| [Errno 2] No such file or directory: '/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/release_data/ccd_cache/components.v20240608.cif': | |
| Traceback (most recent call last): | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/data/infer_data_pipeline.py", line 211, in __getitem__ | |
| data, atom_array, _ = self.process_one( | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/data/infer_data_pipeline.py", line 101, in process_one | |
| sample2feat = SampleDictToFeatures( | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/data/json_to_feature.py", line 34, in __init__ | |
| self.input_dict = add_entity_atom_array(single_sample_dict) | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/data/json_parser.py", line 605, in add_entity_atom_array | |
| atom_info = build_polymer(entity_info) | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/data/json_parser.py", line 345, in build_polymer | |
| chain_array = _build_polymer_atom_array(ccd_seqs) | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/data/json_parser.py", line 290, in _build_polymer_atom_array | |
| residue = ccd.get_component_atom_array( | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/data/ccd.py", line 88, in get_component_atom_array | |
| ccd_cif = biotite_load_ccd_cif() | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/data/ccd.py", line 45, in biotite_load_ccd_cif | |
| return pdbx.CIFFile.read(COMPONENTS_FILE) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/biotite/structure/io/pdbx/cif.py", line 844, in read | |
| with open(file, "r") as f: | |
| FileNotFoundError: [Errno 2] No such file or directory: '/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/release_data/ccd_cache/components.v20240608.cif' | |
| [Rank 0]7r6r Protenix.get_pairformer_output() missing 1 required positional argument: 'pairformer_stack': | |
| Traceback (most recent call last): | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/runner/inference.py", line 260, in infer_predict | |
| prediction, label_dict = runner.predict(data) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context | |
| return func(*args, **kwargs) | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/runner/inference.py", line 156, in predict | |
| prediction, label_dict, _ = self.model( | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 865, in forward | |
| pred_dict, label_dict, log_dict, time_tracker = self.main_inference_loop( | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 339, in main_inference_loop | |
| pred_dict, log_dict, time_tracker = self._main_inference_loop( | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 401, in _main_inference_loop | |
| s_inputs, s, z = self.get_pairformer_output( | |
| TypeError: Protenix.get_pairformer_output() missing 1 required positional argument: 'pairformer_stack' | |
| [Rank 0]7r6r [Errno 28] No space left on device: '/n/home09/zaixizhang/.cache/torch_extensions/py310_cu121/evoformer_attn/lock': | |
| Traceback (most recent call last): | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/runner/inference.py", line 260, in infer_predict | |
| prediction, label_dict = runner.predict(data) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context | |
| return func(*args, **kwargs) | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/runner/inference.py", line 156, in predict | |
| prediction, label_dict, _ = self.model( | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 866, in forward | |
| pred_dict, label_dict, log_dict, time_tracker = self.main_inference_loop( | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 339, in main_inference_loop | |
| pred_dict, log_dict, time_tracker = self._main_inference_loop( | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 401, in _main_inference_loop | |
| s_inputs, s, z = self.get_pairformer_output( | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 202, in get_pairformer_output | |
| z = self.msa_module( | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/modules/pairformer.py", line 732, in forward | |
| msa_sample, z = checkpoint_blocks( | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/openfold_local/utils/checkpointing.py", line 85, in checkpoint_blocks | |
| return exec(blocks, args) | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/openfold_local/utils/checkpointing.py", line 72, in exec | |
| a = wrap(block(*a)) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/modules/pairformer.py", line 517, in forward | |
| _, z = self.pair_stack( | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/modules/pairformer.py", line 132, in forward | |
| z += self.tri_att_start( | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/openfold_local/model/triangular_attention.py", line 129, in forward | |
| x = self._chunk( | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/openfold_local/model/triangular_attention.py", line 74, in _chunk | |
| return chunk_layer( | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/openfold_local/utils/chunk_utils.py", line 285, in chunk_layer | |
| output_chunk = layer(**chunks) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/openfold_local/model/primitives.py", line 560, in forward | |
| o = _deepspeed_evo_attn(q, k, v, biases) | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/openfold_local/model/primitives.py", line 730, in _deepspeed_evo_attn | |
| o = DS4Sci_EvoformerAttention(q, k, v, biases) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/deepspeed/ops/deepspeed4science/evoformer_attn.py", line 106, in DS4Sci_EvoformerAttention | |
| return EvoformerFusedAttention.apply(Q, K, V, biases[0], biases[1]) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/autograd/function.py", line 598, in apply | |
| return super().apply(*args, **kwargs) # type: ignore[misc] | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/deepspeed/ops/deepspeed4science/evoformer_attn.py", line 71, in forward | |
| o, lse = _attention(q, k, v, bias1_, bias2_) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/deepspeed/ops/deepspeed4science/evoformer_attn.py", line 24, in _attention | |
| kernel_ = EvoformerAttnBuilder().load() | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py", line 508, in load | |
| return self.jit_load(verbose) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py", line 555, in jit_load | |
| op_module = load(name=self.name, | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/utils/cpp_extension.py", line 1309, in load | |
| return _jit_compile( | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/utils/cpp_extension.py", line 1696, in _jit_compile | |
| if baton.try_acquire(): | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/utils/file_baton.py", line 29, in try_acquire | |
| self.fd = os.open(self.lock_file_path, os.O_CREAT | os.O_EXCL) | |
| OSError: [Errno 28] No space left on device: '/n/home09/zaixizhang/.cache/torch_extensions/py310_cu121/evoformer_attn/lock' | |
| [Rank 0]7r6r 'profile': | |
| Traceback (most recent call last): | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/runner/inference.py", line 260, in infer_predict | |
| prediction, label_dict = runner.predict(data) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context | |
| return func(*args, **kwargs) | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/runner/inference.py", line 156, in predict | |
| prediction, label_dict, _ = self.model( | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 866, in forward | |
| pred_dict, label_dict, log_dict, time_tracker = self.main_inference_loop( | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 339, in main_inference_loop | |
| pred_dict, log_dict, time_tracker = self._main_inference_loop( | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 523, in _main_inference_loop | |
| s_inputs_clean, s_clean, z_clean = self.get_pairformer_output( | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 159, in get_pairformer_output | |
| s_inputs = self.input_embedder( | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/modules/embedders.py", line 81, in forward | |
| + [ | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/modules/embedders.py", line 82, in <listcomp> | |
| input_feature_dict[name].reshape(*batch_shape, d) | |
| KeyError: 'profile' | |
| [Rank 0]7r6r too many values to unpack (expected 3): | |
| Traceback (most recent call last): | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/runner/inference.py", line 260, in infer_predict | |
| prediction, label_dict = runner.predict(data) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context | |
| return func(*args, **kwargs) | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/runner/inference.py", line 156, in predict | |
| prediction, label_dict, _ = self.model( | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl | |
| return self._call_impl(*args, **kwargs) | |
| File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 867, in forward | |
| pred_dict, label_dict, log_dict, time_tracker = self.main_inference_loop( | |
| File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 339, in main_inference_loop | |
| pred_dict, log_dict, time_tracker = self._main_inference_loop( | |
| ValueError: too many values to unpack (expected 3) | |