[Errno 2] No such file or directory: '/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/release_data/ccd_cache/components.v20240608.cif': Traceback (most recent call last): File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/data/infer_data_pipeline.py", line 211, in __getitem__ data, atom_array, _ = self.process_one( File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/data/infer_data_pipeline.py", line 101, in process_one sample2feat = SampleDictToFeatures( File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/data/json_to_feature.py", line 34, in __init__ self.input_dict = add_entity_atom_array(single_sample_dict) File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/data/json_parser.py", line 605, in add_entity_atom_array atom_info = build_polymer(entity_info) File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/data/json_parser.py", line 345, in build_polymer chain_array = _build_polymer_atom_array(ccd_seqs) File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/data/json_parser.py", line 290, in _build_polymer_atom_array residue = ccd.get_component_atom_array( File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/data/ccd.py", line 88, in get_component_atom_array ccd_cif = biotite_load_ccd_cif() File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/data/ccd.py", line 45, in biotite_load_ccd_cif return pdbx.CIFFile.read(COMPONENTS_FILE) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/biotite/structure/io/pdbx/cif.py", line 844, in read with open(file, "r") as f: FileNotFoundError: [Errno 2] No such file or directory: '/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/release_data/ccd_cache/components.v20240608.cif' [Rank 0]7r6r Protenix.get_pairformer_output() missing 1 required positional argument: 'pairformer_stack': Traceback (most recent call last): File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/runner/inference.py", line 260, in infer_predict prediction, label_dict = runner.predict(data) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context return func(*args, **kwargs) File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/runner/inference.py", line 156, in predict prediction, label_dict, _ = self.model( File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl return forward_call(*args, **kwargs) File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 865, in forward pred_dict, label_dict, log_dict, time_tracker = self.main_inference_loop( File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 339, in main_inference_loop pred_dict, log_dict, time_tracker = self._main_inference_loop( File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 401, in _main_inference_loop s_inputs, s, z = self.get_pairformer_output( TypeError: Protenix.get_pairformer_output() missing 1 required positional argument: 'pairformer_stack' [Rank 0]7r6r [Errno 28] No space left on device: '/n/home09/zaixizhang/.cache/torch_extensions/py310_cu121/evoformer_attn/lock': Traceback (most recent call last): File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/runner/inference.py", line 260, in infer_predict prediction, label_dict = runner.predict(data) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context return func(*args, **kwargs) File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/runner/inference.py", line 156, in predict prediction, label_dict, _ = self.model( File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl return forward_call(*args, **kwargs) File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 866, in forward pred_dict, label_dict, log_dict, time_tracker = self.main_inference_loop( File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 339, in main_inference_loop pred_dict, log_dict, time_tracker = self._main_inference_loop( File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 401, in _main_inference_loop s_inputs, s, z = self.get_pairformer_output( File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 202, in get_pairformer_output z = self.msa_module( File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl return forward_call(*args, **kwargs) File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/modules/pairformer.py", line 732, in forward msa_sample, z = checkpoint_blocks( File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/openfold_local/utils/checkpointing.py", line 85, in checkpoint_blocks return exec(blocks, args) File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/openfold_local/utils/checkpointing.py", line 72, in exec a = wrap(block(*a)) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl return forward_call(*args, **kwargs) File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/modules/pairformer.py", line 517, in forward _, z = self.pair_stack( File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl return forward_call(*args, **kwargs) File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/modules/pairformer.py", line 132, in forward z += self.tri_att_start( File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl return forward_call(*args, **kwargs) File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/openfold_local/model/triangular_attention.py", line 129, in forward x = self._chunk( File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/openfold_local/model/triangular_attention.py", line 74, in _chunk return chunk_layer( File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/openfold_local/utils/chunk_utils.py", line 285, in chunk_layer output_chunk = layer(**chunks) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl return forward_call(*args, **kwargs) File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/openfold_local/model/primitives.py", line 560, in forward o = _deepspeed_evo_attn(q, k, v, biases) File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/openfold_local/model/primitives.py", line 730, in _deepspeed_evo_attn o = DS4Sci_EvoformerAttention(q, k, v, biases) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/deepspeed/ops/deepspeed4science/evoformer_attn.py", line 106, in DS4Sci_EvoformerAttention return EvoformerFusedAttention.apply(Q, K, V, biases[0], biases[1]) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/autograd/function.py", line 598, in apply return super().apply(*args, **kwargs) # type: ignore[misc] File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/deepspeed/ops/deepspeed4science/evoformer_attn.py", line 71, in forward o, lse = _attention(q, k, v, bias1_, bias2_) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/deepspeed/ops/deepspeed4science/evoformer_attn.py", line 24, in _attention kernel_ = EvoformerAttnBuilder().load() File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py", line 508, in load return self.jit_load(verbose) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/deepspeed/ops/op_builder/builder.py", line 555, in jit_load op_module = load(name=self.name, File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/utils/cpp_extension.py", line 1309, in load return _jit_compile( File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/utils/cpp_extension.py", line 1696, in _jit_compile if baton.try_acquire(): File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/utils/file_baton.py", line 29, in try_acquire self.fd = os.open(self.lock_file_path, os.O_CREAT | os.O_EXCL) OSError: [Errno 28] No space left on device: '/n/home09/zaixizhang/.cache/torch_extensions/py310_cu121/evoformer_attn/lock' [Rank 0]7r6r 'profile': Traceback (most recent call last): File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/runner/inference.py", line 260, in infer_predict prediction, label_dict = runner.predict(data) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context return func(*args, **kwargs) File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/runner/inference.py", line 156, in predict prediction, label_dict, _ = self.model( File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl return forward_call(*args, **kwargs) File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 866, in forward pred_dict, label_dict, log_dict, time_tracker = self.main_inference_loop( File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 339, in main_inference_loop pred_dict, log_dict, time_tracker = self._main_inference_loop( File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 523, in _main_inference_loop s_inputs_clean, s_clean, z_clean = self.get_pairformer_output( File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 159, in get_pairformer_output s_inputs = self.input_embedder( File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl return forward_call(*args, **kwargs) File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/modules/embedders.py", line 81, in forward + [ File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/modules/embedders.py", line 82, in input_feature_dict[name].reshape(*batch_shape, d) KeyError: 'profile' [Rank 0]7r6r too many values to unpack (expected 3): Traceback (most recent call last): File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/runner/inference.py", line 260, in infer_predict prediction, label_dict = runner.predict(data) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context return func(*args, **kwargs) File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/runner/inference.py", line 156, in predict prediction, label_dict, _ = self.model( File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl return self._call_impl(*args, **kwargs) File "/n/home09/zaixizhang/miniconda3/envs/pro_new/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl return forward_call(*args, **kwargs) File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 867, in forward pred_dict, label_dict, log_dict, time_tracker = self.main_inference_loop( File "/n/holylabs/LABS/mzitnik_lab/Users/zaixizhang/Protenix_new/protenix/model/protenix_edit.py", line 339, in main_inference_loop pred_dict, log_dict, time_tracker = self._main_inference_loop( ValueError: too many values to unpack (expected 3)