fixed hunyan

models/model_3d_generator.py

@@ -67,79 +67,139 @@ class Hunyuan3DGenerator:
             logger.info("🚀 Starting 3D model loading process...")
             
             try:
-                # Import Hunyuan3D components
-                logger.info("📦 Importing transformers components...")
-                from transformers import AutoModel, AutoProcessor
-                
-                model_id = self.lite_model_id if self.use_lite else self.model_id
-                logger.info(f"📦 Loading model: {model_id}")
-                
-                # Load preprocessor
-                logger.info("📦 Loading preprocessor...")
-                self.preprocessor = AutoProcessor.from_pretrained(model_id)
-                logger.info("✅ Preprocessor loaded successfully")
-                
-                # Load model with optimizations
-                torch_dtype = torch.float16 if self.device == "cuda" else torch.float32
-                logger.info(f"📦 Using torch dtype: {torch_dtype}")
-                
-                # Disable torch.compile to avoid dynamo issues
-                logger.info("📦 Disabling torch compile to avoid dynamo issues...")
-                torch._dynamo.config.suppress_errors = True
-                
-                logger.info("📦 Loading 3D model with safe device handling...")
-                self.model = AutoModel.from_pretrained(
-                    model_id,
-                    torch_dtype=torch_dtype,
-                    low_cpu_mem_usage=True,
-                    device_map=None,  # Avoid auto device mapping to prevent meta tensor issues
-                    trust_remote_code=True
-                )
-                logger.info("✅ 3D model loaded from pretrained")
-                
-                # Safe device movement
-                logger.info(f"📦 Moving model to device: {self.device}")
+                # Try to import Hunyuan3D components
+                logger.info("📦 Attempting to import Hunyuan3D components...")
                 try:
-                    # Check if model is a meta tensor
-                    if hasattr(self.model, 'to_empty'):
-                        # Use to_empty for meta tensors
-                        logger.info("📦 Using to_empty() for safe device movement...")
-                        self.model = self.model.to_empty(device=self.device, dtype=torch_dtype)
-                    else:
-                        # Standard device movement
-                        logger.info("📦 Using standard to() for device movement...")
-                        self.model = self.model.to(self.device, dtype=torch_dtype)
-                    logger.info("✅ Model successfully moved to device")
+                    from hy3dshape.pipelines import Hunyuan3DDiTFlowMatchingPipeline
+                    from hy3dshape.rembg import BackgroundRemover
+                    logger.info("✅ Hunyuan3D components imported successfully")
+                    
+                    # Load the pipeline
+                    model_id = self.lite_model_id if self.use_lite else self.model_id
+                    logger.info(f"📦 Loading Hunyuan3D pipeline: {model_id}")
+                    
+                    self.model = Hunyuan3DDiTFlowMatchingPipeline.from_pretrained(model_id)
+                    self.bg_remover = BackgroundRemover()
+                    
+                    logger.info("✅ Hunyuan3D pipeline loaded successfully")
+                    
+                except ImportError as import_error:
+                    logger.error(f"❌ Failed to import Hunyuan3D components: {import_error}")
+                    logger.info("🔄 Hunyuan3D not installed, trying alternative approach...")
+                    
+                    # Fallback: Try using transformers AutoModel
+                    logger.info("📦 Importing transformers components...")
+                    from transformers import AutoModel, AutoProcessor
+                    
+                    model_id = self.lite_model_id if self.use_lite else self.model_id
+                    logger.info(f"📦 Loading model: {model_id}")
+                    
+                    # Check if model exists on HuggingFace
+                    try:
+                        from huggingface_hub import model_info
+                        info = model_info(model_id)
+                        logger.info(f"✅ Model found on HuggingFace: {info.modelId}")
+                    except Exception as hub_error:
+                        logger.error(f"❌ Model not found on HuggingFace: {hub_error}")
+                        logger.info("🔄 Using fallback 3D generation")
+                        self.model = "fallback"
+                        return
+                    
+                    # Load preprocessor
+                    logger.info("📦 Loading preprocessor...")
+                    try:
+                        self.preprocessor = AutoProcessor.from_pretrained(model_id)
+                        logger.info("✅ Preprocessor loaded successfully")
+                    except Exception as proc_error:
+                        logger.error(f"❌ Preprocessor loading failed: {proc_error}")
+                        logger.info("🔄 Using fallback mode")
+                        self.model = "fallback"
+                        return
+                    
+                    # Load model with optimizations
+                    torch_dtype = torch.float16 if self.device == "cuda" else torch.float32
+                    logger.info(f"📦 Using torch dtype: {torch_dtype}")
+                    
+                    # Disable torch.compile to avoid dynamo issues
+                    logger.info("📦 Disabling torch compile to avoid dynamo issues...")
+                    torch._dynamo.config.suppress_errors = True
+                    
+                    logger.info("📦 Loading 3D model with safe device handling...")
+                    
+                    # Try loading with different strategies
+                    loading_successful = False
+                    
+                    # Strategy 1: Load directly to device
+                    try:
+                        logger.info("📦 Strategy 1: Direct device loading...")
+                        self.model = AutoModel.from_pretrained(
+                            model_id,
+                            torch_dtype=torch_dtype,
+                            device_map={"": self.device},
+                            low_cpu_mem_usage=True,
+                            trust_remote_code=True
+                        )
+                        loading_successful = True
+                        logger.info("✅ Direct device loading successful")
+                    except Exception as e1:
+                        logger.error(f"❌ Strategy 1 failed: {e1}")
                     
-                except RuntimeError as device_error:
-                    logger.error(f"❌ Device movement failed: {device_error}")
-                    if "meta tensor" in str(device_error):
-                        logger.info("🔄 Attempting CPU fallback for meta tensor issue...")
-                        self.device = "cpu"
+                    # Strategy 2: Load to CPU first
+                    if not loading_successful:
                         try:
-                            if hasattr(self.model, 'to_empty'):
-                                self.model = self.model.to_empty(device="cpu", dtype=torch_dtype)
-                            else:
-                                self.model = self.model.to("cpu", dtype=torch_dtype)
-                            logger.info("✅ Fallback to CPU successful")
-                        except Exception as cpu_error:
-                            logger.error(f"❌ CPU fallback also failed: {cpu_error}")
-                            logger.info("🔄 Using fallback mode...")
-                            self.model = "fallback"
-                    else:
-                        logger.error(f"❌ Non-meta tensor device error: {device_error}")
-                        logger.info("🔄 Using fallback mode...")
+                            logger.info("📦 Strategy 2: CPU-first loading...")
+                            # Load model to CPU first to avoid meta tensor issues
+                            self.model = AutoModel.from_pretrained(
+                                model_id,
+                                torch_dtype=torch.float32,  # Use float32 for CPU loading
+                                low_cpu_mem_usage=True,
+                                device_map=None,  # No device mapping initially
+                                trust_remote_code=True
+                            )
+                            logger.info("✅ 3D model loaded to CPU")
+                            
+                            # Now safely move to target device
+                            logger.info(f"📦 Moving model to target device: {self.device}")
+                            try:
+                                if self.device == "cuda":
+                                    # Convert to appropriate dtype for GPU
+                                    self.model = self.model.to(device=self.device, dtype=torch.float16)
+                                    logger.info("✅ Model moved to CUDA with fp16")
+                                else:
+                                    # Keep on CPU
+                                    self.model = self.model.to(device="cpu", dtype=torch.float32)
+                                    logger.info("✅ Model kept on CPU with fp32")
+                                loading_successful = True
+                                
+                            except Exception as device_error:
+                                logger.error(f"❌ Device movement failed: {device_error}")
+                                logger.info("🔄 Falling back to CPU...")
+                                self.device = "cpu"
+                                if self.model is not None:
+                                    self.model = self.model.to("cpu", dtype=torch.float32)
+                                    loading_successful = True
+                                else:
+                                    logger.error("❌ Model is None, using fallback mode")
+                                    self.model = "fallback"
+                        except Exception as e2:
+                            logger.error(f"❌ Strategy 2 failed: {e2}")
+                    
+                    # If all strategies failed, use fallback
+                    if not loading_successful:
+                        logger.error("❌ All loading strategies failed")
+                        logger.info("🔄 Using fallback 3D generation")
                         self.model = "fallback"
-                
-                # Enable optimizations safely
-                logger.info("📦 Applying model optimizations...")
-                if self.model != "fallback" and hasattr(self.model, 'enable_attention_slicing'):
-                    self.model.enable_attention_slicing()
-                    logger.info("✅ Attention slicing enabled")
-                else:
-                    logger.info("⚠️ Attention slicing not available")
-                
-                logger.info("🎉 3D model loading completed successfully!")
+                        return
+                    
+                    # Enable optimizations safely
+                    logger.info("📦 Applying model optimizations...")
+                    if self.model != "fallback" and hasattr(self.model, 'enable_attention_slicing'):
+                        self.model.enable_attention_slicing()
+                        logger.info("✅ Attention slicing enabled")
+                    else:
+                        logger.info("⚠️ Attention slicing not available")
+                    
+                    logger.info("🎉 3D model loading completed successfully!")
                 
             except Exception as e:
                 logger.error(f"❌ Failed to load Hunyuan3D model: {e}")
@@ -184,78 +244,117 @@ class Hunyuan3DGenerator:
             else:
                 logger.info("🖼️ Input is already PIL Image")
             
-            # Ensure RGB
+            # Ensure RGBA for Hunyuan3D
             logger.info(f"🖼️ Image mode: {image.mode}")
-            if image.mode != 'RGB':
-                logger.info("🖼️ Converting image to RGB mode")
-                image = image.convert('RGB')
+            if image.mode != 'RGBA':
+                logger.info("🖼️ Converting image to RGBA mode")
+                image = image.convert('RGBA')
             
             logger.info(f"🖼️ Final image size: {image.size}")
             
-            # Resize for processing
-            logger.info("🖼️ Resizing image for processing (512x512)...")
-            image = image.resize((512, 512), Image.Resampling.LANCZOS)
-            logger.info("✅ Image resized successfully")
-            
             # Remove background if requested
-            if remove_background:
+            if remove_background and image.mode == 'RGB':
                 logger.info("🎭 Removing background from image...")
                 try:
-                    image = self._remove_background(image)
-                    logger.info("✅ Background removal completed")
+                    if hasattr(self, 'bg_remover'):
+                        # Use Hunyuan3D's background remover
+                        image = self.bg_remover(image)
+                        logger.info("✅ Background removed using Hunyuan3D remover")
+                    else:
+                        # Use fallback background removal
+                        image = self._remove_background(image)
+                        logger.info("✅ Background removed using fallback method")
                 except Exception as bg_error:
                     logger.error(f"❌ Background removal failed: {bg_error}")
                     logger.info("🔄 Continuing with original image...")
             
-            # Process with model
-            logger.info("🧠 Starting model inference...")
-            with torch.no_grad():
+            # Check if we have the Hunyuan3D pipeline
+            if hasattr(self.model, '__call__') and hasattr(self, 'bg_remover'):
+                # Using Hunyuan3D pipeline
+                logger.info("🧠 Using Hunyuan3D pipeline for 3D generation...")
+                
                 try:
-                    # Preprocess image
-                    logger.info("🔄 Preprocessing image for model...")
-                    inputs = self.preprocessor(images=image, return_tensors="pt")
-                    logger.info(f"🔄 Input tensor shape: {inputs['pixel_values'].shape if 'pixel_values' in inputs else 'unknown'}")
-                    
-                    # Move inputs to device safely
-                    logger.info(f"🔄 Moving inputs to device: {self.device}")
-                    try:
-                        # Avoid device-related dynamo issues
-                        device_str = str(self.device)  # Convert to string to avoid torch.device in dynamo
-                        inputs = {k: v.to(device_str) for k, v in inputs.items() if hasattr(v, 'to')}
-                        logger.info("✅ Inputs moved to device successfully")
-                    except Exception as device_error:
-                        logger.error(f"❌ Failed to move inputs to device: {device_error}")
-                        raise device_error
+                    # Generate 3D model using Hunyuan3D
+                    logger.info("🚀 Starting Hunyuan3D generation...")
+                    mesh_outputs = self.model(image=image)
                     
-                    # Generate 3D
-                    logger.info("🚀 Starting 3D generation inference...")
-                    logger.info(f"🚀 Parameters: steps={self.num_inference_steps}, guidance={self.guidance_scale}")
-                    
-                    outputs = self.model.generate(
-                        **inputs,
-                        num_inference_steps=self.num_inference_steps,
-                        guidance_scale=self.guidance_scale,
-                        texture_resolution=texture_resolution
-                    )
-                    logger.info("✅ 3D generation completed successfully")
-                    
-                    # Extract mesh
-                    logger.info("🔧 Extracting mesh from model outputs...")
-                    mesh = self._extract_mesh(outputs)
-                    logger.info("✅ Mesh extraction completed")
-                    
-                except Exception as inference_error:
-                    logger.error(f"❌ Model inference failed: {inference_error}")
-                    logger.error(f"❌ Inference error type: {type(inference_error).__name__}")
-                    raise inference_error
-            
-            # Save mesh
-            logger.info("💾 Saving generated mesh...")
-            mesh_path = self._save_mesh(mesh)
-            logger.info(f"✅ Mesh saved to: {mesh_path}")
+                    if isinstance(mesh_outputs, list) and len(mesh_outputs) > 0:
+                        mesh = mesh_outputs[0]
+                        logger.info("✅ 3D mesh generated successfully")
+                        
+                        # Save mesh
+                        logger.info("💾 Saving generated mesh...")
+                        mesh_path = self._save_mesh(mesh)
+                        logger.info(f"✅ Mesh saved to: {mesh_path}")
+                        
+                        return mesh_path
+                    else:
+                        logger.error("❌ No mesh output from Hunyuan3D")
+                        raise Exception("Empty mesh output")
+                        
+                except Exception as hunyuan_error:
+                    logger.error(f"❌ Hunyuan3D generation failed: {hunyuan_error}")
+                    logger.info("🔄 Falling back to alternative generation...")
+                    return self._generate_fallback_3d(image)
             
-            logger.info("🎉 3D generation process completed successfully!")
-            return mesh_path
+            else:
+                # Using transformers-based approach (original code)
+                logger.info("🧠 Using transformers-based 3D generation...")
+                
+                # Resize for processing
+                logger.info("🖼️ Resizing image for processing (512x512)...")
+                image = image.resize((512, 512), Image.Resampling.LANCZOS)
+                logger.info("✅ Image resized successfully")
+                
+                # Process with model
+                logger.info("🧠 Starting model inference...")
+                with torch.no_grad():
+                    try:
+                        # Preprocess image
+                        logger.info("🔄 Preprocessing image for model...")
+                        inputs = self.preprocessor(images=image, return_tensors="pt")
+                        logger.info(f"🔄 Input tensor shape: {inputs['pixel_values'].shape if 'pixel_values' in inputs else 'unknown'}")
+                        
+                        # Move inputs to device safely
+                        logger.info(f"🔄 Moving inputs to device: {self.device}")
+                        try:
+                            # Avoid device-related dynamo issues
+                            device_str = str(self.device)  # Convert to string to avoid torch.device in dynamo
+                            inputs = {k: v.to(device_str) for k, v in inputs.items() if hasattr(v, 'to')}
+                            logger.info("✅ Inputs moved to device successfully")
+                        except Exception as device_error:
+                            logger.error(f"❌ Failed to move inputs to device: {device_error}")
+                            raise device_error
+                        
+                        # Generate 3D
+                        logger.info("🚀 Starting 3D generation inference...")
+                        logger.info(f"🚀 Parameters: steps={self.num_inference_steps}, guidance={self.guidance_scale}")
+                        
+                        outputs = self.model.generate(
+                            **inputs,
+                            num_inference_steps=self.num_inference_steps,
+                            guidance_scale=self.guidance_scale,
+                            texture_resolution=texture_resolution
+                        )
+                        logger.info("✅ 3D generation completed successfully")
+                        
+                        # Extract mesh
+                        logger.info("🔧 Extracting mesh from model outputs...")
+                        mesh = self._extract_mesh(outputs)
+                        logger.info("✅ Mesh extraction completed")
+                        
+                    except Exception as inference_error:
+                        logger.error(f"❌ Model inference failed: {inference_error}")
+                        logger.error(f"❌ Inference error type: {type(inference_error).__name__}")
+                        raise inference_error
+                
+                # Save mesh
+                logger.info("💾 Saving generated mesh...")
+                mesh_path = self._save_mesh(mesh)
+                logger.info(f"✅ Mesh saved to: {mesh_path}")
+                
+                logger.info("🎉 3D generation process completed successfully!")
+                return mesh_path
             
         except Exception as e:
             logger.error(f"❌ 3D generation error: {e}")

requirements_hunyuan3d.txt

@@ -0,0 +1,37 @@
+# Additional requirements for Hunyuan3D-2.1 support
+# Install these in addition to the main requirements.txt
+
+# Hunyuan3D specific dependencies
+# Note: The hy3dshape and hy3dpaint modules need to be installed from the Hunyuan3D repository
+# git clone https://huggingface.co/spaces/tencent/Hunyuan3D-2.1
+# Then copy the hy3dshape and hy3dpaint folders to your project
+
+# Core dependencies that Hunyuan3D uses
+trimesh>=3.23.5
+pygltflib>=1.16.1
+scikit-image>=0.22.0
+opencv-python>=4.8.0
+Pillow>=10.0.0
+numpy>=1.24.0
+torch>=2.0.0
+torchvision>=0.15.0
+transformers>=4.35.0
+diffusers>=0.25.0
+accelerate>=0.25.0
+safetensors>=0.4.0
+huggingface-hub>=0.19.0
+spaces>=0.19.0
+
+# For background removal
+rembg>=2.0.50
+
+# For 3D visualization and processing
+pymeshlab>=2022.2.post3
+open3d>=0.17.0
+
+# Optional: For mesh optimization
+xatlas>=0.0.7
+pyvista>=0.42.0
+
+# Note: Some dependencies might require specific versions or CUDA compatibility
+# Adjust versions based on your system configuration