# # Copyright (C) 2023, Inria # GRAPHDECO research group, https://team.inria.fr/graphdeco # All rights reserved. # # This software is free for non-commercial, research and evaluation use # under the terms of the LICENSE.md file. # # For inquiries contact george.drettakis@inria.fr # import torch import math from scene.gaussian_model import GaussianModel from utils.pose_utils import get_camera_from_tensor, quadmultiply from utils.graphics_utils import depth_to_normal ### if use [diff-gaussian-rasterization](https://github.com/graphdeco-inria/diff-gaussian-rasterization) from diff_gaussian_rasterization import ( GaussianRasterizationSettings, GaussianRasterizer, ) from utils.sh_utils import eval_sh def render( viewpoint_camera, pc: GaussianModel, pipe, bg_color: torch.Tensor, scaling_modifier=1.0, override_color=None, camera_pose=None, ): """ Render the scene. Background tensor (bg_color) must be on GPU! """ # Create zero tensor. We will use it to make pytorch return gradients of the 2D (screen-space) means screenspace_points = ( torch.zeros_like( pc.get_xyz, dtype=pc.get_xyz.dtype, requires_grad=True, device="cuda" ) + 0 ) try: screenspace_points.retain_grad() except: pass # Set up rasterization configuration tanfovx = math.tan(viewpoint_camera.FoVx * 0.5) tanfovy = math.tan(viewpoint_camera.FoVy * 0.5) # Set camera pose as identity. Then, we will transform the Gaussians around camera_pose w2c = torch.eye(4).cuda() projmatrix = ( w2c.unsqueeze(0).bmm(viewpoint_camera.projection_matrix.unsqueeze(0)) ).squeeze(0) camera_pos = w2c.inverse()[3, :3] raster_settings = GaussianRasterizationSettings( image_height=int(viewpoint_camera.image_height), image_width=int(viewpoint_camera.image_width), tanfovx=tanfovx, tanfovy=tanfovy, bg=bg_color, scale_modifier=scaling_modifier, # viewmatrix=viewpoint_camera.world_view_transform, # projmatrix=viewpoint_camera.full_proj_transform, viewmatrix=w2c, projmatrix=projmatrix, sh_degree=pc.active_sh_degree, # campos=viewpoint_camera.camera_center, campos=camera_pos, prefiltered=False, debug=pipe.debug, ) rasterizer = GaussianRasterizer(raster_settings=raster_settings) # means3D = pc.get_xyz rel_w2c = get_camera_from_tensor(camera_pose) # Transform mean and rot of Gaussians to camera frame gaussians_xyz = pc._xyz.clone() gaussians_rot = pc._rotation.clone() xyz_ones = torch.ones(gaussians_xyz.shape[0], 1).cuda().float() xyz_homo = torch.cat((gaussians_xyz, xyz_ones), dim=1) gaussians_xyz_trans = (rel_w2c @ xyz_homo.T).T[:, :3] gaussians_rot_trans = quadmultiply(camera_pose[:4], gaussians_rot) means3D = gaussians_xyz_trans means2D = screenspace_points opacity = pc.get_opacity # If precomputed 3d covariance is provided, use it. If not, then it will be computed from # scaling / rotation by the rasterizer. scales = None rotations = None cov3D_precomp = None if pipe.compute_cov3D_python: cov3D_precomp = pc.get_covariance(scaling_modifier) else: scales = pc.get_scaling rotations = gaussians_rot_trans # pc.get_rotation # If precomputed colors are provided, use them. Otherwise, if it is desired to precompute colors # from SHs in Python, do it. If not, then SH -> RGB conversion will be done by rasterizer. shs = None colors_precomp = None if override_color is None: if pipe.convert_SHs_python: shs_view = pc.get_features.transpose(1, 2).view( -1, 3, (pc.max_sh_degree + 1) ** 2 ) dir_pp = pc.get_xyz - viewpoint_camera.camera_center.repeat( pc.get_features.shape[0], 1 ) dir_pp_normalized = dir_pp / dir_pp.norm(dim=1, keepdim=True) sh2rgb = eval_sh(pc.active_sh_degree, shs_view, dir_pp_normalized) colors_precomp = torch.clamp_min(sh2rgb + 0.5, 0.0) else: shs = pc.get_features else: colors_precomp = override_color # Rasterize visible Gaussians to image, obtain their radii (on screen). rendered_image, radii = rasterizer( means3D=means3D, means2D=means2D, shs=shs, colors_precomp=colors_precomp, opacities=opacity, scales=scales, rotations=rotations, cov3D_precomp=cov3D_precomp, ) # Those Gaussians that were frustum culled or had a radius of 0 were not visible. # They will be excluded from value updates used in the splitting criteria. return { "render": rendered_image, "viewspace_points": screenspace_points, "visibility_filter": radii > 0, "radii": radii, } ### if use [gsplat](https://github.com/nerfstudio-project/gsplat) from gsplat import rasterization def render_gsplat( viewpoint_camera, pc : GaussianModel, pipe, bg_color : torch.Tensor, scaling_modifier = 1.0, override_color = None, camera_pose = None, fov = None, render_mode="RGB"): """ Render the scene. Background tensor (bg_color) must be on GPU! """ if fov is None: FoVx = viewpoint_camera.FoVx FoVy = viewpoint_camera.FoVy else: FoVx = fov[0] FoVy = fov[1] tanfovx = math.tan(FoVx * 0.5) tanfovy = math.tan(FoVy * 0.5) focal_length_x = viewpoint_camera.image_width / (2 * tanfovx) focal_length_y = viewpoint_camera.image_height / (2 * tanfovy) K = torch.tensor( [ [focal_length_x, 0, viewpoint_camera.image_width / 2.0], [0, focal_length_y, viewpoint_camera.image_height / 2.0], [0, 0, 1], ], device="cuda", ) means3D = pc.get_xyz opacity = pc.get_opacity scales = pc.get_scaling * scaling_modifier rotations = pc.get_rotation if override_color is not None: colors = override_color # [N, 3] sh_degree = None else: colors = pc.get_features # [N, K, 3] sh_degree = pc.active_sh_degree if camera_pose is None: viewmat = viewpoint_camera.world_view_transform.transpose(0, 1) # [4, 4] else: viewmat = get_camera_from_tensor(camera_pose) render_colors, render_alphas, info = rasterization( means=means3D, # [N, 3] quats=rotations, # [N, 4] scales=scales, # [N, 3] opacities=opacity.squeeze(-1), # [N,] colors=colors, viewmats=viewmat[None], # [1, 4, 4] Ks=K[None], # [1, 3, 3] backgrounds=bg_color[None], width=int(viewpoint_camera.image_width), height=int(viewpoint_camera.image_height), packed=False, sh_degree=sh_degree, render_mode=render_mode, ) if "D" in render_mode: if "+" in render_mode: depth_map = render_colors[..., -1:] else: depth_map = render_colors normals_surf = depth_to_normal( depth_map, torch.inverse(viewmat[None]), K[None]) normals_surf = normals_surf * (render_alphas).detach() render_colors = torch.cat([render_colors, normals_surf], dim=-1) # [1, H, W, 3] -> [3, H, W] rendered_image = render_colors[0].permute(2, 0, 1) radii = info["radii"].squeeze(0) # [N,] try: info["means2d"].retain_grad() # [1, N, 2] except: pass # Those Gaussians that were frustum culled or had a radius of 0 were not visible. # They will be excluded from value updates used in the splitting criteria. return {"render": rendered_image, "viewspace_points": info["means2d"], "visibility_filter" : radii > 0, "radii": radii}