Rendering bug with many Gaussians and cameras

[cwchenwang]

When I tried to render gaussians of a large scene (9 M Gaussians) with the many cameras (20), the latter ones seem to just crash. (I used the same 20 below for debugging.

If less cameras are used (e.g. 5), there is no problem. I have uploaded the GS to huggingface, below is a reproduction:

import torch
from gsplat import rasterization
import numpy as np
import os
from huggingface_hub import hf_hub_download

def download_data():
    """Download data files from Hugging Face if they don't exist locally"""
    
    repo_id = "chenwang/gsplat-sh-bug-reproduction"
    
    files_to_download = [
        "means3D.pt",
        "rotations.pt", 
        "scales.pt",
        "opacity.pt",
        "shs.pt",
        "W2C.pt",
        "intr.pt",
        "bg_color.pt"
    ]
    
    print("Checking for data files...")
    
    for filename in files_to_download:
        if not os.path.exists(filename):
            print(f"Downloading {filename} from Hugging Face...")
            try:
                hf_hub_download(
                    repo_id=repo_id,
                    filename=filename,
                    repo_type="dataset",
                    local_dir="."
                )
                print(f"✓ Downloaded {filename}")
            except Exception as e:
                print(f"✗ Failed to download {filename}: {e}")
                print(f"Please manually download from: https://huggingface.co/datasets/{repo_id}")
                return False
        else:
            print(f"✓ Found {filename} locally")
    
    return True

def reproduce_sh_bug():
    """Reproduce the spherical harmonics bug with identical cameras"""
    
    # Download data if needed
    if not download_data():
        print("Failed to download required data files. Exiting.")
        return
    
    print("\nLoading data...")
    # Load the data
    means3D = torch.load("means3D.pt", map_location="cpu").to('cuda')
    rotations = torch.load("rotations.pt", map_location="cpu").to('cuda') 
    scales = torch.load("scales.pt", map_location="cpu").to('cuda')
    opacity = torch.load("opacity.pt", map_location="cpu").to('cuda')
    shs = torch.load("shs.pt", map_location="cpu").to('cuda')
    W2C = torch.load("W2C.pt", map_location="cpu").to('cuda')
    intr = torch.load("intr.pt", map_location="cpu").to('cuda')
    bg_color = torch.load("bg_color.pt", map_location="cpu").to('cuda')
    
    print(f"Data shapes:")
    print(f"  means3D: {means3D.shape}")
    print(f"  shs: {shs.shape}")
    print(f"  W2C: {W2C.shape}")
    print(f"  Total Gaussians: {means3D.shape[0]:,}")
    
    # Render with a subset of cameras
    num_cameras = 20  # Use fewer cameras to reduce memory usage
    width, height = 512, 288
    sh_degree = 3
    z_near = 0.1
    
    print(f"\nRendering with {num_cameras} cameras...")
    render_colors, _, _ = rasterization(
        means3D, rotations, scales, opacity.squeeze(), 
        shs, W2C[:num_cameras], intr[:num_cameras], width, height, 
        sh_degree=sh_degree, backgrounds=bg_color[:num_cameras], 
        near_plane=z_near, packed=False
    )
    
    print(f"Render output shape: {render_colors.shape}")
    
    # Compare results between cameras
    print((W2C[0] - W2C[-1]).sum())
    print((intr[0] - intr[-1]).sum())
    print((render_colors[0] - render_colors[-1]).sum())

if __name__ == "__main__":
    reproduce_sh_bug()

I did some debugging and found that the SphericalHarmonics seem to return different results even with the same cameras, but I am not sure about the real reason.