Nerf To 3dgs Migrator

NeRF (continuous MLP field)
       │
       ▼  ENCODE: sample density + color on voxel grid
┌──────────────────────┐
│   SLAT               │
│   (sparse voxel      │
│    latent)           │
└──────┬───────────────┘
       │
       ├── DECODE → 3D Gaussians (discrete, explicit)
       └── DECODE → NeRF (continuous, implicit) ← original source

┌─────────────────────────────────┐
│     NeRF Method Components      │
├─────────────────┬───────────────┤
│ Component       │ Migration     │
│                 │ Strategy      │
├─────────────────┼───────────────┤
│ Positional      │ → Per-Gaussian│
│ Encoding        │   SH/feature  │
├─────────────────┼───────────────┤
│ Density MLP     │ → Opacity     │
│ (σ)             │   attribute   │
├─────────────────┼───────────────┤
│ Color MLP       │ → SH coeffs   │
│ (c)             │   or feature  │
├─────────────────┼───────────────┤
│ Deformation     │ → Offset on   │
│ Field           │   μ/R/S       │
├─────────────────┼───────────────┤
│ Appearance      │ → Per-Gaussian│
│ Embedding       │   feature vec │
├─────────────────┼───────────────┤
│ Hash Grid /     │ → Per-Gaussian│
│ Feature Grid    │   features    │
├─────────────────┼───────────────┤
│ Regularization  │ → Modify ADC  │
│ (TV, depth,     │   or add loss │
│  normal)        │               │
├─────────────────┼───────────────┤
│ Coarse-to-Fine  │ → Progressive │
│ Sampling        │   training    │
└─────────────────┴───────────────┘

# Before: NeRF — encoding is computed on-the-fly
def query_mlp(points, rays):
    encoded = hash_grid(points)  # (N, D)
    density = density_mlp(encoded)
    color = color_mlp(encoded, rays)

# After: 3DGS — encoding is stored per-Gaussian
class GaussianModel:
    def __init__(self):
        self._xyz = nn.Parameter(...)       # position (N, 3)
        self._opacity = nn.Parameter(...)   # opacity (N, 1)
        self._features = nn.Parameter(...)  # encoded features (N, D)  ← NEW
        self._sh = nn.Parameter(...)        # SH coefficients (N, 3*K)

# NeRF: α = 1 - exp(-σ * δ) where δ is step size
# 3DGS: α = sigmoid(raw_opacity)

# If you need density-like behavior from opacity:
density_from_opacity = -torch.log(1 - opacity + 1e-6) / voxel_size

# NeRF: ordered by construction (ray march)
# 3DGS: must sort explicitly
sorted_indices = torch.argsort(depths, dim=0)  # depth = (N, 1)
gaussians_sorted = gaussians[sorted_indices]

# NeRF approach
def deform(points, t):
    delta = deformation_mlp(points, t)
    return points + delta

# 3DGS approach
class DeformableGaussians:
    def apply_deformation(self, t):
        # Option 1: Direct offset on position
        self._xyz = self.base_xyz + self.deformation_net(self.base_xyz, t)

        # Option 2: Offset on rotation and scale too
        self._rotation = self.base_rotation + delta_rotation(t)
        self._scaling = self.base_scaling * scale_factor(t)

# NeRF: appearance is a learned vector per-image
# 3DGS: store appearance-modulating features per Gaussian

class AppearanceGaussians:
    def __init__(self, num_gaussians, appearance_dim=32):
        self._appearance = nn.Parameter(
            torch.randn(num_gaussians, appearance_dim) * 0.01
        )

    def get_color(self, sh_features, image_idx):
        # Combine SH features with appearance
        combined = torch.cat([sh_features, self._appearance], dim=-1)
        return self.color_net(combined)