Spaces:
Running on Zero
Running on Zero
Update project and repository links
Browse filesSync the latest Hugging Face demo branch and point the UI to the official zju3dv project page and GitHub repository.
- .gitignore +0 -1
- INSTALL.md +1 -1
- README.md +19 -26
- app.py +1 -1
- docs/inference.md +1 -2
- src/demo/hf_ui.py +3 -3
- src/demo/infer_batch_images.py +5 -1
- src/demo/infer_single_image.py +23 -1
- src/model/encoder/depth/depthpro/depthpro_wrapper.py +12 -47
- src/model/encoder/depth/infinidepth/infinidepth_wrapper.py +1 -8
- src/model/encoder/encoder_infinidepth_query.py +0 -7
- src/model/encoder/encoder_infinisplat.py +0 -8
.gitignore
CHANGED
|
@@ -14,7 +14,6 @@ __pycache__/
|
|
| 14 |
|
| 15 |
# Runtime outputs
|
| 16 |
/outputs/
|
| 17 |
-
/wandb/
|
| 18 |
/logs/
|
| 19 |
/cache/
|
| 20 |
/trash/
|
|
|
|
| 14 |
|
| 15 |
# Runtime outputs
|
| 16 |
/outputs/
|
|
|
|
| 17 |
/logs/
|
| 18 |
/cache/
|
| 19 |
/trash/
|
INSTALL.md
CHANGED
|
@@ -34,7 +34,7 @@ PLY export works without the dependencies in this section.
|
|
| 34 |
Install `gsplat` only when novel-view video rendering is needed:
|
| 35 |
|
| 36 |
```bash
|
| 37 |
-
uv pip install git+https://github.com/nerfstudio-project/gsplat.git
|
| 38 |
```
|
| 39 |
|
| 40 |
Interactive HTML export requires Node.js and the PlayCanvas `splat-transform` CLI, but does not require `gsplat`:
|
|
|
|
| 34 |
Install `gsplat` only when novel-view video rendering is needed:
|
| 35 |
|
| 36 |
```bash
|
| 37 |
+
uv pip install git+https://github.com/nerfstudio-project/gsplat.git --no-build-isolation
|
| 38 |
```
|
| 39 |
|
| 40 |
Interactive HTML export requires Node.js and the PlayCanvas `splat-transform` CLI, but does not require `gsplat`:
|
README.md
CHANGED
|
@@ -19,35 +19,37 @@ preload_from_hub:
|
|
| 19 |
|
| 20 |
<h1>🌌 InfiniSplat: Implicit Gaussian Decoding for Large-Baseline Monocular View Synthesis</h1>
|
| 21 |
|
|
|
|
|
|
|
| 22 |
<div align="center">
|
| 23 |
-
<a href="https://
|
| 24 |
<img src="https://img.shields.io/badge/Project-Page-red?logo=googlechrome&logoColor=red">
|
| 25 |
</a>
|
| 26 |
<a href="#">
|
| 27 |
<img src="https://img.shields.io/badge/arXiv-Paper-blue?logo=arxiv&logoColor=blue">
|
| 28 |
</a>
|
| 29 |
-
<a href="https://
|
| 30 |
<img src="https://img.shields.io/badge/Gallery-Visualization-green?logo=googlephotos&logoColor=white">
|
| 31 |
</a>
|
| 32 |
-
<a href="https://huggingface.co/PLUS-WAVE/InfiniSplat">
|
| 33 |
-
<img src="https://img.shields.io/badge/HuggingFace-
|
| 34 |
</a>
|
| 35 |
</div>
|
| 36 |
|
| 37 |
<p align="center">
|
| 38 |
-
<a href="https://plus-wave.github.io/">Jiawei Wang*</a> •
|
| 39 |
-
<a href="https://ritianyu.github.io/">Hao Yu*</a> •
|
| 40 |
<a href="https://github.com/Edisoneh">Yongzhen Hu</a> •
|
| 41 |
<a href="https://github.com/shmily768">Xinyi Yang</a> •
|
| 42 |
<a href="#">Tao Ni</a> •
|
| 43 |
<a href="#">Xin Zhan</a> •
|
| 44 |
-
<a href="#">Junbo Chen</a> <br>
|
| 45 |
<a href="https://xzhou.me/">Xiaowei Zhou</a> •
|
| 46 |
<a href="https://csse.szu.edu.cn/staff/ruizhenhu/">Ruizhen Hu</a> •
|
| 47 |
-
<a href="https://pengsida.net/">Sida Peng</a>
|
| 48 |
</p>
|
| 49 |
|
| 50 |
-
<
|
| 51 |
|
| 52 |
</div>
|
| 53 |
|
|
@@ -59,6 +61,8 @@ preload_from_hub:
|
|
| 59 |
|
| 60 |
## 📣 News
|
| 61 |
|
|
|
|
|
|
|
| 62 |
> **[2026-07]** 🎉 Inference code for RGB-only and depth-sensor-guided 3D Gaussian reconstruction is available now!
|
| 63 |
|
| 64 |
## 🧩 What can InfiniSplat do?
|
|
@@ -108,30 +112,19 @@ python -m src.demo.infer_batch_images \
|
|
| 108 |
--input examples/data/lidar_demo
|
| 109 |
```
|
| 110 |
|
| 111 |
-
See [docs/inference.md](docs/inference.md) for camera parameters,
|
| 112 |
-
|
| 113 |
-
## Hugging Face Space
|
| 114 |
|
| 115 |
-
|
| 116 |
-
is loaded once at startup, GPU inference writes a temporary CPU artifact, and CPU
|
| 117 |
-
post-processing filters spatial outliers before exporting the Gaussian PLY. The
|
| 118 |
-
PLY download becomes available before the optimized SOG viewer is encoded. Browser
|
| 119 |
-
viewer assets are prepared once at startup, and the standalone HTML is bundled
|
| 120 |
-
after the interactive viewer starts loading.
|
| 121 |
-
The app reuses `checkpoints/infinisplat_rgb.ckpt` when it exists locally and
|
| 122 |
-
otherwise downloads the released RGB checkpoint from `PLUS-WAVE/InfiniSplat`.
|
| 123 |
-
This RGB path does not require xFormers or any custom CUDA extension.
|
| 124 |
|
| 125 |
-
|
| 126 |
|
| 127 |
```bash
|
| 128 |
-
|
| 129 |
-
python app.py
|
| 130 |
```
|
| 131 |
|
| 132 |
-
|
| 133 |
|
| 134 |
-
|
| 135 |
|
| 136 |
## 🙏 Acknowledgments
|
| 137 |
|
|
|
|
| 19 |
|
| 20 |
<h1>🌌 InfiniSplat: Implicit Gaussian Decoding for Large-Baseline Monocular View Synthesis</h1>
|
| 21 |
|
| 22 |
+
<p align="center"><strong>Conditionally accepted to SIGGRAPH Asia 2026 (Journal Track)</strong></p>
|
| 23 |
+
|
| 24 |
<div align="center">
|
| 25 |
+
<a href="https://zju3dv.github.io/InfiniSplat">
|
| 26 |
<img src="https://img.shields.io/badge/Project-Page-red?logo=googlechrome&logoColor=red">
|
| 27 |
</a>
|
| 28 |
<a href="#">
|
| 29 |
<img src="https://img.shields.io/badge/arXiv-Paper-blue?logo=arxiv&logoColor=blue">
|
| 30 |
</a>
|
| 31 |
+
<a href="https://zju3dv.github.io/InfiniSplat/#visualization">
|
| 32 |
<img src="https://img.shields.io/badge/Gallery-Visualization-green?logo=googlephotos&logoColor=white">
|
| 33 |
</a>
|
| 34 |
+
<a href="https://huggingface.co/spaces/PLUS-WAVE/InfiniSplat">
|
| 35 |
+
<img src="https://img.shields.io/badge/HuggingFace-Demo-yellow?logo=huggingface&logoColor=yellow">
|
| 36 |
</a>
|
| 37 |
</div>
|
| 38 |
|
| 39 |
<p align="center">
|
| 40 |
+
<a href="https://plus-wave.github.io/">Jiawei Wang<sup>*</sup></a> •
|
| 41 |
+
<a href="https://ritianyu.github.io/">Hao Yu<sup>*</sup></a> •
|
| 42 |
<a href="https://github.com/Edisoneh">Yongzhen Hu</a> •
|
| 43 |
<a href="https://github.com/shmily768">Xinyi Yang</a> •
|
| 44 |
<a href="#">Tao Ni</a> •
|
| 45 |
<a href="#">Xin Zhan</a> •
|
| 46 |
+
<a href="#">Junbo Chen<sup>†</sup></a> <br>
|
| 47 |
<a href="https://xzhou.me/">Xiaowei Zhou</a> •
|
| 48 |
<a href="https://csse.szu.edu.cn/staff/ruizhenhu/">Ruizhen Hu</a> •
|
| 49 |
+
<a href="https://pengsida.net/">Sida Peng<sup>†</sup></a>
|
| 50 |
</p>
|
| 51 |
|
| 52 |
+
<p align="center"><sup>*</sup> Equal contribution. <sup>†</sup> Corresponding authors.</p>
|
| 53 |
|
| 54 |
</div>
|
| 55 |
|
|
|
|
| 61 |
|
| 62 |
## 📣 News
|
| 63 |
|
| 64 |
+
> **[2026-07]** 🎉 InfiniSplat has been conditionally accepted to SIGGRAPH Asia 2026 (Journal Track)!
|
| 65 |
+
|
| 66 |
> **[2026-07]** 🎉 Inference code for RGB-only and depth-sensor-guided 3D Gaussian reconstruction is available now!
|
| 67 |
|
| 68 |
## 🧩 What can InfiniSplat do?
|
|
|
|
| 112 |
--input examples/data/lidar_demo
|
| 113 |
```
|
| 114 |
|
| 115 |
+
See [docs/inference.md](docs/inference.md) for camera parameters, output control, and other optional arguments.
|
|
|
|
|
|
|
| 116 |
|
| 117 |
+
## 🖥️ Web Demo
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 118 |
|
| 119 |
+
The local Gradio demo provides RGB reconstruction, staged PLY and standalone HTML downloads, and an interactive Gaussian viewer. After preparing the inference environment, launch it with:
|
| 120 |
|
| 121 |
```bash
|
| 122 |
+
python demo.py
|
|
|
|
| 123 |
```
|
| 124 |
|
| 125 |
+
Open `http://127.0.0.1:7860` in a browser. The demo reuses `checkpoints/infinisplat_rgb.ckpt` when available and otherwise downloads the released RGB checkpoint from `PLUS-WAVE/InfiniSplat`. Set `INFINISPLAT_CHECKPOINT` to use a checkpoint at another path.
|
| 126 |
|
| 127 |
+
The hosted version is available on the [Hugging Face Space](https://huggingface.co/spaces/PLUS-WAVE/InfiniSplat).
|
| 128 |
|
| 129 |
## 🙏 Acknowledgments
|
| 130 |
|
app.py
CHANGED
|
@@ -20,7 +20,7 @@ if __name__ == "__main__":
|
|
| 20 |
server_port=int(os.environ.get("PORT", "7860")),
|
| 21 |
allowed_paths=[str(OUTPUT_ROOT)],
|
| 22 |
max_file_size="20mb",
|
| 23 |
-
show_error=
|
| 24 |
ssr_mode=False,
|
| 25 |
footer_links=[],
|
| 26 |
theme=APP_THEME,
|
|
|
|
| 20 |
server_port=int(os.environ.get("PORT", "7860")),
|
| 21 |
allowed_paths=[str(OUTPUT_ROOT)],
|
| 22 |
max_file_size="20mb",
|
| 23 |
+
show_error=False,
|
| 24 |
ssr_mode=False,
|
| 25 |
footer_links=[],
|
| 26 |
theme=APP_THEME,
|
docs/inference.md
CHANGED
|
@@ -19,7 +19,7 @@ The selected checkpoint contains the full inference model. Encoder and decoder w
|
|
| 19 |
|
| 20 |
`--input` accepts one image or a directory. Supported image extensions are `.jpg`, `.jpeg`, `.png`, `.bmp`, and `.webp`. Directory entries are sorted to make batch selection deterministic.
|
| 21 |
|
| 22 |
-
|
| 23 |
|
| 24 |
When `--input` is omitted, each mode uses its bundled example directory:
|
| 25 |
|
|
@@ -133,7 +133,6 @@ The HTML converter reads `config/viewer_settings.json` directly. It does not gen
|
|
| 133 |
| `--checkpoint PATH` | Override the checkpoint selected by the mode. |
|
| 134 |
| `--input PATH` | Process one image or a directory. |
|
| 135 |
| `--output-dir PATH` | Override `outputs/demo/<mode>`. |
|
| 136 |
-
| `--recursive` | Search input subdirectories. |
|
| 137 |
| `--limit N` | Process at most `N` selected images; `0` means all. |
|
| 138 |
| `--overwrite` | Recompute outputs that already exist. |
|
| 139 |
| `--device DEVICE` | Override automatic device selection, for example `cuda:0`. |
|
|
|
|
| 19 |
|
| 20 |
`--input` accepts one image or a directory. Supported image extensions are `.jpg`, `.jpeg`, `.png`, `.bmp`, and `.webp`. Directory entries are sorted to make batch selection deterministic.
|
| 21 |
|
| 22 |
+
Only files directly inside the selected directory are scanned. Use `--limit N` to process only the first `N` selected images. A limit of `0` means no cap.
|
| 23 |
|
| 24 |
When `--input` is omitted, each mode uses its bundled example directory:
|
| 25 |
|
|
|
|
| 133 |
| `--checkpoint PATH` | Override the checkpoint selected by the mode. |
|
| 134 |
| `--input PATH` | Process one image or a directory. |
|
| 135 |
| `--output-dir PATH` | Override `outputs/demo/<mode>`. |
|
|
|
|
| 136 |
| `--limit N` | Process at most `N` selected images; `0` means all. |
|
| 137 |
| `--overwrite` | Recompute outputs that already exist. |
|
| 138 |
| `--device DEVICE` | Override automatic device selection, for example `cuda:0`. |
|
src/demo/hf_ui.py
CHANGED
|
@@ -31,8 +31,8 @@ GPU_DURATION_SECONDS = 6
|
|
| 31 |
REQUEST_CACHE_SECONDS = 3600
|
| 32 |
CACHE_CLEANUP_INTERVAL_SECONDS = 3600
|
| 33 |
FULL_TITLE = "Implicit Gaussian Decoding for Large-Baseline Monocular View Synthesis"
|
| 34 |
-
GITHUB_URL = "https://github.com/
|
| 35 |
-
PROJECT_PAGE_URL = "https://
|
| 36 |
INPUT_IMAGE_HINT = "Better for indoor scenes due to HyperSim-only training"
|
| 37 |
REPO_ROOT = Path(__file__).resolve().parents[2]
|
| 38 |
RGB_EXAMPLE_DIR = REPO_ROOT / "examples/data/rgb_demo"
|
|
@@ -625,7 +625,7 @@ def _gradio_file_url(path: Path) -> str:
|
|
| 625 |
|
| 626 |
|
| 627 |
def _log_timing(stage: str, request_dir: Path, started_at: float, **metrics) -> None:
|
| 628 |
-
"""Emit one structured timing record for
|
| 629 |
payload = {
|
| 630 |
"stage": stage,
|
| 631 |
"request": request_dir.name,
|
|
|
|
| 31 |
REQUEST_CACHE_SECONDS = 3600
|
| 32 |
CACHE_CLEANUP_INTERVAL_SECONDS = 3600
|
| 33 |
FULL_TITLE = "Implicit Gaussian Decoding for Large-Baseline Monocular View Synthesis"
|
| 34 |
+
GITHUB_URL = "https://github.com/zju3dv/InfiniSplat"
|
| 35 |
+
PROJECT_PAGE_URL = "https://zju3dv.github.io/InfiniSplat"
|
| 36 |
INPUT_IMAGE_HINT = "Better for indoor scenes due to HyperSim-only training"
|
| 37 |
REPO_ROOT = Path(__file__).resolve().parents[2]
|
| 38 |
RGB_EXAMPLE_DIR = REPO_ROOT / "examples/data/rgb_demo"
|
|
|
|
| 625 |
|
| 626 |
|
| 627 |
def _log_timing(stage: str, request_dir: Path, started_at: float, **metrics) -> None:
|
| 628 |
+
"""Emit one structured timing record for demo performance checks."""
|
| 629 |
payload = {
|
| 630 |
"stage": stage,
|
| 631 |
"request": request_dir.name,
|
src/demo/infer_batch_images.py
CHANGED
|
@@ -189,6 +189,10 @@ def _resolve_prompt_depth_path(args: argparse.Namespace, image_path: Path) -> Pa
|
|
| 189 |
Prompt depth path for this image, or None when prompt depth is disabled.
|
| 190 |
"""
|
| 191 |
if args.prompt_depth is not None:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 192 |
return args.prompt_depth
|
| 193 |
prompt_depth_dir = args.prompt_depth_dir
|
| 194 |
if prompt_depth_dir is None and args.mode == "lidar":
|
|
@@ -202,7 +206,7 @@ def _resolve_prompt_depth_path(args: argparse.Namespace, image_path: Path) -> Pa
|
|
| 202 |
for ext in PROMPT_DEPTH_EXTENSIONS
|
| 203 |
]
|
| 204 |
for candidate in candidates:
|
| 205 |
-
if candidate.
|
| 206 |
return candidate
|
| 207 |
raise FileNotFoundError(
|
| 208 |
"Prompt depth file not found for "
|
|
|
|
| 189 |
Prompt depth path for this image, or None when prompt depth is disabled.
|
| 190 |
"""
|
| 191 |
if args.prompt_depth is not None:
|
| 192 |
+
if not args.prompt_depth.exists():
|
| 193 |
+
raise FileNotFoundError(f"Prompt depth file not found: {args.prompt_depth}")
|
| 194 |
+
if not args.prompt_depth.is_file():
|
| 195 |
+
raise ValueError(f"Prompt depth path is not a file: {args.prompt_depth}")
|
| 196 |
return args.prompt_depth
|
| 197 |
prompt_depth_dir = args.prompt_depth_dir
|
| 198 |
if prompt_depth_dir is None and args.mode == "lidar":
|
|
|
|
| 206 |
for ext in PROMPT_DEPTH_EXTENSIONS
|
| 207 |
]
|
| 208 |
for candidate in candidates:
|
| 209 |
+
if candidate.is_file() and candidate.resolve() != image_path.resolve():
|
| 210 |
return candidate
|
| 211 |
raise FileNotFoundError(
|
| 212 |
"Prompt depth file not found for "
|
src/demo/infer_single_image.py
CHANGED
|
@@ -99,8 +99,16 @@ def _as_float(value: Any) -> float | None:
|
|
| 99 |
return None
|
| 100 |
|
| 101 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
def convert_focallength_mm_to_px(width: float, height: float, focal_mm: float) -> float:
|
| 103 |
"""Convert 35mm-equivalent focal length in millimeters to pixels."""
|
|
|
|
|
|
|
|
|
|
| 104 |
return focal_mm * math.sqrt(width**2.0 + height**2.0) / math.sqrt(36.0**2 + 24.0**2)
|
| 105 |
|
| 106 |
|
|
@@ -110,6 +118,9 @@ def build_intrinsics_from_focal_px(
|
|
| 110 |
height: int,
|
| 111 |
) -> torch.Tensor:
|
| 112 |
"""Create a pixel-space OpenCV intrinsics matrix."""
|
|
|
|
|
|
|
|
|
|
| 113 |
return torch.tensor(
|
| 114 |
[
|
| 115 |
[focal_length_px, 0.0, (width - 1) / 2.0],
|
|
@@ -190,9 +201,18 @@ def _load_intrinsics_override(path: Path) -> torch.Tensor:
|
|
| 190 |
)
|
| 191 |
else:
|
| 192 |
matrix = container
|
| 193 |
-
|
|
|
|
|
|
|
|
|
|
| 194 |
if intrinsics_px.shape != (3, 3):
|
| 195 |
raise ValueError(f"Expected a 3x3 intrinsics matrix in {path}, got {tuple(intrinsics_px.shape)}.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 196 |
return intrinsics_px
|
| 197 |
|
| 198 |
|
|
@@ -439,6 +459,8 @@ def load_prompt_depth_tensors(
|
|
| 439 |
"""Load one prompt depth file and build prompt tensors for InfiniDepth demo inference."""
|
| 440 |
if not prompt_depth_path.exists():
|
| 441 |
raise FileNotFoundError(f"Prompt depth file not found: {prompt_depth_path}")
|
|
|
|
|
|
|
| 442 |
|
| 443 |
_, prompt_depth_tensor, _ = load_depth(
|
| 444 |
depth_path=prompt_depth_path,
|
|
|
|
| 99 |
return None
|
| 100 |
|
| 101 |
|
| 102 |
+
def _validate_positive_finite(value: float, name: str) -> None:
|
| 103 |
+
if not math.isfinite(float(value)) or float(value) <= 0.0:
|
| 104 |
+
raise ValueError(f"{name} must be a positive finite value, got {value}.")
|
| 105 |
+
|
| 106 |
+
|
| 107 |
def convert_focallength_mm_to_px(width: float, height: float, focal_mm: float) -> float:
|
| 108 |
"""Convert 35mm-equivalent focal length in millimeters to pixels."""
|
| 109 |
+
_validate_positive_finite(width, "Image width")
|
| 110 |
+
_validate_positive_finite(height, "Image height")
|
| 111 |
+
_validate_positive_finite(focal_mm, "Focal length in millimeters")
|
| 112 |
return focal_mm * math.sqrt(width**2.0 + height**2.0) / math.sqrt(36.0**2 + 24.0**2)
|
| 113 |
|
| 114 |
|
|
|
|
| 118 |
height: int,
|
| 119 |
) -> torch.Tensor:
|
| 120 |
"""Create a pixel-space OpenCV intrinsics matrix."""
|
| 121 |
+
_validate_positive_finite(focal_length_px, "Focal length in pixels")
|
| 122 |
+
_validate_positive_finite(width, "Image width")
|
| 123 |
+
_validate_positive_finite(height, "Image height")
|
| 124 |
return torch.tensor(
|
| 125 |
[
|
| 126 |
[focal_length_px, 0.0, (width - 1) / 2.0],
|
|
|
|
| 201 |
)
|
| 202 |
else:
|
| 203 |
matrix = container
|
| 204 |
+
try:
|
| 205 |
+
intrinsics_px = torch.tensor(matrix, dtype=torch.float32)
|
| 206 |
+
except (TypeError, ValueError) as exc:
|
| 207 |
+
raise ValueError(f"Intrinsics matrix in {path} must contain only numeric values.") from exc
|
| 208 |
if intrinsics_px.shape != (3, 3):
|
| 209 |
raise ValueError(f"Expected a 3x3 intrinsics matrix in {path}, got {tuple(intrinsics_px.shape)}.")
|
| 210 |
+
if not torch.isfinite(intrinsics_px).all():
|
| 211 |
+
raise ValueError(f"Intrinsics matrix in {path} must contain only finite values.")
|
| 212 |
+
if intrinsics_px[0, 0] <= 0.0 or intrinsics_px[1, 1] <= 0.0:
|
| 213 |
+
raise ValueError(f"Intrinsics matrix in {path} must have positive fx and fy.")
|
| 214 |
+
if abs(float(torch.linalg.det(intrinsics_px).item())) <= torch.finfo(intrinsics_px.dtype).eps:
|
| 215 |
+
raise ValueError(f"Intrinsics matrix in {path} must be non-singular.")
|
| 216 |
return intrinsics_px
|
| 217 |
|
| 218 |
|
|
|
|
| 459 |
"""Load one prompt depth file and build prompt tensors for InfiniDepth demo inference."""
|
| 460 |
if not prompt_depth_path.exists():
|
| 461 |
raise FileNotFoundError(f"Prompt depth file not found: {prompt_depth_path}")
|
| 462 |
+
if not prompt_depth_path.is_file():
|
| 463 |
+
raise ValueError(f"Prompt depth path is not a file: {prompt_depth_path}")
|
| 464 |
|
| 465 |
_, prompt_depth_tensor, _ = load_depth(
|
| 466 |
depth_path=prompt_depth_path,
|
src/model/encoder/depth/depthpro/depthpro_wrapper.py
CHANGED
|
@@ -1,32 +1,15 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
-
from pathlib import Path
|
| 4 |
from typing import Dict
|
| 5 |
|
| 6 |
import torch
|
| 7 |
-
from omegaconf import DictConfig
|
| 8 |
from torch import nn
|
| 9 |
|
| 10 |
from .depth_pro import DepthProConfig, create_model_and_transforms
|
| 11 |
|
| 12 |
|
| 13 |
-
def
|
| 14 |
-
|
| 15 |
-
"float16": torch.float16,
|
| 16 |
-
"fp16": torch.float16,
|
| 17 |
-
"half": torch.float16,
|
| 18 |
-
"float32": torch.float32,
|
| 19 |
-
"fp32": torch.float32,
|
| 20 |
-
"float": torch.float32,
|
| 21 |
-
"bfloat16": torch.bfloat16,
|
| 22 |
-
"bf16": torch.bfloat16,
|
| 23 |
-
}
|
| 24 |
-
dtype = precision_map.get((precision or "float32").lower())
|
| 25 |
-
if dtype is None:
|
| 26 |
-
raise ValueError(f"Unsupported DepthPro precision: {precision}")
|
| 27 |
-
if device.type == "cpu" and dtype != torch.float32:
|
| 28 |
-
return torch.float32
|
| 29 |
-
return dtype
|
| 30 |
|
| 31 |
|
| 32 |
def _as_depth_map(depth: torch.Tensor, batch_size: int) -> torch.Tensor:
|
|
@@ -67,27 +50,18 @@ def _as_normalized_intrinsics_batch(
|
|
| 67 |
class DepthPro(nn.Module):
|
| 68 |
"""InfiniSplat wrapper around the vendored upstream DepthPro package."""
|
| 69 |
|
| 70 |
-
def __init__(self
|
| 71 |
super().__init__()
|
| 72 |
-
resolved_config = config if config is not None else {}
|
| 73 |
-
self.config = resolved_config
|
| 74 |
-
ckpt_path_value = resolved_config.get("ckpt_path")
|
| 75 |
-
ckpt_path = Path(ckpt_path_value) if ckpt_path_value else None
|
| 76 |
-
if ckpt_path is not None and not ckpt_path.is_file():
|
| 77 |
-
raise FileNotFoundError(f"DepthPro checkpoint not found: {ckpt_path}")
|
| 78 |
-
|
| 79 |
-
self.freeze_depthpro = bool(resolved_config.get("freeze_depthpro", True))
|
| 80 |
self.device_hint = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 81 |
-
self.
|
| 82 |
-
self.precision = _resolve_precision(self.precision_name, self.device_hint)
|
| 83 |
|
| 84 |
depthpro_cfg = DepthProConfig(
|
| 85 |
-
patch_encoder_preset=
|
| 86 |
-
image_encoder_preset=
|
| 87 |
-
decoder_features=
|
| 88 |
-
checkpoint_uri=
|
| 89 |
-
fov_encoder_preset=
|
| 90 |
-
use_fov_head=
|
| 91 |
)
|
| 92 |
self.model, self.transform = create_model_and_transforms(
|
| 93 |
config=depthpro_cfg,
|
|
@@ -95,16 +69,7 @@ class DepthPro(nn.Module):
|
|
| 95 |
precision=self.precision,
|
| 96 |
)
|
| 97 |
|
| 98 |
-
|
| 99 |
-
for param in self.model.parameters():
|
| 100 |
-
param.requires_grad_(False)
|
| 101 |
-
self.model.eval()
|
| 102 |
-
|
| 103 |
-
def train(self, mode: bool = True):
|
| 104 |
-
super().train(mode)
|
| 105 |
-
if self.freeze_depthpro:
|
| 106 |
-
self.model.eval()
|
| 107 |
-
return self
|
| 108 |
|
| 109 |
def _prepare_input(self, image: torch.Tensor) -> torch.Tensor:
|
| 110 |
if image.ndim != 4:
|
|
@@ -118,7 +83,7 @@ class DepthPro(nn.Module):
|
|
| 118 |
|
| 119 |
def _ensure_runtime_dtype(self) -> None:
|
| 120 |
param = next(self.model.parameters())
|
| 121 |
-
desired_dtype =
|
| 122 |
if param.dtype != desired_dtype:
|
| 123 |
self.model.to(dtype=desired_dtype)
|
| 124 |
self.precision = desired_dtype
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
|
|
|
| 3 |
from typing import Dict
|
| 4 |
|
| 5 |
import torch
|
|
|
|
| 6 |
from torch import nn
|
| 7 |
|
| 8 |
from .depth_pro import DepthProConfig, create_model_and_transforms
|
| 9 |
|
| 10 |
|
| 11 |
+
def _inference_precision(device: torch.device) -> torch.dtype:
|
| 12 |
+
return torch.float16 if device.type == "cuda" else torch.float32
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
|
| 15 |
def _as_depth_map(depth: torch.Tensor, batch_size: int) -> torch.Tensor:
|
|
|
|
| 50 |
class DepthPro(nn.Module):
|
| 51 |
"""InfiniSplat wrapper around the vendored upstream DepthPro package."""
|
| 52 |
|
| 53 |
+
def __init__(self):
|
| 54 |
super().__init__()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
self.device_hint = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 56 |
+
self.precision = _inference_precision(self.device_hint)
|
|
|
|
| 57 |
|
| 58 |
depthpro_cfg = DepthProConfig(
|
| 59 |
+
patch_encoder_preset="dinov2l16_384",
|
| 60 |
+
image_encoder_preset="dinov2l16_384",
|
| 61 |
+
decoder_features=256,
|
| 62 |
+
checkpoint_uri=None,
|
| 63 |
+
fov_encoder_preset="dinov2l16_384",
|
| 64 |
+
use_fov_head=True,
|
| 65 |
)
|
| 66 |
self.model, self.transform = create_model_and_transforms(
|
| 67 |
config=depthpro_cfg,
|
|
|
|
| 69 |
precision=self.precision,
|
| 70 |
)
|
| 71 |
|
| 72 |
+
self.model.eval()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
|
| 74 |
def _prepare_input(self, image: torch.Tensor) -> torch.Tensor:
|
| 75 |
if image.ndim != 4:
|
|
|
|
| 83 |
|
| 84 |
def _ensure_runtime_dtype(self) -> None:
|
| 85 |
param = next(self.model.parameters())
|
| 86 |
+
desired_dtype = _inference_precision(param.device)
|
| 87 |
if param.dtype != desired_dtype:
|
| 88 |
self.model.to(dtype=desired_dtype)
|
| 89 |
self.precision = desired_dtype
|
src/model/encoder/depth/infinidepth/infinidepth_wrapper.py
CHANGED
|
@@ -11,7 +11,7 @@ from src.model.encoder.depth.infinidepth.sampling_utils import make_2d_uniform_c
|
|
| 11 |
|
| 12 |
|
| 13 |
class InfiniDepth(nn.Module):
|
| 14 |
-
"""
|
| 15 |
|
| 16 |
def __init__(self) -> None:
|
| 17 |
super().__init__()
|
|
@@ -20,15 +20,8 @@ class InfiniDepth(nn.Module):
|
|
| 20 |
geometry_type="disparity",
|
| 21 |
use_prompt=True,
|
| 22 |
)
|
| 23 |
-
for parameter in self.model.parameters():
|
| 24 |
-
parameter.requires_grad_(False)
|
| 25 |
self.model.eval()
|
| 26 |
|
| 27 |
-
def train(self, mode: bool = True):
|
| 28 |
-
super().train(mode)
|
| 29 |
-
self.model.eval()
|
| 30 |
-
return self
|
| 31 |
-
|
| 32 |
@torch.inference_mode()
|
| 33 |
def forward(self, batch: dict[str, torch.Tensor]) -> torch.Tensor:
|
| 34 |
"""Predict a dense depth map from RGB and sparse disparity prompts."""
|
|
|
|
| 11 |
|
| 12 |
|
| 13 |
class InfiniDepth(nn.Module):
|
| 14 |
+
"""Prompt-conditioned InfiniDepth dense-depth predictor."""
|
| 15 |
|
| 16 |
def __init__(self) -> None:
|
| 17 |
super().__init__()
|
|
|
|
| 20 |
geometry_type="disparity",
|
| 21 |
use_prompt=True,
|
| 22 |
)
|
|
|
|
|
|
|
| 23 |
self.model.eval()
|
| 24 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
@torch.inference_mode()
|
| 26 |
def forward(self, batch: dict[str, torch.Tensor]) -> torch.Tensor:
|
| 27 |
"""Predict a dense depth map from RGB and sparse disparity prompts."""
|
src/model/encoder/encoder_infinidepth_query.py
CHANGED
|
@@ -36,8 +36,6 @@ class EncoderInfiniDepthQuery(Encoder[EncoderInfiniDepthQueryCfg]):
|
|
| 36 |
|
| 37 |
self.depth_predictor = InfiniDepth()
|
| 38 |
self.depth_predictor.eval()
|
| 39 |
-
for param in self.depth_predictor.parameters():
|
| 40 |
-
param.requires_grad_(False)
|
| 41 |
|
| 42 |
self.image_feature_branch = DinoBasicImageFeatureBranch(
|
| 43 |
backbone_type=cfg.image_backbone_type,
|
|
@@ -51,11 +49,6 @@ class EncoderInfiniDepthQuery(Encoder[EncoderInfiniDepthQueryCfg]):
|
|
| 51 |
|
| 52 |
self.gaussian_decoder = GaussianDecoder(cfg=cfg.gaussian_decoder)
|
| 53 |
|
| 54 |
-
def train(self, mode: bool = True):
|
| 55 |
-
super().train(mode)
|
| 56 |
-
self.depth_predictor.eval()
|
| 57 |
-
return self
|
| 58 |
-
|
| 59 |
def _sample_map(
|
| 60 |
self,
|
| 61 |
feature_map: torch.Tensor,
|
|
|
|
| 36 |
|
| 37 |
self.depth_predictor = InfiniDepth()
|
| 38 |
self.depth_predictor.eval()
|
|
|
|
|
|
|
| 39 |
|
| 40 |
self.image_feature_branch = DinoBasicImageFeatureBranch(
|
| 41 |
backbone_type=cfg.image_backbone_type,
|
|
|
|
| 49 |
|
| 50 |
self.gaussian_decoder = GaussianDecoder(cfg=cfg.gaussian_decoder)
|
| 51 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
def _sample_map(
|
| 53 |
self,
|
| 54 |
feature_map: torch.Tensor,
|
src/model/encoder/encoder_infinisplat.py
CHANGED
|
@@ -111,8 +111,6 @@ class EncoderInfiniSplat(Encoder[EncoderInfiniSplatCfg]):
|
|
| 111 |
|
| 112 |
self.depth_predictor = DepthPro()
|
| 113 |
self.depth_predictor.eval()
|
| 114 |
-
for param in self.depth_predictor.parameters():
|
| 115 |
-
param.requires_grad_(False)
|
| 116 |
|
| 117 |
self.image_feature_branch = DinoBasicImageFeatureBranch(
|
| 118 |
backbone_type=cfg.image_backbone_type,
|
|
@@ -126,11 +124,6 @@ class EncoderInfiniSplat(Encoder[EncoderInfiniSplatCfg]):
|
|
| 126 |
|
| 127 |
self.gaussian_decoder = GaussianDecoder(cfg=cfg.gaussian_decoder)
|
| 128 |
|
| 129 |
-
def train(self, mode: bool = True):
|
| 130 |
-
super().train(mode)
|
| 131 |
-
self.depth_predictor.eval()
|
| 132 |
-
return self
|
| 133 |
-
|
| 134 |
def _sample_map(
|
| 135 |
self,
|
| 136 |
feature_map: torch.Tensor,
|
|
@@ -246,7 +239,6 @@ class EncoderInfiniSplat(Encoder[EncoderInfiniSplatCfg]):
|
|
| 246 |
"InfiniSplat expects the selected depth model to return a single dense depth layer."
|
| 247 |
)
|
| 248 |
|
| 249 |
-
# Sample selection is discrete; keep gradients on sampled depths only.
|
| 250 |
sampling_output_flat = self._sample_sparse_coords(
|
| 251 |
dense_depthmap_flat=dense_depthmap_flat.detach(),
|
| 252 |
intrinsics_flat=intrinsics_flat,
|
|
|
|
| 111 |
|
| 112 |
self.depth_predictor = DepthPro()
|
| 113 |
self.depth_predictor.eval()
|
|
|
|
|
|
|
| 114 |
|
| 115 |
self.image_feature_branch = DinoBasicImageFeatureBranch(
|
| 116 |
backbone_type=cfg.image_backbone_type,
|
|
|
|
| 124 |
|
| 125 |
self.gaussian_decoder = GaussianDecoder(cfg=cfg.gaussian_decoder)
|
| 126 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 127 |
def _sample_map(
|
| 128 |
self,
|
| 129 |
feature_map: torch.Tensor,
|
|
|
|
| 239 |
"InfiniSplat expects the selected depth model to return a single dense depth layer."
|
| 240 |
)
|
| 241 |
|
|
|
|
| 242 |
sampling_output_flat = self._sample_sparse_coords(
|
| 243 |
dense_depthmap_flat=dense_depthmap_flat.detach(),
|
| 244 |
intrinsics_flat=intrinsics_flat,
|