mirror of
https://github.com/SkyworkAI/Matrix-3D
synced 2026-11-21 23:36:52 +08:00
134 lines
6.8 KiB
Python
134 lines
6.8 KiB
Python
import os
|
|
os.environ['OPENCV_IO_ENABLE_OPENEXR'] = '1'
|
|
from pathlib import Path
|
|
import sys
|
|
sys.path.append(str(Path(__file__).absolute().parents[1]))
|
|
|
|
from typing import *
|
|
import itertools
|
|
import json
|
|
import warnings
|
|
|
|
import cv2
|
|
import numpy as np
|
|
import torch
|
|
from PIL import Image
|
|
from tqdm import tqdm
|
|
import trimesh
|
|
import trimesh.visual
|
|
import click
|
|
|
|
from moge.model import MoGeModel
|
|
from moge.utils.io import save_glb, save_ply
|
|
from moge.utils.vis import colorize_depth, colorize_normal
|
|
import utils3d
|
|
|
|
|
|
@click.command(help='Inference script for the MoGe model.')
|
|
@click.option('--input', 'input_path', type=click.Path(exists=True), help='Input image or folder path. "jpg" and "png" are supported.')
|
|
@click.option('--fov_x', 'fov_x_', type=float, default=None, help='If camera parameters are known, set the horizontal field of view in degrees. Otherwise, MoGe will estimate it.')
|
|
@click.option('--output', 'output_path', type=click.Path(), help='Output folder path')
|
|
@click.option('--pretrained', 'pretrained_model_name_or_path', type=str, default='/mnt/workspace/zhongqi.yang/VideoInpainting_new/ViewCrafter/extern/MoGe/checkpoints/model.pt', help='Pretrained model name or path. Default is "Ruicheng/moge-vitl"')
|
|
@click.option('--device', 'device_name', type=str, default='cuda', help='Device name (e.g. "cuda", "cuda:0", "cpu"). Default is "cuda"')
|
|
@click.option('--resize', 'resize_to', type=int, default=None, help='Resize the image(s) & output maps to a specific size. Default is None (no resizing).')
|
|
@click.option('--resolution_level', type=int, default=9, help='An integer [0-9] for the resolution level of inference. The higher, the better but slower. Default is 9. Note that it is irrelevant to the output resolution.')
|
|
@click.option('--threshold', type=float, default=0.03, help='Threshold for removing edges. Default is 0.03. Smaller value removes more edges. "inf" means no thresholding.')
|
|
@click.option('--maps', 'save_maps_', is_flag=True, help='Whether to save the output maps and fov(image, depth, mask, points, fov).')
|
|
@click.option('--glb', 'save_glb_', is_flag=True, help='Whether to save the output as a.glb file. The color will be saved as a texture.')
|
|
@click.option('--ply', 'save_ply_', is_flag=True, help='Whether to save the output as a.ply file. The color will be saved as vertex colors.')
|
|
@click.option('--show', 'show', is_flag=True, help='Whether show the output in a window. Note that this requires pyglet<2 installed as required by trimesh.')
|
|
def main(
|
|
input_path: str,
|
|
fov_x_: float,
|
|
output_path: str,
|
|
pretrained_model_name_or_path: str,
|
|
device_name: str,
|
|
resize_to: int,
|
|
resolution_level: int,
|
|
threshold: float,
|
|
save_maps_: bool,
|
|
save_glb_: bool,
|
|
save_ply_: bool,
|
|
show: bool,
|
|
):
|
|
device = torch.device(device_name)
|
|
|
|
include_suffices = ['jpg', 'png', 'jpeg', 'JPG', 'PNG', 'JPEG']
|
|
if Path(input_path).is_dir():
|
|
image_paths = sorted(itertools.chain(*(Path(input_path).rglob(f'*.{suffix}') for suffix in include_suffices)))
|
|
else:
|
|
image_paths = [Path(input_path)]
|
|
|
|
if len(image_paths) == 0:
|
|
raise FileNotFoundError(f'No image files found in {input_path}')
|
|
|
|
model = MoGeModel.from_pretrained(pretrained_model_name_or_path).to(device).eval()
|
|
|
|
for image_path in (pbar := tqdm(image_paths, desc='Inference', disable=len(image_paths) <= 1)):
|
|
image = cv2.cvtColor(cv2.imread(str(image_path)), cv2.COLOR_BGR2RGB)
|
|
height, width = image.shape[:2]
|
|
if resize_to is not None:
|
|
height, width = min(resize_to, int(resize_to * height / width)), min(resize_to, int(resize_to * width / height))
|
|
image = cv2.resize(image, (width, height), cv2.INTER_AREA)
|
|
image_tensor = torch.tensor(image / 255, dtype=torch.float32, device=device).permute(2, 0, 1)
|
|
|
|
# Inference
|
|
output = model.infer(image_tensor, fov_x=fov_x_)
|
|
points, depth, mask, intrinsics = output['points'].cpu().numpy(), output['depth'].cpu().numpy(), output['mask'].cpu().numpy(), output['intrinsics'].cpu().numpy()
|
|
# yep; there is intrinsics.
|
|
normals, normals_mask = utils3d.numpy.points_to_normals(points, mask=mask)
|
|
|
|
# Write outputs
|
|
if not any([save_maps_, save_glb_, save_ply_]):
|
|
warnings.warn('No output format specified. Please use "--maps", "--glb", or "--ply" to specify the output.')
|
|
|
|
save_path = Path(output_path, image_path.relative_to(input_path).parent, image_path.stem)
|
|
save_path.mkdir(exist_ok=True, parents=True)
|
|
|
|
if save_maps_:
|
|
cv2.imwrite(str(save_path / 'image.jpg'), cv2.cvtColor(image, cv2.COLOR_RGB2BGR))
|
|
cv2.imwrite(str(save_path / 'depth_vis.png'), cv2.cvtColor(colorize_depth(depth), cv2.COLOR_RGB2BGR))
|
|
cv2.imwrite(str(save_path / 'depth.exr'), depth, [cv2.IMWRITE_EXR_TYPE, cv2.IMWRITE_EXR_TYPE_FLOAT])
|
|
cv2.imwrite(str(save_path / 'mask.png'), (mask * 255).astype(np.uint8))
|
|
cv2.imwrite(str(save_path / 'points.exr'), cv2.cvtColor(points, cv2.COLOR_RGB2BGR), [cv2.IMWRITE_EXR_TYPE, cv2.IMWRITE_EXR_TYPE_FLOAT])
|
|
fov_x, fov_y = utils3d.numpy.intrinsics_to_fov(intrinsics)
|
|
with open(save_path / 'fov.json', 'w') as f:
|
|
json.dump({
|
|
'fov_x': round(float(np.rad2deg(fov_x)), 2),
|
|
'fov_y': round(float(np.rad2deg(fov_y)), 2),
|
|
}, f)
|
|
|
|
# Export mesh & visulization
|
|
if save_glb_ or save_ply_ or show:
|
|
faces, vertices, vertex_colors, vertex_uvs = utils3d.numpy.image_mesh(
|
|
points,
|
|
image.astype(np.float32) / 255,
|
|
utils3d.numpy.image_uv(width=width, height=height),
|
|
mask=mask & ~(utils3d.numpy.depth_edge(depth, rtol=threshold, mask=mask) & utils3d.numpy.normals_edge(normals, tol=5, mask=normals_mask)),
|
|
tri=True
|
|
)
|
|
# When exporting the model, follow the OpenGL coordinate conventions:
|
|
# - world coordinate system: x right, y up, z backward.
|
|
# - texture coordinate system: (0, 0) for left-bottom, (1, 1) for right-top.
|
|
vertices, vertex_uvs = vertices * [1, -1, -1], vertex_uvs * [1, -1] + [0, 1]
|
|
mm = depth<1e6
|
|
print(f"vertices range check: {vertices[...,2].max()} {vertices[...,2].min()} {depth[mm].max()} {depth[mm].min()} {vertices.shape} {depth.shape}")
|
|
print(f"intrinsics check: {intrinsics}")
|
|
if save_glb_:
|
|
save_glb(save_path / 'mesh.glb', vertices, faces, vertex_uvs, image)
|
|
|
|
if save_ply_:
|
|
save_ply(save_path / 'mesh.ply', vertices, faces, vertex_colors)
|
|
|
|
if show:
|
|
trimesh.Trimesh(
|
|
vertices=vertices,
|
|
vertex_colors=vertex_colors,
|
|
faces=faces,
|
|
process=False
|
|
).show()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|