Files
2025-08-11 17:51:56 +08:00

1388 lines
56 KiB
Python

import os
import sys
import torch
import numpy as np
import trimesh
from numbers import Number
from typing import *
os.environ['OPENCV_IO_ENABLE_OPENEXR'] = "1"
import trimesh
import math
import json
import cv2
from scipy import ndimage
from scipy.spatial.transform import Rotation as R
import nvdiffrast.torch as dr
import time
import skimage as ski
import warnings
class no_warnings:
def __init__(self, action: str = 'ignore', **kwargs):
self.action = action
self.filter_kwargs = kwargs
def __call__(self, fn):
@wraps(fn)
def wrapper(*args, **kwargs):
with warnings.catch_warnings():
warnings.simplefilter(self.action, **self.filter_kwargs)
return fn(*args, **kwargs)
return wrapper
def __enter__(self):
self.warnings_manager = warnings.catch_warnings()
self.warnings_manager.__enter__()
warnings.simplefilter(self.action, **self.filter_kwargs)
def __exit__(self, exc_type, exc_val, exc_tb):
self.warnings_manager.__exit__(exc_type, exc_val, exc_tb)
def directions_to_spherical_uv_torch(directions):
directions = directions / torch.linalg.norm(directions, dim=-1, keepdims=True)
u = 1 - torch.arctan2(directions[..., 1], directions[..., 0]) / (2 * torch.pi) % 1.0
v = torch.arccos(directions[..., 2]) / torch.pi
return torch.stack([u, v], dim=-1)
def image_uv(
height: int,
width: int,
left: int = None,
top: int = None,
right: int = None,
bottom: int = None,
dtype: np.dtype = np.float32
) -> np.ndarray:
if left is None: left = 0
if top is None: top = 0
if right is None: right = width
if bottom is None: bottom = height
u = np.linspace((left + 0.5) / width, (right - 0.5) / width, right - left, dtype=dtype)
v = np.linspace((top + 0.5) / height, (bottom - 0.5) / height, bottom - top, dtype=dtype)
u, v = np.meshgrid(u, v, indexing='xy')
return np.stack([u, v], axis=2)
def spherical_uv_to_directions(uv: np.ndarray):
theta, phi = (1 - uv[..., 0]) * (2 * np.pi), uv[..., 1] * np.pi
directions = np.stack([np.sin(phi) * np.cos(theta), np.sin(phi) * np.sin(theta), np.cos(phi)], axis=-1)
return directions
def uv_to_pixel_torch(
uv,
width,
height,
):
"""
Args:
pixel (np.ndarray): [..., 2] pixel coordinrates defined in image space, x range is (0, W - 1), y range is (0, H - 1)
width (int | np.ndarray): [...] image width(s)
height (int | np.ndarray): [...] image height(s)
Returns:
(np.ndarray): [..., 2] pixel coordinrates defined in uv space, the range is (0, 1)
"""
if not isinstance(width, int):
pixel = uv * torch.stack([width, height], dim=-1).to(uv.dtype).to(uv.device) - 0.5
else:
Ns = uv.ndim
tmp = torch.tensor([width, height]).to(uv.dtype).to(uv.device)
for i in range(Ns - 1):
tmp = tmp.unsqueeze(0)
pixel = uv * tmp - 0.5
return pixel
def depth_edge_torch(depth: np.ndarray, atol: float = None, rtol: float = None, kernel_size: int = 3, mask = None) -> np.ndarray:
"""
Compute the edge mask from depth map. The edge is defined as the pixels whose neighbors have large difference in depth.
Args:
depth (np.ndarray): shape (..., height, width), linear depth map
atol (float): absolute tolerance
rtol (float): relative tolerance
Returns:
edge (np.ndarray): shape (..., height, width) of dtype torch.bool
"""
func = torch.nn.MaxPool2d(kernel_size, stride=1, padding=kernel_size//2)
if mask is None:
# diff = (max_pool_2d(depth, kernel_size, stride=1, padding=kernel_size // 2) + max_pool_2d(-depth, kernel_size, stride=1, padding=kernel_size // 2))
diff = (func(depth[None,None]) + func(-depth[None,None]))[0,0]
else:
diff = (func(torch.where(mask, depth, -torch.inf)[None,None]) + func(torch.where(mask, -depth, -torch.inf)[None,None]))[0,0]
edge = torch.zeros_like(depth, dtype=torch.bool, device=depth.device)
if atol is not None:
edge |= diff > atol
if rtol is not None:
edge |= diff / depth > rtol
return edge
def correct_pano_depth_batch(pano_depth):
# ok...
# pano_depth: [N, H, W] numpy array
# pano_rgb: [H, W, 3] numpy array
# spherical_uv_to_directions(uv: np.ndarray):
N, H, W = pano_depth.shape
uv = image_uv(width=W, height=H)
directions = spherical_uv_to_directions(uv)
pivot_directions = np.stack([directions[0, W//2],directions[H//2, W//2],directions[H-1, W//2],directions[H//2, 0],directions[H//2, W//4],directions[H//2, 3*W//4]])
cos = (np.sum(pivot_directions[:,None,None,None,:] * directions[None], axis=-1)).max(axis=0)
pano_depth_correct = pano_depth / cos
return pano_depth_correct
def unreal_to_opencv_w2c(pitch, yaw, roll, x, y, z):
# Step 1: 从欧拉角构造旋转矩阵(Unreal顺序)
r_unreal = R.from_euler('YXZ', [yaw, pitch, roll], degrees=True)
R_unreal = r_unreal.as_matrix() # 3x3旋转矩阵
# Step 3: 世界到相机坐标(变换旋转和平移)
R_cv = R_unreal
# t_cv = np.array([x, y, -z])
t_cv = np.array([y, -z, x])
# Step 4: 构造4x4的 W2C 矩阵
T_c2w = np.eye(4)
T_c2w[:3, :3] = R_cv
T_c2w[:3, 3] = t_cv
#print(f"tcv: {t_cv}")
T_w2c = np.linalg.inv(T_c2w)
return T_w2c
def image_uv_torch(height: int, width: int, left: int = None, top: int = None, right: int = None, bottom: int = None, device: torch.device = None, dtype: torch.dtype = None) -> torch.Tensor:
if left is None: left = 0
if top is None: top = 0
if right is None: right = width
if bottom is None: bottom = height
u = torch.linspace((left + 0.5) / width, (right - 0.5) / width, right - left, device=device, dtype=dtype)
v = torch.linspace((top + 0.5) / height, (bottom - 0.5) / height, bottom - top, device=device, dtype=dtype)
u, v = torch.meshgrid(u, v, indexing='xy')
uv = torch.stack([u, v], dim=-1)
return uv
def spherical_uv_to_directions_torch(uv):
# device = uv.device
theta, phi = (1 - uv[..., 0]) * (2 * torch.pi), uv[..., 1] * torch.pi
directions = torch.stack([torch.sin(phi) * torch.cos(theta), torch.sin(phi) * torch.sin(theta), torch.cos(phi)], axis=-1)
return directions
def get_pano_pcs_torch(pano_depth):
device = pano_depth.device
H, W = pano_depth.shape
uv = image_uv_torch(width=W, height=H, device=device)
directions = spherical_uv_to_directions_torch(uv)
vertices_cam = pano_depth[:,:,None] * directions
return vertices_cam.reshape((-1,3))
def get_world_pcs_pano_torch(csv_cam, depth):
device = depth.device
# 提取深度在相机系下的点云;
rot_matrix = torch.tensor([
[0,1,0],[0,0,-1],[-1,0,0.]
], dtype=depth.dtype,device=device)
cam_pcs = get_pano_pcs_torch(depth) @ rot_matrix.T
X_,Y_,Z_,p,y,r = csv_cam
w2c = unreal_to_opencv_w2c(p,y,r,X_,Y_,Z_)
c2w = torch.from_numpy(np.linalg.inv(w2c)).to(device).to(depth.dtype)
# print(cam_pcs.dtype, c2w.dtype)
world_pcs = cam_pcs @ c2w[:3,:3].T + c2w[:3,3][None]
return world_pcs
def get_world_pcs_pano_torch_Rt(Rt, depth):
device = depth.device
# 提取深度在相机系下的点云;
rot_matrix = torch.tensor([
[0,1,0],[0,0,-1],[-1,0,0.]
], dtype=depth.dtype,device=device)
cam_pcs = get_pano_pcs_torch(depth) @ rot_matrix.T
w2c = Rt
c2w = torch.from_numpy(np.linalg.inv(w2c)).to(device).to(depth.dtype)
# print(cam_pcs.dtype, c2w.dtype)
world_pcs = cam_pcs @ c2w[:3,:3].T + c2w[:3,3][None]
return world_pcs
def get_mesh_from_pano(pano_depth, pano_mask, csv_cam,dtype=np.float32):
# ok...
# pano_depth: [H, W] numpy array
# pano_rgb: [H, W, 3] numpy array
# spherical_uv_to_directions(uv: np.ndarray):
H, W = pano_depth.shape
uv = image_uv(width=W, height=H)
pano_depth_ = torch.from_numpy(pano_depth).float()
wrd_pcs = get_world_pcs_pano_torch(csv_cam, pano_depth_)
vertices = wrd_pcs.reshape(H,W,3).numpy()
#directions = spherical_uv_to_directions(uv)
#vertices = pano_depth[:,:,None] * directions + origin_position
pts_topology = np.zeros((H - 1, W, 2, 3), dtype=np.int32)
index_ = np.arange(H*W).astype(np.int32).reshape(H,W)
pts_topology[:,:-1,0,0] = index_[:-1,:-1]
pts_topology[:,:-1,0,2] = index_[:-1,1:]
pts_topology[:,:-1,0,1] = index_[1:,:-1]
pts_topology[:,-1,0,0] = index_[:-1,-1]
pts_topology[:,-1,0,2] = index_[:-1,0]
pts_topology[:,-1,0,1] = index_[1:,-1]
pts_topology[:,:,1,0] = index_[:-1,1:]
pts_topology[:,:,1,1] = index_[1:,:-1]
pts_topology[:,:,1,2] = index_[1:,1:]
pts_topology[:,-1,1,0] = index_[:-1,0]
pts_topology[:,-1,1,1] = index_[1:,-1]
pts_topology[:,-1,1,2] = index_[1:,0]
pano_mask_ext = np.zeros((H,W+1),dtype=np.bool_)
pano_mask_ext[:,:-1] = pano_mask[:,:]
pano_mask_ext[:,-1] = pano_mask[:,0]
topo_mask_0 = pano_mask_ext[:-1,:-1] + pano_mask_ext[:-1,1:] + pano_mask_ext[1:,:-1]
topo_mask_1 = pano_mask_ext[:-1,1:] + pano_mask_ext[1:,:-1] + pano_mask_ext[1:,1:]
topo0 = pts_topology[:,:,0][topo_mask_0]
topo1 = pts_topology[:,:,1][topo_mask_1]
topo = np.concatenate([topo0, topo1], axis=0)
vertices = vertices.reshape(-1,3)
mesh_trimesh = trimesh.Trimesh(vertices=vertices, faces=topo)
print(f"mesh construction complete; mesh info: {mesh_trimesh.vertices.shape} {mesh_trimesh.faces.shape}")
return mesh_trimesh
def get_mesh_from_pano_Rt(pano_depth, pano_mask, Rt, dtype=np.float32, pano_rgb=None, alpha_mask=None):
# ok...
# pano_depth: [H, W] numpy array
# pano_rgb: [H, W, 3] numpy array
# spherical_uv_to_directions(uv: np.ndarray):
H, W = pano_depth.shape
uv = image_uv(width=W, height=H)
pano_depth_ = torch.from_numpy(pano_depth).float()
wrd_pcs = get_world_pcs_pano_torch_Rt(Rt, pano_depth_)
vertices = wrd_pcs.reshape(H,W,3).numpy()
#directions = spherical_uv_to_directions(uv)
#vertices = pano_depth[:,:,None] * directions + origin_position
pts_topology = np.zeros((H - 1, W, 2, 3), dtype=np.int32)
index_ = np.arange(H*W).astype(np.int32).reshape(H,W)
pts_topology[:,:-1,0,0] = index_[:-1,:-1]
pts_topology[:,:-1,0,2] = index_[:-1,1:]
pts_topology[:,:-1,0,1] = index_[1:,:-1]
pts_topology[:,-1,0,0] = index_[:-1,-1]
pts_topology[:,-1,0,2] = index_[:-1,0]
pts_topology[:,-1,0,1] = index_[1:,-1]
pts_topology[:,:-1,1,0] = index_[:-1,1:]
pts_topology[:,:-1,1,1] = index_[1:,:-1]
pts_topology[:,:-1,1,2] = index_[1:,1:]
pts_topology[:,-1,1,0] = index_[:-1,0]
pts_topology[:,-1,1,1] = index_[1:,-1]
pts_topology[:,-1,1,2] = index_[1:,0]
pano_mask_ext = np.zeros((H,W+1),dtype=np.bool_)
pano_mask_ext[:,:-1] = pano_mask[:,:]
pano_mask_ext[:,-1] = pano_mask[:,0]
topo_mask_0 = pano_mask_ext[:-1,:-1] + pano_mask_ext[:-1,1:] + pano_mask_ext[1:,:-1]
topo_mask_1 = pano_mask_ext[:-1,1:] + pano_mask_ext[1:,:-1] + pano_mask_ext[1:,1:]
topo0 = pts_topology[:,:,0][topo_mask_0]
topo1 = pts_topology[:,:,1][topo_mask_1]
topo = np.concatenate([topo0, topo1], axis=0)
vertices = vertices.reshape(-1,3)
if pano_rgb is None and alpha_mask is None:
mesh_trimesh = trimesh.Trimesh(vertices=vertices, faces=topo)
elif alpha_mask is None:
pano_rgb = pano_rgb.reshape(-1,3)
mesh_trimesh = trimesh.Trimesh(vertices=vertices, faces=topo, vertex_colors = pano_rgb)
elif alpha_mask is not None:
if pano_rgb is not None:
pano_rgb = pano_rgb.reshape(-1,3)
else:
pano_rgb = np.ones((H*W,3),dtype=np.float32)
pano_rgb_ = np.ones((H*W,4),dtype=np.float32)
pano_rgb_[:,:3] = pano_rgb
alpha_mask = alpha_mask.reshape(-1)
pano_rgb_[~alpha_mask] = 0.
mesh_trimesh = trimesh.Trimesh(vertices=vertices, faces=topo, vertex_colors = pano_rgb_)
print(f"mesh construction complete; mesh info: {mesh_trimesh.vertices.shape} {mesh_trimesh.faces.shape}")
return mesh_trimesh
def get_diffrast_camera_parameter_from_cv(K, H, W, near, far, device):
K_ = torch.zeros((4,4), dtype=torch.float32, device=device)
K_[0,0] = K[0,0] * 2. / W
K_[1,1] = K[1,1] * 2. / H
K_[2,2] = (far + near) / (far - near)
K_[2,3] = - 2. * (near * far) / (far - near)
K_[3,2] = 1.
return K_
def get_mesh_render_depth(mesh, Ks, Rts, H, W, near, far, device):
#print("begin",Ks.shape, Rts.shape)
with torch.no_grad():
if isinstance(Ks, np.ndarray):
Ks = torch.from_numpy(Ks).float().to(device)
Rts = torch.from_numpy(Rts).float().to(device)
#print("sdafa0")
glctx = dr.RasterizeCudaContext(device=device)
#glctx = dr.RasterizeGLContext(device=device)
#print("sdafa_u")
pos_tensor = torch.from_numpy(np.array(mesh.vertices)).float().to(device)
tri_tensor = torch.from_numpy(np.array(mesh.faces)).int().to(device)
pos_qc = torch.ones((Rts.shape[0],pos_tensor.shape[0],4),dtype=torch.float32, device=device)
N = Rts.shape[0]
all_images = []
#print("sdafa")
print(pos_tensor.shape, Rts.shape)
pos_tensor_cam = pos_tensor[None] @ Rts[:,:3,:3].permute(0,2,1) + Rts[:,:3,3][:,None,:]
K_ = get_diffrast_camera_parameter_from_cv(Ks[0], H, W, near, far, device)
pos_qc[:,:,:3] = pos_tensor_cam
dist = pos_tensor_cam.norm(dim=-1)[:,:,None].repeat(1,1,2)
dist[:,:,1] = 1.
pos_rast = (pos_qc @ (K_.T)[None])
rast, _ = dr.rasterize(glctx, pos_rast, tri_tensor, resolution=[H, W],grad_db=False)
out, _ = dr.interpolate(dist, rast, tri_tensor)
img = out.cpu().numpy()[:, :, :, :] # Flip vertically.
all_images=(img[:,:,:,:2])
all_canvas = all_images[:,:,:,0]
all_canvas_mask = all_images[:,:,:,1] > 1. - 1e-4
#all_canvas,all_canvas_mask = fill_image_nvdiffrast(all_canvas,all_canvas_mask)
return all_canvas,all_canvas_mask
def get_mesh_render_color(mesh, Ks, Rts, H, W, near, far, device):
with torch.no_grad():
if isinstance(Ks, np.ndarray):
Ks = torch.from_numpy(Ks).float().to(device)
Rts = torch.from_numpy(Rts).float().to(device)
# import pdb
# pdb.set_trace()
glctx = dr.RasterizeCudaContext(device=device)
pos_tensor = torch.from_numpy(np.array(mesh.vertices)).float().to(device)
col_tensor = torch.from_numpy(np.array(mesh.visual.vertex_colors)).float().to(device)
print(col_tensor.shape,col_tensor.max())
tri_tensor = torch.from_numpy(np.array(mesh.faces)).int().to(device)
if col_tensor.max() > 5:
col_tensor /= 255.
pos_qc = torch.ones((pos_tensor.shape[0],4),dtype=torch.float32, device=device)
N = Rts.shape[0]
all_images = []
for i in range(N):
pos_tensor_cam = pos_tensor @ Rts[i,:3,:3].T + Rts[i:i+1,:3,3]
K_ = get_diffrast_camera_parameter_from_cv(Ks[i], H, W, near, far, device)
pos_qc[:,:3] = pos_tensor_cam
pos_rast = (pos_qc @ K_.T)[None]
rast, _ = dr.rasterize(glctx, pos_rast, tri_tensor, resolution=[H, W])
out, _ = dr.interpolate(col_tensor[None], rast, tri_tensor)
# out = dr.antialias(out, rast, pos_rast, tri_tensor, topology_hash=None, pos_gradient_boost=1.0)
#out = dr.antialias(out, rast, pos_rast, tri_tensor, topology_hash=None, pos_gradient_boost=1.0)
img = out.cpu().numpy()[0, :, :, :] # Flip vertically.
#img = np.clip(np.rint(img * 255), 0, 255).astype(np.uint8) # Quantize to np.uint8
all_images.append(img)
all_images = np.stack(all_images, axis=0)
all_canvas = all_images[:,:,:,:3]
all_canvas_mask = all_images[:,:,:,3] > 0.999
#all_canvas,all_canvas_mask = fill_image_nvdiffrast(all_canvas,all_canvas_mask)
return all_canvas,all_canvas_mask
def csv_cam_to_opencv(csv_cams):
frame_size = len(csv_cams)
rot_matrix = np.array([
[0,1,0],[0,0,-1],[-1,0,0.]
], dtype=np.float32)
all_w2cs = []
for i in range(frame_size):
X_,Y_,Z_,p,y,r = csv_cams[i]
cur_w2c = (unreal_to_opencv_w2c(p,y,r,X_,Y_,Z_))
all_w2cs.append(cur_w2c)
return np.stack(all_w2cs)
def read_cam_csv(pose_file):
cam_params = []
with open(pose_file, 'r') as f:
next(f) # Skip header
for line in f:
frame_data = line.strip().split(',')
if len(frame_data) != 3:
continue
# Parse position
pos = frame_data[1].split()
X = float(pos[0].split('=')[1])
Y = float(pos[1].split('=')[1])
Z = float(pos[2].split('=')[1])
# Parse rotation
rot = frame_data[2].split()
P = float(rot[0].split('=')[1])
Yaw = float(rot[1].split('=')[1])
R = float(rot[2].split('=')[1])
cam_params.append((X, Y, Z, P, Yaw, R))
return cam_params
def project_cv(
points: np.ndarray,
extrinsics: np.ndarray = None,
intrinsics: np.ndarray = None
) -> Tuple[np.ndarray, np.ndarray]:
"""
Project 3D points to 2D following the OpenCV convention
Args:
points (np.ndarray): [..., N, 3] or [..., N, 4] 3D points to project, if the last
dimension is 4, the points are assumed to be in homogeneous coordinates
extrinsics (np.ndarray): [..., 4, 4] extrinsics matrix
intrinsics (np.ndarray): [..., 3, 3] intrinsics matrix
Returns:
uv_coord (np.ndarray): [..., N, 2] uv coordinates, value ranging in [0, 1].
The origin (0., 0.) is corresponding to the left & top
linear_depth (np.ndarray): [..., N] linear depth
"""
assert intrinsics is not None, "intrinsics matrix is required"
if points.shape[-1] == 3:
points = np.concatenate([points, np.ones_like(points[..., :1])], axis=-1)
if extrinsics is not None:
points = points @ extrinsics.swapaxes(-1, -2)
points = points[..., :3] @ intrinsics.swapaxes(-1, -2)
with no_warnings():
uv_coord = points[..., :2] / points[..., 2:]
linear_depth = points[..., 2]
return uv_coord, linear_depth
def uv_to_pixel(
uv: np.ndarray,
width: Union[int, np.ndarray],
height: Union[int, np.ndarray]
) -> np.ndarray:
"""
Args:
pixel (np.ndarray): [..., 2] pixel coordinrates defined in image space, x range is (0, W - 1), y range is (0, H - 1)
width (int | np.ndarray): [...] image width(s)
height (int | np.ndarray): [...] image height(s)
Returns:
(np.ndarray): [..., 2] pixel coordinrates defined in uv space, the range is (0, 1)
"""
pixel = uv * np.stack([width, height], axis=-1).astype(uv.dtype) - 0.5
return pixel
def merge_panorama_image(width: int, height: int, distance_maps, pred_masks, extrinsics, intrinsics_, init_frame=None, init_mask=None, mode="overwrite"):
# preprocess:
N = len(intrinsics_)
intrinsics = []
H, W = distance_maps[0].shape[:2]
for i in range(N):
ss = intrinsics_[i].copy()
ss[0,...] /= W
ss[1,...] /= H
intrinsics.append(ss)
uv = image_uv(width=width, height=height)
spherical_directions = spherical_uv_to_directions(uv)
# Warp each view to the panorama
panorama_log_distance_grad_maps, panorama_grad_masks = [], []
panorama_log_distance_laplacian_maps, panorama_laplacian_masks = [], []
panorama_pred_masks = []
panorama_log_distance_map = init_frame
panorama_pred_mask = init_mask
for i in range(len(distance_maps)):
projected_uv, projected_depth = project_cv(spherical_directions, extrinsics=extrinsics[i], intrinsics=intrinsics[i])
# print(projected_uv.shape, projected_depth.shape)
projection_valid_mask = (projected_depth >= 0) & (projected_uv >= 0).all(axis=-1) & (projected_uv <= 1).all(axis=-1)
projected_pixels = uv_to_pixel(np.clip(projected_uv, 0, 1), width=distance_maps[i].shape[1], height=distance_maps[i].shape[0]).astype(np.float32)
panorama_log_distance_map_r = np.where(projection_valid_mask, cv2.remap(distance_maps[i][...,0], projected_pixels[..., 0], projected_pixels[..., 1], cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE), 0)
panorama_log_distance_map_g = np.where(projection_valid_mask, cv2.remap(distance_maps[i][...,1], projected_pixels[..., 0], projected_pixels[..., 1], cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE), 0)
panorama_log_distance_map_b = np.where(projection_valid_mask, cv2.remap(distance_maps[i][...,2], projected_pixels[..., 0], projected_pixels[..., 1], cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE), 0)
panorama_log_distance_map_ = np.stack([panorama_log_distance_map_r, panorama_log_distance_map_g, panorama_log_distance_map_b], axis=-1)
panorama_pred_mask_ = projection_valid_mask & (cv2.remap(pred_masks[i].astype(np.uint8), projected_pixels[..., 0], projected_pixels[..., 1], cv2.INTER_NEAREST, borderMode=cv2.BORDER_REPLICATE) > 0)
# print(panorama_log_distance_map.shape, panorama_pred_mask.shape)
# print(panorama_pred_mask_.dtype, panorama_pred_mask_.shape)
# cv2.imwrite("./sdasd.png", panorama_pred_mask_.astype(np.uint8)*255)
if panorama_log_distance_map is None:
panorama_log_distance_map = panorama_log_distance_map_
panorama_pred_mask = panorama_pred_mask_
else:
if mode=="overwrite":
panorama_log_distance_map[panorama_pred_mask_] = panorama_log_distance_map_[panorama_pred_mask_]
else:
uu = panorama_pred_mask_ * panorama_pred_mask
uu1 = (~panorama_pred_mask_) * panorama_pred_mask
uu2 = (panorama_pred_mask_) * (~panorama_pred_mask)
panorama_log_distance_map[uu] = (panorama_log_distance_map_[uu] + panorama_log_distance_map[uu])/2.
panorama_log_distance_map[uu2] = (panorama_log_distance_map_[uu2])
panorama_pred_mask += panorama_pred_mask_
return panorama_log_distance_map, panorama_pred_mask
def merge_panorama_depth(width: int, height: int, distance_maps, pred_masks, extrinsics, intrinsics_, init_frame=None, init_mask=None, mode="overwrite"):
# preprocess:
N = len(intrinsics_)
intrinsics = []
H, W = distance_maps[0].shape[:2]
for i in range(N):
ss = intrinsics_[i].copy()
ss[0,...] /= W
ss[1,...] /= H
intrinsics.append(ss)
uv = image_uv(width=width, height=height)
spherical_directions = spherical_uv_to_directions(uv)
# Warp each view to the panorama
panorama_log_distance_grad_maps, panorama_grad_masks = [], []
panorama_log_distance_laplacian_maps, panorama_laplacian_masks = [], []
panorama_pred_masks = []
panorama_log_distance_map = init_frame
panorama_pred_mask = init_mask
for i in range(len(distance_maps)):
projected_uv, projected_depth = project_cv(spherical_directions, extrinsics=extrinsics[i], intrinsics=intrinsics[i])
# print(projected_uv.shape, projected_depth.shape)
projection_valid_mask = (projected_depth >= 0) & (projected_uv >= 0).all(axis=-1) & (projected_uv <= 1).all(axis=-1)
projected_pixels = uv_to_pixel(np.clip(projected_uv, 0, 1), width=distance_maps[i].shape[1], height=distance_maps[i].shape[0]).astype(np.float32)
#torch.where(condition, input, other, *, out=None)
panorama_log_distance_map_ = np.where(projection_valid_mask, cv2.remap(distance_maps[i], projected_pixels[..., 0], projected_pixels[..., 1], cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE), 0)
panorama_pred_mask_ = projection_valid_mask & (cv2.remap(pred_masks[i].astype(np.uint8), projected_pixels[..., 0], projected_pixels[..., 1], cv2.INTER_NEAREST, borderMode=cv2.BORDER_REPLICATE) > 0)
# print(panorama_log_distance_map.shape, panorama_pred_mask.shape)
# print(panorama_pred_mask_.dtype, panorama_pred_mask_.shape)
# cv2.imwrite("./sdasd.png", panorama_pred_mask_.astype(np.uint8)*255)
if panorama_log_distance_map is None:
panorama_log_distance_map = panorama_log_distance_map_
panorama_pred_mask = panorama_pred_mask_
else:
panorama_log_distance_map[panorama_pred_mask_] = panorama_log_distance_map_[panorama_pred_mask_]
panorama_pred_mask[panorama_pred_mask_] = True
return panorama_log_distance_map, panorama_pred_mask
def project_cv_torch(points, extrinsics, intrinsics):
#points: [H,W,3]
#extrinsics: [4,4]
#intrinsics: [3,3]
eps = 1e-5
points_cam = points @ (extrinsics[:3,:3].T)[None] + extrinsics[:3,3][None,None]
points_pix = points_cam @ (intrinsics.T)[None]
tmp = points_pix[:,:,2:].clone()
tmp_mask = torch.abs(points_pix[:,:,2:]) < eps
tmp[tmp_mask] = eps
points_pix = points_pix / tmp
return points_pix, tmp[...,0]
def merge_panorama_depth_torch(width: int, height: int, distance_maps, pred_masks, extrinsics, intrinsics_, device):
# preprocess:
N = len(intrinsics_)
intrinsics = intrinsics_.clone()
H, W = distance_maps[0].shape[:2]
intrinsics[:,0,:] /= W
intrinsics[:,1,:] /= H
uv = image_uv_torch(width=width, height=height,device=device,dtype=torch.float32)
spherical_directions = spherical_uv_to_directions_torch(uv)
# Warp each view to the panorama
panorama_log_distance_grad_maps, panorama_grad_masks = [], []
panorama_log_distance_laplacian_maps, panorama_laplacian_masks = [], []
panorama_pred_masks = []
panorama_log_distance_map = None
for i in range(len(distance_maps)):
projected_uv, projected_depth = project_cv_torch(spherical_directions, extrinsics=extrinsics[i], intrinsics=intrinsics[i])
# print(projected_uv.shape, projected_depth.shape)
projection_valid_mask = (projected_depth >= 1e-5) & (projected_uv >= 0).all(dim=-1) & (projected_uv <= 1).all(dim=-1)
projected_pixels = (torch.clip((projected_uv * 2. - 1.),-1.,1.))
#torch.where(condition, input, other, *, out=None)
panorama_log_distance_map_ = torch.where(projection_valid_mask, torch.nn.functional.grid_sample(distance_maps[i][None,None], projected_pixels[None,:,:,:2], align_corners=True), 0).squeeze()
if panorama_log_distance_map is None:
panorama_log_distance_map = panorama_log_distance_map_
else:
panorama_log_distance_map[projection_valid_mask] = panorama_log_distance_map_[projection_valid_mask]
return panorama_log_distance_map
def merge_panorama_color_torch(width: int, height: int, distance_maps, pred_masks, extrinsics, intrinsics_, device):
# preprocess:
N = len(intrinsics_)
intrinsics = intrinsics_.clone()
H, W = distance_maps[0].shape[:2]
intrinsics[:,0,:] /= W
intrinsics[:,1,:] /= H
uv = image_uv_torch(width=width, height=height,device=device,dtype=torch.float32)
spherical_directions = spherical_uv_to_directions_torch(uv)
# Warp each view to the panorama
panorama_log_distance_grad_maps, panorama_grad_masks = [], []
panorama_log_distance_laplacian_maps, panorama_laplacian_masks = [], []
panorama_pred_masks = []
panorama_log_distance_map = None
for i in range(len(distance_maps)):
projected_uv, projected_depth = project_cv_torch(spherical_directions, extrinsics=extrinsics[i], intrinsics=intrinsics[i])
# print(projected_uv.shape, projected_depth.shape)
projection_valid_mask = (projected_depth >= 1e-5) & (projected_uv >= 0).all(dim=-1) & (projected_uv <= 1).all(dim=-1)
projected_pixels = (torch.clip((projected_uv * 2. - 1.),-1.,1.))
#torch.where(condition, input, other, *, out=None)
panorama_log_distance_map_ = torch.where(projection_valid_mask, torch.nn.functional.grid_sample(distance_maps[i][None].permute(0,3,1,2), projected_pixels[None,:,:,:2], align_corners=True), 0).squeeze().permute(1,2,0)
if panorama_log_distance_map is None:
panorama_log_distance_map = panorama_log_distance_map_
else:
panorama_log_distance_map[projection_valid_mask] = panorama_log_distance_map_[projection_valid_mask]
return panorama_log_distance_map
def mesh_pano_render(mesh, Rts, height, width, near, far, device):
# the rotational matrix for turning render views;
#print(f"dsadsad:{Rts.shape}")
mat = np.array(
[
[
[1,0,0],
[0,1,0],
[0,0,1]
],
[
[0,0,-1],
[0,1,0],
[1,0,0]
],
[
[-1,0,0],
[0,1,0],
[0,0,-1]
],
[
[0,0,1],
[0,1,0],
[-1,0,0]
],
[
[1,0,0],
[0,0,1],
[0,-1,0]
],
[
[1,0,0],
[0,0,-1],
[0,1,0]
],
],
dtype=np.float32
)
#the extrinsic matrix for gluing images to panorama;
mat_glue = np.array(
[
[
[0,0,-1],
[0,-1,0],
[-1,0,0]
],
[
[0,0,-1],
[-1,0,0],
[0,1,0]
],
[
[0,0,-1],
[0,1,0],
[1,0,0]
],
[
[0,0,-1],
[1,0,0],
[0,-1,0]
],
[
[-1,0,0],
[0,-1,0],
[0,0,1]
],
[
[1,0,0],
[0,-1,0],
[0,0,-1]
],
],
dtype=np.float32
)
mat_glue = np.array([
[
[0,-1,0],
[1,0,0],
[0,0,1],
]
]) @ mat_glue
tmp = np.eye(4).astype(np.float32)[None].repeat(6,0)
tmp[:,:3,:3] = mat_glue
K = np.array([
[256.,0,256.],
[0.,256,256.],
[0.,0,1.],
]).astype(np.float32)
all_pano_depth_map = []
all_pano_mask_map = []
N = Rts.shape[0]
render_batch_size = 5
for i in range(0,N,render_batch_size):
cur_process_size = min(render_batch_size,N-i)
cur_Rt = Rts[i:i+cur_process_size][:,None,:,:]
panorama_render_Rts = cur_Rt.repeat(6,1)
panorama_render_Rts[:,:,:3,:] = mat[None] @ panorama_render_Rts[:,:,:3,:]
render_depth,render_mask = get_mesh_render_depth(mesh, (K[None].repeat(6*cur_process_size,0)), (panorama_render_Rts.reshape(cur_process_size*6,4,4)), 512,512,near,far, device)
#print(render_depth.max(), render_depth.min())
render_depth[~render_mask] = torch.inf
for j in range(cur_process_size):
panorama_log_distance_map = merge_panorama_depth_torch(width,height,torch.from_numpy(render_depth[6*j:6*(j+1)]).to(device),None,torch.from_numpy(tmp).to(device), torch.from_numpy(K[None].repeat(6*render_batch_size,0)).to(device), device)
# print(f"panorama check: {panorama_log_distance_map.max()} {panorama_log_distance_map.min()}")
all_pano_depth_map.append(panorama_log_distance_map.cpu().numpy())
all_pano_mask_map.append((torch.ones_like(panorama_log_distance_map).bool()).cpu().numpy())
return all_pano_depth_map, all_pano_mask_map
def mesh_pano_render_color(mesh, Rts, height, width, near, far, device):
# the rotational matrix for turning render views;
#print(f"dsadsad:{Rts.shape}")
mat = np.array(
[
[
[1,0,0],
[0,1,0],
[0,0,1]
],
[
[0,0,-1],
[0,1,0],
[1,0,0]
],
[
[-1,0,0],
[0,1,0],
[0,0,-1]
],
[
[0,0,1],
[0,1,0],
[-1,0,0]
],
[
[1,0,0],
[0,0,1],
[0,-1,0]
],
[
[1,0,0],
[0,0,-1],
[0,1,0]
],
],
dtype=np.float32
)
#the extrinsic matrix for gluing images to panorama;
mat_glue = np.array(
[
[
[0,0,-1],
[0,-1,0],
[-1,0,0]
],
[
[0,0,-1],
[-1,0,0],
[0,1,0]
],
[
[0,0,-1],
[0,1,0],
[1,0,0]
],
[
[0,0,-1],
[1,0,0],
[0,-1,0]
],
[
[-1,0,0],
[0,-1,0],
[0,0,1]
],
[
[1,0,0],
[0,-1,0],
[0,0,-1]
],
],
dtype=np.float32
)
mat_glue = np.array([
[
[0,-1,0],
[1,0,0],
[0,0,1],
]
]) @ mat_glue
tmp = np.eye(4).astype(np.float32)[None].repeat(6,0)
tmp[:,:3,:3] = mat_glue
K = np.array([
[256.,0,256.],
[0.,256,256.],
[0.,0,1.],
]).astype(np.float32)
all_pano_depth_map = []
all_pano_mask_map = []
N = Rts.shape[0]
render_batch_size = 5
for i in range(0,N,render_batch_size):
cur_process_size = min(render_batch_size,N-i)
cur_Rt = Rts[i:i+cur_process_size][:,None,:,:]
panorama_render_Rts = cur_Rt.repeat(6,1)
panorama_render_Rts[:,:,:3,:] = mat[None] @ panorama_render_Rts[:,:,:3,:]
render_color,render_mask = get_mesh_render_color(mesh, (K[None].repeat(6*cur_process_size,0)), (panorama_render_Rts.reshape(cur_process_size*6,4,4)), 512,512,near,far, device)
render_color[~render_mask] = torch.inf
for j in range(cur_process_size):
panorama_log_distance_map = merge_panorama_color_torch(width,height,torch.from_numpy(render_color[6*j:6*(j+1)]).to(device),None,torch.from_numpy(tmp).to(device), torch.from_numpy(K[None].repeat(6*render_batch_size,0)).to(device), device)
msk = panorama_log_distance_map<torch.inf
panorama_log_distance_map[panorama_log_distance_map==torch.inf] = 0.
all_pano_depth_map.append(panorama_log_distance_map.cpu().numpy())
all_pano_mask_map.append(msk[:,:,0].cpu().numpy())
return all_pano_depth_map, all_pano_mask_map
def generate_mask_video_projection_torch_simulate(csv_cams, depths,first_frame_rgb, skybox_masks = None, apply_boundary_occulusion=True):
device = depths.device
H, W = depths.shape[1:3]
frame_size = len(csv_cams)
# 1. generate first frame deoth edge mask;
edge_mask_first_frame = ~(depth_edge_torch(depths[0],rtol=0.05))
rot_matrix = torch.tensor([
[0,1,0],[0,0,-1],[-1,0,0.]
], dtype=depths.dtype,device=device)
# print(mask.sum())
all_w2cs = []
for i in range(frame_size):
X_,Y_,Z_,p,y,r = csv_cams[i]
# print(f"union check: {X_} {Y_} {Z_} {frame_size}")
cur_w2c = unreal_to_opencv_w2c(p,y,r,X_,Y_,Z_)
all_w2cs.append(cur_w2c)
# raise ValueError("ssad")
all_w2cs = torch.from_numpy(np.stack(all_w2cs)).to(device).to(depths.dtype)
all_c2ws = torch.linalg.inv(all_w2cs)
# print(f"check: {all_w2cs[:10]}")
all_canvas_mask = torch.zeros((frame_size, H, W), dtype=torch.bool, device=device)
all_canvas_rgb = torch.zeros((frame_size, H, W, 3), dtype=torch.float32, device=device)
if apply_boundary_occulusion:
mesh_occulusion = get_mesh_from_pano(depths[0].cpu().numpy(),(~edge_mask_first_frame).cpu().numpy(), csv_cams[0])
near = 1e-3
far = depths[i].max() * 1.1
rendered_depth_, rendered_mask_ = mesh_pano_render(mesh_occulusion, all_w2cs[1:].cpu().numpy(), H, W, near, far, device)
# debug
for i in range(1,frame_size):
if apply_boundary_occulusion:
rendered_depth = rendered_depth_[i-1].reshape(-1)
rendered_depth = torch.from_numpy(rendered_depth).float().to(device)
#rendered_mask = torch.from_numpy(rendered_mask).bool().to(device)
wrd_pcs = get_world_pcs_pano_torch(csv_cams[i], depths[i])
cur_pcs_cam = wrd_pcs @ all_w2cs[0,:3,:3].T + all_w2cs[0,:3,3][None]
pcs_cur_c2w_sph = cur_pcs_cam @ rot_matrix
pcs_cur_c2w_norm = torch.linalg.norm(pcs_cur_c2w_sph, dim=-1)
pcs_cur_c2w_direction = pcs_cur_c2w_sph / (pcs_cur_c2w_norm[...,None] + 1e-8)
pcs_cur_c2w_uv = directions_to_spherical_uv_torch(pcs_cur_c2w_direction)
pcs_cur_c2w_pix = uv_to_pixel_torch(pcs_cur_c2w_uv, width=W, height=H).to(torch.float32)
pcs_cur_c2w_pix_normalized = pcs_cur_c2w_pix.clone()
pcs_cur_c2w_pix_normalized[...,1] = pcs_cur_c2w_pix[...,1]/H * 2. - 1.
pcs_cur_c2w_pix_normalized[...,0] = pcs_cur_c2w_pix[...,0]/W * 2. - 1.
#print(f"range check: ", pcs_cur_c2w_pix_normalized.max(), pcs_cur_c2w_pix_normalized.min())
sampled_depth_value = torch.nn.functional.grid_sample(depths[0][None,None], pcs_cur_c2w_pix_normalized[None,None], mode='bilinear', padding_mode='zeros', align_corners=True).reshape(-1)
sampled_rgb_value = torch.nn.functional.grid_sample(first_frame_rgb[None].permute(0,3,1,2), pcs_cur_c2w_pix_normalized[None,None], mode='bilinear', padding_mode='zeros', align_corners=True).squeeze().permute(1,0)
sampled_mask_value = torch.nn.functional.grid_sample(edge_mask_first_frame[None,None].float(), pcs_cur_c2w_pix_normalized[None,None], mode='nearest', padding_mode='zeros', align_corners=True).reshape(-1)
mask_pixvalid = (pcs_cur_c2w_pix_normalized[...,0] >=-1.) * (pcs_cur_c2w_pix_normalized[...,1] >=-1.) * (pcs_cur_c2w_pix_normalized[...,0] <=1.) * (pcs_cur_c2w_pix_normalized[...,1] <=1.)
mask_depth_value = (sampled_depth_value * 1.02 > pcs_cur_c2w_norm)
mask_mask_value = sampled_mask_value == 1.
mask_valid = mask_pixvalid * mask_depth_value * mask_mask_value
if skybox_masks is not None:
#print("sad")
sampled_skybox_value = torch.nn.functional.grid_sample(skybox_masks[0][None,None].float(), pcs_cur_c2w_pix_normalized[None,None], mode='nearest', padding_mode='zeros', align_corners=True).reshape(-1) == 1.
mask_skybox_value = (sampled_skybox_value * skybox_masks[i].reshape(-1)) * mask_pixvalid * mask_mask_value
mask_valid[mask_skybox_value] = True
if apply_boundary_occulusion:
mask_valid = mask_valid * (depths[i].reshape(-1) < 1.1 * rendered_depth)
mask_valid_reshape = mask_valid.reshape(H,W)
sampled_rgb_valid = sampled_rgb_value[mask_valid]
all_canvas_rgb[i][mask_valid_reshape] = sampled_rgb_valid
all_canvas_mask[i][mask_valid_reshape] = True
all_canvas_rgb[0] = first_frame_rgb
all_canvas_mask[0,...] = True
return all_canvas_rgb, all_canvas_mask
def find_mask_cc(mask):
canvas = np.zeros_like(mask).astype(np.int32)
canvas[mask] = 1
labels, numbers = ski.measure.label(mask, background=None, return_num=True, connectivity=2)
return labels, numbers
def depth_repair(depth, small_area_threshold=1600):
max_depth = depth.max()
mask_valid = depth < 0.99 * max_depth
labels, numbers = find_mask_cc(mask_valid)
for i in range(numbers):
cur_mask = labels == i+1
if cur_mask.sum() < small_area_threshold:
depth[cur_mask] = max_depth
return depth
def generate_pc_render(Rts,first_frame_rgb, first_frame_depth, first_frame_Rt=None):
device = first_frame_depth.device
H, W = first_frame_depth.shape[:2]
frame_size = len(Rts)
# 1. generate first frame deoth edge mask;
edge_mask_first_frame = ~(depth_edge_torch(first_frame_depth,rtol=0.05))
rot_matrix = torch.tensor([
[0,1,0],[0,0,-1],[-1,0,0.]
], dtype=first_frame_depth.dtype,device=device)
# raise ValueError("ssad")
all_w2cs = Rts
all_c2ws = torch.linalg.inv(all_w2cs)
# print(f"check: {all_w2cs[:10]}")
all_canvas_mask = torch.zeros((frame_size, H, W), dtype=torch.bool, device=device)
all_canvas_rgb = torch.zeros((frame_size, H, W, 3), dtype=torch.float32, device=device)
#if apply_boundary_occulusion:
#mesh_occulusion = get_mesh_from_pano(first_frame_depth.cpu().numpy(),(~edge_mask_first_frame).cpu().numpy(), csv_cams[0])
#get_mesh_from_pano_Rt(pano_depth, pano_mask, Rt, dtype=np.float32, pano_rgb=None, alpha_mask=None):
'''mesh_occulusion = get_mesh_from_pano_Rt(first_frame_depth.cpu().numpy(),(~edge_mask_first_frame).cpu().numpy(), Rts[0])
near = 1e-3
far = first_frame_depth.max() * 1.1
rendered_depth, rendered_occulusion_mask = mesh_pano_render(mesh_occulusion, all_w2cs.cpu().numpy(), H, W, near, far, device)'''
if first_frame_Rt is None:
first_frame_Rt = Rts[0]
mesh_full = get_mesh_from_pano_Rt(first_frame_depth.cpu().numpy(),np.ones((H,W),dtype=np.bool_), first_frame_Rt.cpu().numpy(), pano_rgb=first_frame_rgb.cpu().numpy(), alpha_mask=(edge_mask_first_frame).cpu().numpy())
#mesh_full = get_mesh_from_pano_Rt(first_frame_depth.cpu().numpy(),(edge_mask_first_frame).cpu().numpy(), Rts[0].cpu().numpy(), pano_rgb=first_frame_rgb.cpu().numpy())
near = 1e-3
far = first_frame_depth.max() * 1.1
print("mesh_pano_render_color")
torch.cuda.empty_cache()
print(torch.cuda.memory_allocated() / 1024**3, "GB used")
print(torch.cuda.memory_reserved() / 1024**3, "GB reserved")
# import pdb
# pdb.set_trace()
torch.cuda.empty_cache()
torch.cuda.synchronize()
torch.cuda.memory_summary()
rendered_cmap, rendered_cmap_mask = mesh_pano_render_color(mesh_full, all_w2cs.cpu().numpy(), H, W, near, far, device)
# debug
for i in range(frame_size):
cmap_mask = rendered_cmap_mask[i]
cmap = rendered_cmap[i]
cmap[~cmap_mask] = 0.
all_canvas_rgb[i] = torch.from_numpy(cmap).float().to(device)
all_canvas_mask[i] = torch.from_numpy(cmap_mask).float().to(device)
all_canvas_rgb[0] = first_frame_rgb
all_canvas_mask[0,...] = True
return all_canvas_rgb, all_canvas_mask
def perform_camera_movement(firstframe_rgb, firstframe_depth, angle=0., movement_ratio = 0.5, frame_size=81):
device = firstframe_rgb.device
depth_np = firstframe_depth.cpu().numpy()
depth_np = depth_repair(depth_np)
firstframe_depth = torch.from_numpy(depth_np).to(device)
H, W = firstframe_rgb.shape[:2]
pvt = int(angle/360. * W + W//2) % W
firstframe_rgb = torch.cat([firstframe_rgb[:,pvt:],firstframe_rgb[:,:pvt]], dim=1)
firstframe_rgb = torch.cat([firstframe_rgb[:,W//2:],firstframe_rgb[:,:W//2]], dim=1)
firstframe_depth = torch.cat([firstframe_depth[:,pvt:],firstframe_depth[:,:pvt]], dim=1)
firstframe_depth = torch.cat([firstframe_depth[:,W//2:],firstframe_depth[:,:W//2]], dim=1)
depth_pvt = firstframe_depth[H//2,int(W//2*1.05)]
Rt0 = torch.tensor(
[
[1.,0,0,0],
[0,1.,0,0],
[0,0,1,0],
[0,0,0,1]
],
dtype=torch.float32,
device=device
)
Rts = Rt0[None].repeat(frame_size,1,1)
for i in range(frame_size):
Rts[i,2,3] = -(depth_pvt * i *movement_ratio)/(frame_size-1)
rgbs, masks = generate_pc_render(Rts, firstframe_rgb, firstframe_depth)
return rgbs, masks, Rts, firstframe_rgb, firstframe_depth
def generate_rail(depth, angle, movement_ratio, frame_size, mode="straight"):
print("generating rail...")
device=depth.device
H, W = depth.shape[:2]
q = int(angle/360. * W + W//2)%W
depth_pvt = depth[int(H//2*1.05),q]
z_axis_ = torch.tensor([0,0,1.]).float().to(device)
x_axis_ = torch.tensor([1,0,0.]).float().to(device)
y_axis = torch.tensor([0,1,0]).float().to(device)
z_axis = x_axis_ * math.sin(math.radians(angle)) + z_axis_ * math.cos(math.radians(angle))
x_axis = z_axis_ * -math.sin(math.radians(angle)) + x_axis_ * math.cos(math.radians(angle))
print(z_axis, x_axis)
if mode == "straight":
Rt0_ = torch.stack([x_axis,y_axis,z_axis],dim=0)
Rt0 = torch.eye(4).float().to(device)
Rt0[:3,:3] = Rt0_
Rts = Rt0[None].repeat(frame_size,1,1)
for i in range(frame_size):
Rts[i,2,3] = -(depth_pvt * i *movement_ratio)/(frame_size-1)
if mode == "s_curve":
l = depth_pvt * movement_ratio
sin_ratio = 0.15
all_Rts = []
for i in range(frame_size):
cur_t = (depth_pvt * i *movement_ratio)/(frame_size-1)
pos = z_axis * (depth_pvt * i *movement_ratio)/(frame_size-1) + x_axis * math.sin(cur_t / l * math.pi * 2) * sin_ratio
direction_ = z_axis + sin_ratio * (2 * math.pi / l) * math.cos(math.pi * 2 / l * cur_t) * x_axis
direction = direction_/direction_.norm()
cur_z_axis = direction
cur_y_axis = y_axis
cur_x_axis = torch.cross(cur_y_axis, cur_z_axis)
cur_c2w = torch.eye(4).float().to(device)
cur_c2w[:3,:3] = torch.stack([cur_x_axis,y_axis,cur_z_axis],dim=1)
cur_c2w[:3,3] = pos
all_Rts.append(torch.linalg.inv(cur_c2w))
Rts = torch.stack(all_Rts,axis=0)
if mode == "r_curve":
print("r curve")
curve_deg = 30.
l = depth_pvt * movement_ratio
r = float(l/math.sin(math.radians(curve_deg)))
c = torch.tensor([r,0,0],dtype=torch.float32,device=device)
#sin_ratio = 0.15
all_Rts = []
for i in range(frame_size):
theta = curve_deg * float(i)/(frame_size-1)
v = r * (-x_axis_ * math.cos(math.radians(theta)) + z_axis_ * math.sin(math.radians(theta)))
pos = c + v
z_axis = (x_axis_ * math.sin(math.radians(theta)) + z_axis_ * math.cos(math.radians(theta)))
x_axis = -(-x_axis_ * math.cos(math.radians(theta)) + z_axis_ * math.sin(math.radians(theta)))
cur_c2w = torch.eye(4).float().to(device)
cur_c2w[:3,:3] = torch.stack([x_axis,y_axis,z_axis],dim=1)
cur_c2w[:3,3] = pos
all_Rts.append(torch.linalg.inv(cur_c2w))
Rts = torch.stack(all_Rts,axis=0)
if mode == "l_curve":
print("l curve")
curve_deg = 30.
l = depth_pvt * movement_ratio
r = float(l/math.sin(math.radians(curve_deg)))
c = torch.tensor([-r,0,0],dtype=torch.float32,device=device)
#sin_ratio = 0.15
all_Rts = []
for i in range(frame_size):
theta = curve_deg * float(i)/(frame_size-1)
v = r * (x_axis_ * math.cos(math.radians(theta)) + z_axis_ * math.sin(math.radians(theta)))
pos = c + v
z_axis = (-x_axis_ * math.sin(math.radians(theta)) + z_axis_ * math.cos(math.radians(theta)))
x_axis = (x_axis_ * math.cos(math.radians(theta)) + z_axis_ * math.sin(math.radians(theta)))
z_axis[0] = -z_axis[0]
x_axis[0] = -x_axis[0]
pos[0] = -pos[0]
cur_c2w = torch.eye(4).float().to(device)
cur_c2w[:3,:3] = torch.stack([x_axis,y_axis,z_axis],dim=1)
cur_c2w[:3,3] = pos
all_Rts.append(torch.linalg.inv(cur_c2w))
Rts = torch.stack(all_Rts,axis=0)
return Rts
def perform_camera_movement_new(firstframe_rgb, firstframe_depth, angle=0., movement_ratio = 0.5, frame_size=81, preset_rail=None,mode="s_curve"):
'''device = firstframe_rgb.device
depth_np = firstframe_depth.cpu().numpy()
depth_np = depth_repair(depth_np)
firstframe_depth = torch.from_numpy(depth_np).to(device)
H, W = firstframe_rgb.shape[:2]
pvt = int(angle/360. * W + W//2) % W
firstframe_rgb = torch.cat([firstframe_rgb[:,pvt:],firstframe_rgb[:,:pvt]], dim=1)
firstframe_rgb = torch.cat([firstframe_rgb[:,W//2:],firstframe_rgb[:,:W//2]], dim=1)
firstframe_depth = torch.cat([firstframe_depth[:,pvt:],firstframe_depth[:,:pvt]], dim=1)
firstframe_depth = torch.cat([firstframe_depth[:,W//2:],firstframe_depth[:,:W//2]], dim=1)
depth_pvt = firstframe_depth[H//2,int(W//2*1.05)]
Rt0 = torch.tensor(
[
[1.,0,0,0],
[0,1.,0,0],
[0,0,1,0],
[0,0,0,1]
],
dtype=torch.float32,
device=device
)
Rts = Rt0[None].repeat(frame_size,1,1)'''
device = firstframe_depth.device
depth_np = firstframe_depth.cpu().numpy()
depth_np = depth_repair(depth_np)
W = firstframe_rgb.shape[1]
firstframe_depth = torch.from_numpy(depth_np).to(device)
q = int(angle/360. * W + W//2)%W
firstframe_rgb = torch.cat([firstframe_rgb[:,q:],firstframe_rgb[:,:q]],dim=1)
firstframe_depth = torch.cat([firstframe_depth[:,q:],firstframe_depth[:,:q]],dim=1)
firstframe_rgb = torch.cat([firstframe_rgb[:,W//2:],firstframe_rgb[:,:W//2]],dim=1)
firstframe_depth = torch.cat([firstframe_depth[:,W//2:],firstframe_depth[:,:W//2]],dim=1)
if preset_rail is None:
Rts = generate_rail(firstframe_depth, 0., movement_ratio, frame_size, mode=mode)
else:
Rts = preset_rail
#depth_pvt = depth[int(H//2*1.05),q]
'''for i in range(frame_size):
Rts[i,2,3] = -(depth_pvt * i *movement_ratio)/(frame_size-1)'''
print(Rts.device, firstframe_rgb.device, firstframe_depth.device)
print(Rts.shape, firstframe_rgb.shape, firstframe_depth.shape)
print(Rts.dtype, firstframe_rgb.dtype, firstframe_depth.dtype)
print(torch.cuda.memory_allocated() / 1024**3, "GB used")
print(torch.cuda.memory_reserved() / 1024**3, "GB reserved")
torch.cuda.empty_cache()
print(torch.cuda.memory_allocated() / 1024**3, "GB used")
print(torch.cuda.memory_reserved() / 1024**3, "GB reserved")
# import pdb
# pdb.set_trace()
rgbs, masks = generate_pc_render(Rts, firstframe_rgb, firstframe_depth)
return rgbs, masks, Rts, firstframe_rgb, firstframe_depth
def intersection_check(firstframe_depth, Rts, max_ratio = 0.5):
# 假设首帧为单位阵,也即首帧的坐标系为世界坐标系。
c2ws = torch.linalg.inv(Rts)
pos = c2ws[:,:3,3]
pos_norm = pos.norm(dim=-1)
proj_pos = pos/pos.norm(dim=-1,keepdim=True)
cos_value = proj_pos[:,2]
sin_value = proj_pos[:,0]
ac = torch.arccos(cos_value)
mask_2pi_minus = sin_value<0
angle_rad = torch.arccos(cos_value)
q = 2.*math.pi-angle_rad[mask_2pi_minus]
angle_rad[mask_2pi_minus] = q
angle_azimuth = torch.rad2deg(angle_rad)
sin_value_elevation = proj_pos[:,1]
angle_rad_elevation = torch.arcsin(sin_value_elevation)
angle_elevation = torch.rad2deg(angle_rad_elevation)
W = firstframe_depth.shape[1]
H = firstframe_depth.shape[0]
pixel_value = torch.zeros((angle_azimuth.shape[0],2),dtype=torch.int32,device=Rts.device)
#pixel_value[:,0] = H//2
pixel_value[:,0] = ((angle_elevation + 90.) / 180. * (H - 1)).long()
pixel_value[:,1] = (((angle_azimuth / 360.) * W).long() + W//2)%W
depth_fetch = firstframe_depth[pixel_value[:,0],pixel_value[:,1]]
# print(depth_fetch.shape,pos_norm.shape, depth_fetch.shape,pixel_value,angle_azimuth)
ratio = pos_norm/depth_fetch
rmax= ratio.max()
print(f"intersection ratio max: {rmax}")
c2ws[:,:3,3] = c2ws[:,:3,3]/rmax * max_ratio
return torch.linalg.inv(c2ws)
def load_rail(json_path):
with open(json_path,"r") as F_:
d = json.load(F_)
arr = np.array(d)
return arr
def perform_camera_movement_with_cam_input(firstframe_rgb, firstframe_depth, angle=0., movement_ratio = 0.5, frame_size=81, preset_rail=None,mode="s_curve"):
device = firstframe_depth.device
depth_np = firstframe_depth.cpu().numpy()
depth_np = depth_repair(depth_np)
if preset_rail is not None:
# 把相机转成和首帧一致
'''
z_axis = x_axis_ * math.sin(math.radians(angle)) + z_axis_ * math.cos(math.radians(angle))
x_axis = z_axis_ * -math.sin(math.radians(angle)) + x_axis_ * math.cos(math.radians(angle))
'''
if isinstance(preset_rail, np.ndarray):
preset_rail = torch.from_numpy(preset_rail).float().to(device)
# print("sd")
firstframe_Rt = preset_rail[0]
firstframe_z = firstframe_Rt[2,:3]
firstframe_x = firstframe_Rt[0,:3]
original_z = torch.tensor([0,0,1.],dtype=torch.float32).to(device)
original_x = torch.tensor([1.,0,0],dtype=torch.float32).to(device)
cos_value = float((firstframe_z * original_z).sum())
sin_value = float((firstframe_z * original_x).sum())
angle_rad = math.acos(cos_value) if sin_value >=0 else 2 * math.pi - math.acos(cos_value)
angle = math.degrees(angle_rad)
preset_rail_ = preset_rail[...] @ torch.linalg.inv(preset_rail[0])[None]
W = firstframe_rgb.shape[1]
firstframe_depth = torch.from_numpy(depth_np).to(device)
q = int(angle/360. * W + W//2)%W
firstframe_rgb = torch.cat([firstframe_rgb[:,q:],firstframe_rgb[:,:q]],dim=1)
firstframe_depth = torch.cat([firstframe_depth[:,q:],firstframe_depth[:,:q]],dim=1)
firstframe_rgb = torch.cat([firstframe_rgb[:,W//2:],firstframe_rgb[:,:W//2]],dim=1)
firstframe_depth = torch.cat([firstframe_depth[:,W//2:],firstframe_depth[:,:W//2]],dim=1)
if preset_rail is None:
Rts = generate_rail(firstframe_depth, 0., movement_ratio, frame_size, mode=mode)
else:
Rts = intersection_check(firstframe_depth,preset_rail_,0.5)
#depth_pvt = depth[int(H//2*1.05),q]
'''for i in range(frame_size):
Rts[i,2,3] = -(depth_pvt * i *movement_ratio)/(frame_size-1)'''
rgbs, masks = generate_pc_render(Rts, firstframe_rgb, firstframe_depth)
return rgbs, masks, Rts, firstframe_rgb, firstframe_depth, angle
if __name__ == "__main__":
print("nv test")
base_dir = "/ai-video-sh/common/common/Datasets/Panorama_Selected0425/CamTracks/POLYGON_CityPack/POLYGON_CityPackData/Day/Rail_PCG_Road2_50_42__Take_183"
debug_dir = "/ai-video-sh/zhongqi.yang/code/zhongqi.yang/Trajcrafter_Training/dataprocess_minimal/debug"
frame_size = 49
filelist = os.listdir(base_dir)
rgbs = []
depths = []
csv_name = None
for i in filelist:
if i.endswith("col.exr"):
rgbs.append(i)
elif i.endswith("depth.exr"):
depths.append(i)
elif i.endswith("csv"):
csv_name = i
rgbs = sorted(rgbs)
depths = sorted(depths)
if csv_name is None:
raise ValueError("csv not found!")
# load csv as ypr.
csv_file = os.path.join(base_dir, csv_name)
csv_parameters = read_cam_csv(csv_file)
csv_parameters = csv_parameters[:frame_size]
all_rgbs = []
all_depths = []
for i in range(frame_size):
depth = cv2.imread(os.path.join(base_dir, depths[i]), cv2.IMREAD_ANYCOLOR|cv2.IMREAD_ANYDEPTH)[...,0].astype(np.float32)
rgb = np.clip(cv2.imread(os.path.join(base_dir, rgbs[i]), cv2.IMREAD_ANYCOLOR|cv2.IMREAD_ANYDEPTH)[...].astype(np.float32), 0, 1.)
all_rgbs.append(rgb)
all_depths.append(depth)
all_rgbs = np.stack(all_rgbs, axis=0)
all_depths = np.stack(all_depths, axis=0)
# (images, depths, K, Rts, resolution, frame_interval = 8, horizontal_cut_num = 4, vertical_cut_num = 1):
K = np.array([
[256.,0,256.],
[0.,256,256.],
[0.,0,1.],
])
Rts = csv_cam_to_opencv(csv_parameters)
all_depths = correct_pano_depth_batch(all_depths)
print(all_depths[0].max(),all_depths[0].min())
#glctx = dr.RasterizeCudaContext(device="cuda:0")
mask = all_depths > 1000.
all_depths[mask] = 1000.
mesh = get_mesh_from_pano(all_depths[0], np.ones_like(all_depths[0]).astype(np.bool_), csv_parameters[0] ,dtype=np.float32)
depth,render_mask = get_mesh_render_depth(mesh, (K[None]), (Rts[:1]), 512,512,1e-3,2000, "cuda:0")
print(depth.max(), depth.min())
mask_ = depth < 1e-5
depth[mask_ ] = 0.1
depth_inverse = 1./depth[0]
depth_inverse_np = (255. * (depth_inverse - depth_inverse.min()) / (depth_inverse.max() - depth_inverse.min())).astype(np.uint8)
cv2.imwrite("./test.png", depth_inverse_np[:,:,None].repeat(3,2))
cv2.imwrite("./rgb_pano.png", (all_rgbs[0] * 255.).astype(np.uint8))