Files
2025-08-11 17:51:56 +08:00

179 lines
6.0 KiB
Python

import os
import subprocess
import tempfile
from typing import Mapping
from PIL import Image
import cv2
from einops import rearrange
import numpy as np
import torch
import torch.nn.functional as F
import torchvision.transforms.functional as transforms_F
from video_to_video.utils.logger import get_logger
logger = get_logger()
def tensor2vid(video, mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]):
mean = torch.tensor(mean, device=video.device).reshape(1, -1, 1, 1, 1)
std = torch.tensor(std, device=video.device).reshape(1, -1, 1, 1, 1)
video = video.mul_(std).add_(mean)
video.clamp_(0, 1)
video = video * 255.0
images = rearrange(video, "b c f h w -> b f h w c")[0]
return images
def preprocess(input_frames, extend):
out_frame_list = []
for pointer in range(len(input_frames)):
frame = input_frames[pointer]
frame = frame[:, :, ::-1]
frame = Image.fromarray(frame.astype("uint8")).convert("RGB")
frame = transforms_F.to_tensor(frame)
out_frame_list.append(frame)
out_frames = torch.stack(out_frame_list, dim=0)
# copy and concat
out_frames = torch.cat((out_frames, out_frames[...,:extend]), dim=-1)
out_frames.clamp_(0, 1)
mean = out_frames.new_tensor([0.5, 0.5, 0.5]).view(-1)
std = out_frames.new_tensor([0.5, 0.5, 0.5]).view(-1)
out_frames.sub_(mean.view(1, -1, 1, 1)).div_(std.view(1, -1, 1, 1))
return out_frames
def adjust_resolution(h, w, up_scale):
if h * w * up_scale * up_scale < 720 * 1280 * 1.5:
up_s = np.sqrt(720 * 1280 * 1.5 / (h * w))
target_h = int(up_s * h // 2 * 2)
target_w = int(up_s * w // 2 * 2)
elif h * w * up_scale * up_scale > 1152 * 2048:
up_s = np.sqrt(1152 * 2048 / (h * w))
target_h = int(up_s * h // 2 * 2)
target_w = int(up_s * w // 2 * 2)
else:
target_h = int(up_scale * h // 2 * 2)
target_w = int(up_scale * w // 2 * 2)
return (target_h, target_w)
def make_mask_cond(in_f_num, interp_f_num):
mask_cond = []
interp_cond = [-1 for _ in range(interp_f_num)]
for i in range(in_f_num):
mask_cond.append(i)
if i != in_f_num - 1:
mask_cond += interp_cond
return mask_cond
def load_prompt_list(file_path):
files = []
with open(file_path, "r") as fin:
for line in fin:
path = line.strip()
if path:
files.append(path)
return files
def load_video(vid_path):
capture = cv2.VideoCapture(vid_path)
_fps = capture.get(cv2.CAP_PROP_FPS)
_total_frame_num = capture.get(cv2.CAP_PROP_FRAME_COUNT)
pointer = 0
frame_list = []
stride = 1
while len(frame_list) < _total_frame_num:
ret, frame = capture.read()
pointer += 1
if (not ret) or (frame is None):
break
if pointer >= _total_frame_num + 1:
break
if pointer % stride == 0:
frame_list.append(frame)
capture.release()
return frame_list, _fps
def save_video(video, save_dir, file_name, fps=16.0):
output_path = os.path.join(save_dir, file_name)
# tensor
if isinstance(video[0], torch.Tensor):
images = [(img.numpy()).astype("uint8") for img in video]
elif isinstance(video[0], np.ndarray):
images = [frame for frame in video]
else:
print(video.__class__)
raise NotImplementedError
temp_dir = tempfile.mkdtemp()
for fid, frame in enumerate(images):
tpth = os.path.join(temp_dir, "%06d.png" % (fid + 1))
cv2.imwrite(tpth, frame[:, :, ::-1])
tmp_path = os.path.join(save_dir, "tmp.mp4")
cmd = f"/ai-video-sh/haoxiang.guo/HoloTime/ffmpeg-7.0.2-amd64-static/ffmpeg -y -f image2 -framerate {fps} -i {temp_dir}/%06d.png \
-vcodec libx264 -crf 17 -pix_fmt yuv420p {tmp_path}"
# import pdb; pdb.set_trace()
# cmd = f"ffmpeg -y -f image2 -framerate {fps} -i {temp_dir}/%06d.png \
# -vcodec libx264 -pix_fmt yuv420p {tmp_path}"
status, output = subprocess.getstatusoutput(cmd)
if status != 0:
logger.error(f"Save Video Error with {output}")
os.system(f"rm -rf {temp_dir}")
os.rename(tmp_path, output_path)
def collate_fn(data, device):
"""Prepare the input just before the forward function.
This method will move the tensors to the right device.
Usually this method does not need to be overridden.
Args:
data: The data out of the dataloader.
device: The device to move data to.
Returns: The processed data.
"""
from torch.utils.data.dataloader import default_collate
def get_class_name(obj):
return obj.__class__.__name__
if isinstance(data, dict) or isinstance(data, Mapping):
return type(data)({k: collate_fn(v, device) if k != "img_metas" else v for k, v in data.items()})
elif isinstance(data, (tuple, list)):
if 0 == len(data):
return torch.Tensor([])
if isinstance(data[0], (int, float)):
return default_collate(data).to(device)
else:
return type(data)(collate_fn(v, device) for v in data)
elif isinstance(data, np.ndarray):
if data.dtype.type is np.str_:
return data
else:
return collate_fn(torch.from_numpy(data), device)
elif isinstance(data, torch.Tensor):
return data.to(device)
elif isinstance(data, (bytes, str, int, float, bool, type(None))):
return data
else:
raise ValueError(f"Unsupported data type {type(data)}")
def blend_h(a, b, blend_extent, left, right):
blend_extent = min(a.shape[-1], b.shape[-1], blend_extent)
for x in range(blend_extent):
if left:
b[..., x] = a[..., -blend_extent
+ x] * (1 - x / blend_extent) + a[..., x] * (
x / blend_extent)
if right:
b[..., -blend_extent + x] = a[..., -blend_extent
+ x] * (1 - x / blend_extent) + a[..., x] * (
x / blend_extent)
return b