huggingface/diffusers (original) (raw)
Describe the bug
Running diffusers.utils.export_to_video() on the output of HunyuanVideoPipeline results in
/app/diffusers/src/diffusers/image_processor.py:147: RuntimeWarning: invalid value encountered in cast
images = (images * 255).round().astype("uint8")
After adding some checks to numpy_to_pil() in image_processor.py I have confirmed that the output contains NaN values
File "/app/pipeline.py", line 37, in <module>
output = pipe(
^^^^^
File "/usr/local/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/app/diffusers/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video.py", line 677, in __call__
video = self.video_processor.postprocess_video(video, output_type=output_type)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/diffusers/src/diffusers/video_processor.py", line 103, in postprocess_video
batch_output = self.postprocess(batch_vid, output_type)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/diffusers/src/diffusers/image_processor.py", line 823, in postprocess
return self.numpy_to_pil(image)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/diffusers/src/diffusers/image_processor.py", line 158, in numpy_to_pil
raise ValueError("Image array contains NaN values")
ValueError: Image array contains NaN values
Reproduction
import os import time
import torch from diffusers import HunyuanVideoPipeline, HunyuanVideoTransformer3DModel from diffusers.utils import export_to_video from huggingface_hub import snapshot_download from torch.profiler import ProfilerActivity, profile, record_function
os.environ["TOKENIZERS_PARALLELISM"] = "false"
MODEL_ID = "tencent/HunyuanVideo" PROMPT = "a whale shark floating through outer space" profile_dir = os.environ.get("PROFILE_OUT_PATH", "./") profile_file_name = os.environ.get("PROFILE_OUT_FILE_NAME", "hunyuan_profile.json") profile_path = os.path.join(profile_dir, profile_file_name)
transformer = HunyuanVideoTransformer3DModel.from_pretrained( MODEL_ID, subfolder="transformer", torch_dtype=torch.float16, revision="refs/pr/18" ) pipe = HunyuanVideoPipeline.from_pretrained( MODEL_ID, transformer=transformer, torch_dtype=torch.float16, revision="refs/pr/18" ) pipe.vae.enable_tiling() pipe.to("cuda")
print(f"\nStarting profiling of {MODEL_ID}\n")
with profile( activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], record_shapes=True ) as prof: with record_function("model_inference"): output = pipe( prompt=PROMPT, height=320, width=512, num_frames=61, num_inference_steps=30, )
Export and print profiling results
print(prof.key_averages().table(sort_by="cuda_time_total", row_limit=10)) prof.export_chrome_trace(profile_path) print(f"{profile_file_name} ready")
export video
video = output.frames[0]
print(" ====== raw video matrix =====") print(video) print()
print(" ====== Exporting video =====") export_to_video(video, "hunyuan_example.mp4", fps=15) print()
Logs
No response
System Info
GPU: AMD MI300X
ARG BASE_IMAGE=python:3.11-slim FROM ${BASE_IMAGE}
ENV PYTHONBUFFERED=true ENV CUDA_VISIBLE_DEVICES=0
WORKDIR /app
Install tools
RUN apt-get update &&
apt-get install -y --no-install-recommends
git
libgl1-mesa-glx
libglib2.0-0
libsm6
libxext6
libxrender-dev
libfontconfig1
ffmpeg
build-essential &&
rm -rf /var/lib/apt/lists/*
install ROCm pytorch and python dependencies
RUN python -m pip install --no-cache-dir
torch torchvision --index-url https://download.pytorch.org/whl/rocm6.2 &&
python -m pip install --no-cache-dir
accelerate transformers sentencepiece protobuf opencv-python imageio imageio-ffmpeg
install diffusers from source to include newest pipeline classes
COPY diffusers diffusers
RUN cd diffusers &&
python -m pip install -e .
Copy the profiling script
ARG PIPELINE_FILE COPY ${PIPELINE_FILE} pipeline.py
run the script
CMD ["python", "pipeline.py"]