Updated _encode_prompt_with_clip and encode_prompt in train_dreamboth_sd3 by SahilCarterr · Pull Request #9800 · huggingface/diffusers (original) (raw)

The LoRA tests are failing. Here's the stacktrace:

stacktrace

_________________ DreamBoothSD3.test_dreambooth_checkpointing __________________ command = ['accelerate', 'launch', '--config_file', '/tmp/tmpnw4j3n3a/default_config.yml', 'examples/dreambooth/train_dreambooth_sd3.py', '--pretrained_model_name_or_path', ...] return_stdout = False def run_command(command: List[str], return_stdout=False): """ Runs command with subprocess.check_output and will potentially return the stdout. Will also properly capture if an error occurred while running command """ try: output = subprocess.check_output(command, stderr=subprocess.STDOUT) if return_stdout: if hasattr(output, "decode"): output = output.decode("utf-8") return output except subprocess.CalledProcessError as e:

      raise SubprocessCallException(
            f"Command `{' '.join(command)}` failed with the following error:\n\n{e.output.decode()}"
        ) from e

E test_examples_utils.SubprocessCallException: Command accelerate launch --config_file /tmp/tmpnw4j3n3a/default_config.yml examples/dreambooth/train_dreambooth_sd3.py --pretrained_model_name_or_path hf-internal-testing/tiny-sd3-pipe --instance_data_dir docs/source/en/imgs --instance_prompt photo --resolution 64 --train_batch_size 1 --gradient_accumulation_steps 1 --max_train_steps 4 --learning_rate 5.0e-04 --scale_lr --lr_scheduler constant --lr_warmup_steps 0 --output_dir /tmp/tmpotpr0ant --checkpointing_steps=2 --seed=0 failed with the following error: E
E 10/30/2024 11:21:33 - INFO - main - Distributed environment: NO E Num processes: 1 E Process index: 0 E Local process index: 0 E Device: cpu E
E Mixed precision type: no E
E You are using a model of type clip_text_model to instantiate a model of type . This is not supported for all configurations of models and can yield errors. E You are using a model of type clip_text_model to instantiate a model of type . This is not supported for all configurations of models and can yield errors. E You are using a model of type t5 to instantiate a model of type . This is not supported for all configurations of models and can yield errors. E {'base_image_seq_len', 'use_dynamic_shifting', 'base_shift', 'max_shift', 'max_image_seq_len'} was not found in config. Values will be initialized to default values. E {'mid_block_add_attention'} was not found in config. Values will be initialized to default values. E {'qk_norm', 'dual_attention_layers'} was not found in config. Values will be initialized to default values. E Traceback (most recent call last): E File "/__w/diffusers/diffusers/examples/dreambooth/train_dreambooth_sd3.py", line 1819, in E main(args) E File "/__w/diffusers/diffusers/examples/dreambooth/train_dreambooth_sd3.py", line 1387, in main E instance_prompt_hidden_states, instance_pooled_prompt_embeds = compute_text_embeddings( E File "/__w/diffusers/diffusers/examples/dreambooth/train_dreambooth_sd3.py", line 1376, in compute_text_embeddings E prompt_embeds, pooled_prompt_embeds = encode_prompt( E File "/__w/diffusers/diffusers/examples/dreambooth/train_dreambooth_sd3.py", line 970, in encode_prompt E t5_prompt_embed = _encode_prompt_with_t5( E TypeError: _encode_prompt_with_t5() got an unexpected keyword argument 'text_input_ids' E Traceback (most recent call last): E File "/opt/venv/bin/accelerate", line 8, in E sys.exit(main()) E File "/opt/venv/lib/python3.10/site-packages/accelerate/commands/accelerate_cli.py", line 48, in main E args.func(args) E File "/opt/venv/lib/python3.10/site-packages/accelerate/commands/launch.py", line 1168, in launch_command E simple_launcher(args) E File "/opt/venv/lib/python3.10/site-packages/accelerate/commands/launch.py", line 763, in simple_launcher E raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd) E subprocess.CalledProcessError: Command '['/opt/venv/bin/python', 'examples/dreambooth/train_dreambooth_sd3.py', '--pretrained_model_name_or_path', 'hf-internal-testing/tiny-sd3-pipe', '--instance_data_dir', 'docs/source/en/imgs', '--instance_prompt', 'photo', '--resolution', '64', '--train_batch_size', '1', '--gradient_accumulation_steps', '1', '--max_train_steps', '4', '--learning_rate', '5.0e-04', '--scale_lr', '--lr_scheduler', 'constant', '--lr_warmup_steps', '0', '--output_dir', '/tmp/tmpotpr0ant', '--checkpointing_steps=2', '--seed=0']' returned non-zero exit status 1. examples/test_examples_utils.py:43: SubprocessCallException _____ DreamBoothSD3.test_dreambooth_checkpointing_checkpoints_total_limit ______ command = ['accelerate', 'launch', '--config_file', '/tmp/tmpnw4j3n3a/default_config.yml', 'examples/dreambooth/train_dreambooth_sd3.py', '--pretrained_model_name_or_path=hf-internal-testing/tiny-sd3-pipe', ...] return_stdout = False def run_command(command: List[str], return_stdout=False): """ Runs command with subprocess.check_output and will potentially return the stdout. Will also properly capture if an error occurred while running command """ try: output = subprocess.check_output(command, stderr=subprocess.STDOUT) if return_stdout: if hasattr(output, "decode"): output = output.decode("utf-8") return output except subprocess.CalledProcessError as e: raise SubprocessCallException( f"Command {' '.join(command)} failed with the following error:\n\n{e.output.decode()}" ) from e E test_examples_utils.SubprocessCallException: Command accelerate launch --config_file /tmp/tmpnw4j3n3a/default_config.yml examples/dreambooth/train_dreambooth_sd3.py --pretrained_model_name_or_path=hf-internal-testing/tiny-sd3-pipe --instance_data_dir=docs/source/en/imgs --output_dir=/tmp/tmpo2cbphhs --instance_prompt=photo --resolution=64 --train_batch_size=1 --gradient_accumulation_steps=1 --max_train_steps=6 --checkpoints_total_limit=2 --checkpointing_steps=2 failed with the following error: E
E 10/30/2024 11:21:41 - INFO - main - Distributed environment: NO E Num processes: 1 E Process index: 0 E Local process index: 0 E Device: cpu E
E Mixed precision type: no E
E You are using a model of type clip_text_model to instantiate a model of type . This is not supported for all configurations of models and can yield errors. E You are using a model of type clip_text_model to instantiate a model of type . This is not supported for all configurations of models and can yield errors. E You are using a model of type t5 to instantiate a model of type . This is not supported for all configurations of models and can yield errors. E {'base_shift', 'base_image_seq_len', 'max_image_seq_len', 'max_shift', 'use_dynamic_shifting'} was not found in config. Values will be initialized to default values. E {'mid_block_add_attention'} was not found in config. Values will be initialized to default values. E {'qk_norm', 'dual_attention_layers'} was not found in config. Values will be initialized to default values. E Traceback (most recent call last): E File "/__w/diffusers/diffusers/examples/dreambooth/train_dreambooth_sd3.py", line 1819, in E main(args) E File "/__w/diffusers/diffusers/examples/dreambooth/train_dreambooth_sd3.py", line 1387, in main E instance_prompt_hidden_states, instance_pooled_prompt_embeds = compute_text_embeddings( E File "/__w/diffusers/diffusers/examples/dreambooth/train_dreambooth_sd3.py", line 1376, in compute_text_embeddings E prompt_embeds, pooled_prompt_embeds = encode_prompt( E File "/__w/diffusers/diffusers/examples/dreambooth/train_dreambooth_sd3.py", line 970, in encode_prompt E t5_prompt_embed = _encode_prompt_with_t5( E TypeError: _encode_prompt_with_t5() got an unexpected keyword argument 'text_input_ids' E Traceback (most recent call last): E File "/opt/venv/bin/accelerate", line 8, in E sys.exit(main()) E File "/opt/venv/lib/python3.10/site-packages/accelerate/commands/accelerate_cli.py", line 48, in main E args.func(args) E File "/opt/venv/lib/python3.10/site-packages/accelerate/commands/launch.py", line 1168, in launch_command E simple_launcher(args) E File "/opt/venv/lib/python3.10/site-packages/accelerate/commands/launch.py", line 763, in simple_launcher E raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd) E subprocess.CalledProcessError: Command '['/opt/venv/bin/python', 'examples/dreambooth/train_dreambooth_sd3.py', '--pretrained_model_name_or_path=hf-internal-testing/tiny-sd3-pipe', '--instance_data_dir=docs/source/en/imgs', '--output_dir=/tmp/tmpo2cbphhs', '--instance_prompt=photo', '--resolution=64', '--train_batch_size=1', '--gradient_accumulation_steps=1', '--max_train_steps=6', '--checkpoints_total_limit=2', '--checkpointing_steps=2']' returned non-zero exit status 1. examples/test_examples_utils.py:43: SubprocessCallException _ DreamBoothSD3.test_dreambooth_checkpointing_checkpoints_total_limit_removes_multiple_checkpoints _ command = ['accelerate', 'launch', '--config_file', '/tmp/tmpnw4j3n3a/default_config.yml', 'examples/dreambooth/train_dreambooth_sd3.py', '--pretrained_model_name_or_path=hf-internal-testing/tiny-sd3-pipe', ...] return_stdout = False def run_command(command: List[str], return_stdout=False): """ Runs command with subprocess.check_output and will potentially return the stdout. Will also properly capture if an error occurred while running command """ try: output = subprocess.check_output(command, stderr=subprocess.STDOUT) if return_stdout: if hasattr(output, "decode"): output = output.decode("utf-8") return output except subprocess.CalledProcessError as e: raise SubprocessCallException( f"Command {' '.join(command)} failed with the following error:\n\n{e.output.decode()}" ) from e E test_examples_utils.SubprocessCallException: Command accelerate launch --config_file /tmp/tmpnw4j3n3a/default_config.yml examples/dreambooth/train_dreambooth_sd3.py --pretrained_model_name_or_path=hf-internal-testing/tiny-sd3-pipe --instance_data_dir=docs/source/en/imgs --output_dir=/tmp/tmp8vofedjj --instance_prompt=photo --resolution=64 --train_batch_size=1 --gradient_accumulation_steps=1 --max_train_steps=4 --checkpointing_steps=2 failed with the following error: E
E 10/30/2024 11:21:48 - INFO - main - Distributed environment: NO E Num processes: 1 E Process index: 0 E Local process index: 0 E Device: cpu E
E Mixed precision type: no E
E You are using a model of type clip_text_model to instantiate a model of type . This is not supported for all configurations of models and can yield errors. E You are using a model of type clip_text_model to instantiate a model of type . This is not supported for all configurations of models and can yield errors. E You are using a model of type t5 to instantiate a model of type . This is not supported for all configurations of models and can yield errors. E {'max_image_seq_len', 'max_shift', 'base_image_seq_len', 'use_dynamic_shifting', 'base_shift'} was not found in config. Values will be initialized to default values. E {'mid_block_add_attention'} was not found in config. Values will be initialized to default values. E {'dual_attention_layers', 'qk_norm'} was not found in config. Values will be initialized to default values. E Traceback (most recent call last): E File "/__w/diffusers/diffusers/examples/dreambooth/train_dreambooth_sd3.py", line 1819, in E main(args) E File "/__w/diffusers/diffusers/examples/dreambooth/train_dreambooth_sd3.py", line 1387, in main E instance_prompt_hidden_states, instance_pooled_prompt_embeds = compute_text_embeddings( E File "/__w/diffusers/diffusers/examples/dreambooth/train_dreambooth_sd3.py", line 1376, in compute_text_embeddings E prompt_embeds, pooled_prompt_embeds = encode_prompt( E File "/__w/diffusers/diffusers/examples/dreambooth/train_dreambooth_sd3.py", line 970, in encode_prompt E t5_prompt_embed = _encode_prompt_with_t5( E TypeError: _encode_prompt_with_t5() got an unexpected keyword argument 'text_input_ids' E Traceback (most recent call last): E File "/opt/venv/bin/accelerate", line 8, in E sys.exit(main()) E File "/opt/venv/lib/python3.10/site-packages/accelerate/commands/accelerate_cli.py", line 48, in main E args.func(args) E File "/opt/venv/lib/python3.10/site-packages/accelerate/commands/launch.py", line 1168, in launch_command E simple_launcher(args) E File "/opt/venv/lib/python3.10/site-packages/accelerate/commands/launch.py", line 763, in simple_launcher E raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd) E subprocess.CalledProcessError: Command '['/opt/venv/bin/python', 'examples/dreambooth/train_dreambooth_sd3.py', '--pretrained_model_name_or_path=hf-internal-testing/tiny-sd3-pipe', '--instance_data_dir=docs/source/en/imgs', '--output_dir=/tmp/tmp8vofedjj', '--instance_prompt=photo', '--resolution=64', '--train_batch_size=1', '--gradient_accumulation_steps=1', '--max_train_steps=4', '--checkpointing_steps=2']' returned non-zero exit status 1. examples/test_examples_utils.py:43: SubprocessCallException

Might possibly have something to do with line 976 - I see that _encode_prompt_with_clip supports the text_input_ids, and it is passed correctly. But _encode_prompt_with_t5 does not seem to support it, yet the ids are passed.