merge model_specific_prompt_kwargs and dataset_name into each task yaml · EvolvingLMMs-Lab/lmms-eval@0581ab3 (original) (raw)

18 files changed

lines changed

Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
1 1 dataset_path: MM-UPD/MM-UPD
2 2 doc_to_target: "answer"
3 -model_specific_prompt_kwargs:
4 - default:
5 - pre_prompt: ""
6 - post_prompt: "\n"
7 3 doc_to_visual: !function utils.mmupd_doc_to_visual
8 4 doc_to_text: !function utils.mmupd_doc_to_text
9 5 doc_to_target: "answer"
@@ -12,7 +8,6 @@ model_specific_generation_kwargs:
12 8 llava:
13 9 image_aspect_ratio: original
14 10 output_type: generate_until
15 -dataset_name: mmaad_base
16 11 generation_kwargs:
17 12 until:
18 13 - "ASSISTANT:"
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
1 1 task: "mmaad_base"
2 2 test_split: test
3 -include: _default_template_mmaad_base_yaml
3 +dataset_name: mmaad_base
4 +model_specific_prompt_kwargs:
5 +default:
6 +pre_prompt: ""
7 +post_prompt: "\n"
8 +include: _default_template_mmupd_yaml
4 9 metric_list:
5 10 - metric: gpt_eval_score
6 11 aggregation: !function utils.mmaad_base
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
1 1 task: "mmaad_instruction"
2 2 test_split: test
3 -include: _default_template_mmaad_instruction_yaml
3 +dataset_name: mmaad_base
4 +model_specific_prompt_kwargs:
5 +default:
6 +pre_prompt: ""
7 +post_prompt: "\nIf all the options are incorrect, answer \"F. None of the above\"."
8 +include: _default_template_mmupd_yaml
4 9 metric_list:
5 10 - metric: gpt_eval_score
6 11 aggregation: !function utils.mmaad_instruction
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
1 1 task: "mmaad_option"
2 2 test_split: test
3 -include: _default_template_mmaad_option_yaml
3 +dataset_name: mmaad_option
4 +model_specific_prompt_kwargs:
5 +default:
6 +pre_prompt: ""
7 +post_prompt: "\nAnswer with the option's letter from the given choices directly."
8 +include: _default_template_mmupd_yaml
4 9 metric_list:
5 10 - metric: gpt_eval_score
6 11 aggregation: !function utils.mmaad_option
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
1 1 task: "mmiasd_base"
2 2 test_split: test
3 -include: _default_template_mmiasd_base_yaml
3 +dataset_name: mmiasd_base
4 +model_specific_prompt_kwargs:
5 +default:
6 +pre_prompt: ""
7 +post_prompt: "\n"
8 +include: _default_template_mmupd_yaml
4 9 metric_list:
5 10 - metric: gpt_eval_score
6 11 aggregation: !function utils.mmiasd_base
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
1 1 task: "mmiasd_instruction"
2 2 test_split: test
3 -include: _default_template_mmiasd_instruction_yaml
3 +dataset_name: mmiasd_base
4 +model_specific_prompt_kwargs:
5 +default:
6 +pre_prompt: ""
7 +post_prompt: "\nIf all the options are incorrect, answer \"F. None of the above\"."
8 +include: _default_template_mmupd_yaml
4 9 metric_list:
5 10 - metric: gpt_eval_score
6 11 aggregation: !function utils.mmiasd_instruction
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
1 1 task: "mmiasd_option"
2 2 test_split: test
3 -include: _default_template_mmiasd_option_yaml
3 +dataset_name: mmiasd_option
4 +model_specific_prompt_kwargs:
5 +default:
6 +pre_prompt: ""
7 +post_prompt: "\nAnswer with the option's letter from the given choices directly."
8 +include: _default_template_mmupd_yaml
4 9 metric_list:
5 10 - metric: gpt_eval_score
6 11 aggregation: !function utils.mmiasd_option
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
1 1 task: "mmivqd_base"
2 2 test_split: test
3 -include: _default_template_mmivqd_base_yaml
3 +dataset_name: mmivqd_base
4 +model_specific_prompt_kwargs:
5 +default:
6 +pre_prompt: ""
7 +post_prompt: "\n"
8 +include: _default_template_mmupd_yaml
4 9 metric_list:
5 10 - metric: gpt_eval_score
6 11 aggregation: !function utils.mmivqd_base
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
1 1 task: "mmivqd_instruction"
2 2 test_split: test
3 -include: _default_template_mmivqd_instruction_yaml
3 +dataset_name: mmivqd_base
4 +model_specific_prompt_kwargs:
5 +default:
6 +pre_prompt: ""
7 +post_prompt: "\nIf the given image is irrelevant to the question, answer \"F. The image and question are irrelevant.\"."
8 +include: _default_template_mmupd_yaml
4 9 metric_list:
5 10 - metric: gpt_eval_score
6 11 aggregation: !function utils.mmivqd_instruction
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
1 1 task: "mmivqd_option"
2 2 test_split: test
3 -include: _default_template_mmivqd_option_yaml
3 +dataset_name: mmivqd_option
4 +model_specific_prompt_kwargs:
5 +default:
6 +pre_prompt: ""
7 +post_prompt: "\nAnswer with the option's letter from the given choices directly."
8 +include: _default_template_mmupd_yaml
4 9 metric_list:
5 10 - metric: gpt_eval_score
6 11 aggregation: !function utils.mmivqd_option