Fix gen kwargs image aspect ratio in internvl2 · EvolvingLMMs-Lab/lmms-eval@2037a86 (original) (raw)

57 files changed

lines changed

Original file line number	Diff line number	Diff line change
@@ -165,7 +165,6 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
165	165	# reset logger
166	166	eval_logger.remove()
167	167	eval_logger.add(sys.stdout, colorize=True, level=args.verbosity)
168		-# eval_logger.add(sys.stderr, level=args.verbosity)
169	168	eval_logger.info(f"Verbosity set to {args.verbosity}")
170	169	os.environ["TOKENIZERS_PARALLELISM"] = "false"
171	170

Original file line number	Diff line number	Diff line change
@@ -37,7 +37,9 @@ def get_context(self, doc, num_fewshot):
37	37	+ (
38	38	str(self.doc_to_target(doc)[0])
39	39	if type(self.doc_to_target(doc)) is list
40		-else self.doc_to_target(doc) if (self.config.doc_to_choice is None or type(self.doc_to_target(doc)) is str) else str(self.doc_to_choice(doc)[self.doc_to_target(doc)])
	40	+else self.doc_to_target(doc)
	41	+if (self.config.doc_to_choice is None or type(self.doc_to_target(doc)) is str)
	42	+else str(self.doc_to_choice(doc)[self.doc_to_target(doc)])
41	43	)
42	44	for doc in selected_docs
43	45	]

Original file line number	Diff line number	Diff line change
@@ -325,7 +325,7 @@ def evaluate(
325	325	# hack: remove image columns to speed avoid loading images and speed up postprocessing
326	326	# reason: doc_iterator will actually load image if it's in the doc.
327	327	docs = task.test_docs() if task.has_test_docs() else task.validation_docs()
328		-if "d170" not in task_name and "dc100" not in task_name and "dc200" not in task_name and "llava_wilder" not in task_name and "live_bench" not in task_name and "wildvision" not in task_name:
	328	+if "d170" not in task_name and "dc100" not in task_name and "dc200" not in task_name and "llava_wilder" not in task_name and "livebench" not in task_name and "wildvision" not in task_name:
329	329	remove_cols = []
330	330	features = docs.features
331	331	# If it is an Image instance or a Sequence of Image instance. Remove it

Original file line number	Diff line number	Diff line change
@@ -4,6 +4,10 @@
4	4	from loguru import logger
5	5	import sys
6	6
	7	+import hf_transfer
	8	+
	9	+os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
	10	+
7	11	logger.remove()
8	12	logger.add(sys.stdout, level="WARNING")
9	13
@@ -34,7 +38,6 @@
34	38	"tinyllava": "TinyLlava",
35	39	"llava_hf": "LlavaHf",
36	40	"longva": "LongVA",
37		-"llava_onevision": "Llava_OneVision",
38	41	"llava_hf": "LlavaHf",
39	42	"longva": "LongVA",
40	43	"vila": "VILA",
@@ -44,8 +47,7 @@
44	47	try:
45	48	exec(f"from .{model_name} import {model_class}")
46	49	except ImportError as e:
47		-# logger.warning(f"Failed to import {model_class} from {model_name}: {e}")
48		-pass
	50	+logger.warning(f"Failed to import {model_class} from {model_name}: {e}")
49	51
50	52	if os.environ.get("LMMS_EVAL_PLUGINS", None):
51	53	# Allow specifying other packages to import models from
@@ -55,8 +57,4 @@
55	57	try:
56	58	exec(f"from {plugin}.models.{model_name} import {model_class}")
57	59	except ImportError:
58		-pass
59		-
60		-import hf_transfer
61		-
62		-os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
	60	+logger.warning(f"Failed to import {model_class} from {model_name}")