Merge pull request #105 from tianyu-z/main · EvolvingLMMs-Lab/lmms-eval@44a3379 (original) (raw)
`@@ -19,7 +19,9 @@ def textvqa_doc_to_visual(doc):
`
19
19
``
20
20
`def textvqa_process_results(doc, result):
`
21
21
`eval_ai_processor = EvalAIAnswerProcessor()
`
22
``
`-
assert len(result) == 1, f"The result should be a list of length 1, but got {len(result)}."
`
``
22
`+
assert (
`
``
23
`+
len(result) == 1
`
``
24
`+
), f"The result should be a list of length 1, but got {len(result)}."
`
23
25
`resAns = eval_ai_processor(result[0])
`
24
26
`accuracy = 0
`
25
27
``
`@@ -30,7 +32,9 @@ def textvqa_process_results(doc, result):
`
30
32
`doc["answers"][i] = eval_ai_processor(doc["answers"][i])
`
31
33
``
32
34
`for i in range(len(doc["answers"])):
`
33
``
`-
otherGTAns = [doc["answers"][j] for j in range(len(doc["answers"])) if i != j]
`
``
35
`+
otherGTAns = [
`
``
36
`+
doc["answers"][j] for j in range(len(doc["answers"])) if i != j
`
``
37
`+
]
`
34
38
`matchingAns = [item for item in otherGTAns if item == resAns]
`
35
39
`acc = min(1, float(len(matchingAns)) / 3)
`
36
40
`gtAcc.append(acc)
`
`@@ -54,12 +58,15 @@ def textvqa_doc_to_text(doc, model_specific_prompt_kwargs=None):
`
54
58
`pre_prompt = model_specific_prompt_kwargs["pre_prompt"]
`
55
59
`if "post_prompt" in model_specific_prompt_kwargs:
`
56
60
`post_prompt = model_specific_prompt_kwargs["post_prompt"]
`
57
``
`-
if "ocr" in model_specific_prompt_kwargs and model_specific_prompt_kwargs["ocr"]:
`
``
61
`+
if (
`
``
62
`+
"ocr" in model_specific_prompt_kwargs
`
``
63
`+
and model_specific_prompt_kwargs["ocr"]
`
``
64
`+
):
`
58
65
`ocr_ref = f"\nReference OCR token: {', '.join(doc['ocr_tokens'])}"
`
59
66
`return f"{pre_prompt}{doc['question'].capitalize()}{ocr_ref}{post_prompt}"
`
60
67
``
61
68
``
62
``
`-
def textvqa_aggreate_submissions(results, args):
`
``
69
`+
def textvqa_aggregate_submissions(results, args):
`
63
70
`now_date_time = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
`
64
71
`path = generate_submission_file(f"textvqa_submission_{now_date_time}.json", args)
`
65
72
`with open(path, "w") as f:
`