add vila · EvolvingLMMs-Lab/lmms-eval@e6844db (original) (raw)
`@@ -59,6 +59,8 @@ def init(
`
59
59
`mm_spatial_pool_mode: str = "average",
`
60
60
`overwrite: bool = True,
`
61
61
`video_decode_backend: str = "pyav",
`
``
62
`+
delay_load: bool = False,
`
``
63
`+
tie_weights: bool = True,
`
62
64
`**kwargs,
`
63
65
` ) -> None:
`
64
66
`super().init()
`
`@@ -86,16 +88,19 @@ def init(
`
86
88
`self.mm_spatial_pool_out_channels = int(mm_spatial_pool_out_channels)
`
87
89
`self.mm_spatial_pool_mode = mm_spatial_pool_mode
`
88
90
`self.max_frames_num = int(max_frames_num)
`
89
``
`-
print(self.max_frames_num)
`
``
91
`+
self.mm_resampler_location = mm_resampler_location
`
``
92
`+
self.delay_load = delay_load
`
90
93
`if self.overwrite == True:
`
91
94
`overwrite_config = {}
`
92
95
`overwrite_config["mm_resampler_type"] = self.mm_resampler_type
`
93
96
`overwrite_config["mm_spatial_pool_stride"] = self.mm_spatial_pool_stride
`
94
97
`overwrite_config["mm_spatial_pool_out_channels"] = self.mm_spatial_pool_out_channels
`
95
98
`overwrite_config["mm_spatial_pool_mode"] = self.mm_spatial_pool_mode
`
96
``
`-
overwrite_config["mm_resampler_location"] = "before"
`
97
``
`-
overwrite_config["patchify_video_feature"] = False
`
98
``
`-
overwrite_config["attn_implementation"] = attn_implementation
`
``
99
`+
overwrite_config["mm_pooling_position"] = self.mm_resampler_location
`
``
100
`+
overwrite_config["mm_newline_position"] = mm_newline_position
`
``
101
`+
overwrite_config["add_faster_video"] = False
`
``
102
`+
overwrite_config["delay_load"] = self.delay_load
`
``
103
`+
overwrite_config["attn_implementation"] = attn_implementation
`
99
104
``
100
105
`cfg_pretrained = AutoConfig.from_pretrained(self.pretrained)
`
101
106
``
`@@ -146,7 +151,8 @@ def init(
`
146
151
``
147
152
`self._config = self._model.config
`
148
153
`self.model.eval()
`
149
``
`-
self.model.tie_weights()
`
``
154
`+
if tie_weights:
`
``
155
`+
self.model.tie_weights()
`
150
156
`self.truncation = truncation
`
151
157
`self.batch_size_per_gpu = int(batch_size)
`
152
158
`self.conv_template = conv_template
`