Upload folder using huggingface_hub

configuration_internvl_chat.py

@@ -27,11 +27,10 @@ class InternVLChatConfig(PretrainedConfig):
             use_backbone_lora=0,
             use_llm_lora=0,
             pad2square=False,
-            select_layer=-4,
+            select_layer=-1,
             force_image_size=None,
             downsample_ratio=0.5,
             template=None,
-            image_fold=False,
             dynamic_image_size=False,
             use_thumbnail=False,
             ps_version='v1',
@@ -62,7 +61,6 @@ class InternVLChatConfig(PretrainedConfig):
         self.force_image_size = force_image_size
         self.downsample_ratio = downsample_ratio
         self.template = template
-        self.image_fold = image_fold
         self.dynamic_image_size = dynamic_image_size
         self.use_thumbnail = use_thumbnail
         self.ps_version = ps_version  # pixel shuffle version
@@ -70,7 +68,6 @@ class InternVLChatConfig(PretrainedConfig):
         self.max_dynamic_patch = max_dynamic_patch
 
         logger.info(f'vision_select_layer: {self.select_layer}')
-        logger.info(f'image_fold: {self.image_fold}')
         logger.info(f'ps_version: {self.ps_version}')
         logger.info(f'min_dynamic_patch: {self.min_dynamic_patch}')
         logger.info(f'max_dynamic_patch: {self.max_dynamic_patch}')
@@ -93,7 +90,6 @@ class InternVLChatConfig(PretrainedConfig):
         output['force_image_size'] = self.force_image_size
         output['downsample_ratio'] = self.downsample_ratio
         output['template'] = self.template
-        output['image_fold'] = self.image_fold
         output['dynamic_image_size'] = self.dynamic_image_size
         output['use_thumbnail'] = self.use_thumbnail
         output['ps_version'] = self.ps_version

modeling_internvl_chat.py

@@ -23,40 +23,6 @@ from .modeling_internlm2 import InternLM2ForCausalLM
 logger = logging.get_logger(__name__)
 
 
-def window_partition(x, window_size):
-    """
-    Args:
-        x: (B, C, H, W)
-        window_size (int): window size, assuming square window
-
-    Returns:
-        windows: (num_windows*B, C, window_size, window_size)
-    """
-    B, C, H, W = x.shape
-    assert H % window_size == 0 and W % window_size == 0, 'H and W must be divisible by window_size'
-
-    x = x.view(B, C, H // window_size, window_size, W // window_size, window_size)
-    windows = x.permute(0, 2, 4, 1, 3, 5).contiguous().view(-1, C, window_size, window_size)
-    return windows
-
-
-def window_reverse(windows, window_size, H, W):
-    """
-    Args:
-        windows: (num_windows*B, window_size, window_size, C)
-        window_size (int): Window size
-        H (int): Height of image
-        W (int): Width of image
-
-    Returns:
-        x: (B, H * W, C)
-    """
-    B = int(windows.shape[0] / (H * W / window_size / window_size))
-    x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1)
-    x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H * W, -1)
-    return x
-
-
 class InternVLChatModel(PreTrainedModel):
     config_class = InternVLChatConfig
     main_input_name = 'pixel_values'
@@ -72,7 +38,6 @@ class InternVLChatModel(PreTrainedModel):
         self.template = config.template
         self.num_image_token = int((image_size // patch_size) ** 2 * (config.downsample_ratio ** 2))
         self.downsample_ratio = config.downsample_ratio
-        self.image_fold = config.image_fold
         self.ps_version = config.ps_version
 
         logger.info(f'num_image_token: {self.num_image_token}')
@@ -242,10 +207,6 @@ class InternVLChatModel(PreTrainedModel):
         return vit_embeds + noise
 
     def extract_feature(self, pixel_values):
-        if self.image_fold:
-            image_size = pixel_values.size(-1)  # B, C, H, W
-            pixel_values = window_partition(pixel_values, window_size=image_size // self.image_fold)  # 4B, C, H/2, W/2
-
         if self.select_layer == -1:
             vit_embeds = self.vision_model(
                 pixel_values=pixel_values,
@@ -261,21 +222,55 @@ class InternVLChatModel(PreTrainedModel):
         if self.training and self.neftune_alpha is not None:
             vit_embeds = self.noised_embed(vit_embeds, self.neftune_alpha)
 
-        if self.image_fold:
-            vit_embeds = window_reverse(vit_embeds, window_size=image_size // (self.image_fold * self.patch_size),
-                                        H=image_size // self.patch_size, W=image_size // self.patch_size)
-
-        # if torch.distributed.get_rank() == 0:
-        #     print("before pixel shuffle:", vit_embeds.shape)
         h = w = int(vit_embeds.shape[1] ** 0.5)
         vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
         vit_embeds = self.pixel_shuffle(vit_embeds, scale_factor=self.downsample_ratio)
         vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], -1, vit_embeds.shape[-1])
-        # if torch.distributed.get_rank() == 0:
-        #     print("after pixel shuffle:", vit_embeds.shape)
         vit_embeds = self.mlp1(vit_embeds)
         return vit_embeds
 
+    def batch_chat(self, tokenizer, pixel_values, image_counts, questions, generation_config, history=None,
+                         return_history=False, IMG_START_TOKEN='<img>', IMG_END_TOKEN='</img>',
+                         IMG_CONTEXT_TOKEN='<IMG_CONTEXT>'):
+        if history is not None or return_history:
+            print("Now multi-turn chat is not supported in batch_chat.")
+            raise NotImplementedError
+        img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
+        self.img_context_token_id = img_context_token_id
+        if tokenizer.convert_tokens_to_ids('<|im_end|>') != 0:
+            eos_token_id = tokenizer.convert_tokens_to_ids('<|im_end|>')  # 92542, InternLM2
+        else:
+            eos_token_id = tokenizer.eos_token_id
+    
+        from .conversation import get_conv_template
+    
+        queries = []
+        image_bs = pixel_values.shape[0]
+        print(f'dynamic ViT batch size: {image_bs}, image_counts: {image_counts}')
+        for idx, image_count in enumerate(image_counts):
+            image_token = IMG_START_TOKEN + IMG_CONTEXT_TOKEN * self.num_image_token * image_count + IMG_END_TOKEN
+            question = image_token + '\n' + questions[idx]
+            template = get_conv_template(self.template)
+            template.append_message(template.roles[0], question)
+            template.append_message(template.roles[1], None)
+            query = template.get_prompt()
+            queries.append(query)
+        tokenizer.padding_side = 'left'
+        model_inputs = tokenizer(queries, return_tensors='pt', padding=True)
+        input_ids = model_inputs['input_ids'].cuda()
+        attention_mask = model_inputs['attention_mask'].cuda()
+        generation_config['eos_token_id'] = eos_token_id
+    
+        generation_output = self.generate(
+            pixel_values=pixel_values,
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            **generation_config
+        )
+        responses = tokenizer.batch_decode(generation_output, skip_special_tokens=True)
+        responses = [response.split('<|im_end|>')[0].strip() for response in responses]  # for InternLM2
+        return responses
+
     def chat(self, tokenizer, pixel_values, question, generation_config, history=None, return_history=False,
              IMG_START_TOKEN='<img>', IMG_END_TOKEN='</img>', IMG_CONTEXT_TOKEN='<IMG_CONTEXT>'):