|
import torch |
|
from new_impl.cv.elasticdnn.api.algs.fm_lora_glip import ElasticDNN_FMLoRAAlg |
|
from methods.elasticdnn.model.base import ElasticDNNUtil |
|
from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util |
|
from glip import FMLoRA_GLIP_Util, ElasticDNN_OfflineMMDetFMModel |
|
from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util |
|
from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util |
|
from methods.elasticdnn.model.vit import ElasticViTUtil |
|
from utils.dl.common.model import LayerActivation, get_module |
|
from utils.common.exp import save_models_dict_for_init, get_res_save_dir |
|
from data import build_scenario |
|
import torch.nn.functional as F |
|
from maskrcnn_benchmark.structures.bounding_box import BoxList |
|
import os |
|
os.environ['TOKENIZERS_PARALLELISM'] = 'true' |
|
|
|
class ElasticDNN_GLIP_OfflineMMDetFMModel(ElasticDNN_OfflineMMDetFMModel): |
|
def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): |
|
raise NotImplementedError |
|
|
|
def get_feature_hook(self) -> LayerActivation: |
|
return LayerActivation(get_module(self.models_dict['main'], 'visual_projection'), True, self.device), 'output' |
|
|
|
def get_elastic_dnn_util(self) -> ElasticDNNUtil: |
|
raise NotImplementedError |
|
|
|
def forward_to_get_task_loss(self, x, y, *args, **kwargs): |
|
|
|
x['for_training'] = True |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
loss_dict = self.infer(x) |
|
losses = sum(loss for loss in loss_dict.values()) |
|
|
|
|
|
return losses |
|
|
|
def get_lora_util(self) -> FMLoRA_Util: |
|
return FMLoRA_GLIP_Util() |
|
|
|
def get_task_head_params(self): |
|
return [] |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__': |
|
|
|
scenario = build_scenario( |
|
source_datasets_name=['MM-COCO2017'], |
|
target_datasets_order=['MM-CityscapesDet'], |
|
da_mode='close_set', |
|
data_dirs={ |
|
'MM-COCO2017': '/data/zql/datasets/coco2017', |
|
'MM-CityscapesDet': '/data/zql/datasets/cityscape' |
|
}, |
|
) |
|
cfg_path = 'new_impl/cv/glip/object_detection/pretrained_model/glip_Swin_T_O365_GoldG.yaml' |
|
model_path = 'new_impl/cv/glip/object_detection/pretrained_model/glip_tiny_model_o365_goldg_cc_sbu.pth' |
|
|
|
from glip import glip_model, build_transform, run_ner, collect_mm_fn |
|
config, gmodel = glip_model(cfg_path, model_path) |
|
transform = build_transform(config, None) |
|
fm_models_dict_path = save_models_dict_for_init({ |
|
'main': gmodel |
|
}, __file__, 'fm_glip_pretrained') |
|
device = 'cuda' |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fm_model = ElasticDNN_GLIP_OfflineMMDetFMModel('fm', fm_models_dict_path, device, collate_fn=collect_mm_fn) |
|
|
|
|
|
models = { |
|
'fm': fm_model |
|
} |
|
import sys |
|
fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__, tag="results")) |
|
|
|
from PIL import Image, ImageDraw |
|
import requests |
|
import numpy as np |
|
from evaluator import MMCOCODecoder |
|
ori_image = Image.open('new_impl/cv/glip/object_detection/000000103759.jpg').convert("RGB") |
|
image = transform(np.asarray(ori_image)[:, :, [2, 1, 0]]) |
|
text = 'orange. umbrella. ' |
|
targets = BoxList(torch.FloatTensor([[0., 0., 0., 0.]]), image_size=image.size()[1:], mode='xyxy') |
|
targets.add_field('caption', text) |
|
targets.add_field('tokens_positive', run_ner(text)) |
|
targets.add_field('labels', torch.LongTensor([0])) |
|
samples = {'images' : [image], 'targets' : [targets]} |
|
|
|
|
|
from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup |
|
fm_lora_alg.run(scenario, hyps={ |
|
'launch_tbboard': False, |
|
'transform' : transform, |
|
'samples_size': samples, |
|
'ab_r': 8, |
|
'train_batch_size': 16, |
|
'val_batch_size': 1, |
|
'num_workers': 16, |
|
'optimizer': 'Adam', |
|
'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, |
|
'scheduler': 'LambdaLR', |
|
'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, |
|
'num_iters': 6500, |
|
'val_freq': 100 |
|
}, collate_fn=collect_mm_fn) |