diff --git a/groundingdino/datasets/.ipynb_checkpoints/__init__-checkpoint.py b/groundingdino/datasets/.ipynb_checkpoints/__init__-checkpoint.py deleted file mode 100644 index b1864e13e91e60944a78de5b7903228cce165cdb..0000000000000000000000000000000000000000 --- a/groundingdino/datasets/.ipynb_checkpoints/__init__-checkpoint.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -import torch.utils.data -import torchvision -from .coco import build as build_coco - - -def get_coco_api_from_dataset(dataset): - for _ in range(10): - # if isinstance(dataset, torchvision.datasets.CocoDetection): - # break - if isinstance(dataset, torch.utils.data.Subset): - dataset = dataset.dataset - if isinstance(dataset, torchvision.datasets.CocoDetection): - return dataset.coco - - -def build_dataset(image_set, args, datasetinfo): - if datasetinfo["dataset_mode"] == 'coco': - return build_coco(image_set, args, datasetinfo) - if datasetinfo["dataset_mode"] == 'odvg': - from .odvg import build_odvg - return build_odvg(image_set, args, datasetinfo) - raise ValueError(f'dataset {args.dataset_file} not supported') diff --git a/groundingdino/datasets/.ipynb_checkpoints/coco-checkpoint.py b/groundingdino/datasets/.ipynb_checkpoints/coco-checkpoint.py deleted file mode 100644 index ae9acf582138bbc22145828c59f96dc17b998749..0000000000000000000000000000000000000000 --- a/groundingdino/datasets/.ipynb_checkpoints/coco-checkpoint.py +++ /dev/null @@ -1,649 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -""" -COCO dataset which returns image_id for evaluation. - -Mostly copy-paste from https://github.com/pytorch/vision/blob/13b35ff/references/detection/coco_utils.py -""" -if __name__=="__main__": - # for debug only - import os, sys - sys.path.append(os.path.dirname(sys.path[0])) -from torchvision.datasets.vision import VisionDataset - -import json -from pathlib import Path -import random -import os -from typing import Any, Callable, List, Optional, Tuple - -from PIL import Image - -import torch -import torch.utils.data -import torchvision -from pycocotools import mask as coco_mask - -from datasets.data_util import preparing_dataset -import datasets.transforms as T -from util.box_ops import box_cxcywh_to_xyxy, box_iou - -__all__ = ['build'] - - -class label2compat(): - def __init__(self) -> None: - self.category_map_str = {"1": 1, "2": 2, "3": 3, "4": 4, "5": 5, "6": 6, "7": 7, "8": 8, "9": 9, "10": 10, "11": 11, "13": 12, "14": 13, "15": 14, "16": 15, "17": 16, "18": 17, "19": 18, "20": 19, "21": 20, "22": 21, "23": 22, "24": 23, "25": 24, "27": 25, "28": 26, "31": 27, "32": 28, "33": 29, "34": 30, "35": 31, "36": 32, "37": 33, "38": 34, "39": 35, "40": 36, "41": 37, "42": 38, "43": 39, "44": 40, "46": 41, "47": 42, "48": 43, "49": 44, "50": 45, "51": 46, "52": 47, "53": 48, "54": 49, "55": 50, "56": 51, "57": 52, "58": 53, "59": 54, "60": 55, "61": 56, "62": 57, "63": 58, "64": 59, "65": 60, "67": 61, "70": 62, "72": 63, "73": 64, "74": 65, "75": 66, "76": 67, "77": 68, "78": 69, "79": 70, "80": 71, "81": 72, "82": 73, "84": 74, "85": 75, "86": 76, "87": 77, "88": 78, "89": 79, "90": 80} - self.category_map = {int(k):v for k,v in self.category_map_str.items()} - - def __call__(self, target, img=None): - labels = target['labels'] - res = torch.zeros(labels.shape, dtype=labels.dtype) - for idx, item in enumerate(labels): - res[idx] = self.category_map[item.item()] - 1 - target['label_compat'] = res - if img is not None: - return target, img - else: - return target - - -class label_compat2onehot(): - def __init__(self, num_class=80, num_output_objs=1): - self.num_class = num_class - self.num_output_objs = num_output_objs - if num_output_objs != 1: - raise DeprecationWarning("num_output_objs!=1, which is only used for comparison") - - def __call__(self, target, img=None): - labels = target['label_compat'] - place_dict = {k:0 for k in range(self.num_class)} - if self.num_output_objs == 1: - res = torch.zeros(self.num_class) - for i in labels: - itm = i.item() - res[itm] = 1.0 - else: - # compat with baseline - res = torch.zeros(self.num_class, self.num_output_objs) - for i in labels: - itm = i.item() - res[itm][place_dict[itm]] = 1.0 - place_dict[itm] += 1 - target['label_compat_onehot'] = res - if img is not None: - return target, img - else: - return target - - -class box_label_catter(): - def __init__(self): - pass - - def __call__(self, target, img=None): - labels = target['label_compat'] - boxes = target['boxes'] - box_label = torch.cat((boxes, labels.unsqueeze(-1)), 1) - target['box_label'] = box_label - if img is not None: - return target, img - else: - return target - - -class RandomSelectBoxlabels(): - def __init__(self, num_classes, leave_one_out=False, blank_prob=0.8, - prob_first_item = 0.0, - prob_random_item = 0.0, - prob_last_item = 0.8, - prob_stop_sign = 0.2 - ) -> None: - self.num_classes = num_classes - self.leave_one_out = leave_one_out - self.blank_prob = blank_prob - - self.set_state(prob_first_item, prob_random_item, prob_last_item, prob_stop_sign) - - - def get_state(self): - return [self.prob_first_item, self.prob_random_item, self.prob_last_item, self.prob_stop_sign] - - def set_state(self, prob_first_item, prob_random_item, prob_last_item, prob_stop_sign): - sum_prob = prob_first_item + prob_random_item + prob_last_item + prob_stop_sign - assert sum_prob - 1 < 1e-6, \ - f"Sum up all prob = {sum_prob}. prob_first_item:{prob_first_item}" \ - + f"prob_random_item:{prob_random_item}, prob_last_item:{prob_last_item}" \ - + f"prob_stop_sign:{prob_stop_sign}" - - self.prob_first_item = prob_first_item - self.prob_random_item = prob_random_item - self.prob_last_item = prob_last_item - self.prob_stop_sign = prob_stop_sign - - - def sample_for_pred_first_item(self, box_label: torch.FloatTensor): - box_label_known = torch.Tensor(0,5) - box_label_unknown = box_label - return box_label_known, box_label_unknown - - def sample_for_pred_random_item(self, box_label: torch.FloatTensor): - n_select = int(random.random() * box_label.shape[0]) - box_label = box_label[torch.randperm(box_label.shape[0])] - box_label_known = box_label[:n_select] - box_label_unknown = box_label[n_select:] - return box_label_known, box_label_unknown - - def sample_for_pred_last_item(self, box_label: torch.FloatTensor): - box_label_perm = box_label[torch.randperm(box_label.shape[0])] - known_label_list = [] - box_label_known = [] - box_label_unknown = [] - for item in box_label_perm: - label_i = item[4].item() - if label_i in known_label_list: - box_label_known.append(item) - else: - # first item - box_label_unknown.append(item) - known_label_list.append(label_i) - box_label_known = torch.stack(box_label_known) if len(box_label_known) > 0 else torch.Tensor(0,5) - box_label_unknown = torch.stack(box_label_unknown) if len(box_label_unknown) > 0 else torch.Tensor(0,5) - return box_label_known, box_label_unknown - - def sample_for_pred_stop_sign(self, box_label: torch.FloatTensor): - box_label_unknown = torch.Tensor(0,5) - box_label_known = box_label - return box_label_known, box_label_unknown - - def __call__(self, target, img=None): - box_label = target['box_label'] # K, 5 - - dice_number = random.random() - - if dice_number < self.prob_first_item: - box_label_known, box_label_unknown = self.sample_for_pred_first_item(box_label) - elif dice_number < self.prob_first_item + self.prob_random_item: - box_label_known, box_label_unknown = self.sample_for_pred_random_item(box_label) - elif dice_number < self.prob_first_item + self.prob_random_item + self.prob_last_item: - box_label_known, box_label_unknown = self.sample_for_pred_last_item(box_label) - else: - box_label_known, box_label_unknown = self.sample_for_pred_stop_sign(box_label) - - target['label_onehot_known'] = label2onehot(box_label_known[:,-1], self.num_classes) - target['label_onehot_unknown'] = label2onehot(box_label_unknown[:, -1], self.num_classes) - target['box_label_known'] = box_label_known - target['box_label_unknown'] = box_label_unknown - - return target, img - - -class RandomDrop(): - def __init__(self, p=0.2) -> None: - self.p = p - - def __call__(self, target, img=None): - known_box = target['box_label_known'] - num_known_box = known_box.size(0) - idxs = torch.rand(num_known_box) - # indices = torch.randperm(num_known_box)[:int((1-self).p*num_known_box + 0.5 + random.random())] - target['box_label_known'] = known_box[idxs > self.p] - return target, img - - -class BboxPertuber(): - def __init__(self, max_ratio = 0.02, generate_samples = 1000) -> None: - self.max_ratio = max_ratio - self.generate_samples = generate_samples - self.samples = self.generate_pertube_samples() - self.idx = 0 - - def generate_pertube_samples(self): - import torch - samples = (torch.rand(self.generate_samples, 5) - 0.5) * 2 * self.max_ratio - return samples - - def __call__(self, target, img): - known_box = target['box_label_known'] # Tensor(K,5), K known bbox - K = known_box.shape[0] - known_box_pertube = torch.zeros(K, 6) # 4:bbox, 1:prob, 1:label - if K == 0: - pass - else: - if self.idx + K > self.generate_samples: - self.idx = 0 - delta = self.samples[self.idx: self.idx + K, :] - known_box_pertube[:, :4] = known_box[:, :4] + delta[:, :4] - iou = (torch.diag(box_iou(box_cxcywh_to_xyxy(known_box[:, :4]), box_cxcywh_to_xyxy(known_box_pertube[:, :4]))[0])) * (1 + delta[:, -1]) - known_box_pertube[:, 4].copy_(iou) - known_box_pertube[:, -1].copy_(known_box[:, -1]) - - target['box_label_known_pertube'] = known_box_pertube - return target, img - - -class RandomCutout(): - def __init__(self, factor=0.5) -> None: - self.factor = factor - - def __call__(self, target, img=None): - unknown_box = target['box_label_unknown'] # Ku, 5 - known_box = target['box_label_known_pertube'] # Kk, 6 - Ku = unknown_box.size(0) - - known_box_add = torch.zeros(Ku, 6) # Ku, 6 - known_box_add[:, :5] = unknown_box - known_box_add[:, 5].uniform_(0.5, 1) - - - known_box_add[:, :2] += known_box_add[:, 2:4] * (torch.rand(Ku, 2) - 0.5) / 2 - known_box_add[:, 2:4] /= 2 - - target['box_label_known_pertube'] = torch.cat((known_box, known_box_add)) - return target, img - - -class RandomSelectBoxes(): - def __init__(self, num_class=80) -> None: - Warning("This is such a slow function and will be deprecated soon!!!") - self.num_class = num_class - - def __call__(self, target, img=None): - boxes = target['boxes'] - labels = target['label_compat'] - - # transform to list of tensors - boxs_list = [[] for i in range(self.num_class)] - for idx, item in enumerate(boxes): - label = labels[idx].item() - boxs_list[label].append(item) - boxs_list_tensor = [torch.stack(i) if len(i) > 0 else torch.Tensor(0,4) for i in boxs_list] - - # random selection - box_known = [] - box_unknown = [] - for idx, item in enumerate(boxs_list_tensor): - ncnt = item.shape[0] - nselect = int(random.random() * ncnt) # close in both sides, much faster than random.randint - - item = item[torch.randperm(ncnt)] - # random.shuffle(item) - box_known.append(item[:nselect]) - box_unknown.append(item[nselect:]) - - # box_known_tensor = [torch.stack(i) if len(i) > 0 else torch.Tensor(0,4) for i in box_known] - # box_unknown_tensor = [torch.stack(i) if len(i) > 0 else torch.Tensor(0,4) for i in box_unknown] - # print('box_unknown_tensor:', box_unknown_tensor) - target['known_box'] = box_known - target['unknown_box'] = box_unknown - return target, img - - -def label2onehot(label, num_classes): - """ - label: Tensor(K) - """ - res = torch.zeros(num_classes) - for i in label: - itm = int(i.item()) - res[itm] = 1.0 - return res - - -class MaskCrop(): - def __init__(self) -> None: - pass - - def __call__(self, target, img): - known_box = target['known_box'] - h,w = img.shape[1:] # h,w - # imgsize = target['orig_size'] # h,w - - scale = torch.Tensor([w, h, w, h]) - - # _cnt = 0 - for boxes in known_box: - if boxes.shape[0] == 0: - continue - box_xyxy = box_cxcywh_to_xyxy(boxes) * scale - for box in box_xyxy: - x1, y1, x2, y2 = [int(i) for i in box.tolist()] - img[:, y1:y2, x1:x2] = 0 - # _cnt += 1 - # print("_cnt:", _cnt) - return target, img - - -dataset_hook_register = { - 'label2compat': label2compat, - 'label_compat2onehot': label_compat2onehot, - 'box_label_catter': box_label_catter, - 'RandomSelectBoxlabels': RandomSelectBoxlabels, - 'RandomSelectBoxes': RandomSelectBoxes, - 'MaskCrop': MaskCrop, - 'BboxPertuber': BboxPertuber, -} - - -class CocoDetection(torchvision.datasets.CocoDetection): - def __init__(self, img_folder, ann_file, transforms, return_masks, aux_target_hacks=None): - super(CocoDetection, self).__init__(img_folder, ann_file) - self._transforms = transforms - self.prepare = ConvertCocoPolysToMask(return_masks) - self.aux_target_hacks = aux_target_hacks - - def change_hack_attr(self, hackclassname, attrkv_dict): - target_class = dataset_hook_register[hackclassname] - for item in self.aux_target_hacks: - if isinstance(item, target_class): - for k,v in attrkv_dict.items(): - setattr(item, k, v) - - def get_hack(self, hackclassname): - target_class = dataset_hook_register[hackclassname] - for item in self.aux_target_hacks: - if isinstance(item, target_class): - return item - - def _load_image(self, id: int) -> Image.Image: - path = self.coco.loadImgs(id)[0]["file_name"] - abs_path = os.path.join(self.root, path) - return Image.open(abs_path).convert("RGB") - - def __getitem__(self, idx): - """ - Output: - - target: dict of multiple items - - boxes: Tensor[num_box, 4]. \ - Init type: x0,y0,x1,y1. unnormalized data. - Final type: cx,cy,w,h. normalized data. - """ - try: - img, target = super(CocoDetection, self).__getitem__(idx) - except: - print("Error idx: {}".format(idx)) - idx += 1 - img, target = super(CocoDetection, self).__getitem__(idx) - image_id = self.ids[idx] - target = {'image_id': image_id, 'annotations': target} - img, target = self.prepare(img, target) - - if self._transforms is not None: - img, target = self._transforms(img, target) - - # convert to needed format - if self.aux_target_hacks is not None: - for hack_runner in self.aux_target_hacks: - target, img = hack_runner(target, img=img) - - return img, target - - -def convert_coco_poly_to_mask(segmentations, height, width): - masks = [] - for polygons in segmentations: - rles = coco_mask.frPyObjects(polygons, height, width) - mask = coco_mask.decode(rles) - if len(mask.shape) < 3: - mask = mask[..., None] - mask = torch.as_tensor(mask, dtype=torch.uint8) - mask = mask.any(dim=2) - masks.append(mask) - if masks: - masks = torch.stack(masks, dim=0) - else: - masks = torch.zeros((0, height, width), dtype=torch.uint8) - return masks - - -class ConvertCocoPolysToMask(object): - def __init__(self, return_masks=False): - self.return_masks = return_masks - - def __call__(self, image, target): - w, h = image.size - - image_id = target["image_id"] - image_id = torch.tensor([image_id]) - - anno = target["annotations"] - - anno = [obj for obj in anno if 'iscrowd' not in obj or obj['iscrowd'] == 0] - - boxes = [obj["bbox"] for obj in anno] - # guard against no boxes via resizing - boxes = torch.as_tensor(boxes, dtype=torch.float32).reshape(-1, 4) - boxes[:, 2:] += boxes[:, :2] - boxes[:, 0::2].clamp_(min=0, max=w) - boxes[:, 1::2].clamp_(min=0, max=h) - - classes = [obj["category_id"] for obj in anno] - classes = torch.tensor(classes, dtype=torch.int64) - - if self.return_masks: - segmentations = [obj["segmentation"] for obj in anno] - masks = convert_coco_poly_to_mask(segmentations, h, w) - - keypoints = None - if anno and "keypoints" in anno[0]: - keypoints = [obj["keypoints"] for obj in anno] - keypoints = torch.as_tensor(keypoints, dtype=torch.float32) - num_keypoints = keypoints.shape[0] - if num_keypoints: - keypoints = keypoints.view(num_keypoints, -1, 3) - - keep = (boxes[:, 3] > boxes[:, 1]) & (boxes[:, 2] > boxes[:, 0]) - boxes = boxes[keep] - classes = classes[keep] - if self.return_masks: - masks = masks[keep] - if keypoints is not None: - keypoints = keypoints[keep] - - target = {} - target["boxes"] = boxes - target["labels"] = classes - if self.return_masks: - target["masks"] = masks - target["image_id"] = image_id - if keypoints is not None: - target["keypoints"] = keypoints - - # for conversion to coco api - area = torch.tensor([obj["area"] for obj in anno]) - iscrowd = torch.tensor([obj["iscrowd"] if "iscrowd" in obj else 0 for obj in anno]) - target["area"] = area[keep] - target["iscrowd"] = iscrowd[keep] - - target["orig_size"] = torch.as_tensor([int(h), int(w)]) - target["size"] = torch.as_tensor([int(h), int(w)]) - - return image, target - - -def make_coco_transforms(image_set, fix_size=False, strong_aug=False, args=None): - - normalize = T.Compose([ - T.ToTensor(), - T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) - ]) - - # config the params for data aug - scales = [480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800] - max_size = 1333 - scales2_resize = [400, 500, 600] - scales2_crop = [384, 600] - - # update args from config files - scales = getattr(args, 'data_aug_scales', scales) - max_size = getattr(args, 'data_aug_max_size', max_size) - scales2_resize = getattr(args, 'data_aug_scales2_resize', scales2_resize) - scales2_crop = getattr(args, 'data_aug_scales2_crop', scales2_crop) - - # resize them - data_aug_scale_overlap = getattr(args, 'data_aug_scale_overlap', None) - if data_aug_scale_overlap is not None and data_aug_scale_overlap > 0: - data_aug_scale_overlap = float(data_aug_scale_overlap) - scales = [int(i*data_aug_scale_overlap) for i in scales] - max_size = int(max_size*data_aug_scale_overlap) - scales2_resize = [int(i*data_aug_scale_overlap) for i in scales2_resize] - scales2_crop = [int(i*data_aug_scale_overlap) for i in scales2_crop] - - datadict_for_print = { - 'scales': scales, - 'max_size': max_size, - 'scales2_resize': scales2_resize, - 'scales2_crop': scales2_crop - } - # print("data_aug_params:", json.dumps(datadict_for_print, indent=2)) - - if image_set == 'train': - if fix_size: - return T.Compose([ - T.RandomHorizontalFlip(), - T.RandomResize([(max_size, max(scales))]), - # T.RandomResize([(512, 512)]), - normalize, - ]) - - if strong_aug: - import datasets.sltransform as SLT - - return T.Compose([ - T.RandomHorizontalFlip(), - T.RandomSelect( - T.RandomResize(scales, max_size=max_size), - T.Compose([ - T.RandomResize(scales2_resize), - T.RandomSizeCrop(*scales2_crop), - T.RandomResize(scales, max_size=max_size), - ]) - ), - SLT.RandomSelectMulti([ - SLT.RandomCrop(), - SLT.LightingNoise(), - SLT.AdjustBrightness(2), - SLT.AdjustContrast(2), - ]), - normalize, - ]) - - return T.Compose([ - T.RandomHorizontalFlip(), - T.RandomSelect( - T.RandomResize(scales, max_size=max_size), - T.Compose([ - T.RandomResize(scales2_resize), - T.RandomSizeCrop(*scales2_crop), - T.RandomResize(scales, max_size=max_size), - ]) - ), - normalize, - ]) - - if image_set in ['val', 'eval_debug', 'train_reg', 'test']: - - if os.environ.get("GFLOPS_DEBUG_SHILONG", False) == 'INFO': - print("Under debug mode for flops calculation only!!!!!!!!!!!!!!!!") - return T.Compose([ - T.ResizeDebug((1280, 800)), - normalize, - ]) - - return T.Compose([ - T.RandomResize([max(scales)], max_size=max_size), - normalize, - ]) - - - - raise ValueError(f'unknown {image_set}') - - -def get_aux_target_hacks_list(image_set, args): - if args.modelname in ['q2bs_mask', 'q2bs']: - aux_target_hacks_list = [ - label2compat(), - label_compat2onehot(), - RandomSelectBoxes(num_class=args.num_classes) - ] - if args.masked_data and image_set == 'train': - # aux_target_hacks_list.append() - aux_target_hacks_list.append(MaskCrop()) - elif args.modelname in ['q2bm_v2', 'q2bs_ce', 'q2op', 'q2ofocal', 'q2opclip', 'q2ocqonly']: - aux_target_hacks_list = [ - label2compat(), - label_compat2onehot(), - box_label_catter(), - RandomSelectBoxlabels(num_classes=args.num_classes, - prob_first_item=args.prob_first_item, - prob_random_item=args.prob_random_item, - prob_last_item=args.prob_last_item, - prob_stop_sign=args.prob_stop_sign, - ), - BboxPertuber(max_ratio=0.02, generate_samples=1000), - ] - elif args.modelname in ['q2omask', 'q2osa']: - if args.coco_aug: - aux_target_hacks_list = [ - label2compat(), - label_compat2onehot(), - box_label_catter(), - RandomSelectBoxlabels(num_classes=args.num_classes, - prob_first_item=args.prob_first_item, - prob_random_item=args.prob_random_item, - prob_last_item=args.prob_last_item, - prob_stop_sign=args.prob_stop_sign, - ), - RandomDrop(p=0.2), - BboxPertuber(max_ratio=0.02, generate_samples=1000), - RandomCutout(factor=0.5) - ] - else: - aux_target_hacks_list = [ - label2compat(), - label_compat2onehot(), - box_label_catter(), - RandomSelectBoxlabels(num_classes=args.num_classes, - prob_first_item=args.prob_first_item, - prob_random_item=args.prob_random_item, - prob_last_item=args.prob_last_item, - prob_stop_sign=args.prob_stop_sign, - ), - BboxPertuber(max_ratio=0.02, generate_samples=1000), - ] - else: - aux_target_hacks_list = None - - return aux_target_hacks_list - - -def build(image_set, args, datasetinfo): - img_folder = datasetinfo["root"] - ann_file = datasetinfo["anno"] - - # copy to local path - if os.environ.get('DATA_COPY_SHILONG') == 'INFO': - preparing_dataset(dict(img_folder=img_folder, ann_file=ann_file), image_set, args) - - try: - strong_aug = args.strong_aug - except: - strong_aug = False - print(img_folder, ann_file) - dataset = CocoDetection(img_folder, ann_file, - transforms=make_coco_transforms(image_set, fix_size=args.fix_size, strong_aug=strong_aug, args=args), - return_masks=args.masks, - aux_target_hacks=None, - ) - return dataset - - -if __name__ == "__main__": - # Objects365 Val example - dataset_o365 = CocoDetection( - '/path/Objects365/train/', - "/path/Objects365/slannos/anno_preprocess_train_v2.json", - transforms=None, - return_masks=False, - ) - print('len(dataset_o365):', len(dataset_o365)) diff --git a/groundingdino/datasets/.ipynb_checkpoints/dataset-checkpoint.py b/groundingdino/datasets/.ipynb_checkpoints/dataset-checkpoint.py deleted file mode 100644 index 5655c00ccdbf0b09d51da8dfa3593a2bf9aee527..0000000000000000000000000000000000000000 --- a/groundingdino/datasets/.ipynb_checkpoints/dataset-checkpoint.py +++ /dev/null @@ -1,44 +0,0 @@ -from __future__ import print_function - -import torch -import torchvision.datasets as datasets -from torch.utils.data import Dataset -from PIL import Image -from .tsv_io import TSVFile -import numpy as np -import base64 -import io - - -class TSVDataset(Dataset): - """ TSV dataset for ImageNet 1K training - """ - def __init__(self, tsv_file, transform=None, target_transform=None): - self.tsv = TSVFile(tsv_file) - self.transform = transform - self.target_transform = target_transform - - def __getitem__(self, index): - """ - Args: - index (int): Index - Returns: - tuple: (image, target) where target is class_index of the target class. - """ - row = self.tsv.seek(index) - image_data = base64.b64decode(row[-1]) - image = Image.open(io.BytesIO(image_data)) - image = image.convert('RGB') - target = int(row[1]) - - if self.transform is not None: - img = self.transform(image) - else: - img = image - if self.target_transform is not None: - target = self.target_transform(target) - - return img, target - - def __len__(self): - return self.tsv.num_rows() diff --git a/groundingdino/datasets/.ipynb_checkpoints/odvg-checkpoint.py b/groundingdino/datasets/.ipynb_checkpoints/odvg-checkpoint.py deleted file mode 100644 index e684a65f01f8b65e5cf24dadecfa4f38cc16d873..0000000000000000000000000000000000000000 --- a/groundingdino/datasets/.ipynb_checkpoints/odvg-checkpoint.py +++ /dev/null @@ -1,258 +0,0 @@ -from torchvision.datasets.vision import VisionDataset -import os.path -from typing import Callable, Optional -import json -from PIL import Image -import torch -import random -import os, sys -sys.path.append(os.path.dirname(sys.path[0])) - -import datasets.transforms as T - -class ODVGDataset(VisionDataset): - """ - Args: - root (string): Root directory where images are downloaded to. - anno (string): Path to json annotation file. - label_map_anno (string): Path to json label mapping file. Only for Object Detection - transform (callable, optional): A function/transform that takes in an PIL image - and returns a transformed version. E.g, ``transforms.PILToTensor`` - target_transform (callable, optional): A function/transform that takes in the - target and transforms it. - transforms (callable, optional): A function/transform that takes input sample and its target as entry - and returns a transformed version. - """ - - def __init__( - self, - root: str, - anno: str, - label_map_anno: str = None, - max_labels: int = 80, - transform: Optional[Callable] = None, - target_transform: Optional[Callable] = None, - transforms: Optional[Callable] = None, - ) -> None: - super().__init__(root, transforms, transform, target_transform) - self.root = root - self.dataset_mode = "OD" if label_map_anno else "VG" - self.max_labels = max_labels - if self.dataset_mode == "OD": - self.load_label_map(label_map_anno) - self._load_metas(anno) - self.get_dataset_info() - - def load_label_map(self, label_map_anno): - with open(label_map_anno, 'r') as file: - self.label_map = json.load(file) - self.label_index = set(self.label_map.keys()) - - def _load_metas(self, anno): - with open(anno, 'r') as f: - self.metas = json.load(f) - - - def get_dataset_info(self): - print(f" == total images: {len(self)}") - if self.dataset_mode == "OD": - print(f" == total labels: {len(self.label_map)}") - - def __getitem__(self, index: int): - meta = self.metas[index] - rel_path = meta["filename"] - abs_path = os.path.join(self.root, rel_path) - if not os.path.exists(abs_path): - raise FileNotFoundError(f"{abs_path} not found.") - image = Image.open(abs_path).convert('RGB') - w, h = image.size - if self.dataset_mode == "OD": - anno = meta["detection"] - instances = [obj for obj in anno["instances"]] - boxes = [obj["bbox"] for obj in instances] - # generate vg_labels - # pos bbox labels - ori_classes = [str(obj["label"]) for obj in instances] - pos_labels = set(ori_classes) - # neg bbox labels - neg_labels = self.label_index.difference(pos_labels) - - vg_labels = list(pos_labels) - num_to_add = min(len(neg_labels), self.max_labels-len(pos_labels)) - if num_to_add > 0: - vg_labels.extend(random.sample(neg_labels, num_to_add)) - - # shuffle - for i in range(len(vg_labels)-1, 0, -1): - j = random.randint(0, i) - vg_labels[i], vg_labels[j] = vg_labels[j], vg_labels[i] - - caption_list = [self.label_map[lb] for lb in vg_labels] - caption_dict = {item:index for index, item in enumerate(caption_list)} - - caption = ' . '.join(caption_list) + ' .' - classes = [caption_dict[self.label_map[str(obj["label"])]] for obj in instances] - boxes = torch.as_tensor(boxes, dtype=torch.float32).reshape(-1, 4) - classes = torch.tensor(classes, dtype=torch.int64) - elif self.dataset_mode == "VG": - anno = meta["Grounding"] - instances = [obj for obj in anno["regions"]] - boxes = [obj["bbox"] for obj in instances] - caption_list = [obj["phrase"] for obj in instances] - c = list(zip(boxes, caption_list)) - random.shuffle(c) - boxes[:], caption_list[:] = zip(*c) - uni_caption_list = list(set(caption_list)) - label_map = {} - for idx in range(len(uni_caption_list)): - label_map[uni_caption_list[idx]] = idx - classes = [label_map[cap] for cap in caption_list] - caption = ' . '.join(uni_caption_list) + ' .' - boxes = torch.as_tensor(boxes, dtype=torch.float32).reshape(-1, 4) - classes = torch.tensor(classes, dtype=torch.int64) - caption_list = uni_caption_list - # print("caption_list" , caption_list) - # print("caption" , caption) - # print("boxes" , boxes) - target = {} - target["image_id"] = rel_path.strip(".jpg") - target["size"] = torch.as_tensor([int(h), int(w)]) - target["cap_list"] = caption_list - target["caption"] = caption - target["boxes"] = boxes - target["labels"] = classes - # print(" image_id " , target["image_id"]) - # size, cap_list, caption, bboxes, labels - - if self.transforms is not None: - image, target = self.transforms(image, target) - - return image, target - - - def __len__(self) -> int: - return len(self.metas) - - -def make_coco_transforms(image_set, fix_size=False, strong_aug=False, args=None): - - normalize = T.Compose([ - T.ToTensor(), - T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) - ]) - - # config the params for data aug - scales = [480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800] - max_size = 1333 - scales2_resize = [400, 500, 600] - scales2_crop = [384, 600] - - # update args from config files - scales = getattr(args, 'data_aug_scales', scales) - max_size = getattr(args, 'data_aug_max_size', max_size) - scales2_resize = getattr(args, 'data_aug_scales2_resize', scales2_resize) - scales2_crop = getattr(args, 'data_aug_scales2_crop', scales2_crop) - - # resize them - data_aug_scale_overlap = getattr(args, 'data_aug_scale_overlap', None) - if data_aug_scale_overlap is not None and data_aug_scale_overlap > 0: - data_aug_scale_overlap = float(data_aug_scale_overlap) - scales = [int(i*data_aug_scale_overlap) for i in scales] - max_size = int(max_size*data_aug_scale_overlap) - scales2_resize = [int(i*data_aug_scale_overlap) for i in scales2_resize] - scales2_crop = [int(i*data_aug_scale_overlap) for i in scales2_crop] - - # datadict_for_print = { - # 'scales': scales, - # 'max_size': max_size, - # 'scales2_resize': scales2_resize, - # 'scales2_crop': scales2_crop - # } - # print("data_aug_params:", json.dumps(datadict_for_print, indent=2)) - - if image_set == 'train': - if fix_size: - return T.Compose([ - T.RandomHorizontalFlip(), - T.RandomResize([(max_size, max(scales))]), - normalize, - ]) - - if strong_aug: - import datasets.sltransform as SLT - - return T.Compose([ - T.RandomHorizontalFlip(), - T.RandomSelect( - T.RandomResize(scales, max_size=max_size), - T.Compose([ - T.RandomResize(scales2_resize), - T.RandomSizeCrop(*scales2_crop), - T.RandomResize(scales, max_size=max_size), - ]) - ), - SLT.RandomSelectMulti([ - SLT.RandomCrop(), - SLT.LightingNoise(), - SLT.AdjustBrightness(2), - SLT.AdjustContrast(2), - ]), - normalize, - ]) - - return T.Compose([ - T.RandomHorizontalFlip(), - T.RandomSelect( - T.RandomResize(scales, max_size=max_size), - T.Compose([ - T.RandomResize(scales2_resize), - T.RandomSizeCrop(*scales2_crop), - T.RandomResize(scales, max_size=max_size), - ]) - ), - normalize, - ]) - - if image_set in ['val', 'eval_debug', 'train_reg', 'test']: - - if os.environ.get("GFLOPS_DEBUG_SHILONG", False) == 'INFO': - print("Under debug mode for flops calculation only!!!!!!!!!!!!!!!!") - return T.Compose([ - T.ResizeDebug((1280, 800)), - normalize, - ]) - - return T.Compose([ - T.RandomResize([max(scales)], max_size=max_size), - normalize, - ]) - - raise ValueError(f'unknown {image_set}') - -def build_odvg(image_set, args, datasetinfo): - img_folder = datasetinfo["root"] - ann_file = datasetinfo["anno"] - label_map = datasetinfo["label_map"] if "label_map" in datasetinfo else None - try: - strong_aug = args.strong_aug - except: - strong_aug = False # False originally - print(img_folder, ann_file, label_map) - dataset = ODVGDataset(img_folder, ann_file, label_map, max_labels=args.max_labels, - transforms=make_coco_transforms(image_set, fix_size=args.fix_size, strong_aug=strong_aug, args=args), - ) - return dataset - - -if __name__=="__main__": - dataset_vg = ODVGDataset("path/GRIT-20M/data/","path/GRIT-20M/anno/grit_odvg_10k.jsonl",) - print(len(dataset_vg)) - data = dataset_vg[random.randint(0, 100)] - print(data) - dataset_od = ODVGDataset("pathl/V3Det/", - "path/V3Det/annotations/v3det_2023_v1_all_odvg.jsonl", - "path/V3Det/annotations/v3det_label_map.json", - ) - print(len(dataset_od)) - data = dataset_od[random.randint(0, 100)] - print(data) \ No newline at end of file diff --git a/groundingdino/datasets/.ipynb_checkpoints/transforms-checkpoint.py b/groundingdino/datasets/.ipynb_checkpoints/transforms-checkpoint.py deleted file mode 100644 index ec3691e7140847167461998e2d2d62c70f5cde56..0000000000000000000000000000000000000000 --- a/groundingdino/datasets/.ipynb_checkpoints/transforms-checkpoint.py +++ /dev/null @@ -1,285 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -""" -Transforms and data augmentation for both image + bbox. -""" -import random - -import PIL -import torch -import torchvision.transforms as T -import torchvision.transforms.functional as F - -from util.box_ops import box_xyxy_to_cxcywh -from util.misc import interpolate - - -def crop(image, target, region): - cropped_image = F.crop(image, *region) - - target = target.copy() - i, j, h, w = region - - # should we do something wrt the original size? - target["size"] = torch.tensor([h, w]) - - fields = ["labels", "area"] - - if "boxes" in target: - boxes = target["boxes"] - max_size = torch.as_tensor([w, h], dtype=torch.float32) - cropped_boxes = boxes - torch.as_tensor([j, i, j, i]) - cropped_boxes = torch.min(cropped_boxes.reshape(-1, 2, 2), max_size) - cropped_boxes = cropped_boxes.clamp(min=0) - area = (cropped_boxes[:, 1, :] - cropped_boxes[:, 0, :]).prod(dim=1) - target["boxes"] = cropped_boxes.reshape(-1, 4) - target["area"] = area - fields.append("boxes") - - if "masks" in target: - # FIXME should we update the area here if there are no boxes? - target['masks'] = target['masks'][:, i:i + h, j:j + w] - fields.append("masks") - - - # remove elements for which the boxes or masks that have zero area - if "boxes" in target or "masks" in target: - # favor boxes selection when defining which elements to keep - # this is compatible with previous implementation - if "boxes" in target: - cropped_boxes = target['boxes'].reshape(-1, 2, 2) - keep = torch.all(cropped_boxes[:, 1, :] > cropped_boxes[:, 0, :], dim=1) - else: - keep = target['masks'].flatten(1).any(1) - - for field in fields: - target[field] = target[field][keep] - - return cropped_image, target - - -def hflip(image, target): - flipped_image = F.hflip(image) - - w, h = image.size - - target = target.copy() - if "boxes" in target: - boxes = target["boxes"] - boxes = boxes[:, [2, 1, 0, 3]] * torch.as_tensor([-1, 1, -1, 1]) + torch.as_tensor([w, 0, w, 0]) - target["boxes"] = boxes - - if "masks" in target: - target['masks'] = target['masks'].flip(-1) - - return flipped_image, target - - -def resize(image, target, size, max_size=None): - # size can be min_size (scalar) or (w, h) tuple - - def get_size_with_aspect_ratio(image_size, size, max_size=None): - w, h = image_size - if max_size is not None: - min_original_size = float(min((w, h))) - max_original_size = float(max((w, h))) - if max_original_size / min_original_size * size > max_size: - size = int(round(max_size * min_original_size / max_original_size)) - - if (w <= h and w == size) or (h <= w and h == size): - return (h, w) - - if w < h: - ow = size - oh = int(size * h / w) - else: - oh = size - ow = int(size * w / h) - - return (oh, ow) - - def get_size(image_size, size, max_size=None): - if isinstance(size, (list, tuple)): - return size[::-1] - else: - return get_size_with_aspect_ratio(image_size, size, max_size) - - size = get_size(image.size, size, max_size) - rescaled_image = F.resize(image, size) - - if target is None: - return rescaled_image, None - - ratios = tuple(float(s) / float(s_orig) for s, s_orig in zip(rescaled_image.size, image.size)) - ratio_width, ratio_height = ratios - - target = target.copy() - if "boxes" in target: - boxes = target["boxes"] - scaled_boxes = boxes * torch.as_tensor([ratio_width, ratio_height, ratio_width, ratio_height]) - target["boxes"] = scaled_boxes - - if "area" in target: - area = target["area"] - scaled_area = area * (ratio_width * ratio_height) - target["area"] = scaled_area - - h, w = size - target["size"] = torch.tensor([h, w]) - - if "masks" in target: - target['masks'] = interpolate( - target['masks'][:, None].float(), size, mode="nearest")[:, 0] > 0.5 - - return rescaled_image, target - - -def pad(image, target, padding): - # assumes that we only pad on the bottom right corners - padded_image = F.pad(image, (0, 0, padding[0], padding[1])) - if target is None: - return padded_image, None - target = target.copy() - # should we do something wrt the original size? - target["size"] = torch.tensor(padded_image.size[::-1]) - if "masks" in target: - target['masks'] = torch.nn.functional.pad(target['masks'], (0, padding[0], 0, padding[1])) - return padded_image, target - - -class ResizeDebug(object): - def __init__(self, size): - self.size = size - - def __call__(self, img, target): - return resize(img, target, self.size) - - -class RandomCrop(object): - def __init__(self, size): - self.size = size - - def __call__(self, img, target): - region = T.RandomCrop.get_params(img, self.size) - return crop(img, target, region) - - -class RandomSizeCrop(object): - def __init__(self, min_size: int, max_size: int): - self.min_size = min_size - self.max_size = max_size - - def __call__(self, img: PIL.Image.Image, target: dict): - w = random.randint(self.min_size, min(img.width, self.max_size)) - h = random.randint(self.min_size, min(img.height, self.max_size)) - region = T.RandomCrop.get_params(img, [h, w]) - return crop(img, target, region) - - -class CenterCrop(object): - def __init__(self, size): - self.size = size - - def __call__(self, img, target): - image_width, image_height = img.size - crop_height, crop_width = self.size - crop_top = int(round((image_height - crop_height) / 2.)) - crop_left = int(round((image_width - crop_width) / 2.)) - return crop(img, target, (crop_top, crop_left, crop_height, crop_width)) - - -class RandomHorizontalFlip(object): - def __init__(self, p=0.5): - self.p = p - - def __call__(self, img, target): - if random.random() < self.p: - return hflip(img, target) - return img, target - - -class RandomResize(object): - def __init__(self, sizes, max_size=None): - assert isinstance(sizes, (list, tuple)) - self.sizes = sizes - self.max_size = max_size - - def __call__(self, img, target=None): - size = random.choice(self.sizes) - return resize(img, target, size, self.max_size) - - -class RandomPad(object): - def __init__(self, max_pad): - self.max_pad = max_pad - - def __call__(self, img, target): - pad_x = random.randint(0, self.max_pad) - pad_y = random.randint(0, self.max_pad) - return pad(img, target, (pad_x, pad_y)) - - -class RandomSelect(object): - """ - Randomly selects between transforms1 and transforms2, - with probability p for transforms1 and (1 - p) for transforms2 - """ - def __init__(self, transforms1, transforms2, p=0.5): - self.transforms1 = transforms1 - self.transforms2 = transforms2 - self.p = p - - def __call__(self, img, target): - if random.random() < self.p: - return self.transforms1(img, target) - return self.transforms2(img, target) - - -class ToTensor(object): - def __call__(self, img, target): - return F.to_tensor(img), target - - -class RandomErasing(object): - - def __init__(self, *args, **kwargs): - self.eraser = T.RandomErasing(*args, **kwargs) - - def __call__(self, img, target): - return self.eraser(img), target - - -class Normalize(object): - def __init__(self, mean, std): - self.mean = mean - self.std = std - - def __call__(self, image, target=None): - image = F.normalize(image, mean=self.mean, std=self.std) - if target is None: - return image, None - target = target.copy() - h, w = image.shape[-2:] - if "boxes" in target: - boxes = target["boxes"] - boxes = box_xyxy_to_cxcywh(boxes) - boxes = boxes / torch.tensor([w, h, w, h], dtype=torch.float32) - target["boxes"] = boxes - return image, target - - -class Compose(object): - def __init__(self, transforms): - self.transforms = transforms - - def __call__(self, image, target): - for t in self.transforms: - image, target = t(image, target) - return image, target - - def __repr__(self): - format_string = self.__class__.__name__ + "(" - for t in self.transforms: - format_string += "\n" - format_string += " {0}".format(t) - format_string += "\n)" - return format_string diff --git a/groundingdino/datasets/__init__.py b/groundingdino/datasets/__init__.py deleted file mode 100644 index b1864e13e91e60944a78de5b7903228cce165cdb..0000000000000000000000000000000000000000 --- a/groundingdino/datasets/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -import torch.utils.data -import torchvision -from .coco import build as build_coco - - -def get_coco_api_from_dataset(dataset): - for _ in range(10): - # if isinstance(dataset, torchvision.datasets.CocoDetection): - # break - if isinstance(dataset, torch.utils.data.Subset): - dataset = dataset.dataset - if isinstance(dataset, torchvision.datasets.CocoDetection): - return dataset.coco - - -def build_dataset(image_set, args, datasetinfo): - if datasetinfo["dataset_mode"] == 'coco': - return build_coco(image_set, args, datasetinfo) - if datasetinfo["dataset_mode"] == 'odvg': - from .odvg import build_odvg - return build_odvg(image_set, args, datasetinfo) - raise ValueError(f'dataset {args.dataset_file} not supported') diff --git a/groundingdino/datasets/__pycache__/__init__.cpython-310.pyc b/groundingdino/datasets/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 4e04470ff23c98ded5470d35072aa81cc2f2b174..0000000000000000000000000000000000000000 Binary files a/groundingdino/datasets/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/datasets/__pycache__/coco.cpython-310.pyc b/groundingdino/datasets/__pycache__/coco.cpython-310.pyc deleted file mode 100644 index 88e1ce7264e09e3a919206df6182baae7e893caa..0000000000000000000000000000000000000000 Binary files a/groundingdino/datasets/__pycache__/coco.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/datasets/__pycache__/coco_eval.cpython-310.pyc b/groundingdino/datasets/__pycache__/coco_eval.cpython-310.pyc deleted file mode 100644 index 72d86d698293b1a22f9b275eef33682605d269f3..0000000000000000000000000000000000000000 Binary files a/groundingdino/datasets/__pycache__/coco_eval.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/datasets/__pycache__/cocogrounding_eval.cpython-310.pyc b/groundingdino/datasets/__pycache__/cocogrounding_eval.cpython-310.pyc deleted file mode 100644 index c66ce7104379793dd55679ad19b21e658486b929..0000000000000000000000000000000000000000 Binary files a/groundingdino/datasets/__pycache__/cocogrounding_eval.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/datasets/__pycache__/data_util.cpython-310.pyc b/groundingdino/datasets/__pycache__/data_util.cpython-310.pyc deleted file mode 100644 index f678a262cb092d2098bc16e678c9e3f0e867151c..0000000000000000000000000000000000000000 Binary files a/groundingdino/datasets/__pycache__/data_util.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/datasets/__pycache__/odvg.cpython-310.pyc b/groundingdino/datasets/__pycache__/odvg.cpython-310.pyc deleted file mode 100644 index c342a7531c3db95f6fd89d441ba72175db9a999f..0000000000000000000000000000000000000000 Binary files a/groundingdino/datasets/__pycache__/odvg.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/datasets/__pycache__/panoptic_eval.cpython-310.pyc b/groundingdino/datasets/__pycache__/panoptic_eval.cpython-310.pyc deleted file mode 100644 index 997c52d82d6d037d850988e186ded2169e6e85ee..0000000000000000000000000000000000000000 Binary files a/groundingdino/datasets/__pycache__/panoptic_eval.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/datasets/__pycache__/random_crop.cpython-310.pyc b/groundingdino/datasets/__pycache__/random_crop.cpython-310.pyc deleted file mode 100644 index dfedd02272360d7d08dfa1b1e983773bcae3906e..0000000000000000000000000000000000000000 Binary files a/groundingdino/datasets/__pycache__/random_crop.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/datasets/__pycache__/sltransform.cpython-310.pyc b/groundingdino/datasets/__pycache__/sltransform.cpython-310.pyc deleted file mode 100644 index 7bdb86094107689d8e6b85eb6a4a49b423190ec4..0000000000000000000000000000000000000000 Binary files a/groundingdino/datasets/__pycache__/sltransform.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/datasets/__pycache__/transforms.cpython-310.pyc b/groundingdino/datasets/__pycache__/transforms.cpython-310.pyc deleted file mode 100644 index 42c5413eafe979a15e4ebf32033f43c1ada5d1d2..0000000000000000000000000000000000000000 Binary files a/groundingdino/datasets/__pycache__/transforms.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/datasets/coco.py b/groundingdino/datasets/coco.py deleted file mode 100644 index ae9acf582138bbc22145828c59f96dc17b998749..0000000000000000000000000000000000000000 --- a/groundingdino/datasets/coco.py +++ /dev/null @@ -1,649 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -""" -COCO dataset which returns image_id for evaluation. - -Mostly copy-paste from https://github.com/pytorch/vision/blob/13b35ff/references/detection/coco_utils.py -""" -if __name__=="__main__": - # for debug only - import os, sys - sys.path.append(os.path.dirname(sys.path[0])) -from torchvision.datasets.vision import VisionDataset - -import json -from pathlib import Path -import random -import os -from typing import Any, Callable, List, Optional, Tuple - -from PIL import Image - -import torch -import torch.utils.data -import torchvision -from pycocotools import mask as coco_mask - -from datasets.data_util import preparing_dataset -import datasets.transforms as T -from util.box_ops import box_cxcywh_to_xyxy, box_iou - -__all__ = ['build'] - - -class label2compat(): - def __init__(self) -> None: - self.category_map_str = {"1": 1, "2": 2, "3": 3, "4": 4, "5": 5, "6": 6, "7": 7, "8": 8, "9": 9, "10": 10, "11": 11, "13": 12, "14": 13, "15": 14, "16": 15, "17": 16, "18": 17, "19": 18, "20": 19, "21": 20, "22": 21, "23": 22, "24": 23, "25": 24, "27": 25, "28": 26, "31": 27, "32": 28, "33": 29, "34": 30, "35": 31, "36": 32, "37": 33, "38": 34, "39": 35, "40": 36, "41": 37, "42": 38, "43": 39, "44": 40, "46": 41, "47": 42, "48": 43, "49": 44, "50": 45, "51": 46, "52": 47, "53": 48, "54": 49, "55": 50, "56": 51, "57": 52, "58": 53, "59": 54, "60": 55, "61": 56, "62": 57, "63": 58, "64": 59, "65": 60, "67": 61, "70": 62, "72": 63, "73": 64, "74": 65, "75": 66, "76": 67, "77": 68, "78": 69, "79": 70, "80": 71, "81": 72, "82": 73, "84": 74, "85": 75, "86": 76, "87": 77, "88": 78, "89": 79, "90": 80} - self.category_map = {int(k):v for k,v in self.category_map_str.items()} - - def __call__(self, target, img=None): - labels = target['labels'] - res = torch.zeros(labels.shape, dtype=labels.dtype) - for idx, item in enumerate(labels): - res[idx] = self.category_map[item.item()] - 1 - target['label_compat'] = res - if img is not None: - return target, img - else: - return target - - -class label_compat2onehot(): - def __init__(self, num_class=80, num_output_objs=1): - self.num_class = num_class - self.num_output_objs = num_output_objs - if num_output_objs != 1: - raise DeprecationWarning("num_output_objs!=1, which is only used for comparison") - - def __call__(self, target, img=None): - labels = target['label_compat'] - place_dict = {k:0 for k in range(self.num_class)} - if self.num_output_objs == 1: - res = torch.zeros(self.num_class) - for i in labels: - itm = i.item() - res[itm] = 1.0 - else: - # compat with baseline - res = torch.zeros(self.num_class, self.num_output_objs) - for i in labels: - itm = i.item() - res[itm][place_dict[itm]] = 1.0 - place_dict[itm] += 1 - target['label_compat_onehot'] = res - if img is not None: - return target, img - else: - return target - - -class box_label_catter(): - def __init__(self): - pass - - def __call__(self, target, img=None): - labels = target['label_compat'] - boxes = target['boxes'] - box_label = torch.cat((boxes, labels.unsqueeze(-1)), 1) - target['box_label'] = box_label - if img is not None: - return target, img - else: - return target - - -class RandomSelectBoxlabels(): - def __init__(self, num_classes, leave_one_out=False, blank_prob=0.8, - prob_first_item = 0.0, - prob_random_item = 0.0, - prob_last_item = 0.8, - prob_stop_sign = 0.2 - ) -> None: - self.num_classes = num_classes - self.leave_one_out = leave_one_out - self.blank_prob = blank_prob - - self.set_state(prob_first_item, prob_random_item, prob_last_item, prob_stop_sign) - - - def get_state(self): - return [self.prob_first_item, self.prob_random_item, self.prob_last_item, self.prob_stop_sign] - - def set_state(self, prob_first_item, prob_random_item, prob_last_item, prob_stop_sign): - sum_prob = prob_first_item + prob_random_item + prob_last_item + prob_stop_sign - assert sum_prob - 1 < 1e-6, \ - f"Sum up all prob = {sum_prob}. prob_first_item:{prob_first_item}" \ - + f"prob_random_item:{prob_random_item}, prob_last_item:{prob_last_item}" \ - + f"prob_stop_sign:{prob_stop_sign}" - - self.prob_first_item = prob_first_item - self.prob_random_item = prob_random_item - self.prob_last_item = prob_last_item - self.prob_stop_sign = prob_stop_sign - - - def sample_for_pred_first_item(self, box_label: torch.FloatTensor): - box_label_known = torch.Tensor(0,5) - box_label_unknown = box_label - return box_label_known, box_label_unknown - - def sample_for_pred_random_item(self, box_label: torch.FloatTensor): - n_select = int(random.random() * box_label.shape[0]) - box_label = box_label[torch.randperm(box_label.shape[0])] - box_label_known = box_label[:n_select] - box_label_unknown = box_label[n_select:] - return box_label_known, box_label_unknown - - def sample_for_pred_last_item(self, box_label: torch.FloatTensor): - box_label_perm = box_label[torch.randperm(box_label.shape[0])] - known_label_list = [] - box_label_known = [] - box_label_unknown = [] - for item in box_label_perm: - label_i = item[4].item() - if label_i in known_label_list: - box_label_known.append(item) - else: - # first item - box_label_unknown.append(item) - known_label_list.append(label_i) - box_label_known = torch.stack(box_label_known) if len(box_label_known) > 0 else torch.Tensor(0,5) - box_label_unknown = torch.stack(box_label_unknown) if len(box_label_unknown) > 0 else torch.Tensor(0,5) - return box_label_known, box_label_unknown - - def sample_for_pred_stop_sign(self, box_label: torch.FloatTensor): - box_label_unknown = torch.Tensor(0,5) - box_label_known = box_label - return box_label_known, box_label_unknown - - def __call__(self, target, img=None): - box_label = target['box_label'] # K, 5 - - dice_number = random.random() - - if dice_number < self.prob_first_item: - box_label_known, box_label_unknown = self.sample_for_pred_first_item(box_label) - elif dice_number < self.prob_first_item + self.prob_random_item: - box_label_known, box_label_unknown = self.sample_for_pred_random_item(box_label) - elif dice_number < self.prob_first_item + self.prob_random_item + self.prob_last_item: - box_label_known, box_label_unknown = self.sample_for_pred_last_item(box_label) - else: - box_label_known, box_label_unknown = self.sample_for_pred_stop_sign(box_label) - - target['label_onehot_known'] = label2onehot(box_label_known[:,-1], self.num_classes) - target['label_onehot_unknown'] = label2onehot(box_label_unknown[:, -1], self.num_classes) - target['box_label_known'] = box_label_known - target['box_label_unknown'] = box_label_unknown - - return target, img - - -class RandomDrop(): - def __init__(self, p=0.2) -> None: - self.p = p - - def __call__(self, target, img=None): - known_box = target['box_label_known'] - num_known_box = known_box.size(0) - idxs = torch.rand(num_known_box) - # indices = torch.randperm(num_known_box)[:int((1-self).p*num_known_box + 0.5 + random.random())] - target['box_label_known'] = known_box[idxs > self.p] - return target, img - - -class BboxPertuber(): - def __init__(self, max_ratio = 0.02, generate_samples = 1000) -> None: - self.max_ratio = max_ratio - self.generate_samples = generate_samples - self.samples = self.generate_pertube_samples() - self.idx = 0 - - def generate_pertube_samples(self): - import torch - samples = (torch.rand(self.generate_samples, 5) - 0.5) * 2 * self.max_ratio - return samples - - def __call__(self, target, img): - known_box = target['box_label_known'] # Tensor(K,5), K known bbox - K = known_box.shape[0] - known_box_pertube = torch.zeros(K, 6) # 4:bbox, 1:prob, 1:label - if K == 0: - pass - else: - if self.idx + K > self.generate_samples: - self.idx = 0 - delta = self.samples[self.idx: self.idx + K, :] - known_box_pertube[:, :4] = known_box[:, :4] + delta[:, :4] - iou = (torch.diag(box_iou(box_cxcywh_to_xyxy(known_box[:, :4]), box_cxcywh_to_xyxy(known_box_pertube[:, :4]))[0])) * (1 + delta[:, -1]) - known_box_pertube[:, 4].copy_(iou) - known_box_pertube[:, -1].copy_(known_box[:, -1]) - - target['box_label_known_pertube'] = known_box_pertube - return target, img - - -class RandomCutout(): - def __init__(self, factor=0.5) -> None: - self.factor = factor - - def __call__(self, target, img=None): - unknown_box = target['box_label_unknown'] # Ku, 5 - known_box = target['box_label_known_pertube'] # Kk, 6 - Ku = unknown_box.size(0) - - known_box_add = torch.zeros(Ku, 6) # Ku, 6 - known_box_add[:, :5] = unknown_box - known_box_add[:, 5].uniform_(0.5, 1) - - - known_box_add[:, :2] += known_box_add[:, 2:4] * (torch.rand(Ku, 2) - 0.5) / 2 - known_box_add[:, 2:4] /= 2 - - target['box_label_known_pertube'] = torch.cat((known_box, known_box_add)) - return target, img - - -class RandomSelectBoxes(): - def __init__(self, num_class=80) -> None: - Warning("This is such a slow function and will be deprecated soon!!!") - self.num_class = num_class - - def __call__(self, target, img=None): - boxes = target['boxes'] - labels = target['label_compat'] - - # transform to list of tensors - boxs_list = [[] for i in range(self.num_class)] - for idx, item in enumerate(boxes): - label = labels[idx].item() - boxs_list[label].append(item) - boxs_list_tensor = [torch.stack(i) if len(i) > 0 else torch.Tensor(0,4) for i in boxs_list] - - # random selection - box_known = [] - box_unknown = [] - for idx, item in enumerate(boxs_list_tensor): - ncnt = item.shape[0] - nselect = int(random.random() * ncnt) # close in both sides, much faster than random.randint - - item = item[torch.randperm(ncnt)] - # random.shuffle(item) - box_known.append(item[:nselect]) - box_unknown.append(item[nselect:]) - - # box_known_tensor = [torch.stack(i) if len(i) > 0 else torch.Tensor(0,4) for i in box_known] - # box_unknown_tensor = [torch.stack(i) if len(i) > 0 else torch.Tensor(0,4) for i in box_unknown] - # print('box_unknown_tensor:', box_unknown_tensor) - target['known_box'] = box_known - target['unknown_box'] = box_unknown - return target, img - - -def label2onehot(label, num_classes): - """ - label: Tensor(K) - """ - res = torch.zeros(num_classes) - for i in label: - itm = int(i.item()) - res[itm] = 1.0 - return res - - -class MaskCrop(): - def __init__(self) -> None: - pass - - def __call__(self, target, img): - known_box = target['known_box'] - h,w = img.shape[1:] # h,w - # imgsize = target['orig_size'] # h,w - - scale = torch.Tensor([w, h, w, h]) - - # _cnt = 0 - for boxes in known_box: - if boxes.shape[0] == 0: - continue - box_xyxy = box_cxcywh_to_xyxy(boxes) * scale - for box in box_xyxy: - x1, y1, x2, y2 = [int(i) for i in box.tolist()] - img[:, y1:y2, x1:x2] = 0 - # _cnt += 1 - # print("_cnt:", _cnt) - return target, img - - -dataset_hook_register = { - 'label2compat': label2compat, - 'label_compat2onehot': label_compat2onehot, - 'box_label_catter': box_label_catter, - 'RandomSelectBoxlabels': RandomSelectBoxlabels, - 'RandomSelectBoxes': RandomSelectBoxes, - 'MaskCrop': MaskCrop, - 'BboxPertuber': BboxPertuber, -} - - -class CocoDetection(torchvision.datasets.CocoDetection): - def __init__(self, img_folder, ann_file, transforms, return_masks, aux_target_hacks=None): - super(CocoDetection, self).__init__(img_folder, ann_file) - self._transforms = transforms - self.prepare = ConvertCocoPolysToMask(return_masks) - self.aux_target_hacks = aux_target_hacks - - def change_hack_attr(self, hackclassname, attrkv_dict): - target_class = dataset_hook_register[hackclassname] - for item in self.aux_target_hacks: - if isinstance(item, target_class): - for k,v in attrkv_dict.items(): - setattr(item, k, v) - - def get_hack(self, hackclassname): - target_class = dataset_hook_register[hackclassname] - for item in self.aux_target_hacks: - if isinstance(item, target_class): - return item - - def _load_image(self, id: int) -> Image.Image: - path = self.coco.loadImgs(id)[0]["file_name"] - abs_path = os.path.join(self.root, path) - return Image.open(abs_path).convert("RGB") - - def __getitem__(self, idx): - """ - Output: - - target: dict of multiple items - - boxes: Tensor[num_box, 4]. \ - Init type: x0,y0,x1,y1. unnormalized data. - Final type: cx,cy,w,h. normalized data. - """ - try: - img, target = super(CocoDetection, self).__getitem__(idx) - except: - print("Error idx: {}".format(idx)) - idx += 1 - img, target = super(CocoDetection, self).__getitem__(idx) - image_id = self.ids[idx] - target = {'image_id': image_id, 'annotations': target} - img, target = self.prepare(img, target) - - if self._transforms is not None: - img, target = self._transforms(img, target) - - # convert to needed format - if self.aux_target_hacks is not None: - for hack_runner in self.aux_target_hacks: - target, img = hack_runner(target, img=img) - - return img, target - - -def convert_coco_poly_to_mask(segmentations, height, width): - masks = [] - for polygons in segmentations: - rles = coco_mask.frPyObjects(polygons, height, width) - mask = coco_mask.decode(rles) - if len(mask.shape) < 3: - mask = mask[..., None] - mask = torch.as_tensor(mask, dtype=torch.uint8) - mask = mask.any(dim=2) - masks.append(mask) - if masks: - masks = torch.stack(masks, dim=0) - else: - masks = torch.zeros((0, height, width), dtype=torch.uint8) - return masks - - -class ConvertCocoPolysToMask(object): - def __init__(self, return_masks=False): - self.return_masks = return_masks - - def __call__(self, image, target): - w, h = image.size - - image_id = target["image_id"] - image_id = torch.tensor([image_id]) - - anno = target["annotations"] - - anno = [obj for obj in anno if 'iscrowd' not in obj or obj['iscrowd'] == 0] - - boxes = [obj["bbox"] for obj in anno] - # guard against no boxes via resizing - boxes = torch.as_tensor(boxes, dtype=torch.float32).reshape(-1, 4) - boxes[:, 2:] += boxes[:, :2] - boxes[:, 0::2].clamp_(min=0, max=w) - boxes[:, 1::2].clamp_(min=0, max=h) - - classes = [obj["category_id"] for obj in anno] - classes = torch.tensor(classes, dtype=torch.int64) - - if self.return_masks: - segmentations = [obj["segmentation"] for obj in anno] - masks = convert_coco_poly_to_mask(segmentations, h, w) - - keypoints = None - if anno and "keypoints" in anno[0]: - keypoints = [obj["keypoints"] for obj in anno] - keypoints = torch.as_tensor(keypoints, dtype=torch.float32) - num_keypoints = keypoints.shape[0] - if num_keypoints: - keypoints = keypoints.view(num_keypoints, -1, 3) - - keep = (boxes[:, 3] > boxes[:, 1]) & (boxes[:, 2] > boxes[:, 0]) - boxes = boxes[keep] - classes = classes[keep] - if self.return_masks: - masks = masks[keep] - if keypoints is not None: - keypoints = keypoints[keep] - - target = {} - target["boxes"] = boxes - target["labels"] = classes - if self.return_masks: - target["masks"] = masks - target["image_id"] = image_id - if keypoints is not None: - target["keypoints"] = keypoints - - # for conversion to coco api - area = torch.tensor([obj["area"] for obj in anno]) - iscrowd = torch.tensor([obj["iscrowd"] if "iscrowd" in obj else 0 for obj in anno]) - target["area"] = area[keep] - target["iscrowd"] = iscrowd[keep] - - target["orig_size"] = torch.as_tensor([int(h), int(w)]) - target["size"] = torch.as_tensor([int(h), int(w)]) - - return image, target - - -def make_coco_transforms(image_set, fix_size=False, strong_aug=False, args=None): - - normalize = T.Compose([ - T.ToTensor(), - T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) - ]) - - # config the params for data aug - scales = [480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800] - max_size = 1333 - scales2_resize = [400, 500, 600] - scales2_crop = [384, 600] - - # update args from config files - scales = getattr(args, 'data_aug_scales', scales) - max_size = getattr(args, 'data_aug_max_size', max_size) - scales2_resize = getattr(args, 'data_aug_scales2_resize', scales2_resize) - scales2_crop = getattr(args, 'data_aug_scales2_crop', scales2_crop) - - # resize them - data_aug_scale_overlap = getattr(args, 'data_aug_scale_overlap', None) - if data_aug_scale_overlap is not None and data_aug_scale_overlap > 0: - data_aug_scale_overlap = float(data_aug_scale_overlap) - scales = [int(i*data_aug_scale_overlap) for i in scales] - max_size = int(max_size*data_aug_scale_overlap) - scales2_resize = [int(i*data_aug_scale_overlap) for i in scales2_resize] - scales2_crop = [int(i*data_aug_scale_overlap) for i in scales2_crop] - - datadict_for_print = { - 'scales': scales, - 'max_size': max_size, - 'scales2_resize': scales2_resize, - 'scales2_crop': scales2_crop - } - # print("data_aug_params:", json.dumps(datadict_for_print, indent=2)) - - if image_set == 'train': - if fix_size: - return T.Compose([ - T.RandomHorizontalFlip(), - T.RandomResize([(max_size, max(scales))]), - # T.RandomResize([(512, 512)]), - normalize, - ]) - - if strong_aug: - import datasets.sltransform as SLT - - return T.Compose([ - T.RandomHorizontalFlip(), - T.RandomSelect( - T.RandomResize(scales, max_size=max_size), - T.Compose([ - T.RandomResize(scales2_resize), - T.RandomSizeCrop(*scales2_crop), - T.RandomResize(scales, max_size=max_size), - ]) - ), - SLT.RandomSelectMulti([ - SLT.RandomCrop(), - SLT.LightingNoise(), - SLT.AdjustBrightness(2), - SLT.AdjustContrast(2), - ]), - normalize, - ]) - - return T.Compose([ - T.RandomHorizontalFlip(), - T.RandomSelect( - T.RandomResize(scales, max_size=max_size), - T.Compose([ - T.RandomResize(scales2_resize), - T.RandomSizeCrop(*scales2_crop), - T.RandomResize(scales, max_size=max_size), - ]) - ), - normalize, - ]) - - if image_set in ['val', 'eval_debug', 'train_reg', 'test']: - - if os.environ.get("GFLOPS_DEBUG_SHILONG", False) == 'INFO': - print("Under debug mode for flops calculation only!!!!!!!!!!!!!!!!") - return T.Compose([ - T.ResizeDebug((1280, 800)), - normalize, - ]) - - return T.Compose([ - T.RandomResize([max(scales)], max_size=max_size), - normalize, - ]) - - - - raise ValueError(f'unknown {image_set}') - - -def get_aux_target_hacks_list(image_set, args): - if args.modelname in ['q2bs_mask', 'q2bs']: - aux_target_hacks_list = [ - label2compat(), - label_compat2onehot(), - RandomSelectBoxes(num_class=args.num_classes) - ] - if args.masked_data and image_set == 'train': - # aux_target_hacks_list.append() - aux_target_hacks_list.append(MaskCrop()) - elif args.modelname in ['q2bm_v2', 'q2bs_ce', 'q2op', 'q2ofocal', 'q2opclip', 'q2ocqonly']: - aux_target_hacks_list = [ - label2compat(), - label_compat2onehot(), - box_label_catter(), - RandomSelectBoxlabels(num_classes=args.num_classes, - prob_first_item=args.prob_first_item, - prob_random_item=args.prob_random_item, - prob_last_item=args.prob_last_item, - prob_stop_sign=args.prob_stop_sign, - ), - BboxPertuber(max_ratio=0.02, generate_samples=1000), - ] - elif args.modelname in ['q2omask', 'q2osa']: - if args.coco_aug: - aux_target_hacks_list = [ - label2compat(), - label_compat2onehot(), - box_label_catter(), - RandomSelectBoxlabels(num_classes=args.num_classes, - prob_first_item=args.prob_first_item, - prob_random_item=args.prob_random_item, - prob_last_item=args.prob_last_item, - prob_stop_sign=args.prob_stop_sign, - ), - RandomDrop(p=0.2), - BboxPertuber(max_ratio=0.02, generate_samples=1000), - RandomCutout(factor=0.5) - ] - else: - aux_target_hacks_list = [ - label2compat(), - label_compat2onehot(), - box_label_catter(), - RandomSelectBoxlabels(num_classes=args.num_classes, - prob_first_item=args.prob_first_item, - prob_random_item=args.prob_random_item, - prob_last_item=args.prob_last_item, - prob_stop_sign=args.prob_stop_sign, - ), - BboxPertuber(max_ratio=0.02, generate_samples=1000), - ] - else: - aux_target_hacks_list = None - - return aux_target_hacks_list - - -def build(image_set, args, datasetinfo): - img_folder = datasetinfo["root"] - ann_file = datasetinfo["anno"] - - # copy to local path - if os.environ.get('DATA_COPY_SHILONG') == 'INFO': - preparing_dataset(dict(img_folder=img_folder, ann_file=ann_file), image_set, args) - - try: - strong_aug = args.strong_aug - except: - strong_aug = False - print(img_folder, ann_file) - dataset = CocoDetection(img_folder, ann_file, - transforms=make_coco_transforms(image_set, fix_size=args.fix_size, strong_aug=strong_aug, args=args), - return_masks=args.masks, - aux_target_hacks=None, - ) - return dataset - - -if __name__ == "__main__": - # Objects365 Val example - dataset_o365 = CocoDetection( - '/path/Objects365/train/', - "/path/Objects365/slannos/anno_preprocess_train_v2.json", - transforms=None, - return_masks=False, - ) - print('len(dataset_o365):', len(dataset_o365)) diff --git a/groundingdino/datasets/coco_eval.py b/groundingdino/datasets/coco_eval.py deleted file mode 100644 index 74bbc0b32b5e4dbe40cddae7ff9add26e3f910f9..0000000000000000000000000000000000000000 --- a/groundingdino/datasets/coco_eval.py +++ /dev/null @@ -1,266 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -""" -COCO evaluator that works in distributed mode. - -Mostly copy-paste from https://github.com/pytorch/vision/blob/edfd5a7/references/detection/coco_eval.py -The difference is that there is less copy-pasting from pycocotools -in the end of the file, as python3 can suppress prints with contextlib -""" -import os -import contextlib -import copy -import numpy as np -import torch - -from pycocotools.cocoeval import COCOeval -from pycocotools.coco import COCO -import pycocotools.mask as mask_util - -from util.misc import all_gather - - -class CocoEvaluator(object): - def __init__(self, coco_gt, iou_types, useCats=True): - assert isinstance(iou_types, (list, tuple)) - coco_gt = copy.deepcopy(coco_gt) - self.coco_gt = coco_gt - - self.iou_types = iou_types - self.coco_eval = {} - for iou_type in iou_types: - self.coco_eval[iou_type] = COCOeval(coco_gt, iouType=iou_type) - self.coco_eval[iou_type].useCats = useCats - - self.img_ids = [] - self.eval_imgs = {k: [] for k in iou_types} - self.useCats = useCats - - def update(self, predictions): - img_ids = list(np.unique(list(predictions.keys()))) - self.img_ids.extend(img_ids) - - for iou_type in self.iou_types: - results = self.prepare(predictions, iou_type) - - # suppress pycocotools prints - with open(os.devnull, 'w') as devnull: - with contextlib.redirect_stdout(devnull): - coco_dt = COCO.loadRes(self.coco_gt, results) if results else COCO() - coco_eval = self.coco_eval[iou_type] - - coco_eval.cocoDt = coco_dt - coco_eval.params.imgIds = list(img_ids) - coco_eval.params.useCats = self.useCats - img_ids, eval_imgs = evaluate(coco_eval) - - self.eval_imgs[iou_type].append(eval_imgs) - - def synchronize_between_processes(self): - for iou_type in self.iou_types: - self.eval_imgs[iou_type] = np.concatenate(self.eval_imgs[iou_type], 2) - create_common_coco_eval(self.coco_eval[iou_type], self.img_ids, self.eval_imgs[iou_type]) - - def accumulate(self): - for coco_eval in self.coco_eval.values(): - coco_eval.accumulate() - - def summarize(self): - for iou_type, coco_eval in self.coco_eval.items(): - print("IoU metric: {}".format(iou_type)) - coco_eval.summarize() - - def prepare(self, predictions, iou_type): - if iou_type == "bbox": - return self.prepare_for_coco_detection(predictions) - elif iou_type == "segm": - return self.prepare_for_coco_segmentation(predictions) - elif iou_type == "keypoints": - return self.prepare_for_coco_keypoint(predictions) - else: - raise ValueError("Unknown iou type {}".format(iou_type)) - - def prepare_for_coco_detection(self, predictions): - coco_results = [] - for original_id, prediction in predictions.items(): - if len(prediction) == 0: - continue - - boxes = prediction["boxes"] - boxes = convert_to_xywh(boxes).tolist() - if not isinstance(prediction["scores"], list): - scores = prediction["scores"].tolist() - else: - scores = prediction["scores"] - if not isinstance(prediction["labels"], list): - labels = prediction["labels"].tolist() - else: - labels = prediction["labels"] - - - try: - coco_results.extend( - [ - { - "image_id": original_id, - "category_id": labels[k], - "bbox": box, - "score": scores[k], - } - for k, box in enumerate(boxes) - ] - ) - except: - import ipdb; ipdb.set_trace() - return coco_results - - def prepare_for_coco_segmentation(self, predictions): - coco_results = [] - for original_id, prediction in predictions.items(): - if len(prediction) == 0: - continue - - scores = prediction["scores"] - labels = prediction["labels"] - masks = prediction["masks"] - - masks = masks > 0.5 - - scores = prediction["scores"].tolist() - labels = prediction["labels"].tolist() - - rles = [ - mask_util.encode(np.array(mask[0, :, :, np.newaxis], dtype=np.uint8, order="F"))[0] - for mask in masks - ] - for rle in rles: - rle["counts"] = rle["counts"].decode("utf-8") - - coco_results.extend( - [ - { - "image_id": original_id, - "category_id": labels[k], - "segmentation": rle, - "score": scores[k], - } - for k, rle in enumerate(rles) - ] - ) - return coco_results - - def prepare_for_coco_keypoint(self, predictions): - coco_results = [] - for original_id, prediction in predictions.items(): - if len(prediction) == 0: - continue - - boxes = prediction["boxes"] - boxes = convert_to_xywh(boxes).tolist() - scores = prediction["scores"].tolist() - labels = prediction["labels"].tolist() - keypoints = prediction["keypoints"] - keypoints = keypoints.flatten(start_dim=1).tolist() - - coco_results.extend( - [ - { - "image_id": original_id, - "category_id": labels[k], - 'keypoints': keypoint, - "score": scores[k], - } - for k, keypoint in enumerate(keypoints) - ] - ) - return coco_results - - -def convert_to_xywh(boxes): - xmin, ymin, xmax, ymax = boxes.unbind(1) - return torch.stack((xmin, ymin, xmax - xmin, ymax - ymin), dim=1) - - -def merge(img_ids, eval_imgs): - all_img_ids = all_gather(img_ids) - all_eval_imgs = all_gather(eval_imgs) - - merged_img_ids = [] - for p in all_img_ids: - merged_img_ids.extend(p) - - merged_eval_imgs = [] - for p in all_eval_imgs: - merged_eval_imgs.append(p) - - merged_img_ids = np.array(merged_img_ids) - merged_eval_imgs = np.concatenate(merged_eval_imgs, 2) - - # keep only unique (and in sorted order) images - merged_img_ids, idx = np.unique(merged_img_ids, return_index=True) - merged_eval_imgs = merged_eval_imgs[..., idx] - - return merged_img_ids, merged_eval_imgs - - -def create_common_coco_eval(coco_eval, img_ids, eval_imgs): - img_ids, eval_imgs = merge(img_ids, eval_imgs) - img_ids = list(img_ids) - eval_imgs = list(eval_imgs.flatten()) - - coco_eval.evalImgs = eval_imgs - coco_eval.params.imgIds = img_ids - coco_eval._paramsEval = copy.deepcopy(coco_eval.params) - - -################################################################# -# From pycocotools, just removed the prints and fixed -# a Python3 bug about unicode not defined -################################################################# - - -def evaluate(self): - ''' - Run per image evaluation on given images and store results (a list of dict) in self.evalImgs - :return: None - ''' - p = self.params - # add backward compatibility if useSegm is specified in params - if p.useSegm is not None: - p.iouType = 'segm' if p.useSegm == 1 else 'bbox' - print('useSegm (deprecated) is not None. Running {} evaluation'.format(p.iouType)) - p.imgIds = list(np.unique(p.imgIds)) - if p.useCats: - p.catIds = list(np.unique(p.catIds)) - p.maxDets = sorted(p.maxDets) - self.params = p - - self._prepare() - # loop through images, area range, max detection number - catIds = p.catIds if p.useCats else [-1] - - if p.iouType == 'segm' or p.iouType == 'bbox': - computeIoU = self.computeIoU - elif p.iouType == 'keypoints': - computeIoU = self.computeOks - self.ious = { - (imgId, catId): computeIoU(imgId, catId) - for imgId in p.imgIds - for catId in catIds} - - evaluateImg = self.evaluateImg - maxDet = p.maxDets[-1] - evalImgs = [ - evaluateImg(imgId, catId, areaRng, maxDet) - for catId in catIds - for areaRng in p.areaRng - for imgId in p.imgIds - ] - # this is NOT in the pycocotools code, but could be done outside - evalImgs = np.asarray(evalImgs).reshape(len(catIds), len(p.areaRng), len(p.imgIds)) - self._paramsEval = copy.deepcopy(self.params) - - return p.imgIds, evalImgs - -################################################################# -# end of straight copy from pycocotools, just removing the prints -################################################################# diff --git a/groundingdino/datasets/coco_panoptic.py b/groundingdino/datasets/coco_panoptic.py deleted file mode 100644 index b24f615c2faa14b422829e2edad996e2b5b84248..0000000000000000000000000000000000000000 --- a/groundingdino/datasets/coco_panoptic.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -import json -from pathlib import Path - -import numpy as np -import torch -from PIL import Image - -from panopticapi.utils import rgb2id -from util.box_ops import masks_to_boxes - -from .coco import make_coco_transforms - - -class CocoPanoptic: - def __init__(self, img_folder, ann_folder, ann_file, transforms=None, return_masks=True): - with open(ann_file, 'r') as f: - self.coco = json.load(f) - - # sort 'images' field so that they are aligned with 'annotations' - # i.e., in alphabetical order - self.coco['images'] = sorted(self.coco['images'], key=lambda x: x['id']) - # sanity check - if "annotations" in self.coco: - for img, ann in zip(self.coco['images'], self.coco['annotations']): - assert img['file_name'][:-4] == ann['file_name'][:-4] - - self.img_folder = img_folder - self.ann_folder = ann_folder - self.ann_file = ann_file - self.transforms = transforms - self.return_masks = return_masks - - def __getitem__(self, idx): - ann_info = self.coco['annotations'][idx] if "annotations" in self.coco else self.coco['images'][idx] - img_path = Path(self.img_folder) / ann_info['file_name'].replace('.png', '.jpg') - ann_path = Path(self.ann_folder) / ann_info['file_name'] - - img = Image.open(img_path).convert('RGB') - w, h = img.size - if "segments_info" in ann_info: - masks = np.asarray(Image.open(ann_path), dtype=np.uint32) - masks = rgb2id(masks) - - ids = np.array([ann['id'] for ann in ann_info['segments_info']]) - masks = masks == ids[:, None, None] - - masks = torch.as_tensor(masks, dtype=torch.uint8) - labels = torch.tensor([ann['category_id'] for ann in ann_info['segments_info']], dtype=torch.int64) - - target = {} - target['image_id'] = torch.tensor([ann_info['image_id'] if "image_id" in ann_info else ann_info["id"]]) - if self.return_masks: - target['masks'] = masks - target['labels'] = labels - - target["boxes"] = masks_to_boxes(masks) - - target['size'] = torch.as_tensor([int(h), int(w)]) - target['orig_size'] = torch.as_tensor([int(h), int(w)]) - if "segments_info" in ann_info: - for name in ['iscrowd', 'area']: - target[name] = torch.tensor([ann[name] for ann in ann_info['segments_info']]) - - if self.transforms is not None: - img, target = self.transforms(img, target) - - return img, target - - def __len__(self): - return len(self.coco['images']) - - def get_height_and_width(self, idx): - img_info = self.coco['images'][idx] - height = img_info['height'] - width = img_info['width'] - return height, width - - -def build(image_set, args): - img_folder_root = Path(args.coco_path) - ann_folder_root = Path(args.coco_panoptic_path) - assert img_folder_root.exists(), f'provided COCO path {img_folder_root} does not exist' - assert ann_folder_root.exists(), f'provided COCO path {ann_folder_root} does not exist' - mode = 'panoptic' - PATHS = { - "train": ("train2017", Path("annotations") / f'{mode}_train2017.json'), - "val": ("val2017", Path("annotations") / f'{mode}_val2017.json'), - } - - img_folder, ann_file = PATHS[image_set] - img_folder_path = img_folder_root / img_folder - ann_folder = ann_folder_root / f'{mode}_{img_folder}' - ann_file = ann_folder_root / ann_file - - dataset = CocoPanoptic(img_folder_path, ann_folder, ann_file, - transforms=make_coco_transforms(image_set), return_masks=args.masks) - - return dataset diff --git a/groundingdino/datasets/cocogrounding_eval.py b/groundingdino/datasets/cocogrounding_eval.py deleted file mode 100644 index 32f86a50a8f1daf36135edefd20574e022fe5ff6..0000000000000000000000000000000000000000 --- a/groundingdino/datasets/cocogrounding_eval.py +++ /dev/null @@ -1,271 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO. Midified by Shilong Liu. -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Copyright (c) Aishwarya Kamath & Nicolas Carion. Licensed under the Apache License 2.0. All Rights Reserved -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -""" -COCO evaluator that works in distributed mode. - -Mostly copy-paste from https://github.com/pytorch/vision/blob/edfd5a7/references/detection/coco_eval.py -The difference is that there is less copy-pasting from pycocotools -in the end of the file, as python3 can suppress prints with contextlib -""" -import contextlib -import copy -import os - -import numpy as np -import pycocotools.mask as mask_util -import torch -from pycocotools.coco import COCO -from pycocotools.cocoeval import COCOeval - -from groundingdino.util.misc import all_gather - - -class CocoGroundingEvaluator(object): - def __init__(self, coco_gt, iou_types, useCats=True): - assert isinstance(iou_types, (list, tuple)) - coco_gt = copy.deepcopy(coco_gt) - self.coco_gt = coco_gt - - self.iou_types = iou_types - self.coco_eval = {} - for iou_type in iou_types: - self.coco_eval[iou_type] = COCOeval(coco_gt, iouType=iou_type) - self.coco_eval[iou_type].useCats = useCats - - self.img_ids = [] - self.eval_imgs = {k: [] for k in iou_types} - self.useCats = useCats - - def update(self, predictions): - img_ids = list(np.unique(list(predictions.keys()))) - self.img_ids.extend(img_ids) - # import pdb;pdb.set_trace() - for iou_type in self.iou_types: - results = self.prepare(predictions, iou_type) - - # suppress pycocotools prints - with open(os.devnull, "w") as devnull: - with contextlib.redirect_stdout(devnull): - coco_dt = COCO.loadRes(self.coco_gt, results) if results else COCO() - - coco_eval = self.coco_eval[iou_type] - - coco_eval.cocoDt = coco_dt - coco_eval.params.imgIds = list(img_ids) - coco_eval.params.useCats = self.useCats - img_ids, eval_imgs = evaluate(coco_eval) - - self.eval_imgs[iou_type].append(eval_imgs) - - def synchronize_between_processes(self): - for iou_type in self.iou_types: - self.eval_imgs[iou_type] = np.concatenate(self.eval_imgs[iou_type], 2) - create_common_coco_eval(self.coco_eval[iou_type], self.img_ids, self.eval_imgs[iou_type]) - - def accumulate(self): - for coco_eval in self.coco_eval.values(): - coco_eval.accumulate() - - def summarize(self): - for iou_type, coco_eval in self.coco_eval.items(): - print("IoU metric: {}".format(iou_type)) - coco_eval.summarize() - - def prepare(self, predictions, iou_type): - if iou_type == "bbox": - return self.prepare_for_coco_detection(predictions) - elif iou_type == "segm": - return self.prepare_for_coco_segmentation(predictions) - elif iou_type == "keypoints": - return self.prepare_for_coco_keypoint(predictions) - else: - raise ValueError("Unknown iou type {}".format(iou_type)) - - def prepare_for_coco_detection(self, predictions): - coco_results = [] - for original_id, prediction in predictions.items(): - if len(prediction) == 0: - continue - - boxes = prediction["boxes"] - boxes = convert_to_xywh(boxes).tolist() - scores = prediction["scores"].tolist() - labels = prediction["labels"].tolist() - - coco_results.extend( - [ - { - "image_id": original_id, - "category_id": labels[k], - "bbox": box, - "score": scores[k], - } - for k, box in enumerate(boxes) - ] - ) - return coco_results - - def prepare_for_coco_segmentation(self, predictions): - coco_results = [] - for original_id, prediction in predictions.items(): - if len(prediction) == 0: - continue - - scores = prediction["scores"] - labels = prediction["labels"] - masks = prediction["masks"] - - masks = masks > 0.5 - - scores = prediction["scores"].tolist() - labels = prediction["labels"].tolist() - - rles = [ - mask_util.encode(np.array(mask[0, :, :, np.newaxis], dtype=np.uint8, order="F"))[0] - for mask in masks - ] - for rle in rles: - rle["counts"] = rle["counts"].decode("utf-8") - - coco_results.extend( - [ - { - "image_id": original_id, - "category_id": labels[k], - "segmentation": rle, - "score": scores[k], - } - for k, rle in enumerate(rles) - ] - ) - return coco_results - - def prepare_for_coco_keypoint(self, predictions): - coco_results = [] - for original_id, prediction in predictions.items(): - if len(prediction) == 0: - continue - - boxes = prediction["boxes"] - boxes = convert_to_xywh(boxes).tolist() - scores = prediction["scores"].tolist() - labels = prediction["labels"].tolist() - keypoints = prediction["keypoints"] - keypoints = keypoints.flatten(start_dim=1).tolist() - - coco_results.extend( - [ - { - "image_id": original_id, - "category_id": labels[k], - "keypoints": keypoint, - "score": scores[k], - } - for k, keypoint in enumerate(keypoints) - ] - ) - return coco_results - - -def convert_to_xywh(boxes): - xmin, ymin, xmax, ymax = boxes.unbind(1) - return torch.stack((xmin, ymin, xmax - xmin, ymax - ymin), dim=1) - - -def merge(img_ids, eval_imgs): - all_img_ids = all_gather(img_ids) - all_eval_imgs = all_gather(eval_imgs) - - merged_img_ids = [] - for p in all_img_ids: - merged_img_ids.extend(p) - - merged_eval_imgs = [] - for p in all_eval_imgs: - merged_eval_imgs.append(p) - - merged_img_ids = np.array(merged_img_ids) - merged_eval_imgs = np.concatenate(merged_eval_imgs, 2) - - # keep only unique (and in sorted order) images - merged_img_ids, idx = np.unique(merged_img_ids, return_index=True) - merged_eval_imgs = merged_eval_imgs[..., idx] - - return merged_img_ids, merged_eval_imgs - - -def create_common_coco_eval(coco_eval, img_ids, eval_imgs): - img_ids, eval_imgs = merge(img_ids, eval_imgs) - img_ids = list(img_ids) - eval_imgs = list(eval_imgs.flatten()) - - coco_eval.evalImgs = eval_imgs - coco_eval.params.imgIds = img_ids - coco_eval._paramsEval = copy.deepcopy(coco_eval.params) - - -################################################################# -# From pycocotools, just removed the prints and fixed -# a Python3 bug about unicode not defined -################################################################# - - -def evaluate(self): - """ - Run per image evaluation on given images and store results (a list of dict) in self.evalImgs - :return: None - """ - # tic = time.time() - # print('Running per image evaluation...') - - # import pdb;pdb.set_trace() - p = self.params - # add backward compatibility if useSegm is specified in params - if p.useSegm is not None: - p.iouType = "segm" if p.useSegm == 1 else "bbox" - print("useSegm (deprecated) is not None. Running {} evaluation".format(p.iouType)) - # print('Evaluate annotation type *{}*'.format(p.iouType)) - p.imgIds = list(np.unique(p.imgIds)) - if p.useCats: - p.catIds = list(np.unique(p.catIds)) - p.maxDets = sorted(p.maxDets) - self.params = p - - self._prepare() - # loop through images, area range, max detection number - catIds = p.catIds if p.useCats else [-1] - - if p.iouType == "segm" or p.iouType == "bbox": - computeIoU = self.computeIoU - elif p.iouType == "keypoints": - computeIoU = self.computeOks - self.ious = { - (imgId, catId): computeIoU(imgId, catId) - for imgId in p.imgIds - for catId in catIds} - - evaluateImg = self.evaluateImg - maxDet = p.maxDets[-1] - evalImgs = [ - evaluateImg(imgId, catId, areaRng, maxDet) - for catId in catIds - for areaRng in p.areaRng - for imgId in p.imgIds - ] - # this is NOT in the pycocotools code, but could be done outside - evalImgs = np.asarray(evalImgs).reshape(len(catIds), len(p.areaRng), len(p.imgIds)) - self._paramsEval = copy.deepcopy(self.params) - # toc = time.time() - # print('DONE (t={:0.2f}s).'.format(toc-tic)) - return p.imgIds, evalImgs - - -################################################################# -# end of straight copy from pycocotools, just removing the prints -################################################################# diff --git a/groundingdino/datasets/data_util.py b/groundingdino/datasets/data_util.py deleted file mode 100644 index 8019ccd7348f87640620c52b7a3f8a1c266859a7..0000000000000000000000000000000000000000 --- a/groundingdino/datasets/data_util.py +++ /dev/null @@ -1,170 +0,0 @@ -import os -import os.path as osp -import shutil -import time -import datetime - -import torch - -from util.slconfig import SLConfig - -class Error(OSError): - pass - -def slcopytree(src, dst, symlinks=False, ignore=None, copy_function=shutil.copyfile, - ignore_dangling_symlinks=False): - """ - modified from shutil.copytree without copystat. - - Recursively copy a directory tree. - - The destination directory must not already exist. - If exception(s) occur, an Error is raised with a list of reasons. - - If the optional symlinks flag is true, symbolic links in the - source tree result in symbolic links in the destination tree; if - it is false, the contents of the files pointed to by symbolic - links are copied. If the file pointed by the symlink doesn't - exist, an exception will be added in the list of errors raised in - an Error exception at the end of the copy process. - - You can set the optional ignore_dangling_symlinks flag to true if you - want to silence this exception. Notice that this has no effect on - platforms that don't support os.symlink. - - The optional ignore argument is a callable. If given, it - is called with the `src` parameter, which is the directory - being visited by copytree(), and `names` which is the list of - `src` contents, as returned by os.listdir(): - - callable(src, names) -> ignored_names - - Since copytree() is called recursively, the callable will be - called once for each directory that is copied. It returns a - list of names relative to the `src` directory that should - not be copied. - - The optional copy_function argument is a callable that will be used - to copy each file. It will be called with the source path and the - destination path as arguments. By default, copy2() is used, but any - function that supports the same signature (like copy()) can be used. - - """ - errors = [] - if os.path.isdir(src): - names = os.listdir(src) - if ignore is not None: - ignored_names = ignore(src, names) - else: - ignored_names = set() - - os.makedirs(dst) - for name in names: - if name in ignored_names: - continue - srcname = os.path.join(src, name) - dstname = os.path.join(dst, name) - try: - if os.path.islink(srcname): - linkto = os.readlink(srcname) - if symlinks: - # We can't just leave it to `copy_function` because legacy - # code with a custom `copy_function` may rely on copytree - # doing the right thing. - os.symlink(linkto, dstname) - else: - # ignore dangling symlink if the flag is on - if not os.path.exists(linkto) and ignore_dangling_symlinks: - continue - # otherwise let the copy occurs. copy2 will raise an error - if os.path.isdir(srcname): - slcopytree(srcname, dstname, symlinks, ignore, - copy_function) - else: - copy_function(srcname, dstname) - elif os.path.isdir(srcname): - slcopytree(srcname, dstname, symlinks, ignore, copy_function) - else: - # Will raise a SpecialFileError for unsupported file types - copy_function(srcname, dstname) - # catch the Error from the recursive copytree so that we can - # continue with other files - except Error as err: - errors.extend(err.args[0]) - except OSError as why: - errors.append((srcname, dstname, str(why))) - else: - copy_function(src, dst) - - if errors: - raise Error(errors) - return dst - -def check_and_copy(src_path, tgt_path): - if os.path.exists(tgt_path): - return None - - return slcopytree(src_path, tgt_path) - - -def remove(srcpath): - if os.path.isdir(srcpath): - return shutil.rmtree(srcpath) - else: - return os.remove(srcpath) - - -def preparing_dataset(pathdict, image_set, args): - start_time = time.time() - dataset_file = args.dataset_file - data_static_info = SLConfig.fromfile('util/static_data_path.py') - static_dict = data_static_info[dataset_file][image_set] - - copyfilelist = [] - for k,tgt_v in pathdict.items(): - if os.path.exists(tgt_v): - if args.local_rank == 0: - print("path <{}> exist. remove it!".format(tgt_v)) - remove(tgt_v) - # continue - - if args.local_rank == 0: - src_v = static_dict[k] - assert isinstance(src_v, str) - if src_v.endswith('.zip'): - # copy - cp_tgt_dir = os.path.dirname(tgt_v) - filename = os.path.basename(src_v) - cp_tgt_path = os.path.join(cp_tgt_dir, filename) - print('Copy from <{}> to <{}>.'.format(src_v, cp_tgt_path)) - os.makedirs(cp_tgt_dir, exist_ok=True) - check_and_copy(src_v, cp_tgt_path) - - # unzip - import zipfile - print("Starting unzip <{}>".format(cp_tgt_path)) - with zipfile.ZipFile(cp_tgt_path, 'r') as zip_ref: - zip_ref.extractall(os.path.dirname(cp_tgt_path)) - - copyfilelist.append(cp_tgt_path) - copyfilelist.append(tgt_v) - else: - print('Copy from <{}> to <{}>.'.format(src_v, tgt_v)) - os.makedirs(os.path.dirname(tgt_v), exist_ok=True) - check_and_copy(src_v, tgt_v) - copyfilelist.append(tgt_v) - - if len(copyfilelist) == 0: - copyfilelist = None - args.copyfilelist = copyfilelist - - if args.distributed: - torch.distributed.barrier() - total_time = time.time() - start_time - if copyfilelist: - total_time_str = str(datetime.timedelta(seconds=int(total_time))) - print('Data copy time {}'.format(total_time_str)) - return copyfilelist - - - \ No newline at end of file diff --git a/groundingdino/datasets/dataset.py b/groundingdino/datasets/dataset.py deleted file mode 100644 index 5655c00ccdbf0b09d51da8dfa3593a2bf9aee527..0000000000000000000000000000000000000000 --- a/groundingdino/datasets/dataset.py +++ /dev/null @@ -1,44 +0,0 @@ -from __future__ import print_function - -import torch -import torchvision.datasets as datasets -from torch.utils.data import Dataset -from PIL import Image -from .tsv_io import TSVFile -import numpy as np -import base64 -import io - - -class TSVDataset(Dataset): - """ TSV dataset for ImageNet 1K training - """ - def __init__(self, tsv_file, transform=None, target_transform=None): - self.tsv = TSVFile(tsv_file) - self.transform = transform - self.target_transform = target_transform - - def __getitem__(self, index): - """ - Args: - index (int): Index - Returns: - tuple: (image, target) where target is class_index of the target class. - """ - row = self.tsv.seek(index) - image_data = base64.b64decode(row[-1]) - image = Image.open(io.BytesIO(image_data)) - image = image.convert('RGB') - target = int(row[1]) - - if self.transform is not None: - img = self.transform(image) - else: - img = image - if self.target_transform is not None: - target = self.target_transform(target) - - return img, target - - def __len__(self): - return self.tsv.num_rows() diff --git a/groundingdino/datasets/odvg.py b/groundingdino/datasets/odvg.py deleted file mode 100644 index e684a65f01f8b65e5cf24dadecfa4f38cc16d873..0000000000000000000000000000000000000000 --- a/groundingdino/datasets/odvg.py +++ /dev/null @@ -1,258 +0,0 @@ -from torchvision.datasets.vision import VisionDataset -import os.path -from typing import Callable, Optional -import json -from PIL import Image -import torch -import random -import os, sys -sys.path.append(os.path.dirname(sys.path[0])) - -import datasets.transforms as T - -class ODVGDataset(VisionDataset): - """ - Args: - root (string): Root directory where images are downloaded to. - anno (string): Path to json annotation file. - label_map_anno (string): Path to json label mapping file. Only for Object Detection - transform (callable, optional): A function/transform that takes in an PIL image - and returns a transformed version. E.g, ``transforms.PILToTensor`` - target_transform (callable, optional): A function/transform that takes in the - target and transforms it. - transforms (callable, optional): A function/transform that takes input sample and its target as entry - and returns a transformed version. - """ - - def __init__( - self, - root: str, - anno: str, - label_map_anno: str = None, - max_labels: int = 80, - transform: Optional[Callable] = None, - target_transform: Optional[Callable] = None, - transforms: Optional[Callable] = None, - ) -> None: - super().__init__(root, transforms, transform, target_transform) - self.root = root - self.dataset_mode = "OD" if label_map_anno else "VG" - self.max_labels = max_labels - if self.dataset_mode == "OD": - self.load_label_map(label_map_anno) - self._load_metas(anno) - self.get_dataset_info() - - def load_label_map(self, label_map_anno): - with open(label_map_anno, 'r') as file: - self.label_map = json.load(file) - self.label_index = set(self.label_map.keys()) - - def _load_metas(self, anno): - with open(anno, 'r') as f: - self.metas = json.load(f) - - - def get_dataset_info(self): - print(f" == total images: {len(self)}") - if self.dataset_mode == "OD": - print(f" == total labels: {len(self.label_map)}") - - def __getitem__(self, index: int): - meta = self.metas[index] - rel_path = meta["filename"] - abs_path = os.path.join(self.root, rel_path) - if not os.path.exists(abs_path): - raise FileNotFoundError(f"{abs_path} not found.") - image = Image.open(abs_path).convert('RGB') - w, h = image.size - if self.dataset_mode == "OD": - anno = meta["detection"] - instances = [obj for obj in anno["instances"]] - boxes = [obj["bbox"] for obj in instances] - # generate vg_labels - # pos bbox labels - ori_classes = [str(obj["label"]) for obj in instances] - pos_labels = set(ori_classes) - # neg bbox labels - neg_labels = self.label_index.difference(pos_labels) - - vg_labels = list(pos_labels) - num_to_add = min(len(neg_labels), self.max_labels-len(pos_labels)) - if num_to_add > 0: - vg_labels.extend(random.sample(neg_labels, num_to_add)) - - # shuffle - for i in range(len(vg_labels)-1, 0, -1): - j = random.randint(0, i) - vg_labels[i], vg_labels[j] = vg_labels[j], vg_labels[i] - - caption_list = [self.label_map[lb] for lb in vg_labels] - caption_dict = {item:index for index, item in enumerate(caption_list)} - - caption = ' . '.join(caption_list) + ' .' - classes = [caption_dict[self.label_map[str(obj["label"])]] for obj in instances] - boxes = torch.as_tensor(boxes, dtype=torch.float32).reshape(-1, 4) - classes = torch.tensor(classes, dtype=torch.int64) - elif self.dataset_mode == "VG": - anno = meta["Grounding"] - instances = [obj for obj in anno["regions"]] - boxes = [obj["bbox"] for obj in instances] - caption_list = [obj["phrase"] for obj in instances] - c = list(zip(boxes, caption_list)) - random.shuffle(c) - boxes[:], caption_list[:] = zip(*c) - uni_caption_list = list(set(caption_list)) - label_map = {} - for idx in range(len(uni_caption_list)): - label_map[uni_caption_list[idx]] = idx - classes = [label_map[cap] for cap in caption_list] - caption = ' . '.join(uni_caption_list) + ' .' - boxes = torch.as_tensor(boxes, dtype=torch.float32).reshape(-1, 4) - classes = torch.tensor(classes, dtype=torch.int64) - caption_list = uni_caption_list - # print("caption_list" , caption_list) - # print("caption" , caption) - # print("boxes" , boxes) - target = {} - target["image_id"] = rel_path.strip(".jpg") - target["size"] = torch.as_tensor([int(h), int(w)]) - target["cap_list"] = caption_list - target["caption"] = caption - target["boxes"] = boxes - target["labels"] = classes - # print(" image_id " , target["image_id"]) - # size, cap_list, caption, bboxes, labels - - if self.transforms is not None: - image, target = self.transforms(image, target) - - return image, target - - - def __len__(self) -> int: - return len(self.metas) - - -def make_coco_transforms(image_set, fix_size=False, strong_aug=False, args=None): - - normalize = T.Compose([ - T.ToTensor(), - T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) - ]) - - # config the params for data aug - scales = [480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800] - max_size = 1333 - scales2_resize = [400, 500, 600] - scales2_crop = [384, 600] - - # update args from config files - scales = getattr(args, 'data_aug_scales', scales) - max_size = getattr(args, 'data_aug_max_size', max_size) - scales2_resize = getattr(args, 'data_aug_scales2_resize', scales2_resize) - scales2_crop = getattr(args, 'data_aug_scales2_crop', scales2_crop) - - # resize them - data_aug_scale_overlap = getattr(args, 'data_aug_scale_overlap', None) - if data_aug_scale_overlap is not None and data_aug_scale_overlap > 0: - data_aug_scale_overlap = float(data_aug_scale_overlap) - scales = [int(i*data_aug_scale_overlap) for i in scales] - max_size = int(max_size*data_aug_scale_overlap) - scales2_resize = [int(i*data_aug_scale_overlap) for i in scales2_resize] - scales2_crop = [int(i*data_aug_scale_overlap) for i in scales2_crop] - - # datadict_for_print = { - # 'scales': scales, - # 'max_size': max_size, - # 'scales2_resize': scales2_resize, - # 'scales2_crop': scales2_crop - # } - # print("data_aug_params:", json.dumps(datadict_for_print, indent=2)) - - if image_set == 'train': - if fix_size: - return T.Compose([ - T.RandomHorizontalFlip(), - T.RandomResize([(max_size, max(scales))]), - normalize, - ]) - - if strong_aug: - import datasets.sltransform as SLT - - return T.Compose([ - T.RandomHorizontalFlip(), - T.RandomSelect( - T.RandomResize(scales, max_size=max_size), - T.Compose([ - T.RandomResize(scales2_resize), - T.RandomSizeCrop(*scales2_crop), - T.RandomResize(scales, max_size=max_size), - ]) - ), - SLT.RandomSelectMulti([ - SLT.RandomCrop(), - SLT.LightingNoise(), - SLT.AdjustBrightness(2), - SLT.AdjustContrast(2), - ]), - normalize, - ]) - - return T.Compose([ - T.RandomHorizontalFlip(), - T.RandomSelect( - T.RandomResize(scales, max_size=max_size), - T.Compose([ - T.RandomResize(scales2_resize), - T.RandomSizeCrop(*scales2_crop), - T.RandomResize(scales, max_size=max_size), - ]) - ), - normalize, - ]) - - if image_set in ['val', 'eval_debug', 'train_reg', 'test']: - - if os.environ.get("GFLOPS_DEBUG_SHILONG", False) == 'INFO': - print("Under debug mode for flops calculation only!!!!!!!!!!!!!!!!") - return T.Compose([ - T.ResizeDebug((1280, 800)), - normalize, - ]) - - return T.Compose([ - T.RandomResize([max(scales)], max_size=max_size), - normalize, - ]) - - raise ValueError(f'unknown {image_set}') - -def build_odvg(image_set, args, datasetinfo): - img_folder = datasetinfo["root"] - ann_file = datasetinfo["anno"] - label_map = datasetinfo["label_map"] if "label_map" in datasetinfo else None - try: - strong_aug = args.strong_aug - except: - strong_aug = False # False originally - print(img_folder, ann_file, label_map) - dataset = ODVGDataset(img_folder, ann_file, label_map, max_labels=args.max_labels, - transforms=make_coco_transforms(image_set, fix_size=args.fix_size, strong_aug=strong_aug, args=args), - ) - return dataset - - -if __name__=="__main__": - dataset_vg = ODVGDataset("path/GRIT-20M/data/","path/GRIT-20M/anno/grit_odvg_10k.jsonl",) - print(len(dataset_vg)) - data = dataset_vg[random.randint(0, 100)] - print(data) - dataset_od = ODVGDataset("pathl/V3Det/", - "path/V3Det/annotations/v3det_2023_v1_all_odvg.jsonl", - "path/V3Det/annotations/v3det_label_map.json", - ) - print(len(dataset_od)) - data = dataset_od[random.randint(0, 100)] - print(data) \ No newline at end of file diff --git a/groundingdino/datasets/panoptic_eval.py b/groundingdino/datasets/panoptic_eval.py deleted file mode 100644 index 9cb4f83409046a5c2a87643ee005e52a440aae74..0000000000000000000000000000000000000000 --- a/groundingdino/datasets/panoptic_eval.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -import json -import os - -import util.misc as utils - -try: - from panopticapi.evaluation import pq_compute -except ImportError: - pass - - -class PanopticEvaluator(object): - def __init__(self, ann_file, ann_folder, output_dir="panoptic_eval"): - self.gt_json = ann_file - self.gt_folder = ann_folder - if utils.is_main_process(): - if not os.path.exists(output_dir): - os.mkdir(output_dir) - self.output_dir = output_dir - self.predictions = [] - - def update(self, predictions): - for p in predictions: - with open(os.path.join(self.output_dir, p["file_name"]), "wb") as f: - f.write(p.pop("png_string")) - - self.predictions += predictions - - def synchronize_between_processes(self): - all_predictions = utils.all_gather(self.predictions) - merged_predictions = [] - for p in all_predictions: - merged_predictions += p - self.predictions = merged_predictions - - def summarize(self): - if utils.is_main_process(): - json_data = {"annotations": self.predictions} - predictions_json = os.path.join(self.output_dir, "predictions.json") - with open(predictions_json, "w") as f: - f.write(json.dumps(json_data)) - return pq_compute(self.gt_json, predictions_json, gt_folder=self.gt_folder, pred_folder=self.output_dir) - return None diff --git a/groundingdino/datasets/random_crop.py b/groundingdino/datasets/random_crop.py deleted file mode 100644 index a23d936048e073423dd176f76607e9fc2b35bb48..0000000000000000000000000000000000000000 --- a/groundingdino/datasets/random_crop.py +++ /dev/null @@ -1,135 +0,0 @@ -import PIL #version 1.2.0 -import torch -import os -import torchvision.transforms.functional as F -import numpy as np -import random - - -def intersect(boxes1, boxes2): - ''' - Find intersection of every box combination between two sets of box - boxes1: bounding boxes 1, a tensor of dimensions (n1, 4) - boxes2: bounding boxes 2, a tensor of dimensions (n2, 4) - - Out: Intersection each of boxes1 with respect to each of boxes2, - a tensor of dimensions (n1, n2) - ''' - n1 = boxes1.size(0) - n2 = boxes2.size(0) - max_xy = torch.min(boxes1[:, 2:].unsqueeze(1).expand(n1, n2, 2), - boxes2[:, 2:].unsqueeze(0).expand(n1, n2, 2)) - - min_xy = torch.max(boxes1[:, :2].unsqueeze(1).expand(n1, n2, 2), - boxes2[:, :2].unsqueeze(0).expand(n1, n2, 2)) - inter = torch.clamp(max_xy - min_xy , min=0) # (n1, n2, 2) - return inter[:, :, 0] * inter[:, :, 1] #(n1, n2) -def find_IoU(boxes1, boxes2): - ''' - Find IoU between every boxes set of boxes - boxes1: a tensor of dimensions (n1, 4) (left, top, right , bottom) - boxes2: a tensor of dimensions (n2, 4) - - Out: IoU each of boxes1 with respect to each of boxes2, a tensor of - dimensions (n1, n2) - - Formula: - (box1 ∩ box2) / (box1 u box2) = (box1 ∩ box2) / (area(box1) + area(box2) - (box1 ∩ box2 )) - ''' - inter = intersect(boxes1, boxes2) - area_boxes1 = (boxes1[:, 2] - boxes1[:, 0]) * (boxes1[:, 3] - boxes1[:, 1]) - area_boxes2 = (boxes2[:, 2] - boxes2[:, 0]) * (boxes2[:, 3] - boxes2[:, 1]) - - area_boxes1 = area_boxes1.unsqueeze(1).expand_as(inter) #(n1, n2) - area_boxes2 = area_boxes2.unsqueeze(0).expand_as(inter) #(n1, n2) - union = (area_boxes1 + area_boxes2 - inter) - return inter / union - - -def random_crop(image, boxes, labels, difficulties=None): - ''' - image: A PIL image - boxes: Bounding boxes, a tensor of dimensions (#objects, 4) - labels: labels of object, a tensor of dimensions (#objects) - difficulties: difficulties of detect object, a tensor of dimensions (#objects) - - Out: cropped image , new boxes, new labels, new difficulties - ''' - if type(image) == PIL.Image.Image: - image = F.to_tensor(image) - original_h = image.size(1) - original_w = image.size(2) - - while True: - mode = random.choice([0.1, 0.3, 0.5, 0.9, None]) - - if mode is None: - return F.to_pil_image(image), boxes, labels, difficulties - - new_image = image - new_boxes = boxes - new_difficulties = difficulties - new_labels = labels - for _ in range(50): - # Crop dimensions: [0.3, 1] of original dimensions - new_h = random.uniform(0.3*original_h, original_h) - new_w = random.uniform(0.3*original_w, original_w) - - # Aspect ratio constraint b/t .5 & 2 - if new_h/new_w < 0.5 or new_h/new_w > 2: - continue - - #Crop coordinate - left = random.uniform(0, original_w - new_w) - right = left + new_w - top = random.uniform(0, original_h - new_h) - bottom = top + new_h - crop = torch.FloatTensor([int(left), int(top), int(right), int(bottom)]) - - # Calculate IoU between the crop and the bounding boxes - overlap = find_IoU(crop.unsqueeze(0), boxes) #(1, #objects) - overlap = overlap.squeeze(0) - - # If not a single bounding box has a IoU of greater than the minimum, try again - if overlap.shape[0] == 0: - continue - if overlap.max().item() < mode: - continue - - #Crop - new_image = image[:, int(top):int(bottom), int(left):int(right)] #(3, new_h, new_w) - - #Center of bounding boxes - center_bb = (boxes[:, :2] + boxes[:, 2:])/2.0 - - #Find bounding box has been had center in crop - center_in_crop = (center_bb[:, 0] >left) * (center_bb[:, 0] < right - ) *(center_bb[:, 1] > top) * (center_bb[:, 1] < bottom) #( #objects) - - if not center_in_crop.any(): - continue - - #take matching bounding box - new_boxes = boxes[center_in_crop, :] - - #take matching labels - new_labels = labels[center_in_crop] - - #take matching difficulities - if difficulties is not None: - new_difficulties = difficulties[center_in_crop] - else: - new_difficulties = None - - #Use the box left and top corner or the crop's - new_boxes[:, :2] = torch.max(new_boxes[:, :2], crop[:2]) - - #adjust to crop - new_boxes[:, :2] -= crop[:2] - - new_boxes[:, 2:] = torch.min(new_boxes[:, 2:],crop[2:]) - - #adjust to crop - new_boxes[:, 2:] -= crop[:2] - - return F.to_pil_image(new_image), new_boxes, new_labels, new_difficulties \ No newline at end of file diff --git a/groundingdino/datasets/sltransform.py b/groundingdino/datasets/sltransform.py deleted file mode 100644 index 7d9d10bc20e159f59e46912057714c527a08d6cd..0000000000000000000000000000000000000000 --- a/groundingdino/datasets/sltransform.py +++ /dev/null @@ -1,247 +0,0 @@ -# modified from https://github.com/anhtuan85/Data-Augmentation-for-Object-Detection/blob/master/augmentation.ipynb - -import PIL #version 1.2.0 -from PIL import Image #version 6.1.0 -import torch -import os -import torchvision.transforms.functional as F -import numpy as np -import random - -from .random_crop import random_crop -from util.box_ops import box_cxcywh_to_xyxy, box_xyxy_to_cxcywh - -class AdjustContrast: - def __init__(self, contrast_factor): - self.contrast_factor = contrast_factor - - def __call__(self, img, target): - """ - img (PIL Image or Tensor): Image to be adjusted. - """ - _contrast_factor = ((random.random() + 1.0) / 2.0) * self.contrast_factor - img = F.adjust_contrast(img, _contrast_factor) - return img, target - -class AdjustBrightness: - def __init__(self, brightness_factor): - self.brightness_factor = brightness_factor - - def __call__(self, img, target): - """ - img (PIL Image or Tensor): Image to be adjusted. - """ - _brightness_factor = ((random.random() + 1.0) / 2.0) * self.brightness_factor - img = F.adjust_brightness(img, _brightness_factor) - return img, target - -def lighting_noise(image): - ''' - color channel swap in image - image: A PIL image - ''' - new_image = image - perms = ((0, 1, 2), (0, 2, 1), (1, 0, 2), - (1, 2, 0), (2, 0, 1), (2, 1, 0)) - swap = perms[random.randint(0, len(perms)- 1)] - new_image = F.to_tensor(new_image) - new_image = new_image[swap, :, :] - new_image = F.to_pil_image(new_image) - return new_image - -class LightingNoise: - def __init__(self) -> None: - pass - - def __call__(self, img, target): - return lighting_noise(img), target - - -def rotate(image, boxes, angle): - ''' - Rotate image and bounding box - image: A Pil image (w, h) - boxes: A tensors of dimensions (#objects, 4) - - Out: rotated image (w, h), rotated boxes - ''' - new_image = image.copy() - new_boxes = boxes.clone() - - #Rotate image, expand = True - w = image.width - h = image.height - cx = w/2 - cy = h/2 - new_image = new_image.rotate(angle, expand=True) - angle = np.radians(angle) - alpha = np.cos(angle) - beta = np.sin(angle) - #Get affine matrix - AffineMatrix = torch.tensor([[alpha, beta, (1-alpha)*cx - beta*cy], - [-beta, alpha, beta*cx + (1-alpha)*cy]]) - - #Rotation boxes - box_width = (boxes[:,2] - boxes[:,0]).reshape(-1,1) - box_height = (boxes[:,3] - boxes[:,1]).reshape(-1,1) - - #Get corners for boxes - x1 = boxes[:,0].reshape(-1,1) - y1 = boxes[:,1].reshape(-1,1) - - x2 = x1 + box_width - y2 = y1 - - x3 = x1 - y3 = y1 + box_height - - x4 = boxes[:,2].reshape(-1,1) - y4 = boxes[:,3].reshape(-1,1) - - corners = torch.stack((x1,y1,x2,y2,x3,y3,x4,y4), dim= 1) - # corners.reshape(-1, 8) #Tensors of dimensions (#objects, 8) - corners = corners.reshape(-1,2) #Tensors of dimension (4* #objects, 2) - corners = torch.cat((corners, torch.ones(corners.shape[0], 1)), dim= 1) #(Tensors of dimension (4* #objects, 3)) - - cos = np.abs(AffineMatrix[0, 0]) - sin = np.abs(AffineMatrix[0, 1]) - - nW = int((h * sin) + (w * cos)) - nH = int((h * cos) + (w * sin)) - AffineMatrix[0, 2] += (nW / 2) - cx - AffineMatrix[1, 2] += (nH / 2) - cy - - - #Apply affine transform - rotate_corners = torch.mm(AffineMatrix, corners.t().to(torch.float64)).t() - rotate_corners = rotate_corners.reshape(-1,8) - - x_corners = rotate_corners[:,[0,2,4,6]] - y_corners = rotate_corners[:,[1,3,5,7]] - - #Get (x_min, y_min, x_max, y_max) - x_min, _ = torch.min(x_corners, dim= 1) - x_min = x_min.reshape(-1, 1) - y_min, _ = torch.min(y_corners, dim= 1) - y_min = y_min.reshape(-1, 1) - x_max, _ = torch.max(x_corners, dim= 1) - x_max = x_max.reshape(-1, 1) - y_max, _ = torch.max(y_corners, dim= 1) - y_max = y_max.reshape(-1, 1) - - new_boxes = torch.cat((x_min, y_min, x_max, y_max), dim= 1) - - scale_x = new_image.width / w - scale_y = new_image.height / h - - #Resize new image to (w, h) - - new_image = new_image.resize((w, h)) - - #Resize boxes - new_boxes /= torch.Tensor([scale_x, scale_y, scale_x, scale_y]) - new_boxes[:, 0] = torch.clamp(new_boxes[:, 0], 0, w) - new_boxes[:, 1] = torch.clamp(new_boxes[:, 1], 0, h) - new_boxes[:, 2] = torch.clamp(new_boxes[:, 2], 0, w) - new_boxes[:, 3] = torch.clamp(new_boxes[:, 3], 0, h) - return new_image, new_boxes - -# def convert_xywh_to_xyxy(boxes: torch.Tensor): -# _boxes = boxes.clone() -# box_xy = _boxes[:, :2] -# box_wh = _boxes[:, 2:] -# box_x1y1 = box_xy - box_wh/2 -# box_x2y2 = box_xy + box_wh/2 -# box_xyxy = torch.cat((box_x1y1, box_x2y2), dim=-1) -# return box_xyxy - -class Rotate: - def __init__(self, angle=10) -> None: - self.angle = angle - - def __call__(self, img, target): - w,h = img.size - whwh = torch.Tensor([w, h, w, h]) - boxes_xyxy = box_cxcywh_to_xyxy(target['boxes']) * whwh - img, boxes_new = rotate(img, boxes_xyxy, self.angle) - target['boxes'] = box_xyxy_to_cxcywh(boxes_new).to(boxes_xyxy.dtype) / (whwh + 1e-3) - return img, target - - -class RandomCrop: - def __init__(self) -> None: - pass - - def __call__(self, img, target): - w,h = img.size - try: - boxes_xyxy = target['boxes'] - labels = target['labels'] - img, new_boxes, new_labels, _ = random_crop(img, boxes_xyxy, labels) - target['boxes'] = new_boxes - target['labels'] = new_labels - except Exception as e: - pass - return img, target - - -class RandomCropDebug: - def __init__(self) -> None: - pass - - def __call__(self, img, target): - boxes_xyxy = target['boxes'].clone() - labels = target['labels'].clone() - img, new_boxes, new_labels, _ = random_crop(img, boxes_xyxy, labels) - target['boxes'] = new_boxes - target['labels'] = new_labels - - - return img, target - -class RandomSelectMulti(object): - """ - Randomly selects between transforms1 and transforms2, - """ - def __init__(self, transformslist, p=-1): - self.transformslist = transformslist - self.p = p - assert p == -1 - - def __call__(self, img, target): - if self.p == -1: - return random.choice(self.transformslist)(img, target) - - -class Albumentations: - def __init__(self): - import albumentations as A - self.transform = A.Compose([ - A.Blur(p=0.01), - A.MedianBlur(p=0.01), - A.ToGray(p=0.01), - A.CLAHE(p=0.01), - A.RandomBrightnessContrast(p=0.005), - A.RandomGamma(p=0.005), - A.ImageCompression(quality_lower=75, p=0.005)], - bbox_params=A.BboxParams(format='pascal_voc', label_fields=['class_labels'])) - - def __call__(self, img, target, p=1.0): - """ - Input: - target['boxes']: xyxy, unnormalized data. - - """ - boxes_raw = target['boxes'] - labels_raw = target['labels'] - img_np = np.array(img) - if self.transform and random.random() < p: - new_res = self.transform(image=img_np, bboxes=boxes_raw, class_labels=labels_raw) # transformed - boxes_new = torch.Tensor(new_res['bboxes']).to(boxes_raw.dtype).reshape_as(boxes_raw) - img_np = new_res['image'] - labels_new = torch.Tensor(new_res['class_labels']).to(labels_raw.dtype) - img_new = Image.fromarray(img_np) - target['boxes'] = boxes_new - target['labels'] = labels_new - - return img_new, target \ No newline at end of file diff --git a/groundingdino/datasets/transforms.py b/groundingdino/datasets/transforms.py deleted file mode 100644 index ec3691e7140847167461998e2d2d62c70f5cde56..0000000000000000000000000000000000000000 --- a/groundingdino/datasets/transforms.py +++ /dev/null @@ -1,285 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -""" -Transforms and data augmentation for both image + bbox. -""" -import random - -import PIL -import torch -import torchvision.transforms as T -import torchvision.transforms.functional as F - -from util.box_ops import box_xyxy_to_cxcywh -from util.misc import interpolate - - -def crop(image, target, region): - cropped_image = F.crop(image, *region) - - target = target.copy() - i, j, h, w = region - - # should we do something wrt the original size? - target["size"] = torch.tensor([h, w]) - - fields = ["labels", "area"] - - if "boxes" in target: - boxes = target["boxes"] - max_size = torch.as_tensor([w, h], dtype=torch.float32) - cropped_boxes = boxes - torch.as_tensor([j, i, j, i]) - cropped_boxes = torch.min(cropped_boxes.reshape(-1, 2, 2), max_size) - cropped_boxes = cropped_boxes.clamp(min=0) - area = (cropped_boxes[:, 1, :] - cropped_boxes[:, 0, :]).prod(dim=1) - target["boxes"] = cropped_boxes.reshape(-1, 4) - target["area"] = area - fields.append("boxes") - - if "masks" in target: - # FIXME should we update the area here if there are no boxes? - target['masks'] = target['masks'][:, i:i + h, j:j + w] - fields.append("masks") - - - # remove elements for which the boxes or masks that have zero area - if "boxes" in target or "masks" in target: - # favor boxes selection when defining which elements to keep - # this is compatible with previous implementation - if "boxes" in target: - cropped_boxes = target['boxes'].reshape(-1, 2, 2) - keep = torch.all(cropped_boxes[:, 1, :] > cropped_boxes[:, 0, :], dim=1) - else: - keep = target['masks'].flatten(1).any(1) - - for field in fields: - target[field] = target[field][keep] - - return cropped_image, target - - -def hflip(image, target): - flipped_image = F.hflip(image) - - w, h = image.size - - target = target.copy() - if "boxes" in target: - boxes = target["boxes"] - boxes = boxes[:, [2, 1, 0, 3]] * torch.as_tensor([-1, 1, -1, 1]) + torch.as_tensor([w, 0, w, 0]) - target["boxes"] = boxes - - if "masks" in target: - target['masks'] = target['masks'].flip(-1) - - return flipped_image, target - - -def resize(image, target, size, max_size=None): - # size can be min_size (scalar) or (w, h) tuple - - def get_size_with_aspect_ratio(image_size, size, max_size=None): - w, h = image_size - if max_size is not None: - min_original_size = float(min((w, h))) - max_original_size = float(max((w, h))) - if max_original_size / min_original_size * size > max_size: - size = int(round(max_size * min_original_size / max_original_size)) - - if (w <= h and w == size) or (h <= w and h == size): - return (h, w) - - if w < h: - ow = size - oh = int(size * h / w) - else: - oh = size - ow = int(size * w / h) - - return (oh, ow) - - def get_size(image_size, size, max_size=None): - if isinstance(size, (list, tuple)): - return size[::-1] - else: - return get_size_with_aspect_ratio(image_size, size, max_size) - - size = get_size(image.size, size, max_size) - rescaled_image = F.resize(image, size) - - if target is None: - return rescaled_image, None - - ratios = tuple(float(s) / float(s_orig) for s, s_orig in zip(rescaled_image.size, image.size)) - ratio_width, ratio_height = ratios - - target = target.copy() - if "boxes" in target: - boxes = target["boxes"] - scaled_boxes = boxes * torch.as_tensor([ratio_width, ratio_height, ratio_width, ratio_height]) - target["boxes"] = scaled_boxes - - if "area" in target: - area = target["area"] - scaled_area = area * (ratio_width * ratio_height) - target["area"] = scaled_area - - h, w = size - target["size"] = torch.tensor([h, w]) - - if "masks" in target: - target['masks'] = interpolate( - target['masks'][:, None].float(), size, mode="nearest")[:, 0] > 0.5 - - return rescaled_image, target - - -def pad(image, target, padding): - # assumes that we only pad on the bottom right corners - padded_image = F.pad(image, (0, 0, padding[0], padding[1])) - if target is None: - return padded_image, None - target = target.copy() - # should we do something wrt the original size? - target["size"] = torch.tensor(padded_image.size[::-1]) - if "masks" in target: - target['masks'] = torch.nn.functional.pad(target['masks'], (0, padding[0], 0, padding[1])) - return padded_image, target - - -class ResizeDebug(object): - def __init__(self, size): - self.size = size - - def __call__(self, img, target): - return resize(img, target, self.size) - - -class RandomCrop(object): - def __init__(self, size): - self.size = size - - def __call__(self, img, target): - region = T.RandomCrop.get_params(img, self.size) - return crop(img, target, region) - - -class RandomSizeCrop(object): - def __init__(self, min_size: int, max_size: int): - self.min_size = min_size - self.max_size = max_size - - def __call__(self, img: PIL.Image.Image, target: dict): - w = random.randint(self.min_size, min(img.width, self.max_size)) - h = random.randint(self.min_size, min(img.height, self.max_size)) - region = T.RandomCrop.get_params(img, [h, w]) - return crop(img, target, region) - - -class CenterCrop(object): - def __init__(self, size): - self.size = size - - def __call__(self, img, target): - image_width, image_height = img.size - crop_height, crop_width = self.size - crop_top = int(round((image_height - crop_height) / 2.)) - crop_left = int(round((image_width - crop_width) / 2.)) - return crop(img, target, (crop_top, crop_left, crop_height, crop_width)) - - -class RandomHorizontalFlip(object): - def __init__(self, p=0.5): - self.p = p - - def __call__(self, img, target): - if random.random() < self.p: - return hflip(img, target) - return img, target - - -class RandomResize(object): - def __init__(self, sizes, max_size=None): - assert isinstance(sizes, (list, tuple)) - self.sizes = sizes - self.max_size = max_size - - def __call__(self, img, target=None): - size = random.choice(self.sizes) - return resize(img, target, size, self.max_size) - - -class RandomPad(object): - def __init__(self, max_pad): - self.max_pad = max_pad - - def __call__(self, img, target): - pad_x = random.randint(0, self.max_pad) - pad_y = random.randint(0, self.max_pad) - return pad(img, target, (pad_x, pad_y)) - - -class RandomSelect(object): - """ - Randomly selects between transforms1 and transforms2, - with probability p for transforms1 and (1 - p) for transforms2 - """ - def __init__(self, transforms1, transforms2, p=0.5): - self.transforms1 = transforms1 - self.transforms2 = transforms2 - self.p = p - - def __call__(self, img, target): - if random.random() < self.p: - return self.transforms1(img, target) - return self.transforms2(img, target) - - -class ToTensor(object): - def __call__(self, img, target): - return F.to_tensor(img), target - - -class RandomErasing(object): - - def __init__(self, *args, **kwargs): - self.eraser = T.RandomErasing(*args, **kwargs) - - def __call__(self, img, target): - return self.eraser(img), target - - -class Normalize(object): - def __init__(self, mean, std): - self.mean = mean - self.std = std - - def __call__(self, image, target=None): - image = F.normalize(image, mean=self.mean, std=self.std) - if target is None: - return image, None - target = target.copy() - h, w = image.shape[-2:] - if "boxes" in target: - boxes = target["boxes"] - boxes = box_xyxy_to_cxcywh(boxes) - boxes = boxes / torch.tensor([w, h, w, h], dtype=torch.float32) - target["boxes"] = boxes - return image, target - - -class Compose(object): - def __init__(self, transforms): - self.transforms = transforms - - def __call__(self, image, target): - for t in self.transforms: - image, target = t(image, target) - return image, target - - def __repr__(self): - format_string = self.__class__.__name__ + "(" - for t in self.transforms: - format_string += "\n" - format_string += " {0}".format(t) - format_string += "\n)" - return format_string diff --git a/groundingdino/models/GroundingDINO/.ipynb_checkpoints/bertwarper-checkpoint.py b/groundingdino/models/GroundingDINO/.ipynb_checkpoints/bertwarper-checkpoint.py deleted file mode 100644 index f0cf9779b270e1aead32845006f8b881fcba37ad..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/.ipynb_checkpoints/bertwarper-checkpoint.py +++ /dev/null @@ -1,273 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ - -import torch -import torch.nn.functional as F -import torch.utils.checkpoint as checkpoint -from torch import Tensor, nn -from torchvision.ops.boxes import nms -from transformers import BertConfig, BertModel, BertPreTrainedModel -from transformers.modeling_outputs import BaseModelOutputWithPoolingAndCrossAttentions - - -class BertModelWarper(nn.Module): - def __init__(self, bert_model): - super().__init__() - # self.bert = bert_modelc - - self.config = bert_model.config - self.embeddings = bert_model.embeddings - self.encoder = bert_model.encoder - self.pooler = bert_model.pooler - - self.get_extended_attention_mask = bert_model.get_extended_attention_mask - self.invert_attention_mask = bert_model.invert_attention_mask - self.get_head_mask = bert_model.get_head_mask - - def forward( - self, - input_ids=None, - attention_mask=None, - token_type_ids=None, - position_ids=None, - head_mask=None, - inputs_embeds=None, - encoder_hidden_states=None, - encoder_attention_mask=None, - past_key_values=None, - use_cache=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): - r""" - encoder_hidden_states (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. - encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - - If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` - (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)` - instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`. - use_cache (:obj:`bool`, `optional`): - If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up - decoding (see :obj:`past_key_values`). - """ - output_attentions = ( - output_attentions if output_attentions is not None else self.config.output_attentions - ) - output_hidden_states = ( - output_hidden_states - if output_hidden_states is not None - else self.config.output_hidden_states - ) - return_dict = return_dict if return_dict is not None else self.config.use_return_dict - - if self.config.is_decoder: - use_cache = use_cache if use_cache is not None else self.config.use_cache - else: - use_cache = False - - if input_ids is not None and inputs_embeds is not None: - raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") - elif input_ids is not None: - input_shape = input_ids.size() - batch_size, seq_length = input_shape - elif inputs_embeds is not None: - input_shape = inputs_embeds.size()[:-1] - batch_size, seq_length = input_shape - else: - raise ValueError("You have to specify either input_ids or inputs_embeds") - - device = input_ids.device if input_ids is not None else inputs_embeds.device - - # past_key_values_length - past_key_values_length = ( - past_key_values[0][0].shape[2] if past_key_values is not None else 0 - ) - - if attention_mask is None: - attention_mask = torch.ones( - ((batch_size, seq_length + past_key_values_length)), device=device - ) - if token_type_ids is None: - token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device) - - # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] - # ourselves in which case we just need to make it broadcastable to all heads. - extended_attention_mask: torch.Tensor = self.get_extended_attention_mask( - attention_mask, input_shape, device - ) - - # If a 2D or 3D attention mask is provided for the cross-attention - # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] - if self.config.is_decoder and encoder_hidden_states is not None: - encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size() - encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length) - if encoder_attention_mask is None: - encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device) - encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask) - else: - encoder_extended_attention_mask = None - # if os.environ.get('IPDB_SHILONG_DEBUG', None) == 'INFO': - # import ipdb; ipdb.set_trace() - - # Prepare head mask if needed - # 1.0 in head_mask indicate we keep the head - # attention_probs has shape bsz x n_heads x N x N - # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] - # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] - head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers) - - embedding_output = self.embeddings( - input_ids=input_ids, - position_ids=position_ids, - token_type_ids=token_type_ids, - inputs_embeds=inputs_embeds, - past_key_values_length=past_key_values_length, - ) - - encoder_outputs = self.encoder( - embedding_output, - attention_mask=extended_attention_mask, - head_mask=head_mask, - encoder_hidden_states=encoder_hidden_states, - encoder_attention_mask=encoder_extended_attention_mask, - past_key_values=past_key_values, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - ) - sequence_output = encoder_outputs[0] - pooled_output = self.pooler(sequence_output) if self.pooler is not None else None - - if not return_dict: - return (sequence_output, pooled_output) + encoder_outputs[1:] - - return BaseModelOutputWithPoolingAndCrossAttentions( - last_hidden_state=sequence_output, - pooler_output=pooled_output, - past_key_values=encoder_outputs.past_key_values, - hidden_states=encoder_outputs.hidden_states, - attentions=encoder_outputs.attentions, - cross_attentions=encoder_outputs.cross_attentions, - ) - - -class TextEncoderShell(nn.Module): - def __init__(self, text_encoder): - super().__init__() - self.text_encoder = text_encoder - self.config = self.text_encoder.config - - def forward(self, **kw): - # feed into text encoder - return self.text_encoder(**kw) - - -def generate_masks_with_special_tokens(tokenized, special_tokens_list, tokenizer): - """Generate attention mask between each pair of special tokens - Args: - input_ids (torch.Tensor): input ids. Shape: [bs, num_token] - special_tokens_mask (list): special tokens mask. - Returns: - torch.Tensor: attention mask between each special tokens. - """ - input_ids = tokenized["input_ids"] - bs, num_token = input_ids.shape - # special_tokens_mask: bs, num_token. 1 for special tokens. 0 for normal tokens - special_tokens_mask = torch.zeros((bs, num_token), device=input_ids.device).bool() - for special_token in special_tokens_list: - special_tokens_mask |= input_ids == special_token - - # idxs: each row is a list of indices of special tokens - idxs = torch.nonzero(special_tokens_mask) - - # generate attention mask and positional ids - attention_mask = ( - torch.eye(num_token, device=input_ids.device).bool().unsqueeze(0).repeat(bs, 1, 1) - ) - position_ids = torch.zeros((bs, num_token), device=input_ids.device) - previous_col = 0 - for i in range(idxs.shape[0]): - row, col = idxs[i] - if (col == 0) or (col == num_token - 1): - attention_mask[row, col, col] = True - position_ids[row, col] = 0 - else: - attention_mask[row, previous_col + 1 : col + 1, previous_col + 1 : col + 1] = True - position_ids[row, previous_col + 1 : col + 1] = torch.arange( - 0, col - previous_col, device=input_ids.device - ) - - previous_col = col - - # # padding mask - # padding_mask = tokenized['attention_mask'] - # attention_mask = attention_mask & padding_mask.unsqueeze(1).bool() & padding_mask.unsqueeze(2).bool() - - return attention_mask, position_ids.to(torch.long) - - -def generate_masks_with_special_tokens_and_transfer_map(tokenized, special_tokens_list, tokenizer): - """Generate attention mask between each pair of special tokens - Args: - input_ids (torch.Tensor): input ids. Shape: [bs, num_token] - special_tokens_mask (list): special tokens mask. - Returns: - torch.Tensor: attention mask between each special tokens. - """ - input_ids = tokenized["input_ids"] - bs, num_token = input_ids.shape - # special_tokens_mask: bs, num_token. 1 for special tokens. 0 for normal tokens - special_tokens_mask = torch.zeros((bs, num_token), device=input_ids.device).bool() - for special_token in special_tokens_list: - special_tokens_mask |= input_ids == special_token - - # idxs: each row is a list of indices of special tokens - idxs = torch.nonzero(special_tokens_mask) - - # generate attention mask and positional ids - attention_mask = ( - torch.eye(num_token, device=input_ids.device).bool().unsqueeze(0).repeat(bs, 1, 1) - ) - position_ids = torch.zeros((bs, num_token), device=input_ids.device) - cate_to_token_mask_list = [[] for _ in range(bs)] - previous_col = 0 - for i in range(idxs.shape[0]): - row, col = idxs[i] - if (col == 0) or (col == num_token - 1): - attention_mask[row, col, col] = True - position_ids[row, col] = 0 - else: - attention_mask[row, previous_col + 1 : col + 1, previous_col + 1 : col + 1] = True - position_ids[row, previous_col + 1 : col + 1] = torch.arange( - 0, col - previous_col, device=input_ids.device - ) - c2t_maski = torch.zeros((num_token), device=input_ids.device).bool() - c2t_maski[previous_col + 1 : col] = True - cate_to_token_mask_list[row].append(c2t_maski) - previous_col = col - - cate_to_token_mask_list = [ - torch.stack(cate_to_token_mask_listi, dim=0) - for cate_to_token_mask_listi in cate_to_token_mask_list - ] - - # # padding mask - # padding_mask = tokenized['attention_mask'] - # attention_mask = attention_mask & padding_mask.unsqueeze(1).bool() & padding_mask.unsqueeze(2).bool() - - return attention_mask, position_ids.to(torch.long), cate_to_token_mask_list diff --git a/groundingdino/models/GroundingDINO/.ipynb_checkpoints/fuse_modules-checkpoint.py b/groundingdino/models/GroundingDINO/.ipynb_checkpoints/fuse_modules-checkpoint.py deleted file mode 100644 index 559c4eb960e5ca1787c27815e9f75ed0d80106ea..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/.ipynb_checkpoints/fuse_modules-checkpoint.py +++ /dev/null @@ -1,298 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ - -import torch -import torch.nn as nn -import torch.nn.functional as F -from timm.models.layers import DropPath -import loralib as lora - -class FeatureResizer(nn.Module): - """ - This class takes as input a set of embeddings of dimension C1 and outputs a set of - embedding of dimension C2, after a linear transformation, dropout and normalization (LN). - """ - - def __init__(self, input_feat_size, output_feat_size, dropout, do_ln=True): - super().__init__() - self.do_ln = do_ln - r = 12 - # Object feature encoding - self.fc = lora.Linear(input_feat_size, output_feat_size,r=r, bias=True) - self.layer_norm = nn.LayerNorm(output_feat_size, eps=1e-12) - self.dropout = nn.Dropout(dropout) - - def forward(self, encoder_features): - x = self.fc(encoder_features) - if self.do_ln: - x = self.layer_norm(x) - output = self.dropout(x) - return output - - -def l1norm(X, dim, eps=1e-8): - """L1-normalize columns of X""" - norm = torch.abs(X).sum(dim=dim, keepdim=True) + eps - X = torch.div(X, norm) - return X - - -def l2norm(X, dim, eps=1e-8): - """L2-normalize columns of X""" - norm = torch.pow(X, 2).sum(dim=dim, keepdim=True).sqrt() + eps - X = torch.div(X, norm) - return X - - -def func_attention(query, context, smooth=1, raw_feature_norm="softmax", eps=1e-8): - """ - query: (n_context, queryL, d) - context: (n_context, sourceL, d) - """ - batch_size_q, queryL = query.size(0), query.size(1) - batch_size, sourceL = context.size(0), context.size(1) - - # Get attention - # --> (batch, d, queryL) - queryT = torch.transpose(query, 1, 2) - - # (batch, sourceL, d)(batch, d, queryL) - # --> (batch, sourceL, queryL) - attn = torch.bmm(context, queryT) - if raw_feature_norm == "softmax": - # --> (batch*sourceL, queryL) - attn = attn.view(batch_size * sourceL, queryL) - attn = nn.Softmax()(attn) - # --> (batch, sourceL, queryL) - attn = attn.view(batch_size, sourceL, queryL) - elif raw_feature_norm == "l2norm": - attn = l2norm(attn, 2) - elif raw_feature_norm == "clipped_l2norm": - attn = nn.LeakyReLU(0.1)(attn) - attn = l2norm(attn, 2) - else: - raise ValueError("unknown first norm type:", raw_feature_norm) - # --> (batch, queryL, sourceL) - attn = torch.transpose(attn, 1, 2).contiguous() - # --> (batch*queryL, sourceL) - attn = attn.view(batch_size * queryL, sourceL) - attn = nn.Softmax()(attn * smooth) - # --> (batch, queryL, sourceL) - attn = attn.view(batch_size, queryL, sourceL) - # --> (batch, sourceL, queryL) - attnT = torch.transpose(attn, 1, 2).contiguous() - - # --> (batch, d, sourceL) - contextT = torch.transpose(context, 1, 2) - # (batch x d x sourceL)(batch x sourceL x queryL) - # --> (batch, d, queryL) - weightedContext = torch.bmm(contextT, attnT) - # --> (batch, queryL, d) - weightedContext = torch.transpose(weightedContext, 1, 2) - - return weightedContext, attnT - - -class BiMultiHeadAttention(nn.Module): - def __init__(self, v_dim, l_dim, embed_dim, num_heads, dropout=0.1, cfg=None): - super(BiMultiHeadAttention, self).__init__() - - self.embed_dim = embed_dim - self.num_heads = num_heads - self.head_dim = embed_dim // num_heads - self.v_dim = v_dim - self.l_dim = l_dim - - assert ( - self.head_dim * self.num_heads == self.embed_dim - ), f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`: {self.num_heads})." - self.scale = self.head_dim ** (-0.5) - self.dropout = dropout - r = 12 - self.v_proj = lora.Linear(self.v_dim, self.embed_dim , r=r) - self.l_proj = lora.Linear(self.l_dim, self.embed_dim , r=r ) - self.values_v_proj = lora.Linear(self.v_dim, self.embed_dim , r=r ) - self.values_l_proj = lora.Linear(self.l_dim, self.embed_dim , r=r ) - - self.out_v_proj = lora.Linear(self.embed_dim, self.v_dim , r=r ) - self.out_l_proj = lora.Linear(self.embed_dim, self.l_dim , r=r ) - - self.stable_softmax_2d = True - self.clamp_min_for_underflow = True - self.clamp_max_for_overflow = True - - self._reset_parameters() - - def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): - return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() - - def _reset_parameters(self): - nn.init.xavier_uniform_(self.v_proj.weight) - self.v_proj.bias.data.fill_(0) - nn.init.xavier_uniform_(self.l_proj.weight) - self.l_proj.bias.data.fill_(0) - nn.init.xavier_uniform_(self.values_v_proj.weight) - self.values_v_proj.bias.data.fill_(0) - nn.init.xavier_uniform_(self.values_l_proj.weight) - self.values_l_proj.bias.data.fill_(0) - nn.init.xavier_uniform_(self.out_v_proj.weight) - self.out_v_proj.bias.data.fill_(0) - nn.init.xavier_uniform_(self.out_l_proj.weight) - self.out_l_proj.bias.data.fill_(0) - - def forward(self, v, l, attention_mask_v=None, attention_mask_l=None): - """_summary_ - - Args: - v (_type_): bs, n_img, dim - l (_type_): bs, n_text, dim - attention_mask_v (_type_, optional): _description_. bs, n_img - attention_mask_l (_type_, optional): _description_. bs, n_text - - Returns: - _type_: _description_ - """ - # if os.environ.get('IPDB_SHILONG_DEBUG', None) == 'INFO': - # import ipdb; ipdb.set_trace() - bsz, tgt_len, _ = v.size() - - query_states = self.v_proj(v) * self.scale - key_states = self._shape(self.l_proj(l), -1, bsz) - value_v_states = self._shape(self.values_v_proj(v), -1, bsz) - value_l_states = self._shape(self.values_l_proj(l), -1, bsz) - - proj_shape = (bsz * self.num_heads, -1, self.head_dim) - query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) - key_states = key_states.view(*proj_shape) - value_v_states = value_v_states.view(*proj_shape) - value_l_states = value_l_states.view(*proj_shape) - - src_len = key_states.size(1) - attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) # bs*nhead, nimg, ntxt - - if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): - raise ValueError( - f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is {attn_weights.size()}" - ) - - if self.stable_softmax_2d: - attn_weights = attn_weights - attn_weights.max() - - if self.clamp_min_for_underflow: - attn_weights = torch.clamp( - attn_weights, min=-50000 - ) # Do not increase -50000, data type half has quite limited range - if self.clamp_max_for_overflow: - attn_weights = torch.clamp( - attn_weights, max=50000 - ) # Do not increase 50000, data type half has quite limited range - - attn_weights_T = attn_weights.transpose(1, 2) - attn_weights_l = attn_weights_T - torch.max(attn_weights_T, dim=-1, keepdim=True)[0] - if self.clamp_min_for_underflow: - attn_weights_l = torch.clamp( - attn_weights_l, min=-50000 - ) # Do not increase -50000, data type half has quite limited range - if self.clamp_max_for_overflow: - attn_weights_l = torch.clamp( - attn_weights_l, max=50000 - ) # Do not increase 50000, data type half has quite limited range - - # mask vison for language - if attention_mask_v is not None: - attention_mask_v = ( - attention_mask_v[:, None, None, :].repeat(1, self.num_heads, 1, 1).flatten(0, 1) - ) - attn_weights_l.masked_fill_(attention_mask_v, float("-inf")) - - attn_weights_l = attn_weights_l.softmax(dim=-1) - - # mask language for vision - if attention_mask_l is not None: - attention_mask_l = ( - attention_mask_l[:, None, None, :].repeat(1, self.num_heads, 1, 1).flatten(0, 1) - ) - attn_weights.masked_fill_(attention_mask_l, float("-inf")) - attn_weights_v = attn_weights.softmax(dim=-1) - - attn_probs_v = F.dropout(attn_weights_v, p=self.dropout, training=self.training) - attn_probs_l = F.dropout(attn_weights_l, p=self.dropout, training=self.training) - - attn_output_v = torch.bmm(attn_probs_v, value_l_states) - attn_output_l = torch.bmm(attn_probs_l, value_v_states) - - if attn_output_v.size() != (bsz * self.num_heads, tgt_len, self.head_dim): - raise ValueError( - f"`attn_output_v` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is {attn_output_v.size()}" - ) - - if attn_output_l.size() != (bsz * self.num_heads, src_len, self.head_dim): - raise ValueError( - f"`attn_output_l` should be of size {(bsz, self.num_heads, src_len, self.head_dim)}, but is {attn_output_l.size()}" - ) - - attn_output_v = attn_output_v.view(bsz, self.num_heads, tgt_len, self.head_dim) - attn_output_v = attn_output_v.transpose(1, 2) - attn_output_v = attn_output_v.reshape(bsz, tgt_len, self.embed_dim) - - attn_output_l = attn_output_l.view(bsz, self.num_heads, src_len, self.head_dim) - attn_output_l = attn_output_l.transpose(1, 2) - attn_output_l = attn_output_l.reshape(bsz, src_len, self.embed_dim) - - attn_output_v = self.out_v_proj(attn_output_v) - attn_output_l = self.out_l_proj(attn_output_l) - - return attn_output_v, attn_output_l - - -# Bi-Direction MHA (text->image, image->text) -class BiAttentionBlock(nn.Module): - def __init__( - self, - v_dim, - l_dim, - embed_dim, - num_heads, - dropout=0.1, - drop_path=0.0, - init_values=1e-4, - cfg=None, - ): - """ - Inputs: - embed_dim - Dimensionality of input and attention feature vectors - hidden_dim - Dimensionality of hidden layer in feed-forward network - (usually 2-4x larger than embed_dim) - num_heads - Number of heads to use in the Multi-Head Attention block - dropout - Amount of dropout to apply in the feed-forward network - """ - super(BiAttentionBlock, self).__init__() - - # pre layer norm - self.layer_norm_v = nn.LayerNorm(v_dim) - self.layer_norm_l = nn.LayerNorm(l_dim) - self.attn = BiMultiHeadAttention( - v_dim=v_dim, l_dim=l_dim, embed_dim=embed_dim, num_heads=num_heads, dropout=dropout - ) - - # add layer scale for training stability - self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() - self.gamma_v = nn.Parameter(init_values * torch.ones((v_dim)), requires_grad=True) - self.gamma_l = nn.Parameter(init_values * torch.ones((l_dim)), requires_grad=True) - - def forward(self, v, l, attention_mask_v=None, attention_mask_l=None): - v = self.layer_norm_v(v) - l = self.layer_norm_l(l) - delta_v, delta_l = self.attn( - v, l, attention_mask_v=attention_mask_v, attention_mask_l=attention_mask_l - ) - # v, l = v + delta_v, l + delta_l - v = v + self.drop_path(self.gamma_v * delta_v) - l = l + self.drop_path(self.gamma_l * delta_l) - return v, l - - # def forward(self, v:List[torch.Tensor], l, attention_mask_v=None, attention_mask_l=None) diff --git a/groundingdino/models/GroundingDINO/.ipynb_checkpoints/groundingdino-checkpoint.py b/groundingdino/models/GroundingDINO/.ipynb_checkpoints/groundingdino-checkpoint.py deleted file mode 100644 index df190bcfd6a4b86caaee4642022dfbb997cfd479..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/.ipynb_checkpoints/groundingdino-checkpoint.py +++ /dev/null @@ -1,857 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Conditional DETR model and criterion classes. -# Copyright (c) 2021 Microsoft. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Modified from DETR (https://github.com/facebookresearch/detr) -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -# ------------------------------------------------------------------------ -# Modified from Deformable DETR (https://github.com/fundamentalvision/Deformable-DETR) -# Copyright (c) 2020 SenseTime. All Rights Reserved. -# ------------------------------------------------------------------------ -import copy -from typing import List - -import torch -import torch.nn.functional as F -from torch import nn -from torchvision.ops.boxes import nms -from transformers import AutoTokenizer, BertModel, BertTokenizer, RobertaModel, RobertaTokenizerFast - -from groundingdino.util import box_ops, get_tokenlizer -from groundingdino.util.misc import ( - NestedTensor, - accuracy, - get_world_size, - interpolate, - inverse_sigmoid, - is_dist_avail_and_initialized, - nested_tensor_from_tensor_list, -) -from groundingdino.util.utils import get_phrases_from_posmap -from groundingdino.util.visualizer import COCOVisualizer -from groundingdino.util.vl_utils import create_positive_map_from_span - -from ..registry import MODULE_BUILD_FUNCS -from .backbone import build_backbone -from .bertwarper import ( - BertModelWarper, - generate_masks_with_special_tokens, - generate_masks_with_special_tokens_and_transfer_map, -) -from .transformer import build_transformer -from .utils import MLP, ContrastiveEmbed, sigmoid_focal_loss - -from .matcher import build_matcher - - - - -class GroundingDINO(nn.Module): - """This is the Cross-Attention Detector module that performs object detection""" - - def __init__( - self, - backbone, - transformer, - num_queries, - aux_loss=False, - iter_update=False, - query_dim=2, - num_feature_levels=1, - nheads=8, - # two stage - two_stage_type="no", # ['no', 'standard'] - dec_pred_bbox_embed_share=True, - two_stage_class_embed_share=True, - two_stage_bbox_embed_share=True, - num_patterns=0, - dn_number=100, - dn_box_noise_scale=0.4, - dn_label_noise_ratio=0.5, - dn_labelbook_size=100, - text_encoder_type="bert-base-uncased", - sub_sentence_present=True, - max_text_len=256, - ): - """Initializes the model. - Parameters: - backbone: torch module of the backbone to be used. See backbone.py - transformer: torch module of the transformer architecture. See transformer.py - num_queries: number of object queries, ie detection slot. This is the maximal number of objects - Conditional DETR can detect in a single image. For COCO, we recommend 100 queries. - aux_loss: True if auxiliary decoding losses (loss at each decoder layer) are to be used. - """ - super().__init__() - self.num_queries = num_queries - self.transformer = transformer - self.hidden_dim = hidden_dim = transformer.d_model - self.num_feature_levels = num_feature_levels - self.nheads = nheads - self.max_text_len = 256 - self.sub_sentence_present = sub_sentence_present - - # setting query dim - self.query_dim = query_dim - assert query_dim == 4 - - # for dn training - self.num_patterns = num_patterns - self.dn_number = dn_number - self.dn_box_noise_scale = dn_box_noise_scale - self.dn_label_noise_ratio = dn_label_noise_ratio - self.dn_labelbook_size = dn_labelbook_size - - # bert - self.tokenizer = get_tokenlizer.get_tokenlizer(text_encoder_type) - self.bert = get_tokenlizer.get_pretrained_language_model(text_encoder_type) - self.bert.pooler.dense.weight.requires_grad_(False) - self.bert.pooler.dense.bias.requires_grad_(False) - self.bert = BertModelWarper(bert_model=self.bert) - - self.feat_map = nn.Linear(self.bert.config.hidden_size, self.hidden_dim, bias=True) - nn.init.constant_(self.feat_map.bias.data, 0) - nn.init.xavier_uniform_(self.feat_map.weight.data) - # freeze - - # special tokens - self.specical_tokens = self.tokenizer.convert_tokens_to_ids(["[CLS]", "[SEP]", ".", "?"]) - - # prepare input projection layers - if num_feature_levels > 1: - num_backbone_outs = len(backbone.num_channels) - input_proj_list = [] - for _ in range(num_backbone_outs): - in_channels = backbone.num_channels[_] - input_proj_list.append( - nn.Sequential( - nn.Conv2d(in_channels, hidden_dim, kernel_size=1), - nn.GroupNorm(32, hidden_dim), - ) - ) - for _ in range(num_feature_levels - num_backbone_outs): - input_proj_list.append( - nn.Sequential( - nn.Conv2d(in_channels, hidden_dim, kernel_size=3, stride=2, padding=1), - nn.GroupNorm(32, hidden_dim), - ) - ) - in_channels = hidden_dim - self.input_proj = nn.ModuleList(input_proj_list) - else: - assert two_stage_type == "no", "two_stage_type should be no if num_feature_levels=1 !!!" - self.input_proj = nn.ModuleList( - [ - nn.Sequential( - nn.Conv2d(backbone.num_channels[-1], hidden_dim, kernel_size=1), - nn.GroupNorm(32, hidden_dim), - ) - ] - ) - - self.backbone = backbone - self.aux_loss = aux_loss - self.box_pred_damping = box_pred_damping = None - - self.iter_update = iter_update - assert iter_update, "Why not iter_update?" - - # prepare pred layers - self.dec_pred_bbox_embed_share = dec_pred_bbox_embed_share - # prepare class & box embed - _class_embed = ContrastiveEmbed() - - _bbox_embed = MLP(hidden_dim, hidden_dim, 4, 3) - nn.init.constant_(_bbox_embed.layers[-1].weight.data, 0) - nn.init.constant_(_bbox_embed.layers[-1].bias.data, 0) - - if dec_pred_bbox_embed_share: - box_embed_layerlist = [_bbox_embed for i in range(transformer.num_decoder_layers)] - else: - box_embed_layerlist = [ - copy.deepcopy(_bbox_embed) for i in range(transformer.num_decoder_layers) - ] - class_embed_layerlist = [_class_embed for i in range(transformer.num_decoder_layers)] - self.bbox_embed = nn.ModuleList(box_embed_layerlist) - self.class_embed = nn.ModuleList(class_embed_layerlist) - self.transformer.decoder.bbox_embed = self.bbox_embed - self.transformer.decoder.class_embed = self.class_embed - - # two stage - self.two_stage_type = two_stage_type - assert two_stage_type in ["no", "standard"], "unknown param {} of two_stage_type".format( - two_stage_type - ) - if two_stage_type != "no": - if two_stage_bbox_embed_share: - assert dec_pred_bbox_embed_share - self.transformer.enc_out_bbox_embed = _bbox_embed - else: - self.transformer.enc_out_bbox_embed = copy.deepcopy(_bbox_embed) - - if two_stage_class_embed_share: - assert dec_pred_bbox_embed_share - self.transformer.enc_out_class_embed = _class_embed - else: - self.transformer.enc_out_class_embed = copy.deepcopy(_class_embed) - - self.refpoint_embed = None - - self._reset_parameters() - - def _reset_parameters(self): - # init input_proj - for proj in self.input_proj: - nn.init.xavier_uniform_(proj[0].weight, gain=1) - nn.init.constant_(proj[0].bias, 0) - - def init_ref_points(self, use_num_queries): - self.refpoint_embed = nn.Embedding(use_num_queries, self.query_dim) - - def forward(self, samples: NestedTensor, targets: List = None, **kw): - """The forward expects a NestedTensor, which consists of: - - samples.tensor: batched images, of shape [batch_size x 3 x H x W] - - samples.mask: a binary mask of shape [batch_size x H x W], containing 1 on padded pixels - - It returns a dict with the following elements: - - "pred_logits": the classification logits (including no-object) for all queries. - Shape= [batch_size x num_queries x num_classes] - - "pred_boxes": The normalized boxes coordinates for all queries, represented as - (center_x, center_y, width, height). These values are normalized in [0, 1], - relative to the size of each individual image (disregarding possible padding). - See PostProcess for information on how to retrieve the unnormalized bounding box. - - "aux_outputs": Optional, only returned when auxilary losses are activated. It is a list of - dictionnaries containing the two above keys for each decoder layer. - """ - if targets is None: - captions = kw["captions"] - else: - captions = [t["caption"] for t in targets] - # encoder texts - - tokenized = self.tokenizer(captions, padding="longest", return_tensors="pt").to( - samples.device - ) - one_hot_token = tokenized - - ( - text_self_attention_masks, - position_ids, - cate_to_token_mask_list, - ) = generate_masks_with_special_tokens_and_transfer_map( - tokenized, self.specical_tokens, self.tokenizer - ) - - if text_self_attention_masks.shape[1] > self.max_text_len: - text_self_attention_masks = text_self_attention_masks[ - :, : self.max_text_len, : self.max_text_len - ] - position_ids = position_ids[:, : self.max_text_len] - tokenized["input_ids"] = tokenized["input_ids"][:, : self.max_text_len] - tokenized["attention_mask"] = tokenized["attention_mask"][:, : self.max_text_len] - tokenized["token_type_ids"] = tokenized["token_type_ids"][:, : self.max_text_len] - - # extract text embeddings - if self.sub_sentence_present: - tokenized_for_encoder = {k: v for k, v in tokenized.items() if k != "attention_mask"} - tokenized_for_encoder["attention_mask"] = text_self_attention_masks - tokenized_for_encoder["position_ids"] = position_ids - else: - tokenized_for_encoder = tokenized - - bert_output = self.bert(**tokenized_for_encoder) # bs, 195, 768 - - encoded_text = self.feat_map(bert_output["last_hidden_state"]) # bs, 195, d_model - text_token_mask = tokenized.attention_mask.bool() # bs, 195 - # text_token_mask: True for nomask, False for mask - # text_self_attention_masks: True for nomask, False for mask - - if encoded_text.shape[1] > self.max_text_len: - encoded_text = encoded_text[:, : self.max_text_len, :] - text_token_mask = text_token_mask[:, : self.max_text_len] - position_ids = position_ids[:, : self.max_text_len] - text_self_attention_masks = text_self_attention_masks[ - :, : self.max_text_len, : self.max_text_len - ] - - text_dict = { - "encoded_text": encoded_text, # bs, 195, d_model - "text_token_mask": text_token_mask, # bs, 195 - "position_ids": position_ids, # bs, 195 - "text_self_attention_masks": text_self_attention_masks, # bs, 195,195 - } - - - if isinstance(samples, (list, torch.Tensor)): - samples = nested_tensor_from_tensor_list(samples) - features, poss = self.backbone(samples) - srcs = [] - masks = [] - for l, feat in enumerate(features): - src, mask = feat.decompose() - srcs.append(self.input_proj[l](src)) - masks.append(mask) - assert mask is not None - if self.num_feature_levels > len(srcs): - _len_srcs = len(srcs) - for l in range(_len_srcs, self.num_feature_levels): - if l == _len_srcs: - src = self.input_proj[l](features[-1].tensors) - else: - src = self.input_proj[l](srcs[-1]) - m = samples.mask - mask = F.interpolate(m[None].float(), size=src.shape[-2:]).to(torch.bool)[0] - pos_l = self.backbone[1](NestedTensor(src, mask)).to(src.dtype) - srcs.append(src) - masks.append(mask) - poss.append(pos_l) - - input_query_bbox = input_query_label = attn_mask = dn_meta = None - hs, reference, hs_enc, ref_enc, init_box_proposal = self.transformer( - srcs, masks, input_query_bbox, poss, input_query_label, attn_mask, text_dict - ) - - - # deformable-detr-like anchor update - outputs_coord_list = [] - for dec_lid, (layer_ref_sig, layer_bbox_embed, layer_hs) in enumerate( - zip(reference[:-1], self.bbox_embed, hs) - ): - layer_delta_unsig = layer_bbox_embed(layer_hs) - layer_outputs_unsig = layer_delta_unsig + inverse_sigmoid(layer_ref_sig) - layer_outputs_unsig = layer_outputs_unsig.sigmoid() - outputs_coord_list.append(layer_outputs_unsig) - outputs_coord_list = torch.stack(outputs_coord_list) - - - outputs_class = torch.stack( - [ - layer_cls_embed(layer_hs, text_dict) - for layer_cls_embed, layer_hs in zip(self.class_embed, hs) - ] - ) - - out = {"pred_logits": outputs_class[-1], "pred_boxes": outputs_coord_list[-1]} - - # Used to calculate losses - bs, len_td = text_dict['text_token_mask'].shape - out['text_mask']=torch.zeros(bs, self.max_text_len, dtype=torch.bool).to( - samples.device - ) - for b in range(bs): - for j in range(len_td): - if text_dict['text_token_mask'][b][j] == True: - out['text_mask'][b][j] = True - - # for intermediate outputs - if self.aux_loss: - out['aux_outputs'] = self._set_aux_loss(outputs_class, outputs_coord_list) - out['token']=one_hot_token - # # for encoder output - if hs_enc is not None: - # prepare intermediate outputs - interm_coord = ref_enc[-1] - interm_class = self.transformer.enc_out_class_embed(hs_enc[-1], text_dict) - out['interm_outputs'] = {'pred_logits': interm_class, 'pred_boxes': interm_coord} - out['interm_outputs_for_matching_pre'] = {'pred_logits': interm_class, 'pred_boxes': init_box_proposal} - - # outputs['pred_logits'].shape - # torch.Size([4, 900, 256]) - - # outputs['pred_boxes'].shape - # torch.Size([4, 900, 4]) - - # outputs['text_mask'].shape - # torch.Size([256]) - - # outputs['text_mask'] - - # outputs['aux_outputs'][0].keys() - # dict_keys(['pred_logits', 'pred_boxes', 'one_hot', 'text_mask']) - - # outputs['aux_outputs'][img_idx] - - # outputs['token'] - # - - # outputs['interm_outputs'].keys() - # dict_keys(['pred_logits', 'pred_boxes', 'one_hot', 'text_mask']) - - - # outputs['interm_outputs_for_matching_pre'].keys() - # dict_keys(['pred_logits', 'pred_boxes']) - - # outputs['one_hot'].shape - # torch.Size([4, 900, 256]) - - return out - - @torch.jit.unused - def _set_aux_loss(self, outputs_class, outputs_coord): - # this is a workaround to make torchscript happy, as torchscript - # doesn't support dictionary with non-homogeneous values, such - # as a dict having both a Tensor and a list. - return [ - {"pred_logits": a, "pred_boxes": b} - for a, b in zip(outputs_class[:-1], outputs_coord[:-1]) - ] - - - - -class SetCriterion(nn.Module): - def __init__(self, matcher, weight_dict, focal_alpha,focal_gamma, losses): - """ Create the criterion. - Parameters: - matcher: module able to compute a matching between targets and proposals - weight_dict: dict containing as key the names of the losses and as values their relative weight. - losses: list of all the losses to be applied. See get_loss for list of available losses. - focal_alpha: alpha in Focal Loss - """ - super().__init__() - self.matcher = matcher - self.weight_dict = weight_dict - self.losses = losses - self.focal_alpha = focal_alpha - self.focal_gamma= focal_gamma - - @torch.no_grad() - def loss_cardinality(self, outputs, targets, indices, num_boxes): - """ Compute the cardinality error, ie the absolute error in the number of predicted non-empty boxes - This is not really a loss, it is intended for logging purposes only. It doesn't propagate gradients - """ - - pred_logits = outputs['pred_logits'] - device = pred_logits.device - tgt_lengths = torch.as_tensor([len(v["labels"]) for v in targets], device=device) - # Count the number of predictions that are NOT "no-object" (which is the last class) - card_pred = (pred_logits.argmax(-1) != pred_logits.shape[-1] - 1).sum(1) - card_err = F.l1_loss(card_pred.float(), tgt_lengths.float()) - losses = {'cardinality_error': card_err} - return losses - - def loss_boxes(self, outputs, targets, indices, num_boxes): - """Compute the losses related to the bounding boxes, the L1 regression loss and the GIoU loss - targets dicts must contain the key "boxes" containing a tensor of dim [nb_target_boxes, 4] - The target boxes are expected in format (center_x, center_y, w, h), normalized by the image size. - """ - assert 'pred_boxes' in outputs - idx = self._get_src_permutation_idx(indices) - src_boxes = outputs['pred_boxes'][idx] - target_boxes = torch.cat([t['boxes'][i] for t, (_, i) in zip(targets, indices)], dim=0) - - loss_bbox = F.l1_loss(src_boxes, target_boxes, reduction='none') - - losses = {} - losses['loss_bbox'] = loss_bbox.sum() / num_boxes - - loss_giou = 1 - torch.diag(box_ops.generalized_box_iou( - box_ops.box_cxcywh_to_xyxy(src_boxes), - box_ops.box_cxcywh_to_xyxy(target_boxes))) - losses['loss_giou'] = loss_giou.sum() / num_boxes - - # calculate the x,y and h,w loss - with torch.no_grad(): - losses['loss_xy'] = loss_bbox[..., :2].sum() / num_boxes - losses['loss_hw'] = loss_bbox[..., 2:].sum() / num_boxes - - - return losses - - - def token_sigmoid_binary_focal_loss(self, outputs, targets, indices, num_boxes): - pred_logits=outputs['pred_logits'] - new_targets=outputs['one_hot'].to(pred_logits.device) - text_mask=outputs['text_mask'] - - assert (new_targets.dim() == 3) - assert (pred_logits.dim() == 3) # batch x from x to - - bs, n, _ = pred_logits.shape - alpha=self.focal_alpha - gamma=self.focal_gamma - if text_mask is not None: - # ODVG: each sample has different mask - text_mask = text_mask.repeat(1, pred_logits.size(1)).view(outputs['text_mask'].shape[0],-1,outputs['text_mask'].shape[1]) - pred_logits = torch.masked_select(pred_logits, text_mask) - new_targets = torch.masked_select(new_targets, text_mask) - - new_targets=new_targets.float() - p = torch.sigmoid(pred_logits) - ce_loss = F.binary_cross_entropy_with_logits(pred_logits, new_targets, reduction="none") - p_t = p * new_targets + (1 - p) * (1 - new_targets) - loss = ce_loss * ((1 - p_t) ** gamma) - - if alpha >= 0: - alpha_t = alpha * new_targets + (1 - alpha) * (1 - new_targets) - loss = alpha_t * loss - - total_num_pos=0 - for batch_indices in indices: - total_num_pos += len(batch_indices[0]) - num_pos_avg_per_gpu = max(total_num_pos , 1.0) - loss=loss.sum()/num_pos_avg_per_gpu - - losses = {'loss_ce': loss} - return losses - - - def _get_src_permutation_idx(self, indices): - # permute predictions following indices - batch_idx = torch.cat([torch.full_like(src, i) for i, (src, _) in enumerate(indices)]) - src_idx = torch.cat([src for (src, _) in indices]) - return batch_idx, src_idx - - def _get_tgt_permutation_idx(self, indices): - # permute targets following indices - batch_idx = torch.cat([torch.full_like(tgt, i) for i, (_, tgt) in enumerate(indices)]) - tgt_idx = torch.cat([tgt for (_, tgt) in indices]) - return batch_idx, tgt_idx - - def get_loss(self, loss, outputs, targets, indices, num_boxes, **kwargs): - loss_map = { - 'labels': self.token_sigmoid_binary_focal_loss, - 'cardinality': self.loss_cardinality, - 'boxes': self.loss_boxes, - } - assert loss in loss_map, f'do you really want to compute {loss} loss?' - return loss_map[loss](outputs, targets, indices, num_boxes, **kwargs) - - def forward(self, outputs, targets, cat_list, caption, return_indices=False): - """ This performs the loss computation. - Parameters: - outputs: dict of tensors, see the output specification of the model for the format - targets: list of dicts, such that len(targets) == batch_size. - The expected keys in each dict depends on the losses applied, see each loss' doc - - return_indices: used for vis. if True, the layer0-5 indices will be returned as well. - """ - device=next(iter(outputs.values())).device - one_hot = torch.zeros(outputs['pred_logits'].size(),dtype=torch.int64) # torch.Size([bs, 900, 256]) - token = outputs['token'] - - label_map_list = [] - indices = [] - for j in range(len(cat_list)): # bs - label_map=[] - for i in range(len(cat_list[j])): - label_id=torch.tensor([i]) - per_label=create_positive_map(token[j], label_id, cat_list[j], caption[j]) - label_map.append(per_label) - label_map=torch.stack(label_map,dim=0).squeeze(1) - label_map_list.append(label_map) - for j in range(len(cat_list)): # bs - for_match = { - "pred_logits" : outputs['pred_logits'][j].unsqueeze(0), - "pred_boxes" : outputs['pred_boxes'][j].unsqueeze(0) - } - inds = self.matcher(for_match, [targets[j]], label_map_list[j]) - indices.extend(inds) - # indices : A list of size batch_size, containing tuples of (index_i, index_j) where: - # - index_i is the indices of the selected predictions (in order) - # - index_j is the indices of the corresponding selected targets (in order) - - # import pdb; pdb.set_trace() - tgt_ids = [v["labels"].cpu() for v in targets] - # len(tgt_ids) == bs - for i in range(len(indices)): - tgt_ids[i]=tgt_ids[i][indices[i][1]] - one_hot[i,indices[i][0]] = label_map_list[i][tgt_ids[i]].to(torch.long) - outputs['one_hot'] = one_hot - if return_indices: - indices0_copy = indices - indices_list = [] - - # Compute the average number of target boxes accross all nodes, for normalization purposes - num_boxes_list = [len(t["labels"]) for t in targets] - num_boxes = sum(num_boxes_list) - num_boxes = torch.as_tensor([num_boxes], dtype=torch.float, device=device) - if is_dist_avail_and_initialized(): - torch.distributed.all_reduce(num_boxes) - num_boxes = torch.clamp(num_boxes / get_world_size(), min=1).item() - - # Compute all the requested losses - losses = {} - for loss in self.losses: - losses.update(self.get_loss(loss, outputs, targets, indices, num_boxes)) - - # In case of auxiliary losses, we repeat this process with the output of each intermediate layer. - if 'aux_outputs' in outputs: - for idx, aux_outputs in enumerate(outputs['aux_outputs']): - indices = [] - for j in range(len(cat_list)): # bs - aux_output_single = { - 'pred_logits' : aux_outputs['pred_logits'][j].unsqueeze(0), - 'pred_boxes': aux_outputs['pred_boxes'][j].unsqueeze(0) - } - inds = self.matcher(aux_output_single, [targets[j]], label_map_list[j]) - indices.extend(inds) - one_hot_aux = torch.zeros(outputs['pred_logits'].size(),dtype=torch.int64) - tgt_ids = [v["labels"].cpu() for v in targets] - for i in range(len(indices)): - tgt_ids[i]=tgt_ids[i][indices[i][1]] - one_hot_aux[i,indices[i][0]] = label_map_list[i][tgt_ids[i]].to(torch.long) - aux_outputs['one_hot'] = one_hot_aux - aux_outputs['text_mask'] = outputs['text_mask'] - if return_indices: - indices_list.append(indices) - for loss in self.losses: - kwargs = {} - l_dict = self.get_loss(loss, aux_outputs, targets, indices, num_boxes, **kwargs) - l_dict = {k + f'_{idx}': v for k, v in l_dict.items()} - losses.update(l_dict) - - # interm_outputs loss - if 'interm_outputs' in outputs: - interm_outputs = outputs['interm_outputs'] - indices = [] - for j in range(len(cat_list)): # bs - interm_output_single = { - 'pred_logits' : interm_outputs['pred_logits'][j].unsqueeze(0), - 'pred_boxes': interm_outputs['pred_boxes'][j].unsqueeze(0) - } - inds = self.matcher(interm_output_single, [targets[j]], label_map_list[j]) - indices.extend(inds) - one_hot_aux = torch.zeros(outputs['pred_logits'].size(),dtype=torch.int64) - tgt_ids = [v["labels"].cpu() for v in targets] - for i in range(len(indices)): - tgt_ids[i]=tgt_ids[i][indices[i][1]] - one_hot_aux[i,indices[i][0]] = label_map_list[i][tgt_ids[i]].to(torch.long) - interm_outputs['one_hot'] = one_hot_aux - interm_outputs['text_mask'] = outputs['text_mask'] - if return_indices: - indices_list.append(indices) - for loss in self.losses: - kwargs = {} - l_dict = self.get_loss(loss, interm_outputs, targets, indices, num_boxes, **kwargs) - l_dict = {k + f'_interm': v for k, v in l_dict.items()} - losses.update(l_dict) - - if return_indices: - indices_list.append(indices0_copy) - return losses, indices_list - - return losses - - -class PostProcess(nn.Module): - """ This module converts the model's output into the format expected by the coco api""" - def __init__(self, num_select=100,text_encoder_type='text_encoder_type', nms_iou_threshold=-1,use_coco_eval=False,args=None) -> None: - super().__init__() - self.num_select = num_select - self.tokenizer = get_tokenlizer.get_tokenlizer(text_encoder_type) - if args.use_coco_eval: - from pycocotools.coco import COCO - coco = COCO(args.coco_val_path) - category_dict = coco.loadCats(coco.getCatIds()) - cat_list = [item['name'] for item in category_dict] - else: - cat_list=args.label_list - caption = " . ".join(cat_list) + ' .' - tokenized = self.tokenizer(caption, padding="longest", return_tensors="pt") - label_list = torch.arange(len(cat_list)) - pos_map=create_positive_map(tokenized,label_list,cat_list,caption) - # build a mapping from label_id to pos_map - if args.use_coco_eval: - id_map = {0: 1, 1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: 7, 7: 8, 8: 9, 9: 10, 10: 11, 11: 13, 12: 14, 13: 15, 14: 16, 15: 17, 16: 18, 17: 19, 18: 20, 19: 21, 20: 22, 21: 23, 22: 24, 23: 25, 24: 27, 25: 28, 26: 31, 27: 32, 28: 33, 29: 34, 30: 35, 31: 36, 32: 37, 33: 38, 34: 39, 35: 40, 36: 41, 37: 42, 38: 43, 39: 44, 40: 46, - 41: 47, 42: 48, 43: 49, 44: 50, 45: 51, 46: 52, 47: 53, 48: 54, 49: 55, 50: 56, 51: 57, 52: 58, 53: 59, 54: 60, 55: 61, 56: 62, 57: 63, 58: 64, 59: 65, 60: 67, 61: 70, 62: 72, 63: 73, 64: 74, 65: 75, 66: 76, 67: 77, 68: 78, 69: 79, 70: 80, 71: 81, 72: 82, 73: 84, 74: 85, 75: 86, 76: 87, 77: 88, 78: 89, 79: 90} - new_pos_map = torch.zeros((91, 256)) - for k, v in id_map.items(): - new_pos_map[v] = pos_map[k] - pos_map=new_pos_map - - - self.nms_iou_threshold=nms_iou_threshold - self.positive_map = pos_map - - @torch.no_grad() - def forward(self, outputs, target_sizes, not_to_xyxy=False, test=False): - """ Perform the computation - Parameters: - outputs: raw outputs of the model - target_sizes: tensor of dimension [batch_size x 2] containing the size of each images of the batch - For evaluation, this must be the original image size (before any data augmentation) - For visualization, this should be the image size after data augment, but before padding - """ - num_select = self.num_select - out_logits, out_bbox = outputs['pred_logits'], outputs['pred_boxes'] - - - prob_to_token = out_logits.sigmoid() - pos_maps = self.positive_map.to(prob_to_token.device) - for label_ind in range(len(pos_maps)): - if pos_maps[label_ind].sum() != 0: - pos_maps[label_ind]=pos_maps[label_ind]/pos_maps[label_ind].sum() - - prob_to_label = prob_to_token @ pos_maps.T - - assert len(out_logits) == len(target_sizes) - assert target_sizes.shape[1] == 2 - - prob = prob_to_label - topk_values, topk_indexes = torch.topk(prob.view(prob.shape[0], -1), num_select, dim=1) - scores = topk_values - topk_boxes = torch.div(topk_indexes, prob.shape[2], rounding_mode='trunc') - labels = topk_indexes % prob.shape[2] - if not_to_xyxy: - boxes = out_bbox - else: - boxes = box_ops.box_cxcywh_to_xyxy(out_bbox) - - # if test: - # assert not not_to_xyxy - # boxes[:,:,2:] = boxes[:,:,2:] - boxes[:,:,:2] - boxes = torch.gather(boxes, 1, topk_boxes.unsqueeze(-1).repeat(1,1,4)) - - # and from relative [0, 1] to absolute [0, height] coordinates - img_h, img_w = target_sizes.unbind(1) - scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1) - boxes = boxes * scale_fct[:, None, :] - - if self.nms_iou_threshold > 0: - item_indices = [nms(b, s, iou_threshold=self.nms_iou_threshold) for b,s in zip(boxes, scores)] - - results = [{'scores': s[i], 'labels': l[i], 'boxes': b[i]} for s, l, b, i in zip(scores, labels, boxes, item_indices)] - else: - results = [{'scores': s, 'labels': l, 'boxes': b} for s, l, b in zip(scores, labels, boxes)] - results = [{'scores': s, 'labels': l, 'boxes': b} for s, l, b in zip(scores, labels, boxes)] - return results - - -@MODULE_BUILD_FUNCS.registe_with_name(module_name="groundingdino") -def build_groundingdino(args): - device = torch.device(args.device) - backbone = build_backbone(args) - transformer = build_transformer(args) - - dn_labelbook_size = args.dn_labelbook_size - dec_pred_bbox_embed_share = args.dec_pred_bbox_embed_share - sub_sentence_present = args.sub_sentence_present - - model = GroundingDINO( - backbone, - transformer, - num_queries=args.num_queries, - aux_loss=args.aux_loss, - iter_update=True, - query_dim=4, - num_feature_levels=args.num_feature_levels, - nheads=args.nheads, - dec_pred_bbox_embed_share=dec_pred_bbox_embed_share, - two_stage_type=args.two_stage_type, - two_stage_bbox_embed_share=args.two_stage_bbox_embed_share, - two_stage_class_embed_share=args.two_stage_class_embed_share, - num_patterns=args.num_patterns, - dn_number=0, - dn_box_noise_scale=args.dn_box_noise_scale, - dn_label_noise_ratio=args.dn_label_noise_ratio, - dn_labelbook_size=dn_labelbook_size, - text_encoder_type=args.text_encoder_type, - sub_sentence_present=sub_sentence_present, - max_text_len=args.max_text_len, - ) - - - - matcher = build_matcher(args) - - # prepare weight dict - weight_dict = {'loss_ce': args.cls_loss_coef, 'loss_bbox': args.bbox_loss_coef} - weight_dict['loss_giou'] = args.giou_loss_coef - clean_weight_dict_wo_dn = copy.deepcopy(weight_dict) - - - - clean_weight_dict = copy.deepcopy(weight_dict) - - # TODO this is a hack - if args.aux_loss: - aux_weight_dict = {} - for i in range(args.dec_layers - 1): - aux_weight_dict.update({k + f'_{i}': v for k, v in clean_weight_dict.items()}) - weight_dict.update(aux_weight_dict) - - if args.two_stage_type != 'no': - interm_weight_dict = {} - try: - no_interm_box_loss = args.no_interm_box_loss - except: - no_interm_box_loss = False - _coeff_weight_dict = { - 'loss_ce': 1.0, - 'loss_bbox': 1.0 if not no_interm_box_loss else 0.0, - 'loss_giou': 1.0 if not no_interm_box_loss else 0.0, - } - try: - interm_loss_coef = args.interm_loss_coef - except: - interm_loss_coef = 1.0 - interm_weight_dict.update({k + f'_interm': v * interm_loss_coef * _coeff_weight_dict[k] for k, v in clean_weight_dict_wo_dn.items()}) - weight_dict.update(interm_weight_dict) - - # losses = ['labels', 'boxes', 'cardinality'] - losses = ['labels', 'boxes'] - - criterion = SetCriterion(matcher=matcher, weight_dict=weight_dict, - focal_alpha=args.focal_alpha, focal_gamma=args.focal_gamma,losses=losses - ) - criterion.to(device) - postprocessors = {'bbox': PostProcess(num_select=args.num_select , text_encoder_type=args.text_encoder_type,nms_iou_threshold=args.nms_iou_threshold,args=args)} - - return model, criterion, postprocessors - -def create_positive_map(tokenized, tokens_positive,cat_list,caption): - """construct a map such that positive_map[i,j] = True iff box i is associated to token j""" - positive_map = torch.zeros((len(tokens_positive), 256), dtype=torch.float) - - for j,label in enumerate(tokens_positive): - - start_ind = caption.find(cat_list[label]) - end_ind = start_ind + len(cat_list[label]) - 1 - beg_pos = tokenized.char_to_token(start_ind) - try: - end_pos = tokenized.char_to_token(end_ind) - except: - end_pos = None - if end_pos is None: - try: - end_pos = tokenized.char_to_token(end_ind - 1) - if end_pos is None: - end_pos = tokenized.char_to_token(end_ind - 2) - except: - end_pos = None - # except Exception as e: - # print("beg:", beg, "end:", end) - # print("token_positive:", tokens_positive) - # # print("beg_pos:", beg_pos, "end_pos:", end_pos) - # raise e - # if beg_pos is None: - # try: - # beg_pos = tokenized.char_to_token(beg + 1) - # if beg_pos is None: - # beg_pos = tokenized.char_to_token(beg + 2) - # except: - # beg_pos = None - # if end_pos is None: - # try: - # end_pos = tokenized.char_to_token(end - 2) - # if end_pos is None: - # end_pos = tokenized.char_to_token(end - 3) - # except: - # end_pos = None - if beg_pos is None or end_pos is None: - continue - if beg_pos < 0 or end_pos < 0: - continue - if beg_pos > end_pos: - continue - # assert beg_pos is not None and end_pos is not None - positive_map[j,beg_pos: end_pos + 1].fill_(1) - return positive_map - - diff --git a/groundingdino/models/GroundingDINO/.ipynb_checkpoints/matcher-checkpoint.py b/groundingdino/models/GroundingDINO/.ipynb_checkpoints/matcher-checkpoint.py deleted file mode 100644 index 69d09a6464801c5c7c37e46a4f457121d7cce61c..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/.ipynb_checkpoints/matcher-checkpoint.py +++ /dev/null @@ -1,218 +0,0 @@ -# ------------------------------------------------------------------------ -# DINO -# Copyright (c) 2022 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Modules to compute the matching cost and solve the corresponding LSAP. -# Copyright (c) 2021 Microsoft. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Modified from DETR (https://github.com/facebookresearch/detr) -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -# ------------------------------------------------------------------------ -# Modified from Deformable DETR (https://github.com/fundamentalvision/Deformable-DETR) -# Copyright (c) 2020 SenseTime. All Rights Reserved. -# ------------------------------------------------------------------------ - - -import torch, os -from torch import nn -from scipy.optimize import linear_sum_assignment - -from util.box_ops import box_cxcywh_to_xyxy, generalized_box_iou - - -class HungarianMatcher(nn.Module): - """This class computes an assignment between the targets and the predictions of the network - For efficiency reasons, the targets don't include the no_object. Because of this, in general, - there are more predictions than targets. In this case, we do a 1-to-1 matching of the best predictions, - while the others are un-matched (and thus treated as non-objects). - """ - - def __init__(self, cost_class: float = 1, cost_bbox: float = 1, cost_giou: float = 1, focal_alpha = 0.25): - """Creates the matcher - Params: - cost_class: This is the relative weight of the classification error in the matching cost - cost_bbox: This is the relative weight of the L1 error of the bounding box coordinates in the matching cost - cost_giou: This is the relative weight of the giou loss of the bounding box in the matching cost - """ - super().__init__() - self.cost_class = cost_class - self.cost_bbox = cost_bbox - self.cost_giou = cost_giou - assert cost_class != 0 or cost_bbox != 0 or cost_giou != 0, "all costs cant be 0" - - self.focal_alpha = focal_alpha - - @torch.no_grad() - def forward(self, outputs, targets, label_map): - """ Performs the matching - Params: - outputs: This is a dict that contains at least these entries: - "pred_logits": Tensor of dim [batch_size, num_queries, num_classes] with the classification logits - "pred_boxes": Tensor of dim [batch_size, num_queries, 4] with the predicted box coordinates - targets: This is a list of targets (len(targets) = batch_size), where each target is a dict containing: - "labels": Tensor of dim [num_target_boxes] (where num_target_boxes is the number of ground-truth - objects in the target) containing the class labels - "boxes": Tensor of dim [num_target_boxes, 4] containing the target box coordinates - Returns: - A list of size batch_size, containing tuples of (index_i, index_j) where: - - index_i is the indices of the selected predictions (in order) - - index_j is the indices of the corresponding selected targets (in order) - For each batch element, it holds: - len(index_i) = len(index_j) = min(num_queries, num_target_boxes) - """ - - bs, num_queries = outputs["pred_logits"].shape[:2] - - # We flatten to compute the cost matrices in a batch - out_prob = outputs["pred_logits"].flatten(0, 1).sigmoid() # [batch_size * num_queries, num_classes] - out_bbox = outputs["pred_boxes"].flatten(0, 1) # [batch_size * num_queries, 4] - - # Also concat the target labels and boxes - tgt_ids = torch.cat([v["labels"] for v in targets]) - tgt_bbox = torch.cat([v["boxes"] for v in targets]) - - # Compute the classification cost. - alpha = self.focal_alpha - gamma = 2.0 - - new_label_map=label_map[tgt_ids.cpu()] - - neg_cost_class = (1 - alpha) * (out_prob ** gamma) * (-(1 - out_prob + 1e-8).log()) - pos_cost_class = alpha * ((1 - out_prob) ** gamma) * (-(out_prob + 1e-8).log()) - new_label_map=new_label_map.to(pos_cost_class.device) - - cost_bbox = torch.cdist(out_bbox, tgt_bbox, p=1) - - # cost_class=(pos_cost_class @ new_label_map.T - neg_cost_class@ new_label_map.T) - cost_class=[] - for idx_map in new_label_map: - idx_map = idx_map / idx_map.sum() - cost_class.append(pos_cost_class @ idx_map - neg_cost_class@ idx_map) - if cost_class: - cost_class=torch.stack(cost_class,dim=0).T - else: - cost_class=torch.zeros_like(cost_bbox) - # Compute the L1 cost between boxes - - - # Compute the giou cost betwen boxes - cost_giou = -generalized_box_iou(box_cxcywh_to_xyxy(out_bbox), box_cxcywh_to_xyxy(tgt_bbox)) - # import pdb;pdb.set_trace() - # Final cost matrix - C = self.cost_bbox * cost_bbox + self.cost_class * cost_class + self.cost_giou * cost_giou - C = C.view(bs, num_queries, -1).cpu() - C[torch.isnan(C)] = 0.0 - C[torch.isinf(C)] = 0.0 - - sizes = [len(v["boxes"]) for v in targets] - try: - indices = [linear_sum_assignment(c[i]) for i, c in enumerate(C.split(sizes, -1))] - except: - print("warning: use SimpleMinsumMatcher") - indices = [] - device = C.device - for i, (c, _size) in enumerate(zip(C.split(sizes, -1), sizes)): - weight_mat = c[i] - idx_i = weight_mat.min(0)[1] - idx_j = torch.arange(_size).to(device) - indices.append((idx_i, idx_j)) - return [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices] - - -class SimpleMinsumMatcher(nn.Module): - """This class computes an assignment between the targets and the predictions of the network - For efficiency reasons, the targets don't include the no_object. Because of this, in general, - there are more predictions than targets. In this case, we do a 1-to-1 matching of the best predictions, - while the others are un-matched (and thus treated as non-objects). - """ - - def __init__(self, cost_class: float = 1, cost_bbox: float = 1, cost_giou: float = 1, focal_alpha = 0.25): - """Creates the matcher - Params: - cost_class: This is the relative weight of the classification error in the matching cost - cost_bbox: This is the relative weight of the L1 error of the bounding box coordinates in the matching cost - cost_giou: This is the relative weight of the giou loss of the bounding box in the matching cost - """ - super().__init__() - self.cost_class = cost_class - self.cost_bbox = cost_bbox - self.cost_giou = cost_giou - assert cost_class != 0 or cost_bbox != 0 or cost_giou != 0, "all costs cant be 0" - - self.focal_alpha = focal_alpha - - @torch.no_grad() - def forward(self, outputs, targets): - """ Performs the matching - Params: - outputs: This is a dict that contains at least these entries: - "pred_logits": Tensor of dim [batch_size, num_queries, num_classes] with the classification logits - "pred_boxes": Tensor of dim [batch_size, num_queries, 4] with the predicted box coordinates - targets: This is a list of targets (len(targets) = batch_size), where each target is a dict containing: - "labels": Tensor of dim [num_target_boxes] (where num_target_boxes is the number of ground-truth - objects in the target) containing the class labels - "boxes": Tensor of dim [num_target_boxes, 4] containing the target box coordinates - Returns: - A list of size batch_size, containing tuples of (index_i, index_j) where: - - index_i is the indices of the selected predictions (in order) - - index_j is the indices of the corresponding selected targets (in order) - For each batch element, it holds: - len(index_i) = len(index_j) = min(num_queries, num_target_boxes) - """ - - bs, num_queries = outputs["pred_logits"].shape[:2] - - # We flatten to compute the cost matrices in a batch - out_prob = outputs["pred_logits"].flatten(0, 1).sigmoid() # [batch_size * num_queries, num_classes] - out_bbox = outputs["pred_boxes"].flatten(0, 1) # [batch_size * num_queries, 4] - - # Also concat the target labels and boxes - tgt_ids = torch.cat([v["labels"] for v in targets]) - tgt_bbox = torch.cat([v["boxes"] for v in targets]) - - # Compute the classification cost. - alpha = self.focal_alpha - gamma = 2.0 - neg_cost_class = (1 - alpha) * (out_prob ** gamma) * (-(1 - out_prob + 1e-8).log()) - pos_cost_class = alpha * ((1 - out_prob) ** gamma) * (-(out_prob + 1e-8).log()) - cost_class = pos_cost_class[:, tgt_ids] - neg_cost_class[:, tgt_ids] - - # Compute the L1 cost between boxes - cost_bbox = torch.cdist(out_bbox, tgt_bbox, p=1) - - # Compute the giou cost betwen boxes - cost_giou = -generalized_box_iou(box_cxcywh_to_xyxy(out_bbox), box_cxcywh_to_xyxy(tgt_bbox)) - - # Final cost matrix - - C = self.cost_bbox * cost_bbox + self.cost_class * cost_class + self.cost_giou * cost_giou - C = C.view(bs, num_queries, -1) - - sizes = [len(v["boxes"]) for v in targets] - indices = [] - device = C.device - for i, (c, _size) in enumerate(zip(C.split(sizes, -1), sizes)): - weight_mat = c[i] - idx_i = weight_mat.min(0)[1] - idx_j = torch.arange(_size).to(device) - indices.append((idx_i, idx_j)) - - return [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices] - - -def build_matcher(args): - assert args.matcher_type in ['HungarianMatcher', 'SimpleMinsumMatcher'], "Unknown args.matcher_type: {}".format(args.matcher_type) - if args.matcher_type == 'HungarianMatcher': - return HungarianMatcher( - cost_class=args.set_cost_class, cost_bbox=args.set_cost_bbox, cost_giou=args.set_cost_giou, - focal_alpha=args.focal_alpha - ) - elif args.matcher_type == 'SimpleMinsumMatcher': - return SimpleMinsumMatcher( - cost_class=args.set_cost_class, cost_bbox=args.set_cost_bbox, cost_giou=args.set_cost_giou, - focal_alpha=args.focal_alpha - ) - else: - raise NotImplementedError("Unknown args.matcher_type: {}".format(args.matcher_type)) \ No newline at end of file diff --git a/groundingdino/models/GroundingDINO/.ipynb_checkpoints/ms_deform_attn-checkpoint.py b/groundingdino/models/GroundingDINO/.ipynb_checkpoints/ms_deform_attn-checkpoint.py deleted file mode 100644 index 4a5222c1b548d44458d68b440149d2c8b0c8e39e..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/.ipynb_checkpoints/ms_deform_attn-checkpoint.py +++ /dev/null @@ -1,416 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Deformable DETR -# Copyright (c) 2020 SenseTime. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------------------------------ -# Modified from: -# https://github.com/fundamentalvision/Deformable-DETR/blob/main/models/ops/functions/ms_deform_attn_func.py -# https://github.com/fundamentalvision/Deformable-DETR/blob/main/models/ops/modules/ms_deform_attn.py -# https://github.com/open-mmlab/mmcv/blob/master/mmcv/ops/multi_scale_deform_attn.py -# ------------------------------------------------------------------------------------------------ - -import math -import warnings -from typing import Optional - -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch.autograd import Function -from torch.autograd.function import once_differentiable -from torch.nn.init import constant_, xavier_uniform_ -import loralib as lora - -try: - # from groundingdino import _C - import MultiScaleDeformableAttention as _C -except: - warnings.warn("Failed to load custom C++ ops. Running on CPU mode Only!") - - -# helpers -def _is_power_of_2(n): - if (not isinstance(n, int)) or (n < 0): - raise ValueError("invalid input for _is_power_of_2: {} (type: {})".format(n, type(n))) - return (n & (n - 1) == 0) and n != 0 - - -class MultiScaleDeformableAttnFunction(Function): - @staticmethod - def forward( - ctx, - value, - value_spatial_shapes, - value_level_start_index, - sampling_locations, - attention_weights, - im2col_step, - ): - ctx.im2col_step = im2col_step - output = _C.ms_deform_attn_forward( - value, - value_spatial_shapes, - value_level_start_index, - sampling_locations, - attention_weights, - ctx.im2col_step, - ) - ctx.save_for_backward( - value, - value_spatial_shapes, - value_level_start_index, - sampling_locations, - attention_weights, - ) - return output - - @staticmethod - @once_differentiable - def backward(ctx, grad_output): - ( - value, - value_spatial_shapes, - value_level_start_index, - sampling_locations, - attention_weights, - ) = ctx.saved_tensors - grad_value, grad_sampling_loc, grad_attn_weight = _C.ms_deform_attn_backward( - value, - value_spatial_shapes, - value_level_start_index, - sampling_locations, - attention_weights, - grad_output, - ctx.im2col_step, - ) - - return grad_value, None, None, grad_sampling_loc, grad_attn_weight, None - - -def multi_scale_deformable_attn_pytorch( - value: torch.Tensor, - value_spatial_shapes: torch.Tensor, - sampling_locations: torch.Tensor, - attention_weights: torch.Tensor, -) -> torch.Tensor: - - bs, _, num_heads, embed_dims = value.shape - _, num_queries, num_heads, num_levels, num_points, _ = sampling_locations.shape - value_list = value.split([H_ * W_ for H_, W_ in value_spatial_shapes], dim=1) - sampling_grids = 2 * sampling_locations - 1 - sampling_value_list = [] - for level, (H_, W_) in enumerate(value_spatial_shapes): - # bs, H_*W_, num_heads, embed_dims -> - # bs, H_*W_, num_heads*embed_dims -> - # bs, num_heads*embed_dims, H_*W_ -> - # bs*num_heads, embed_dims, H_, W_ - value_l_ = ( - value_list[level].flatten(2).transpose(1, 2).reshape(bs * num_heads, embed_dims, H_, W_) - ) - # bs, num_queries, num_heads, num_points, 2 -> - # bs, num_heads, num_queries, num_points, 2 -> - # bs*num_heads, num_queries, num_points, 2 - sampling_grid_l_ = sampling_grids[:, :, :, level].transpose(1, 2).flatten(0, 1) - # bs*num_heads, embed_dims, num_queries, num_points - sampling_value_l_ = F.grid_sample( - value_l_, sampling_grid_l_, mode="bilinear", padding_mode="zeros", align_corners=False - ) - sampling_value_list.append(sampling_value_l_) - # (bs, num_queries, num_heads, num_levels, num_points) -> - # (bs, num_heads, num_queries, num_levels, num_points) -> - # (bs, num_heads, 1, num_queries, num_levels*num_points) - attention_weights = attention_weights.transpose(1, 2).reshape( - bs * num_heads, 1, num_queries, num_levels * num_points - ) - output = ( - (torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights) - .sum(-1) - .view(bs, num_heads * embed_dims, num_queries) - ) - return output.transpose(1, 2).contiguous() - - -class MultiScaleDeformableAttention(nn.Module): - """Multi-Scale Deformable Attention Module used in Deformable-DETR - - `Deformable DETR: Deformable Transformers for End-to-End Object Detection. - `_. - - Args: - embed_dim (int): The embedding dimension of Attention. Default: 256. - num_heads (int): The number of attention heads. Default: 8. - num_levels (int): The number of feature map used in Attention. Default: 4. - num_points (int): The number of sampling points for each query - in each head. Default: 4. - img2col_steps (int): The step used in image_to_column. Defualt: 64. - dropout (float): Dropout layer used in output. Default: 0.1. - batch_first (bool): if ``True``, then the input and output tensor will be - provided as `(bs, n, embed_dim)`. Default: False. `(n, bs, embed_dim)` - """ - - def __init__( - self, - embed_dim: int = 256, - num_heads: int = 8, - num_levels: int = 4, - num_points: int = 4, - img2col_step: int = 64, - batch_first: bool = False, - ): - super().__init__() - if embed_dim % num_heads != 0: - raise ValueError( - "embed_dim must be divisible by num_heads, but got {} and {}".format( - embed_dim, num_heads - ) - ) - head_dim = embed_dim // num_heads - - self.batch_first = batch_first - - if not _is_power_of_2(head_dim): - warnings.warn( - """ - You'd better set d_model in MSDeformAttn to make sure that - each dim of the attention head a power of 2, which is more efficient. - """ - ) - - self.im2col_step = img2col_step - self.embed_dim = embed_dim - self.num_heads = num_heads - self.num_levels = num_levels - self.num_points = num_points - r = 12 - self.sampling_offsets = lora.Linear(embed_dim, num_heads * num_levels * num_points * 2 , r=r ) - self.attention_weights = lora.Linear(embed_dim, num_heads * num_levels * num_points , r=r) - self.value_proj = lora.Linear(embed_dim, embed_dim , r=r) - self.output_proj = lora.Linear(embed_dim, embed_dim , r=r) - - self.init_weights() - - def _reset_parameters(self): - return self.init_weights() - - def init_weights(self): - """ - Default initialization for Parameters of Module. - """ - constant_(self.sampling_offsets.weight.data, 0.0) - thetas = torch.arange(self.num_heads, dtype=torch.float32) * ( - 2.0 * math.pi / self.num_heads - ) - grid_init = torch.stack([thetas.cos(), thetas.sin()], -1) - grid_init = ( - (grid_init / grid_init.abs().max(-1, keepdim=True)[0]) - .view(self.num_heads, 1, 1, 2) - .repeat(1, self.num_levels, self.num_points, 1) - ) - for i in range(self.num_points): - grid_init[:, :, i, :] *= i + 1 - with torch.no_grad(): - self.sampling_offsets.bias = nn.Parameter(grid_init.view(-1)) - constant_(self.attention_weights.weight.data, 0.0) - constant_(self.attention_weights.bias.data, 0.0) - xavier_uniform_(self.value_proj.weight.data) - constant_(self.value_proj.bias.data, 0.0) - xavier_uniform_(self.output_proj.weight.data) - constant_(self.output_proj.bias.data, 0.0) - - def freeze_sampling_offsets(self): - print("Freeze sampling offsets") - self.sampling_offsets.weight.requires_grad = False - self.sampling_offsets.bias.requires_grad = False - - def freeze_attention_weights(self): - print("Freeze attention weights") - self.attention_weights.weight.requires_grad = False - self.attention_weights.bias.requires_grad = False - - def forward( - self, - query: torch.Tensor, - key: Optional[torch.Tensor] = None, - value: Optional[torch.Tensor] = None, - query_pos: Optional[torch.Tensor] = None, - key_padding_mask: Optional[torch.Tensor] = None, - reference_points: Optional[torch.Tensor] = None, - spatial_shapes: Optional[torch.Tensor] = None, - level_start_index: Optional[torch.Tensor] = None, - **kwargs - ) -> torch.Tensor: - - """Forward Function of MultiScaleDeformableAttention - - Args: - query (torch.Tensor): Query embeddings with shape - `(num_query, bs, embed_dim)` - key (torch.Tensor): Key embeddings with shape - `(num_key, bs, embed_dim)` - value (torch.Tensor): Value embeddings with shape - `(num_key, bs, embed_dim)` - query_pos (torch.Tensor): The position embedding for `query`. Default: None. - key_padding_mask (torch.Tensor): ByteTensor for `query`, with shape `(bs, num_key)`, - indicating which elements within `key` to be ignored in attention. - reference_points (torch.Tensor): The normalized reference points - with shape `(bs, num_query, num_levels, 2)`, - all elements is range in [0, 1], top-left (0, 0), - bottom-right (1, 1), including padding are. - or `(N, Length_{query}, num_levels, 4)`, add additional - two dimensions `(h, w)` to form reference boxes. - spatial_shapes (torch.Tensor): Spatial shape of features in different levels. - With shape `(num_levels, 2)`, last dimension represents `(h, w)`. - level_start_index (torch.Tensor): The start index of each level. A tensor with - shape `(num_levels, )` which can be represented as - `[0, h_0 * w_0, h_0 * w_0 + h_1 * w_1, ...]`. - - Returns: - torch.Tensor: forward results with shape `(num_query, bs, embed_dim)` - """ - - if value is None: - value = query - - if query_pos is not None: - query = query + query_pos - - if not self.batch_first: - # change to (bs, num_query ,embed_dims) - query = query.permute(1, 0, 2) - value = value.permute(1, 0, 2) - - bs, num_query, _ = query.shape - bs, num_value, _ = value.shape - - assert (spatial_shapes[:, 0] * spatial_shapes[:, 1]).sum() == num_value - - value = self.value_proj(value) - if key_padding_mask is not None: - value = value.masked_fill(key_padding_mask[..., None], float(0)) - value = value.view(bs, num_value, self.num_heads, -1) - sampling_offsets = self.sampling_offsets(query).view( - bs, num_query, self.num_heads, self.num_levels, self.num_points, 2 - ) - attention_weights = self.attention_weights(query).view( - bs, num_query, self.num_heads, self.num_levels * self.num_points - ) - attention_weights = attention_weights.softmax(-1) - attention_weights = attention_weights.view( - bs, - num_query, - self.num_heads, - self.num_levels, - self.num_points, - ) - - # bs, num_query, num_heads, num_levels, num_points, 2 - if reference_points.shape[-1] == 2: - offset_normalizer = torch.stack([spatial_shapes[..., 1], spatial_shapes[..., 0]], -1) - sampling_locations = ( - reference_points[:, :, None, :, None, :] - + sampling_offsets / offset_normalizer[None, None, None, :, None, :] - ) - elif reference_points.shape[-1] == 4: - sampling_locations = ( - reference_points[:, :, None, :, None, :2] - + sampling_offsets - / self.num_points - * reference_points[:, :, None, :, None, 2:] - * 0.5 - ) - else: - raise ValueError( - "Last dim of reference_points must be 2 or 4, but get {} instead.".format( - reference_points.shape[-1] - ) - ) - - if torch.cuda.is_available() and value.is_cuda: - halffloat = False - if value.dtype == torch.float16: - halffloat = True - value = value.float() - sampling_locations = sampling_locations.float() - attention_weights = attention_weights.float() - - output = MultiScaleDeformableAttnFunction.apply( - value, - spatial_shapes, - level_start_index, - sampling_locations, - attention_weights, - self.im2col_step, - ) - - if halffloat: - output = output.half() - else: - output = multi_scale_deformable_attn_pytorch( - value, spatial_shapes, sampling_locations, attention_weights - ) - - output = self.output_proj(output) - - if not self.batch_first: - output = output.permute(1, 0, 2) - - return output - - -def create_dummy_class(klass, dependency, message=""): - """ - When a dependency of a class is not available, create a dummy class which throws ImportError - when used. - - Args: - klass (str): name of the class. - dependency (str): name of the dependency. - message: extra message to print - Returns: - class: a class object - """ - err = "Cannot import '{}', therefore '{}' is not available.".format(dependency, klass) - if message: - err = err + " " + message - - class _DummyMetaClass(type): - # throw error on class attribute access - def __getattr__(_, __): # noqa: B902 - raise ImportError(err) - - class _Dummy(object, metaclass=_DummyMetaClass): - # throw error on constructor - def __init__(self, *args, **kwargs): - raise ImportError(err) - - return _Dummy - - -def create_dummy_func(func, dependency, message=""): - """ - When a dependency of a function is not available, create a dummy function which throws - ImportError when used. - - Args: - func (str): name of the function. - dependency (str or list[str]): name(s) of the dependency. - message: extra message to print - Returns: - function: a function object - """ - err = "Cannot import '{}', therefore '{}' is not available.".format(dependency, func) - if message: - err = err + " " + message - - if isinstance(dependency, (list, tuple)): - dependency = ",".join(dependency) - - def _dummy(*args, **kwargs): - raise ImportError(err) - - return _dummy diff --git a/groundingdino/models/GroundingDINO/.ipynb_checkpoints/transformer-checkpoint.py b/groundingdino/models/GroundingDINO/.ipynb_checkpoints/transformer-checkpoint.py deleted file mode 100644 index 379cbb5857cf953f337312ba5e40014efe83618f..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/.ipynb_checkpoints/transformer-checkpoint.py +++ /dev/null @@ -1,969 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# DINO -# Copyright (c) 2022 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Conditional DETR Transformer class. -# Copyright (c) 2021 Microsoft. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Modified from DETR (https://github.com/facebookresearch/detr) -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -# ------------------------------------------------------------------------ - -from typing import Optional - -import torch -import torch.utils.checkpoint as checkpoint -from torch import Tensor, nn - -from groundingdino.util.misc import inverse_sigmoid -import loralib as lora -from .fuse_modules import BiAttentionBlock -from .ms_deform_attn import MultiScaleDeformableAttention as MSDeformAttn -from .transformer_vanilla import TransformerEncoderLayer -from .utils import ( - MLP, - _get_activation_fn, - _get_clones, - gen_encoder_output_proposals, - gen_sineembed_for_position, - get_sine_pos_embed, -) - - -class Transformer(nn.Module): - def __init__( - self, - d_model=256, - nhead=8, - num_queries=300, - num_encoder_layers=6, - num_unicoder_layers=0, - num_decoder_layers=6, - dim_feedforward=2048, - dropout=0.0, - activation="relu", - normalize_before=False, - return_intermediate_dec=False, - query_dim=4, - num_patterns=0, - # for deformable encoder - num_feature_levels=1, - enc_n_points=4, - dec_n_points=4, - # init query - learnable_tgt_init=False, - # two stage - two_stage_type="no", # ['no', 'standard', 'early', 'combine', 'enceachlayer', 'enclayer1'] - embed_init_tgt=False, - # for text - use_text_enhancer=False, - use_fusion_layer=False, - use_checkpoint=False, - use_transformer_ckpt=False, - use_text_cross_attention=False, - text_dropout=0.1, - fusion_dropout=0.1, - fusion_droppath=0.0, - ): - super().__init__() - self.num_feature_levels = num_feature_levels - self.num_encoder_layers = num_encoder_layers - self.num_unicoder_layers = num_unicoder_layers - self.num_decoder_layers = num_decoder_layers - self.num_queries = num_queries - assert query_dim == 4 - - # choose encoder layer type - encoder_layer = DeformableTransformerEncoderLayer( - d_model, dim_feedforward, dropout, activation, num_feature_levels, nhead, enc_n_points - ) - - if use_text_enhancer: - text_enhance_layer = TransformerEncoderLayer( - d_model=d_model, - nhead=nhead // 2, - dim_feedforward=dim_feedforward // 2, - dropout=text_dropout, - ) - else: - text_enhance_layer = None - - if use_fusion_layer: - feature_fusion_layer = BiAttentionBlock( - v_dim=d_model, - l_dim=d_model, - embed_dim=dim_feedforward // 2, - num_heads=nhead // 2, - dropout=fusion_dropout, - drop_path=fusion_droppath, - ) - else: - feature_fusion_layer = None - - encoder_norm = nn.LayerNorm(d_model) if normalize_before else None - assert encoder_norm is None - self.encoder = TransformerEncoder( - encoder_layer, - num_encoder_layers, - d_model=d_model, - num_queries=num_queries, - text_enhance_layer=text_enhance_layer, - feature_fusion_layer=feature_fusion_layer, - use_checkpoint=use_checkpoint, - use_transformer_ckpt=use_transformer_ckpt, - ) - - # choose decoder layer type - decoder_layer = DeformableTransformerDecoderLayer( - d_model, - dim_feedforward, - dropout, - activation, - num_feature_levels, - nhead, - dec_n_points, - use_text_cross_attention=use_text_cross_attention, - ) - - decoder_norm = nn.LayerNorm(d_model) - self.decoder = TransformerDecoder( - decoder_layer, - num_decoder_layers, - decoder_norm, - return_intermediate=return_intermediate_dec, - d_model=d_model, - query_dim=query_dim, - num_feature_levels=num_feature_levels, - ) - - self.d_model = d_model - self.nhead = nhead - self.dec_layers = num_decoder_layers - self.num_queries = num_queries # useful for single stage model only - self.num_patterns = num_patterns - if not isinstance(num_patterns, int): - Warning("num_patterns should be int but {}".format(type(num_patterns))) - self.num_patterns = 0 - - if num_feature_levels > 1: - if self.num_encoder_layers > 0: - self.level_embed = nn.Parameter(torch.Tensor(num_feature_levels, d_model)) - else: - self.level_embed = None - - self.learnable_tgt_init = learnable_tgt_init - assert learnable_tgt_init, "why not learnable_tgt_init" - self.embed_init_tgt = embed_init_tgt - if (two_stage_type != "no" and embed_init_tgt) or (two_stage_type == "no"): - self.tgt_embed = nn.Embedding(self.num_queries, d_model) - nn.init.normal_(self.tgt_embed.weight.data) - else: - self.tgt_embed = None - - # for two stage - self.two_stage_type = two_stage_type - assert two_stage_type in ["no", "standard"], "unknown param {} of two_stage_type".format( - two_stage_type - ) - if two_stage_type == "standard": - # anchor selection at the output of encoder - r = 24 - self.enc_output = lora.Linear(d_model, d_model , r=r) - self.enc_output_norm = nn.LayerNorm(d_model) - self.two_stage_wh_embedding = None - - if two_stage_type == "no": - self.init_ref_points(num_queries) # init self.refpoint_embed - - self.enc_out_class_embed = None - self.enc_out_bbox_embed = None - - self._reset_parameters() - - def _reset_parameters(self): - for p in self.parameters(): - if p.dim() > 1: - nn.init.xavier_uniform_(p) - for m in self.modules(): - if isinstance(m, MSDeformAttn): - m._reset_parameters() - if self.num_feature_levels > 1 and self.level_embed is not None: - nn.init.normal_(self.level_embed) - - def get_valid_ratio(self, mask): - _, H, W = mask.shape - valid_H = torch.sum(~mask[:, :, 0], 1) - valid_W = torch.sum(~mask[:, 0, :], 1) - valid_ratio_h = valid_H.float() / H - valid_ratio_w = valid_W.float() / W - valid_ratio = torch.stack([valid_ratio_w, valid_ratio_h], -1) - return valid_ratio - - def init_ref_points(self, use_num_queries): - self.refpoint_embed = nn.Embedding(use_num_queries, 4) - - def forward(self, srcs, masks, refpoint_embed, pos_embeds, tgt, attn_mask=None, text_dict=None): - """ - Input: - - srcs: List of multi features [bs, ci, hi, wi] - - masks: List of multi masks [bs, hi, wi] - - refpoint_embed: [bs, num_dn, 4]. None in infer - - pos_embeds: List of multi pos embeds [bs, ci, hi, wi] - - tgt: [bs, num_dn, d_model]. None in infer - - """ - # prepare input for encoder - src_flatten = [] - mask_flatten = [] - lvl_pos_embed_flatten = [] - spatial_shapes = [] - for lvl, (src, mask, pos_embed) in enumerate(zip(srcs, masks, pos_embeds)): - bs, c, h, w = src.shape - spatial_shape = (h, w) - spatial_shapes.append(spatial_shape) - - src = src.flatten(2).transpose(1, 2) # bs, hw, c - mask = mask.flatten(1) # bs, hw - pos_embed = pos_embed.flatten(2).transpose(1, 2) # bs, hw, c - if self.num_feature_levels > 1 and self.level_embed is not None: - lvl_pos_embed = pos_embed + self.level_embed[lvl].view(1, 1, -1) - else: - lvl_pos_embed = pos_embed - lvl_pos_embed_flatten.append(lvl_pos_embed) - src_flatten.append(src) - mask_flatten.append(mask) - src_flatten = torch.cat(src_flatten, 1) # bs, \sum{hxw}, c - mask_flatten = torch.cat(mask_flatten, 1) # bs, \sum{hxw} - lvl_pos_embed_flatten = torch.cat(lvl_pos_embed_flatten, 1) # bs, \sum{hxw}, c - spatial_shapes = torch.as_tensor( - spatial_shapes, dtype=torch.long, device=src_flatten.device - ) - level_start_index = torch.cat( - (spatial_shapes.new_zeros((1,)), spatial_shapes.prod(1).cumsum(0)[:-1]) - ) - valid_ratios = torch.stack([self.get_valid_ratio(m) for m in masks], 1) - - # two stage - enc_topk_proposals = enc_refpoint_embed = None - - ######################################################### - # Begin Encoder - ######################################################### - memory, memory_text = self.encoder( - src_flatten, - pos=lvl_pos_embed_flatten, - level_start_index=level_start_index, - spatial_shapes=spatial_shapes, - valid_ratios=valid_ratios, - key_padding_mask=mask_flatten, - memory_text=text_dict["encoded_text"], - text_attention_mask=~text_dict["text_token_mask"], - # we ~ the mask . False means use the token; True means pad the token - position_ids=text_dict["position_ids"], - text_self_attention_masks=text_dict["text_self_attention_masks"], - ) - ######################################################### - # End Encoder - # - memory: bs, \sum{hw}, c - # - mask_flatten: bs, \sum{hw} - # - lvl_pos_embed_flatten: bs, \sum{hw}, c - # - enc_intermediate_output: None or (nenc+1, bs, nq, c) or (nenc, bs, nq, c) - # - enc_intermediate_refpoints: None or (nenc+1, bs, nq, c) or (nenc, bs, nq, c) - ######################################################### - text_dict["encoded_text"] = memory_text - # if os.environ.get("SHILONG_AMP_INFNAN_DEBUG") == '1': - # if memory.isnan().any() | memory.isinf().any(): - # import ipdb; ipdb.set_trace() - - if self.two_stage_type == "standard": #把encoder的输出作为proposal - output_memory, output_proposals = gen_encoder_output_proposals( - memory, mask_flatten, spatial_shapes - ) - output_memory = self.enc_output_norm(self.enc_output(output_memory)) - - if text_dict is not None: - enc_outputs_class_unselected = self.enc_out_class_embed(output_memory, text_dict) - else: - enc_outputs_class_unselected = self.enc_out_class_embed(output_memory) - - topk_logits = enc_outputs_class_unselected.max(-1)[0] - enc_outputs_coord_unselected = ( - self.enc_out_bbox_embed(output_memory) + output_proposals - ) # (bs, \sum{hw}, 4) unsigmoid - topk = self.num_queries - - topk_proposals = torch.topk(topk_logits, topk, dim=1)[1] # bs, nq - - # gather boxes - refpoint_embed_undetach = torch.gather( - enc_outputs_coord_unselected, 1, topk_proposals.unsqueeze(-1).repeat(1, 1, 4) - ) # unsigmoid - refpoint_embed_ = refpoint_embed_undetach.detach() - init_box_proposal = torch.gather( - output_proposals, 1, topk_proposals.unsqueeze(-1).repeat(1, 1, 4) - ).sigmoid() # sigmoid - - # gather tgt - tgt_undetach = torch.gather( - output_memory, 1, topk_proposals.unsqueeze(-1).repeat(1, 1, self.d_model) - ) - if self.embed_init_tgt: - tgt_ = ( - self.tgt_embed.weight[:, None, :].repeat(1, bs, 1).transpose(0, 1) - ) # nq, bs, d_model - else: - tgt_ = tgt_undetach.detach() - - if refpoint_embed is not None: - refpoint_embed = torch.cat([refpoint_embed, refpoint_embed_], dim=1) - tgt = torch.cat([tgt, tgt_], dim=1) - else: - refpoint_embed, tgt = refpoint_embed_, tgt_ - - elif self.two_stage_type == "no": - tgt_ = ( - self.tgt_embed.weight[:, None, :].repeat(1, bs, 1).transpose(0, 1) - ) # nq, bs, d_model - refpoint_embed_ = ( - self.refpoint_embed.weight[:, None, :].repeat(1, bs, 1).transpose(0, 1) - ) # nq, bs, 4 - - if refpoint_embed is not None: - refpoint_embed = torch.cat([refpoint_embed, refpoint_embed_], dim=1) - tgt = torch.cat([tgt, tgt_], dim=1) - else: - refpoint_embed, tgt = refpoint_embed_, tgt_ - - if self.num_patterns > 0: - tgt_embed = tgt.repeat(1, self.num_patterns, 1) - refpoint_embed = refpoint_embed.repeat(1, self.num_patterns, 1) - tgt_pat = self.patterns.weight[None, :, :].repeat_interleave( - self.num_queries, 1 - ) # 1, n_q*n_pat, d_model - tgt = tgt_embed + tgt_pat - - init_box_proposal = refpoint_embed_.sigmoid() - - else: - raise NotImplementedError("unknown two_stage_type {}".format(self.two_stage_type)) - ######################################################### - # End preparing tgt - # - tgt: bs, NQ, d_model - # - refpoint_embed(unsigmoid): bs, NQ, d_model - ######################################################### - - ######################################################### - # Begin Decoder - ######################################################### - - #memory torch.Size([2, 16320, 256]) - - # import pdb;pdb.set_trace() - hs, references = self.decoder( - tgt=tgt.transpose(0, 1), - memory=memory.transpose(0, 1), - memory_key_padding_mask=mask_flatten, - pos=lvl_pos_embed_flatten.transpose(0, 1), - refpoints_unsigmoid=refpoint_embed.transpose(0, 1), - level_start_index=level_start_index, - spatial_shapes=spatial_shapes, - valid_ratios=valid_ratios, - tgt_mask=attn_mask, - memory_text=text_dict["encoded_text"], - text_attention_mask=~text_dict["text_token_mask"], - # we ~ the mask . False means use the token; True means pad the token - ) - ######################################################### - # End Decoder - # hs: n_dec, bs, nq, d_model - # references: n_dec+1, bs, nq, query_dim - ######################################################### - - ######################################################### - # Begin postprocess - ######################################################### - if self.two_stage_type == "standard": - hs_enc = tgt_undetach.unsqueeze(0) - ref_enc = refpoint_embed_undetach.sigmoid().unsqueeze(0) - else: - hs_enc = ref_enc = None - ######################################################### - # End postprocess - # hs_enc: (n_enc+1, bs, nq, d_model) or (1, bs, nq, d_model) or (n_enc, bs, nq, d_model) or None - # ref_enc: (n_enc+1, bs, nq, query_dim) or (1, bs, nq, query_dim) or (n_enc, bs, nq, d_model) or None - ######################################################### - - return hs, references, hs_enc, ref_enc, init_box_proposal - # hs: (n_dec, bs, nq, d_model) - # references: sigmoid coordinates. (n_dec+1, bs, bq, 4) - # hs_enc: (n_enc+1, bs, nq, d_model) or (1, bs, nq, d_model) or None - # ref_enc: sigmoid coordinates. \ - # (n_enc+1, bs, nq, query_dim) or (1, bs, nq, query_dim) or None - - -class TransformerEncoder(nn.Module): - def __init__( - self, - encoder_layer, - num_layers, - d_model=256, - num_queries=300, - enc_layer_share=False, - text_enhance_layer=None, - feature_fusion_layer=None, - use_checkpoint=False, - use_transformer_ckpt=False, - ): - """_summary_ - - Args: - encoder_layer (_type_): _description_ - num_layers (_type_): _description_ - norm (_type_, optional): _description_. Defaults to None. - d_model (int, optional): _description_. Defaults to 256. - num_queries (int, optional): _description_. Defaults to 300. - enc_layer_share (bool, optional): _description_. Defaults to False. - - """ - super().__init__() - # prepare layers - self.layers = [] - self.text_layers = [] - self.fusion_layers = [] - if num_layers > 0: - self.layers = _get_clones(encoder_layer, num_layers, layer_share=enc_layer_share) - - if text_enhance_layer is not None: - self.text_layers = _get_clones( - text_enhance_layer, num_layers, layer_share=enc_layer_share - ) - if feature_fusion_layer is not None: - self.fusion_layers = _get_clones( - feature_fusion_layer, num_layers, layer_share=enc_layer_share - ) - else: - self.layers = [] - del encoder_layer - - if text_enhance_layer is not None: - self.text_layers = [] - del text_enhance_layer - if feature_fusion_layer is not None: - self.fusion_layers = [] - del feature_fusion_layer - - self.query_scale = None - self.num_queries = num_queries - self.num_layers = num_layers - self.d_model = d_model - - self.use_checkpoint = use_checkpoint - self.use_transformer_ckpt = use_transformer_ckpt - - @staticmethod - def get_reference_points(spatial_shapes, valid_ratios, device): - reference_points_list = [] - for lvl, (H_, W_) in enumerate(spatial_shapes): - - ref_y, ref_x = torch.meshgrid( - torch.linspace(0.5, H_ - 0.5, H_, dtype=torch.float32, device=device), - torch.linspace(0.5, W_ - 0.5, W_, dtype=torch.float32, device=device), - ) - ref_y = ref_y.reshape(-1)[None] / (valid_ratios[:, None, lvl, 1] * H_) - ref_x = ref_x.reshape(-1)[None] / (valid_ratios[:, None, lvl, 0] * W_) - ref = torch.stack((ref_x, ref_y), -1) - reference_points_list.append(ref) - reference_points = torch.cat(reference_points_list, 1) - reference_points = reference_points[:, :, None] * valid_ratios[:, None] - return reference_points - - def forward( - self, - # for images - src: Tensor, - pos: Tensor, - spatial_shapes: Tensor, - level_start_index: Tensor, - valid_ratios: Tensor, - key_padding_mask: Tensor, - # for texts - memory_text: Tensor = None, - text_attention_mask: Tensor = None, - pos_text: Tensor = None, - text_self_attention_masks: Tensor = None, - position_ids: Tensor = None, - ): - """ - Input: - - src: [bs, sum(hi*wi), 256] - - pos: pos embed for src. [bs, sum(hi*wi), 256] - - spatial_shapes: h,w of each level [num_level, 2] - - level_start_index: [num_level] start point of level in sum(hi*wi). - - valid_ratios: [bs, num_level, 2] - - key_padding_mask: [bs, sum(hi*wi)] - - - memory_text: bs, n_text, 256 - - text_attention_mask: bs, n_text - False for no padding; True for padding - - pos_text: bs, n_text, 256 - - - position_ids: bs, n_text - Intermedia: - - reference_points: [bs, sum(hi*wi), num_level, 2] - Outpus: - - output: [bs, sum(hi*wi), 256] - """ - - output = src - - # preparation and reshape - if self.num_layers > 0: - reference_points = self.get_reference_points( - spatial_shapes, valid_ratios, device=src.device - ) - - if self.text_layers: - # generate pos_text - bs, n_text, text_dim = memory_text.shape - if pos_text is None and position_ids is None: - pos_text = ( - torch.arange(n_text, device=memory_text.device) - .float() - .unsqueeze(0) - .unsqueeze(-1) - .repeat(bs, 1, 1) - ) - pos_text = get_sine_pos_embed(pos_text, num_pos_feats=256, exchange_xy=False) - if position_ids is not None: - pos_text = get_sine_pos_embed( - position_ids[..., None], num_pos_feats=256, exchange_xy=False - ) - - # main process - for layer_id, layer in enumerate(self.layers): - # if output.isnan().any() or memory_text.isnan().any(): - # if os.environ.get('IPDB_SHILONG_DEBUG', None) == 'INFO': - # import ipdb; ipdb.set_trace() - if self.fusion_layers: - if self.use_checkpoint: - output, memory_text = checkpoint.checkpoint( - self.fusion_layers[layer_id], - output, - memory_text, - key_padding_mask, - text_attention_mask, - ) - else: - output, memory_text = self.fusion_layers[layer_id]( - v=output, - l=memory_text, - attention_mask_v=key_padding_mask, - attention_mask_l=text_attention_mask, - ) - - if self.text_layers: - memory_text = self.text_layers[layer_id]( - src=memory_text.transpose(0, 1), - src_mask=~text_self_attention_masks, # note we use ~ for mask here - src_key_padding_mask=text_attention_mask, - pos=(pos_text.transpose(0, 1) if pos_text is not None else None), - ).transpose(0, 1) - - # main process - if self.use_transformer_ckpt: - output = checkpoint.checkpoint( - layer, - output, - pos, - reference_points, - spatial_shapes, - level_start_index, - key_padding_mask, - ) - else: - output = layer( - src=output, - pos=pos, - reference_points=reference_points, - spatial_shapes=spatial_shapes, - level_start_index=level_start_index, - key_padding_mask=key_padding_mask, - ) - - return output, memory_text - - -class TransformerDecoder(nn.Module): - def __init__( - self, - decoder_layer, - num_layers, - norm=None, - return_intermediate=False, - d_model=256, - query_dim=4, - num_feature_levels=1, - ): - super().__init__() - if num_layers > 0: - self.layers = _get_clones(decoder_layer, num_layers) - else: - self.layers = [] - self.num_layers = num_layers - self.norm = norm - self.return_intermediate = return_intermediate - assert return_intermediate, "support return_intermediate only" - self.query_dim = query_dim - assert query_dim in [2, 4], "query_dim should be 2/4 but {}".format(query_dim) - self.num_feature_levels = num_feature_levels - - self.ref_point_head = MLP(query_dim // 2 * d_model, d_model, d_model, 2) - self.query_pos_sine_scale = None - - self.query_scale = None - self.bbox_embed = None - self.class_embed = None - - self.d_model = d_model - - self.ref_anchor_head = None - - def forward( - self, - tgt, - memory, - tgt_mask: Optional[Tensor] = None, - memory_mask: Optional[Tensor] = None, - tgt_key_padding_mask: Optional[Tensor] = None, - memory_key_padding_mask: Optional[Tensor] = None, - pos: Optional[Tensor] = None, - refpoints_unsigmoid: Optional[Tensor] = None, # num_queries, bs, 2 - # for memory - level_start_index: Optional[Tensor] = None, # num_levels - spatial_shapes: Optional[Tensor] = None, # bs, num_levels, 2 - valid_ratios: Optional[Tensor] = None, - # for text - memory_text: Optional[Tensor] = None, - text_attention_mask: Optional[Tensor] = None, - ): - """ - Input: - - tgt: nq, bs, d_model - - memory: hw, bs, d_model - - pos: hw, bs, d_model - - refpoints_unsigmoid: nq, bs, 2/4 - - valid_ratios/spatial_shapes: bs, nlevel, 2 - """ - output = tgt - - intermediate = [] - reference_points = refpoints_unsigmoid.sigmoid() - ref_points = [reference_points] - - - - for layer_id, layer in enumerate(self.layers): - - if reference_points.shape[-1] == 4: - reference_points_input = ( - reference_points[:, :, None] - * torch.cat([valid_ratios, valid_ratios], -1)[None, :] - ) # nq, bs, nlevel, 4 - else: - assert reference_points.shape[-1] == 2 - reference_points_input = reference_points[:, :, None] * valid_ratios[None, :] - query_sine_embed = gen_sineembed_for_position( - reference_points_input[:, :, 0, :] - ) # nq, bs, 256*2 - - # conditional query - raw_query_pos = self.ref_point_head(query_sine_embed) # nq, bs, 256 - pos_scale = self.query_scale(output) if self.query_scale is not None else 1 - query_pos = pos_scale * raw_query_pos - # if os.environ.get("SHILONG_AMP_INFNAN_DEBUG") == '1': - # if query_pos.isnan().any() | query_pos.isinf().any(): - # import ipdb; ipdb.set_trace() - - # main process - output = layer( - tgt=output, - tgt_query_pos=query_pos, - tgt_query_sine_embed=query_sine_embed, - tgt_key_padding_mask=tgt_key_padding_mask, - tgt_reference_points=reference_points_input, - memory_text=memory_text, - text_attention_mask=text_attention_mask, - memory=memory, - memory_key_padding_mask=memory_key_padding_mask, - memory_level_start_index=level_start_index, - memory_spatial_shapes=spatial_shapes, - memory_pos=pos, - self_attn_mask=tgt_mask, - cross_attn_mask=memory_mask, - ) - if output.isnan().any() | output.isinf().any(): - print(f"output layer_id {layer_id} is nan") - try: - num_nan = output.isnan().sum().item() - num_inf = output.isinf().sum().item() - print(f"num_nan {num_nan}, num_inf {num_inf}") - except Exception as e: - print(e) - # if os.environ.get("SHILONG_AMP_INFNAN_DEBUG") == '1': - # import ipdb; ipdb.set_trace() - - # iter update - if self.bbox_embed is not None: - # box_holder = self.bbox_embed(output) - # box_holder[..., :self.query_dim] += inverse_sigmoid(reference_points) - # new_reference_points = box_holder[..., :self.query_dim].sigmoid() - - reference_before_sigmoid = inverse_sigmoid(reference_points) - delta_unsig = self.bbox_embed[layer_id](output) - outputs_unsig = delta_unsig + reference_before_sigmoid - new_reference_points = outputs_unsig.sigmoid() - - reference_points = new_reference_points.detach() - # if layer_id != self.num_layers - 1: - ref_points.append(new_reference_points) - - intermediate.append(self.norm(output)) - - # import pdb;pdb.set_trace() - - return [ - [itm_out.transpose(0, 1) for itm_out in intermediate], - [itm_refpoint.transpose(0, 1) for itm_refpoint in ref_points], - ] - - -class DeformableTransformerEncoderLayer(nn.Module): - def __init__( - self, - d_model=256, - d_ffn=1024, - dropout=0.1, - activation="relu", - n_levels=4, - n_heads=8, - n_points=4, - ): - super().__init__() - - # self attention - self.self_attn = MSDeformAttn( - embed_dim=d_model, - num_levels=n_levels, - num_heads=n_heads, - num_points=n_points, - batch_first=True, - ) - self.dropout1 = nn.Dropout(dropout) - self.norm1 = nn.LayerNorm(d_model) - r =12 - # ffn - self.linear1 = lora.Linear(d_model, d_ffn , r=r ) - self.activation = _get_activation_fn(activation, d_model=d_ffn) - self.dropout2 = nn.Dropout(dropout) - self.linear2 = lora.Linear(d_ffn, d_model , r=r) - self.dropout3 = nn.Dropout(dropout) - self.norm2 = nn.LayerNorm(d_model) - - @staticmethod - def with_pos_embed(tensor, pos): - return tensor if pos is None else tensor + pos - - def forward_ffn(self, src): - src2 = self.linear2(self.dropout2(self.activation(self.linear1(src)))) - src = src + self.dropout3(src2) - src = self.norm2(src) - return src - - def forward( - self, src, pos, reference_points, spatial_shapes, level_start_index, key_padding_mask=None - ): - # self attention - # import ipdb; ipdb.set_trace() - src2 = self.self_attn( - query=self.with_pos_embed(src, pos), - reference_points=reference_points, - value=src, - spatial_shapes=spatial_shapes, - level_start_index=level_start_index, - key_padding_mask=key_padding_mask, - ) - src = src + self.dropout1(src2) - src = self.norm1(src) - - # ffn - src = self.forward_ffn(src) - - return src - - -class DeformableTransformerDecoderLayer(nn.Module): - def __init__( - self, - d_model=256, - d_ffn=1024, - dropout=0.1, - activation="relu", - n_levels=4, - n_heads=8, - n_points=4, - use_text_feat_guide=False, - use_text_cross_attention=False, - ): - super().__init__() - - # cross attention - self.cross_attn = MSDeformAttn( - embed_dim=d_model, - num_levels=n_levels, - num_heads=n_heads, - num_points=n_points, - batch_first=True, - ) - self.dropout1 = nn.Dropout(dropout) if dropout > 0 else nn.Identity() - self.norm1 = nn.LayerNorm(d_model) - - # cross attention text - if use_text_cross_attention: - self.ca_text = nn.MultiheadAttention(d_model, n_heads, dropout=dropout) - self.catext_dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity() - self.catext_norm = nn.LayerNorm(d_model) - - # self attention - self.self_attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout) - self.dropout2 = nn.Dropout(dropout) if dropout > 0 else nn.Identity() - self.norm2 = nn.LayerNorm(d_model) - - # ffn - r = 12 - self.linear1 = lora.Linear(d_model, d_ffn , r=r) - self.activation = _get_activation_fn(activation, d_model=d_ffn, batch_dim=1) - self.dropout3 = nn.Dropout(dropout) if dropout > 0 else nn.Identity() - self.linear2 = lora.Linear(d_ffn, d_model , r=r ) - self.dropout4 = nn.Dropout(dropout) if dropout > 0 else nn.Identity() - self.norm3 = nn.LayerNorm(d_model) - - self.key_aware_proj = None - self.use_text_feat_guide = use_text_feat_guide - assert not use_text_feat_guide - self.use_text_cross_attention = use_text_cross_attention - - def rm_self_attn_modules(self): - self.self_attn = None - self.dropout2 = None - self.norm2 = None - - @staticmethod - def with_pos_embed(tensor, pos): - return tensor if pos is None else tensor + pos - - def forward_ffn(self, tgt): - with torch.cuda.amp.autocast(enabled=False): - tgt2 = self.linear2(self.dropout3(self.activation(self.linear1(tgt)))) - tgt = tgt + self.dropout4(tgt2) - tgt = self.norm3(tgt) - return tgt - - def forward( - self, - # for tgt - tgt: Optional[Tensor], # nq, bs, d_model - tgt_query_pos: Optional[Tensor] = None, # pos for query. MLP(Sine(pos)) - tgt_query_sine_embed: Optional[Tensor] = None, # pos for query. Sine(pos) - tgt_key_padding_mask: Optional[Tensor] = None, - tgt_reference_points: Optional[Tensor] = None, # nq, bs, 4 - memory_text: Optional[Tensor] = None, # bs, num_token, d_model - text_attention_mask: Optional[Tensor] = None, # bs, num_token - # for memory - memory: Optional[Tensor] = None, # hw, bs, d_model - memory_key_padding_mask: Optional[Tensor] = None, - memory_level_start_index: Optional[Tensor] = None, # num_levels - memory_spatial_shapes: Optional[Tensor] = None, # bs, num_levels, 2 - memory_pos: Optional[Tensor] = None, # pos for memory - # sa - self_attn_mask: Optional[Tensor] = None, # mask used for self-attention - cross_attn_mask: Optional[Tensor] = None, # mask used for cross-attention - ): - """ - Input: - - tgt/tgt_query_pos: nq, bs, d_model - - - """ - assert cross_attn_mask is None - - # self attention - if self.self_attn is not None: - # import ipdb; ipdb.set_trace() - q = k = self.with_pos_embed(tgt, tgt_query_pos) - tgt2 = self.self_attn(q, k, tgt, attn_mask=self_attn_mask)[0] - tgt = tgt + self.dropout2(tgt2) - tgt = self.norm2(tgt) - - if self.use_text_cross_attention: - tgt2 = self.ca_text( - self.with_pos_embed(tgt, tgt_query_pos), - memory_text.transpose(0, 1), - memory_text.transpose(0, 1), - key_padding_mask=text_attention_mask, - )[0] - tgt = tgt + self.catext_dropout(tgt2) - tgt = self.catext_norm(tgt) - - tgt2 = self.cross_attn( - query=self.with_pos_embed(tgt, tgt_query_pos).transpose(0, 1), - reference_points=tgt_reference_points.transpose(0, 1).contiguous(), - value=memory.transpose(0, 1), - spatial_shapes=memory_spatial_shapes, - level_start_index=memory_level_start_index, - key_padding_mask=memory_key_padding_mask, - ).transpose(0, 1) - tgt = tgt + self.dropout1(tgt2) - tgt = self.norm1(tgt) - - # ffn - tgt = self.forward_ffn(tgt) - - return tgt - - -def build_transformer(args): - return Transformer( - d_model=args.hidden_dim, - dropout=args.dropout, - nhead=args.nheads, - num_queries=args.num_queries, - dim_feedforward=args.dim_feedforward, - num_encoder_layers=args.enc_layers, - num_decoder_layers=args.dec_layers, - normalize_before=args.pre_norm, - return_intermediate_dec=True, - query_dim=args.query_dim, - activation=args.transformer_activation, - num_patterns=args.num_patterns, - num_feature_levels=args.num_feature_levels, - enc_n_points=args.enc_n_points, - dec_n_points=args.dec_n_points, - learnable_tgt_init=True, - # two stage - two_stage_type=args.two_stage_type, # ['no', 'standard', 'early'] - embed_init_tgt=args.embed_init_tgt, - use_text_enhancer=args.use_text_enhancer, - use_fusion_layer=args.use_fusion_layer, - use_checkpoint=args.use_checkpoint, - use_transformer_ckpt=args.use_transformer_ckpt, - use_text_cross_attention=args.use_text_cross_attention, - text_dropout=args.text_dropout, - fusion_dropout=args.fusion_dropout, - fusion_droppath=args.fusion_droppath, - ) diff --git a/groundingdino/models/GroundingDINO/.ipynb_checkpoints/transformer_vanilla-checkpoint.py b/groundingdino/models/GroundingDINO/.ipynb_checkpoints/transformer_vanilla-checkpoint.py deleted file mode 100644 index 9840aaac0d325dbaefd50d23bd3705e967609287..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/.ipynb_checkpoints/transformer_vanilla-checkpoint.py +++ /dev/null @@ -1,125 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Copyright (c) Aishwarya Kamath & Nicolas Carion. Licensed under the Apache License 2.0. All Rights Reserved -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -""" -DETR Transformer class. - -Copy-paste from torch.nn.Transformer with modifications: - * positional encodings are passed in MHattention - * extra LN at the end of encoder is removed - * decoder returns a stack of activations from all decoding layers -""" -from typing import Optional - -import torch -import torch.nn.functional as F -from torch import Tensor, nn -import loralib as lora - -from .utils import ( - MLP, - _get_activation_fn, - _get_clones, - gen_encoder_output_proposals, - gen_sineembed_for_position, - sigmoid_focal_loss, -) - - -class TextTransformer(nn.Module): - def __init__(self, num_layers, d_model=256, nheads=8, dim_feedforward=2048, dropout=0.1): - super().__init__() - self.num_layers = num_layers - self.d_model = d_model - self.nheads = nheads - self.dim_feedforward = dim_feedforward - self.norm = None - - single_encoder_layer = TransformerEncoderLayer( - d_model=d_model, nhead=nheads, dim_feedforward=dim_feedforward, dropout=dropout - ) - self.layers = _get_clones(single_encoder_layer, num_layers) - - def forward(self, memory_text: torch.Tensor, text_attention_mask: torch.Tensor): - """ - - Args: - text_attention_mask: bs, num_token - memory_text: bs, num_token, d_model - - Raises: - RuntimeError: _description_ - - Returns: - output: bs, num_token, d_model - """ - - output = memory_text.transpose(0, 1) - - for layer in self.layers: - output = layer(output, src_key_padding_mask=text_attention_mask) - - if self.norm is not None: - output = self.norm(output) - - return output.transpose(0, 1) - - -class TransformerEncoderLayer(nn.Module): - def __init__( - self, - d_model, - nhead, - dim_feedforward=2048, - dropout=0.1, - activation="relu", - normalize_before=False, - ): - super().__init__() - self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) - r = 12 - # Implementation of Feedforward model - self.linear1 = lora.Linear(d_model, dim_feedforward , r=r) - self.dropout = nn.Dropout(dropout) - self.linear2 = lora.Linear(dim_feedforward, d_model , r=r) - - self.norm1 = nn.LayerNorm(d_model) - self.norm2 = nn.LayerNorm(d_model) - self.dropout1 = nn.Dropout(dropout) - self.dropout2 = nn.Dropout(dropout) - - self.activation = _get_activation_fn(activation) - self.normalize_before = normalize_before - self.nhead = nhead - - def with_pos_embed(self, tensor, pos: Optional[Tensor]): - return tensor if pos is None else tensor + pos - - def forward( - self, - src, - src_mask: Optional[Tensor] = None, - src_key_padding_mask: Optional[Tensor] = None, - pos: Optional[Tensor] = None, - ): - # repeat attn mask - if src_mask.dim() == 3 and src_mask.shape[0] == src.shape[1]: - # bs, num_q, num_k - src_mask = src_mask.repeat(self.nhead, 1, 1) - - q = k = self.with_pos_embed(src, pos) - - src2 = self.self_attn(q, k, value=src, attn_mask=src_mask)[0] - - # src2 = self.self_attn(q, k, value=src, attn_mask=src_mask, key_padding_mask=src_key_padding_mask)[0] - src = src + self.dropout1(src2) - src = self.norm1(src) - src2 = self.linear2(self.dropout(self.activation(self.linear1(src)))) - src = src + self.dropout2(src2) - src = self.norm2(src) - return src diff --git a/groundingdino/models/GroundingDINO/.ipynb_checkpoints/utils-checkpoint.py b/groundingdino/models/GroundingDINO/.ipynb_checkpoints/utils-checkpoint.py deleted file mode 100644 index 6f295d43befccceec8ff5285beed1f0b99836013..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/.ipynb_checkpoints/utils-checkpoint.py +++ /dev/null @@ -1,274 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ - -import copy -import math - -import torch -import torch.nn.functional as F -from torch import Tensor, nn -import loralib as lora - -def _get_clones(module, N, layer_share=False): - # import ipdb; ipdb.set_trace() - if layer_share: - return nn.ModuleList([module for i in range(N)]) - else: - return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) - - -def get_sine_pos_embed( - pos_tensor: torch.Tensor, - num_pos_feats: int = 128, - temperature: int = 10000, - exchange_xy: bool = True, -): - """generate sine position embedding from a position tensor - Args: - pos_tensor (torch.Tensor): shape: [..., n]. - num_pos_feats (int): projected shape for each float in the tensor. - temperature (int): temperature in the sine/cosine function. - exchange_xy (bool, optional): exchange pos x and pos y. \ - For example, input tensor is [x,y], the results will be [pos(y), pos(x)]. Defaults to True. - Returns: - pos_embed (torch.Tensor): shape: [..., n*num_pos_feats]. - """ - scale = 2 * math.pi - dim_t = torch.arange(num_pos_feats, dtype=torch.float32, device=pos_tensor.device) - dim_t = temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / num_pos_feats) - - def sine_func(x: torch.Tensor): - sin_x = x * scale / dim_t - sin_x = torch.stack((sin_x[..., 0::2].sin(), sin_x[..., 1::2].cos()), dim=3).flatten(2) - return sin_x - - pos_res = [sine_func(x) for x in pos_tensor.split([1] * pos_tensor.shape[-1], dim=-1)] - if exchange_xy: - pos_res[0], pos_res[1] = pos_res[1], pos_res[0] - pos_res = torch.cat(pos_res, dim=-1) - return pos_res - - -def gen_encoder_output_proposals( - memory: Tensor, memory_padding_mask: Tensor, spatial_shapes: Tensor, learnedwh=None -): - """ - Input: - - memory: bs, \sum{hw}, d_model - - memory_padding_mask: bs, \sum{hw} - - spatial_shapes: nlevel, 2 - - learnedwh: 2 - Output: - - output_memory: bs, \sum{hw}, d_model - - output_proposals: bs, \sum{hw}, 4 - """ - N_, S_, C_ = memory.shape - proposals = [] - _cur = 0 - for lvl, (H_, W_) in enumerate(spatial_shapes): - mask_flatten_ = memory_padding_mask[:, _cur : (_cur + H_ * W_)].view(N_, H_, W_, 1) - valid_H = torch.sum(~mask_flatten_[:, :, 0, 0], 1) - valid_W = torch.sum(~mask_flatten_[:, 0, :, 0], 1) - - # import ipdb; ipdb.set_trace() - - grid_y, grid_x = torch.meshgrid( - torch.linspace(0, H_ - 1, H_, dtype=torch.float32, device=memory.device), - torch.linspace(0, W_ - 1, W_, dtype=torch.float32, device=memory.device), - ) - grid = torch.cat([grid_x.unsqueeze(-1), grid_y.unsqueeze(-1)], -1) # H_, W_, 2 - - scale = torch.cat([valid_W.unsqueeze(-1), valid_H.unsqueeze(-1)], 1).view(N_, 1, 1, 2) - grid = (grid.unsqueeze(0).expand(N_, -1, -1, -1) + 0.5) / scale - - if learnedwh is not None: - # import ipdb; ipdb.set_trace() - wh = torch.ones_like(grid) * learnedwh.sigmoid() * (2.0**lvl) - else: - wh = torch.ones_like(grid) * 0.05 * (2.0**lvl) - - # scale = torch.cat([W_[None].unsqueeze(-1), H_[None].unsqueeze(-1)], 1).view(1, 1, 1, 2).repeat(N_, 1, 1, 1) - # grid = (grid.unsqueeze(0).expand(N_, -1, -1, -1) + 0.5) / scale - # wh = torch.ones_like(grid) / scale - proposal = torch.cat((grid, wh), -1).view(N_, -1, 4) - proposals.append(proposal) - _cur += H_ * W_ - # import ipdb; ipdb.set_trace() - output_proposals = torch.cat(proposals, 1) - output_proposals_valid = ((output_proposals > 0.01) & (output_proposals < 0.99)).all( - -1, keepdim=True - ) - output_proposals = torch.log(output_proposals / (1 - output_proposals)) # unsigmoid - output_proposals = output_proposals.masked_fill(memory_padding_mask.unsqueeze(-1), float("inf")) - output_proposals = output_proposals.masked_fill(~output_proposals_valid, float("inf")) - - output_memory = memory - output_memory = output_memory.masked_fill(memory_padding_mask.unsqueeze(-1), float(0)) - output_memory = output_memory.masked_fill(~output_proposals_valid, float(0)) - - # output_memory = output_memory.masked_fill(memory_padding_mask.unsqueeze(-1), float('inf')) - # output_memory = output_memory.masked_fill(~output_proposals_valid, float('inf')) - - return output_memory, output_proposals - - -class RandomBoxPerturber: - def __init__( - self, x_noise_scale=0.2, y_noise_scale=0.2, w_noise_scale=0.2, h_noise_scale=0.2 - ) -> None: - self.noise_scale = torch.Tensor( - [x_noise_scale, y_noise_scale, w_noise_scale, h_noise_scale] - ) - - def __call__(self, refanchors: Tensor) -> Tensor: - nq, bs, query_dim = refanchors.shape - device = refanchors.device - - noise_raw = torch.rand_like(refanchors) - noise_scale = self.noise_scale.to(device)[:query_dim] - - new_refanchors = refanchors * (1 + (noise_raw - 0.5) * noise_scale) - return new_refanchors.clamp_(0, 1) - - -def sigmoid_focal_loss( - inputs, targets, num_boxes, alpha: float = 0.25, gamma: float = 2, no_reduction=False -): - """ - Loss used in RetinaNet for dense detection: https://arxiv.org/abs/1708.02002. - Args: - inputs: A float tensor of arbitrary shape. - The predictions for each example. - targets: A float tensor with the same shape as inputs. Stores the binary - classification label for each element in inputs - (0 for the negative class and 1 for the positive class). - alpha: (optional) Weighting factor in range (0,1) to balance - positive vs negative examples. Default = -1 (no weighting). - gamma: Exponent of the modulating factor (1 - p_t) to - balance easy vs hard examples. - Returns: - Loss tensor - """ - prob = inputs.sigmoid() - ce_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction="none") - p_t = prob * targets + (1 - prob) * (1 - targets) - loss = ce_loss * ((1 - p_t) ** gamma) - - if alpha >= 0: - alpha_t = alpha * targets + (1 - alpha) * (1 - targets) - loss = alpha_t * loss - - if no_reduction: - return loss - - return loss.mean(1).sum() / num_boxes - - -class MLP(nn.Module): - """Very simple multi-layer perceptron (also called FFN)""" - - def __init__(self, input_dim, hidden_dim, output_dim, num_layers): - super().__init__() - r = 12 - self.num_layers = num_layers - h = [hidden_dim] * (num_layers - 1) - self.layers = nn.ModuleList( - lora.Linear(n, k , r=r) for n, k in zip([input_dim] + h, h + [output_dim]) - ) - - def forward(self, x): - for i, layer in enumerate(self.layers): - x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x) - return x - - -def _get_activation_fn(activation, d_model=256, batch_dim=0): - """Return an activation function given a string""" - if activation == "relu": - return F.relu - if activation == "gelu": - return F.gelu - if activation == "glu": - return F.glu - if activation == "prelu": - return nn.PReLU() - if activation == "selu": - return F.selu - - raise RuntimeError(f"activation should be relu/gelu, not {activation}.") - - -def gen_sineembed_for_position(pos_tensor): - # n_query, bs, _ = pos_tensor.size() - # sineembed_tensor = torch.zeros(n_query, bs, 256) - scale = 2 * math.pi - dim_t = torch.arange(128, dtype=torch.float32, device=pos_tensor.device) - dim_t = 10000 ** (2 * (torch.div(dim_t, 2, rounding_mode='floor')) / 128) - x_embed = pos_tensor[:, :, 0] * scale - y_embed = pos_tensor[:, :, 1] * scale - pos_x = x_embed[:, :, None] / dim_t - pos_y = y_embed[:, :, None] / dim_t - pos_x = torch.stack((pos_x[:, :, 0::2].sin(), pos_x[:, :, 1::2].cos()), dim=3).flatten(2) - pos_y = torch.stack((pos_y[:, :, 0::2].sin(), pos_y[:, :, 1::2].cos()), dim=3).flatten(2) - if pos_tensor.size(-1) == 2: - pos = torch.cat((pos_y, pos_x), dim=2) - elif pos_tensor.size(-1) == 4: - w_embed = pos_tensor[:, :, 2] * scale - pos_w = w_embed[:, :, None] / dim_t - pos_w = torch.stack((pos_w[:, :, 0::2].sin(), pos_w[:, :, 1::2].cos()), dim=3).flatten(2) - - h_embed = pos_tensor[:, :, 3] * scale - pos_h = h_embed[:, :, None] / dim_t - pos_h = torch.stack((pos_h[:, :, 0::2].sin(), pos_h[:, :, 1::2].cos()), dim=3).flatten(2) - - pos = torch.cat((pos_y, pos_x, pos_w, pos_h), dim=2) - else: - raise ValueError("Unknown pos_tensor shape(-1):{}".format(pos_tensor.size(-1))) - return pos - - -class ContrastiveEmbed(nn.Module): - def __init__(self, max_text_len=256): - """ - Args: - max_text_len: max length of text. - """ - super().__init__() - self.max_text_len = max_text_len - - def forward(self, x, text_dict): - """_summary_ - - Args: - x (_type_): _description_ - text_dict (_type_): _description_ - { - 'encoded_text': encoded_text, # bs, 195, d_model - 'text_token_mask': text_token_mask, # bs, 195 - # True for used tokens. False for padding tokens - } - Returns: - _type_: _description_ - """ - assert isinstance(text_dict, dict) - # print(x) #torch.Size([2, 16320, 256]) - # print(text_dict) - - # import pdb;pdb.set_trace() - y = text_dict["encoded_text"] #torch.Size([2, 195, 256]) - text_token_mask = text_dict["text_token_mask"] - - res = x @ y.transpose(-1, -2) - res.masked_fill_(~text_token_mask[:, None, :], float("-inf")) - # 接着，对res进行掩码操作，将未使用的文本token（即padding的token）对应的得分置为负无穷float("-inf")。这是为了在计算相似度时，排除padding部分的影响。 - - - # padding to max_text_len - new_res = torch.full((*res.shape[:-1], self.max_text_len), float("-inf"), device=res.device) - new_res[..., : res.shape[-1]] = res #torch.Size([2, 16320, 195]) - - return new_res diff --git a/groundingdino/models/GroundingDINO/__pycache__/__init__.cpython-310.pyc b/groundingdino/models/GroundingDINO/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 547f457316a6ca609e8b956a68d70d4820845395..0000000000000000000000000000000000000000 Binary files a/groundingdino/models/GroundingDINO/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/models/GroundingDINO/__pycache__/bertwarper.cpython-310.pyc b/groundingdino/models/GroundingDINO/__pycache__/bertwarper.cpython-310.pyc deleted file mode 100644 index 7cf3a5fa5acf9c1bbbb74eca535f3a2e57f44d1f..0000000000000000000000000000000000000000 Binary files a/groundingdino/models/GroundingDINO/__pycache__/bertwarper.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/models/GroundingDINO/__pycache__/fuse_modules.cpython-310.pyc b/groundingdino/models/GroundingDINO/__pycache__/fuse_modules.cpython-310.pyc deleted file mode 100644 index 1203f178d44a0f538a12f29553d2d47258516361..0000000000000000000000000000000000000000 Binary files a/groundingdino/models/GroundingDINO/__pycache__/fuse_modules.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/models/GroundingDINO/__pycache__/groundingdino.cpython-310.pyc b/groundingdino/models/GroundingDINO/__pycache__/groundingdino.cpython-310.pyc deleted file mode 100644 index d9f6f75cdc2e264bacb84084802818f57e433833..0000000000000000000000000000000000000000 Binary files a/groundingdino/models/GroundingDINO/__pycache__/groundingdino.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/models/GroundingDINO/__pycache__/matcher.cpython-310.pyc b/groundingdino/models/GroundingDINO/__pycache__/matcher.cpython-310.pyc deleted file mode 100644 index 962c7994ab4ac2e9f4d8fb21aa3a0339817c42d2..0000000000000000000000000000000000000000 Binary files a/groundingdino/models/GroundingDINO/__pycache__/matcher.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/models/GroundingDINO/__pycache__/ms_deform_attn.cpython-310.pyc b/groundingdino/models/GroundingDINO/__pycache__/ms_deform_attn.cpython-310.pyc deleted file mode 100644 index 084364fc54407d388dcf552bcf94fcf811033491..0000000000000000000000000000000000000000 Binary files a/groundingdino/models/GroundingDINO/__pycache__/ms_deform_attn.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/models/GroundingDINO/__pycache__/transformer.cpython-310.pyc b/groundingdino/models/GroundingDINO/__pycache__/transformer.cpython-310.pyc deleted file mode 100644 index 4433c3134a6f113a2ef81835be9e49c10ac8c3c6..0000000000000000000000000000000000000000 Binary files a/groundingdino/models/GroundingDINO/__pycache__/transformer.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/models/GroundingDINO/__pycache__/transformer_vanilla.cpython-310.pyc b/groundingdino/models/GroundingDINO/__pycache__/transformer_vanilla.cpython-310.pyc deleted file mode 100644 index 682a0d5044a10e90d6bf5b93334bbfa96ddbdcd5..0000000000000000000000000000000000000000 Binary files a/groundingdino/models/GroundingDINO/__pycache__/transformer_vanilla.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/models/GroundingDINO/__pycache__/utils.cpython-310.pyc b/groundingdino/models/GroundingDINO/__pycache__/utils.cpython-310.pyc deleted file mode 100644 index 0426d224a2bdb33bec933c5dfa3e899dd56a8c46..0000000000000000000000000000000000000000 Binary files a/groundingdino/models/GroundingDINO/__pycache__/utils.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/models/GroundingDINO/backbone/.ipynb_checkpoints/__init__-checkpoint.py b/groundingdino/models/GroundingDINO/backbone/.ipynb_checkpoints/__init__-checkpoint.py deleted file mode 100644 index 76e4b272b479a26c63d120c818c140870cd8c287..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/backbone/.ipynb_checkpoints/__init__-checkpoint.py +++ /dev/null @@ -1 +0,0 @@ -from .backbone import build_backbone diff --git a/groundingdino/models/GroundingDINO/backbone/.ipynb_checkpoints/backbone-checkpoint.py b/groundingdino/models/GroundingDINO/backbone/.ipynb_checkpoints/backbone-checkpoint.py deleted file mode 100644 index c8340c723fad8e07e2fc62daaa3912487498814b..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/backbone/.ipynb_checkpoints/backbone-checkpoint.py +++ /dev/null @@ -1,221 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Conditional DETR -# Copyright (c) 2021 Microsoft. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Copied from DETR (https://github.com/facebookresearch/detr) -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -# ------------------------------------------------------------------------ - -""" -Backbone modules. -""" - -from typing import Dict, List - -import torch -import torch.nn.functional as F -import torchvision -from torch import nn -from torchvision.models._utils import IntermediateLayerGetter - -from groundingdino.util.misc import NestedTensor, clean_state_dict, is_main_process - -from .position_encoding import build_position_encoding -from .swin_transformer import build_swin_transformer - - -class FrozenBatchNorm2d(torch.nn.Module): - """ - BatchNorm2d where the batch statistics and the affine parameters are fixed. - - Copy-paste from torchvision.misc.ops with added eps before rqsrt, - without which any other models than torchvision.models.resnet[18,34,50,101] - produce nans. - """ - - def __init__(self, n): - super(FrozenBatchNorm2d, self).__init__() - self.register_buffer("weight", torch.ones(n)) - self.register_buffer("bias", torch.zeros(n)) - self.register_buffer("running_mean", torch.zeros(n)) - self.register_buffer("running_var", torch.ones(n)) - - def _load_from_state_dict( - self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs - ): - num_batches_tracked_key = prefix + "num_batches_tracked" - if num_batches_tracked_key in state_dict: - del state_dict[num_batches_tracked_key] - - super(FrozenBatchNorm2d, self)._load_from_state_dict( - state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs - ) - - def forward(self, x): - # move reshapes to the beginning - # to make it fuser-friendly - w = self.weight.reshape(1, -1, 1, 1) - b = self.bias.reshape(1, -1, 1, 1) - rv = self.running_var.reshape(1, -1, 1, 1) - rm = self.running_mean.reshape(1, -1, 1, 1) - eps = 1e-5 - scale = w * (rv + eps).rsqrt() - bias = b - rm * scale - return x * scale + bias - - -class BackboneBase(nn.Module): - def __init__( - self, - backbone: nn.Module, - train_backbone: bool, - num_channels: int, - return_interm_indices: list, - ): - super().__init__() - for name, parameter in backbone.named_parameters(): - if ( - not train_backbone - or "layer2" not in name - and "layer3" not in name - and "layer4" not in name - ): - parameter.requires_grad_(False) - - return_layers = {} - for idx, layer_index in enumerate(return_interm_indices): - return_layers.update( - {"layer{}".format(5 - len(return_interm_indices) + idx): "{}".format(layer_index)} - ) - - # if len: - # if use_stage1_feature: - # return_layers = {"layer1": "0", "layer2": "1", "layer3": "2", "layer4": "3"} - # else: - # return_layers = {"layer2": "0", "layer3": "1", "layer4": "2"} - # else: - # return_layers = {'layer4': "0"} - self.body = IntermediateLayerGetter(backbone, return_layers=return_layers) - self.num_channels = num_channels - - def forward(self, tensor_list: NestedTensor): - xs = self.body(tensor_list.tensors) - out: Dict[str, NestedTensor] = {} - for name, x in xs.items(): - m = tensor_list.mask - assert m is not None - mask = F.interpolate(m[None].float(), size=x.shape[-2:]).to(torch.bool)[0] - out[name] = NestedTensor(x, mask) - # import ipdb; ipdb.set_trace() - return out - - -class Backbone(BackboneBase): - """ResNet backbone with frozen BatchNorm.""" - - def __init__( - self, - name: str, - train_backbone: bool, - dilation: bool, - return_interm_indices: list, - batch_norm=FrozenBatchNorm2d, - ): - if name in ["resnet18", "resnet34", "resnet50", "resnet101"]: - backbone = getattr(torchvision.models, name)( - replace_stride_with_dilation=[False, False, dilation], - pretrained=is_main_process(), - norm_layer=batch_norm, - ) - else: - raise NotImplementedError("Why you can get here with name {}".format(name)) - # num_channels = 512 if name in ('resnet18', 'resnet34') else 2048 - assert name not in ("resnet18", "resnet34"), "Only resnet50 and resnet101 are available." - assert return_interm_indices in [[0, 1, 2, 3], [1, 2, 3], [3]] - num_channels_all = [256, 512, 1024, 2048] - num_channels = num_channels_all[4 - len(return_interm_indices) :] - super().__init__(backbone, train_backbone, num_channels, return_interm_indices) - - -class Joiner(nn.Sequential): - def __init__(self, backbone, position_embedding): - super().__init__(backbone, position_embedding) - - def forward(self, tensor_list: NestedTensor): - xs = self[0](tensor_list) - out: List[NestedTensor] = [] - pos = [] - for name, x in xs.items(): - out.append(x) - # position encoding - pos.append(self[1](x).to(x.tensors.dtype)) - - return out, pos - - -def build_backbone(args): - """ - Useful args: - - backbone: backbone name - - lr_backbone: - - dilation - - return_interm_indices: available: [0,1,2,3], [1,2,3], [3] - - backbone_freeze_keywords: - - use_checkpoint: for swin only for now - - """ - position_embedding = build_position_encoding(args) - train_backbone = True - if not train_backbone: - raise ValueError("Please set lr_backbone > 0") - return_interm_indices = args.return_interm_indices - assert return_interm_indices in [[0, 1, 2, 3], [1, 2, 3], [3]] - args.backbone_freeze_keywords - use_checkpoint = getattr(args, "use_checkpoint", False) - - if args.backbone in ["resnet50", "resnet101"]: - backbone = Backbone( - args.backbone, - train_backbone, - args.dilation, - return_interm_indices, - batch_norm=FrozenBatchNorm2d, - ) - bb_num_channels = backbone.num_channels - elif args.backbone in [ - "swin_T_224_1k", - "swin_B_224_22k", - "swin_B_384_22k", - "swin_L_224_22k", - "swin_L_384_22k", - ]: - pretrain_img_size = int(args.backbone.split("_")[-2]) - backbone = build_swin_transformer( - args.backbone, - pretrain_img_size=pretrain_img_size, - out_indices=tuple(return_interm_indices), - dilation=False, - use_checkpoint=use_checkpoint, - ) - - bb_num_channels = backbone.num_features[4 - len(return_interm_indices) :] - else: - raise NotImplementedError("Unknown backbone {}".format(args.backbone)) - - assert len(bb_num_channels) == len( - return_interm_indices - ), f"len(bb_num_channels) {len(bb_num_channels)} != len(return_interm_indices) {len(return_interm_indices)}" - - model = Joiner(backbone, position_embedding) - model.num_channels = bb_num_channels - assert isinstance( - bb_num_channels, List - ), "bb_num_channels is expected to be a List but {}".format(type(bb_num_channels)) - # import ipdb; ipdb.set_trace() - return model diff --git a/groundingdino/models/GroundingDINO/backbone/.ipynb_checkpoints/position_encoding-checkpoint.py b/groundingdino/models/GroundingDINO/backbone/.ipynb_checkpoints/position_encoding-checkpoint.py deleted file mode 100644 index eac7e896bbe85a670824bfe8ef487d0535d5bd99..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/backbone/.ipynb_checkpoints/position_encoding-checkpoint.py +++ /dev/null @@ -1,186 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# DINO -# Copyright (c) 2022 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Conditional DETR -# Copyright (c) 2021 Microsoft. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Copied from DETR (https://github.com/facebookresearch/detr) -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -# ------------------------------------------------------------------------ - -""" -Various positional encodings for the transformer. -""" -import math - -import torch -from torch import nn - -from groundingdino.util.misc import NestedTensor - - -class PositionEmbeddingSine(nn.Module): - """ - This is a more standard version of the position embedding, very similar to the one - used by the Attention is all you need paper, generalized to work on images. - """ - - def __init__(self, num_pos_feats=64, temperature=10000, normalize=False, scale=None): - super().__init__() - self.num_pos_feats = num_pos_feats - self.temperature = temperature - self.normalize = normalize - if scale is not None and normalize is False: - raise ValueError("normalize should be True if scale is passed") - if scale is None: - scale = 2 * math.pi - self.scale = scale - - def forward(self, tensor_list: NestedTensor): - x = tensor_list.tensors - mask = tensor_list.mask - assert mask is not None - not_mask = ~mask - y_embed = not_mask.cumsum(1, dtype=torch.float32) - x_embed = not_mask.cumsum(2, dtype=torch.float32) - if self.normalize: - eps = 1e-6 - # if os.environ.get("SHILONG_AMP", None) == '1': - # eps = 1e-4 - # else: - # eps = 1e-6 - y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale - x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale - - dim_t = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device) - dim_t = self.temperature ** (2 * (dim_t // 2) / self.num_pos_feats) - - pos_x = x_embed[:, :, :, None] / dim_t - pos_y = y_embed[:, :, :, None] / dim_t - pos_x = torch.stack( - (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4 - ).flatten(3) - pos_y = torch.stack( - (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4 - ).flatten(3) - pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) - return pos - - -class PositionEmbeddingSineHW(nn.Module): - """ - This is a more standard version of the position embedding, very similar to the one - used by the Attention is all you need paper, generalized to work on images. - """ - - def __init__( - self, num_pos_feats=64, temperatureH=10000, temperatureW=10000, normalize=False, scale=None - ): - super().__init__() - self.num_pos_feats = num_pos_feats - self.temperatureH = temperatureH - self.temperatureW = temperatureW - self.normalize = normalize - if scale is not None and normalize is False: - raise ValueError("normalize should be True if scale is passed") - if scale is None: - scale = 2 * math.pi - self.scale = scale - - def forward(self, tensor_list: NestedTensor): - x = tensor_list.tensors - mask = tensor_list.mask - assert mask is not None - not_mask = ~mask - y_embed = not_mask.cumsum(1, dtype=torch.float32) - x_embed = not_mask.cumsum(2, dtype=torch.float32) - - # import ipdb; ipdb.set_trace() - - if self.normalize: - eps = 1e-6 - y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale - x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale - - dim_tx = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device) - dim_tx = self.temperatureW ** (2 * (torch.div(dim_tx, 2, rounding_mode='floor')) / self.num_pos_feats) - pos_x = x_embed[:, :, :, None] / dim_tx - - dim_ty = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device) - dim_ty = self.temperatureH ** (2 * (torch.div(dim_ty, 2, rounding_mode='floor')) / self.num_pos_feats) - pos_y = y_embed[:, :, :, None] / dim_ty - - pos_x = torch.stack( - (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4 - ).flatten(3) - pos_y = torch.stack( - (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4 - ).flatten(3) - pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) - - # import ipdb; ipdb.set_trace() - - return pos - - -class PositionEmbeddingLearned(nn.Module): - """ - Absolute pos embedding, learned. - """ - - def __init__(self, num_pos_feats=256): - super().__init__() - self.row_embed = nn.Embedding(50, num_pos_feats) - self.col_embed = nn.Embedding(50, num_pos_feats) - self.reset_parameters() - - def reset_parameters(self): - nn.init.uniform_(self.row_embed.weight) - nn.init.uniform_(self.col_embed.weight) - - def forward(self, tensor_list: NestedTensor): - x = tensor_list.tensors - h, w = x.shape[-2:] - i = torch.arange(w, device=x.device) - j = torch.arange(h, device=x.device) - x_emb = self.col_embed(i) - y_emb = self.row_embed(j) - pos = ( - torch.cat( - [ - x_emb.unsqueeze(0).repeat(h, 1, 1), - y_emb.unsqueeze(1).repeat(1, w, 1), - ], - dim=-1, - ) - .permute(2, 0, 1) - .unsqueeze(0) - .repeat(x.shape[0], 1, 1, 1) - ) - return pos - - -def build_position_encoding(args): - N_steps = args.hidden_dim // 2 - if args.position_embedding in ("v2", "sine"): - # TODO find a better way of exposing other arguments - position_embedding = PositionEmbeddingSineHW( - N_steps, - temperatureH=args.pe_temperatureH, - temperatureW=args.pe_temperatureW, - normalize=True, - ) - elif args.position_embedding in ("v3", "learned"): - position_embedding = PositionEmbeddingLearned(N_steps) - else: - raise ValueError(f"not supported {args.position_embedding}") - - return position_embedding diff --git a/groundingdino/models/GroundingDINO/backbone/.ipynb_checkpoints/swin_transformer-checkpoint.py b/groundingdino/models/GroundingDINO/backbone/.ipynb_checkpoints/swin_transformer-checkpoint.py deleted file mode 100644 index 0cab63162840b6adf4f3404a6ced06c8c4e85ed6..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/backbone/.ipynb_checkpoints/swin_transformer-checkpoint.py +++ /dev/null @@ -1,804 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# DINO -# Copyright (c) 2022 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# -------------------------------------------------------- -# modified from https://github.com/SwinTransformer/Swin-Transformer-Object-Detection/blob/master/mmdet/models/backbones/swin_transformer.py -# -------------------------------------------------------- - -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.utils.checkpoint as checkpoint -from timm.models.layers import DropPath, to_2tuple, trunc_normal_ -import loralib as lora -from groundingdino.util.misc import NestedTensor - - -class Mlp(nn.Module): - """Multilayer perceptron.""" - - def __init__( - self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.0 - ): - super().__init__() - r = 12 - out_features = out_features or in_features - hidden_features = hidden_features or in_features - self.fc1 = lora.Linear(in_features, hidden_features , r=r) - self.act = act_layer() - self.fc2 = lora.Linear(hidden_features, out_features , r=r) - self.drop = nn.Dropout(drop) - - def forward(self, x): - x = self.fc1(x) - x = self.act(x) - x = self.drop(x) - x = self.fc2(x) - x = self.drop(x) - return x - - -def window_partition(x, window_size): - """ - Args: - x: (B, H, W, C) - window_size (int): window size - Returns: - windows: (num_windows*B, window_size, window_size, C) - """ - B, H, W, C = x.shape - x = x.view(B, H // window_size, window_size, W // window_size, window_size, C) - windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) - return windows - - -def window_reverse(windows, window_size, H, W): - """ - Args: - windows: (num_windows*B, window_size, window_size, C) - window_size (int): Window size - H (int): Height of image - W (int): Width of image - Returns: - x: (B, H, W, C) - """ - B = int(windows.shape[0] / (H * W / window_size / window_size)) - x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1) - x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) - return x - - -class WindowAttention(nn.Module): - """Window based multi-head self attention (W-MSA) module with relative position bias. - It supports both of shifted and non-shifted window. - Args: - dim (int): Number of input channels. - window_size (tuple[int]): The height and width of the window. - num_heads (int): Number of attention heads. - qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True - qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set - attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 - proj_drop (float, optional): Dropout ratio of output. Default: 0.0 - """ - - def __init__( - self, - dim, - window_size, - num_heads, - qkv_bias=True, - qk_scale=None, - attn_drop=0.0, - proj_drop=0.0, - ): - - super().__init__() - self.dim = dim - self.window_size = window_size # Wh, Ww - self.num_heads = num_heads - head_dim = dim // num_heads - self.scale = qk_scale or head_dim**-0.5 - - # define a parameter table of relative position bias - self.relative_position_bias_table = nn.Parameter( - torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads) - ) # 2*Wh-1 * 2*Ww-1, nH - - # get pair-wise relative position index for each token inside the window - coords_h = torch.arange(self.window_size[0]) - coords_w = torch.arange(self.window_size[1]) - coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww - coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww - relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww - relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 - relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0 - relative_coords[:, :, 1] += self.window_size[1] - 1 - relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 - relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww - self.register_buffer("relative_position_index", relative_position_index) - r = 12 - self.qkv = lora.Linear(dim, dim * 3, r=r , bias=qkv_bias) - self.attn_drop = nn.Dropout(attn_drop) - self.proj = lora.Linear(dim, dim , r=r) - self.proj_drop = nn.Dropout(proj_drop) - - trunc_normal_(self.relative_position_bias_table, std=0.02) - self.softmax = nn.Softmax(dim=-1) - - def forward(self, x, mask=None): - """Forward function. - Args: - x: input features with shape of (num_windows*B, N, C) - mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None - """ - B_, N, C = x.shape - qkv = ( - self.qkv(x) - .reshape(B_, N, 3, self.num_heads, C // self.num_heads) - .permute(2, 0, 3, 1, 4) - ) - q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) - - q = q * self.scale - attn = q @ k.transpose(-2, -1) - - relative_position_bias = self.relative_position_bias_table[ - self.relative_position_index.view(-1) - ].view( - self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1 - ) # Wh*Ww,Wh*Ww,nH - relative_position_bias = relative_position_bias.permute( - 2, 0, 1 - ).contiguous() # nH, Wh*Ww, Wh*Ww - attn = attn + relative_position_bias.unsqueeze(0) - - if mask is not None: - nW = mask.shape[0] - attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0) - attn = attn.view(-1, self.num_heads, N, N) - attn = self.softmax(attn) - else: - attn = self.softmax(attn) - - attn = self.attn_drop(attn) - - x = (attn @ v).transpose(1, 2).reshape(B_, N, C) - x = self.proj(x) - x = self.proj_drop(x) - return x - - -class SwinTransformerBlock(nn.Module): - """Swin Transformer Block. - Args: - dim (int): Number of input channels. - num_heads (int): Number of attention heads. - window_size (int): Window size. - shift_size (int): Shift size for SW-MSA. - mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. - qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True - qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. - drop (float, optional): Dropout rate. Default: 0.0 - attn_drop (float, optional): Attention dropout rate. Default: 0.0 - drop_path (float, optional): Stochastic depth rate. Default: 0.0 - act_layer (nn.Module, optional): Activation layer. Default: nn.GELU - norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm - """ - - def __init__( - self, - dim, - num_heads, - window_size=7, - shift_size=0, - mlp_ratio=4.0, - qkv_bias=True, - qk_scale=None, - drop=0.0, - attn_drop=0.0, - drop_path=0.0, - act_layer=nn.GELU, - norm_layer=nn.LayerNorm, - ): - super().__init__() - self.dim = dim - self.num_heads = num_heads - self.window_size = window_size - self.shift_size = shift_size - self.mlp_ratio = mlp_ratio - assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size" - - self.norm1 = norm_layer(dim) - self.attn = WindowAttention( - dim, - window_size=to_2tuple(self.window_size), - num_heads=num_heads, - qkv_bias=qkv_bias, - qk_scale=qk_scale, - attn_drop=attn_drop, - proj_drop=drop, - ) - - self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() - self.norm2 = norm_layer(dim) - mlp_hidden_dim = int(dim * mlp_ratio) - self.mlp = Mlp( - in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop - ) - - self.H = None - self.W = None - - def forward(self, x, mask_matrix): - """Forward function. - Args: - x: Input feature, tensor size (B, H*W, C). - H, W: Spatial resolution of the input feature. - mask_matrix: Attention mask for cyclic shift. - """ - B, L, C = x.shape - H, W = self.H, self.W - assert L == H * W, "input feature has wrong size" - - shortcut = x - x = self.norm1(x) - x = x.view(B, H, W, C) - - # pad feature maps to multiples of window size - pad_l = pad_t = 0 - pad_r = (self.window_size - W % self.window_size) % self.window_size - pad_b = (self.window_size - H % self.window_size) % self.window_size - x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b)) - _, Hp, Wp, _ = x.shape - - # cyclic shift - if self.shift_size > 0: - shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) - attn_mask = mask_matrix - else: - shifted_x = x - attn_mask = None - - # partition windows - x_windows = window_partition( - shifted_x, self.window_size - ) # nW*B, window_size, window_size, C - x_windows = x_windows.view( - -1, self.window_size * self.window_size, C - ) # nW*B, window_size*window_size, C - - # W-MSA/SW-MSA - attn_windows = self.attn(x_windows, mask=attn_mask) # nW*B, window_size*window_size, C - - # merge windows - attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C) - shifted_x = window_reverse(attn_windows, self.window_size, Hp, Wp) # B H' W' C - - # reverse cyclic shift - if self.shift_size > 0: - x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2)) - else: - x = shifted_x - - if pad_r > 0 or pad_b > 0: - x = x[:, :H, :W, :].contiguous() - - x = x.view(B, H * W, C) - - # FFN - x = shortcut + self.drop_path(x) - x = x + self.drop_path(self.mlp(self.norm2(x))) - - return x - - -class PatchMerging(nn.Module): - """Patch Merging Layer - Args: - dim (int): Number of input channels. - norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm - """ - - def __init__(self, dim, norm_layer=nn.LayerNorm): - super().__init__() - r = 24 - self.dim = dim - self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False) - self.norm = norm_layer(4 * dim) - - def forward(self, x, H, W): - """Forward function. - Args: - x: Input feature, tensor size (B, H*W, C). - H, W: Spatial resolution of the input feature. - """ - B, L, C = x.shape - assert L == H * W, "input feature has wrong size" - - x = x.view(B, H, W, C) - - # padding - pad_input = (H % 2 == 1) or (W % 2 == 1) - if pad_input: - x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2)) - - x0 = x[:, 0::2, 0::2, :] # B H/2 W/2 C - x1 = x[:, 1::2, 0::2, :] # B H/2 W/2 C - x2 = x[:, 0::2, 1::2, :] # B H/2 W/2 C - x3 = x[:, 1::2, 1::2, :] # B H/2 W/2 C - x = torch.cat([x0, x1, x2, x3], -1) # B H/2 W/2 4*C - x = x.view(B, -1, 4 * C) # B H/2*W/2 4*C - - x = self.norm(x) - x = self.reduction(x) - - return x - - -class BasicLayer(nn.Module): - """A basic Swin Transformer layer for one stage. - Args: - dim (int): Number of feature channels - depth (int): Depths of this stage. - num_heads (int): Number of attention head. - window_size (int): Local window size. Default: 7. - mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. - qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True - qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. - drop (float, optional): Dropout rate. Default: 0.0 - attn_drop (float, optional): Attention dropout rate. Default: 0.0 - drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 - norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm - downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None - use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. - """ - - def __init__( - self, - dim, - depth, - num_heads, - window_size=7, - mlp_ratio=4.0, - qkv_bias=True, - qk_scale=None, - drop=0.0, - attn_drop=0.0, - drop_path=0.0, - norm_layer=nn.LayerNorm, - downsample=None, - use_checkpoint=False, - ): - super().__init__() - self.window_size = window_size - self.shift_size = window_size // 2 - self.depth = depth - self.use_checkpoint = use_checkpoint - - # build blocks - self.blocks = nn.ModuleList( - [ - SwinTransformerBlock( - dim=dim, - num_heads=num_heads, - window_size=window_size, - shift_size=0 if (i % 2 == 0) else window_size // 2, - mlp_ratio=mlp_ratio, - qkv_bias=qkv_bias, - qk_scale=qk_scale, - drop=drop, - attn_drop=attn_drop, - drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path, - norm_layer=norm_layer, - ) - for i in range(depth) - ] - ) - - # patch merging layer - if downsample is not None: - self.downsample = downsample(dim=dim, norm_layer=norm_layer) - else: - self.downsample = None - - def forward(self, x, H, W): - """Forward function. - Args: - x: Input feature, tensor size (B, H*W, C). - H, W: Spatial resolution of the input feature. - """ - - # calculate attention mask for SW-MSA - Hp = int(np.ceil(H / self.window_size)) * self.window_size - Wp = int(np.ceil(W / self.window_size)) * self.window_size - img_mask = torch.zeros((1, Hp, Wp, 1), device=x.device) # 1 Hp Wp 1 - h_slices = ( - slice(0, -self.window_size), - slice(-self.window_size, -self.shift_size), - slice(-self.shift_size, None), - ) - w_slices = ( - slice(0, -self.window_size), - slice(-self.window_size, -self.shift_size), - slice(-self.shift_size, None), - ) - cnt = 0 - for h in h_slices: - for w in w_slices: - img_mask[:, h, w, :] = cnt - cnt += 1 - - mask_windows = window_partition( - img_mask, self.window_size - ) # nW, window_size, window_size, 1 - mask_windows = mask_windows.view(-1, self.window_size * self.window_size) - attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) - attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill( - attn_mask == 0, float(0.0) - ) - - for blk in self.blocks: - blk.H, blk.W = H, W - if self.use_checkpoint: - x = checkpoint.checkpoint(blk, x, attn_mask) - else: - x = blk(x, attn_mask) - if self.downsample is not None: - x_down = self.downsample(x, H, W) - Wh, Ww = (H + 1) // 2, (W + 1) // 2 - return x, H, W, x_down, Wh, Ww - else: - return x, H, W, x, H, W - - -class PatchEmbed(nn.Module): - """Image to Patch Embedding - Args: - patch_size (int): Patch token size. Default: 4. - in_chans (int): Number of input image channels. Default: 3. - embed_dim (int): Number of linear projection output channels. Default: 96. - norm_layer (nn.Module, optional): Normalization layer. Default: None - """ - - def __init__(self, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None): - super().__init__() - patch_size = to_2tuple(patch_size) - self.patch_size = patch_size - - self.in_chans = in_chans - self.embed_dim = embed_dim - - self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) - if norm_layer is not None: - self.norm = norm_layer(embed_dim) - else: - self.norm = None - - def forward(self, x): - """Forward function.""" - # padding - _, _, H, W = x.size() - if W % self.patch_size[1] != 0: - x = F.pad(x, (0, self.patch_size[1] - W % self.patch_size[1])) - if H % self.patch_size[0] != 0: - x = F.pad(x, (0, 0, 0, self.patch_size[0] - H % self.patch_size[0])) - - x = self.proj(x) # B C Wh Ww - if self.norm is not None: - Wh, Ww = x.size(2), x.size(3) - x = x.flatten(2).transpose(1, 2) - x = self.norm(x) - x = x.transpose(1, 2).view(-1, self.embed_dim, Wh, Ww) - - return x - - -class SwinTransformer(nn.Module): - """Swin Transformer backbone. - A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows` - - https://arxiv.org/pdf/2103.14030 - Args: - pretrain_img_size (int): Input image size for training the pretrained model, - used in absolute postion embedding. Default 224. - patch_size (int | tuple(int)): Patch size. Default: 4. - in_chans (int): Number of input image channels. Default: 3. - embed_dim (int): Number of linear projection output channels. Default: 96. - depths (tuple[int]): Depths of each Swin Transformer stage. - num_heads (tuple[int]): Number of attention head of each stage. - window_size (int): Window size. Default: 7. - mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. - qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True - qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. - drop_rate (float): Dropout rate. - attn_drop_rate (float): Attention dropout rate. Default: 0. - drop_path_rate (float): Stochastic depth rate. Default: 0.2. - norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm. - ape (bool): If True, add absolute position embedding to the patch embedding. Default: False. - patch_norm (bool): If True, add normalization after patch embedding. Default: True. - out_indices (Sequence[int]): Output from which stages. - frozen_stages (int): Stages to be frozen (stop grad and set eval mode). - -1 means not freezing any parameters. - use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. - dilation (bool): if True, the output size if 16x downsample, ow 32x downsample. - """ - - def __init__( - self, - pretrain_img_size=224, - patch_size=4, - in_chans=3, - embed_dim=96, - depths=[2, 2, 6, 2], - num_heads=[3, 6, 12, 24], - window_size=7, - mlp_ratio=4.0, - qkv_bias=True, - qk_scale=None, - drop_rate=0.0, - attn_drop_rate=0.0, - drop_path_rate=0.2, - norm_layer=nn.LayerNorm, - ape=False, - patch_norm=True, - out_indices=(0, 1, 2, 3), - frozen_stages=-1, - dilation=False, - use_checkpoint=False, - ): - super().__init__() - - self.pretrain_img_size = pretrain_img_size - self.num_layers = len(depths) - self.embed_dim = embed_dim - self.ape = ape - self.patch_norm = patch_norm - self.out_indices = out_indices - self.frozen_stages = frozen_stages - self.dilation = dilation - - # if use_checkpoint: - # print("use_checkpoint!!!!!!!!!!!!!!!!!!!!!!!!") - - # split image into non-overlapping patches - self.patch_embed = PatchEmbed( - patch_size=patch_size, - in_chans=in_chans, - embed_dim=embed_dim, - norm_layer=norm_layer if self.patch_norm else None, - ) - - # absolute position embedding - if self.ape: - pretrain_img_size = to_2tuple(pretrain_img_size) - patch_size = to_2tuple(patch_size) - patches_resolution = [ - pretrain_img_size[0] // patch_size[0], - pretrain_img_size[1] // patch_size[1], - ] - - self.absolute_pos_embed = nn.Parameter( - torch.zeros(1, embed_dim, patches_resolution[0], patches_resolution[1]) - ) - trunc_normal_(self.absolute_pos_embed, std=0.02) - - self.pos_drop = nn.Dropout(p=drop_rate) - - # stochastic depth - dpr = [ - x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) - ] # stochastic depth decay rule - - # build layers - self.layers = nn.ModuleList() - # prepare downsample list - downsamplelist = [PatchMerging for i in range(self.num_layers)] - downsamplelist[-1] = None - num_features = [int(embed_dim * 2**i) for i in range(self.num_layers)] - if self.dilation: - downsamplelist[-2] = None - num_features[-1] = int(embed_dim * 2 ** (self.num_layers - 1)) // 2 - for i_layer in range(self.num_layers): - layer = BasicLayer( - # dim=int(embed_dim * 2 ** i_layer), - dim=num_features[i_layer], - depth=depths[i_layer], - num_heads=num_heads[i_layer], - window_size=window_size, - mlp_ratio=mlp_ratio, - qkv_bias=qkv_bias, - qk_scale=qk_scale, - drop=drop_rate, - attn_drop=attn_drop_rate, - drop_path=dpr[sum(depths[:i_layer]) : sum(depths[: i_layer + 1])], - norm_layer=norm_layer, - # downsample=PatchMerging if (i_layer < self.num_layers - 1) else None, - downsample=downsamplelist[i_layer], - use_checkpoint=use_checkpoint, - ) - self.layers.append(layer) - - # num_features = [int(embed_dim * 2 ** i) for i in range(self.num_layers)] - self.num_features = num_features - - # add a norm layer for each output - for i_layer in out_indices: - layer = norm_layer(num_features[i_layer]) - layer_name = f"norm{i_layer}" - self.add_module(layer_name, layer) - - self._freeze_stages() - - def _freeze_stages(self): - if self.frozen_stages >= 0: - self.patch_embed.eval() - for param in self.patch_embed.parameters(): - param.requires_grad = False - - if self.frozen_stages >= 1 and self.ape: - self.absolute_pos_embed.requires_grad = False - - if self.frozen_stages >= 2: - self.pos_drop.eval() - for i in range(0, self.frozen_stages - 1): - m = self.layers[i] - m.eval() - for param in m.parameters(): - param.requires_grad = False - - # def init_weights(self, pretrained=None): - # """Initialize the weights in backbone. - # Args: - # pretrained (str, optional): Path to pre-trained weights. - # Defaults to None. - # """ - - # def _init_weights(m): - # if isinstance(m, nn.Linear): - # trunc_normal_(m.weight, std=.02) - # if isinstance(m, nn.Linear) and m.bias is not None: - # nn.init.constant_(m.bias, 0) - # elif isinstance(m, nn.LayerNorm): - # nn.init.constant_(m.bias, 0) - # nn.init.constant_(m.weight, 1.0) - - # if isinstance(pretrained, str): - # self.apply(_init_weights) - # logger = get_root_logger() - # load_checkpoint(self, pretrained, strict=False, logger=logger) - # elif pretrained is None: - # self.apply(_init_weights) - # else: - # raise TypeError('pretrained must be a str or None') - - def forward_raw(self, x): - """Forward function.""" - x = self.patch_embed(x) - - Wh, Ww = x.size(2), x.size(3) - if self.ape: - # interpolate the position embedding to the corresponding size - absolute_pos_embed = F.interpolate( - self.absolute_pos_embed, size=(Wh, Ww), mode="bicubic" - ) - x = (x + absolute_pos_embed).flatten(2).transpose(1, 2) # B Wh*Ww C - else: - x = x.flatten(2).transpose(1, 2) - x = self.pos_drop(x) - - outs = [] - for i in range(self.num_layers): - layer = self.layers[i] - x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww) - # import ipdb; ipdb.set_trace() - - if i in self.out_indices: - norm_layer = getattr(self, f"norm{i}") - x_out = norm_layer(x_out) - - out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous() - outs.append(out) - # in: - # torch.Size([2, 3, 1024, 1024]) - # outs: - # [torch.Size([2, 192, 256, 256]), torch.Size([2, 384, 128, 128]), \ - # torch.Size([2, 768, 64, 64]), torch.Size([2, 1536, 32, 32])] - return tuple(outs) - - def forward(self, tensor_list: NestedTensor): - x = tensor_list.tensors - - """Forward function.""" - x = self.patch_embed(x) - - Wh, Ww = x.size(2), x.size(3) - if self.ape: - # interpolate the position embedding to the corresponding size - absolute_pos_embed = F.interpolate( - self.absolute_pos_embed, size=(Wh, Ww), mode="bicubic" - ) - x = (x + absolute_pos_embed).flatten(2).transpose(1, 2) # B Wh*Ww C - else: - x = x.flatten(2).transpose(1, 2) - x = self.pos_drop(x) - - outs = [] - for i in range(self.num_layers): - layer = self.layers[i] - x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww) - - if i in self.out_indices: - norm_layer = getattr(self, f"norm{i}") - x_out = norm_layer(x_out) - - out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous() - outs.append(out) - # in: - # torch.Size([2, 3, 1024, 1024]) - # out: - # [torch.Size([2, 192, 256, 256]), torch.Size([2, 384, 128, 128]), \ - # torch.Size([2, 768, 64, 64]), torch.Size([2, 1536, 32, 32])] - - # collect for nesttensors - outs_dict = {} - for idx, out_i in enumerate(outs): - m = tensor_list.mask - assert m is not None - mask = F.interpolate(m[None].float(), size=out_i.shape[-2:]).to(torch.bool)[0] - outs_dict[idx] = NestedTensor(out_i, mask) - - return outs_dict - - def train(self, mode=True): - """Convert the model into training mode while keep layers freezed.""" - super(SwinTransformer, self).train(mode) - self._freeze_stages() - - -def build_swin_transformer(modelname, pretrain_img_size, **kw): - assert modelname in [ - "swin_T_224_1k", - "swin_B_224_22k", - "swin_B_384_22k", - "swin_L_224_22k", - "swin_L_384_22k", - ] - - model_para_dict = { - "swin_T_224_1k": dict( - embed_dim=96, depths=[2, 2, 6, 2], num_heads=[3, 6, 12, 24], window_size=7 - ), - "swin_B_224_22k": dict( - embed_dim=128, depths=[2, 2, 18, 2], num_heads=[4, 8, 16, 32], window_size=7 - ), - "swin_B_384_22k": dict( - embed_dim=128, depths=[2, 2, 18, 2], num_heads=[4, 8, 16, 32], window_size=12 - ), - "swin_L_224_22k": dict( - embed_dim=192, depths=[2, 2, 18, 2], num_heads=[6, 12, 24, 48], window_size=7 - ), - "swin_L_384_22k": dict( - embed_dim=192, depths=[2, 2, 18, 2], num_heads=[6, 12, 24, 48], window_size=12 - ), - } - kw_cgf = model_para_dict[modelname] - kw_cgf.update(kw) - model = SwinTransformer(pretrain_img_size=pretrain_img_size, **kw_cgf) - return model - - -if __name__ == "__main__": - model = build_swin_transformer("swin_L_384_22k", 384, dilation=True) - x = torch.rand(2, 3, 1024, 1024) - y = model.forward_raw(x) - import ipdb - - ipdb.set_trace() - x = torch.rand(2, 3, 384, 384) - y = model.forward_raw(x) diff --git a/groundingdino/models/GroundingDINO/backbone/__pycache__/__init__.cpython-310.pyc b/groundingdino/models/GroundingDINO/backbone/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index e2ab670e8ab5143166f123e968e5080c975cacc1..0000000000000000000000000000000000000000 Binary files a/groundingdino/models/GroundingDINO/backbone/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/models/GroundingDINO/backbone/__pycache__/backbone.cpython-310.pyc b/groundingdino/models/GroundingDINO/backbone/__pycache__/backbone.cpython-310.pyc deleted file mode 100644 index 74bbcea77bbb7fa029abf15d1a8aa071164e633d..0000000000000000000000000000000000000000 Binary files a/groundingdino/models/GroundingDINO/backbone/__pycache__/backbone.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/models/GroundingDINO/backbone/__pycache__/position_encoding.cpython-310.pyc b/groundingdino/models/GroundingDINO/backbone/__pycache__/position_encoding.cpython-310.pyc deleted file mode 100644 index c08fafa9db27553aa00d8cc54f69ded5b59eaaa9..0000000000000000000000000000000000000000 Binary files a/groundingdino/models/GroundingDINO/backbone/__pycache__/position_encoding.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/models/GroundingDINO/backbone/__pycache__/swin_transformer.cpython-310.pyc b/groundingdino/models/GroundingDINO/backbone/__pycache__/swin_transformer.cpython-310.pyc deleted file mode 100644 index 2eefb4164f28219c03fe31615e90dac0ba823bf5..0000000000000000000000000000000000000000 Binary files a/groundingdino/models/GroundingDINO/backbone/__pycache__/swin_transformer.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/models/GroundingDINO/backbone/backbone.py b/groundingdino/models/GroundingDINO/backbone/backbone.py deleted file mode 100644 index c8340c723fad8e07e2fc62daaa3912487498814b..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/backbone/backbone.py +++ /dev/null @@ -1,221 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Conditional DETR -# Copyright (c) 2021 Microsoft. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Copied from DETR (https://github.com/facebookresearch/detr) -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -# ------------------------------------------------------------------------ - -""" -Backbone modules. -""" - -from typing import Dict, List - -import torch -import torch.nn.functional as F -import torchvision -from torch import nn -from torchvision.models._utils import IntermediateLayerGetter - -from groundingdino.util.misc import NestedTensor, clean_state_dict, is_main_process - -from .position_encoding import build_position_encoding -from .swin_transformer import build_swin_transformer - - -class FrozenBatchNorm2d(torch.nn.Module): - """ - BatchNorm2d where the batch statistics and the affine parameters are fixed. - - Copy-paste from torchvision.misc.ops with added eps before rqsrt, - without which any other models than torchvision.models.resnet[18,34,50,101] - produce nans. - """ - - def __init__(self, n): - super(FrozenBatchNorm2d, self).__init__() - self.register_buffer("weight", torch.ones(n)) - self.register_buffer("bias", torch.zeros(n)) - self.register_buffer("running_mean", torch.zeros(n)) - self.register_buffer("running_var", torch.ones(n)) - - def _load_from_state_dict( - self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs - ): - num_batches_tracked_key = prefix + "num_batches_tracked" - if num_batches_tracked_key in state_dict: - del state_dict[num_batches_tracked_key] - - super(FrozenBatchNorm2d, self)._load_from_state_dict( - state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs - ) - - def forward(self, x): - # move reshapes to the beginning - # to make it fuser-friendly - w = self.weight.reshape(1, -1, 1, 1) - b = self.bias.reshape(1, -1, 1, 1) - rv = self.running_var.reshape(1, -1, 1, 1) - rm = self.running_mean.reshape(1, -1, 1, 1) - eps = 1e-5 - scale = w * (rv + eps).rsqrt() - bias = b - rm * scale - return x * scale + bias - - -class BackboneBase(nn.Module): - def __init__( - self, - backbone: nn.Module, - train_backbone: bool, - num_channels: int, - return_interm_indices: list, - ): - super().__init__() - for name, parameter in backbone.named_parameters(): - if ( - not train_backbone - or "layer2" not in name - and "layer3" not in name - and "layer4" not in name - ): - parameter.requires_grad_(False) - - return_layers = {} - for idx, layer_index in enumerate(return_interm_indices): - return_layers.update( - {"layer{}".format(5 - len(return_interm_indices) + idx): "{}".format(layer_index)} - ) - - # if len: - # if use_stage1_feature: - # return_layers = {"layer1": "0", "layer2": "1", "layer3": "2", "layer4": "3"} - # else: - # return_layers = {"layer2": "0", "layer3": "1", "layer4": "2"} - # else: - # return_layers = {'layer4': "0"} - self.body = IntermediateLayerGetter(backbone, return_layers=return_layers) - self.num_channels = num_channels - - def forward(self, tensor_list: NestedTensor): - xs = self.body(tensor_list.tensors) - out: Dict[str, NestedTensor] = {} - for name, x in xs.items(): - m = tensor_list.mask - assert m is not None - mask = F.interpolate(m[None].float(), size=x.shape[-2:]).to(torch.bool)[0] - out[name] = NestedTensor(x, mask) - # import ipdb; ipdb.set_trace() - return out - - -class Backbone(BackboneBase): - """ResNet backbone with frozen BatchNorm.""" - - def __init__( - self, - name: str, - train_backbone: bool, - dilation: bool, - return_interm_indices: list, - batch_norm=FrozenBatchNorm2d, - ): - if name in ["resnet18", "resnet34", "resnet50", "resnet101"]: - backbone = getattr(torchvision.models, name)( - replace_stride_with_dilation=[False, False, dilation], - pretrained=is_main_process(), - norm_layer=batch_norm, - ) - else: - raise NotImplementedError("Why you can get here with name {}".format(name)) - # num_channels = 512 if name in ('resnet18', 'resnet34') else 2048 - assert name not in ("resnet18", "resnet34"), "Only resnet50 and resnet101 are available." - assert return_interm_indices in [[0, 1, 2, 3], [1, 2, 3], [3]] - num_channels_all = [256, 512, 1024, 2048] - num_channels = num_channels_all[4 - len(return_interm_indices) :] - super().__init__(backbone, train_backbone, num_channels, return_interm_indices) - - -class Joiner(nn.Sequential): - def __init__(self, backbone, position_embedding): - super().__init__(backbone, position_embedding) - - def forward(self, tensor_list: NestedTensor): - xs = self[0](tensor_list) - out: List[NestedTensor] = [] - pos = [] - for name, x in xs.items(): - out.append(x) - # position encoding - pos.append(self[1](x).to(x.tensors.dtype)) - - return out, pos - - -def build_backbone(args): - """ - Useful args: - - backbone: backbone name - - lr_backbone: - - dilation - - return_interm_indices: available: [0,1,2,3], [1,2,3], [3] - - backbone_freeze_keywords: - - use_checkpoint: for swin only for now - - """ - position_embedding = build_position_encoding(args) - train_backbone = True - if not train_backbone: - raise ValueError("Please set lr_backbone > 0") - return_interm_indices = args.return_interm_indices - assert return_interm_indices in [[0, 1, 2, 3], [1, 2, 3], [3]] - args.backbone_freeze_keywords - use_checkpoint = getattr(args, "use_checkpoint", False) - - if args.backbone in ["resnet50", "resnet101"]: - backbone = Backbone( - args.backbone, - train_backbone, - args.dilation, - return_interm_indices, - batch_norm=FrozenBatchNorm2d, - ) - bb_num_channels = backbone.num_channels - elif args.backbone in [ - "swin_T_224_1k", - "swin_B_224_22k", - "swin_B_384_22k", - "swin_L_224_22k", - "swin_L_384_22k", - ]: - pretrain_img_size = int(args.backbone.split("_")[-2]) - backbone = build_swin_transformer( - args.backbone, - pretrain_img_size=pretrain_img_size, - out_indices=tuple(return_interm_indices), - dilation=False, - use_checkpoint=use_checkpoint, - ) - - bb_num_channels = backbone.num_features[4 - len(return_interm_indices) :] - else: - raise NotImplementedError("Unknown backbone {}".format(args.backbone)) - - assert len(bb_num_channels) == len( - return_interm_indices - ), f"len(bb_num_channels) {len(bb_num_channels)} != len(return_interm_indices) {len(return_interm_indices)}" - - model = Joiner(backbone, position_embedding) - model.num_channels = bb_num_channels - assert isinstance( - bb_num_channels, List - ), "bb_num_channels is expected to be a List but {}".format(type(bb_num_channels)) - # import ipdb; ipdb.set_trace() - return model diff --git a/groundingdino/models/GroundingDINO/backbone/position_encoding.py b/groundingdino/models/GroundingDINO/backbone/position_encoding.py deleted file mode 100644 index eac7e896bbe85a670824bfe8ef487d0535d5bd99..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/backbone/position_encoding.py +++ /dev/null @@ -1,186 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# DINO -# Copyright (c) 2022 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Conditional DETR -# Copyright (c) 2021 Microsoft. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Copied from DETR (https://github.com/facebookresearch/detr) -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -# ------------------------------------------------------------------------ - -""" -Various positional encodings for the transformer. -""" -import math - -import torch -from torch import nn - -from groundingdino.util.misc import NestedTensor - - -class PositionEmbeddingSine(nn.Module): - """ - This is a more standard version of the position embedding, very similar to the one - used by the Attention is all you need paper, generalized to work on images. - """ - - def __init__(self, num_pos_feats=64, temperature=10000, normalize=False, scale=None): - super().__init__() - self.num_pos_feats = num_pos_feats - self.temperature = temperature - self.normalize = normalize - if scale is not None and normalize is False: - raise ValueError("normalize should be True if scale is passed") - if scale is None: - scale = 2 * math.pi - self.scale = scale - - def forward(self, tensor_list: NestedTensor): - x = tensor_list.tensors - mask = tensor_list.mask - assert mask is not None - not_mask = ~mask - y_embed = not_mask.cumsum(1, dtype=torch.float32) - x_embed = not_mask.cumsum(2, dtype=torch.float32) - if self.normalize: - eps = 1e-6 - # if os.environ.get("SHILONG_AMP", None) == '1': - # eps = 1e-4 - # else: - # eps = 1e-6 - y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale - x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale - - dim_t = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device) - dim_t = self.temperature ** (2 * (dim_t // 2) / self.num_pos_feats) - - pos_x = x_embed[:, :, :, None] / dim_t - pos_y = y_embed[:, :, :, None] / dim_t - pos_x = torch.stack( - (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4 - ).flatten(3) - pos_y = torch.stack( - (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4 - ).flatten(3) - pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) - return pos - - -class PositionEmbeddingSineHW(nn.Module): - """ - This is a more standard version of the position embedding, very similar to the one - used by the Attention is all you need paper, generalized to work on images. - """ - - def __init__( - self, num_pos_feats=64, temperatureH=10000, temperatureW=10000, normalize=False, scale=None - ): - super().__init__() - self.num_pos_feats = num_pos_feats - self.temperatureH = temperatureH - self.temperatureW = temperatureW - self.normalize = normalize - if scale is not None and normalize is False: - raise ValueError("normalize should be True if scale is passed") - if scale is None: - scale = 2 * math.pi - self.scale = scale - - def forward(self, tensor_list: NestedTensor): - x = tensor_list.tensors - mask = tensor_list.mask - assert mask is not None - not_mask = ~mask - y_embed = not_mask.cumsum(1, dtype=torch.float32) - x_embed = not_mask.cumsum(2, dtype=torch.float32) - - # import ipdb; ipdb.set_trace() - - if self.normalize: - eps = 1e-6 - y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale - x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale - - dim_tx = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device) - dim_tx = self.temperatureW ** (2 * (torch.div(dim_tx, 2, rounding_mode='floor')) / self.num_pos_feats) - pos_x = x_embed[:, :, :, None] / dim_tx - - dim_ty = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device) - dim_ty = self.temperatureH ** (2 * (torch.div(dim_ty, 2, rounding_mode='floor')) / self.num_pos_feats) - pos_y = y_embed[:, :, :, None] / dim_ty - - pos_x = torch.stack( - (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4 - ).flatten(3) - pos_y = torch.stack( - (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4 - ).flatten(3) - pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) - - # import ipdb; ipdb.set_trace() - - return pos - - -class PositionEmbeddingLearned(nn.Module): - """ - Absolute pos embedding, learned. - """ - - def __init__(self, num_pos_feats=256): - super().__init__() - self.row_embed = nn.Embedding(50, num_pos_feats) - self.col_embed = nn.Embedding(50, num_pos_feats) - self.reset_parameters() - - def reset_parameters(self): - nn.init.uniform_(self.row_embed.weight) - nn.init.uniform_(self.col_embed.weight) - - def forward(self, tensor_list: NestedTensor): - x = tensor_list.tensors - h, w = x.shape[-2:] - i = torch.arange(w, device=x.device) - j = torch.arange(h, device=x.device) - x_emb = self.col_embed(i) - y_emb = self.row_embed(j) - pos = ( - torch.cat( - [ - x_emb.unsqueeze(0).repeat(h, 1, 1), - y_emb.unsqueeze(1).repeat(1, w, 1), - ], - dim=-1, - ) - .permute(2, 0, 1) - .unsqueeze(0) - .repeat(x.shape[0], 1, 1, 1) - ) - return pos - - -def build_position_encoding(args): - N_steps = args.hidden_dim // 2 - if args.position_embedding in ("v2", "sine"): - # TODO find a better way of exposing other arguments - position_embedding = PositionEmbeddingSineHW( - N_steps, - temperatureH=args.pe_temperatureH, - temperatureW=args.pe_temperatureW, - normalize=True, - ) - elif args.position_embedding in ("v3", "learned"): - position_embedding = PositionEmbeddingLearned(N_steps) - else: - raise ValueError(f"not supported {args.position_embedding}") - - return position_embedding diff --git a/groundingdino/models/GroundingDINO/backbone/swin_transformer.py b/groundingdino/models/GroundingDINO/backbone/swin_transformer.py deleted file mode 100644 index 0cab63162840b6adf4f3404a6ced06c8c4e85ed6..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/backbone/swin_transformer.py +++ /dev/null @@ -1,804 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# DINO -# Copyright (c) 2022 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# -------------------------------------------------------- -# modified from https://github.com/SwinTransformer/Swin-Transformer-Object-Detection/blob/master/mmdet/models/backbones/swin_transformer.py -# -------------------------------------------------------- - -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.utils.checkpoint as checkpoint -from timm.models.layers import DropPath, to_2tuple, trunc_normal_ -import loralib as lora -from groundingdino.util.misc import NestedTensor - - -class Mlp(nn.Module): - """Multilayer perceptron.""" - - def __init__( - self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.0 - ): - super().__init__() - r = 12 - out_features = out_features or in_features - hidden_features = hidden_features or in_features - self.fc1 = lora.Linear(in_features, hidden_features , r=r) - self.act = act_layer() - self.fc2 = lora.Linear(hidden_features, out_features , r=r) - self.drop = nn.Dropout(drop) - - def forward(self, x): - x = self.fc1(x) - x = self.act(x) - x = self.drop(x) - x = self.fc2(x) - x = self.drop(x) - return x - - -def window_partition(x, window_size): - """ - Args: - x: (B, H, W, C) - window_size (int): window size - Returns: - windows: (num_windows*B, window_size, window_size, C) - """ - B, H, W, C = x.shape - x = x.view(B, H // window_size, window_size, W // window_size, window_size, C) - windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) - return windows - - -def window_reverse(windows, window_size, H, W): - """ - Args: - windows: (num_windows*B, window_size, window_size, C) - window_size (int): Window size - H (int): Height of image - W (int): Width of image - Returns: - x: (B, H, W, C) - """ - B = int(windows.shape[0] / (H * W / window_size / window_size)) - x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1) - x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) - return x - - -class WindowAttention(nn.Module): - """Window based multi-head self attention (W-MSA) module with relative position bias. - It supports both of shifted and non-shifted window. - Args: - dim (int): Number of input channels. - window_size (tuple[int]): The height and width of the window. - num_heads (int): Number of attention heads. - qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True - qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set - attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 - proj_drop (float, optional): Dropout ratio of output. Default: 0.0 - """ - - def __init__( - self, - dim, - window_size, - num_heads, - qkv_bias=True, - qk_scale=None, - attn_drop=0.0, - proj_drop=0.0, - ): - - super().__init__() - self.dim = dim - self.window_size = window_size # Wh, Ww - self.num_heads = num_heads - head_dim = dim // num_heads - self.scale = qk_scale or head_dim**-0.5 - - # define a parameter table of relative position bias - self.relative_position_bias_table = nn.Parameter( - torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads) - ) # 2*Wh-1 * 2*Ww-1, nH - - # get pair-wise relative position index for each token inside the window - coords_h = torch.arange(self.window_size[0]) - coords_w = torch.arange(self.window_size[1]) - coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww - coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww - relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww - relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 - relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0 - relative_coords[:, :, 1] += self.window_size[1] - 1 - relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 - relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww - self.register_buffer("relative_position_index", relative_position_index) - r = 12 - self.qkv = lora.Linear(dim, dim * 3, r=r , bias=qkv_bias) - self.attn_drop = nn.Dropout(attn_drop) - self.proj = lora.Linear(dim, dim , r=r) - self.proj_drop = nn.Dropout(proj_drop) - - trunc_normal_(self.relative_position_bias_table, std=0.02) - self.softmax = nn.Softmax(dim=-1) - - def forward(self, x, mask=None): - """Forward function. - Args: - x: input features with shape of (num_windows*B, N, C) - mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None - """ - B_, N, C = x.shape - qkv = ( - self.qkv(x) - .reshape(B_, N, 3, self.num_heads, C // self.num_heads) - .permute(2, 0, 3, 1, 4) - ) - q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) - - q = q * self.scale - attn = q @ k.transpose(-2, -1) - - relative_position_bias = self.relative_position_bias_table[ - self.relative_position_index.view(-1) - ].view( - self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1 - ) # Wh*Ww,Wh*Ww,nH - relative_position_bias = relative_position_bias.permute( - 2, 0, 1 - ).contiguous() # nH, Wh*Ww, Wh*Ww - attn = attn + relative_position_bias.unsqueeze(0) - - if mask is not None: - nW = mask.shape[0] - attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0) - attn = attn.view(-1, self.num_heads, N, N) - attn = self.softmax(attn) - else: - attn = self.softmax(attn) - - attn = self.attn_drop(attn) - - x = (attn @ v).transpose(1, 2).reshape(B_, N, C) - x = self.proj(x) - x = self.proj_drop(x) - return x - - -class SwinTransformerBlock(nn.Module): - """Swin Transformer Block. - Args: - dim (int): Number of input channels. - num_heads (int): Number of attention heads. - window_size (int): Window size. - shift_size (int): Shift size for SW-MSA. - mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. - qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True - qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. - drop (float, optional): Dropout rate. Default: 0.0 - attn_drop (float, optional): Attention dropout rate. Default: 0.0 - drop_path (float, optional): Stochastic depth rate. Default: 0.0 - act_layer (nn.Module, optional): Activation layer. Default: nn.GELU - norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm - """ - - def __init__( - self, - dim, - num_heads, - window_size=7, - shift_size=0, - mlp_ratio=4.0, - qkv_bias=True, - qk_scale=None, - drop=0.0, - attn_drop=0.0, - drop_path=0.0, - act_layer=nn.GELU, - norm_layer=nn.LayerNorm, - ): - super().__init__() - self.dim = dim - self.num_heads = num_heads - self.window_size = window_size - self.shift_size = shift_size - self.mlp_ratio = mlp_ratio - assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size" - - self.norm1 = norm_layer(dim) - self.attn = WindowAttention( - dim, - window_size=to_2tuple(self.window_size), - num_heads=num_heads, - qkv_bias=qkv_bias, - qk_scale=qk_scale, - attn_drop=attn_drop, - proj_drop=drop, - ) - - self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() - self.norm2 = norm_layer(dim) - mlp_hidden_dim = int(dim * mlp_ratio) - self.mlp = Mlp( - in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop - ) - - self.H = None - self.W = None - - def forward(self, x, mask_matrix): - """Forward function. - Args: - x: Input feature, tensor size (B, H*W, C). - H, W: Spatial resolution of the input feature. - mask_matrix: Attention mask for cyclic shift. - """ - B, L, C = x.shape - H, W = self.H, self.W - assert L == H * W, "input feature has wrong size" - - shortcut = x - x = self.norm1(x) - x = x.view(B, H, W, C) - - # pad feature maps to multiples of window size - pad_l = pad_t = 0 - pad_r = (self.window_size - W % self.window_size) % self.window_size - pad_b = (self.window_size - H % self.window_size) % self.window_size - x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b)) - _, Hp, Wp, _ = x.shape - - # cyclic shift - if self.shift_size > 0: - shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) - attn_mask = mask_matrix - else: - shifted_x = x - attn_mask = None - - # partition windows - x_windows = window_partition( - shifted_x, self.window_size - ) # nW*B, window_size, window_size, C - x_windows = x_windows.view( - -1, self.window_size * self.window_size, C - ) # nW*B, window_size*window_size, C - - # W-MSA/SW-MSA - attn_windows = self.attn(x_windows, mask=attn_mask) # nW*B, window_size*window_size, C - - # merge windows - attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C) - shifted_x = window_reverse(attn_windows, self.window_size, Hp, Wp) # B H' W' C - - # reverse cyclic shift - if self.shift_size > 0: - x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2)) - else: - x = shifted_x - - if pad_r > 0 or pad_b > 0: - x = x[:, :H, :W, :].contiguous() - - x = x.view(B, H * W, C) - - # FFN - x = shortcut + self.drop_path(x) - x = x + self.drop_path(self.mlp(self.norm2(x))) - - return x - - -class PatchMerging(nn.Module): - """Patch Merging Layer - Args: - dim (int): Number of input channels. - norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm - """ - - def __init__(self, dim, norm_layer=nn.LayerNorm): - super().__init__() - r = 24 - self.dim = dim - self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False) - self.norm = norm_layer(4 * dim) - - def forward(self, x, H, W): - """Forward function. - Args: - x: Input feature, tensor size (B, H*W, C). - H, W: Spatial resolution of the input feature. - """ - B, L, C = x.shape - assert L == H * W, "input feature has wrong size" - - x = x.view(B, H, W, C) - - # padding - pad_input = (H % 2 == 1) or (W % 2 == 1) - if pad_input: - x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2)) - - x0 = x[:, 0::2, 0::2, :] # B H/2 W/2 C - x1 = x[:, 1::2, 0::2, :] # B H/2 W/2 C - x2 = x[:, 0::2, 1::2, :] # B H/2 W/2 C - x3 = x[:, 1::2, 1::2, :] # B H/2 W/2 C - x = torch.cat([x0, x1, x2, x3], -1) # B H/2 W/2 4*C - x = x.view(B, -1, 4 * C) # B H/2*W/2 4*C - - x = self.norm(x) - x = self.reduction(x) - - return x - - -class BasicLayer(nn.Module): - """A basic Swin Transformer layer for one stage. - Args: - dim (int): Number of feature channels - depth (int): Depths of this stage. - num_heads (int): Number of attention head. - window_size (int): Local window size. Default: 7. - mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. - qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True - qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. - drop (float, optional): Dropout rate. Default: 0.0 - attn_drop (float, optional): Attention dropout rate. Default: 0.0 - drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 - norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm - downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None - use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. - """ - - def __init__( - self, - dim, - depth, - num_heads, - window_size=7, - mlp_ratio=4.0, - qkv_bias=True, - qk_scale=None, - drop=0.0, - attn_drop=0.0, - drop_path=0.0, - norm_layer=nn.LayerNorm, - downsample=None, - use_checkpoint=False, - ): - super().__init__() - self.window_size = window_size - self.shift_size = window_size // 2 - self.depth = depth - self.use_checkpoint = use_checkpoint - - # build blocks - self.blocks = nn.ModuleList( - [ - SwinTransformerBlock( - dim=dim, - num_heads=num_heads, - window_size=window_size, - shift_size=0 if (i % 2 == 0) else window_size // 2, - mlp_ratio=mlp_ratio, - qkv_bias=qkv_bias, - qk_scale=qk_scale, - drop=drop, - attn_drop=attn_drop, - drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path, - norm_layer=norm_layer, - ) - for i in range(depth) - ] - ) - - # patch merging layer - if downsample is not None: - self.downsample = downsample(dim=dim, norm_layer=norm_layer) - else: - self.downsample = None - - def forward(self, x, H, W): - """Forward function. - Args: - x: Input feature, tensor size (B, H*W, C). - H, W: Spatial resolution of the input feature. - """ - - # calculate attention mask for SW-MSA - Hp = int(np.ceil(H / self.window_size)) * self.window_size - Wp = int(np.ceil(W / self.window_size)) * self.window_size - img_mask = torch.zeros((1, Hp, Wp, 1), device=x.device) # 1 Hp Wp 1 - h_slices = ( - slice(0, -self.window_size), - slice(-self.window_size, -self.shift_size), - slice(-self.shift_size, None), - ) - w_slices = ( - slice(0, -self.window_size), - slice(-self.window_size, -self.shift_size), - slice(-self.shift_size, None), - ) - cnt = 0 - for h in h_slices: - for w in w_slices: - img_mask[:, h, w, :] = cnt - cnt += 1 - - mask_windows = window_partition( - img_mask, self.window_size - ) # nW, window_size, window_size, 1 - mask_windows = mask_windows.view(-1, self.window_size * self.window_size) - attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) - attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill( - attn_mask == 0, float(0.0) - ) - - for blk in self.blocks: - blk.H, blk.W = H, W - if self.use_checkpoint: - x = checkpoint.checkpoint(blk, x, attn_mask) - else: - x = blk(x, attn_mask) - if self.downsample is not None: - x_down = self.downsample(x, H, W) - Wh, Ww = (H + 1) // 2, (W + 1) // 2 - return x, H, W, x_down, Wh, Ww - else: - return x, H, W, x, H, W - - -class PatchEmbed(nn.Module): - """Image to Patch Embedding - Args: - patch_size (int): Patch token size. Default: 4. - in_chans (int): Number of input image channels. Default: 3. - embed_dim (int): Number of linear projection output channels. Default: 96. - norm_layer (nn.Module, optional): Normalization layer. Default: None - """ - - def __init__(self, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None): - super().__init__() - patch_size = to_2tuple(patch_size) - self.patch_size = patch_size - - self.in_chans = in_chans - self.embed_dim = embed_dim - - self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) - if norm_layer is not None: - self.norm = norm_layer(embed_dim) - else: - self.norm = None - - def forward(self, x): - """Forward function.""" - # padding - _, _, H, W = x.size() - if W % self.patch_size[1] != 0: - x = F.pad(x, (0, self.patch_size[1] - W % self.patch_size[1])) - if H % self.patch_size[0] != 0: - x = F.pad(x, (0, 0, 0, self.patch_size[0] - H % self.patch_size[0])) - - x = self.proj(x) # B C Wh Ww - if self.norm is not None: - Wh, Ww = x.size(2), x.size(3) - x = x.flatten(2).transpose(1, 2) - x = self.norm(x) - x = x.transpose(1, 2).view(-1, self.embed_dim, Wh, Ww) - - return x - - -class SwinTransformer(nn.Module): - """Swin Transformer backbone. - A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows` - - https://arxiv.org/pdf/2103.14030 - Args: - pretrain_img_size (int): Input image size for training the pretrained model, - used in absolute postion embedding. Default 224. - patch_size (int | tuple(int)): Patch size. Default: 4. - in_chans (int): Number of input image channels. Default: 3. - embed_dim (int): Number of linear projection output channels. Default: 96. - depths (tuple[int]): Depths of each Swin Transformer stage. - num_heads (tuple[int]): Number of attention head of each stage. - window_size (int): Window size. Default: 7. - mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. - qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True - qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. - drop_rate (float): Dropout rate. - attn_drop_rate (float): Attention dropout rate. Default: 0. - drop_path_rate (float): Stochastic depth rate. Default: 0.2. - norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm. - ape (bool): If True, add absolute position embedding to the patch embedding. Default: False. - patch_norm (bool): If True, add normalization after patch embedding. Default: True. - out_indices (Sequence[int]): Output from which stages. - frozen_stages (int): Stages to be frozen (stop grad and set eval mode). - -1 means not freezing any parameters. - use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. - dilation (bool): if True, the output size if 16x downsample, ow 32x downsample. - """ - - def __init__( - self, - pretrain_img_size=224, - patch_size=4, - in_chans=3, - embed_dim=96, - depths=[2, 2, 6, 2], - num_heads=[3, 6, 12, 24], - window_size=7, - mlp_ratio=4.0, - qkv_bias=True, - qk_scale=None, - drop_rate=0.0, - attn_drop_rate=0.0, - drop_path_rate=0.2, - norm_layer=nn.LayerNorm, - ape=False, - patch_norm=True, - out_indices=(0, 1, 2, 3), - frozen_stages=-1, - dilation=False, - use_checkpoint=False, - ): - super().__init__() - - self.pretrain_img_size = pretrain_img_size - self.num_layers = len(depths) - self.embed_dim = embed_dim - self.ape = ape - self.patch_norm = patch_norm - self.out_indices = out_indices - self.frozen_stages = frozen_stages - self.dilation = dilation - - # if use_checkpoint: - # print("use_checkpoint!!!!!!!!!!!!!!!!!!!!!!!!") - - # split image into non-overlapping patches - self.patch_embed = PatchEmbed( - patch_size=patch_size, - in_chans=in_chans, - embed_dim=embed_dim, - norm_layer=norm_layer if self.patch_norm else None, - ) - - # absolute position embedding - if self.ape: - pretrain_img_size = to_2tuple(pretrain_img_size) - patch_size = to_2tuple(patch_size) - patches_resolution = [ - pretrain_img_size[0] // patch_size[0], - pretrain_img_size[1] // patch_size[1], - ] - - self.absolute_pos_embed = nn.Parameter( - torch.zeros(1, embed_dim, patches_resolution[0], patches_resolution[1]) - ) - trunc_normal_(self.absolute_pos_embed, std=0.02) - - self.pos_drop = nn.Dropout(p=drop_rate) - - # stochastic depth - dpr = [ - x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) - ] # stochastic depth decay rule - - # build layers - self.layers = nn.ModuleList() - # prepare downsample list - downsamplelist = [PatchMerging for i in range(self.num_layers)] - downsamplelist[-1] = None - num_features = [int(embed_dim * 2**i) for i in range(self.num_layers)] - if self.dilation: - downsamplelist[-2] = None - num_features[-1] = int(embed_dim * 2 ** (self.num_layers - 1)) // 2 - for i_layer in range(self.num_layers): - layer = BasicLayer( - # dim=int(embed_dim * 2 ** i_layer), - dim=num_features[i_layer], - depth=depths[i_layer], - num_heads=num_heads[i_layer], - window_size=window_size, - mlp_ratio=mlp_ratio, - qkv_bias=qkv_bias, - qk_scale=qk_scale, - drop=drop_rate, - attn_drop=attn_drop_rate, - drop_path=dpr[sum(depths[:i_layer]) : sum(depths[: i_layer + 1])], - norm_layer=norm_layer, - # downsample=PatchMerging if (i_layer < self.num_layers - 1) else None, - downsample=downsamplelist[i_layer], - use_checkpoint=use_checkpoint, - ) - self.layers.append(layer) - - # num_features = [int(embed_dim * 2 ** i) for i in range(self.num_layers)] - self.num_features = num_features - - # add a norm layer for each output - for i_layer in out_indices: - layer = norm_layer(num_features[i_layer]) - layer_name = f"norm{i_layer}" - self.add_module(layer_name, layer) - - self._freeze_stages() - - def _freeze_stages(self): - if self.frozen_stages >= 0: - self.patch_embed.eval() - for param in self.patch_embed.parameters(): - param.requires_grad = False - - if self.frozen_stages >= 1 and self.ape: - self.absolute_pos_embed.requires_grad = False - - if self.frozen_stages >= 2: - self.pos_drop.eval() - for i in range(0, self.frozen_stages - 1): - m = self.layers[i] - m.eval() - for param in m.parameters(): - param.requires_grad = False - - # def init_weights(self, pretrained=None): - # """Initialize the weights in backbone. - # Args: - # pretrained (str, optional): Path to pre-trained weights. - # Defaults to None. - # """ - - # def _init_weights(m): - # if isinstance(m, nn.Linear): - # trunc_normal_(m.weight, std=.02) - # if isinstance(m, nn.Linear) and m.bias is not None: - # nn.init.constant_(m.bias, 0) - # elif isinstance(m, nn.LayerNorm): - # nn.init.constant_(m.bias, 0) - # nn.init.constant_(m.weight, 1.0) - - # if isinstance(pretrained, str): - # self.apply(_init_weights) - # logger = get_root_logger() - # load_checkpoint(self, pretrained, strict=False, logger=logger) - # elif pretrained is None: - # self.apply(_init_weights) - # else: - # raise TypeError('pretrained must be a str or None') - - def forward_raw(self, x): - """Forward function.""" - x = self.patch_embed(x) - - Wh, Ww = x.size(2), x.size(3) - if self.ape: - # interpolate the position embedding to the corresponding size - absolute_pos_embed = F.interpolate( - self.absolute_pos_embed, size=(Wh, Ww), mode="bicubic" - ) - x = (x + absolute_pos_embed).flatten(2).transpose(1, 2) # B Wh*Ww C - else: - x = x.flatten(2).transpose(1, 2) - x = self.pos_drop(x) - - outs = [] - for i in range(self.num_layers): - layer = self.layers[i] - x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww) - # import ipdb; ipdb.set_trace() - - if i in self.out_indices: - norm_layer = getattr(self, f"norm{i}") - x_out = norm_layer(x_out) - - out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous() - outs.append(out) - # in: - # torch.Size([2, 3, 1024, 1024]) - # outs: - # [torch.Size([2, 192, 256, 256]), torch.Size([2, 384, 128, 128]), \ - # torch.Size([2, 768, 64, 64]), torch.Size([2, 1536, 32, 32])] - return tuple(outs) - - def forward(self, tensor_list: NestedTensor): - x = tensor_list.tensors - - """Forward function.""" - x = self.patch_embed(x) - - Wh, Ww = x.size(2), x.size(3) - if self.ape: - # interpolate the position embedding to the corresponding size - absolute_pos_embed = F.interpolate( - self.absolute_pos_embed, size=(Wh, Ww), mode="bicubic" - ) - x = (x + absolute_pos_embed).flatten(2).transpose(1, 2) # B Wh*Ww C - else: - x = x.flatten(2).transpose(1, 2) - x = self.pos_drop(x) - - outs = [] - for i in range(self.num_layers): - layer = self.layers[i] - x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww) - - if i in self.out_indices: - norm_layer = getattr(self, f"norm{i}") - x_out = norm_layer(x_out) - - out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous() - outs.append(out) - # in: - # torch.Size([2, 3, 1024, 1024]) - # out: - # [torch.Size([2, 192, 256, 256]), torch.Size([2, 384, 128, 128]), \ - # torch.Size([2, 768, 64, 64]), torch.Size([2, 1536, 32, 32])] - - # collect for nesttensors - outs_dict = {} - for idx, out_i in enumerate(outs): - m = tensor_list.mask - assert m is not None - mask = F.interpolate(m[None].float(), size=out_i.shape[-2:]).to(torch.bool)[0] - outs_dict[idx] = NestedTensor(out_i, mask) - - return outs_dict - - def train(self, mode=True): - """Convert the model into training mode while keep layers freezed.""" - super(SwinTransformer, self).train(mode) - self._freeze_stages() - - -def build_swin_transformer(modelname, pretrain_img_size, **kw): - assert modelname in [ - "swin_T_224_1k", - "swin_B_224_22k", - "swin_B_384_22k", - "swin_L_224_22k", - "swin_L_384_22k", - ] - - model_para_dict = { - "swin_T_224_1k": dict( - embed_dim=96, depths=[2, 2, 6, 2], num_heads=[3, 6, 12, 24], window_size=7 - ), - "swin_B_224_22k": dict( - embed_dim=128, depths=[2, 2, 18, 2], num_heads=[4, 8, 16, 32], window_size=7 - ), - "swin_B_384_22k": dict( - embed_dim=128, depths=[2, 2, 18, 2], num_heads=[4, 8, 16, 32], window_size=12 - ), - "swin_L_224_22k": dict( - embed_dim=192, depths=[2, 2, 18, 2], num_heads=[6, 12, 24, 48], window_size=7 - ), - "swin_L_384_22k": dict( - embed_dim=192, depths=[2, 2, 18, 2], num_heads=[6, 12, 24, 48], window_size=12 - ), - } - kw_cgf = model_para_dict[modelname] - kw_cgf.update(kw) - model = SwinTransformer(pretrain_img_size=pretrain_img_size, **kw_cgf) - return model - - -if __name__ == "__main__": - model = build_swin_transformer("swin_L_384_22k", 384, dilation=True) - x = torch.rand(2, 3, 1024, 1024) - y = model.forward_raw(x) - import ipdb - - ipdb.set_trace() - x = torch.rand(2, 3, 384, 384) - y = model.forward_raw(x) diff --git a/groundingdino/models/GroundingDINO/fuse_modules.py b/groundingdino/models/GroundingDINO/fuse_modules.py deleted file mode 100644 index 559c4eb960e5ca1787c27815e9f75ed0d80106ea..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/fuse_modules.py +++ /dev/null @@ -1,298 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ - -import torch -import torch.nn as nn -import torch.nn.functional as F -from timm.models.layers import DropPath -import loralib as lora - -class FeatureResizer(nn.Module): - """ - This class takes as input a set of embeddings of dimension C1 and outputs a set of - embedding of dimension C2, after a linear transformation, dropout and normalization (LN). - """ - - def __init__(self, input_feat_size, output_feat_size, dropout, do_ln=True): - super().__init__() - self.do_ln = do_ln - r = 12 - # Object feature encoding - self.fc = lora.Linear(input_feat_size, output_feat_size,r=r, bias=True) - self.layer_norm = nn.LayerNorm(output_feat_size, eps=1e-12) - self.dropout = nn.Dropout(dropout) - - def forward(self, encoder_features): - x = self.fc(encoder_features) - if self.do_ln: - x = self.layer_norm(x) - output = self.dropout(x) - return output - - -def l1norm(X, dim, eps=1e-8): - """L1-normalize columns of X""" - norm = torch.abs(X).sum(dim=dim, keepdim=True) + eps - X = torch.div(X, norm) - return X - - -def l2norm(X, dim, eps=1e-8): - """L2-normalize columns of X""" - norm = torch.pow(X, 2).sum(dim=dim, keepdim=True).sqrt() + eps - X = torch.div(X, norm) - return X - - -def func_attention(query, context, smooth=1, raw_feature_norm="softmax", eps=1e-8): - """ - query: (n_context, queryL, d) - context: (n_context, sourceL, d) - """ - batch_size_q, queryL = query.size(0), query.size(1) - batch_size, sourceL = context.size(0), context.size(1) - - # Get attention - # --> (batch, d, queryL) - queryT = torch.transpose(query, 1, 2) - - # (batch, sourceL, d)(batch, d, queryL) - # --> (batch, sourceL, queryL) - attn = torch.bmm(context, queryT) - if raw_feature_norm == "softmax": - # --> (batch*sourceL, queryL) - attn = attn.view(batch_size * sourceL, queryL) - attn = nn.Softmax()(attn) - # --> (batch, sourceL, queryL) - attn = attn.view(batch_size, sourceL, queryL) - elif raw_feature_norm == "l2norm": - attn = l2norm(attn, 2) - elif raw_feature_norm == "clipped_l2norm": - attn = nn.LeakyReLU(0.1)(attn) - attn = l2norm(attn, 2) - else: - raise ValueError("unknown first norm type:", raw_feature_norm) - # --> (batch, queryL, sourceL) - attn = torch.transpose(attn, 1, 2).contiguous() - # --> (batch*queryL, sourceL) - attn = attn.view(batch_size * queryL, sourceL) - attn = nn.Softmax()(attn * smooth) - # --> (batch, queryL, sourceL) - attn = attn.view(batch_size, queryL, sourceL) - # --> (batch, sourceL, queryL) - attnT = torch.transpose(attn, 1, 2).contiguous() - - # --> (batch, d, sourceL) - contextT = torch.transpose(context, 1, 2) - # (batch x d x sourceL)(batch x sourceL x queryL) - # --> (batch, d, queryL) - weightedContext = torch.bmm(contextT, attnT) - # --> (batch, queryL, d) - weightedContext = torch.transpose(weightedContext, 1, 2) - - return weightedContext, attnT - - -class BiMultiHeadAttention(nn.Module): - def __init__(self, v_dim, l_dim, embed_dim, num_heads, dropout=0.1, cfg=None): - super(BiMultiHeadAttention, self).__init__() - - self.embed_dim = embed_dim - self.num_heads = num_heads - self.head_dim = embed_dim // num_heads - self.v_dim = v_dim - self.l_dim = l_dim - - assert ( - self.head_dim * self.num_heads == self.embed_dim - ), f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`: {self.num_heads})." - self.scale = self.head_dim ** (-0.5) - self.dropout = dropout - r = 12 - self.v_proj = lora.Linear(self.v_dim, self.embed_dim , r=r) - self.l_proj = lora.Linear(self.l_dim, self.embed_dim , r=r ) - self.values_v_proj = lora.Linear(self.v_dim, self.embed_dim , r=r ) - self.values_l_proj = lora.Linear(self.l_dim, self.embed_dim , r=r ) - - self.out_v_proj = lora.Linear(self.embed_dim, self.v_dim , r=r ) - self.out_l_proj = lora.Linear(self.embed_dim, self.l_dim , r=r ) - - self.stable_softmax_2d = True - self.clamp_min_for_underflow = True - self.clamp_max_for_overflow = True - - self._reset_parameters() - - def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): - return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() - - def _reset_parameters(self): - nn.init.xavier_uniform_(self.v_proj.weight) - self.v_proj.bias.data.fill_(0) - nn.init.xavier_uniform_(self.l_proj.weight) - self.l_proj.bias.data.fill_(0) - nn.init.xavier_uniform_(self.values_v_proj.weight) - self.values_v_proj.bias.data.fill_(0) - nn.init.xavier_uniform_(self.values_l_proj.weight) - self.values_l_proj.bias.data.fill_(0) - nn.init.xavier_uniform_(self.out_v_proj.weight) - self.out_v_proj.bias.data.fill_(0) - nn.init.xavier_uniform_(self.out_l_proj.weight) - self.out_l_proj.bias.data.fill_(0) - - def forward(self, v, l, attention_mask_v=None, attention_mask_l=None): - """_summary_ - - Args: - v (_type_): bs, n_img, dim - l (_type_): bs, n_text, dim - attention_mask_v (_type_, optional): _description_. bs, n_img - attention_mask_l (_type_, optional): _description_. bs, n_text - - Returns: - _type_: _description_ - """ - # if os.environ.get('IPDB_SHILONG_DEBUG', None) == 'INFO': - # import ipdb; ipdb.set_trace() - bsz, tgt_len, _ = v.size() - - query_states = self.v_proj(v) * self.scale - key_states = self._shape(self.l_proj(l), -1, bsz) - value_v_states = self._shape(self.values_v_proj(v), -1, bsz) - value_l_states = self._shape(self.values_l_proj(l), -1, bsz) - - proj_shape = (bsz * self.num_heads, -1, self.head_dim) - query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) - key_states = key_states.view(*proj_shape) - value_v_states = value_v_states.view(*proj_shape) - value_l_states = value_l_states.view(*proj_shape) - - src_len = key_states.size(1) - attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) # bs*nhead, nimg, ntxt - - if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): - raise ValueError( - f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is {attn_weights.size()}" - ) - - if self.stable_softmax_2d: - attn_weights = attn_weights - attn_weights.max() - - if self.clamp_min_for_underflow: - attn_weights = torch.clamp( - attn_weights, min=-50000 - ) # Do not increase -50000, data type half has quite limited range - if self.clamp_max_for_overflow: - attn_weights = torch.clamp( - attn_weights, max=50000 - ) # Do not increase 50000, data type half has quite limited range - - attn_weights_T = attn_weights.transpose(1, 2) - attn_weights_l = attn_weights_T - torch.max(attn_weights_T, dim=-1, keepdim=True)[0] - if self.clamp_min_for_underflow: - attn_weights_l = torch.clamp( - attn_weights_l, min=-50000 - ) # Do not increase -50000, data type half has quite limited range - if self.clamp_max_for_overflow: - attn_weights_l = torch.clamp( - attn_weights_l, max=50000 - ) # Do not increase 50000, data type half has quite limited range - - # mask vison for language - if attention_mask_v is not None: - attention_mask_v = ( - attention_mask_v[:, None, None, :].repeat(1, self.num_heads, 1, 1).flatten(0, 1) - ) - attn_weights_l.masked_fill_(attention_mask_v, float("-inf")) - - attn_weights_l = attn_weights_l.softmax(dim=-1) - - # mask language for vision - if attention_mask_l is not None: - attention_mask_l = ( - attention_mask_l[:, None, None, :].repeat(1, self.num_heads, 1, 1).flatten(0, 1) - ) - attn_weights.masked_fill_(attention_mask_l, float("-inf")) - attn_weights_v = attn_weights.softmax(dim=-1) - - attn_probs_v = F.dropout(attn_weights_v, p=self.dropout, training=self.training) - attn_probs_l = F.dropout(attn_weights_l, p=self.dropout, training=self.training) - - attn_output_v = torch.bmm(attn_probs_v, value_l_states) - attn_output_l = torch.bmm(attn_probs_l, value_v_states) - - if attn_output_v.size() != (bsz * self.num_heads, tgt_len, self.head_dim): - raise ValueError( - f"`attn_output_v` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is {attn_output_v.size()}" - ) - - if attn_output_l.size() != (bsz * self.num_heads, src_len, self.head_dim): - raise ValueError( - f"`attn_output_l` should be of size {(bsz, self.num_heads, src_len, self.head_dim)}, but is {attn_output_l.size()}" - ) - - attn_output_v = attn_output_v.view(bsz, self.num_heads, tgt_len, self.head_dim) - attn_output_v = attn_output_v.transpose(1, 2) - attn_output_v = attn_output_v.reshape(bsz, tgt_len, self.embed_dim) - - attn_output_l = attn_output_l.view(bsz, self.num_heads, src_len, self.head_dim) - attn_output_l = attn_output_l.transpose(1, 2) - attn_output_l = attn_output_l.reshape(bsz, src_len, self.embed_dim) - - attn_output_v = self.out_v_proj(attn_output_v) - attn_output_l = self.out_l_proj(attn_output_l) - - return attn_output_v, attn_output_l - - -# Bi-Direction MHA (text->image, image->text) -class BiAttentionBlock(nn.Module): - def __init__( - self, - v_dim, - l_dim, - embed_dim, - num_heads, - dropout=0.1, - drop_path=0.0, - init_values=1e-4, - cfg=None, - ): - """ - Inputs: - embed_dim - Dimensionality of input and attention feature vectors - hidden_dim - Dimensionality of hidden layer in feed-forward network - (usually 2-4x larger than embed_dim) - num_heads - Number of heads to use in the Multi-Head Attention block - dropout - Amount of dropout to apply in the feed-forward network - """ - super(BiAttentionBlock, self).__init__() - - # pre layer norm - self.layer_norm_v = nn.LayerNorm(v_dim) - self.layer_norm_l = nn.LayerNorm(l_dim) - self.attn = BiMultiHeadAttention( - v_dim=v_dim, l_dim=l_dim, embed_dim=embed_dim, num_heads=num_heads, dropout=dropout - ) - - # add layer scale for training stability - self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() - self.gamma_v = nn.Parameter(init_values * torch.ones((v_dim)), requires_grad=True) - self.gamma_l = nn.Parameter(init_values * torch.ones((l_dim)), requires_grad=True) - - def forward(self, v, l, attention_mask_v=None, attention_mask_l=None): - v = self.layer_norm_v(v) - l = self.layer_norm_l(l) - delta_v, delta_l = self.attn( - v, l, attention_mask_v=attention_mask_v, attention_mask_l=attention_mask_l - ) - # v, l = v + delta_v, l + delta_l - v = v + self.drop_path(self.gamma_v * delta_v) - l = l + self.drop_path(self.gamma_l * delta_l) - return v, l - - # def forward(self, v:List[torch.Tensor], l, attention_mask_v=None, attention_mask_l=None) diff --git a/groundingdino/models/GroundingDINO/groundingdino.py b/groundingdino/models/GroundingDINO/groundingdino.py deleted file mode 100644 index df190bcfd6a4b86caaee4642022dfbb997cfd479..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/groundingdino.py +++ /dev/null @@ -1,857 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Conditional DETR model and criterion classes. -# Copyright (c) 2021 Microsoft. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Modified from DETR (https://github.com/facebookresearch/detr) -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -# ------------------------------------------------------------------------ -# Modified from Deformable DETR (https://github.com/fundamentalvision/Deformable-DETR) -# Copyright (c) 2020 SenseTime. All Rights Reserved. -# ------------------------------------------------------------------------ -import copy -from typing import List - -import torch -import torch.nn.functional as F -from torch import nn -from torchvision.ops.boxes import nms -from transformers import AutoTokenizer, BertModel, BertTokenizer, RobertaModel, RobertaTokenizerFast - -from groundingdino.util import box_ops, get_tokenlizer -from groundingdino.util.misc import ( - NestedTensor, - accuracy, - get_world_size, - interpolate, - inverse_sigmoid, - is_dist_avail_and_initialized, - nested_tensor_from_tensor_list, -) -from groundingdino.util.utils import get_phrases_from_posmap -from groundingdino.util.visualizer import COCOVisualizer -from groundingdino.util.vl_utils import create_positive_map_from_span - -from ..registry import MODULE_BUILD_FUNCS -from .backbone import build_backbone -from .bertwarper import ( - BertModelWarper, - generate_masks_with_special_tokens, - generate_masks_with_special_tokens_and_transfer_map, -) -from .transformer import build_transformer -from .utils import MLP, ContrastiveEmbed, sigmoid_focal_loss - -from .matcher import build_matcher - - - - -class GroundingDINO(nn.Module): - """This is the Cross-Attention Detector module that performs object detection""" - - def __init__( - self, - backbone, - transformer, - num_queries, - aux_loss=False, - iter_update=False, - query_dim=2, - num_feature_levels=1, - nheads=8, - # two stage - two_stage_type="no", # ['no', 'standard'] - dec_pred_bbox_embed_share=True, - two_stage_class_embed_share=True, - two_stage_bbox_embed_share=True, - num_patterns=0, - dn_number=100, - dn_box_noise_scale=0.4, - dn_label_noise_ratio=0.5, - dn_labelbook_size=100, - text_encoder_type="bert-base-uncased", - sub_sentence_present=True, - max_text_len=256, - ): - """Initializes the model. - Parameters: - backbone: torch module of the backbone to be used. See backbone.py - transformer: torch module of the transformer architecture. See transformer.py - num_queries: number of object queries, ie detection slot. This is the maximal number of objects - Conditional DETR can detect in a single image. For COCO, we recommend 100 queries. - aux_loss: True if auxiliary decoding losses (loss at each decoder layer) are to be used. - """ - super().__init__() - self.num_queries = num_queries - self.transformer = transformer - self.hidden_dim = hidden_dim = transformer.d_model - self.num_feature_levels = num_feature_levels - self.nheads = nheads - self.max_text_len = 256 - self.sub_sentence_present = sub_sentence_present - - # setting query dim - self.query_dim = query_dim - assert query_dim == 4 - - # for dn training - self.num_patterns = num_patterns - self.dn_number = dn_number - self.dn_box_noise_scale = dn_box_noise_scale - self.dn_label_noise_ratio = dn_label_noise_ratio - self.dn_labelbook_size = dn_labelbook_size - - # bert - self.tokenizer = get_tokenlizer.get_tokenlizer(text_encoder_type) - self.bert = get_tokenlizer.get_pretrained_language_model(text_encoder_type) - self.bert.pooler.dense.weight.requires_grad_(False) - self.bert.pooler.dense.bias.requires_grad_(False) - self.bert = BertModelWarper(bert_model=self.bert) - - self.feat_map = nn.Linear(self.bert.config.hidden_size, self.hidden_dim, bias=True) - nn.init.constant_(self.feat_map.bias.data, 0) - nn.init.xavier_uniform_(self.feat_map.weight.data) - # freeze - - # special tokens - self.specical_tokens = self.tokenizer.convert_tokens_to_ids(["[CLS]", "[SEP]", ".", "?"]) - - # prepare input projection layers - if num_feature_levels > 1: - num_backbone_outs = len(backbone.num_channels) - input_proj_list = [] - for _ in range(num_backbone_outs): - in_channels = backbone.num_channels[_] - input_proj_list.append( - nn.Sequential( - nn.Conv2d(in_channels, hidden_dim, kernel_size=1), - nn.GroupNorm(32, hidden_dim), - ) - ) - for _ in range(num_feature_levels - num_backbone_outs): - input_proj_list.append( - nn.Sequential( - nn.Conv2d(in_channels, hidden_dim, kernel_size=3, stride=2, padding=1), - nn.GroupNorm(32, hidden_dim), - ) - ) - in_channels = hidden_dim - self.input_proj = nn.ModuleList(input_proj_list) - else: - assert two_stage_type == "no", "two_stage_type should be no if num_feature_levels=1 !!!" - self.input_proj = nn.ModuleList( - [ - nn.Sequential( - nn.Conv2d(backbone.num_channels[-1], hidden_dim, kernel_size=1), - nn.GroupNorm(32, hidden_dim), - ) - ] - ) - - self.backbone = backbone - self.aux_loss = aux_loss - self.box_pred_damping = box_pred_damping = None - - self.iter_update = iter_update - assert iter_update, "Why not iter_update?" - - # prepare pred layers - self.dec_pred_bbox_embed_share = dec_pred_bbox_embed_share - # prepare class & box embed - _class_embed = ContrastiveEmbed() - - _bbox_embed = MLP(hidden_dim, hidden_dim, 4, 3) - nn.init.constant_(_bbox_embed.layers[-1].weight.data, 0) - nn.init.constant_(_bbox_embed.layers[-1].bias.data, 0) - - if dec_pred_bbox_embed_share: - box_embed_layerlist = [_bbox_embed for i in range(transformer.num_decoder_layers)] - else: - box_embed_layerlist = [ - copy.deepcopy(_bbox_embed) for i in range(transformer.num_decoder_layers) - ] - class_embed_layerlist = [_class_embed for i in range(transformer.num_decoder_layers)] - self.bbox_embed = nn.ModuleList(box_embed_layerlist) - self.class_embed = nn.ModuleList(class_embed_layerlist) - self.transformer.decoder.bbox_embed = self.bbox_embed - self.transformer.decoder.class_embed = self.class_embed - - # two stage - self.two_stage_type = two_stage_type - assert two_stage_type in ["no", "standard"], "unknown param {} of two_stage_type".format( - two_stage_type - ) - if two_stage_type != "no": - if two_stage_bbox_embed_share: - assert dec_pred_bbox_embed_share - self.transformer.enc_out_bbox_embed = _bbox_embed - else: - self.transformer.enc_out_bbox_embed = copy.deepcopy(_bbox_embed) - - if two_stage_class_embed_share: - assert dec_pred_bbox_embed_share - self.transformer.enc_out_class_embed = _class_embed - else: - self.transformer.enc_out_class_embed = copy.deepcopy(_class_embed) - - self.refpoint_embed = None - - self._reset_parameters() - - def _reset_parameters(self): - # init input_proj - for proj in self.input_proj: - nn.init.xavier_uniform_(proj[0].weight, gain=1) - nn.init.constant_(proj[0].bias, 0) - - def init_ref_points(self, use_num_queries): - self.refpoint_embed = nn.Embedding(use_num_queries, self.query_dim) - - def forward(self, samples: NestedTensor, targets: List = None, **kw): - """The forward expects a NestedTensor, which consists of: - - samples.tensor: batched images, of shape [batch_size x 3 x H x W] - - samples.mask: a binary mask of shape [batch_size x H x W], containing 1 on padded pixels - - It returns a dict with the following elements: - - "pred_logits": the classification logits (including no-object) for all queries. - Shape= [batch_size x num_queries x num_classes] - - "pred_boxes": The normalized boxes coordinates for all queries, represented as - (center_x, center_y, width, height). These values are normalized in [0, 1], - relative to the size of each individual image (disregarding possible padding). - See PostProcess for information on how to retrieve the unnormalized bounding box. - - "aux_outputs": Optional, only returned when auxilary losses are activated. It is a list of - dictionnaries containing the two above keys for each decoder layer. - """ - if targets is None: - captions = kw["captions"] - else: - captions = [t["caption"] for t in targets] - # encoder texts - - tokenized = self.tokenizer(captions, padding="longest", return_tensors="pt").to( - samples.device - ) - one_hot_token = tokenized - - ( - text_self_attention_masks, - position_ids, - cate_to_token_mask_list, - ) = generate_masks_with_special_tokens_and_transfer_map( - tokenized, self.specical_tokens, self.tokenizer - ) - - if text_self_attention_masks.shape[1] > self.max_text_len: - text_self_attention_masks = text_self_attention_masks[ - :, : self.max_text_len, : self.max_text_len - ] - position_ids = position_ids[:, : self.max_text_len] - tokenized["input_ids"] = tokenized["input_ids"][:, : self.max_text_len] - tokenized["attention_mask"] = tokenized["attention_mask"][:, : self.max_text_len] - tokenized["token_type_ids"] = tokenized["token_type_ids"][:, : self.max_text_len] - - # extract text embeddings - if self.sub_sentence_present: - tokenized_for_encoder = {k: v for k, v in tokenized.items() if k != "attention_mask"} - tokenized_for_encoder["attention_mask"] = text_self_attention_masks - tokenized_for_encoder["position_ids"] = position_ids - else: - tokenized_for_encoder = tokenized - - bert_output = self.bert(**tokenized_for_encoder) # bs, 195, 768 - - encoded_text = self.feat_map(bert_output["last_hidden_state"]) # bs, 195, d_model - text_token_mask = tokenized.attention_mask.bool() # bs, 195 - # text_token_mask: True for nomask, False for mask - # text_self_attention_masks: True for nomask, False for mask - - if encoded_text.shape[1] > self.max_text_len: - encoded_text = encoded_text[:, : self.max_text_len, :] - text_token_mask = text_token_mask[:, : self.max_text_len] - position_ids = position_ids[:, : self.max_text_len] - text_self_attention_masks = text_self_attention_masks[ - :, : self.max_text_len, : self.max_text_len - ] - - text_dict = { - "encoded_text": encoded_text, # bs, 195, d_model - "text_token_mask": text_token_mask, # bs, 195 - "position_ids": position_ids, # bs, 195 - "text_self_attention_masks": text_self_attention_masks, # bs, 195,195 - } - - - if isinstance(samples, (list, torch.Tensor)): - samples = nested_tensor_from_tensor_list(samples) - features, poss = self.backbone(samples) - srcs = [] - masks = [] - for l, feat in enumerate(features): - src, mask = feat.decompose() - srcs.append(self.input_proj[l](src)) - masks.append(mask) - assert mask is not None - if self.num_feature_levels > len(srcs): - _len_srcs = len(srcs) - for l in range(_len_srcs, self.num_feature_levels): - if l == _len_srcs: - src = self.input_proj[l](features[-1].tensors) - else: - src = self.input_proj[l](srcs[-1]) - m = samples.mask - mask = F.interpolate(m[None].float(), size=src.shape[-2:]).to(torch.bool)[0] - pos_l = self.backbone[1](NestedTensor(src, mask)).to(src.dtype) - srcs.append(src) - masks.append(mask) - poss.append(pos_l) - - input_query_bbox = input_query_label = attn_mask = dn_meta = None - hs, reference, hs_enc, ref_enc, init_box_proposal = self.transformer( - srcs, masks, input_query_bbox, poss, input_query_label, attn_mask, text_dict - ) - - - # deformable-detr-like anchor update - outputs_coord_list = [] - for dec_lid, (layer_ref_sig, layer_bbox_embed, layer_hs) in enumerate( - zip(reference[:-1], self.bbox_embed, hs) - ): - layer_delta_unsig = layer_bbox_embed(layer_hs) - layer_outputs_unsig = layer_delta_unsig + inverse_sigmoid(layer_ref_sig) - layer_outputs_unsig = layer_outputs_unsig.sigmoid() - outputs_coord_list.append(layer_outputs_unsig) - outputs_coord_list = torch.stack(outputs_coord_list) - - - outputs_class = torch.stack( - [ - layer_cls_embed(layer_hs, text_dict) - for layer_cls_embed, layer_hs in zip(self.class_embed, hs) - ] - ) - - out = {"pred_logits": outputs_class[-1], "pred_boxes": outputs_coord_list[-1]} - - # Used to calculate losses - bs, len_td = text_dict['text_token_mask'].shape - out['text_mask']=torch.zeros(bs, self.max_text_len, dtype=torch.bool).to( - samples.device - ) - for b in range(bs): - for j in range(len_td): - if text_dict['text_token_mask'][b][j] == True: - out['text_mask'][b][j] = True - - # for intermediate outputs - if self.aux_loss: - out['aux_outputs'] = self._set_aux_loss(outputs_class, outputs_coord_list) - out['token']=one_hot_token - # # for encoder output - if hs_enc is not None: - # prepare intermediate outputs - interm_coord = ref_enc[-1] - interm_class = self.transformer.enc_out_class_embed(hs_enc[-1], text_dict) - out['interm_outputs'] = {'pred_logits': interm_class, 'pred_boxes': interm_coord} - out['interm_outputs_for_matching_pre'] = {'pred_logits': interm_class, 'pred_boxes': init_box_proposal} - - # outputs['pred_logits'].shape - # torch.Size([4, 900, 256]) - - # outputs['pred_boxes'].shape - # torch.Size([4, 900, 4]) - - # outputs['text_mask'].shape - # torch.Size([256]) - - # outputs['text_mask'] - - # outputs['aux_outputs'][0].keys() - # dict_keys(['pred_logits', 'pred_boxes', 'one_hot', 'text_mask']) - - # outputs['aux_outputs'][img_idx] - - # outputs['token'] - # - - # outputs['interm_outputs'].keys() - # dict_keys(['pred_logits', 'pred_boxes', 'one_hot', 'text_mask']) - - - # outputs['interm_outputs_for_matching_pre'].keys() - # dict_keys(['pred_logits', 'pred_boxes']) - - # outputs['one_hot'].shape - # torch.Size([4, 900, 256]) - - return out - - @torch.jit.unused - def _set_aux_loss(self, outputs_class, outputs_coord): - # this is a workaround to make torchscript happy, as torchscript - # doesn't support dictionary with non-homogeneous values, such - # as a dict having both a Tensor and a list. - return [ - {"pred_logits": a, "pred_boxes": b} - for a, b in zip(outputs_class[:-1], outputs_coord[:-1]) - ] - - - - -class SetCriterion(nn.Module): - def __init__(self, matcher, weight_dict, focal_alpha,focal_gamma, losses): - """ Create the criterion. - Parameters: - matcher: module able to compute a matching between targets and proposals - weight_dict: dict containing as key the names of the losses and as values their relative weight. - losses: list of all the losses to be applied. See get_loss for list of available losses. - focal_alpha: alpha in Focal Loss - """ - super().__init__() - self.matcher = matcher - self.weight_dict = weight_dict - self.losses = losses - self.focal_alpha = focal_alpha - self.focal_gamma= focal_gamma - - @torch.no_grad() - def loss_cardinality(self, outputs, targets, indices, num_boxes): - """ Compute the cardinality error, ie the absolute error in the number of predicted non-empty boxes - This is not really a loss, it is intended for logging purposes only. It doesn't propagate gradients - """ - - pred_logits = outputs['pred_logits'] - device = pred_logits.device - tgt_lengths = torch.as_tensor([len(v["labels"]) for v in targets], device=device) - # Count the number of predictions that are NOT "no-object" (which is the last class) - card_pred = (pred_logits.argmax(-1) != pred_logits.shape[-1] - 1).sum(1) - card_err = F.l1_loss(card_pred.float(), tgt_lengths.float()) - losses = {'cardinality_error': card_err} - return losses - - def loss_boxes(self, outputs, targets, indices, num_boxes): - """Compute the losses related to the bounding boxes, the L1 regression loss and the GIoU loss - targets dicts must contain the key "boxes" containing a tensor of dim [nb_target_boxes, 4] - The target boxes are expected in format (center_x, center_y, w, h), normalized by the image size. - """ - assert 'pred_boxes' in outputs - idx = self._get_src_permutation_idx(indices) - src_boxes = outputs['pred_boxes'][idx] - target_boxes = torch.cat([t['boxes'][i] for t, (_, i) in zip(targets, indices)], dim=0) - - loss_bbox = F.l1_loss(src_boxes, target_boxes, reduction='none') - - losses = {} - losses['loss_bbox'] = loss_bbox.sum() / num_boxes - - loss_giou = 1 - torch.diag(box_ops.generalized_box_iou( - box_ops.box_cxcywh_to_xyxy(src_boxes), - box_ops.box_cxcywh_to_xyxy(target_boxes))) - losses['loss_giou'] = loss_giou.sum() / num_boxes - - # calculate the x,y and h,w loss - with torch.no_grad(): - losses['loss_xy'] = loss_bbox[..., :2].sum() / num_boxes - losses['loss_hw'] = loss_bbox[..., 2:].sum() / num_boxes - - - return losses - - - def token_sigmoid_binary_focal_loss(self, outputs, targets, indices, num_boxes): - pred_logits=outputs['pred_logits'] - new_targets=outputs['one_hot'].to(pred_logits.device) - text_mask=outputs['text_mask'] - - assert (new_targets.dim() == 3) - assert (pred_logits.dim() == 3) # batch x from x to - - bs, n, _ = pred_logits.shape - alpha=self.focal_alpha - gamma=self.focal_gamma - if text_mask is not None: - # ODVG: each sample has different mask - text_mask = text_mask.repeat(1, pred_logits.size(1)).view(outputs['text_mask'].shape[0],-1,outputs['text_mask'].shape[1]) - pred_logits = torch.masked_select(pred_logits, text_mask) - new_targets = torch.masked_select(new_targets, text_mask) - - new_targets=new_targets.float() - p = torch.sigmoid(pred_logits) - ce_loss = F.binary_cross_entropy_with_logits(pred_logits, new_targets, reduction="none") - p_t = p * new_targets + (1 - p) * (1 - new_targets) - loss = ce_loss * ((1 - p_t) ** gamma) - - if alpha >= 0: - alpha_t = alpha * new_targets + (1 - alpha) * (1 - new_targets) - loss = alpha_t * loss - - total_num_pos=0 - for batch_indices in indices: - total_num_pos += len(batch_indices[0]) - num_pos_avg_per_gpu = max(total_num_pos , 1.0) - loss=loss.sum()/num_pos_avg_per_gpu - - losses = {'loss_ce': loss} - return losses - - - def _get_src_permutation_idx(self, indices): - # permute predictions following indices - batch_idx = torch.cat([torch.full_like(src, i) for i, (src, _) in enumerate(indices)]) - src_idx = torch.cat([src for (src, _) in indices]) - return batch_idx, src_idx - - def _get_tgt_permutation_idx(self, indices): - # permute targets following indices - batch_idx = torch.cat([torch.full_like(tgt, i) for i, (_, tgt) in enumerate(indices)]) - tgt_idx = torch.cat([tgt for (_, tgt) in indices]) - return batch_idx, tgt_idx - - def get_loss(self, loss, outputs, targets, indices, num_boxes, **kwargs): - loss_map = { - 'labels': self.token_sigmoid_binary_focal_loss, - 'cardinality': self.loss_cardinality, - 'boxes': self.loss_boxes, - } - assert loss in loss_map, f'do you really want to compute {loss} loss?' - return loss_map[loss](outputs, targets, indices, num_boxes, **kwargs) - - def forward(self, outputs, targets, cat_list, caption, return_indices=False): - """ This performs the loss computation. - Parameters: - outputs: dict of tensors, see the output specification of the model for the format - targets: list of dicts, such that len(targets) == batch_size. - The expected keys in each dict depends on the losses applied, see each loss' doc - - return_indices: used for vis. if True, the layer0-5 indices will be returned as well. - """ - device=next(iter(outputs.values())).device - one_hot = torch.zeros(outputs['pred_logits'].size(),dtype=torch.int64) # torch.Size([bs, 900, 256]) - token = outputs['token'] - - label_map_list = [] - indices = [] - for j in range(len(cat_list)): # bs - label_map=[] - for i in range(len(cat_list[j])): - label_id=torch.tensor([i]) - per_label=create_positive_map(token[j], label_id, cat_list[j], caption[j]) - label_map.append(per_label) - label_map=torch.stack(label_map,dim=0).squeeze(1) - label_map_list.append(label_map) - for j in range(len(cat_list)): # bs - for_match = { - "pred_logits" : outputs['pred_logits'][j].unsqueeze(0), - "pred_boxes" : outputs['pred_boxes'][j].unsqueeze(0) - } - inds = self.matcher(for_match, [targets[j]], label_map_list[j]) - indices.extend(inds) - # indices : A list of size batch_size, containing tuples of (index_i, index_j) where: - # - index_i is the indices of the selected predictions (in order) - # - index_j is the indices of the corresponding selected targets (in order) - - # import pdb; pdb.set_trace() - tgt_ids = [v["labels"].cpu() for v in targets] - # len(tgt_ids) == bs - for i in range(len(indices)): - tgt_ids[i]=tgt_ids[i][indices[i][1]] - one_hot[i,indices[i][0]] = label_map_list[i][tgt_ids[i]].to(torch.long) - outputs['one_hot'] = one_hot - if return_indices: - indices0_copy = indices - indices_list = [] - - # Compute the average number of target boxes accross all nodes, for normalization purposes - num_boxes_list = [len(t["labels"]) for t in targets] - num_boxes = sum(num_boxes_list) - num_boxes = torch.as_tensor([num_boxes], dtype=torch.float, device=device) - if is_dist_avail_and_initialized(): - torch.distributed.all_reduce(num_boxes) - num_boxes = torch.clamp(num_boxes / get_world_size(), min=1).item() - - # Compute all the requested losses - losses = {} - for loss in self.losses: - losses.update(self.get_loss(loss, outputs, targets, indices, num_boxes)) - - # In case of auxiliary losses, we repeat this process with the output of each intermediate layer. - if 'aux_outputs' in outputs: - for idx, aux_outputs in enumerate(outputs['aux_outputs']): - indices = [] - for j in range(len(cat_list)): # bs - aux_output_single = { - 'pred_logits' : aux_outputs['pred_logits'][j].unsqueeze(0), - 'pred_boxes': aux_outputs['pred_boxes'][j].unsqueeze(0) - } - inds = self.matcher(aux_output_single, [targets[j]], label_map_list[j]) - indices.extend(inds) - one_hot_aux = torch.zeros(outputs['pred_logits'].size(),dtype=torch.int64) - tgt_ids = [v["labels"].cpu() for v in targets] - for i in range(len(indices)): - tgt_ids[i]=tgt_ids[i][indices[i][1]] - one_hot_aux[i,indices[i][0]] = label_map_list[i][tgt_ids[i]].to(torch.long) - aux_outputs['one_hot'] = one_hot_aux - aux_outputs['text_mask'] = outputs['text_mask'] - if return_indices: - indices_list.append(indices) - for loss in self.losses: - kwargs = {} - l_dict = self.get_loss(loss, aux_outputs, targets, indices, num_boxes, **kwargs) - l_dict = {k + f'_{idx}': v for k, v in l_dict.items()} - losses.update(l_dict) - - # interm_outputs loss - if 'interm_outputs' in outputs: - interm_outputs = outputs['interm_outputs'] - indices = [] - for j in range(len(cat_list)): # bs - interm_output_single = { - 'pred_logits' : interm_outputs['pred_logits'][j].unsqueeze(0), - 'pred_boxes': interm_outputs['pred_boxes'][j].unsqueeze(0) - } - inds = self.matcher(interm_output_single, [targets[j]], label_map_list[j]) - indices.extend(inds) - one_hot_aux = torch.zeros(outputs['pred_logits'].size(),dtype=torch.int64) - tgt_ids = [v["labels"].cpu() for v in targets] - for i in range(len(indices)): - tgt_ids[i]=tgt_ids[i][indices[i][1]] - one_hot_aux[i,indices[i][0]] = label_map_list[i][tgt_ids[i]].to(torch.long) - interm_outputs['one_hot'] = one_hot_aux - interm_outputs['text_mask'] = outputs['text_mask'] - if return_indices: - indices_list.append(indices) - for loss in self.losses: - kwargs = {} - l_dict = self.get_loss(loss, interm_outputs, targets, indices, num_boxes, **kwargs) - l_dict = {k + f'_interm': v for k, v in l_dict.items()} - losses.update(l_dict) - - if return_indices: - indices_list.append(indices0_copy) - return losses, indices_list - - return losses - - -class PostProcess(nn.Module): - """ This module converts the model's output into the format expected by the coco api""" - def __init__(self, num_select=100,text_encoder_type='text_encoder_type', nms_iou_threshold=-1,use_coco_eval=False,args=None) -> None: - super().__init__() - self.num_select = num_select - self.tokenizer = get_tokenlizer.get_tokenlizer(text_encoder_type) - if args.use_coco_eval: - from pycocotools.coco import COCO - coco = COCO(args.coco_val_path) - category_dict = coco.loadCats(coco.getCatIds()) - cat_list = [item['name'] for item in category_dict] - else: - cat_list=args.label_list - caption = " . ".join(cat_list) + ' .' - tokenized = self.tokenizer(caption, padding="longest", return_tensors="pt") - label_list = torch.arange(len(cat_list)) - pos_map=create_positive_map(tokenized,label_list,cat_list,caption) - # build a mapping from label_id to pos_map - if args.use_coco_eval: - id_map = {0: 1, 1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: 7, 7: 8, 8: 9, 9: 10, 10: 11, 11: 13, 12: 14, 13: 15, 14: 16, 15: 17, 16: 18, 17: 19, 18: 20, 19: 21, 20: 22, 21: 23, 22: 24, 23: 25, 24: 27, 25: 28, 26: 31, 27: 32, 28: 33, 29: 34, 30: 35, 31: 36, 32: 37, 33: 38, 34: 39, 35: 40, 36: 41, 37: 42, 38: 43, 39: 44, 40: 46, - 41: 47, 42: 48, 43: 49, 44: 50, 45: 51, 46: 52, 47: 53, 48: 54, 49: 55, 50: 56, 51: 57, 52: 58, 53: 59, 54: 60, 55: 61, 56: 62, 57: 63, 58: 64, 59: 65, 60: 67, 61: 70, 62: 72, 63: 73, 64: 74, 65: 75, 66: 76, 67: 77, 68: 78, 69: 79, 70: 80, 71: 81, 72: 82, 73: 84, 74: 85, 75: 86, 76: 87, 77: 88, 78: 89, 79: 90} - new_pos_map = torch.zeros((91, 256)) - for k, v in id_map.items(): - new_pos_map[v] = pos_map[k] - pos_map=new_pos_map - - - self.nms_iou_threshold=nms_iou_threshold - self.positive_map = pos_map - - @torch.no_grad() - def forward(self, outputs, target_sizes, not_to_xyxy=False, test=False): - """ Perform the computation - Parameters: - outputs: raw outputs of the model - target_sizes: tensor of dimension [batch_size x 2] containing the size of each images of the batch - For evaluation, this must be the original image size (before any data augmentation) - For visualization, this should be the image size after data augment, but before padding - """ - num_select = self.num_select - out_logits, out_bbox = outputs['pred_logits'], outputs['pred_boxes'] - - - prob_to_token = out_logits.sigmoid() - pos_maps = self.positive_map.to(prob_to_token.device) - for label_ind in range(len(pos_maps)): - if pos_maps[label_ind].sum() != 0: - pos_maps[label_ind]=pos_maps[label_ind]/pos_maps[label_ind].sum() - - prob_to_label = prob_to_token @ pos_maps.T - - assert len(out_logits) == len(target_sizes) - assert target_sizes.shape[1] == 2 - - prob = prob_to_label - topk_values, topk_indexes = torch.topk(prob.view(prob.shape[0], -1), num_select, dim=1) - scores = topk_values - topk_boxes = torch.div(topk_indexes, prob.shape[2], rounding_mode='trunc') - labels = topk_indexes % prob.shape[2] - if not_to_xyxy: - boxes = out_bbox - else: - boxes = box_ops.box_cxcywh_to_xyxy(out_bbox) - - # if test: - # assert not not_to_xyxy - # boxes[:,:,2:] = boxes[:,:,2:] - boxes[:,:,:2] - boxes = torch.gather(boxes, 1, topk_boxes.unsqueeze(-1).repeat(1,1,4)) - - # and from relative [0, 1] to absolute [0, height] coordinates - img_h, img_w = target_sizes.unbind(1) - scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1) - boxes = boxes * scale_fct[:, None, :] - - if self.nms_iou_threshold > 0: - item_indices = [nms(b, s, iou_threshold=self.nms_iou_threshold) for b,s in zip(boxes, scores)] - - results = [{'scores': s[i], 'labels': l[i], 'boxes': b[i]} for s, l, b, i in zip(scores, labels, boxes, item_indices)] - else: - results = [{'scores': s, 'labels': l, 'boxes': b} for s, l, b in zip(scores, labels, boxes)] - results = [{'scores': s, 'labels': l, 'boxes': b} for s, l, b in zip(scores, labels, boxes)] - return results - - -@MODULE_BUILD_FUNCS.registe_with_name(module_name="groundingdino") -def build_groundingdino(args): - device = torch.device(args.device) - backbone = build_backbone(args) - transformer = build_transformer(args) - - dn_labelbook_size = args.dn_labelbook_size - dec_pred_bbox_embed_share = args.dec_pred_bbox_embed_share - sub_sentence_present = args.sub_sentence_present - - model = GroundingDINO( - backbone, - transformer, - num_queries=args.num_queries, - aux_loss=args.aux_loss, - iter_update=True, - query_dim=4, - num_feature_levels=args.num_feature_levels, - nheads=args.nheads, - dec_pred_bbox_embed_share=dec_pred_bbox_embed_share, - two_stage_type=args.two_stage_type, - two_stage_bbox_embed_share=args.two_stage_bbox_embed_share, - two_stage_class_embed_share=args.two_stage_class_embed_share, - num_patterns=args.num_patterns, - dn_number=0, - dn_box_noise_scale=args.dn_box_noise_scale, - dn_label_noise_ratio=args.dn_label_noise_ratio, - dn_labelbook_size=dn_labelbook_size, - text_encoder_type=args.text_encoder_type, - sub_sentence_present=sub_sentence_present, - max_text_len=args.max_text_len, - ) - - - - matcher = build_matcher(args) - - # prepare weight dict - weight_dict = {'loss_ce': args.cls_loss_coef, 'loss_bbox': args.bbox_loss_coef} - weight_dict['loss_giou'] = args.giou_loss_coef - clean_weight_dict_wo_dn = copy.deepcopy(weight_dict) - - - - clean_weight_dict = copy.deepcopy(weight_dict) - - # TODO this is a hack - if args.aux_loss: - aux_weight_dict = {} - for i in range(args.dec_layers - 1): - aux_weight_dict.update({k + f'_{i}': v for k, v in clean_weight_dict.items()}) - weight_dict.update(aux_weight_dict) - - if args.two_stage_type != 'no': - interm_weight_dict = {} - try: - no_interm_box_loss = args.no_interm_box_loss - except: - no_interm_box_loss = False - _coeff_weight_dict = { - 'loss_ce': 1.0, - 'loss_bbox': 1.0 if not no_interm_box_loss else 0.0, - 'loss_giou': 1.0 if not no_interm_box_loss else 0.0, - } - try: - interm_loss_coef = args.interm_loss_coef - except: - interm_loss_coef = 1.0 - interm_weight_dict.update({k + f'_interm': v * interm_loss_coef * _coeff_weight_dict[k] for k, v in clean_weight_dict_wo_dn.items()}) - weight_dict.update(interm_weight_dict) - - # losses = ['labels', 'boxes', 'cardinality'] - losses = ['labels', 'boxes'] - - criterion = SetCriterion(matcher=matcher, weight_dict=weight_dict, - focal_alpha=args.focal_alpha, focal_gamma=args.focal_gamma,losses=losses - ) - criterion.to(device) - postprocessors = {'bbox': PostProcess(num_select=args.num_select , text_encoder_type=args.text_encoder_type,nms_iou_threshold=args.nms_iou_threshold,args=args)} - - return model, criterion, postprocessors - -def create_positive_map(tokenized, tokens_positive,cat_list,caption): - """construct a map such that positive_map[i,j] = True iff box i is associated to token j""" - positive_map = torch.zeros((len(tokens_positive), 256), dtype=torch.float) - - for j,label in enumerate(tokens_positive): - - start_ind = caption.find(cat_list[label]) - end_ind = start_ind + len(cat_list[label]) - 1 - beg_pos = tokenized.char_to_token(start_ind) - try: - end_pos = tokenized.char_to_token(end_ind) - except: - end_pos = None - if end_pos is None: - try: - end_pos = tokenized.char_to_token(end_ind - 1) - if end_pos is None: - end_pos = tokenized.char_to_token(end_ind - 2) - except: - end_pos = None - # except Exception as e: - # print("beg:", beg, "end:", end) - # print("token_positive:", tokens_positive) - # # print("beg_pos:", beg_pos, "end_pos:", end_pos) - # raise e - # if beg_pos is None: - # try: - # beg_pos = tokenized.char_to_token(beg + 1) - # if beg_pos is None: - # beg_pos = tokenized.char_to_token(beg + 2) - # except: - # beg_pos = None - # if end_pos is None: - # try: - # end_pos = tokenized.char_to_token(end - 2) - # if end_pos is None: - # end_pos = tokenized.char_to_token(end - 3) - # except: - # end_pos = None - if beg_pos is None or end_pos is None: - continue - if beg_pos < 0 or end_pos < 0: - continue - if beg_pos > end_pos: - continue - # assert beg_pos is not None and end_pos is not None - positive_map[j,beg_pos: end_pos + 1].fill_(1) - return positive_map - - diff --git a/groundingdino/models/GroundingDINO/matcher.py b/groundingdino/models/GroundingDINO/matcher.py deleted file mode 100644 index 69d09a6464801c5c7c37e46a4f457121d7cce61c..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/matcher.py +++ /dev/null @@ -1,218 +0,0 @@ -# ------------------------------------------------------------------------ -# DINO -# Copyright (c) 2022 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Modules to compute the matching cost and solve the corresponding LSAP. -# Copyright (c) 2021 Microsoft. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Modified from DETR (https://github.com/facebookresearch/detr) -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -# ------------------------------------------------------------------------ -# Modified from Deformable DETR (https://github.com/fundamentalvision/Deformable-DETR) -# Copyright (c) 2020 SenseTime. All Rights Reserved. -# ------------------------------------------------------------------------ - - -import torch, os -from torch import nn -from scipy.optimize import linear_sum_assignment - -from util.box_ops import box_cxcywh_to_xyxy, generalized_box_iou - - -class HungarianMatcher(nn.Module): - """This class computes an assignment between the targets and the predictions of the network - For efficiency reasons, the targets don't include the no_object. Because of this, in general, - there are more predictions than targets. In this case, we do a 1-to-1 matching of the best predictions, - while the others are un-matched (and thus treated as non-objects). - """ - - def __init__(self, cost_class: float = 1, cost_bbox: float = 1, cost_giou: float = 1, focal_alpha = 0.25): - """Creates the matcher - Params: - cost_class: This is the relative weight of the classification error in the matching cost - cost_bbox: This is the relative weight of the L1 error of the bounding box coordinates in the matching cost - cost_giou: This is the relative weight of the giou loss of the bounding box in the matching cost - """ - super().__init__() - self.cost_class = cost_class - self.cost_bbox = cost_bbox - self.cost_giou = cost_giou - assert cost_class != 0 or cost_bbox != 0 or cost_giou != 0, "all costs cant be 0" - - self.focal_alpha = focal_alpha - - @torch.no_grad() - def forward(self, outputs, targets, label_map): - """ Performs the matching - Params: - outputs: This is a dict that contains at least these entries: - "pred_logits": Tensor of dim [batch_size, num_queries, num_classes] with the classification logits - "pred_boxes": Tensor of dim [batch_size, num_queries, 4] with the predicted box coordinates - targets: This is a list of targets (len(targets) = batch_size), where each target is a dict containing: - "labels": Tensor of dim [num_target_boxes] (where num_target_boxes is the number of ground-truth - objects in the target) containing the class labels - "boxes": Tensor of dim [num_target_boxes, 4] containing the target box coordinates - Returns: - A list of size batch_size, containing tuples of (index_i, index_j) where: - - index_i is the indices of the selected predictions (in order) - - index_j is the indices of the corresponding selected targets (in order) - For each batch element, it holds: - len(index_i) = len(index_j) = min(num_queries, num_target_boxes) - """ - - bs, num_queries = outputs["pred_logits"].shape[:2] - - # We flatten to compute the cost matrices in a batch - out_prob = outputs["pred_logits"].flatten(0, 1).sigmoid() # [batch_size * num_queries, num_classes] - out_bbox = outputs["pred_boxes"].flatten(0, 1) # [batch_size * num_queries, 4] - - # Also concat the target labels and boxes - tgt_ids = torch.cat([v["labels"] for v in targets]) - tgt_bbox = torch.cat([v["boxes"] for v in targets]) - - # Compute the classification cost. - alpha = self.focal_alpha - gamma = 2.0 - - new_label_map=label_map[tgt_ids.cpu()] - - neg_cost_class = (1 - alpha) * (out_prob ** gamma) * (-(1 - out_prob + 1e-8).log()) - pos_cost_class = alpha * ((1 - out_prob) ** gamma) * (-(out_prob + 1e-8).log()) - new_label_map=new_label_map.to(pos_cost_class.device) - - cost_bbox = torch.cdist(out_bbox, tgt_bbox, p=1) - - # cost_class=(pos_cost_class @ new_label_map.T - neg_cost_class@ new_label_map.T) - cost_class=[] - for idx_map in new_label_map: - idx_map = idx_map / idx_map.sum() - cost_class.append(pos_cost_class @ idx_map - neg_cost_class@ idx_map) - if cost_class: - cost_class=torch.stack(cost_class,dim=0).T - else: - cost_class=torch.zeros_like(cost_bbox) - # Compute the L1 cost between boxes - - - # Compute the giou cost betwen boxes - cost_giou = -generalized_box_iou(box_cxcywh_to_xyxy(out_bbox), box_cxcywh_to_xyxy(tgt_bbox)) - # import pdb;pdb.set_trace() - # Final cost matrix - C = self.cost_bbox * cost_bbox + self.cost_class * cost_class + self.cost_giou * cost_giou - C = C.view(bs, num_queries, -1).cpu() - C[torch.isnan(C)] = 0.0 - C[torch.isinf(C)] = 0.0 - - sizes = [len(v["boxes"]) for v in targets] - try: - indices = [linear_sum_assignment(c[i]) for i, c in enumerate(C.split(sizes, -1))] - except: - print("warning: use SimpleMinsumMatcher") - indices = [] - device = C.device - for i, (c, _size) in enumerate(zip(C.split(sizes, -1), sizes)): - weight_mat = c[i] - idx_i = weight_mat.min(0)[1] - idx_j = torch.arange(_size).to(device) - indices.append((idx_i, idx_j)) - return [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices] - - -class SimpleMinsumMatcher(nn.Module): - """This class computes an assignment between the targets and the predictions of the network - For efficiency reasons, the targets don't include the no_object. Because of this, in general, - there are more predictions than targets. In this case, we do a 1-to-1 matching of the best predictions, - while the others are un-matched (and thus treated as non-objects). - """ - - def __init__(self, cost_class: float = 1, cost_bbox: float = 1, cost_giou: float = 1, focal_alpha = 0.25): - """Creates the matcher - Params: - cost_class: This is the relative weight of the classification error in the matching cost - cost_bbox: This is the relative weight of the L1 error of the bounding box coordinates in the matching cost - cost_giou: This is the relative weight of the giou loss of the bounding box in the matching cost - """ - super().__init__() - self.cost_class = cost_class - self.cost_bbox = cost_bbox - self.cost_giou = cost_giou - assert cost_class != 0 or cost_bbox != 0 or cost_giou != 0, "all costs cant be 0" - - self.focal_alpha = focal_alpha - - @torch.no_grad() - def forward(self, outputs, targets): - """ Performs the matching - Params: - outputs: This is a dict that contains at least these entries: - "pred_logits": Tensor of dim [batch_size, num_queries, num_classes] with the classification logits - "pred_boxes": Tensor of dim [batch_size, num_queries, 4] with the predicted box coordinates - targets: This is a list of targets (len(targets) = batch_size), where each target is a dict containing: - "labels": Tensor of dim [num_target_boxes] (where num_target_boxes is the number of ground-truth - objects in the target) containing the class labels - "boxes": Tensor of dim [num_target_boxes, 4] containing the target box coordinates - Returns: - A list of size batch_size, containing tuples of (index_i, index_j) where: - - index_i is the indices of the selected predictions (in order) - - index_j is the indices of the corresponding selected targets (in order) - For each batch element, it holds: - len(index_i) = len(index_j) = min(num_queries, num_target_boxes) - """ - - bs, num_queries = outputs["pred_logits"].shape[:2] - - # We flatten to compute the cost matrices in a batch - out_prob = outputs["pred_logits"].flatten(0, 1).sigmoid() # [batch_size * num_queries, num_classes] - out_bbox = outputs["pred_boxes"].flatten(0, 1) # [batch_size * num_queries, 4] - - # Also concat the target labels and boxes - tgt_ids = torch.cat([v["labels"] for v in targets]) - tgt_bbox = torch.cat([v["boxes"] for v in targets]) - - # Compute the classification cost. - alpha = self.focal_alpha - gamma = 2.0 - neg_cost_class = (1 - alpha) * (out_prob ** gamma) * (-(1 - out_prob + 1e-8).log()) - pos_cost_class = alpha * ((1 - out_prob) ** gamma) * (-(out_prob + 1e-8).log()) - cost_class = pos_cost_class[:, tgt_ids] - neg_cost_class[:, tgt_ids] - - # Compute the L1 cost between boxes - cost_bbox = torch.cdist(out_bbox, tgt_bbox, p=1) - - # Compute the giou cost betwen boxes - cost_giou = -generalized_box_iou(box_cxcywh_to_xyxy(out_bbox), box_cxcywh_to_xyxy(tgt_bbox)) - - # Final cost matrix - - C = self.cost_bbox * cost_bbox + self.cost_class * cost_class + self.cost_giou * cost_giou - C = C.view(bs, num_queries, -1) - - sizes = [len(v["boxes"]) for v in targets] - indices = [] - device = C.device - for i, (c, _size) in enumerate(zip(C.split(sizes, -1), sizes)): - weight_mat = c[i] - idx_i = weight_mat.min(0)[1] - idx_j = torch.arange(_size).to(device) - indices.append((idx_i, idx_j)) - - return [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices] - - -def build_matcher(args): - assert args.matcher_type in ['HungarianMatcher', 'SimpleMinsumMatcher'], "Unknown args.matcher_type: {}".format(args.matcher_type) - if args.matcher_type == 'HungarianMatcher': - return HungarianMatcher( - cost_class=args.set_cost_class, cost_bbox=args.set_cost_bbox, cost_giou=args.set_cost_giou, - focal_alpha=args.focal_alpha - ) - elif args.matcher_type == 'SimpleMinsumMatcher': - return SimpleMinsumMatcher( - cost_class=args.set_cost_class, cost_bbox=args.set_cost_bbox, cost_giou=args.set_cost_giou, - focal_alpha=args.focal_alpha - ) - else: - raise NotImplementedError("Unknown args.matcher_type: {}".format(args.matcher_type)) \ No newline at end of file diff --git a/groundingdino/models/GroundingDINO/ms_deform_attn.py b/groundingdino/models/GroundingDINO/ms_deform_attn.py deleted file mode 100644 index 4a5222c1b548d44458d68b440149d2c8b0c8e39e..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ms_deform_attn.py +++ /dev/null @@ -1,416 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Deformable DETR -# Copyright (c) 2020 SenseTime. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------------------------------ -# Modified from: -# https://github.com/fundamentalvision/Deformable-DETR/blob/main/models/ops/functions/ms_deform_attn_func.py -# https://github.com/fundamentalvision/Deformable-DETR/blob/main/models/ops/modules/ms_deform_attn.py -# https://github.com/open-mmlab/mmcv/blob/master/mmcv/ops/multi_scale_deform_attn.py -# ------------------------------------------------------------------------------------------------ - -import math -import warnings -from typing import Optional - -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch.autograd import Function -from torch.autograd.function import once_differentiable -from torch.nn.init import constant_, xavier_uniform_ -import loralib as lora - -try: - # from groundingdino import _C - import MultiScaleDeformableAttention as _C -except: - warnings.warn("Failed to load custom C++ ops. Running on CPU mode Only!") - - -# helpers -def _is_power_of_2(n): - if (not isinstance(n, int)) or (n < 0): - raise ValueError("invalid input for _is_power_of_2: {} (type: {})".format(n, type(n))) - return (n & (n - 1) == 0) and n != 0 - - -class MultiScaleDeformableAttnFunction(Function): - @staticmethod - def forward( - ctx, - value, - value_spatial_shapes, - value_level_start_index, - sampling_locations, - attention_weights, - im2col_step, - ): - ctx.im2col_step = im2col_step - output = _C.ms_deform_attn_forward( - value, - value_spatial_shapes, - value_level_start_index, - sampling_locations, - attention_weights, - ctx.im2col_step, - ) - ctx.save_for_backward( - value, - value_spatial_shapes, - value_level_start_index, - sampling_locations, - attention_weights, - ) - return output - - @staticmethod - @once_differentiable - def backward(ctx, grad_output): - ( - value, - value_spatial_shapes, - value_level_start_index, - sampling_locations, - attention_weights, - ) = ctx.saved_tensors - grad_value, grad_sampling_loc, grad_attn_weight = _C.ms_deform_attn_backward( - value, - value_spatial_shapes, - value_level_start_index, - sampling_locations, - attention_weights, - grad_output, - ctx.im2col_step, - ) - - return grad_value, None, None, grad_sampling_loc, grad_attn_weight, None - - -def multi_scale_deformable_attn_pytorch( - value: torch.Tensor, - value_spatial_shapes: torch.Tensor, - sampling_locations: torch.Tensor, - attention_weights: torch.Tensor, -) -> torch.Tensor: - - bs, _, num_heads, embed_dims = value.shape - _, num_queries, num_heads, num_levels, num_points, _ = sampling_locations.shape - value_list = value.split([H_ * W_ for H_, W_ in value_spatial_shapes], dim=1) - sampling_grids = 2 * sampling_locations - 1 - sampling_value_list = [] - for level, (H_, W_) in enumerate(value_spatial_shapes): - # bs, H_*W_, num_heads, embed_dims -> - # bs, H_*W_, num_heads*embed_dims -> - # bs, num_heads*embed_dims, H_*W_ -> - # bs*num_heads, embed_dims, H_, W_ - value_l_ = ( - value_list[level].flatten(2).transpose(1, 2).reshape(bs * num_heads, embed_dims, H_, W_) - ) - # bs, num_queries, num_heads, num_points, 2 -> - # bs, num_heads, num_queries, num_points, 2 -> - # bs*num_heads, num_queries, num_points, 2 - sampling_grid_l_ = sampling_grids[:, :, :, level].transpose(1, 2).flatten(0, 1) - # bs*num_heads, embed_dims, num_queries, num_points - sampling_value_l_ = F.grid_sample( - value_l_, sampling_grid_l_, mode="bilinear", padding_mode="zeros", align_corners=False - ) - sampling_value_list.append(sampling_value_l_) - # (bs, num_queries, num_heads, num_levels, num_points) -> - # (bs, num_heads, num_queries, num_levels, num_points) -> - # (bs, num_heads, 1, num_queries, num_levels*num_points) - attention_weights = attention_weights.transpose(1, 2).reshape( - bs * num_heads, 1, num_queries, num_levels * num_points - ) - output = ( - (torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights) - .sum(-1) - .view(bs, num_heads * embed_dims, num_queries) - ) - return output.transpose(1, 2).contiguous() - - -class MultiScaleDeformableAttention(nn.Module): - """Multi-Scale Deformable Attention Module used in Deformable-DETR - - `Deformable DETR: Deformable Transformers for End-to-End Object Detection. - `_. - - Args: - embed_dim (int): The embedding dimension of Attention. Default: 256. - num_heads (int): The number of attention heads. Default: 8. - num_levels (int): The number of feature map used in Attention. Default: 4. - num_points (int): The number of sampling points for each query - in each head. Default: 4. - img2col_steps (int): The step used in image_to_column. Defualt: 64. - dropout (float): Dropout layer used in output. Default: 0.1. - batch_first (bool): if ``True``, then the input and output tensor will be - provided as `(bs, n, embed_dim)`. Default: False. `(n, bs, embed_dim)` - """ - - def __init__( - self, - embed_dim: int = 256, - num_heads: int = 8, - num_levels: int = 4, - num_points: int = 4, - img2col_step: int = 64, - batch_first: bool = False, - ): - super().__init__() - if embed_dim % num_heads != 0: - raise ValueError( - "embed_dim must be divisible by num_heads, but got {} and {}".format( - embed_dim, num_heads - ) - ) - head_dim = embed_dim // num_heads - - self.batch_first = batch_first - - if not _is_power_of_2(head_dim): - warnings.warn( - """ - You'd better set d_model in MSDeformAttn to make sure that - each dim of the attention head a power of 2, which is more efficient. - """ - ) - - self.im2col_step = img2col_step - self.embed_dim = embed_dim - self.num_heads = num_heads - self.num_levels = num_levels - self.num_points = num_points - r = 12 - self.sampling_offsets = lora.Linear(embed_dim, num_heads * num_levels * num_points * 2 , r=r ) - self.attention_weights = lora.Linear(embed_dim, num_heads * num_levels * num_points , r=r) - self.value_proj = lora.Linear(embed_dim, embed_dim , r=r) - self.output_proj = lora.Linear(embed_dim, embed_dim , r=r) - - self.init_weights() - - def _reset_parameters(self): - return self.init_weights() - - def init_weights(self): - """ - Default initialization for Parameters of Module. - """ - constant_(self.sampling_offsets.weight.data, 0.0) - thetas = torch.arange(self.num_heads, dtype=torch.float32) * ( - 2.0 * math.pi / self.num_heads - ) - grid_init = torch.stack([thetas.cos(), thetas.sin()], -1) - grid_init = ( - (grid_init / grid_init.abs().max(-1, keepdim=True)[0]) - .view(self.num_heads, 1, 1, 2) - .repeat(1, self.num_levels, self.num_points, 1) - ) - for i in range(self.num_points): - grid_init[:, :, i, :] *= i + 1 - with torch.no_grad(): - self.sampling_offsets.bias = nn.Parameter(grid_init.view(-1)) - constant_(self.attention_weights.weight.data, 0.0) - constant_(self.attention_weights.bias.data, 0.0) - xavier_uniform_(self.value_proj.weight.data) - constant_(self.value_proj.bias.data, 0.0) - xavier_uniform_(self.output_proj.weight.data) - constant_(self.output_proj.bias.data, 0.0) - - def freeze_sampling_offsets(self): - print("Freeze sampling offsets") - self.sampling_offsets.weight.requires_grad = False - self.sampling_offsets.bias.requires_grad = False - - def freeze_attention_weights(self): - print("Freeze attention weights") - self.attention_weights.weight.requires_grad = False - self.attention_weights.bias.requires_grad = False - - def forward( - self, - query: torch.Tensor, - key: Optional[torch.Tensor] = None, - value: Optional[torch.Tensor] = None, - query_pos: Optional[torch.Tensor] = None, - key_padding_mask: Optional[torch.Tensor] = None, - reference_points: Optional[torch.Tensor] = None, - spatial_shapes: Optional[torch.Tensor] = None, - level_start_index: Optional[torch.Tensor] = None, - **kwargs - ) -> torch.Tensor: - - """Forward Function of MultiScaleDeformableAttention - - Args: - query (torch.Tensor): Query embeddings with shape - `(num_query, bs, embed_dim)` - key (torch.Tensor): Key embeddings with shape - `(num_key, bs, embed_dim)` - value (torch.Tensor): Value embeddings with shape - `(num_key, bs, embed_dim)` - query_pos (torch.Tensor): The position embedding for `query`. Default: None. - key_padding_mask (torch.Tensor): ByteTensor for `query`, with shape `(bs, num_key)`, - indicating which elements within `key` to be ignored in attention. - reference_points (torch.Tensor): The normalized reference points - with shape `(bs, num_query, num_levels, 2)`, - all elements is range in [0, 1], top-left (0, 0), - bottom-right (1, 1), including padding are. - or `(N, Length_{query}, num_levels, 4)`, add additional - two dimensions `(h, w)` to form reference boxes. - spatial_shapes (torch.Tensor): Spatial shape of features in different levels. - With shape `(num_levels, 2)`, last dimension represents `(h, w)`. - level_start_index (torch.Tensor): The start index of each level. A tensor with - shape `(num_levels, )` which can be represented as - `[0, h_0 * w_0, h_0 * w_0 + h_1 * w_1, ...]`. - - Returns: - torch.Tensor: forward results with shape `(num_query, bs, embed_dim)` - """ - - if value is None: - value = query - - if query_pos is not None: - query = query + query_pos - - if not self.batch_first: - # change to (bs, num_query ,embed_dims) - query = query.permute(1, 0, 2) - value = value.permute(1, 0, 2) - - bs, num_query, _ = query.shape - bs, num_value, _ = value.shape - - assert (spatial_shapes[:, 0] * spatial_shapes[:, 1]).sum() == num_value - - value = self.value_proj(value) - if key_padding_mask is not None: - value = value.masked_fill(key_padding_mask[..., None], float(0)) - value = value.view(bs, num_value, self.num_heads, -1) - sampling_offsets = self.sampling_offsets(query).view( - bs, num_query, self.num_heads, self.num_levels, self.num_points, 2 - ) - attention_weights = self.attention_weights(query).view( - bs, num_query, self.num_heads, self.num_levels * self.num_points - ) - attention_weights = attention_weights.softmax(-1) - attention_weights = attention_weights.view( - bs, - num_query, - self.num_heads, - self.num_levels, - self.num_points, - ) - - # bs, num_query, num_heads, num_levels, num_points, 2 - if reference_points.shape[-1] == 2: - offset_normalizer = torch.stack([spatial_shapes[..., 1], spatial_shapes[..., 0]], -1) - sampling_locations = ( - reference_points[:, :, None, :, None, :] - + sampling_offsets / offset_normalizer[None, None, None, :, None, :] - ) - elif reference_points.shape[-1] == 4: - sampling_locations = ( - reference_points[:, :, None, :, None, :2] - + sampling_offsets - / self.num_points - * reference_points[:, :, None, :, None, 2:] - * 0.5 - ) - else: - raise ValueError( - "Last dim of reference_points must be 2 or 4, but get {} instead.".format( - reference_points.shape[-1] - ) - ) - - if torch.cuda.is_available() and value.is_cuda: - halffloat = False - if value.dtype == torch.float16: - halffloat = True - value = value.float() - sampling_locations = sampling_locations.float() - attention_weights = attention_weights.float() - - output = MultiScaleDeformableAttnFunction.apply( - value, - spatial_shapes, - level_start_index, - sampling_locations, - attention_weights, - self.im2col_step, - ) - - if halffloat: - output = output.half() - else: - output = multi_scale_deformable_attn_pytorch( - value, spatial_shapes, sampling_locations, attention_weights - ) - - output = self.output_proj(output) - - if not self.batch_first: - output = output.permute(1, 0, 2) - - return output - - -def create_dummy_class(klass, dependency, message=""): - """ - When a dependency of a class is not available, create a dummy class which throws ImportError - when used. - - Args: - klass (str): name of the class. - dependency (str): name of the dependency. - message: extra message to print - Returns: - class: a class object - """ - err = "Cannot import '{}', therefore '{}' is not available.".format(dependency, klass) - if message: - err = err + " " + message - - class _DummyMetaClass(type): - # throw error on class attribute access - def __getattr__(_, __): # noqa: B902 - raise ImportError(err) - - class _Dummy(object, metaclass=_DummyMetaClass): - # throw error on constructor - def __init__(self, *args, **kwargs): - raise ImportError(err) - - return _Dummy - - -def create_dummy_func(func, dependency, message=""): - """ - When a dependency of a function is not available, create a dummy function which throws - ImportError when used. - - Args: - func (str): name of the function. - dependency (str or list[str]): name(s) of the dependency. - message: extra message to print - Returns: - function: a function object - """ - err = "Cannot import '{}', therefore '{}' is not available.".format(dependency, func) - if message: - err = err + " " + message - - if isinstance(dependency, (list, tuple)): - dependency = ",".join(dependency) - - def _dummy(*args, **kwargs): - raise ImportError(err) - - return _dummy diff --git a/groundingdino/models/GroundingDINO/ops/MultiScaleDeformableAttention.egg-info/PKG-INFO b/groundingdino/models/GroundingDINO/ops/MultiScaleDeformableAttention.egg-info/PKG-INFO deleted file mode 100644 index 18ccefb15d11821485ec63f66385d7cde57009e9..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/MultiScaleDeformableAttention.egg-info/PKG-INFO +++ /dev/null @@ -1,11 +0,0 @@ -Metadata-Version: 2.1 -Name: MultiScaleDeformableAttention -Version: 1.0 -Summary: PyTorch Wrapper for CUDA Functions of Multi-Scale Deformable Attention -Home-page: https://github.com/fundamentalvision/Deformable-DETR -Author: Weijie Su -License: UNKNOWN -Platform: UNKNOWN - -UNKNOWN - diff --git a/groundingdino/models/GroundingDINO/ops/MultiScaleDeformableAttention.egg-info/SOURCES.txt b/groundingdino/models/GroundingDINO/ops/MultiScaleDeformableAttention.egg-info/SOURCES.txt deleted file mode 100644 index 277b127056d8f3b5e8d4101cad608d535b0a467e..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/MultiScaleDeformableAttention.egg-info/SOURCES.txt +++ /dev/null @@ -1,12 +0,0 @@ -setup.py -/home/jamada/jupyterlab/projects/gdino-peft/vlm4eo/Open-GroundingDino/models/GroundingDINO/ops/src/vision.cpp -/home/jamada/jupyterlab/projects/gdino-peft/vlm4eo/Open-GroundingDino/models/GroundingDINO/ops/src/cpu/ms_deform_attn_cpu.cpp -/home/jamada/jupyterlab/projects/gdino-peft/vlm4eo/Open-GroundingDino/models/GroundingDINO/ops/src/cuda/ms_deform_attn_cuda.cu -MultiScaleDeformableAttention.egg-info/PKG-INFO -MultiScaleDeformableAttention.egg-info/SOURCES.txt -MultiScaleDeformableAttention.egg-info/dependency_links.txt -MultiScaleDeformableAttention.egg-info/top_level.txt -functions/__init__.py -functions/ms_deform_attn_func.py -modules/__init__.py -modules/ms_deform_attn.py \ No newline at end of file diff --git a/groundingdino/models/GroundingDINO/ops/MultiScaleDeformableAttention.egg-info/dependency_links.txt b/groundingdino/models/GroundingDINO/ops/MultiScaleDeformableAttention.egg-info/dependency_links.txt deleted file mode 100644 index 8b137891791fe96927ad78e64b0aad7bded08bdc..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/MultiScaleDeformableAttention.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/groundingdino/models/GroundingDINO/ops/MultiScaleDeformableAttention.egg-info/top_level.txt b/groundingdino/models/GroundingDINO/ops/MultiScaleDeformableAttention.egg-info/top_level.txt deleted file mode 100644 index 25d8f7790d14d04a74c6acec779aedb3688ef630..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/MultiScaleDeformableAttention.egg-info/top_level.txt +++ /dev/null @@ -1,3 +0,0 @@ -MultiScaleDeformableAttention -functions -modules diff --git a/groundingdino/models/GroundingDINO/ops/build/lib.linux-x86_64-3.10/functions/__init__.py b/groundingdino/models/GroundingDINO/ops/build/lib.linux-x86_64-3.10/functions/__init__.py deleted file mode 100644 index 8a2197bda3199aa32cafc5b9d396479609853dd2..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/build/lib.linux-x86_64-3.10/functions/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# ------------------------------------------------------------------------------------------------ -# Deformable DETR -# Copyright (c) 2020 SenseTime. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------------------------------ -# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 -# ------------------------------------------------------------------------------------------------ - -from .ms_deform_attn_func import MSDeformAttnFunction - diff --git a/groundingdino/models/GroundingDINO/ops/build/lib.linux-x86_64-3.10/functions/ms_deform_attn_func.py b/groundingdino/models/GroundingDINO/ops/build/lib.linux-x86_64-3.10/functions/ms_deform_attn_func.py deleted file mode 100644 index 8c5df8cf5d23aca963eec6c1133c180b37289607..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/build/lib.linux-x86_64-3.10/functions/ms_deform_attn_func.py +++ /dev/null @@ -1,61 +0,0 @@ -# ------------------------------------------------------------------------------------------------ -# Deformable DETR -# Copyright (c) 2020 SenseTime. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------------------------------ -# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 -# ------------------------------------------------------------------------------------------------ - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import division - -import torch -import torch.nn.functional as F -from torch.autograd import Function -from torch.autograd.function import once_differentiable - -import MultiScaleDeformableAttention as MSDA - - -class MSDeformAttnFunction(Function): - @staticmethod - def forward(ctx, value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, im2col_step): - ctx.im2col_step = im2col_step - output = MSDA.ms_deform_attn_forward( - value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, ctx.im2col_step) - ctx.save_for_backward(value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights) - return output - - @staticmethod - @once_differentiable - def backward(ctx, grad_output): - value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights = ctx.saved_tensors - grad_value, grad_sampling_loc, grad_attn_weight = \ - MSDA.ms_deform_attn_backward( - value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, grad_output, ctx.im2col_step) - - return grad_value, None, None, grad_sampling_loc, grad_attn_weight, None - - -def ms_deform_attn_core_pytorch(value, value_spatial_shapes, sampling_locations, attention_weights): - # for debug and test only, - # need to use cuda version instead - N_, S_, M_, D_ = value.shape - _, Lq_, M_, L_, P_, _ = sampling_locations.shape - value_list = value.split([H_ * W_ for H_, W_ in value_spatial_shapes], dim=1) - sampling_grids = 2 * sampling_locations - 1 - sampling_value_list = [] - for lid_, (H_, W_) in enumerate(value_spatial_shapes): - # N_, H_*W_, M_, D_ -> N_, H_*W_, M_*D_ -> N_, M_*D_, H_*W_ -> N_*M_, D_, H_, W_ - value_l_ = value_list[lid_].flatten(2).transpose(1, 2).reshape(N_*M_, D_, H_, W_) - # N_, Lq_, M_, P_, 2 -> N_, M_, Lq_, P_, 2 -> N_*M_, Lq_, P_, 2 - sampling_grid_l_ = sampling_grids[:, :, :, lid_].transpose(1, 2).flatten(0, 1) - # N_*M_, D_, Lq_, P_ - sampling_value_l_ = F.grid_sample(value_l_, sampling_grid_l_, - mode='bilinear', padding_mode='zeros', align_corners=False) - sampling_value_list.append(sampling_value_l_) - # (N_, Lq_, M_, L_, P_) -> (N_, M_, Lq_, L_, P_) -> (N_, M_, 1, Lq_, L_*P_) - attention_weights = attention_weights.transpose(1, 2).reshape(N_*M_, 1, Lq_, L_*P_) - output = (torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights).sum(-1).view(N_, M_*D_, Lq_) - return output.transpose(1, 2).contiguous() diff --git a/groundingdino/models/GroundingDINO/ops/build/lib.linux-x86_64-3.10/modules/__init__.py b/groundingdino/models/GroundingDINO/ops/build/lib.linux-x86_64-3.10/modules/__init__.py deleted file mode 100644 index f82cb1ad9d634a87b54ba6a71b58a230bcade5fe..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/build/lib.linux-x86_64-3.10/modules/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# ------------------------------------------------------------------------------------------------ -# Deformable DETR -# Copyright (c) 2020 SenseTime. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------------------------------ -# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 -# ------------------------------------------------------------------------------------------------ - -from .ms_deform_attn import MSDeformAttn diff --git a/groundingdino/models/GroundingDINO/ops/build/lib.linux-x86_64-3.10/modules/ms_deform_attn.py b/groundingdino/models/GroundingDINO/ops/build/lib.linux-x86_64-3.10/modules/ms_deform_attn.py deleted file mode 100644 index 0c22ac8b7611a7920115ce253bc9ef5510744dfd..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/build/lib.linux-x86_64-3.10/modules/ms_deform_attn.py +++ /dev/null @@ -1,126 +0,0 @@ -# ------------------------------------------------------------------------------------------------ -# Deformable DETR -# Copyright (c) 2020 SenseTime. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------------------------------ -# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 -# ------------------------------------------------------------------------------------------------ - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import division - -import warnings -import math - -import torch -from torch import nn -import torch.nn.functional as F -from torch.nn.init import xavier_uniform_, constant_ - -from ..functions import MSDeformAttnFunction - - -def _is_power_of_2(n): - if (not isinstance(n, int)) or (n < 0): - raise ValueError("invalid input for _is_power_of_2: {} (type: {})".format(n, type(n))) - return (n & (n-1) == 0) and n != 0 - - -class MSDeformAttn(nn.Module): - def __init__(self, d_model=256, n_levels=4, n_heads=8, n_points=4): - """ - Multi-Scale Deformable Attention Module - :param d_model hidden dimension - :param n_levels number of feature levels - :param n_heads number of attention heads - :param n_points number of sampling points per attention head per feature level - """ - super().__init__() - if d_model % n_heads != 0: - raise ValueError('d_model must be divisible by n_heads, but got {} and {}'.format(d_model, n_heads)) - _d_per_head = d_model // n_heads - # you'd better set _d_per_head to a power of 2 which is more efficient in our CUDA implementation - if not _is_power_of_2(_d_per_head): - warnings.warn("You'd better set d_model in MSDeformAttn to make the dimension of each attention head a power of 2 " - "which is more efficient in our CUDA implementation.") - - self.im2col_step = 64 - - self.d_model = d_model - self.n_levels = n_levels - self.n_heads = n_heads - self.n_points = n_points - - self.sampling_offsets = nn.Linear(d_model, n_heads * n_levels * n_points * 2) - self.attention_weights = nn.Linear(d_model, n_heads * n_levels * n_points) - self.value_proj = nn.Linear(d_model, d_model) - self.output_proj = nn.Linear(d_model, d_model) - - self._reset_parameters() - - def _reset_parameters(self): - constant_(self.sampling_offsets.weight.data, 0.) - thetas = torch.arange(self.n_heads, dtype=torch.float32) * (2.0 * math.pi / self.n_heads) - grid_init = torch.stack([thetas.cos(), thetas.sin()], -1) - grid_init = (grid_init / grid_init.abs().max(-1, keepdim=True)[0]).view(self.n_heads, 1, 1, 2).repeat(1, self.n_levels, self.n_points, 1) - for i in range(self.n_points): - grid_init[:, :, i, :] *= i + 1 - with torch.no_grad(): - self.sampling_offsets.bias = nn.Parameter(grid_init.view(-1)) - constant_(self.attention_weights.weight.data, 0.) - constant_(self.attention_weights.bias.data, 0.) - xavier_uniform_(self.value_proj.weight.data) - constant_(self.value_proj.bias.data, 0.) - xavier_uniform_(self.output_proj.weight.data) - constant_(self.output_proj.bias.data, 0.) - - def forward(self, query, reference_points, input_flatten, input_spatial_shapes, input_level_start_index, input_padding_mask=None): - """ - :param query (N, Length_{query}, C) - :param reference_points (N, Length_{query}, n_levels, 2), range in [0, 1], top-left (0,0), bottom-right (1, 1), including padding area - or (N, Length_{query}, n_levels, 4), add additional (w, h) to form reference boxes - :param input_flatten (N, \sum_{l=0}^{L-1} H_l \cdot W_l, C) - :param input_spatial_shapes (n_levels, 2), [(H_0, W_0), (H_1, W_1), ..., (H_{L-1}, W_{L-1})] - :param input_level_start_index (n_levels, ), [0, H_0*W_0, H_0*W_0+H_1*W_1, H_0*W_0+H_1*W_1+H_2*W_2, ..., H_0*W_0+H_1*W_1+...+H_{L-1}*W_{L-1}] - :param input_padding_mask (N, \sum_{l=0}^{L-1} H_l \cdot W_l), True for padding elements, False for non-padding elements - - :return output (N, Length_{query}, C) - """ - N, Len_q, _ = query.shape - N, Len_in, _ = input_flatten.shape - assert (input_spatial_shapes[:, 0] * input_spatial_shapes[:, 1]).sum() == Len_in - - value = self.value_proj(input_flatten) - if input_padding_mask is not None: - value = value.masked_fill(input_padding_mask[..., None], float(0)) - value = value.view(N, Len_in, self.n_heads, self.d_model // self.n_heads) - sampling_offsets = self.sampling_offsets(query).view(N, Len_q, self.n_heads, self.n_levels, self.n_points, 2) - attention_weights = self.attention_weights(query).view(N, Len_q, self.n_heads, self.n_levels * self.n_points) - attention_weights = F.softmax(attention_weights, -1).view(N, Len_q, self.n_heads, self.n_levels, self.n_points) - # N, Len_q, n_heads, n_levels, n_points, 2 - if reference_points.shape[-1] == 2: - offset_normalizer = torch.stack([input_spatial_shapes[..., 1], input_spatial_shapes[..., 0]], -1) - sampling_locations = reference_points[:, :, None, :, None, :] \ - + sampling_offsets / offset_normalizer[None, None, None, :, None, :] - elif reference_points.shape[-1] == 4: - sampling_locations = reference_points[:, :, None, :, None, :2] \ - + sampling_offsets / self.n_points * reference_points[:, :, None, :, None, 2:] * 0.5 - else: - raise ValueError( - 'Last dim of reference_points must be 2 or 4, but get {} instead.'.format(reference_points.shape[-1])) - - # for amp - if value.dtype == torch.float16: - # for mixed precision - output = MSDeformAttnFunction.apply( - value.to(torch.float32), input_spatial_shapes, input_level_start_index, sampling_locations.to(torch.float32), attention_weights, self.im2col_step) - output = output.to(torch.float16) - output = self.output_proj(output) - return output - - - output = MSDeformAttnFunction.apply( - value, input_spatial_shapes, input_level_start_index, sampling_locations, attention_weights, self.im2col_step) - output = self.output_proj(output) - return output diff --git a/groundingdino/models/GroundingDINO/ops/functions/__init__.py b/groundingdino/models/GroundingDINO/ops/functions/__init__.py deleted file mode 100644 index 8a2197bda3199aa32cafc5b9d396479609853dd2..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/functions/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# ------------------------------------------------------------------------------------------------ -# Deformable DETR -# Copyright (c) 2020 SenseTime. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------------------------------ -# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 -# ------------------------------------------------------------------------------------------------ - -from .ms_deform_attn_func import MSDeformAttnFunction - diff --git a/groundingdino/models/GroundingDINO/ops/functions/__pycache__/__init__.cpython-310.pyc b/groundingdino/models/GroundingDINO/ops/functions/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index fd566f034004c883e15b80a375266e8e9f14e9cc..0000000000000000000000000000000000000000 Binary files a/groundingdino/models/GroundingDINO/ops/functions/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/models/GroundingDINO/ops/functions/__pycache__/ms_deform_attn_func.cpython-310.pyc b/groundingdino/models/GroundingDINO/ops/functions/__pycache__/ms_deform_attn_func.cpython-310.pyc deleted file mode 100644 index cbec019b2dd0b2775594a623448a7e0c616ef3eb..0000000000000000000000000000000000000000 Binary files a/groundingdino/models/GroundingDINO/ops/functions/__pycache__/ms_deform_attn_func.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/models/GroundingDINO/ops/functions/ms_deform_attn_func.py b/groundingdino/models/GroundingDINO/ops/functions/ms_deform_attn_func.py deleted file mode 100644 index 8c5df8cf5d23aca963eec6c1133c180b37289607..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/functions/ms_deform_attn_func.py +++ /dev/null @@ -1,61 +0,0 @@ -# ------------------------------------------------------------------------------------------------ -# Deformable DETR -# Copyright (c) 2020 SenseTime. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------------------------------ -# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 -# ------------------------------------------------------------------------------------------------ - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import division - -import torch -import torch.nn.functional as F -from torch.autograd import Function -from torch.autograd.function import once_differentiable - -import MultiScaleDeformableAttention as MSDA - - -class MSDeformAttnFunction(Function): - @staticmethod - def forward(ctx, value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, im2col_step): - ctx.im2col_step = im2col_step - output = MSDA.ms_deform_attn_forward( - value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, ctx.im2col_step) - ctx.save_for_backward(value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights) - return output - - @staticmethod - @once_differentiable - def backward(ctx, grad_output): - value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights = ctx.saved_tensors - grad_value, grad_sampling_loc, grad_attn_weight = \ - MSDA.ms_deform_attn_backward( - value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, grad_output, ctx.im2col_step) - - return grad_value, None, None, grad_sampling_loc, grad_attn_weight, None - - -def ms_deform_attn_core_pytorch(value, value_spatial_shapes, sampling_locations, attention_weights): - # for debug and test only, - # need to use cuda version instead - N_, S_, M_, D_ = value.shape - _, Lq_, M_, L_, P_, _ = sampling_locations.shape - value_list = value.split([H_ * W_ for H_, W_ in value_spatial_shapes], dim=1) - sampling_grids = 2 * sampling_locations - 1 - sampling_value_list = [] - for lid_, (H_, W_) in enumerate(value_spatial_shapes): - # N_, H_*W_, M_, D_ -> N_, H_*W_, M_*D_ -> N_, M_*D_, H_*W_ -> N_*M_, D_, H_, W_ - value_l_ = value_list[lid_].flatten(2).transpose(1, 2).reshape(N_*M_, D_, H_, W_) - # N_, Lq_, M_, P_, 2 -> N_, M_, Lq_, P_, 2 -> N_*M_, Lq_, P_, 2 - sampling_grid_l_ = sampling_grids[:, :, :, lid_].transpose(1, 2).flatten(0, 1) - # N_*M_, D_, Lq_, P_ - sampling_value_l_ = F.grid_sample(value_l_, sampling_grid_l_, - mode='bilinear', padding_mode='zeros', align_corners=False) - sampling_value_list.append(sampling_value_l_) - # (N_, Lq_, M_, L_, P_) -> (N_, M_, Lq_, L_, P_) -> (N_, M_, 1, Lq_, L_*P_) - attention_weights = attention_weights.transpose(1, 2).reshape(N_*M_, 1, Lq_, L_*P_) - output = (torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights).sum(-1).view(N_, M_*D_, Lq_) - return output.transpose(1, 2).contiguous() diff --git a/groundingdino/models/GroundingDINO/ops/modules/__init__.py b/groundingdino/models/GroundingDINO/ops/modules/__init__.py deleted file mode 100644 index f82cb1ad9d634a87b54ba6a71b58a230bcade5fe..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/modules/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# ------------------------------------------------------------------------------------------------ -# Deformable DETR -# Copyright (c) 2020 SenseTime. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------------------------------ -# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 -# ------------------------------------------------------------------------------------------------ - -from .ms_deform_attn import MSDeformAttn diff --git a/groundingdino/models/GroundingDINO/ops/modules/ms_deform_attn.py b/groundingdino/models/GroundingDINO/ops/modules/ms_deform_attn.py deleted file mode 100644 index 0c22ac8b7611a7920115ce253bc9ef5510744dfd..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/modules/ms_deform_attn.py +++ /dev/null @@ -1,126 +0,0 @@ -# ------------------------------------------------------------------------------------------------ -# Deformable DETR -# Copyright (c) 2020 SenseTime. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------------------------------ -# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 -# ------------------------------------------------------------------------------------------------ - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import division - -import warnings -import math - -import torch -from torch import nn -import torch.nn.functional as F -from torch.nn.init import xavier_uniform_, constant_ - -from ..functions import MSDeformAttnFunction - - -def _is_power_of_2(n): - if (not isinstance(n, int)) or (n < 0): - raise ValueError("invalid input for _is_power_of_2: {} (type: {})".format(n, type(n))) - return (n & (n-1) == 0) and n != 0 - - -class MSDeformAttn(nn.Module): - def __init__(self, d_model=256, n_levels=4, n_heads=8, n_points=4): - """ - Multi-Scale Deformable Attention Module - :param d_model hidden dimension - :param n_levels number of feature levels - :param n_heads number of attention heads - :param n_points number of sampling points per attention head per feature level - """ - super().__init__() - if d_model % n_heads != 0: - raise ValueError('d_model must be divisible by n_heads, but got {} and {}'.format(d_model, n_heads)) - _d_per_head = d_model // n_heads - # you'd better set _d_per_head to a power of 2 which is more efficient in our CUDA implementation - if not _is_power_of_2(_d_per_head): - warnings.warn("You'd better set d_model in MSDeformAttn to make the dimension of each attention head a power of 2 " - "which is more efficient in our CUDA implementation.") - - self.im2col_step = 64 - - self.d_model = d_model - self.n_levels = n_levels - self.n_heads = n_heads - self.n_points = n_points - - self.sampling_offsets = nn.Linear(d_model, n_heads * n_levels * n_points * 2) - self.attention_weights = nn.Linear(d_model, n_heads * n_levels * n_points) - self.value_proj = nn.Linear(d_model, d_model) - self.output_proj = nn.Linear(d_model, d_model) - - self._reset_parameters() - - def _reset_parameters(self): - constant_(self.sampling_offsets.weight.data, 0.) - thetas = torch.arange(self.n_heads, dtype=torch.float32) * (2.0 * math.pi / self.n_heads) - grid_init = torch.stack([thetas.cos(), thetas.sin()], -1) - grid_init = (grid_init / grid_init.abs().max(-1, keepdim=True)[0]).view(self.n_heads, 1, 1, 2).repeat(1, self.n_levels, self.n_points, 1) - for i in range(self.n_points): - grid_init[:, :, i, :] *= i + 1 - with torch.no_grad(): - self.sampling_offsets.bias = nn.Parameter(grid_init.view(-1)) - constant_(self.attention_weights.weight.data, 0.) - constant_(self.attention_weights.bias.data, 0.) - xavier_uniform_(self.value_proj.weight.data) - constant_(self.value_proj.bias.data, 0.) - xavier_uniform_(self.output_proj.weight.data) - constant_(self.output_proj.bias.data, 0.) - - def forward(self, query, reference_points, input_flatten, input_spatial_shapes, input_level_start_index, input_padding_mask=None): - """ - :param query (N, Length_{query}, C) - :param reference_points (N, Length_{query}, n_levels, 2), range in [0, 1], top-left (0,0), bottom-right (1, 1), including padding area - or (N, Length_{query}, n_levels, 4), add additional (w, h) to form reference boxes - :param input_flatten (N, \sum_{l=0}^{L-1} H_l \cdot W_l, C) - :param input_spatial_shapes (n_levels, 2), [(H_0, W_0), (H_1, W_1), ..., (H_{L-1}, W_{L-1})] - :param input_level_start_index (n_levels, ), [0, H_0*W_0, H_0*W_0+H_1*W_1, H_0*W_0+H_1*W_1+H_2*W_2, ..., H_0*W_0+H_1*W_1+...+H_{L-1}*W_{L-1}] - :param input_padding_mask (N, \sum_{l=0}^{L-1} H_l \cdot W_l), True for padding elements, False for non-padding elements - - :return output (N, Length_{query}, C) - """ - N, Len_q, _ = query.shape - N, Len_in, _ = input_flatten.shape - assert (input_spatial_shapes[:, 0] * input_spatial_shapes[:, 1]).sum() == Len_in - - value = self.value_proj(input_flatten) - if input_padding_mask is not None: - value = value.masked_fill(input_padding_mask[..., None], float(0)) - value = value.view(N, Len_in, self.n_heads, self.d_model // self.n_heads) - sampling_offsets = self.sampling_offsets(query).view(N, Len_q, self.n_heads, self.n_levels, self.n_points, 2) - attention_weights = self.attention_weights(query).view(N, Len_q, self.n_heads, self.n_levels * self.n_points) - attention_weights = F.softmax(attention_weights, -1).view(N, Len_q, self.n_heads, self.n_levels, self.n_points) - # N, Len_q, n_heads, n_levels, n_points, 2 - if reference_points.shape[-1] == 2: - offset_normalizer = torch.stack([input_spatial_shapes[..., 1], input_spatial_shapes[..., 0]], -1) - sampling_locations = reference_points[:, :, None, :, None, :] \ - + sampling_offsets / offset_normalizer[None, None, None, :, None, :] - elif reference_points.shape[-1] == 4: - sampling_locations = reference_points[:, :, None, :, None, :2] \ - + sampling_offsets / self.n_points * reference_points[:, :, None, :, None, 2:] * 0.5 - else: - raise ValueError( - 'Last dim of reference_points must be 2 or 4, but get {} instead.'.format(reference_points.shape[-1])) - - # for amp - if value.dtype == torch.float16: - # for mixed precision - output = MSDeformAttnFunction.apply( - value.to(torch.float32), input_spatial_shapes, input_level_start_index, sampling_locations.to(torch.float32), attention_weights, self.im2col_step) - output = output.to(torch.float16) - output = self.output_proj(output) - return output - - - output = MSDeformAttnFunction.apply( - value, input_spatial_shapes, input_level_start_index, sampling_locations, attention_weights, self.im2col_step) - output = self.output_proj(output) - return output diff --git a/groundingdino/models/GroundingDINO/ops/setup.py b/groundingdino/models/GroundingDINO/ops/setup.py deleted file mode 100644 index c76d823200c0a3bbad37444958e8af8889fe653e..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/setup.py +++ /dev/null @@ -1,73 +0,0 @@ -# ------------------------------------------------------------------------------------------------ -# Deformable DETR -# Copyright (c) 2020 SenseTime. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------------------------------ -# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 -# ------------------------------------------------------------------------------------------------ - -import os -import glob - -import torch - -from torch.utils.cpp_extension import CUDA_HOME -from torch.utils.cpp_extension import CppExtension -from torch.utils.cpp_extension import CUDAExtension - -from setuptools import find_packages -from setuptools import setup - -requirements = ["torch", "torchvision"] - -def get_extensions(): - this_dir = os.path.dirname(os.path.abspath(__file__)) - extensions_dir = os.path.join(this_dir, "src") - - main_file = glob.glob(os.path.join(extensions_dir, "*.cpp")) - source_cpu = glob.glob(os.path.join(extensions_dir, "cpu", "*.cpp")) - source_cuda = glob.glob(os.path.join(extensions_dir, "cuda", "*.cu")) - - sources = main_file + source_cpu - extension = CppExtension - extra_compile_args = {"cxx": []} - define_macros = [] - - - - if torch.cuda.is_available() and CUDA_HOME is not None: - extension = CUDAExtension - sources += source_cuda - define_macros += [("WITH_CUDA", None)] - extra_compile_args["nvcc"] = [ - "-DCUDA_HAS_FP16=1", - "-D__CUDA_NO_HALF_OPERATORS__", - "-D__CUDA_NO_HALF_CONVERSIONS__", - "-D__CUDA_NO_HALF2_OPERATORS__", - ] - else: - raise NotImplementedError('Cuda is not availabel') - - sources = [os.path.join(extensions_dir, s) for s in sources] - include_dirs = [extensions_dir] - ext_modules = [ - extension( - "MultiScaleDeformableAttention", - sources, - include_dirs=include_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - ) - ] - return ext_modules - -setup( - name="MultiScaleDeformableAttention", - version="1.0", - author="Weijie Su", - url="https://github.com/fundamentalvision/Deformable-DETR", - description="PyTorch Wrapper for CUDA Functions of Multi-Scale Deformable Attention", - packages=find_packages(exclude=("configs", "tests",)), - ext_modules=get_extensions(), - cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension}, -) diff --git a/groundingdino/models/GroundingDINO/ops/src/cpu/ms_deform_attn_cpu.cpp b/groundingdino/models/GroundingDINO/ops/src/cpu/ms_deform_attn_cpu.cpp deleted file mode 100644 index e1bf854de1f3860d20b6fef5c1a17817c268e70a..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/src/cpu/ms_deform_attn_cpu.cpp +++ /dev/null @@ -1,41 +0,0 @@ -/*! -************************************************************************************************** -* Deformable DETR -* Copyright (c) 2020 SenseTime. All Rights Reserved. -* Licensed under the Apache License, Version 2.0 [see LICENSE for details] -************************************************************************************************** -* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 -************************************************************************************************** -*/ - -#include - -#include -#include - - -at::Tensor -ms_deform_attn_cpu_forward( - const at::Tensor &value, - const at::Tensor &spatial_shapes, - const at::Tensor &level_start_index, - const at::Tensor &sampling_loc, - const at::Tensor &attn_weight, - const int im2col_step) -{ - AT_ERROR("Not implement on cpu"); -} - -std::vector -ms_deform_attn_cpu_backward( - const at::Tensor &value, - const at::Tensor &spatial_shapes, - const at::Tensor &level_start_index, - const at::Tensor &sampling_loc, - const at::Tensor &attn_weight, - const at::Tensor &grad_output, - const int im2col_step) -{ - AT_ERROR("Not implement on cpu"); -} - diff --git a/groundingdino/models/GroundingDINO/ops/src/cpu/ms_deform_attn_cpu.h b/groundingdino/models/GroundingDINO/ops/src/cpu/ms_deform_attn_cpu.h deleted file mode 100644 index 81b7b58a3d9502bbb684dc84687a526dedf94cae..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/src/cpu/ms_deform_attn_cpu.h +++ /dev/null @@ -1,33 +0,0 @@ -/*! -************************************************************************************************** -* Deformable DETR -* Copyright (c) 2020 SenseTime. All Rights Reserved. -* Licensed under the Apache License, Version 2.0 [see LICENSE for details] -************************************************************************************************** -* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 -************************************************************************************************** -*/ - -#pragma once -#include - -at::Tensor -ms_deform_attn_cpu_forward( - const at::Tensor &value, - const at::Tensor &spatial_shapes, - const at::Tensor &level_start_index, - const at::Tensor &sampling_loc, - const at::Tensor &attn_weight, - const int im2col_step); - -std::vector -ms_deform_attn_cpu_backward( - const at::Tensor &value, - const at::Tensor &spatial_shapes, - const at::Tensor &level_start_index, - const at::Tensor &sampling_loc, - const at::Tensor &attn_weight, - const at::Tensor &grad_output, - const int im2col_step); - - diff --git a/groundingdino/models/GroundingDINO/ops/src/cuda/ms_deform_attn_cuda.cu b/groundingdino/models/GroundingDINO/ops/src/cuda/ms_deform_attn_cuda.cu deleted file mode 100644 index d6d583647cce987196d5ad1968a8a365a379e774..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/src/cuda/ms_deform_attn_cuda.cu +++ /dev/null @@ -1,153 +0,0 @@ -/*! -************************************************************************************************** -* Deformable DETR -* Copyright (c) 2020 SenseTime. All Rights Reserved. -* Licensed under the Apache License, Version 2.0 [see LICENSE for details] -************************************************************************************************** -* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 -************************************************************************************************** -*/ - -#include -#include "cuda/ms_deform_im2col_cuda.cuh" - -#include -#include -#include -#include - - -at::Tensor ms_deform_attn_cuda_forward( - const at::Tensor &value, - const at::Tensor &spatial_shapes, - const at::Tensor &level_start_index, - const at::Tensor &sampling_loc, - const at::Tensor &attn_weight, - const int im2col_step) -{ - AT_ASSERTM(value.is_contiguous(), "value tensor has to be contiguous"); - AT_ASSERTM(spatial_shapes.is_contiguous(), "spatial_shapes tensor has to be contiguous"); - AT_ASSERTM(level_start_index.is_contiguous(), "level_start_index tensor has to be contiguous"); - AT_ASSERTM(sampling_loc.is_contiguous(), "sampling_loc tensor has to be contiguous"); - AT_ASSERTM(attn_weight.is_contiguous(), "attn_weight tensor has to be contiguous"); - - AT_ASSERTM(value.type().is_cuda(), "value must be a CUDA tensor"); - AT_ASSERTM(spatial_shapes.type().is_cuda(), "spatial_shapes must be a CUDA tensor"); - AT_ASSERTM(level_start_index.type().is_cuda(), "level_start_index must be a CUDA tensor"); - AT_ASSERTM(sampling_loc.type().is_cuda(), "sampling_loc must be a CUDA tensor"); - AT_ASSERTM(attn_weight.type().is_cuda(), "attn_weight must be a CUDA tensor"); - - const int batch = value.size(0); - const int spatial_size = value.size(1); - const int num_heads = value.size(2); - const int channels = value.size(3); - - const int num_levels = spatial_shapes.size(0); - - const int num_query = sampling_loc.size(1); - const int num_point = sampling_loc.size(4); - - const int im2col_step_ = std::min(batch, im2col_step); - - AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); - - auto output = at::zeros({batch, num_query, num_heads, channels}, value.options()); - - const int batch_n = im2col_step_; - auto output_n = output.view({batch/im2col_step_, batch_n, num_query, num_heads, channels}); - auto per_value_size = spatial_size * num_heads * channels; - auto per_sample_loc_size = num_query * num_heads * num_levels * num_point * 2; - auto per_attn_weight_size = num_query * num_heads * num_levels * num_point; - for (int n = 0; n < batch/im2col_step_; ++n) - { - auto columns = output_n.select(0, n); - AT_DISPATCH_FLOATING_TYPES(value.type(), "ms_deform_attn_forward_cuda", ([&] { - ms_deformable_im2col_cuda(at::cuda::getCurrentCUDAStream(), - value.data() + n * im2col_step_ * per_value_size, - spatial_shapes.data(), - level_start_index.data(), - sampling_loc.data() + n * im2col_step_ * per_sample_loc_size, - attn_weight.data() + n * im2col_step_ * per_attn_weight_size, - batch_n, spatial_size, num_heads, channels, num_levels, num_query, num_point, - columns.data()); - - })); - } - - output = output.view({batch, num_query, num_heads*channels}); - - return output; -} - - -std::vector ms_deform_attn_cuda_backward( - const at::Tensor &value, - const at::Tensor &spatial_shapes, - const at::Tensor &level_start_index, - const at::Tensor &sampling_loc, - const at::Tensor &attn_weight, - const at::Tensor &grad_output, - const int im2col_step) -{ - - AT_ASSERTM(value.is_contiguous(), "value tensor has to be contiguous"); - AT_ASSERTM(spatial_shapes.is_contiguous(), "spatial_shapes tensor has to be contiguous"); - AT_ASSERTM(level_start_index.is_contiguous(), "level_start_index tensor has to be contiguous"); - AT_ASSERTM(sampling_loc.is_contiguous(), "sampling_loc tensor has to be contiguous"); - AT_ASSERTM(attn_weight.is_contiguous(), "attn_weight tensor has to be contiguous"); - AT_ASSERTM(grad_output.is_contiguous(), "grad_output tensor has to be contiguous"); - - AT_ASSERTM(value.type().is_cuda(), "value must be a CUDA tensor"); - AT_ASSERTM(spatial_shapes.type().is_cuda(), "spatial_shapes must be a CUDA tensor"); - AT_ASSERTM(level_start_index.type().is_cuda(), "level_start_index must be a CUDA tensor"); - AT_ASSERTM(sampling_loc.type().is_cuda(), "sampling_loc must be a CUDA tensor"); - AT_ASSERTM(attn_weight.type().is_cuda(), "attn_weight must be a CUDA tensor"); - AT_ASSERTM(grad_output.type().is_cuda(), "grad_output must be a CUDA tensor"); - - const int batch = value.size(0); - const int spatial_size = value.size(1); - const int num_heads = value.size(2); - const int channels = value.size(3); - - const int num_levels = spatial_shapes.size(0); - - const int num_query = sampling_loc.size(1); - const int num_point = sampling_loc.size(4); - - const int im2col_step_ = std::min(batch, im2col_step); - - AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); - - auto grad_value = at::zeros_like(value); - auto grad_sampling_loc = at::zeros_like(sampling_loc); - auto grad_attn_weight = at::zeros_like(attn_weight); - - const int batch_n = im2col_step_; - auto per_value_size = spatial_size * num_heads * channels; - auto per_sample_loc_size = num_query * num_heads * num_levels * num_point * 2; - auto per_attn_weight_size = num_query * num_heads * num_levels * num_point; - auto grad_output_n = grad_output.view({batch/im2col_step_, batch_n, num_query, num_heads, channels}); - - for (int n = 0; n < batch/im2col_step_; ++n) - { - auto grad_output_g = grad_output_n.select(0, n); - AT_DISPATCH_FLOATING_TYPES(value.type(), "ms_deform_attn_backward_cuda", ([&] { - ms_deformable_col2im_cuda(at::cuda::getCurrentCUDAStream(), - grad_output_g.data(), - value.data() + n * im2col_step_ * per_value_size, - spatial_shapes.data(), - level_start_index.data(), - sampling_loc.data() + n * im2col_step_ * per_sample_loc_size, - attn_weight.data() + n * im2col_step_ * per_attn_weight_size, - batch_n, spatial_size, num_heads, channels, num_levels, num_query, num_point, - grad_value.data() + n * im2col_step_ * per_value_size, - grad_sampling_loc.data() + n * im2col_step_ * per_sample_loc_size, - grad_attn_weight.data() + n * im2col_step_ * per_attn_weight_size); - - })); - } - - return { - grad_value, grad_sampling_loc, grad_attn_weight - }; -} \ No newline at end of file diff --git a/groundingdino/models/GroundingDINO/ops/src/cuda/ms_deform_attn_cuda.h b/groundingdino/models/GroundingDINO/ops/src/cuda/ms_deform_attn_cuda.h deleted file mode 100644 index c7ae53f99c820ce6193b608ad344550348a0b42c..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/src/cuda/ms_deform_attn_cuda.h +++ /dev/null @@ -1,30 +0,0 @@ -/*! -************************************************************************************************** -* Deformable DETR -* Copyright (c) 2020 SenseTime. All Rights Reserved. -* Licensed under the Apache License, Version 2.0 [see LICENSE for details] -************************************************************************************************** -* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 -************************************************************************************************** -*/ - -#pragma once -#include - -at::Tensor ms_deform_attn_cuda_forward( - const at::Tensor &value, - const at::Tensor &spatial_shapes, - const at::Tensor &level_start_index, - const at::Tensor &sampling_loc, - const at::Tensor &attn_weight, - const int im2col_step); - -std::vector ms_deform_attn_cuda_backward( - const at::Tensor &value, - const at::Tensor &spatial_shapes, - const at::Tensor &level_start_index, - const at::Tensor &sampling_loc, - const at::Tensor &attn_weight, - const at::Tensor &grad_output, - const int im2col_step); - diff --git a/groundingdino/models/GroundingDINO/ops/src/cuda/ms_deform_im2col_cuda.cuh b/groundingdino/models/GroundingDINO/ops/src/cuda/ms_deform_im2col_cuda.cuh deleted file mode 100644 index 6bc2acb7aea0eab2e9e91e769a16861e1652c284..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/src/cuda/ms_deform_im2col_cuda.cuh +++ /dev/null @@ -1,1327 +0,0 @@ -/*! -************************************************************************** -* Deformable DETR -* Copyright (c) 2020 SenseTime. All Rights Reserved. -* Licensed under the Apache License, Version 2.0 [see LICENSE for details] -************************************************************************** -* Modified from DCN (https://github.com/msracver/Deformable-ConvNets) -* Copyright (c) 2018 Microsoft -************************************************************************** -*/ - -#include -#include -#include - -#include -#include - -#include - -#define CUDA_KERNEL_LOOP(i, n) \ - for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ - i < (n); \ - i += blockDim.x * gridDim.x) - -const int CUDA_NUM_THREADS = 1024; -inline int GET_BLOCKS(const int N, const int num_threads) -{ - return (N + num_threads - 1) / num_threads; -} - - -template -__device__ scalar_t ms_deform_attn_im2col_bilinear(const scalar_t* &bottom_data, - const int &height, const int &width, const int &nheads, const int &channels, - const scalar_t &h, const scalar_t &w, const int &m, const int &c) -{ - const int h_low = floor(h); - const int w_low = floor(w); - const int h_high = h_low + 1; - const int w_high = w_low + 1; - - const scalar_t lh = h - h_low; - const scalar_t lw = w - w_low; - const scalar_t hh = 1 - lh, hw = 1 - lw; - - const int w_stride = nheads * channels; - const int h_stride = width * w_stride; - const int h_low_ptr_offset = h_low * h_stride; - const int h_high_ptr_offset = h_low_ptr_offset + h_stride; - const int w_low_ptr_offset = w_low * w_stride; - const int w_high_ptr_offset = w_low_ptr_offset + w_stride; - const int base_ptr = m * channels + c; - - scalar_t v1 = 0; - if (h_low >= 0 && w_low >= 0) - { - const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; - v1 = bottom_data[ptr1]; - } - scalar_t v2 = 0; - if (h_low >= 0 && w_high <= width - 1) - { - const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; - v2 = bottom_data[ptr2]; - } - scalar_t v3 = 0; - if (h_high <= height - 1 && w_low >= 0) - { - const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; - v3 = bottom_data[ptr3]; - } - scalar_t v4 = 0; - if (h_high <= height - 1 && w_high <= width - 1) - { - const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; - v4 = bottom_data[ptr4]; - } - - const scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; - - const scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); - return val; -} - - -template -__device__ void ms_deform_attn_col2im_bilinear(const scalar_t* &bottom_data, - const int &height, const int &width, const int &nheads, const int &channels, - const scalar_t &h, const scalar_t &w, const int &m, const int &c, - const scalar_t &top_grad, - const scalar_t &attn_weight, - scalar_t* &grad_value, - scalar_t* grad_sampling_loc, - scalar_t* grad_attn_weight) -{ - const int h_low = floor(h); - const int w_low = floor(w); - const int h_high = h_low + 1; - const int w_high = w_low + 1; - - const scalar_t lh = h - h_low; - const scalar_t lw = w - w_low; - const scalar_t hh = 1 - lh, hw = 1 - lw; - - const int w_stride = nheads * channels; - const int h_stride = width * w_stride; - const int h_low_ptr_offset = h_low * h_stride; - const int h_high_ptr_offset = h_low_ptr_offset + h_stride; - const int w_low_ptr_offset = w_low * w_stride; - const int w_high_ptr_offset = w_low_ptr_offset + w_stride; - const int base_ptr = m * channels + c; - - const scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; - const scalar_t top_grad_value = top_grad * attn_weight; - scalar_t grad_h_weight = 0, grad_w_weight = 0; - - scalar_t v1 = 0; - if (h_low >= 0 && w_low >= 0) - { - const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; - v1 = bottom_data[ptr1]; - grad_h_weight -= hw * v1; - grad_w_weight -= hh * v1; - atomicAdd(grad_value+ptr1, w1*top_grad_value); - } - scalar_t v2 = 0; - if (h_low >= 0 && w_high <= width - 1) - { - const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; - v2 = bottom_data[ptr2]; - grad_h_weight -= lw * v2; - grad_w_weight += hh * v2; - atomicAdd(grad_value+ptr2, w2*top_grad_value); - } - scalar_t v3 = 0; - if (h_high <= height - 1 && w_low >= 0) - { - const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; - v3 = bottom_data[ptr3]; - grad_h_weight += hw * v3; - grad_w_weight -= lh * v3; - atomicAdd(grad_value+ptr3, w3*top_grad_value); - } - scalar_t v4 = 0; - if (h_high <= height - 1 && w_high <= width - 1) - { - const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; - v4 = bottom_data[ptr4]; - grad_h_weight += lw * v4; - grad_w_weight += lh * v4; - atomicAdd(grad_value+ptr4, w4*top_grad_value); - } - - const scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); - *grad_attn_weight = top_grad * val; - *grad_sampling_loc = width * grad_w_weight * top_grad_value; - *(grad_sampling_loc + 1) = height * grad_h_weight * top_grad_value; -} - - -template -__device__ void ms_deform_attn_col2im_bilinear_gm(const scalar_t* &bottom_data, - const int &height, const int &width, const int &nheads, const int &channels, - const scalar_t &h, const scalar_t &w, const int &m, const int &c, - const scalar_t &top_grad, - const scalar_t &attn_weight, - scalar_t* &grad_value, - scalar_t* grad_sampling_loc, - scalar_t* grad_attn_weight) -{ - const int h_low = floor(h); - const int w_low = floor(w); - const int h_high = h_low + 1; - const int w_high = w_low + 1; - - const scalar_t lh = h - h_low; - const scalar_t lw = w - w_low; - const scalar_t hh = 1 - lh, hw = 1 - lw; - - const int w_stride = nheads * channels; - const int h_stride = width * w_stride; - const int h_low_ptr_offset = h_low * h_stride; - const int h_high_ptr_offset = h_low_ptr_offset + h_stride; - const int w_low_ptr_offset = w_low * w_stride; - const int w_high_ptr_offset = w_low_ptr_offset + w_stride; - const int base_ptr = m * channels + c; - - const scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; - const scalar_t top_grad_value = top_grad * attn_weight; - scalar_t grad_h_weight = 0, grad_w_weight = 0; - - scalar_t v1 = 0; - if (h_low >= 0 && w_low >= 0) - { - const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; - v1 = bottom_data[ptr1]; - grad_h_weight -= hw * v1; - grad_w_weight -= hh * v1; - atomicAdd(grad_value+ptr1, w1*top_grad_value); - } - scalar_t v2 = 0; - if (h_low >= 0 && w_high <= width - 1) - { - const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; - v2 = bottom_data[ptr2]; - grad_h_weight -= lw * v2; - grad_w_weight += hh * v2; - atomicAdd(grad_value+ptr2, w2*top_grad_value); - } - scalar_t v3 = 0; - if (h_high <= height - 1 && w_low >= 0) - { - const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; - v3 = bottom_data[ptr3]; - grad_h_weight += hw * v3; - grad_w_weight -= lh * v3; - atomicAdd(grad_value+ptr3, w3*top_grad_value); - } - scalar_t v4 = 0; - if (h_high <= height - 1 && w_high <= width - 1) - { - const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; - v4 = bottom_data[ptr4]; - grad_h_weight += lw * v4; - grad_w_weight += lh * v4; - atomicAdd(grad_value+ptr4, w4*top_grad_value); - } - - const scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); - atomicAdd(grad_attn_weight, top_grad * val); - atomicAdd(grad_sampling_loc, width * grad_w_weight * top_grad_value); - atomicAdd(grad_sampling_loc + 1, height * grad_h_weight * top_grad_value); -} - - -template -__global__ void ms_deformable_im2col_gpu_kernel(const int n, - const scalar_t *data_value, - const int64_t *data_spatial_shapes, - const int64_t *data_level_start_index, - const scalar_t *data_sampling_loc, - const scalar_t *data_attn_weight, - const int batch_size, - const int spatial_size, - const int num_heads, - const int channels, - const int num_levels, - const int num_query, - const int num_point, - scalar_t *data_col) -{ - CUDA_KERNEL_LOOP(index, n) - { - int _temp = index; - const int c_col = _temp % channels; - _temp /= channels; - const int sampling_index = _temp; - const int m_col = _temp % num_heads; - _temp /= num_heads; - const int q_col = _temp % num_query; - _temp /= num_query; - const int b_col = _temp; - - scalar_t *data_col_ptr = data_col + index; - int data_weight_ptr = sampling_index * num_levels * num_point; - int data_loc_w_ptr = data_weight_ptr << 1; - const int qid_stride = num_heads * channels; - const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; - scalar_t col = 0; - - for (int l_col=0; l_col < num_levels; ++l_col) - { - const int level_start_id = data_level_start_index[l_col]; - const int spatial_h_ptr = l_col << 1; - const int spatial_h = data_spatial_shapes[spatial_h_ptr]; - const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; - const scalar_t *data_value_ptr = data_value + (data_value_ptr_init_offset + level_start_id * qid_stride); - for (int p_col=0; p_col < num_point; ++p_col) - { - const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; - const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; - const scalar_t weight = data_attn_weight[data_weight_ptr]; - - const scalar_t h_im = loc_h * spatial_h - 0.5; - const scalar_t w_im = loc_w * spatial_w - 0.5; - - if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) - { - col += ms_deform_attn_im2col_bilinear(data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col) * weight; - } - - data_weight_ptr += 1; - data_loc_w_ptr += 2; - } - } - *data_col_ptr = col; - } -} - -template -__global__ void ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1(const int n, - const scalar_t *grad_col, - const scalar_t *data_value, - const int64_t *data_spatial_shapes, - const int64_t *data_level_start_index, - const scalar_t *data_sampling_loc, - const scalar_t *data_attn_weight, - const int batch_size, - const int spatial_size, - const int num_heads, - const int channels, - const int num_levels, - const int num_query, - const int num_point, - scalar_t *grad_value, - scalar_t *grad_sampling_loc, - scalar_t *grad_attn_weight) -{ - CUDA_KERNEL_LOOP(index, n) - { - __shared__ scalar_t cache_grad_sampling_loc[blockSize * 2]; - __shared__ scalar_t cache_grad_attn_weight[blockSize]; - unsigned int tid = threadIdx.x; - int _temp = index; - const int c_col = _temp % channels; - _temp /= channels; - const int sampling_index = _temp; - const int m_col = _temp % num_heads; - _temp /= num_heads; - const int q_col = _temp % num_query; - _temp /= num_query; - const int b_col = _temp; - - const scalar_t top_grad = grad_col[index]; - - int data_weight_ptr = sampling_index * num_levels * num_point; - int data_loc_w_ptr = data_weight_ptr << 1; - const int grad_sampling_ptr = data_weight_ptr; - grad_sampling_loc += grad_sampling_ptr << 1; - grad_attn_weight += grad_sampling_ptr; - const int grad_weight_stride = 1; - const int grad_loc_stride = 2; - const int qid_stride = num_heads * channels; - const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; - - for (int l_col=0; l_col < num_levels; ++l_col) - { - const int level_start_id = data_level_start_index[l_col]; - const int spatial_h_ptr = l_col << 1; - const int spatial_h = data_spatial_shapes[spatial_h_ptr]; - const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; - const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; - const scalar_t *data_value_ptr = data_value + value_ptr_offset; - scalar_t *grad_value_ptr = grad_value + value_ptr_offset; - - for (int p_col=0; p_col < num_point; ++p_col) - { - const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; - const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; - const scalar_t weight = data_attn_weight[data_weight_ptr]; - - const scalar_t h_im = loc_h * spatial_h - 0.5; - const scalar_t w_im = loc_w * spatial_w - 0.5; - *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; - *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; - *(cache_grad_attn_weight+threadIdx.x)=0; - if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) - { - ms_deform_attn_col2im_bilinear( - data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, - top_grad, weight, grad_value_ptr, - cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); - } - - __syncthreads(); - if (tid == 0) - { - scalar_t _grad_w=cache_grad_sampling_loc[0], _grad_h=cache_grad_sampling_loc[1], _grad_a=cache_grad_attn_weight[0]; - int sid=2; - for (unsigned int tid = 1; tid < blockSize; ++tid) - { - _grad_w += cache_grad_sampling_loc[sid]; - _grad_h += cache_grad_sampling_loc[sid + 1]; - _grad_a += cache_grad_attn_weight[tid]; - sid += 2; - } - - - *grad_sampling_loc = _grad_w; - *(grad_sampling_loc + 1) = _grad_h; - *grad_attn_weight = _grad_a; - } - __syncthreads(); - - data_weight_ptr += 1; - data_loc_w_ptr += 2; - grad_attn_weight += grad_weight_stride; - grad_sampling_loc += grad_loc_stride; - } - } - } -} - - -template -__global__ void ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2(const int n, - const scalar_t *grad_col, - const scalar_t *data_value, - const int64_t *data_spatial_shapes, - const int64_t *data_level_start_index, - const scalar_t *data_sampling_loc, - const scalar_t *data_attn_weight, - const int batch_size, - const int spatial_size, - const int num_heads, - const int channels, - const int num_levels, - const int num_query, - const int num_point, - scalar_t *grad_value, - scalar_t *grad_sampling_loc, - scalar_t *grad_attn_weight) -{ - CUDA_KERNEL_LOOP(index, n) - { - __shared__ scalar_t cache_grad_sampling_loc[blockSize * 2]; - __shared__ scalar_t cache_grad_attn_weight[blockSize]; - unsigned int tid = threadIdx.x; - int _temp = index; - const int c_col = _temp % channels; - _temp /= channels; - const int sampling_index = _temp; - const int m_col = _temp % num_heads; - _temp /= num_heads; - const int q_col = _temp % num_query; - _temp /= num_query; - const int b_col = _temp; - - const scalar_t top_grad = grad_col[index]; - - int data_weight_ptr = sampling_index * num_levels * num_point; - int data_loc_w_ptr = data_weight_ptr << 1; - const int grad_sampling_ptr = data_weight_ptr; - grad_sampling_loc += grad_sampling_ptr << 1; - grad_attn_weight += grad_sampling_ptr; - const int grad_weight_stride = 1; - const int grad_loc_stride = 2; - const int qid_stride = num_heads * channels; - const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; - - for (int l_col=0; l_col < num_levels; ++l_col) - { - const int level_start_id = data_level_start_index[l_col]; - const int spatial_h_ptr = l_col << 1; - const int spatial_h = data_spatial_shapes[spatial_h_ptr]; - const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; - const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; - const scalar_t *data_value_ptr = data_value + value_ptr_offset; - scalar_t *grad_value_ptr = grad_value + value_ptr_offset; - - for (int p_col=0; p_col < num_point; ++p_col) - { - const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; - const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; - const scalar_t weight = data_attn_weight[data_weight_ptr]; - - const scalar_t h_im = loc_h * spatial_h - 0.5; - const scalar_t w_im = loc_w * spatial_w - 0.5; - *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; - *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; - *(cache_grad_attn_weight+threadIdx.x)=0; - if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) - { - ms_deform_attn_col2im_bilinear( - data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, - top_grad, weight, grad_value_ptr, - cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); - } - - __syncthreads(); - - for (unsigned int s=blockSize/2; s>0; s>>=1) - { - if (tid < s) { - const unsigned int xid1 = tid << 1; - const unsigned int xid2 = (tid + s) << 1; - cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + s]; - cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2]; - cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1]; - } - __syncthreads(); - } - - if (tid == 0) - { - *grad_sampling_loc = cache_grad_sampling_loc[0]; - *(grad_sampling_loc + 1) = cache_grad_sampling_loc[1]; - *grad_attn_weight = cache_grad_attn_weight[0]; - } - __syncthreads(); - - data_weight_ptr += 1; - data_loc_w_ptr += 2; - grad_attn_weight += grad_weight_stride; - grad_sampling_loc += grad_loc_stride; - } - } - } -} - - -template -__global__ void ms_deformable_col2im_gpu_kernel_shm_reduce_v1(const int n, - const scalar_t *grad_col, - const scalar_t *data_value, - const int64_t *data_spatial_shapes, - const int64_t *data_level_start_index, - const scalar_t *data_sampling_loc, - const scalar_t *data_attn_weight, - const int batch_size, - const int spatial_size, - const int num_heads, - const int channels, - const int num_levels, - const int num_query, - const int num_point, - scalar_t *grad_value, - scalar_t *grad_sampling_loc, - scalar_t *grad_attn_weight) -{ - CUDA_KERNEL_LOOP(index, n) - { - extern __shared__ int _s[]; - scalar_t* cache_grad_sampling_loc = (scalar_t*)_s; - scalar_t* cache_grad_attn_weight = cache_grad_sampling_loc + 2 * blockDim.x; - unsigned int tid = threadIdx.x; - int _temp = index; - const int c_col = _temp % channels; - _temp /= channels; - const int sampling_index = _temp; - const int m_col = _temp % num_heads; - _temp /= num_heads; - const int q_col = _temp % num_query; - _temp /= num_query; - const int b_col = _temp; - - const scalar_t top_grad = grad_col[index]; - - int data_weight_ptr = sampling_index * num_levels * num_point; - int data_loc_w_ptr = data_weight_ptr << 1; - const int grad_sampling_ptr = data_weight_ptr; - grad_sampling_loc += grad_sampling_ptr << 1; - grad_attn_weight += grad_sampling_ptr; - const int grad_weight_stride = 1; - const int grad_loc_stride = 2; - const int qid_stride = num_heads * channels; - const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; - - for (int l_col=0; l_col < num_levels; ++l_col) - { - const int level_start_id = data_level_start_index[l_col]; - const int spatial_h_ptr = l_col << 1; - const int spatial_h = data_spatial_shapes[spatial_h_ptr]; - const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; - const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; - const scalar_t *data_value_ptr = data_value + value_ptr_offset; - scalar_t *grad_value_ptr = grad_value + value_ptr_offset; - - for (int p_col=0; p_col < num_point; ++p_col) - { - const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; - const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; - const scalar_t weight = data_attn_weight[data_weight_ptr]; - - const scalar_t h_im = loc_h * spatial_h - 0.5; - const scalar_t w_im = loc_w * spatial_w - 0.5; - *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; - *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; - *(cache_grad_attn_weight+threadIdx.x)=0; - if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) - { - ms_deform_attn_col2im_bilinear( - data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, - top_grad, weight, grad_value_ptr, - cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); - } - - __syncthreads(); - if (tid == 0) - { - scalar_t _grad_w=cache_grad_sampling_loc[0], _grad_h=cache_grad_sampling_loc[1], _grad_a=cache_grad_attn_weight[0]; - int sid=2; - for (unsigned int tid = 1; tid < blockDim.x; ++tid) - { - _grad_w += cache_grad_sampling_loc[sid]; - _grad_h += cache_grad_sampling_loc[sid + 1]; - _grad_a += cache_grad_attn_weight[tid]; - sid += 2; - } - - - *grad_sampling_loc = _grad_w; - *(grad_sampling_loc + 1) = _grad_h; - *grad_attn_weight = _grad_a; - } - __syncthreads(); - - data_weight_ptr += 1; - data_loc_w_ptr += 2; - grad_attn_weight += grad_weight_stride; - grad_sampling_loc += grad_loc_stride; - } - } - } -} - -template -__global__ void ms_deformable_col2im_gpu_kernel_shm_reduce_v2(const int n, - const scalar_t *grad_col, - const scalar_t *data_value, - const int64_t *data_spatial_shapes, - const int64_t *data_level_start_index, - const scalar_t *data_sampling_loc, - const scalar_t *data_attn_weight, - const int batch_size, - const int spatial_size, - const int num_heads, - const int channels, - const int num_levels, - const int num_query, - const int num_point, - scalar_t *grad_value, - scalar_t *grad_sampling_loc, - scalar_t *grad_attn_weight) -{ - CUDA_KERNEL_LOOP(index, n) - { - extern __shared__ int _s[]; - scalar_t* cache_grad_sampling_loc = (scalar_t*)_s; - scalar_t* cache_grad_attn_weight = cache_grad_sampling_loc + 2 * blockDim.x; - unsigned int tid = threadIdx.x; - int _temp = index; - const int c_col = _temp % channels; - _temp /= channels; - const int sampling_index = _temp; - const int m_col = _temp % num_heads; - _temp /= num_heads; - const int q_col = _temp % num_query; - _temp /= num_query; - const int b_col = _temp; - - const scalar_t top_grad = grad_col[index]; - - int data_weight_ptr = sampling_index * num_levels * num_point; - int data_loc_w_ptr = data_weight_ptr << 1; - const int grad_sampling_ptr = data_weight_ptr; - grad_sampling_loc += grad_sampling_ptr << 1; - grad_attn_weight += grad_sampling_ptr; - const int grad_weight_stride = 1; - const int grad_loc_stride = 2; - const int qid_stride = num_heads * channels; - const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; - - for (int l_col=0; l_col < num_levels; ++l_col) - { - const int level_start_id = data_level_start_index[l_col]; - const int spatial_h_ptr = l_col << 1; - const int spatial_h = data_spatial_shapes[spatial_h_ptr]; - const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; - const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; - const scalar_t *data_value_ptr = data_value + value_ptr_offset; - scalar_t *grad_value_ptr = grad_value + value_ptr_offset; - - for (int p_col=0; p_col < num_point; ++p_col) - { - const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; - const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; - const scalar_t weight = data_attn_weight[data_weight_ptr]; - - const scalar_t h_im = loc_h * spatial_h - 0.5; - const scalar_t w_im = loc_w * spatial_w - 0.5; - *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; - *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; - *(cache_grad_attn_weight+threadIdx.x)=0; - if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) - { - ms_deform_attn_col2im_bilinear( - data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, - top_grad, weight, grad_value_ptr, - cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); - } - - __syncthreads(); - - for (unsigned int s=blockDim.x/2, spre=blockDim.x; s>0; s>>=1, spre>>=1) - { - if (tid < s) { - const unsigned int xid1 = tid << 1; - const unsigned int xid2 = (tid + s) << 1; - cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + s]; - cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2]; - cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1]; - if (tid + (s << 1) < spre) - { - cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + (s << 1)]; - cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2 + (s << 1)]; - cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1 + (s << 1)]; - } - } - __syncthreads(); - } - - if (tid == 0) - { - *grad_sampling_loc = cache_grad_sampling_loc[0]; - *(grad_sampling_loc + 1) = cache_grad_sampling_loc[1]; - *grad_attn_weight = cache_grad_attn_weight[0]; - } - __syncthreads(); - - data_weight_ptr += 1; - data_loc_w_ptr += 2; - grad_attn_weight += grad_weight_stride; - grad_sampling_loc += grad_loc_stride; - } - } - } -} - -template -__global__ void ms_deformable_col2im_gpu_kernel_shm_reduce_v2_multi_blocks(const int n, - const scalar_t *grad_col, - const scalar_t *data_value, - const int64_t *data_spatial_shapes, - const int64_t *data_level_start_index, - const scalar_t *data_sampling_loc, - const scalar_t *data_attn_weight, - const int batch_size, - const int spatial_size, - const int num_heads, - const int channels, - const int num_levels, - const int num_query, - const int num_point, - scalar_t *grad_value, - scalar_t *grad_sampling_loc, - scalar_t *grad_attn_weight) -{ - CUDA_KERNEL_LOOP(index, n) - { - extern __shared__ int _s[]; - scalar_t* cache_grad_sampling_loc = (scalar_t*)_s; - scalar_t* cache_grad_attn_weight = cache_grad_sampling_loc + 2 * blockDim.x; - unsigned int tid = threadIdx.x; - int _temp = index; - const int c_col = _temp % channels; - _temp /= channels; - const int sampling_index = _temp; - const int m_col = _temp % num_heads; - _temp /= num_heads; - const int q_col = _temp % num_query; - _temp /= num_query; - const int b_col = _temp; - - const scalar_t top_grad = grad_col[index]; - - int data_weight_ptr = sampling_index * num_levels * num_point; - int data_loc_w_ptr = data_weight_ptr << 1; - const int grad_sampling_ptr = data_weight_ptr; - grad_sampling_loc += grad_sampling_ptr << 1; - grad_attn_weight += grad_sampling_ptr; - const int grad_weight_stride = 1; - const int grad_loc_stride = 2; - const int qid_stride = num_heads * channels; - const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; - - for (int l_col=0; l_col < num_levels; ++l_col) - { - const int level_start_id = data_level_start_index[l_col]; - const int spatial_h_ptr = l_col << 1; - const int spatial_h = data_spatial_shapes[spatial_h_ptr]; - const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; - const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; - const scalar_t *data_value_ptr = data_value + value_ptr_offset; - scalar_t *grad_value_ptr = grad_value + value_ptr_offset; - - for (int p_col=0; p_col < num_point; ++p_col) - { - const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; - const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; - const scalar_t weight = data_attn_weight[data_weight_ptr]; - - const scalar_t h_im = loc_h * spatial_h - 0.5; - const scalar_t w_im = loc_w * spatial_w - 0.5; - *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; - *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; - *(cache_grad_attn_weight+threadIdx.x)=0; - if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) - { - ms_deform_attn_col2im_bilinear( - data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, - top_grad, weight, grad_value_ptr, - cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); - } - - __syncthreads(); - - for (unsigned int s=blockDim.x/2, spre=blockDim.x; s>0; s>>=1, spre>>=1) - { - if (tid < s) { - const unsigned int xid1 = tid << 1; - const unsigned int xid2 = (tid + s) << 1; - cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + s]; - cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2]; - cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1]; - if (tid + (s << 1) < spre) - { - cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + (s << 1)]; - cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2 + (s << 1)]; - cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1 + (s << 1)]; - } - } - __syncthreads(); - } - - if (tid == 0) - { - atomicAdd(grad_sampling_loc, cache_grad_sampling_loc[0]); - atomicAdd(grad_sampling_loc + 1, cache_grad_sampling_loc[1]); - atomicAdd(grad_attn_weight, cache_grad_attn_weight[0]); - } - __syncthreads(); - - data_weight_ptr += 1; - data_loc_w_ptr += 2; - grad_attn_weight += grad_weight_stride; - grad_sampling_loc += grad_loc_stride; - } - } - } -} - - -template -__global__ void ms_deformable_col2im_gpu_kernel_gm(const int n, - const scalar_t *grad_col, - const scalar_t *data_value, - const int64_t *data_spatial_shapes, - const int64_t *data_level_start_index, - const scalar_t *data_sampling_loc, - const scalar_t *data_attn_weight, - const int batch_size, - const int spatial_size, - const int num_heads, - const int channels, - const int num_levels, - const int num_query, - const int num_point, - scalar_t *grad_value, - scalar_t *grad_sampling_loc, - scalar_t *grad_attn_weight) -{ - CUDA_KERNEL_LOOP(index, n) - { - int _temp = index; - const int c_col = _temp % channels; - _temp /= channels; - const int sampling_index = _temp; - const int m_col = _temp % num_heads; - _temp /= num_heads; - const int q_col = _temp % num_query; - _temp /= num_query; - const int b_col = _temp; - - const scalar_t top_grad = grad_col[index]; - - int data_weight_ptr = sampling_index * num_levels * num_point; - int data_loc_w_ptr = data_weight_ptr << 1; - const int grad_sampling_ptr = data_weight_ptr; - grad_sampling_loc += grad_sampling_ptr << 1; - grad_attn_weight += grad_sampling_ptr; - const int grad_weight_stride = 1; - const int grad_loc_stride = 2; - const int qid_stride = num_heads * channels; - const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; - - for (int l_col=0; l_col < num_levels; ++l_col) - { - const int level_start_id = data_level_start_index[l_col]; - const int spatial_h_ptr = l_col << 1; - const int spatial_h = data_spatial_shapes[spatial_h_ptr]; - const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; - const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; - const scalar_t *data_value_ptr = data_value + value_ptr_offset; - scalar_t *grad_value_ptr = grad_value + value_ptr_offset; - - for (int p_col=0; p_col < num_point; ++p_col) - { - const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; - const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; - const scalar_t weight = data_attn_weight[data_weight_ptr]; - - const scalar_t h_im = loc_h * spatial_h - 0.5; - const scalar_t w_im = loc_w * spatial_w - 0.5; - if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) - { - ms_deform_attn_col2im_bilinear_gm( - data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, - top_grad, weight, grad_value_ptr, - grad_sampling_loc, grad_attn_weight); - } - data_weight_ptr += 1; - data_loc_w_ptr += 2; - grad_attn_weight += grad_weight_stride; - grad_sampling_loc += grad_loc_stride; - } - } - } -} - - -template -void ms_deformable_im2col_cuda(cudaStream_t stream, - const scalar_t* data_value, - const int64_t* data_spatial_shapes, - const int64_t* data_level_start_index, - const scalar_t* data_sampling_loc, - const scalar_t* data_attn_weight, - const int batch_size, - const int spatial_size, - const int num_heads, - const int channels, - const int num_levels, - const int num_query, - const int num_point, - scalar_t* data_col) -{ - const int num_kernels = batch_size * num_query * num_heads * channels; - const int num_actual_kernels = batch_size * num_query * num_heads * channels; - const int num_threads = CUDA_NUM_THREADS; - ms_deformable_im2col_gpu_kernel - <<>>( - num_kernels, data_value, data_spatial_shapes, data_level_start_index, data_sampling_loc, data_attn_weight, - batch_size, spatial_size, num_heads, channels, num_levels, num_query, num_point, data_col); - - cudaError_t err = cudaGetLastError(); - if (err != cudaSuccess) - { - printf("error in ms_deformable_im2col_cuda: %s\n", cudaGetErrorString(err)); - } - -} - -template -void ms_deformable_col2im_cuda(cudaStream_t stream, - const scalar_t* grad_col, - const scalar_t* data_value, - const int64_t * data_spatial_shapes, - const int64_t * data_level_start_index, - const scalar_t * data_sampling_loc, - const scalar_t * data_attn_weight, - const int batch_size, - const int spatial_size, - const int num_heads, - const int channels, - const int num_levels, - const int num_query, - const int num_point, - scalar_t* grad_value, - scalar_t* grad_sampling_loc, - scalar_t* grad_attn_weight) -{ - const int num_threads = (channels > CUDA_NUM_THREADS)?CUDA_NUM_THREADS:channels; - const int num_kernels = batch_size * num_query * num_heads * channels; - const int num_actual_kernels = batch_size * num_query * num_heads * channels; - if (channels > 1024) - { - if ((channels & 1023) == 0) - { - ms_deformable_col2im_gpu_kernel_shm_reduce_v2_multi_blocks - <<>>( - num_kernels, - grad_col, - data_value, - data_spatial_shapes, - data_level_start_index, - data_sampling_loc, - data_attn_weight, - batch_size, - spatial_size, - num_heads, - channels, - num_levels, - num_query, - num_point, - grad_value, - grad_sampling_loc, - grad_attn_weight); - } - else - { - ms_deformable_col2im_gpu_kernel_gm - <<>>( - num_kernels, - grad_col, - data_value, - data_spatial_shapes, - data_level_start_index, - data_sampling_loc, - data_attn_weight, - batch_size, - spatial_size, - num_heads, - channels, - num_levels, - num_query, - num_point, - grad_value, - grad_sampling_loc, - grad_attn_weight); - } - } - else{ - switch(channels) - { - case 1: - ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 - <<>>( - num_kernels, - grad_col, - data_value, - data_spatial_shapes, - data_level_start_index, - data_sampling_loc, - data_attn_weight, - batch_size, - spatial_size, - num_heads, - channels, - num_levels, - num_query, - num_point, - grad_value, - grad_sampling_loc, - grad_attn_weight); - break; - case 2: - ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 - <<>>( - num_kernels, - grad_col, - data_value, - data_spatial_shapes, - data_level_start_index, - data_sampling_loc, - data_attn_weight, - batch_size, - spatial_size, - num_heads, - channels, - num_levels, - num_query, - num_point, - grad_value, - grad_sampling_loc, - grad_attn_weight); - break; - case 4: - ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 - <<>>( - num_kernels, - grad_col, - data_value, - data_spatial_shapes, - data_level_start_index, - data_sampling_loc, - data_attn_weight, - batch_size, - spatial_size, - num_heads, - channels, - num_levels, - num_query, - num_point, - grad_value, - grad_sampling_loc, - grad_attn_weight); - break; - case 8: - ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 - <<>>( - num_kernels, - grad_col, - data_value, - data_spatial_shapes, - data_level_start_index, - data_sampling_loc, - data_attn_weight, - batch_size, - spatial_size, - num_heads, - channels, - num_levels, - num_query, - num_point, - grad_value, - grad_sampling_loc, - grad_attn_weight); - break; - case 16: - ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 - <<>>( - num_kernels, - grad_col, - data_value, - data_spatial_shapes, - data_level_start_index, - data_sampling_loc, - data_attn_weight, - batch_size, - spatial_size, - num_heads, - channels, - num_levels, - num_query, - num_point, - grad_value, - grad_sampling_loc, - grad_attn_weight); - break; - case 32: - ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 - <<>>( - num_kernels, - grad_col, - data_value, - data_spatial_shapes, - data_level_start_index, - data_sampling_loc, - data_attn_weight, - batch_size, - spatial_size, - num_heads, - channels, - num_levels, - num_query, - num_point, - grad_value, - grad_sampling_loc, - grad_attn_weight); - break; - case 64: - ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 - <<>>( - num_kernels, - grad_col, - data_value, - data_spatial_shapes, - data_level_start_index, - data_sampling_loc, - data_attn_weight, - batch_size, - spatial_size, - num_heads, - channels, - num_levels, - num_query, - num_point, - grad_value, - grad_sampling_loc, - grad_attn_weight); - break; - case 128: - ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 - <<>>( - num_kernels, - grad_col, - data_value, - data_spatial_shapes, - data_level_start_index, - data_sampling_loc, - data_attn_weight, - batch_size, - spatial_size, - num_heads, - channels, - num_levels, - num_query, - num_point, - grad_value, - grad_sampling_loc, - grad_attn_weight); - break; - case 256: - ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 - <<>>( - num_kernels, - grad_col, - data_value, - data_spatial_shapes, - data_level_start_index, - data_sampling_loc, - data_attn_weight, - batch_size, - spatial_size, - num_heads, - channels, - num_levels, - num_query, - num_point, - grad_value, - grad_sampling_loc, - grad_attn_weight); - break; - case 512: - ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 - <<>>( - num_kernels, - grad_col, - data_value, - data_spatial_shapes, - data_level_start_index, - data_sampling_loc, - data_attn_weight, - batch_size, - spatial_size, - num_heads, - channels, - num_levels, - num_query, - num_point, - grad_value, - grad_sampling_loc, - grad_attn_weight); - break; - case 1024: - ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 - <<>>( - num_kernels, - grad_col, - data_value, - data_spatial_shapes, - data_level_start_index, - data_sampling_loc, - data_attn_weight, - batch_size, - spatial_size, - num_heads, - channels, - num_levels, - num_query, - num_point, - grad_value, - grad_sampling_loc, - grad_attn_weight); - break; - default: - if (channels < 64) - { - ms_deformable_col2im_gpu_kernel_shm_reduce_v1 - <<>>( - num_kernels, - grad_col, - data_value, - data_spatial_shapes, - data_level_start_index, - data_sampling_loc, - data_attn_weight, - batch_size, - spatial_size, - num_heads, - channels, - num_levels, - num_query, - num_point, - grad_value, - grad_sampling_loc, - grad_attn_weight); - } - else - { - ms_deformable_col2im_gpu_kernel_shm_reduce_v2 - <<>>( - num_kernels, - grad_col, - data_value, - data_spatial_shapes, - data_level_start_index, - data_sampling_loc, - data_attn_weight, - batch_size, - spatial_size, - num_heads, - channels, - num_levels, - num_query, - num_point, - grad_value, - grad_sampling_loc, - grad_attn_weight); - } - } - } - cudaError_t err = cudaGetLastError(); - if (err != cudaSuccess) - { - printf("error in ms_deformable_col2im_cuda: %s\n", cudaGetErrorString(err)); - } - -} \ No newline at end of file diff --git a/groundingdino/models/GroundingDINO/ops/src/ms_deform_attn.h b/groundingdino/models/GroundingDINO/ops/src/ms_deform_attn.h deleted file mode 100644 index ac0ef2ec25f7d0ee51ca2d807b159ddf85652017..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/src/ms_deform_attn.h +++ /dev/null @@ -1,62 +0,0 @@ -/*! -************************************************************************************************** -* Deformable DETR -* Copyright (c) 2020 SenseTime. All Rights Reserved. -* Licensed under the Apache License, Version 2.0 [see LICENSE for details] -************************************************************************************************** -* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 -************************************************************************************************** -*/ - -#pragma once - -#include "cpu/ms_deform_attn_cpu.h" - -#ifdef WITH_CUDA -#include "cuda/ms_deform_attn_cuda.h" -#endif - - -at::Tensor -ms_deform_attn_forward( - const at::Tensor &value, - const at::Tensor &spatial_shapes, - const at::Tensor &level_start_index, - const at::Tensor &sampling_loc, - const at::Tensor &attn_weight, - const int im2col_step) -{ - if (value.type().is_cuda()) - { -#ifdef WITH_CUDA - return ms_deform_attn_cuda_forward( - value, spatial_shapes, level_start_index, sampling_loc, attn_weight, im2col_step); -#else - AT_ERROR("Not compiled with GPU support"); -#endif - } - AT_ERROR("Not implemented on the CPU"); -} - -std::vector -ms_deform_attn_backward( - const at::Tensor &value, - const at::Tensor &spatial_shapes, - const at::Tensor &level_start_index, - const at::Tensor &sampling_loc, - const at::Tensor &attn_weight, - const at::Tensor &grad_output, - const int im2col_step) -{ - if (value.type().is_cuda()) - { -#ifdef WITH_CUDA - return ms_deform_attn_cuda_backward( - value, spatial_shapes, level_start_index, sampling_loc, attn_weight, grad_output, im2col_step); -#else - AT_ERROR("Not compiled with GPU support"); -#endif - } - AT_ERROR("Not implemented on the CPU"); -} - diff --git a/groundingdino/models/GroundingDINO/ops/src/vision.cpp b/groundingdino/models/GroundingDINO/ops/src/vision.cpp deleted file mode 100644 index 2201f63a51dca16d0b31148ed2c9e8e47ec15bdc..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/src/vision.cpp +++ /dev/null @@ -1,16 +0,0 @@ -/*! -************************************************************************************************** -* Deformable DETR -* Copyright (c) 2020 SenseTime. All Rights Reserved. -* Licensed under the Apache License, Version 2.0 [see LICENSE for details] -************************************************************************************************** -* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 -************************************************************************************************** -*/ - -#include "ms_deform_attn.h" - -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { - m.def("ms_deform_attn_forward", &ms_deform_attn_forward, "ms_deform_attn_forward"); - m.def("ms_deform_attn_backward", &ms_deform_attn_backward, "ms_deform_attn_backward"); -} diff --git a/groundingdino/models/GroundingDINO/ops/test.py b/groundingdino/models/GroundingDINO/ops/test.py deleted file mode 100644 index c3a1b918492835d45d19741b49754119047e5c5c..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/ops/test.py +++ /dev/null @@ -1,86 +0,0 @@ -# ------------------------------------------------------------------------------------------------ -# Deformable DETR -# Copyright (c) 2020 SenseTime. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------------------------------ -# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 -# ------------------------------------------------------------------------------------------------ - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import division - -import time -import torch -import torch.nn as nn -from torch.autograd import gradcheck - -from functions.ms_deform_attn_func import MSDeformAttnFunction, ms_deform_attn_core_pytorch - - -N, M, D = 1, 2, 2 -Lq, L, P = 2, 2, 2 -shapes = torch.as_tensor([(6, 4), (3, 2)], dtype=torch.long).cuda() -level_start_index = torch.cat((shapes.new_zeros((1, )), shapes.prod(1).cumsum(0)[:-1])) -S = sum([(H*W).item() for H, W in shapes]) - - -torch.manual_seed(3) - - -@torch.no_grad() -def check_forward_equal_with_pytorch_double(): - value = torch.rand(N, S, M, D).cuda() * 0.01 - sampling_locations = torch.rand(N, Lq, M, L, P, 2).cuda() - attention_weights = torch.rand(N, Lq, M, L, P).cuda() + 1e-5 - attention_weights /= attention_weights.sum(-1, keepdim=True).sum(-2, keepdim=True) - im2col_step = 2 - output_pytorch = ms_deform_attn_core_pytorch(value.double(), shapes, sampling_locations.double(), attention_weights.double()).detach().cpu() - output_cuda = MSDeformAttnFunction.apply(value.double(), shapes, level_start_index, sampling_locations.double(), attention_weights.double(), im2col_step).detach().cpu() - fwdok = torch.allclose(output_cuda, output_pytorch) - max_abs_err = (output_cuda - output_pytorch).abs().max() - max_rel_err = ((output_cuda - output_pytorch).abs() / output_pytorch.abs()).max() - - print(f'* {fwdok} check_forward_equal_with_pytorch_double: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') - - -@torch.no_grad() -def check_forward_equal_with_pytorch_float(): - value = torch.rand(N, S, M, D).cuda() * 0.01 - sampling_locations = torch.rand(N, Lq, M, L, P, 2).cuda() - attention_weights = torch.rand(N, Lq, M, L, P).cuda() + 1e-5 - attention_weights /= attention_weights.sum(-1, keepdim=True).sum(-2, keepdim=True) - im2col_step = 2 - output_pytorch = ms_deform_attn_core_pytorch(value, shapes, sampling_locations, attention_weights).detach().cpu() - output_cuda = MSDeformAttnFunction.apply(value, shapes, level_start_index, sampling_locations, attention_weights, im2col_step).detach().cpu() - fwdok = torch.allclose(output_cuda, output_pytorch, rtol=1e-2, atol=1e-3) - max_abs_err = (output_cuda - output_pytorch).abs().max() - max_rel_err = ((output_cuda - output_pytorch).abs() / output_pytorch.abs()).max() - - print(f'* {fwdok} check_forward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') - - -def check_gradient_numerical(channels=4, grad_value=True, grad_sampling_loc=True, grad_attn_weight=True): - - value = torch.rand(N, S, M, channels).cuda() * 0.01 - sampling_locations = torch.rand(N, Lq, M, L, P, 2).cuda() - attention_weights = torch.rand(N, Lq, M, L, P).cuda() + 1e-5 - attention_weights /= attention_weights.sum(-1, keepdim=True).sum(-2, keepdim=True) - im2col_step = 2 - func = MSDeformAttnFunction.apply - - value.requires_grad = grad_value - sampling_locations.requires_grad = grad_sampling_loc - attention_weights.requires_grad = grad_attn_weight - - gradok = gradcheck(func, (value.double(), shapes, level_start_index, sampling_locations.double(), attention_weights.double(), im2col_step)) - - print(f'* {gradok} check_gradient_numerical(D={channels})') - - -if __name__ == '__main__': - check_forward_equal_with_pytorch_double() - check_forward_equal_with_pytorch_float() - - for channels in [30, 32, 64, 71]: - check_gradient_numerical(channels, True, True, True) diff --git a/groundingdino/models/GroundingDINO/transformer.py b/groundingdino/models/GroundingDINO/transformer.py deleted file mode 100644 index 379cbb5857cf953f337312ba5e40014efe83618f..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/transformer.py +++ /dev/null @@ -1,969 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# DINO -# Copyright (c) 2022 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Conditional DETR Transformer class. -# Copyright (c) 2021 Microsoft. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Modified from DETR (https://github.com/facebookresearch/detr) -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -# ------------------------------------------------------------------------ - -from typing import Optional - -import torch -import torch.utils.checkpoint as checkpoint -from torch import Tensor, nn - -from groundingdino.util.misc import inverse_sigmoid -import loralib as lora -from .fuse_modules import BiAttentionBlock -from .ms_deform_attn import MultiScaleDeformableAttention as MSDeformAttn -from .transformer_vanilla import TransformerEncoderLayer -from .utils import ( - MLP, - _get_activation_fn, - _get_clones, - gen_encoder_output_proposals, - gen_sineembed_for_position, - get_sine_pos_embed, -) - - -class Transformer(nn.Module): - def __init__( - self, - d_model=256, - nhead=8, - num_queries=300, - num_encoder_layers=6, - num_unicoder_layers=0, - num_decoder_layers=6, - dim_feedforward=2048, - dropout=0.0, - activation="relu", - normalize_before=False, - return_intermediate_dec=False, - query_dim=4, - num_patterns=0, - # for deformable encoder - num_feature_levels=1, - enc_n_points=4, - dec_n_points=4, - # init query - learnable_tgt_init=False, - # two stage - two_stage_type="no", # ['no', 'standard', 'early', 'combine', 'enceachlayer', 'enclayer1'] - embed_init_tgt=False, - # for text - use_text_enhancer=False, - use_fusion_layer=False, - use_checkpoint=False, - use_transformer_ckpt=False, - use_text_cross_attention=False, - text_dropout=0.1, - fusion_dropout=0.1, - fusion_droppath=0.0, - ): - super().__init__() - self.num_feature_levels = num_feature_levels - self.num_encoder_layers = num_encoder_layers - self.num_unicoder_layers = num_unicoder_layers - self.num_decoder_layers = num_decoder_layers - self.num_queries = num_queries - assert query_dim == 4 - - # choose encoder layer type - encoder_layer = DeformableTransformerEncoderLayer( - d_model, dim_feedforward, dropout, activation, num_feature_levels, nhead, enc_n_points - ) - - if use_text_enhancer: - text_enhance_layer = TransformerEncoderLayer( - d_model=d_model, - nhead=nhead // 2, - dim_feedforward=dim_feedforward // 2, - dropout=text_dropout, - ) - else: - text_enhance_layer = None - - if use_fusion_layer: - feature_fusion_layer = BiAttentionBlock( - v_dim=d_model, - l_dim=d_model, - embed_dim=dim_feedforward // 2, - num_heads=nhead // 2, - dropout=fusion_dropout, - drop_path=fusion_droppath, - ) - else: - feature_fusion_layer = None - - encoder_norm = nn.LayerNorm(d_model) if normalize_before else None - assert encoder_norm is None - self.encoder = TransformerEncoder( - encoder_layer, - num_encoder_layers, - d_model=d_model, - num_queries=num_queries, - text_enhance_layer=text_enhance_layer, - feature_fusion_layer=feature_fusion_layer, - use_checkpoint=use_checkpoint, - use_transformer_ckpt=use_transformer_ckpt, - ) - - # choose decoder layer type - decoder_layer = DeformableTransformerDecoderLayer( - d_model, - dim_feedforward, - dropout, - activation, - num_feature_levels, - nhead, - dec_n_points, - use_text_cross_attention=use_text_cross_attention, - ) - - decoder_norm = nn.LayerNorm(d_model) - self.decoder = TransformerDecoder( - decoder_layer, - num_decoder_layers, - decoder_norm, - return_intermediate=return_intermediate_dec, - d_model=d_model, - query_dim=query_dim, - num_feature_levels=num_feature_levels, - ) - - self.d_model = d_model - self.nhead = nhead - self.dec_layers = num_decoder_layers - self.num_queries = num_queries # useful for single stage model only - self.num_patterns = num_patterns - if not isinstance(num_patterns, int): - Warning("num_patterns should be int but {}".format(type(num_patterns))) - self.num_patterns = 0 - - if num_feature_levels > 1: - if self.num_encoder_layers > 0: - self.level_embed = nn.Parameter(torch.Tensor(num_feature_levels, d_model)) - else: - self.level_embed = None - - self.learnable_tgt_init = learnable_tgt_init - assert learnable_tgt_init, "why not learnable_tgt_init" - self.embed_init_tgt = embed_init_tgt - if (two_stage_type != "no" and embed_init_tgt) or (two_stage_type == "no"): - self.tgt_embed = nn.Embedding(self.num_queries, d_model) - nn.init.normal_(self.tgt_embed.weight.data) - else: - self.tgt_embed = None - - # for two stage - self.two_stage_type = two_stage_type - assert two_stage_type in ["no", "standard"], "unknown param {} of two_stage_type".format( - two_stage_type - ) - if two_stage_type == "standard": - # anchor selection at the output of encoder - r = 24 - self.enc_output = lora.Linear(d_model, d_model , r=r) - self.enc_output_norm = nn.LayerNorm(d_model) - self.two_stage_wh_embedding = None - - if two_stage_type == "no": - self.init_ref_points(num_queries) # init self.refpoint_embed - - self.enc_out_class_embed = None - self.enc_out_bbox_embed = None - - self._reset_parameters() - - def _reset_parameters(self): - for p in self.parameters(): - if p.dim() > 1: - nn.init.xavier_uniform_(p) - for m in self.modules(): - if isinstance(m, MSDeformAttn): - m._reset_parameters() - if self.num_feature_levels > 1 and self.level_embed is not None: - nn.init.normal_(self.level_embed) - - def get_valid_ratio(self, mask): - _, H, W = mask.shape - valid_H = torch.sum(~mask[:, :, 0], 1) - valid_W = torch.sum(~mask[:, 0, :], 1) - valid_ratio_h = valid_H.float() / H - valid_ratio_w = valid_W.float() / W - valid_ratio = torch.stack([valid_ratio_w, valid_ratio_h], -1) - return valid_ratio - - def init_ref_points(self, use_num_queries): - self.refpoint_embed = nn.Embedding(use_num_queries, 4) - - def forward(self, srcs, masks, refpoint_embed, pos_embeds, tgt, attn_mask=None, text_dict=None): - """ - Input: - - srcs: List of multi features [bs, ci, hi, wi] - - masks: List of multi masks [bs, hi, wi] - - refpoint_embed: [bs, num_dn, 4]. None in infer - - pos_embeds: List of multi pos embeds [bs, ci, hi, wi] - - tgt: [bs, num_dn, d_model]. None in infer - - """ - # prepare input for encoder - src_flatten = [] - mask_flatten = [] - lvl_pos_embed_flatten = [] - spatial_shapes = [] - for lvl, (src, mask, pos_embed) in enumerate(zip(srcs, masks, pos_embeds)): - bs, c, h, w = src.shape - spatial_shape = (h, w) - spatial_shapes.append(spatial_shape) - - src = src.flatten(2).transpose(1, 2) # bs, hw, c - mask = mask.flatten(1) # bs, hw - pos_embed = pos_embed.flatten(2).transpose(1, 2) # bs, hw, c - if self.num_feature_levels > 1 and self.level_embed is not None: - lvl_pos_embed = pos_embed + self.level_embed[lvl].view(1, 1, -1) - else: - lvl_pos_embed = pos_embed - lvl_pos_embed_flatten.append(lvl_pos_embed) - src_flatten.append(src) - mask_flatten.append(mask) - src_flatten = torch.cat(src_flatten, 1) # bs, \sum{hxw}, c - mask_flatten = torch.cat(mask_flatten, 1) # bs, \sum{hxw} - lvl_pos_embed_flatten = torch.cat(lvl_pos_embed_flatten, 1) # bs, \sum{hxw}, c - spatial_shapes = torch.as_tensor( - spatial_shapes, dtype=torch.long, device=src_flatten.device - ) - level_start_index = torch.cat( - (spatial_shapes.new_zeros((1,)), spatial_shapes.prod(1).cumsum(0)[:-1]) - ) - valid_ratios = torch.stack([self.get_valid_ratio(m) for m in masks], 1) - - # two stage - enc_topk_proposals = enc_refpoint_embed = None - - ######################################################### - # Begin Encoder - ######################################################### - memory, memory_text = self.encoder( - src_flatten, - pos=lvl_pos_embed_flatten, - level_start_index=level_start_index, - spatial_shapes=spatial_shapes, - valid_ratios=valid_ratios, - key_padding_mask=mask_flatten, - memory_text=text_dict["encoded_text"], - text_attention_mask=~text_dict["text_token_mask"], - # we ~ the mask . False means use the token; True means pad the token - position_ids=text_dict["position_ids"], - text_self_attention_masks=text_dict["text_self_attention_masks"], - ) - ######################################################### - # End Encoder - # - memory: bs, \sum{hw}, c - # - mask_flatten: bs, \sum{hw} - # - lvl_pos_embed_flatten: bs, \sum{hw}, c - # - enc_intermediate_output: None or (nenc+1, bs, nq, c) or (nenc, bs, nq, c) - # - enc_intermediate_refpoints: None or (nenc+1, bs, nq, c) or (nenc, bs, nq, c) - ######################################################### - text_dict["encoded_text"] = memory_text - # if os.environ.get("SHILONG_AMP_INFNAN_DEBUG") == '1': - # if memory.isnan().any() | memory.isinf().any(): - # import ipdb; ipdb.set_trace() - - if self.two_stage_type == "standard": #把encoder的输出作为proposal - output_memory, output_proposals = gen_encoder_output_proposals( - memory, mask_flatten, spatial_shapes - ) - output_memory = self.enc_output_norm(self.enc_output(output_memory)) - - if text_dict is not None: - enc_outputs_class_unselected = self.enc_out_class_embed(output_memory, text_dict) - else: - enc_outputs_class_unselected = self.enc_out_class_embed(output_memory) - - topk_logits = enc_outputs_class_unselected.max(-1)[0] - enc_outputs_coord_unselected = ( - self.enc_out_bbox_embed(output_memory) + output_proposals - ) # (bs, \sum{hw}, 4) unsigmoid - topk = self.num_queries - - topk_proposals = torch.topk(topk_logits, topk, dim=1)[1] # bs, nq - - # gather boxes - refpoint_embed_undetach = torch.gather( - enc_outputs_coord_unselected, 1, topk_proposals.unsqueeze(-1).repeat(1, 1, 4) - ) # unsigmoid - refpoint_embed_ = refpoint_embed_undetach.detach() - init_box_proposal = torch.gather( - output_proposals, 1, topk_proposals.unsqueeze(-1).repeat(1, 1, 4) - ).sigmoid() # sigmoid - - # gather tgt - tgt_undetach = torch.gather( - output_memory, 1, topk_proposals.unsqueeze(-1).repeat(1, 1, self.d_model) - ) - if self.embed_init_tgt: - tgt_ = ( - self.tgt_embed.weight[:, None, :].repeat(1, bs, 1).transpose(0, 1) - ) # nq, bs, d_model - else: - tgt_ = tgt_undetach.detach() - - if refpoint_embed is not None: - refpoint_embed = torch.cat([refpoint_embed, refpoint_embed_], dim=1) - tgt = torch.cat([tgt, tgt_], dim=1) - else: - refpoint_embed, tgt = refpoint_embed_, tgt_ - - elif self.two_stage_type == "no": - tgt_ = ( - self.tgt_embed.weight[:, None, :].repeat(1, bs, 1).transpose(0, 1) - ) # nq, bs, d_model - refpoint_embed_ = ( - self.refpoint_embed.weight[:, None, :].repeat(1, bs, 1).transpose(0, 1) - ) # nq, bs, 4 - - if refpoint_embed is not None: - refpoint_embed = torch.cat([refpoint_embed, refpoint_embed_], dim=1) - tgt = torch.cat([tgt, tgt_], dim=1) - else: - refpoint_embed, tgt = refpoint_embed_, tgt_ - - if self.num_patterns > 0: - tgt_embed = tgt.repeat(1, self.num_patterns, 1) - refpoint_embed = refpoint_embed.repeat(1, self.num_patterns, 1) - tgt_pat = self.patterns.weight[None, :, :].repeat_interleave( - self.num_queries, 1 - ) # 1, n_q*n_pat, d_model - tgt = tgt_embed + tgt_pat - - init_box_proposal = refpoint_embed_.sigmoid() - - else: - raise NotImplementedError("unknown two_stage_type {}".format(self.two_stage_type)) - ######################################################### - # End preparing tgt - # - tgt: bs, NQ, d_model - # - refpoint_embed(unsigmoid): bs, NQ, d_model - ######################################################### - - ######################################################### - # Begin Decoder - ######################################################### - - #memory torch.Size([2, 16320, 256]) - - # import pdb;pdb.set_trace() - hs, references = self.decoder( - tgt=tgt.transpose(0, 1), - memory=memory.transpose(0, 1), - memory_key_padding_mask=mask_flatten, - pos=lvl_pos_embed_flatten.transpose(0, 1), - refpoints_unsigmoid=refpoint_embed.transpose(0, 1), - level_start_index=level_start_index, - spatial_shapes=spatial_shapes, - valid_ratios=valid_ratios, - tgt_mask=attn_mask, - memory_text=text_dict["encoded_text"], - text_attention_mask=~text_dict["text_token_mask"], - # we ~ the mask . False means use the token; True means pad the token - ) - ######################################################### - # End Decoder - # hs: n_dec, bs, nq, d_model - # references: n_dec+1, bs, nq, query_dim - ######################################################### - - ######################################################### - # Begin postprocess - ######################################################### - if self.two_stage_type == "standard": - hs_enc = tgt_undetach.unsqueeze(0) - ref_enc = refpoint_embed_undetach.sigmoid().unsqueeze(0) - else: - hs_enc = ref_enc = None - ######################################################### - # End postprocess - # hs_enc: (n_enc+1, bs, nq, d_model) or (1, bs, nq, d_model) or (n_enc, bs, nq, d_model) or None - # ref_enc: (n_enc+1, bs, nq, query_dim) or (1, bs, nq, query_dim) or (n_enc, bs, nq, d_model) or None - ######################################################### - - return hs, references, hs_enc, ref_enc, init_box_proposal - # hs: (n_dec, bs, nq, d_model) - # references: sigmoid coordinates. (n_dec+1, bs, bq, 4) - # hs_enc: (n_enc+1, bs, nq, d_model) or (1, bs, nq, d_model) or None - # ref_enc: sigmoid coordinates. \ - # (n_enc+1, bs, nq, query_dim) or (1, bs, nq, query_dim) or None - - -class TransformerEncoder(nn.Module): - def __init__( - self, - encoder_layer, - num_layers, - d_model=256, - num_queries=300, - enc_layer_share=False, - text_enhance_layer=None, - feature_fusion_layer=None, - use_checkpoint=False, - use_transformer_ckpt=False, - ): - """_summary_ - - Args: - encoder_layer (_type_): _description_ - num_layers (_type_): _description_ - norm (_type_, optional): _description_. Defaults to None. - d_model (int, optional): _description_. Defaults to 256. - num_queries (int, optional): _description_. Defaults to 300. - enc_layer_share (bool, optional): _description_. Defaults to False. - - """ - super().__init__() - # prepare layers - self.layers = [] - self.text_layers = [] - self.fusion_layers = [] - if num_layers > 0: - self.layers = _get_clones(encoder_layer, num_layers, layer_share=enc_layer_share) - - if text_enhance_layer is not None: - self.text_layers = _get_clones( - text_enhance_layer, num_layers, layer_share=enc_layer_share - ) - if feature_fusion_layer is not None: - self.fusion_layers = _get_clones( - feature_fusion_layer, num_layers, layer_share=enc_layer_share - ) - else: - self.layers = [] - del encoder_layer - - if text_enhance_layer is not None: - self.text_layers = [] - del text_enhance_layer - if feature_fusion_layer is not None: - self.fusion_layers = [] - del feature_fusion_layer - - self.query_scale = None - self.num_queries = num_queries - self.num_layers = num_layers - self.d_model = d_model - - self.use_checkpoint = use_checkpoint - self.use_transformer_ckpt = use_transformer_ckpt - - @staticmethod - def get_reference_points(spatial_shapes, valid_ratios, device): - reference_points_list = [] - for lvl, (H_, W_) in enumerate(spatial_shapes): - - ref_y, ref_x = torch.meshgrid( - torch.linspace(0.5, H_ - 0.5, H_, dtype=torch.float32, device=device), - torch.linspace(0.5, W_ - 0.5, W_, dtype=torch.float32, device=device), - ) - ref_y = ref_y.reshape(-1)[None] / (valid_ratios[:, None, lvl, 1] * H_) - ref_x = ref_x.reshape(-1)[None] / (valid_ratios[:, None, lvl, 0] * W_) - ref = torch.stack((ref_x, ref_y), -1) - reference_points_list.append(ref) - reference_points = torch.cat(reference_points_list, 1) - reference_points = reference_points[:, :, None] * valid_ratios[:, None] - return reference_points - - def forward( - self, - # for images - src: Tensor, - pos: Tensor, - spatial_shapes: Tensor, - level_start_index: Tensor, - valid_ratios: Tensor, - key_padding_mask: Tensor, - # for texts - memory_text: Tensor = None, - text_attention_mask: Tensor = None, - pos_text: Tensor = None, - text_self_attention_masks: Tensor = None, - position_ids: Tensor = None, - ): - """ - Input: - - src: [bs, sum(hi*wi), 256] - - pos: pos embed for src. [bs, sum(hi*wi), 256] - - spatial_shapes: h,w of each level [num_level, 2] - - level_start_index: [num_level] start point of level in sum(hi*wi). - - valid_ratios: [bs, num_level, 2] - - key_padding_mask: [bs, sum(hi*wi)] - - - memory_text: bs, n_text, 256 - - text_attention_mask: bs, n_text - False for no padding; True for padding - - pos_text: bs, n_text, 256 - - - position_ids: bs, n_text - Intermedia: - - reference_points: [bs, sum(hi*wi), num_level, 2] - Outpus: - - output: [bs, sum(hi*wi), 256] - """ - - output = src - - # preparation and reshape - if self.num_layers > 0: - reference_points = self.get_reference_points( - spatial_shapes, valid_ratios, device=src.device - ) - - if self.text_layers: - # generate pos_text - bs, n_text, text_dim = memory_text.shape - if pos_text is None and position_ids is None: - pos_text = ( - torch.arange(n_text, device=memory_text.device) - .float() - .unsqueeze(0) - .unsqueeze(-1) - .repeat(bs, 1, 1) - ) - pos_text = get_sine_pos_embed(pos_text, num_pos_feats=256, exchange_xy=False) - if position_ids is not None: - pos_text = get_sine_pos_embed( - position_ids[..., None], num_pos_feats=256, exchange_xy=False - ) - - # main process - for layer_id, layer in enumerate(self.layers): - # if output.isnan().any() or memory_text.isnan().any(): - # if os.environ.get('IPDB_SHILONG_DEBUG', None) == 'INFO': - # import ipdb; ipdb.set_trace() - if self.fusion_layers: - if self.use_checkpoint: - output, memory_text = checkpoint.checkpoint( - self.fusion_layers[layer_id], - output, - memory_text, - key_padding_mask, - text_attention_mask, - ) - else: - output, memory_text = self.fusion_layers[layer_id]( - v=output, - l=memory_text, - attention_mask_v=key_padding_mask, - attention_mask_l=text_attention_mask, - ) - - if self.text_layers: - memory_text = self.text_layers[layer_id]( - src=memory_text.transpose(0, 1), - src_mask=~text_self_attention_masks, # note we use ~ for mask here - src_key_padding_mask=text_attention_mask, - pos=(pos_text.transpose(0, 1) if pos_text is not None else None), - ).transpose(0, 1) - - # main process - if self.use_transformer_ckpt: - output = checkpoint.checkpoint( - layer, - output, - pos, - reference_points, - spatial_shapes, - level_start_index, - key_padding_mask, - ) - else: - output = layer( - src=output, - pos=pos, - reference_points=reference_points, - spatial_shapes=spatial_shapes, - level_start_index=level_start_index, - key_padding_mask=key_padding_mask, - ) - - return output, memory_text - - -class TransformerDecoder(nn.Module): - def __init__( - self, - decoder_layer, - num_layers, - norm=None, - return_intermediate=False, - d_model=256, - query_dim=4, - num_feature_levels=1, - ): - super().__init__() - if num_layers > 0: - self.layers = _get_clones(decoder_layer, num_layers) - else: - self.layers = [] - self.num_layers = num_layers - self.norm = norm - self.return_intermediate = return_intermediate - assert return_intermediate, "support return_intermediate only" - self.query_dim = query_dim - assert query_dim in [2, 4], "query_dim should be 2/4 but {}".format(query_dim) - self.num_feature_levels = num_feature_levels - - self.ref_point_head = MLP(query_dim // 2 * d_model, d_model, d_model, 2) - self.query_pos_sine_scale = None - - self.query_scale = None - self.bbox_embed = None - self.class_embed = None - - self.d_model = d_model - - self.ref_anchor_head = None - - def forward( - self, - tgt, - memory, - tgt_mask: Optional[Tensor] = None, - memory_mask: Optional[Tensor] = None, - tgt_key_padding_mask: Optional[Tensor] = None, - memory_key_padding_mask: Optional[Tensor] = None, - pos: Optional[Tensor] = None, - refpoints_unsigmoid: Optional[Tensor] = None, # num_queries, bs, 2 - # for memory - level_start_index: Optional[Tensor] = None, # num_levels - spatial_shapes: Optional[Tensor] = None, # bs, num_levels, 2 - valid_ratios: Optional[Tensor] = None, - # for text - memory_text: Optional[Tensor] = None, - text_attention_mask: Optional[Tensor] = None, - ): - """ - Input: - - tgt: nq, bs, d_model - - memory: hw, bs, d_model - - pos: hw, bs, d_model - - refpoints_unsigmoid: nq, bs, 2/4 - - valid_ratios/spatial_shapes: bs, nlevel, 2 - """ - output = tgt - - intermediate = [] - reference_points = refpoints_unsigmoid.sigmoid() - ref_points = [reference_points] - - - - for layer_id, layer in enumerate(self.layers): - - if reference_points.shape[-1] == 4: - reference_points_input = ( - reference_points[:, :, None] - * torch.cat([valid_ratios, valid_ratios], -1)[None, :] - ) # nq, bs, nlevel, 4 - else: - assert reference_points.shape[-1] == 2 - reference_points_input = reference_points[:, :, None] * valid_ratios[None, :] - query_sine_embed = gen_sineembed_for_position( - reference_points_input[:, :, 0, :] - ) # nq, bs, 256*2 - - # conditional query - raw_query_pos = self.ref_point_head(query_sine_embed) # nq, bs, 256 - pos_scale = self.query_scale(output) if self.query_scale is not None else 1 - query_pos = pos_scale * raw_query_pos - # if os.environ.get("SHILONG_AMP_INFNAN_DEBUG") == '1': - # if query_pos.isnan().any() | query_pos.isinf().any(): - # import ipdb; ipdb.set_trace() - - # main process - output = layer( - tgt=output, - tgt_query_pos=query_pos, - tgt_query_sine_embed=query_sine_embed, - tgt_key_padding_mask=tgt_key_padding_mask, - tgt_reference_points=reference_points_input, - memory_text=memory_text, - text_attention_mask=text_attention_mask, - memory=memory, - memory_key_padding_mask=memory_key_padding_mask, - memory_level_start_index=level_start_index, - memory_spatial_shapes=spatial_shapes, - memory_pos=pos, - self_attn_mask=tgt_mask, - cross_attn_mask=memory_mask, - ) - if output.isnan().any() | output.isinf().any(): - print(f"output layer_id {layer_id} is nan") - try: - num_nan = output.isnan().sum().item() - num_inf = output.isinf().sum().item() - print(f"num_nan {num_nan}, num_inf {num_inf}") - except Exception as e: - print(e) - # if os.environ.get("SHILONG_AMP_INFNAN_DEBUG") == '1': - # import ipdb; ipdb.set_trace() - - # iter update - if self.bbox_embed is not None: - # box_holder = self.bbox_embed(output) - # box_holder[..., :self.query_dim] += inverse_sigmoid(reference_points) - # new_reference_points = box_holder[..., :self.query_dim].sigmoid() - - reference_before_sigmoid = inverse_sigmoid(reference_points) - delta_unsig = self.bbox_embed[layer_id](output) - outputs_unsig = delta_unsig + reference_before_sigmoid - new_reference_points = outputs_unsig.sigmoid() - - reference_points = new_reference_points.detach() - # if layer_id != self.num_layers - 1: - ref_points.append(new_reference_points) - - intermediate.append(self.norm(output)) - - # import pdb;pdb.set_trace() - - return [ - [itm_out.transpose(0, 1) for itm_out in intermediate], - [itm_refpoint.transpose(0, 1) for itm_refpoint in ref_points], - ] - - -class DeformableTransformerEncoderLayer(nn.Module): - def __init__( - self, - d_model=256, - d_ffn=1024, - dropout=0.1, - activation="relu", - n_levels=4, - n_heads=8, - n_points=4, - ): - super().__init__() - - # self attention - self.self_attn = MSDeformAttn( - embed_dim=d_model, - num_levels=n_levels, - num_heads=n_heads, - num_points=n_points, - batch_first=True, - ) - self.dropout1 = nn.Dropout(dropout) - self.norm1 = nn.LayerNorm(d_model) - r =12 - # ffn - self.linear1 = lora.Linear(d_model, d_ffn , r=r ) - self.activation = _get_activation_fn(activation, d_model=d_ffn) - self.dropout2 = nn.Dropout(dropout) - self.linear2 = lora.Linear(d_ffn, d_model , r=r) - self.dropout3 = nn.Dropout(dropout) - self.norm2 = nn.LayerNorm(d_model) - - @staticmethod - def with_pos_embed(tensor, pos): - return tensor if pos is None else tensor + pos - - def forward_ffn(self, src): - src2 = self.linear2(self.dropout2(self.activation(self.linear1(src)))) - src = src + self.dropout3(src2) - src = self.norm2(src) - return src - - def forward( - self, src, pos, reference_points, spatial_shapes, level_start_index, key_padding_mask=None - ): - # self attention - # import ipdb; ipdb.set_trace() - src2 = self.self_attn( - query=self.with_pos_embed(src, pos), - reference_points=reference_points, - value=src, - spatial_shapes=spatial_shapes, - level_start_index=level_start_index, - key_padding_mask=key_padding_mask, - ) - src = src + self.dropout1(src2) - src = self.norm1(src) - - # ffn - src = self.forward_ffn(src) - - return src - - -class DeformableTransformerDecoderLayer(nn.Module): - def __init__( - self, - d_model=256, - d_ffn=1024, - dropout=0.1, - activation="relu", - n_levels=4, - n_heads=8, - n_points=4, - use_text_feat_guide=False, - use_text_cross_attention=False, - ): - super().__init__() - - # cross attention - self.cross_attn = MSDeformAttn( - embed_dim=d_model, - num_levels=n_levels, - num_heads=n_heads, - num_points=n_points, - batch_first=True, - ) - self.dropout1 = nn.Dropout(dropout) if dropout > 0 else nn.Identity() - self.norm1 = nn.LayerNorm(d_model) - - # cross attention text - if use_text_cross_attention: - self.ca_text = nn.MultiheadAttention(d_model, n_heads, dropout=dropout) - self.catext_dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity() - self.catext_norm = nn.LayerNorm(d_model) - - # self attention - self.self_attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout) - self.dropout2 = nn.Dropout(dropout) if dropout > 0 else nn.Identity() - self.norm2 = nn.LayerNorm(d_model) - - # ffn - r = 12 - self.linear1 = lora.Linear(d_model, d_ffn , r=r) - self.activation = _get_activation_fn(activation, d_model=d_ffn, batch_dim=1) - self.dropout3 = nn.Dropout(dropout) if dropout > 0 else nn.Identity() - self.linear2 = lora.Linear(d_ffn, d_model , r=r ) - self.dropout4 = nn.Dropout(dropout) if dropout > 0 else nn.Identity() - self.norm3 = nn.LayerNorm(d_model) - - self.key_aware_proj = None - self.use_text_feat_guide = use_text_feat_guide - assert not use_text_feat_guide - self.use_text_cross_attention = use_text_cross_attention - - def rm_self_attn_modules(self): - self.self_attn = None - self.dropout2 = None - self.norm2 = None - - @staticmethod - def with_pos_embed(tensor, pos): - return tensor if pos is None else tensor + pos - - def forward_ffn(self, tgt): - with torch.cuda.amp.autocast(enabled=False): - tgt2 = self.linear2(self.dropout3(self.activation(self.linear1(tgt)))) - tgt = tgt + self.dropout4(tgt2) - tgt = self.norm3(tgt) - return tgt - - def forward( - self, - # for tgt - tgt: Optional[Tensor], # nq, bs, d_model - tgt_query_pos: Optional[Tensor] = None, # pos for query. MLP(Sine(pos)) - tgt_query_sine_embed: Optional[Tensor] = None, # pos for query. Sine(pos) - tgt_key_padding_mask: Optional[Tensor] = None, - tgt_reference_points: Optional[Tensor] = None, # nq, bs, 4 - memory_text: Optional[Tensor] = None, # bs, num_token, d_model - text_attention_mask: Optional[Tensor] = None, # bs, num_token - # for memory - memory: Optional[Tensor] = None, # hw, bs, d_model - memory_key_padding_mask: Optional[Tensor] = None, - memory_level_start_index: Optional[Tensor] = None, # num_levels - memory_spatial_shapes: Optional[Tensor] = None, # bs, num_levels, 2 - memory_pos: Optional[Tensor] = None, # pos for memory - # sa - self_attn_mask: Optional[Tensor] = None, # mask used for self-attention - cross_attn_mask: Optional[Tensor] = None, # mask used for cross-attention - ): - """ - Input: - - tgt/tgt_query_pos: nq, bs, d_model - - - """ - assert cross_attn_mask is None - - # self attention - if self.self_attn is not None: - # import ipdb; ipdb.set_trace() - q = k = self.with_pos_embed(tgt, tgt_query_pos) - tgt2 = self.self_attn(q, k, tgt, attn_mask=self_attn_mask)[0] - tgt = tgt + self.dropout2(tgt2) - tgt = self.norm2(tgt) - - if self.use_text_cross_attention: - tgt2 = self.ca_text( - self.with_pos_embed(tgt, tgt_query_pos), - memory_text.transpose(0, 1), - memory_text.transpose(0, 1), - key_padding_mask=text_attention_mask, - )[0] - tgt = tgt + self.catext_dropout(tgt2) - tgt = self.catext_norm(tgt) - - tgt2 = self.cross_attn( - query=self.with_pos_embed(tgt, tgt_query_pos).transpose(0, 1), - reference_points=tgt_reference_points.transpose(0, 1).contiguous(), - value=memory.transpose(0, 1), - spatial_shapes=memory_spatial_shapes, - level_start_index=memory_level_start_index, - key_padding_mask=memory_key_padding_mask, - ).transpose(0, 1) - tgt = tgt + self.dropout1(tgt2) - tgt = self.norm1(tgt) - - # ffn - tgt = self.forward_ffn(tgt) - - return tgt - - -def build_transformer(args): - return Transformer( - d_model=args.hidden_dim, - dropout=args.dropout, - nhead=args.nheads, - num_queries=args.num_queries, - dim_feedforward=args.dim_feedforward, - num_encoder_layers=args.enc_layers, - num_decoder_layers=args.dec_layers, - normalize_before=args.pre_norm, - return_intermediate_dec=True, - query_dim=args.query_dim, - activation=args.transformer_activation, - num_patterns=args.num_patterns, - num_feature_levels=args.num_feature_levels, - enc_n_points=args.enc_n_points, - dec_n_points=args.dec_n_points, - learnable_tgt_init=True, - # two stage - two_stage_type=args.two_stage_type, # ['no', 'standard', 'early'] - embed_init_tgt=args.embed_init_tgt, - use_text_enhancer=args.use_text_enhancer, - use_fusion_layer=args.use_fusion_layer, - use_checkpoint=args.use_checkpoint, - use_transformer_ckpt=args.use_transformer_ckpt, - use_text_cross_attention=args.use_text_cross_attention, - text_dropout=args.text_dropout, - fusion_dropout=args.fusion_dropout, - fusion_droppath=args.fusion_droppath, - ) diff --git a/groundingdino/models/GroundingDINO/transformer_vanilla.py b/groundingdino/models/GroundingDINO/transformer_vanilla.py deleted file mode 100644 index 9840aaac0d325dbaefd50d23bd3705e967609287..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/transformer_vanilla.py +++ /dev/null @@ -1,125 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Copyright (c) Aishwarya Kamath & Nicolas Carion. Licensed under the Apache License 2.0. All Rights Reserved -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -""" -DETR Transformer class. - -Copy-paste from torch.nn.Transformer with modifications: - * positional encodings are passed in MHattention - * extra LN at the end of encoder is removed - * decoder returns a stack of activations from all decoding layers -""" -from typing import Optional - -import torch -import torch.nn.functional as F -from torch import Tensor, nn -import loralib as lora - -from .utils import ( - MLP, - _get_activation_fn, - _get_clones, - gen_encoder_output_proposals, - gen_sineembed_for_position, - sigmoid_focal_loss, -) - - -class TextTransformer(nn.Module): - def __init__(self, num_layers, d_model=256, nheads=8, dim_feedforward=2048, dropout=0.1): - super().__init__() - self.num_layers = num_layers - self.d_model = d_model - self.nheads = nheads - self.dim_feedforward = dim_feedforward - self.norm = None - - single_encoder_layer = TransformerEncoderLayer( - d_model=d_model, nhead=nheads, dim_feedforward=dim_feedforward, dropout=dropout - ) - self.layers = _get_clones(single_encoder_layer, num_layers) - - def forward(self, memory_text: torch.Tensor, text_attention_mask: torch.Tensor): - """ - - Args: - text_attention_mask: bs, num_token - memory_text: bs, num_token, d_model - - Raises: - RuntimeError: _description_ - - Returns: - output: bs, num_token, d_model - """ - - output = memory_text.transpose(0, 1) - - for layer in self.layers: - output = layer(output, src_key_padding_mask=text_attention_mask) - - if self.norm is not None: - output = self.norm(output) - - return output.transpose(0, 1) - - -class TransformerEncoderLayer(nn.Module): - def __init__( - self, - d_model, - nhead, - dim_feedforward=2048, - dropout=0.1, - activation="relu", - normalize_before=False, - ): - super().__init__() - self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) - r = 12 - # Implementation of Feedforward model - self.linear1 = lora.Linear(d_model, dim_feedforward , r=r) - self.dropout = nn.Dropout(dropout) - self.linear2 = lora.Linear(dim_feedforward, d_model , r=r) - - self.norm1 = nn.LayerNorm(d_model) - self.norm2 = nn.LayerNorm(d_model) - self.dropout1 = nn.Dropout(dropout) - self.dropout2 = nn.Dropout(dropout) - - self.activation = _get_activation_fn(activation) - self.normalize_before = normalize_before - self.nhead = nhead - - def with_pos_embed(self, tensor, pos: Optional[Tensor]): - return tensor if pos is None else tensor + pos - - def forward( - self, - src, - src_mask: Optional[Tensor] = None, - src_key_padding_mask: Optional[Tensor] = None, - pos: Optional[Tensor] = None, - ): - # repeat attn mask - if src_mask.dim() == 3 and src_mask.shape[0] == src.shape[1]: - # bs, num_q, num_k - src_mask = src_mask.repeat(self.nhead, 1, 1) - - q = k = self.with_pos_embed(src, pos) - - src2 = self.self_attn(q, k, value=src, attn_mask=src_mask)[0] - - # src2 = self.self_attn(q, k, value=src, attn_mask=src_mask, key_padding_mask=src_key_padding_mask)[0] - src = src + self.dropout1(src2) - src = self.norm1(src) - src2 = self.linear2(self.dropout(self.activation(self.linear1(src)))) - src = src + self.dropout2(src2) - src = self.norm2(src) - return src diff --git a/groundingdino/models/GroundingDINO/utils.py b/groundingdino/models/GroundingDINO/utils.py deleted file mode 100644 index 6f295d43befccceec8ff5285beed1f0b99836013..0000000000000000000000000000000000000000 --- a/groundingdino/models/GroundingDINO/utils.py +++ /dev/null @@ -1,274 +0,0 @@ -# ------------------------------------------------------------------------ -# Grounding DINO -# url: https://github.com/IDEA-Research/GroundingDINO -# Copyright (c) 2023 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ - -import copy -import math - -import torch -import torch.nn.functional as F -from torch import Tensor, nn -import loralib as lora - -def _get_clones(module, N, layer_share=False): - # import ipdb; ipdb.set_trace() - if layer_share: - return nn.ModuleList([module for i in range(N)]) - else: - return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) - - -def get_sine_pos_embed( - pos_tensor: torch.Tensor, - num_pos_feats: int = 128, - temperature: int = 10000, - exchange_xy: bool = True, -): - """generate sine position embedding from a position tensor - Args: - pos_tensor (torch.Tensor): shape: [..., n]. - num_pos_feats (int): projected shape for each float in the tensor. - temperature (int): temperature in the sine/cosine function. - exchange_xy (bool, optional): exchange pos x and pos y. \ - For example, input tensor is [x,y], the results will be [pos(y), pos(x)]. Defaults to True. - Returns: - pos_embed (torch.Tensor): shape: [..., n*num_pos_feats]. - """ - scale = 2 * math.pi - dim_t = torch.arange(num_pos_feats, dtype=torch.float32, device=pos_tensor.device) - dim_t = temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / num_pos_feats) - - def sine_func(x: torch.Tensor): - sin_x = x * scale / dim_t - sin_x = torch.stack((sin_x[..., 0::2].sin(), sin_x[..., 1::2].cos()), dim=3).flatten(2) - return sin_x - - pos_res = [sine_func(x) for x in pos_tensor.split([1] * pos_tensor.shape[-1], dim=-1)] - if exchange_xy: - pos_res[0], pos_res[1] = pos_res[1], pos_res[0] - pos_res = torch.cat(pos_res, dim=-1) - return pos_res - - -def gen_encoder_output_proposals( - memory: Tensor, memory_padding_mask: Tensor, spatial_shapes: Tensor, learnedwh=None -): - """ - Input: - - memory: bs, \sum{hw}, d_model - - memory_padding_mask: bs, \sum{hw} - - spatial_shapes: nlevel, 2 - - learnedwh: 2 - Output: - - output_memory: bs, \sum{hw}, d_model - - output_proposals: bs, \sum{hw}, 4 - """ - N_, S_, C_ = memory.shape - proposals = [] - _cur = 0 - for lvl, (H_, W_) in enumerate(spatial_shapes): - mask_flatten_ = memory_padding_mask[:, _cur : (_cur + H_ * W_)].view(N_, H_, W_, 1) - valid_H = torch.sum(~mask_flatten_[:, :, 0, 0], 1) - valid_W = torch.sum(~mask_flatten_[:, 0, :, 0], 1) - - # import ipdb; ipdb.set_trace() - - grid_y, grid_x = torch.meshgrid( - torch.linspace(0, H_ - 1, H_, dtype=torch.float32, device=memory.device), - torch.linspace(0, W_ - 1, W_, dtype=torch.float32, device=memory.device), - ) - grid = torch.cat([grid_x.unsqueeze(-1), grid_y.unsqueeze(-1)], -1) # H_, W_, 2 - - scale = torch.cat([valid_W.unsqueeze(-1), valid_H.unsqueeze(-1)], 1).view(N_, 1, 1, 2) - grid = (grid.unsqueeze(0).expand(N_, -1, -1, -1) + 0.5) / scale - - if learnedwh is not None: - # import ipdb; ipdb.set_trace() - wh = torch.ones_like(grid) * learnedwh.sigmoid() * (2.0**lvl) - else: - wh = torch.ones_like(grid) * 0.05 * (2.0**lvl) - - # scale = torch.cat([W_[None].unsqueeze(-1), H_[None].unsqueeze(-1)], 1).view(1, 1, 1, 2).repeat(N_, 1, 1, 1) - # grid = (grid.unsqueeze(0).expand(N_, -1, -1, -1) + 0.5) / scale - # wh = torch.ones_like(grid) / scale - proposal = torch.cat((grid, wh), -1).view(N_, -1, 4) - proposals.append(proposal) - _cur += H_ * W_ - # import ipdb; ipdb.set_trace() - output_proposals = torch.cat(proposals, 1) - output_proposals_valid = ((output_proposals > 0.01) & (output_proposals < 0.99)).all( - -1, keepdim=True - ) - output_proposals = torch.log(output_proposals / (1 - output_proposals)) # unsigmoid - output_proposals = output_proposals.masked_fill(memory_padding_mask.unsqueeze(-1), float("inf")) - output_proposals = output_proposals.masked_fill(~output_proposals_valid, float("inf")) - - output_memory = memory - output_memory = output_memory.masked_fill(memory_padding_mask.unsqueeze(-1), float(0)) - output_memory = output_memory.masked_fill(~output_proposals_valid, float(0)) - - # output_memory = output_memory.masked_fill(memory_padding_mask.unsqueeze(-1), float('inf')) - # output_memory = output_memory.masked_fill(~output_proposals_valid, float('inf')) - - return output_memory, output_proposals - - -class RandomBoxPerturber: - def __init__( - self, x_noise_scale=0.2, y_noise_scale=0.2, w_noise_scale=0.2, h_noise_scale=0.2 - ) -> None: - self.noise_scale = torch.Tensor( - [x_noise_scale, y_noise_scale, w_noise_scale, h_noise_scale] - ) - - def __call__(self, refanchors: Tensor) -> Tensor: - nq, bs, query_dim = refanchors.shape - device = refanchors.device - - noise_raw = torch.rand_like(refanchors) - noise_scale = self.noise_scale.to(device)[:query_dim] - - new_refanchors = refanchors * (1 + (noise_raw - 0.5) * noise_scale) - return new_refanchors.clamp_(0, 1) - - -def sigmoid_focal_loss( - inputs, targets, num_boxes, alpha: float = 0.25, gamma: float = 2, no_reduction=False -): - """ - Loss used in RetinaNet for dense detection: https://arxiv.org/abs/1708.02002. - Args: - inputs: A float tensor of arbitrary shape. - The predictions for each example. - targets: A float tensor with the same shape as inputs. Stores the binary - classification label for each element in inputs - (0 for the negative class and 1 for the positive class). - alpha: (optional) Weighting factor in range (0,1) to balance - positive vs negative examples. Default = -1 (no weighting). - gamma: Exponent of the modulating factor (1 - p_t) to - balance easy vs hard examples. - Returns: - Loss tensor - """ - prob = inputs.sigmoid() - ce_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction="none") - p_t = prob * targets + (1 - prob) * (1 - targets) - loss = ce_loss * ((1 - p_t) ** gamma) - - if alpha >= 0: - alpha_t = alpha * targets + (1 - alpha) * (1 - targets) - loss = alpha_t * loss - - if no_reduction: - return loss - - return loss.mean(1).sum() / num_boxes - - -class MLP(nn.Module): - """Very simple multi-layer perceptron (also called FFN)""" - - def __init__(self, input_dim, hidden_dim, output_dim, num_layers): - super().__init__() - r = 12 - self.num_layers = num_layers - h = [hidden_dim] * (num_layers - 1) - self.layers = nn.ModuleList( - lora.Linear(n, k , r=r) for n, k in zip([input_dim] + h, h + [output_dim]) - ) - - def forward(self, x): - for i, layer in enumerate(self.layers): - x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x) - return x - - -def _get_activation_fn(activation, d_model=256, batch_dim=0): - """Return an activation function given a string""" - if activation == "relu": - return F.relu - if activation == "gelu": - return F.gelu - if activation == "glu": - return F.glu - if activation == "prelu": - return nn.PReLU() - if activation == "selu": - return F.selu - - raise RuntimeError(f"activation should be relu/gelu, not {activation}.") - - -def gen_sineembed_for_position(pos_tensor): - # n_query, bs, _ = pos_tensor.size() - # sineembed_tensor = torch.zeros(n_query, bs, 256) - scale = 2 * math.pi - dim_t = torch.arange(128, dtype=torch.float32, device=pos_tensor.device) - dim_t = 10000 ** (2 * (torch.div(dim_t, 2, rounding_mode='floor')) / 128) - x_embed = pos_tensor[:, :, 0] * scale - y_embed = pos_tensor[:, :, 1] * scale - pos_x = x_embed[:, :, None] / dim_t - pos_y = y_embed[:, :, None] / dim_t - pos_x = torch.stack((pos_x[:, :, 0::2].sin(), pos_x[:, :, 1::2].cos()), dim=3).flatten(2) - pos_y = torch.stack((pos_y[:, :, 0::2].sin(), pos_y[:, :, 1::2].cos()), dim=3).flatten(2) - if pos_tensor.size(-1) == 2: - pos = torch.cat((pos_y, pos_x), dim=2) - elif pos_tensor.size(-1) == 4: - w_embed = pos_tensor[:, :, 2] * scale - pos_w = w_embed[:, :, None] / dim_t - pos_w = torch.stack((pos_w[:, :, 0::2].sin(), pos_w[:, :, 1::2].cos()), dim=3).flatten(2) - - h_embed = pos_tensor[:, :, 3] * scale - pos_h = h_embed[:, :, None] / dim_t - pos_h = torch.stack((pos_h[:, :, 0::2].sin(), pos_h[:, :, 1::2].cos()), dim=3).flatten(2) - - pos = torch.cat((pos_y, pos_x, pos_w, pos_h), dim=2) - else: - raise ValueError("Unknown pos_tensor shape(-1):{}".format(pos_tensor.size(-1))) - return pos - - -class ContrastiveEmbed(nn.Module): - def __init__(self, max_text_len=256): - """ - Args: - max_text_len: max length of text. - """ - super().__init__() - self.max_text_len = max_text_len - - def forward(self, x, text_dict): - """_summary_ - - Args: - x (_type_): _description_ - text_dict (_type_): _description_ - { - 'encoded_text': encoded_text, # bs, 195, d_model - 'text_token_mask': text_token_mask, # bs, 195 - # True for used tokens. False for padding tokens - } - Returns: - _type_: _description_ - """ - assert isinstance(text_dict, dict) - # print(x) #torch.Size([2, 16320, 256]) - # print(text_dict) - - # import pdb;pdb.set_trace() - y = text_dict["encoded_text"] #torch.Size([2, 195, 256]) - text_token_mask = text_dict["text_token_mask"] - - res = x @ y.transpose(-1, -2) - res.masked_fill_(~text_token_mask[:, None, :], float("-inf")) - # 接着，对res进行掩码操作，将未使用的文本token（即padding的token）对应的得分置为负无穷float("-inf")。这是为了在计算相似度时，排除padding部分的影响。 - - - # padding to max_text_len - new_res = torch.full((*res.shape[:-1], self.max_text_len), float("-inf"), device=res.device) - new_res[..., : res.shape[-1]] = res #torch.Size([2, 16320, 195]) - - return new_res diff --git a/groundingdino/models/__init__.py b/groundingdino/models/__init__.py deleted file mode 100644 index 8b123bbf7ac35014e68a70c2056f1325389f0ecc..0000000000000000000000000000000000000000 --- a/groundingdino/models/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# ------------------------------------------------------------------------ -# DINO -# Copyright (c) 2022 IDEA. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -from .GroundingDINO import build_groundingdino - -def build_model(args): - return build(args) diff --git a/groundingdino/models/__pycache__/__init__.cpython-310.pyc b/groundingdino/models/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 1a1b0c87c8031e520d5870a9135ae067ad890b6d..0000000000000000000000000000000000000000 Binary files a/groundingdino/models/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/models/__pycache__/registry.cpython-310.pyc b/groundingdino/models/__pycache__/registry.cpython-310.pyc deleted file mode 100644 index ee952834e314615a5725499dde2e4651d0fec831..0000000000000000000000000000000000000000 Binary files a/groundingdino/models/__pycache__/registry.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/models/registry.py b/groundingdino/models/registry.py deleted file mode 100644 index f438c6e3918a84cc2004b5da9c1d79d18cfb3118..0000000000000000000000000000000000000000 --- a/groundingdino/models/registry.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- -# @Author: Yihao Chen -# @Date: 2021-08-16 16:03:17 -# @Last Modified by: Shilong Liu -# @Last Modified time: 2022-01-23 15:26 -# modified from mmcv - -import inspect -from functools import partial - - -class Registry(object): - - def __init__(self, name): - self._name = name - self._module_dict = dict() - - def __repr__(self): - format_str = self.__class__.__name__ + '(name={}, items={})'.format( - self._name, list(self._module_dict.keys())) - return format_str - - def __len__(self): - return len(self._module_dict) - - @property - def name(self): - return self._name - - @property - def module_dict(self): - return self._module_dict - - def get(self, key): - return self._module_dict.get(key, None) - - def registe_with_name(self, module_name=None, force=False): - return partial(self.register, module_name=module_name, force=force) - - def register(self, module_build_function, module_name=None, force=False): - """Register a module build function. - Args: - module (:obj:`nn.Module`): Module to be registered. - """ - if not inspect.isfunction(module_build_function): - raise TypeError('module_build_function must be a function, but got {}'.format( - type(module_build_function))) - if module_name is None: - module_name = module_build_function.__name__ - if not force and module_name in self._module_dict: - raise KeyError('{} is already registered in {}'.format( - module_name, self.name)) - self._module_dict[module_name] = module_build_function - - return module_build_function - -MODULE_BUILD_FUNCS = Registry('model build functions') - diff --git a/groundingdino/util/.ipynb_checkpoints/__init__-checkpoint.py b/groundingdino/util/.ipynb_checkpoints/__init__-checkpoint.py deleted file mode 100644 index 168f9979a4623806934b0ff1102ac166704e7dec..0000000000000000000000000000000000000000 --- a/groundingdino/util/.ipynb_checkpoints/__init__-checkpoint.py +++ /dev/null @@ -1 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved diff --git a/groundingdino/util/.ipynb_checkpoints/get_tokenlizer-checkpoint.py b/groundingdino/util/.ipynb_checkpoints/get_tokenlizer-checkpoint.py deleted file mode 100644 index 79faf9ed22659249e0123a9f2d8d0d2f646ee8f4..0000000000000000000000000000000000000000 --- a/groundingdino/util/.ipynb_checkpoints/get_tokenlizer-checkpoint.py +++ /dev/null @@ -1,30 +0,0 @@ -from transformers import AutoTokenizer, BertModel, BertTokenizer, RobertaModel, RobertaTokenizerFast -import os - -def get_tokenlizer(text_encoder_type): - if not isinstance(text_encoder_type, str): - # print("text_encoder_type is not a str") - if hasattr(text_encoder_type, "text_encoder_type"): - text_encoder_type = text_encoder_type.text_encoder_type - elif text_encoder_type.get("text_encoder_type", False): - text_encoder_type = text_encoder_type.get("text_encoder_type") - elif os.path.isdir(text_encoder_type) and os.path.exists(text_encoder_type): - pass - else: - raise ValueError( - "Unknown type of text_encoder_type: {}".format(type(text_encoder_type)) - ) - print("final text_encoder_type: {}".format(text_encoder_type)) - tokenizer = AutoTokenizer.from_pretrained(text_encoder_type) - print("load tokenizer done.") - return tokenizer - - -def get_pretrained_language_model(text_encoder_type): - if text_encoder_type == "bert-base-uncased" or (os.path.isdir(text_encoder_type) and os.path.exists(text_encoder_type)): - return BertModel.from_pretrained(text_encoder_type) - if text_encoder_type == "roberta-base": - return RobertaModel.from_pretrained(text_encoder_type) - - - raise ValueError("Unknown text_encoder_type {}".format(text_encoder_type)) diff --git a/groundingdino/util/.ipynb_checkpoints/utils-checkpoint.py b/groundingdino/util/.ipynb_checkpoints/utils-checkpoint.py deleted file mode 100644 index 8cf83ae03a7865bc48493be16e8b1b2d53a1b09f..0000000000000000000000000000000000000000 --- a/groundingdino/util/.ipynb_checkpoints/utils-checkpoint.py +++ /dev/null @@ -1,610 +0,0 @@ -import argparse -import json -import warnings -from collections import OrderedDict -from copy import deepcopy -from typing import Any, Dict, List - -import numpy as np -import torch -from transformers import AutoTokenizer - -from groundingdino.util.slconfig import SLConfig - - -def slprint(x, name="x"): - if isinstance(x, (torch.Tensor, np.ndarray)): - print(f"{name}.shape:", x.shape) - elif isinstance(x, (tuple, list)): - print("type x:", type(x)) - for i in range(min(10, len(x))): - slprint(x[i], f"{name}[{i}]") - elif isinstance(x, dict): - for k, v in x.items(): - slprint(v, f"{name}[{k}]") - else: - print(f"{name}.type:", type(x)) - - -def clean_state_dict(state_dict): - new_state_dict = OrderedDict() - for k, v in state_dict.items(): - if k[:7] == "module.": - k = k[7:] # remove `module.` - new_state_dict[k] = v - return new_state_dict - - -def renorm( - img: torch.FloatTensor, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] -) -> torch.FloatTensor: - # img: tensor(3,H,W) or tensor(B,3,H,W) - # return: same as img - assert img.dim() == 3 or img.dim() == 4, "img.dim() should be 3 or 4 but %d" % img.dim() - if img.dim() == 3: - assert img.size(0) == 3, 'img.size(0) shoule be 3 but "%d". (%s)' % ( - img.size(0), - str(img.size()), - ) - img_perm = img.permute(1, 2, 0) - mean = torch.Tensor(mean) - std = torch.Tensor(std) - img_res = img_perm * std + mean - return img_res.permute(2, 0, 1) - else: # img.dim() == 4 - assert img.size(1) == 3, 'img.size(1) shoule be 3 but "%d". (%s)' % ( - img.size(1), - str(img.size()), - ) - img_perm = img.permute(0, 2, 3, 1) - mean = torch.Tensor(mean) - std = torch.Tensor(std) - img_res = img_perm * std + mean - return img_res.permute(0, 3, 1, 2) - - -class CocoClassMapper: - def __init__(self) -> None: - self.category_map_str = { - "1": 1, - "2": 2, - "3": 3, - "4": 4, - "5": 5, - "6": 6, - "7": 7, - "8": 8, - "9": 9, - "10": 10, - "11": 11, - "13": 12, - "14": 13, - "15": 14, - "16": 15, - "17": 16, - "18": 17, - "19": 18, - "20": 19, - "21": 20, - "22": 21, - "23": 22, - "24": 23, - "25": 24, - "27": 25, - "28": 26, - "31": 27, - "32": 28, - "33": 29, - "34": 30, - "35": 31, - "36": 32, - "37": 33, - "38": 34, - "39": 35, - "40": 36, - "41": 37, - "42": 38, - "43": 39, - "44": 40, - "46": 41, - "47": 42, - "48": 43, - "49": 44, - "50": 45, - "51": 46, - "52": 47, - "53": 48, - "54": 49, - "55": 50, - "56": 51, - "57": 52, - "58": 53, - "59": 54, - "60": 55, - "61": 56, - "62": 57, - "63": 58, - "64": 59, - "65": 60, - "67": 61, - "70": 62, - "72": 63, - "73": 64, - "74": 65, - "75": 66, - "76": 67, - "77": 68, - "78": 69, - "79": 70, - "80": 71, - "81": 72, - "82": 73, - "84": 74, - "85": 75, - "86": 76, - "87": 77, - "88": 78, - "89": 79, - "90": 80, - } - self.origin2compact_mapper = {int(k): v - 1 for k, v in self.category_map_str.items()} - self.compact2origin_mapper = {int(v - 1): int(k) for k, v in self.category_map_str.items()} - - def origin2compact(self, idx): - return self.origin2compact_mapper[int(idx)] - - def compact2origin(self, idx): - return self.compact2origin_mapper[int(idx)] - - -def to_device(item, device): - if isinstance(item, torch.Tensor): - return item.to(device) - elif isinstance(item, list): - return [to_device(i, device) for i in item] - elif isinstance(item, dict): - return {k: to_device(v, device) for k, v in item.items()} - else: - raise NotImplementedError( - "Call Shilong if you use other containers! type: {}".format(type(item)) - ) - - -# -def get_gaussian_mean(x, axis, other_axis, softmax=True): - """ - - Args: - x (float): Input images(BxCxHxW) - axis (int): The index for weighted mean - other_axis (int): The other index - - Returns: weighted index for axis, BxC - - """ - mat2line = torch.sum(x, axis=other_axis) - # mat2line = mat2line / mat2line.mean() * 10 - if softmax: - u = torch.softmax(mat2line, axis=2) - else: - u = mat2line / (mat2line.sum(2, keepdim=True) + 1e-6) - size = x.shape[axis] - ind = torch.linspace(0, 1, size).to(x.device) - batch = x.shape[0] - channel = x.shape[1] - index = ind.repeat([batch, channel, 1]) - mean_position = torch.sum(index * u, dim=2) - return mean_position - - -def get_expected_points_from_map(hm, softmax=True): - """get_gaussian_map_from_points - B,C,H,W -> B,N,2 float(0, 1) float(0, 1) - softargmax function - - Args: - hm (float): Input images(BxCxHxW) - - Returns: - weighted index for axis, BxCx2. float between 0 and 1. - - """ - # hm = 10*hm - B, C, H, W = hm.shape - y_mean = get_gaussian_mean(hm, 2, 3, softmax=softmax) # B,C - x_mean = get_gaussian_mean(hm, 3, 2, softmax=softmax) # B,C - # return torch.cat((x_mean.unsqueeze(-1), y_mean.unsqueeze(-1)), 2) - return torch.stack([x_mean, y_mean], dim=2) - - -# Positional encoding (section 5.1) -# borrow from nerf -class Embedder: - def __init__(self, **kwargs): - self.kwargs = kwargs - self.create_embedding_fn() - - def create_embedding_fn(self): - embed_fns = [] - d = self.kwargs["input_dims"] - out_dim = 0 - if self.kwargs["include_input"]: - embed_fns.append(lambda x: x) - out_dim += d - - max_freq = self.kwargs["max_freq_log2"] - N_freqs = self.kwargs["num_freqs"] - - if self.kwargs["log_sampling"]: - freq_bands = 2.0 ** torch.linspace(0.0, max_freq, steps=N_freqs) - else: - freq_bands = torch.linspace(2.0**0.0, 2.0**max_freq, steps=N_freqs) - - for freq in freq_bands: - for p_fn in self.kwargs["periodic_fns"]: - embed_fns.append(lambda x, p_fn=p_fn, freq=freq: p_fn(x * freq)) - out_dim += d - - self.embed_fns = embed_fns - self.out_dim = out_dim - - def embed(self, inputs): - return torch.cat([fn(inputs) for fn in self.embed_fns], -1) - - -def get_embedder(multires, i=0): - import torch.nn as nn - - if i == -1: - return nn.Identity(), 3 - - embed_kwargs = { - "include_input": True, - "input_dims": 3, - "max_freq_log2": multires - 1, - "num_freqs": multires, - "log_sampling": True, - "periodic_fns": [torch.sin, torch.cos], - } - - embedder_obj = Embedder(**embed_kwargs) - embed = lambda x, eo=embedder_obj: eo.embed(x) - return embed, embedder_obj.out_dim - - -class APOPMeter: - def __init__(self) -> None: - self.tp = 0 - self.fp = 0 - self.tn = 0 - self.fn = 0 - - def update(self, pred, gt): - """ - Input: - pred, gt: Tensor() - """ - assert pred.shape == gt.shape - self.tp += torch.logical_and(pred == 1, gt == 1).sum().item() - self.fp += torch.logical_and(pred == 1, gt == 0).sum().item() - self.tn += torch.logical_and(pred == 0, gt == 0).sum().item() - self.tn += torch.logical_and(pred == 1, gt == 0).sum().item() - - def update_cm(self, tp, fp, tn, fn): - self.tp += tp - self.fp += fp - self.tn += tn - self.tn += fn - - -def inverse_sigmoid(x, eps=1e-5): - x = x.clamp(min=0, max=1) - x1 = x.clamp(min=eps) - x2 = (1 - x).clamp(min=eps) - return torch.log(x1 / x2) - - -def get_raw_dict(args): - """ - return the dicf contained in args. - - e.g: - >>> with open(path, 'w') as f: - json.dump(get_raw_dict(args), f, indent=2) - """ - if isinstance(args, argparse.Namespace): - return vars(args) - elif isinstance(args, dict): - return args - elif isinstance(args, SLConfig): - return args._cfg_dict - else: - raise NotImplementedError("Unknown type {}".format(type(args))) - - -def stat_tensors(tensor): - assert tensor.dim() == 1 - tensor_sm = tensor.softmax(0) - entropy = (tensor_sm * torch.log(tensor_sm + 1e-9)).sum() - - return { - "max": tensor.max(), - "min": tensor.min(), - "mean": tensor.mean(), - "var": tensor.var(), - "std": tensor.var() ** 0.5, - "entropy": entropy, - } - - -class NiceRepr: - """Inherit from this class and define ``__nice__`` to "nicely" print your - objects. - - Defines ``__str__`` and ``__repr__`` in terms of ``__nice__`` function - Classes that inherit from :class:`NiceRepr` should redefine ``__nice__``. - If the inheriting class has a ``__len__``, method then the default - ``__nice__`` method will return its length. - - Example: - >>> class Foo(NiceRepr): - ... def __nice__(self): - ... return 'info' - >>> foo = Foo() - >>> assert str(foo) == '' - >>> assert repr(foo).startswith('>> class Bar(NiceRepr): - ... pass - >>> bar = Bar() - >>> import pytest - >>> with pytest.warns(None) as record: - >>> assert 'object at' in str(bar) - >>> assert 'object at' in repr(bar) - - Example: - >>> class Baz(NiceRepr): - ... def __len__(self): - ... return 5 - >>> baz = Baz() - >>> assert str(baz) == '' - """ - - def __nice__(self): - """str: a "nice" summary string describing this module""" - if hasattr(self, "__len__"): - # It is a common pattern for objects to use __len__ in __nice__ - # As a convenience we define a default __nice__ for these objects - return str(len(self)) - else: - # In all other cases force the subclass to overload __nice__ - raise NotImplementedError(f"Define the __nice__ method for {self.__class__!r}") - - def __repr__(self): - """str: the string of the module""" - try: - nice = self.__nice__() - classname = self.__class__.__name__ - return f"<{classname}({nice}) at {hex(id(self))}>" - except NotImplementedError as ex: - warnings.warn(str(ex), category=RuntimeWarning) - return object.__repr__(self) - - def __str__(self): - """str: the string of the module""" - try: - classname = self.__class__.__name__ - nice = self.__nice__() - return f"<{classname}({nice})>" - except NotImplementedError as ex: - warnings.warn(str(ex), category=RuntimeWarning) - return object.__repr__(self) - - -def ensure_rng(rng=None): - """Coerces input into a random number generator. - - If the input is None, then a global random state is returned. - - If the input is a numeric value, then that is used as a seed to construct a - random state. Otherwise the input is returned as-is. - - Adapted from [1]_. - - Args: - rng (int | numpy.random.RandomState | None): - if None, then defaults to the global rng. Otherwise this can be an - integer or a RandomState class - Returns: - (numpy.random.RandomState) : rng - - a numpy random number generator - - References: - .. [1] https://gitlab.kitware.com/computer-vision/kwarray/blob/master/kwarray/util_random.py#L270 # noqa: E501 - """ - - if rng is None: - rng = np.random.mtrand._rand - elif isinstance(rng, int): - rng = np.random.RandomState(rng) - else: - rng = rng - return rng - - -def random_boxes(num=1, scale=1, rng=None): - """Simple version of ``kwimage.Boxes.random`` - - Returns: - Tensor: shape (n, 4) in x1, y1, x2, y2 format. - - References: - https://gitlab.kitware.com/computer-vision/kwimage/blob/master/kwimage/structs/boxes.py#L1390 - - Example: - >>> num = 3 - >>> scale = 512 - >>> rng = 0 - >>> boxes = random_boxes(num, scale, rng) - >>> print(boxes) - tensor([[280.9925, 278.9802, 308.6148, 366.1769], - [216.9113, 330.6978, 224.0446, 456.5878], - [405.3632, 196.3221, 493.3953, 270.7942]]) - """ - rng = ensure_rng(rng) - - tlbr = rng.rand(num, 4).astype(np.float32) - - tl_x = np.minimum(tlbr[:, 0], tlbr[:, 2]) - tl_y = np.minimum(tlbr[:, 1], tlbr[:, 3]) - br_x = np.maximum(tlbr[:, 0], tlbr[:, 2]) - br_y = np.maximum(tlbr[:, 1], tlbr[:, 3]) - - tlbr[:, 0] = tl_x * scale - tlbr[:, 1] = tl_y * scale - tlbr[:, 2] = br_x * scale - tlbr[:, 3] = br_y * scale - - boxes = torch.from_numpy(tlbr) - return boxes - - -class ModelEma(torch.nn.Module): - def __init__(self, model, decay=0.9997, device=None): - super(ModelEma, self).__init__() - # make a copy of the model for accumulating moving average of weights - self.module = deepcopy(model) - self.module.eval() - - # import ipdb; ipdb.set_trace() - - self.decay = decay - self.device = device # perform ema on different device from model if set - if self.device is not None: - self.module.to(device=device) - - def _update(self, model, update_fn): - with torch.no_grad(): - for ema_v, model_v in zip( - self.module.state_dict().values(), model.state_dict().values() - ): - if self.device is not None: - model_v = model_v.to(device=self.device) - ema_v.copy_(update_fn(ema_v, model_v)) - - def update(self, model): - self._update(model, update_fn=lambda e, m: self.decay * e + (1.0 - self.decay) * m) - - def set(self, model): - self._update(model, update_fn=lambda e, m: m) - - -class BestMetricSingle: - def __init__(self, init_res=0.0, better="large") -> None: - self.init_res = init_res - self.best_res = init_res - self.best_ep = -1 - - self.better = better - assert better in ["large", "small"] - - def isbetter(self, new_res, old_res): - if self.better == "large": - return new_res > old_res - if self.better == "small": - return new_res < old_res - - def update(self, new_res, ep): - if self.isbetter(new_res, self.best_res): - self.best_res = new_res - self.best_ep = ep - return True - return False - - def __str__(self) -> str: - return "best_res: {}\t best_ep: {}".format(self.best_res, self.best_ep) - - def __repr__(self) -> str: - return self.__str__() - - def summary(self) -> dict: - return { - "best_res": self.best_res, - "best_ep": self.best_ep, - } - - -class BestMetricHolder: - def __init__(self, init_res=0.0, better="large", use_ema=False) -> None: - self.best_all = BestMetricSingle(init_res, better) - self.use_ema = use_ema - if use_ema: - self.best_ema = BestMetricSingle(init_res, better) - self.best_regular = BestMetricSingle(init_res, better) - - def update(self, new_res, epoch, is_ema=False): - """ - return if the results is the best. - """ - if not self.use_ema: - return self.best_all.update(new_res, epoch) - else: - if is_ema: - self.best_ema.update(new_res, epoch) - return self.best_all.update(new_res, epoch) - else: - self.best_regular.update(new_res, epoch) - return self.best_all.update(new_res, epoch) - - def summary(self): - if not self.use_ema: - return self.best_all.summary() - - res = {} - res.update({f"all_{k}": v for k, v in self.best_all.summary().items()}) - res.update({f"regular_{k}": v for k, v in self.best_regular.summary().items()}) - res.update({f"ema_{k}": v for k, v in self.best_ema.summary().items()}) - return res - - def __repr__(self) -> str: - return json.dumps(self.summary(), indent=2) - - def __str__(self) -> str: - return self.__repr__() - - -def targets_to(targets: List[Dict[str, Any]], device): - """Moves the target dicts to the given device.""" - excluded_keys = [ - "questionId", - "tokens_positive", - "strings_positive", - "tokens", - "dataset_name", - "sentence_id", - "original_img_id", - "nb_eval", - "task_id", - "original_id", - "token_span", - "caption", - "dataset_type", - ] - return [ - {k: v.to(device) if k not in excluded_keys else v for k, v in t.items()} for t in targets - ] - - -def get_phrases_from_posmap( - posmap: torch.BoolTensor, tokenized: Dict, tokenizer: AutoTokenizer, left_idx: int = 0, right_idx: int = 255 -): - assert isinstance(posmap, torch.Tensor), "posmap must be torch.Tensor" - if posmap.dim() == 1: - posmap[0: left_idx + 1] = False - posmap[right_idx:] = False - non_zero_idx = posmap.nonzero(as_tuple=True)[0].tolist() - token_ids = [tokenized["input_ids"][i] for i in non_zero_idx] - return tokenizer.decode(token_ids) - else: - raise NotImplementedError("posmap must be 1-dim") diff --git a/groundingdino/util/__pycache__/__init__.cpython-310.pyc b/groundingdino/util/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 38bf0959c9426666610e97bf60a72072385ecc9b..0000000000000000000000000000000000000000 Binary files a/groundingdino/util/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/util/__pycache__/box_ops.cpython-310.pyc b/groundingdino/util/__pycache__/box_ops.cpython-310.pyc deleted file mode 100644 index 026e62a90ab0ed87d2cbd756fa24b381102673c8..0000000000000000000000000000000000000000 Binary files a/groundingdino/util/__pycache__/box_ops.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/util/__pycache__/get_tokenlizer.cpython-310.pyc b/groundingdino/util/__pycache__/get_tokenlizer.cpython-310.pyc deleted file mode 100644 index 7722834bfce06e6fc03c8f0d5efe99a58a92e695..0000000000000000000000000000000000000000 Binary files a/groundingdino/util/__pycache__/get_tokenlizer.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/util/__pycache__/misc.cpython-310.pyc b/groundingdino/util/__pycache__/misc.cpython-310.pyc deleted file mode 100644 index 5b30db6917c7bccc53e77c795bf62eb748eaecac..0000000000000000000000000000000000000000 Binary files a/groundingdino/util/__pycache__/misc.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/util/__pycache__/slconfig.cpython-310.pyc b/groundingdino/util/__pycache__/slconfig.cpython-310.pyc deleted file mode 100644 index fd3ccd8d8091fac42b5cdb5b568fcef6acee416a..0000000000000000000000000000000000000000 Binary files a/groundingdino/util/__pycache__/slconfig.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/util/__pycache__/utils.cpython-310.pyc b/groundingdino/util/__pycache__/utils.cpython-310.pyc deleted file mode 100644 index b458a90ee4aaafffc0b69ea45d7b69b07c89c4f8..0000000000000000000000000000000000000000 Binary files a/groundingdino/util/__pycache__/utils.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/util/__pycache__/visualizer.cpython-310.pyc b/groundingdino/util/__pycache__/visualizer.cpython-310.pyc deleted file mode 100644 index 75fe4bac2f8a2f8910213693a11383505e15e0fe..0000000000000000000000000000000000000000 Binary files a/groundingdino/util/__pycache__/visualizer.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/util/__pycache__/vl_utils.cpython-310.pyc b/groundingdino/util/__pycache__/vl_utils.cpython-310.pyc deleted file mode 100644 index 6ed2e5026bf16aa26ab2d0fb7d6acd348ca78dbd..0000000000000000000000000000000000000000 Binary files a/groundingdino/util/__pycache__/vl_utils.cpython-310.pyc and /dev/null differ diff --git a/groundingdino/util/get_tokenlizer.py b/groundingdino/util/get_tokenlizer.py deleted file mode 100644 index 79faf9ed22659249e0123a9f2d8d0d2f646ee8f4..0000000000000000000000000000000000000000 --- a/groundingdino/util/get_tokenlizer.py +++ /dev/null @@ -1,30 +0,0 @@ -from transformers import AutoTokenizer, BertModel, BertTokenizer, RobertaModel, RobertaTokenizerFast -import os - -def get_tokenlizer(text_encoder_type): - if not isinstance(text_encoder_type, str): - # print("text_encoder_type is not a str") - if hasattr(text_encoder_type, "text_encoder_type"): - text_encoder_type = text_encoder_type.text_encoder_type - elif text_encoder_type.get("text_encoder_type", False): - text_encoder_type = text_encoder_type.get("text_encoder_type") - elif os.path.isdir(text_encoder_type) and os.path.exists(text_encoder_type): - pass - else: - raise ValueError( - "Unknown type of text_encoder_type: {}".format(type(text_encoder_type)) - ) - print("final text_encoder_type: {}".format(text_encoder_type)) - tokenizer = AutoTokenizer.from_pretrained(text_encoder_type) - print("load tokenizer done.") - return tokenizer - - -def get_pretrained_language_model(text_encoder_type): - if text_encoder_type == "bert-base-uncased" or (os.path.isdir(text_encoder_type) and os.path.exists(text_encoder_type)): - return BertModel.from_pretrained(text_encoder_type) - if text_encoder_type == "roberta-base": - return RobertaModel.from_pretrained(text_encoder_type) - - - raise ValueError("Unknown text_encoder_type {}".format(text_encoder_type))