def get_clip_relevance(model: nn.Module, pil_img: Image, text_list: List[str], device: str, vis_start_layer: int = 11,
                       text_start_layer: int = 11, transform: Callable = None, tokenize: Callable = None) -> Tuple:
    """
    Get text relevance and image relevance from CLIP model.
    Args:
        model (`nn.Module`):
            CLIP model to visualize.
        pil_img (`Image`):
            Input image.
        text_list (`List[str]`):
            List of text str.
        device (`str`):
            Device to use.
        vis_start_layer (`int`):
            Start layer for visualization.
        text_start_layer (`int`):
            Start layer for text.
        transform (`Callable`):
            Transform function for image.
        tokenize (`Callable`):
            Tokenize function for text.

    Returns:
        (`Tuple`):
            text_relevance, image_relevance, text_tokens, img_tensor
    """
    if transform is None:
        def _transform(n_px):
            return Compose([
                Resize(n_px, interpolation=Image.BICUBIC),
                CenterCrop(n_px),
                lambda image: image.convert('RGB'),
                ToTensor(),
                Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)),
            ])

        transform = _transform(224)
    if tokenize is None:
        tokenize = clip.tokenize

    img_tensor = transform(pil_img).unsqueeze(0).to(device)
    text_tokens = tokenize(text_list).to(device)
    batch_size = text_tokens.shape[0]
    img_tensors = img_tensor.repeat(batch_size, 1, 1, 1)
    logits_per_image, _ = model(img_tensors, text_tokens, device=device)
    index = list(range(batch_size))
    one_hot = np.zeros((logits_per_image.shape[0], logits_per_image.shape[1]), dtype=np.float32)
    one_hot[torch.arange(logits_per_image.shape[0]), index] = 1
    one_hot = torch.from_numpy(one_hot).requires_grad_(True)
    one_hot = torch.sum(one_hot.to(device) * logits_per_image)
    model.zero_grad()

    image_attn_blocks = list(dict(model.visual.transformer.resblocks.named_children()).values())
    num_tokens = image_attn_blocks[0].attn_probs.shape[-1]
    relevance = torch.eye(num_tokens, num_tokens, dtype=image_attn_blocks[0].attn_probs.dtype).to(device)
    relevance = relevance.unsqueeze(0).expand(batch_size, num_tokens, num_tokens)
    for i, blk in enumerate(image_attn_blocks):
        if i < vis_start_layer:
            continue
        grad = torch.autograd.grad(one_hot, [blk.attn_probs], retain_graph=True)[0].detach()
        cam = blk.attn_probs.detach()
        cam = cam.reshape(-1, cam.shape[-1], cam.shape[-1])
        grad = grad.reshape(-1, grad.shape[-1], grad.shape[-1])
        cam = grad * cam
        cam = cam.reshape(batch_size, -1, cam.shape[-1], cam.shape[-1])
        cam = cam.clamp(min=0).mean(dim=1)
        relevance = relevance + torch.bmm(cam, relevance)
    image_relevance = relevance[:, 0, 1:]

    text_attn_blocks = list(dict(model.transformer.resblocks.named_children()).values())
    num_tokens = text_attn_blocks[0].attn_probs.shape[-1]
    rel_text = torch.eye(num_tokens, num_tokens, dtype=text_attn_blocks[0].attn_probs.dtype).to(device)
    rel_text = rel_text.unsqueeze(0).expand(batch_size, num_tokens, num_tokens)
    for i, blk in enumerate(text_attn_blocks):
        if i < text_start_layer:
            continue
        grad = torch.autograd.grad(one_hot, [blk.attn_probs], retain_graph=True)[0].detach()
        cam = blk.attn_probs.detach()
        cam = cam.reshape(-1, cam.shape[-1], cam.shape[-1])
        grad = grad.reshape(-1, grad.shape[-1], grad.shape[-1])
        cam = grad * cam
        cam = cam.reshape(batch_size, -1, cam.shape[-1], cam.shape[-1])
        cam = cam.clamp(min=0).mean(dim=1)
        rel_text = rel_text + torch.bmm(cam, rel_text)
    text_relevance = rel_text
    return text_relevance, image_relevance, text_tokens, img_tensor
Get text relevance and image relevance from CLIP model.
Args:
model (`nn.Module`):
CLIP model to visualize.
pil_img (`Image`):
Input image.
text_list (`List[str]`):
List of text str.
device (`str`):
Device to use.
vis_start_layer (`int`):
Start layer for visualization.
text_start_layer (`int`):
Start layer for text.
transform (`Callable`):
Transform function for image.
tokenize (`Callable`):
Tokenize function for text.

Returns:
(`Tuple`):
text_relevance, image_relevance, text_tokens, img_tensor
towhee/models/visualization/clip_visualization.py