def attention_ref(qkv, attn_mask, dropout_p, upcast=False, causal=False):
    """
    Reference implementation of scaled dot-product attention. For benchmarking
    purposes, when possible we use torch.nn.functional.scaled_dot_product_attention,
    which calls optimized mem.effient and flash attention kernels.
    """
    if True or (causal and attn_mask is not None):
        # SDPA doesn't support causal and custom masks simultaneously,
        # fall back on manual implementation
        return attention_ref_math(
            qkv, attn_mask, dropout_p, upcast=upcast, causal=causal
        )
    return attention_ref_sdpa(qkv, attn_mask, dropout_p, upcast=upcast, causal=causal)
Reference implementation of scaled dot-product attention. For benchmarking
purposes, when possible we use torch.nn.functional.scaled_dot_product_attention,
which calls optimized mem.effient and flash attention kernels.
tests/unittest/ops/test_attention.py