def dist_reduce(value: _V, reduce_op) -> _V:
    """
    Reduces the given `value` across all distributed worker nodes according the given
    reduction operation.

    If called outside of a distributed context, it will just return `value`.

    # Parameters

    value : `_V`
        The value to reduce across distributed nodes.
    reduce_op : `torch.distributed.ReduceOp`
        The [reduction operation](https://pytorch.org/docs/stable/distributed.html#torch.distributed.ReduceOp)
        to use.
    **kwargs : `Any`
        Additional arguments used to construct the tensor that will wrap `value`.

    # Returns

    `_V`
        The final value.
    """
    if not is_distributed():
        return value
    device = distributed_device()
    if isinstance(value, torch.Tensor):
        value_tensor = value.clone().to(device)
    else:
        value_tensor = torch.tensor(value, device=device)
    dist.all_reduce(value_tensor, op=reduce_op)

    if isinstance(value, torch.Tensor):
        return value_tensor
    return value_tensor.item()  # type: ignore[return-value]
Reduces the given `value` across all distributed worker nodes according the given
reduction operation.

If called outside of a distributed context, it will just return `value`.

# Parameters

value : `_V`
The value to reduce across distributed nodes.
reduce_op : `torch.distributed.ReduceOp`
The [reduction operation](https://pytorch.org/docs/stable/distributed.html#torch.distributed.ReduceOp)
to use.
**kwargs : `Any`
Additional arguments used to construct the tensor that will wrap `value`.

# Returns

`_V`
The final value.
allennlp/nn/util.py