<filename>colossalai/engine/gradient_handler/_zero_gradient_handler.py
from colossalai.registry import GRADIENT_HANDLER
from ._base_gradient_handler import BaseGradientHandler


@GRADIENT_HANDLER.register_module
class ZeROGradientHandler(BaseGradientHandler):
    """A helper class to handle all-reduce operations in a data parallel group.
    A all-reduce collective communication will be operated in
    :func:`handle_gradient` among a data parallel group.
    This class is specialized with ZeRO optimization.
    """

    def handle_gradient(self):
        """A method running a all-reduce operation in a data parallel group.
        """
        self._optimizer.sync_grad()

<filename>colossalai/engine/gradient_handler/_zero_gradient_handler.py
from colossalai.registry import GRADIENT_HANDLER
from ._base_gradient_handler import BaseGradientHandler


@GRADIENT_HANDLER.register_module
class ZeROGradientHandler(BaseGradientHandler):
    """A helper class to handle all-reduce operations in a data parallel group.
    A all-reduce collective communication will be operated in
    :func:`handle_gradient` among a data parallel group.
    This class is specialized with ZeRO optimization.
    """

    def handle_gradient(self):
        """A method running a all-reduce operation in a data parallel group.
        """
        self._optimizer.sync_grad()

A helper class to handle all-reduce operations in a data parallel group.     A all-reduce collective communication will be operated in     :func:`handle_gradient` among a data parallel group.     This class is specialized with ZeRO optimization. A method running a all-reduce operation in a data parallel group.