    def __init__(
        self,
        max_latency_in_ms: int,
        max_batch_size: int,
        *,
        shared_sema: t.Optional[NonBlockSema] = None,
        fallback: t.Callable[[], T_OUT],
        get_batch_size: t.Callable[[T_IN], int] = lambda x: x.sample.batch_size,
        batch_dim: tuple[int, int] = (0, 0),
    ) -> None:
        ...
        """
        params:
            * max_latency_in_ms: max_latency_in_ms for inbound tasks in milliseconds
            * max_batch_size: max batch size of inbound tasks
            * shared_sema: semaphore to limit concurrent outbound tasks
            * get_batch_size: callable to get batch size from inputs
            * batch_dim: tuple of (input_dim, output_dim) of batch
            * fallback: callable to return fallback result
        raises:
            * all possible exceptions the decorated function has
        """
        self.max_latency = max_latency_in_ms / 1000.0
        self.fallback = fallback
        self.optimizer = Optimizer(self.max_latency)
        self.max_batch_size = int(max_batch_size)

        self._controller = None
        self._queue: collections.deque[Job[T_IN, T_OUT]] = (
            collections.deque()
        )  # TODO(bojiang): maxlen
        self.get_batch_size = get_batch_size
        self.batch_dim = batch_dim
        # at most 1 batch can be processed at the same time
        self._sema = shared_sema if shared_sema else NonBlockSema(1)
params:
* max_latency_in_ms: max_latency_in_ms for inbound tasks in milliseconds
* max_batch_size: max batch size of inbound tasks
* shared_sema: semaphore to limit concurrent outbound tasks
* get_batch_size: callable to get batch size from inputs
* batch_dim: tuple of (input_dim, output_dim) of batch
* fallback: callable to return fallback result
raises:
* all possible exceptions the decorated function has
src/bentoml/_internal/marshal/dispatcher.py