    async def controller(self) -> None:
        """
        A standalone coroutine to wait/dispatch calling.
        """
        try:
            logger.debug("Starting dispatcher optimizer training...")
            # warm up the model
            await self.train_optimizer(
                self.optimizer.N_SKIPPED_SAMPLE, self.optimizer.N_SKIPPED_SAMPLE + 6, 1
            )

            logger.debug("Dispatcher finished warming up model.")

            await self.train_optimizer(6, 1, 1)
            self.optimizer.trigger_refresh()
            logger.debug("Dispatcher finished optimizer training request 1.")

            await self.train_optimizer(5, 1, 2)
            self.optimizer.trigger_refresh()
            logger.debug("Dispatcher finished optimizer training request 2.")

            await self.train_optimizer(3, 1, 3)
            self.optimizer.trigger_refresh()
            logger.debug("Dispatcher finished optimizer training request 3.")

            if self.optimizer.o_a + self.optimizer.o_b >= self.max_latency:
                logger.warning(
                    "BentoML has detected that a service has a max latency that is likely too low for serving. If many 503 errors are encountered, try raising the 'runner.max_latency' in your BentoML configuration YAML file."
                )
            logger.debug("Dispatcher optimizer training complete.")
        except Exception:  # pylint: disable=broad-except
            logger.exception("Error training optimizer")

        while True:
            try:
                async with self._wake_event:  # block until there's any request in queue
                    await self._wake_event.wait_for(self._queue.__len__)

                n = len(self._queue)
                dt = self.TICKET_INTERVAL
                decay = 0.95  # the decay rate of wait time
                now = time.time()
                # the wait time of the first request
                w0 = now - self._queue[0].enqueue_time
                # the wait time of the last request
                wn = now - self._queue[-1].enqueue_time
                a = self.optimizer.o_a
                b = self.optimizer.o_b

                # the estimated latency of the first request if we began processing now
                latency_0 = w0 + a * n + b

                if n > 1 and latency_0 >= self.max_latency:
                    self._queue.popleft().future.cancel()
                    continue
                if self._sema.is_locked():
                    if n == 1 and w0 >= self.max_latency:
                        self._queue.popleft().future.cancel()
                        continue
                    await asyncio.sleep(self.TICKET_INTERVAL)
                    continue
                if (
                    n < self.max_batch_size
                    and n * (wn + dt + (a or 0)) <= self.optimizer.wait * decay
                ):
                    await asyncio.sleep(self.TICKET_INTERVAL)
                    continue
                # call
                self._sema.acquire()
                inputs_info = tuple(self._get_inputs())
                self._loop.create_task(self.outbound_call(inputs_info))
            except Exception:
                logger.exception("Error processing batch requests")
A standalone coroutine to wait/dispatch calling.
src/bentoml/_internal/marshal/dispatcher.py