Upload 163 files

api/core/agent/base_agent_runner.py

@@ -0,0 +1,539 @@
+import json
+import logging
+import uuid
+from datetime import UTC, datetime
+from typing import Optional, Union, cast
+
+from core.agent.entities import AgentEntity, AgentToolEntity
+from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
+from core.app.apps.agent_chat.app_config_manager import AgentChatAppConfig
+from core.app.apps.base_app_queue_manager import AppQueueManager
+from core.app.apps.base_app_runner import AppRunner
+from core.app.entities.app_invoke_entities import (
+    AgentChatAppGenerateEntity,
+    ModelConfigWithCredentialsEntity,
+)
+from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler
+from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler
+from core.file import file_manager
+from core.memory.token_buffer_memory import TokenBufferMemory
+from core.model_manager import ModelInstance
+from core.model_runtime.entities import (
+    AssistantPromptMessage,
+    LLMUsage,
+    PromptMessage,
+    PromptMessageContent,
+    PromptMessageTool,
+    SystemPromptMessage,
+    TextPromptMessageContent,
+    ToolPromptMessage,
+    UserPromptMessage,
+)
+from core.model_runtime.entities.message_entities import ImagePromptMessageContent
+from core.model_runtime.entities.model_entities import ModelFeature
+from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
+from core.model_runtime.utils.encoders import jsonable_encoder
+from core.prompt.utils.extract_thread_messages import extract_thread_messages
+from core.tools.entities.tool_entities import (
+    ToolParameter,
+    ToolRuntimeVariablePool,
+)
+from core.tools.tool.dataset_retriever_tool import DatasetRetrieverTool
+from core.tools.tool.tool import Tool
+from core.tools.tool_manager import ToolManager
+from extensions.ext_database import db
+from factories import file_factory
+from models.model import Conversation, Message, MessageAgentThought, MessageFile
+from models.tools import ToolConversationVariables
+
+logger = logging.getLogger(__name__)
+
+
+class BaseAgentRunner(AppRunner):
+    def __init__(
+        self,
+        *,
+        tenant_id: str,
+        application_generate_entity: AgentChatAppGenerateEntity,
+        conversation: Conversation,
+        app_config: AgentChatAppConfig,
+        model_config: ModelConfigWithCredentialsEntity,
+        config: AgentEntity,
+        queue_manager: AppQueueManager,
+        message: Message,
+        user_id: str,
+        memory: Optional[TokenBufferMemory] = None,
+        prompt_messages: Optional[list[PromptMessage]] = None,
+        variables_pool: Optional[ToolRuntimeVariablePool] = None,
+        db_variables: Optional[ToolConversationVariables] = None,
+        model_instance: ModelInstance,
+    ) -> None:
+        self.tenant_id = tenant_id
+        self.application_generate_entity = application_generate_entity
+        self.conversation = conversation
+        self.app_config = app_config
+        self.model_config = model_config
+        self.config = config
+        self.queue_manager = queue_manager
+        self.message = message
+        self.user_id = user_id
+        self.memory = memory
+        self.history_prompt_messages = self.organize_agent_history(prompt_messages=prompt_messages or [])
+        self.variables_pool = variables_pool
+        self.db_variables_pool = db_variables
+        self.model_instance = model_instance
+
+        # init callback
+        self.agent_callback = DifyAgentCallbackHandler()
+        # init dataset tools
+        hit_callback = DatasetIndexToolCallbackHandler(
+            queue_manager=queue_manager,
+            app_id=self.app_config.app_id,
+            message_id=message.id,
+            user_id=user_id,
+            invoke_from=self.application_generate_entity.invoke_from,
+        )
+        self.dataset_tools = DatasetRetrieverTool.get_dataset_tools(
+            tenant_id=tenant_id,
+            dataset_ids=app_config.dataset.dataset_ids if app_config.dataset else [],
+            retrieve_config=app_config.dataset.retrieve_config if app_config.dataset else None,
+            return_resource=app_config.additional_features.show_retrieve_source,
+            invoke_from=application_generate_entity.invoke_from,
+            hit_callback=hit_callback,
+        )
+        # get how many agent thoughts have been created
+        self.agent_thought_count = (
+            db.session.query(MessageAgentThought)
+            .filter(
+                MessageAgentThought.message_id == self.message.id,
+            )
+            .count()
+        )
+        db.session.close()
+
+        # check if model supports stream tool call
+        llm_model = cast(LargeLanguageModel, model_instance.model_type_instance)
+        model_schema = llm_model.get_model_schema(model_instance.model, model_instance.credentials)
+        features = model_schema.features if model_schema and model_schema.features else []
+        self.stream_tool_call = ModelFeature.STREAM_TOOL_CALL in features
+        self.files = application_generate_entity.files if ModelFeature.VISION in features else []
+        self.query: Optional[str] = ""
+        self._current_thoughts: list[PromptMessage] = []
+
+    def _repack_app_generate_entity(
+        self, app_generate_entity: AgentChatAppGenerateEntity
+    ) -> AgentChatAppGenerateEntity:
+        """
+        Repack app generate entity
+        """
+        if app_generate_entity.app_config.prompt_template.simple_prompt_template is None:
+            app_generate_entity.app_config.prompt_template.simple_prompt_template = ""
+
+        return app_generate_entity
+
+    def _convert_tool_to_prompt_message_tool(self, tool: AgentToolEntity) -> tuple[PromptMessageTool, Tool]:
+        """
+        convert tool to prompt message tool
+        """
+        tool_entity = ToolManager.get_agent_tool_runtime(
+            tenant_id=self.tenant_id,
+            app_id=self.app_config.app_id,
+            agent_tool=tool,
+            invoke_from=self.application_generate_entity.invoke_from,
+        )
+        tool_entity.load_variables(self.variables_pool)
+
+        message_tool = PromptMessageTool(
+            name=tool.tool_name,
+            description=tool_entity.description.llm if tool_entity.description else "",
+            parameters={
+                "type": "object",
+                "properties": {},
+                "required": [],
+            },
+        )
+
+        parameters = tool_entity.get_all_runtime_parameters()
+        for parameter in parameters:
+            if parameter.form != ToolParameter.ToolParameterForm.LLM:
+                continue
+
+            parameter_type = parameter.type.as_normal_type()
+            if parameter.type in {
+                ToolParameter.ToolParameterType.SYSTEM_FILES,
+                ToolParameter.ToolParameterType.FILE,
+                ToolParameter.ToolParameterType.FILES,
+            }:
+                continue
+            enum = []
+            if parameter.type == ToolParameter.ToolParameterType.SELECT:
+                enum = [option.value for option in parameter.options] if parameter.options else []
+
+            message_tool.parameters["properties"][parameter.name] = {
+                "type": parameter_type,
+                "description": parameter.llm_description or "",
+            }
+
+            if len(enum) > 0:
+                message_tool.parameters["properties"][parameter.name]["enum"] = enum
+
+            if parameter.required:
+                message_tool.parameters["required"].append(parameter.name)
+
+        return message_tool, tool_entity
+
+    def _convert_dataset_retriever_tool_to_prompt_message_tool(self, tool: DatasetRetrieverTool) -> PromptMessageTool:
+        """
+        convert dataset retriever tool to prompt message tool
+        """
+        prompt_tool = PromptMessageTool(
+            name=tool.identity.name if tool.identity else "unknown",
+            description=tool.description.llm if tool.description else "",
+            parameters={
+                "type": "object",
+                "properties": {},
+                "required": [],
+            },
+        )
+
+        for parameter in tool.get_runtime_parameters():
+            parameter_type = "string"
+
+            prompt_tool.parameters["properties"][parameter.name] = {
+                "type": parameter_type,
+                "description": parameter.llm_description or "",
+            }
+
+            if parameter.required:
+                if parameter.name not in prompt_tool.parameters["required"]:
+                    prompt_tool.parameters["required"].append(parameter.name)
+
+        return prompt_tool
+
+    def _init_prompt_tools(self) -> tuple[dict[str, Tool], list[PromptMessageTool]]:
+        """
+        Init tools
+        """
+        tool_instances = {}
+        prompt_messages_tools = []
+
+        for tool in self.app_config.agent.tools or [] if self.app_config.agent else []:
+            try:
+                prompt_tool, tool_entity = self._convert_tool_to_prompt_message_tool(tool)
+            except Exception:
+                # api tool may be deleted
+                continue
+            # save tool entity
+            tool_instances[tool.tool_name] = tool_entity
+            # save prompt tool
+            prompt_messages_tools.append(prompt_tool)
+
+        # convert dataset tools into ModelRuntime Tool format
+        for dataset_tool in self.dataset_tools:
+            prompt_tool = self._convert_dataset_retriever_tool_to_prompt_message_tool(dataset_tool)
+            # save prompt tool
+            prompt_messages_tools.append(prompt_tool)
+            # save tool entity
+            if dataset_tool.identity is not None:
+                tool_instances[dataset_tool.identity.name] = dataset_tool
+
+        return tool_instances, prompt_messages_tools
+
+    def update_prompt_message_tool(self, tool: Tool, prompt_tool: PromptMessageTool) -> PromptMessageTool:
+        """
+        update prompt message tool
+        """
+        # try to get tool runtime parameters
+        tool_runtime_parameters = tool.get_runtime_parameters()
+
+        for parameter in tool_runtime_parameters:
+            if parameter.form != ToolParameter.ToolParameterForm.LLM:
+                continue
+
+            parameter_type = parameter.type.as_normal_type()
+            if parameter.type in {
+                ToolParameter.ToolParameterType.SYSTEM_FILES,
+                ToolParameter.ToolParameterType.FILE,
+                ToolParameter.ToolParameterType.FILES,
+            }:
+                continue
+            enum = []
+            if parameter.type == ToolParameter.ToolParameterType.SELECT:
+                enum = [option.value for option in parameter.options] if parameter.options else []
+
+            prompt_tool.parameters["properties"][parameter.name] = {
+                "type": parameter_type,
+                "description": parameter.llm_description or "",
+            }
+
+            if len(enum) > 0:
+                prompt_tool.parameters["properties"][parameter.name]["enum"] = enum
+
+            if parameter.required:
+                if parameter.name not in prompt_tool.parameters["required"]:
+                    prompt_tool.parameters["required"].append(parameter.name)
+
+        return prompt_tool
+
+    def create_agent_thought(
+        self, message_id: str, message: str, tool_name: str, tool_input: str, messages_ids: list[str]
+    ) -> MessageAgentThought:
+        """
+        Create agent thought
+        """
+        thought = MessageAgentThought(
+            message_id=message_id,
+            message_chain_id=None,
+            thought="",
+            tool=tool_name,
+            tool_labels_str="{}",
+            tool_meta_str="{}",
+            tool_input=tool_input,
+            message=message,
+            message_token=0,
+            message_unit_price=0,
+            message_price_unit=0,
+            message_files=json.dumps(messages_ids) if messages_ids else "",
+            answer="",
+            observation="",
+            answer_token=0,
+            answer_unit_price=0,
+            answer_price_unit=0,
+            tokens=0,
+            total_price=0,
+            position=self.agent_thought_count + 1,
+            currency="USD",
+            latency=0,
+            created_by_role="account",
+            created_by=self.user_id,
+        )
+
+        db.session.add(thought)
+        db.session.commit()
+        db.session.refresh(thought)
+        db.session.close()
+
+        self.agent_thought_count += 1
+
+        return thought
+
+    def save_agent_thought(
+        self,
+        agent_thought: MessageAgentThought,
+        tool_name: str,
+        tool_input: Union[str, dict],
+        thought: str,
+        observation: Union[str, dict, None],
+        tool_invoke_meta: Union[str, dict, None],
+        answer: str,
+        messages_ids: list[str],
+        llm_usage: LLMUsage | None = None,
+    ):
+        """
+        Save agent thought
+        """
+        queried_thought = (
+            db.session.query(MessageAgentThought).filter(MessageAgentThought.id == agent_thought.id).first()
+        )
+        if not queried_thought:
+            raise ValueError(f"Agent thought {agent_thought.id} not found")
+        agent_thought = queried_thought
+
+        if thought:
+            agent_thought.thought = thought
+
+        if tool_name:
+            agent_thought.tool = tool_name
+
+        if tool_input:
+            if isinstance(tool_input, dict):
+                try:
+                    tool_input = json.dumps(tool_input, ensure_ascii=False)
+                except Exception as e:
+                    tool_input = json.dumps(tool_input)
+
+            agent_thought.tool_input = tool_input
+
+        if observation:
+            if isinstance(observation, dict):
+                try:
+                    observation = json.dumps(observation, ensure_ascii=False)
+                except Exception as e:
+                    observation = json.dumps(observation)
+
+            agent_thought.observation = observation
+
+        if answer:
+            agent_thought.answer = answer
+
+        if messages_ids is not None and len(messages_ids) > 0:
+            agent_thought.message_files = json.dumps(messages_ids)
+
+        if llm_usage:
+            agent_thought.message_token = llm_usage.prompt_tokens
+            agent_thought.message_price_unit = llm_usage.prompt_price_unit
+            agent_thought.message_unit_price = llm_usage.prompt_unit_price
+            agent_thought.answer_token = llm_usage.completion_tokens
+            agent_thought.answer_price_unit = llm_usage.completion_price_unit
+            agent_thought.answer_unit_price = llm_usage.completion_unit_price
+            agent_thought.tokens = llm_usage.total_tokens
+            agent_thought.total_price = llm_usage.total_price
+
+        # check if tool labels is not empty
+        labels = agent_thought.tool_labels or {}
+        tools = agent_thought.tool.split(";") if agent_thought.tool else []
+        for tool in tools:
+            if not tool:
+                continue
+            if tool not in labels:
+                tool_label = ToolManager.get_tool_label(tool)
+                if tool_label:
+                    labels[tool] = tool_label.to_dict()
+                else:
+                    labels[tool] = {"en_US": tool, "zh_Hans": tool}
+
+        agent_thought.tool_labels_str = json.dumps(labels)
+
+        if tool_invoke_meta is not None:
+            if isinstance(tool_invoke_meta, dict):
+                try:
+                    tool_invoke_meta = json.dumps(tool_invoke_meta, ensure_ascii=False)
+                except Exception as e:
+                    tool_invoke_meta = json.dumps(tool_invoke_meta)
+
+            agent_thought.tool_meta_str = tool_invoke_meta
+
+        db.session.commit()
+        db.session.close()
+
+    def update_db_variables(self, tool_variables: ToolRuntimeVariablePool, db_variables: ToolConversationVariables):
+        """
+        convert tool variables to db variables
+        """
+        queried_variables = (
+            db.session.query(ToolConversationVariables)
+            .filter(
+                ToolConversationVariables.conversation_id == self.message.conversation_id,
+            )
+            .first()
+        )
+
+        if not queried_variables:
+            return
+
+        db_variables = queried_variables
+
+        db_variables.updated_at = datetime.now(UTC).replace(tzinfo=None)
+        db_variables.variables_str = json.dumps(jsonable_encoder(tool_variables.pool))
+        db.session.commit()
+        db.session.close()
+
+    def organize_agent_history(self, prompt_messages: list[PromptMessage]) -> list[PromptMessage]:
+        """
+        Organize agent history
+        """
+        result: list[PromptMessage] = []
+        # check if there is a system message in the beginning of the conversation
+        for prompt_message in prompt_messages:
+            if isinstance(prompt_message, SystemPromptMessage):
+                result.append(prompt_message)
+
+        messages: list[Message] = (
+            db.session.query(Message)
+            .filter(
+                Message.conversation_id == self.message.conversation_id,
+            )
+            .order_by(Message.created_at.desc())
+            .all()
+        )
+
+        messages = list(reversed(extract_thread_messages(messages)))
+
+        for message in messages:
+            if message.id == self.message.id:
+                continue
+
+            result.append(self.organize_agent_user_prompt(message))
+            agent_thoughts: list[MessageAgentThought] = message.agent_thoughts
+            if agent_thoughts:
+                for agent_thought in agent_thoughts:
+                    tools = agent_thought.tool
+                    if tools:
+                        tools = tools.split(";")
+                        tool_calls: list[AssistantPromptMessage.ToolCall] = []
+                        tool_call_response: list[ToolPromptMessage] = []
+                        try:
+                            tool_inputs = json.loads(agent_thought.tool_input)
+                        except Exception as e:
+                            tool_inputs = {tool: {} for tool in tools}
+                        try:
+                            tool_responses = json.loads(agent_thought.observation)
+                        except Exception as e:
+                            tool_responses = dict.fromkeys(tools, agent_thought.observation)
+
+                        for tool in tools:
+                            # generate a uuid for tool call
+                            tool_call_id = str(uuid.uuid4())
+                            tool_calls.append(
+                                AssistantPromptMessage.ToolCall(
+                                    id=tool_call_id,
+                                    type="function",
+                                    function=AssistantPromptMessage.ToolCall.ToolCallFunction(
+                                        name=tool,
+                                        arguments=json.dumps(tool_inputs.get(tool, {})),
+                                    ),
+                                )
+                            )
+                            tool_call_response.append(
+                                ToolPromptMessage(
+                                    content=tool_responses.get(tool, agent_thought.observation),
+                                    name=tool,
+                                    tool_call_id=tool_call_id,
+                                )
+                            )
+
+                        result.extend(
+                            [
+                                AssistantPromptMessage(
+                                    content=agent_thought.thought,
+                                    tool_calls=tool_calls,
+                                ),
+                                *tool_call_response,
+                            ]
+                        )
+                    if not tools:
+                        result.append(AssistantPromptMessage(content=agent_thought.thought))
+            else:
+                if message.answer:
+                    result.append(AssistantPromptMessage(content=message.answer))
+
+        db.session.close()
+
+        return result
+
+    def organize_agent_user_prompt(self, message: Message) -> UserPromptMessage:
+        files = db.session.query(MessageFile).filter(MessageFile.message_id == message.id).all()
+        if not files:
+            return UserPromptMessage(content=message.query)
+        file_extra_config = FileUploadConfigManager.convert(message.app_model_config.to_dict())
+        if not file_extra_config:
+            return UserPromptMessage(content=message.query)
+
+        image_detail_config = file_extra_config.image_config.detail if file_extra_config.image_config else None
+        image_detail_config = image_detail_config or ImagePromptMessageContent.DETAIL.LOW
+
+        file_objs = file_factory.build_from_message_files(
+            message_files=files, tenant_id=self.tenant_id, config=file_extra_config
+        )
+        if not file_objs:
+            return UserPromptMessage(content=message.query)
+        prompt_message_contents: list[PromptMessageContent] = []
+        prompt_message_contents.append(TextPromptMessageContent(data=message.query))
+        for file in file_objs:
+            prompt_message_contents.append(
+                file_manager.to_prompt_message_content(
+                    file,
+                    image_detail_config=image_detail_config,
+                )
+            )
+        return UserPromptMessage(content=prompt_message_contents)

api/core/agent/cot_agent_runner.py

@@ -0,0 +1,437 @@
+import json
+from abc import ABC, abstractmethod
+from collections.abc import Generator, Mapping
+from typing import Any, Optional
+
+from core.agent.base_agent_runner import BaseAgentRunner
+from core.agent.entities import AgentScratchpadUnit
+from core.agent.output_parser.cot_output_parser import CotAgentOutputParser
+from core.app.apps.base_app_queue_manager import PublishFrom
+from core.app.entities.queue_entities import QueueAgentThoughtEvent, QueueMessageEndEvent, QueueMessageFileEvent
+from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
+from core.model_runtime.entities.message_entities import (
+    AssistantPromptMessage,
+    PromptMessage,
+    PromptMessageTool,
+    ToolPromptMessage,
+    UserPromptMessage,
+)
+from core.ops.ops_trace_manager import TraceQueueManager
+from core.prompt.agent_history_prompt_transform import AgentHistoryPromptTransform
+from core.tools.entities.tool_entities import ToolInvokeMeta
+from core.tools.tool.tool import Tool
+from core.tools.tool_engine import ToolEngine
+from models.model import Message
+
+
+class CotAgentRunner(BaseAgentRunner, ABC):
+    _is_first_iteration = True
+    _ignore_observation_providers = ["wenxin"]
+    _historic_prompt_messages: list[PromptMessage] | None = None
+    _agent_scratchpad: list[AgentScratchpadUnit] | None = None
+    _instruction: str = ""  # FIXME this must be str for now
+    _query: str | None = None
+    _prompt_messages_tools: list[PromptMessageTool] = []
+
+    def run(
+        self,
+        message: Message,
+        query: str,
+        inputs: Mapping[str, str],
+    ) -> Generator:
+        """
+        Run Cot agent application
+        """
+        app_generate_entity = self.application_generate_entity
+        self._repack_app_generate_entity(app_generate_entity)
+        self._init_react_state(query)
+
+        trace_manager = app_generate_entity.trace_manager
+
+        # check model mode
+        if "Observation" not in app_generate_entity.model_conf.stop:
+            if app_generate_entity.model_conf.provider not in self._ignore_observation_providers:
+                app_generate_entity.model_conf.stop.append("Observation")
+
+        app_config = self.app_config
+
+        # init instruction
+        inputs = inputs or {}
+        instruction = app_config.prompt_template.simple_prompt_template
+        self._instruction = self._fill_in_inputs_from_external_data_tools(instruction=instruction or "", inputs=inputs)
+
+        iteration_step = 1
+        max_iteration_steps = min(app_config.agent.max_iteration if app_config.agent else 5, 5) + 1
+
+        # convert tools into ModelRuntime Tool format
+        tool_instances, self._prompt_messages_tools = self._init_prompt_tools()
+
+        function_call_state = True
+        llm_usage: dict[str, Optional[LLMUsage]] = {"usage": None}
+        final_answer = ""
+
+        def increase_usage(final_llm_usage_dict: dict[str, Optional[LLMUsage]], usage: LLMUsage):
+            if not final_llm_usage_dict["usage"]:
+                final_llm_usage_dict["usage"] = usage
+            else:
+                llm_usage = final_llm_usage_dict["usage"]
+                llm_usage.prompt_tokens += usage.prompt_tokens
+                llm_usage.completion_tokens += usage.completion_tokens
+                llm_usage.prompt_price += usage.prompt_price
+                llm_usage.completion_price += usage.completion_price
+                llm_usage.total_price += usage.total_price
+
+        model_instance = self.model_instance
+
+        while function_call_state and iteration_step <= max_iteration_steps:
+            # continue to run until there is not any tool call
+            function_call_state = False
+
+            if iteration_step == max_iteration_steps:
+                # the last iteration, remove all tools
+                self._prompt_messages_tools = []
+
+            message_file_ids: list[str] = []
+
+            agent_thought = self.create_agent_thought(
+                message_id=message.id, message="", tool_name="", tool_input="", messages_ids=message_file_ids
+            )
+
+            if iteration_step > 1:
+                self.queue_manager.publish(
+                    QueueAgentThoughtEvent(agent_thought_id=agent_thought.id), PublishFrom.APPLICATION_MANAGER
+                )
+
+            # recalc llm max tokens
+            prompt_messages = self._organize_prompt_messages()
+            self.recalc_llm_max_tokens(self.model_config, prompt_messages)
+            # invoke model
+            chunks = model_instance.invoke_llm(
+                prompt_messages=prompt_messages,
+                model_parameters=app_generate_entity.model_conf.parameters,
+                tools=[],
+                stop=app_generate_entity.model_conf.stop,
+                stream=True,
+                user=self.user_id,
+                callbacks=[],
+            )
+
+            if not isinstance(chunks, Generator):
+                raise ValueError("Expected streaming response from LLM")
+
+            # check llm result
+            if not chunks:
+                raise ValueError("failed to invoke llm")
+
+            usage_dict: dict[str, Optional[LLMUsage]] = {"usage": None}
+            react_chunks = CotAgentOutputParser.handle_react_stream_output(chunks, usage_dict)
+            scratchpad = AgentScratchpadUnit(
+                agent_response="",
+                thought="",
+                action_str="",
+                observation="",
+                action=None,
+            )
+
+            # publish agent thought if it's first iteration
+            if iteration_step == 1:
+                self.queue_manager.publish(
+                    QueueAgentThoughtEvent(agent_thought_id=agent_thought.id), PublishFrom.APPLICATION_MANAGER
+                )
+
+            for chunk in react_chunks:
+                if isinstance(chunk, AgentScratchpadUnit.Action):
+                    action = chunk
+                    # detect action
+                    if scratchpad.agent_response is not None:
+                        scratchpad.agent_response += json.dumps(chunk.model_dump())
+                    scratchpad.action_str = json.dumps(chunk.model_dump())
+                    scratchpad.action = action
+                else:
+                    if scratchpad.agent_response is not None:
+                        scratchpad.agent_response += chunk
+                    if scratchpad.thought is not None:
+                        scratchpad.thought += chunk
+                    yield LLMResultChunk(
+                        model=self.model_config.model,
+                        prompt_messages=prompt_messages,
+                        system_fingerprint="",
+                        delta=LLMResultChunkDelta(index=0, message=AssistantPromptMessage(content=chunk), usage=None),
+                    )
+            if scratchpad.thought is not None:
+                scratchpad.thought = scratchpad.thought.strip() or "I am thinking about how to help you"
+            if self._agent_scratchpad is not None:
+                self._agent_scratchpad.append(scratchpad)
+
+            # get llm usage
+            if "usage" in usage_dict:
+                if usage_dict["usage"] is not None:
+                    increase_usage(llm_usage, usage_dict["usage"])
+            else:
+                usage_dict["usage"] = LLMUsage.empty_usage()
+
+            self.save_agent_thought(
+                agent_thought=agent_thought,
+                tool_name=(scratchpad.action.action_name if scratchpad.action and not scratchpad.is_final() else ""),
+                tool_input={scratchpad.action.action_name: scratchpad.action.action_input} if scratchpad.action else {},
+                tool_invoke_meta={},
+                thought=scratchpad.thought or "",
+                observation="",
+                answer=scratchpad.agent_response or "",
+                messages_ids=[],
+                llm_usage=usage_dict["usage"],
+            )
+
+            if not scratchpad.is_final():
+                self.queue_manager.publish(
+                    QueueAgentThoughtEvent(agent_thought_id=agent_thought.id), PublishFrom.APPLICATION_MANAGER
+                )
+
+            if not scratchpad.action:
+                # failed to extract action, return final answer directly
+                final_answer = ""
+            else:
+                if scratchpad.action.action_name.lower() == "final answer":
+                    # action is final answer, return final answer directly
+                    try:
+                        if isinstance(scratchpad.action.action_input, dict):
+                            final_answer = json.dumps(scratchpad.action.action_input)
+                        elif isinstance(scratchpad.action.action_input, str):
+                            final_answer = scratchpad.action.action_input
+                        else:
+                            final_answer = f"{scratchpad.action.action_input}"
+                    except json.JSONDecodeError:
+                        final_answer = f"{scratchpad.action.action_input}"
+                else:
+                    function_call_state = True
+                    # action is tool call, invoke tool
+                    tool_invoke_response, tool_invoke_meta = self._handle_invoke_action(
+                        action=scratchpad.action,
+                        tool_instances=tool_instances,
+                        message_file_ids=message_file_ids,
+                        trace_manager=trace_manager,
+                    )
+                    scratchpad.observation = tool_invoke_response
+                    scratchpad.agent_response = tool_invoke_response
+
+                    self.save_agent_thought(
+                        agent_thought=agent_thought,
+                        tool_name=scratchpad.action.action_name,
+                        tool_input={scratchpad.action.action_name: scratchpad.action.action_input},
+                        thought=scratchpad.thought or "",
+                        observation={scratchpad.action.action_name: tool_invoke_response},
+                        tool_invoke_meta={scratchpad.action.action_name: tool_invoke_meta.to_dict()},
+                        answer=scratchpad.agent_response,
+                        messages_ids=message_file_ids,
+                        llm_usage=usage_dict["usage"],
+                    )
+
+                    self.queue_manager.publish(
+                        QueueAgentThoughtEvent(agent_thought_id=agent_thought.id), PublishFrom.APPLICATION_MANAGER
+                    )
+
+                # update prompt tool message
+                for prompt_tool in self._prompt_messages_tools:
+                    self.update_prompt_message_tool(tool_instances[prompt_tool.name], prompt_tool)
+
+            iteration_step += 1
+
+        yield LLMResultChunk(
+            model=model_instance.model,
+            prompt_messages=prompt_messages,
+            delta=LLMResultChunkDelta(
+                index=0, message=AssistantPromptMessage(content=final_answer), usage=llm_usage["usage"]
+            ),
+            system_fingerprint="",
+        )
+
+        # save agent thought
+        self.save_agent_thought(
+            agent_thought=agent_thought,
+            tool_name="",
+            tool_input={},
+            tool_invoke_meta={},
+            thought=final_answer,
+            observation={},
+            answer=final_answer,
+            messages_ids=[],
+        )
+        if self.variables_pool is not None and self.db_variables_pool is not None:
+            self.update_db_variables(self.variables_pool, self.db_variables_pool)
+        # publish end event
+        self.queue_manager.publish(
+            QueueMessageEndEvent(
+                llm_result=LLMResult(
+                    model=model_instance.model,
+                    prompt_messages=prompt_messages,
+                    message=AssistantPromptMessage(content=final_answer),
+                    usage=llm_usage["usage"] or LLMUsage.empty_usage(),
+                    system_fingerprint="",
+                )
+            ),
+            PublishFrom.APPLICATION_MANAGER,
+        )
+
+    def _handle_invoke_action(
+        self,
+        action: AgentScratchpadUnit.Action,
+        tool_instances: dict[str, Tool],
+        message_file_ids: list[str],
+        trace_manager: Optional[TraceQueueManager] = None,
+    ) -> tuple[str, ToolInvokeMeta]:
+        """
+        handle invoke action
+        :param action: action
+        :param tool_instances: tool instances
+        :param message_file_ids: message file ids
+        :param trace_manager: trace manager
+        :return: observation, meta
+        """
+        # action is tool call, invoke tool
+        tool_call_name = action.action_name
+        tool_call_args = action.action_input
+        tool_instance = tool_instances.get(tool_call_name)
+
+        if not tool_instance:
+            answer = f"there is not a tool named {tool_call_name}"
+            return answer, ToolInvokeMeta.error_instance(answer)
+
+        if isinstance(tool_call_args, str):
+            try:
+                tool_call_args = json.loads(tool_call_args)
+            except json.JSONDecodeError:
+                pass
+
+        # invoke tool
+        tool_invoke_response, message_files, tool_invoke_meta = ToolEngine.agent_invoke(
+            tool=tool_instance,
+            tool_parameters=tool_call_args,
+            user_id=self.user_id,
+            tenant_id=self.tenant_id,
+            message=self.message,
+            invoke_from=self.application_generate_entity.invoke_from,
+            agent_tool_callback=self.agent_callback,
+            trace_manager=trace_manager,
+        )
+
+        # publish files
+        for message_file_id, save_as in message_files:
+            if save_as is not None and self.variables_pool:
+                # FIXME the save_as type is confusing, it should be a string or not
+                self.variables_pool.set_file(tool_name=tool_call_name, value=message_file_id, name=str(save_as))
+
+            # publish message file
+            self.queue_manager.publish(
+                QueueMessageFileEvent(message_file_id=message_file_id), PublishFrom.APPLICATION_MANAGER
+            )
+            # add message file ids
+            message_file_ids.append(message_file_id)
+
+        return tool_invoke_response, tool_invoke_meta
+
+    def _convert_dict_to_action(self, action: dict) -> AgentScratchpadUnit.Action:
+        """
+        convert dict to action
+        """
+        return AgentScratchpadUnit.Action(action_name=action["action"], action_input=action["action_input"])
+
+    def _fill_in_inputs_from_external_data_tools(self, instruction: str, inputs: Mapping[str, Any]) -> str:
+        """
+        fill in inputs from external data tools
+        """
+        for key, value in inputs.items():
+            try:
+                instruction = instruction.replace(f"{{{{{key}}}}}", str(value))
+            except Exception as e:
+                continue
+
+        return instruction
+
+    def _init_react_state(self, query) -> None:
+        """
+        init agent scratchpad
+        """
+        self._query = query
+        self._agent_scratchpad = []
+        self._historic_prompt_messages = self._organize_historic_prompt_messages()
+
+    @abstractmethod
+    def _organize_prompt_messages(self) -> list[PromptMessage]:
+        """
+        organize prompt messages
+        """
+
+    def _format_assistant_message(self, agent_scratchpad: list[AgentScratchpadUnit]) -> str:
+        """
+        format assistant message
+        """
+        message = ""
+        for scratchpad in agent_scratchpad:
+            if scratchpad.is_final():
+                message += f"Final Answer: {scratchpad.agent_response}"
+            else:
+                message += f"Thought: {scratchpad.thought}\n\n"
+                if scratchpad.action_str:
+                    message += f"Action: {scratchpad.action_str}\n\n"
+                if scratchpad.observation:
+                    message += f"Observation: {scratchpad.observation}\n\n"
+
+        return message
+
+    def _organize_historic_prompt_messages(
+        self, current_session_messages: Optional[list[PromptMessage]] = None
+    ) -> list[PromptMessage]:
+        """
+        organize historic prompt messages
+        """
+        result: list[PromptMessage] = []
+        scratchpads: list[AgentScratchpadUnit] = []
+        current_scratchpad: AgentScratchpadUnit | None = None
+
+        for message in self.history_prompt_messages:
+            if isinstance(message, AssistantPromptMessage):
+                if not current_scratchpad:
+                    if not isinstance(message.content, str | None):
+                        raise NotImplementedError("expected str type")
+                    current_scratchpad = AgentScratchpadUnit(
+                        agent_response=message.content,
+                        thought=message.content or "I am thinking about how to help you",
+                        action_str="",
+                        action=None,
+                        observation=None,
+                    )
+                    scratchpads.append(current_scratchpad)
+                if message.tool_calls:
+                    try:
+                        current_scratchpad.action = AgentScratchpadUnit.Action(
+                            action_name=message.tool_calls[0].function.name,
+                            action_input=json.loads(message.tool_calls[0].function.arguments),
+                        )
+                        current_scratchpad.action_str = json.dumps(current_scratchpad.action.to_dict())
+                    except:
+                        pass
+            elif isinstance(message, ToolPromptMessage):
+                if not current_scratchpad:
+                    continue
+                if isinstance(message.content, str):
+                    current_scratchpad.observation = message.content
+                else:
+                    raise NotImplementedError("expected str type")
+            elif isinstance(message, UserPromptMessage):
+                if scratchpads:
+                    result.append(AssistantPromptMessage(content=self._format_assistant_message(scratchpads)))
+                    scratchpads = []
+                    current_scratchpad = None
+
+                result.append(message)
+
+        if scratchpads:
+            result.append(AssistantPromptMessage(content=self._format_assistant_message(scratchpads)))
+
+        historic_prompts = AgentHistoryPromptTransform(
+            model_config=self.model_config,
+            prompt_messages=current_session_messages or [],
+            history_messages=result,
+            memory=self.memory,
+        ).get_prompt()
+        return historic_prompts

api/core/agent/cot_chat_agent_runner.py

@@ -0,0 +1,117 @@
+import json
+
+from core.agent.cot_agent_runner import CotAgentRunner
+from core.file import file_manager
+from core.model_runtime.entities import (
+    AssistantPromptMessage,
+    PromptMessage,
+    PromptMessageContent,
+    SystemPromptMessage,
+    TextPromptMessageContent,
+    UserPromptMessage,
+)
+from core.model_runtime.entities.message_entities import ImagePromptMessageContent
+from core.model_runtime.utils.encoders import jsonable_encoder
+
+
+class CotChatAgentRunner(CotAgentRunner):
+    def _organize_system_prompt(self) -> SystemPromptMessage:
+        """
+        Organize system prompt
+        """
+        if not self.app_config.agent:
+            raise ValueError("Agent configuration is not set")
+
+        prompt_entity = self.app_config.agent.prompt
+        if not prompt_entity:
+            raise ValueError("Agent prompt configuration is not set")
+        first_prompt = prompt_entity.first_prompt
+
+        system_prompt = (
+            first_prompt.replace("{{instruction}}", self._instruction)
+            .replace("{{tools}}", json.dumps(jsonable_encoder(self._prompt_messages_tools)))
+            .replace("{{tool_names}}", ", ".join([tool.name for tool in self._prompt_messages_tools]))
+        )
+
+        return SystemPromptMessage(content=system_prompt)
+
+    def _organize_user_query(self, query, prompt_messages: list[PromptMessage]) -> list[PromptMessage]:
+        """
+        Organize user query
+        """
+        if self.files:
+            prompt_message_contents: list[PromptMessageContent] = []
+            prompt_message_contents.append(TextPromptMessageContent(data=query))
+
+            # get image detail config
+            image_detail_config = (
+                self.application_generate_entity.file_upload_config.image_config.detail
+                if (
+                    self.application_generate_entity.file_upload_config
+                    and self.application_generate_entity.file_upload_config.image_config
+                )
+                else None
+            )
+            image_detail_config = image_detail_config or ImagePromptMessageContent.DETAIL.LOW
+            for file in self.files:
+                prompt_message_contents.append(
+                    file_manager.to_prompt_message_content(
+                        file,
+                        image_detail_config=image_detail_config,
+                    )
+                )
+
+            prompt_messages.append(UserPromptMessage(content=prompt_message_contents))
+        else:
+            prompt_messages.append(UserPromptMessage(content=query))
+
+        return prompt_messages
+
+    def _organize_prompt_messages(self) -> list[PromptMessage]:
+        """
+        Organize
+        """
+        # organize system prompt
+        system_message = self._organize_system_prompt()
+
+        # organize current assistant messages
+        agent_scratchpad = self._agent_scratchpad
+        if not agent_scratchpad:
+            assistant_messages = []
+        else:
+            assistant_message = AssistantPromptMessage(content="")
+            assistant_message.content = ""  # FIXME: type check tell mypy that assistant_message.content is str
+            for unit in agent_scratchpad:
+                if unit.is_final():
+                    assistant_message.content += f"Final Answer: {unit.agent_response}"
+                else:
+                    assistant_message.content += f"Thought: {unit.thought}\n\n"
+                    if unit.action_str:
+                        assistant_message.content += f"Action: {unit.action_str}\n\n"
+                    if unit.observation:
+                        assistant_message.content += f"Observation: {unit.observation}\n\n"
+
+            assistant_messages = [assistant_message]
+
+        # query messages
+        query_messages = self._organize_user_query(self._query, [])
+
+        if assistant_messages:
+            # organize historic prompt messages
+            historic_messages = self._organize_historic_prompt_messages(
+                [system_message, *query_messages, *assistant_messages, UserPromptMessage(content="continue")]
+            )
+            messages = [
+                system_message,
+                *historic_messages,
+                *query_messages,
+                *assistant_messages,
+                UserPromptMessage(content="continue"),
+            ]
+        else:
+            # organize historic prompt messages
+            historic_messages = self._organize_historic_prompt_messages([system_message, *query_messages])
+            messages = [system_message, *historic_messages, *query_messages]
+
+        # join all messages
+        return messages

api/core/agent/cot_completion_agent_runner.py

@@ -0,0 +1,88 @@
+import json
+from typing import Optional
+
+from core.agent.cot_agent_runner import CotAgentRunner
+from core.model_runtime.entities.message_entities import (
+    AssistantPromptMessage,
+    PromptMessage,
+    TextPromptMessageContent,
+    UserPromptMessage,
+)
+from core.model_runtime.utils.encoders import jsonable_encoder
+
+
+class CotCompletionAgentRunner(CotAgentRunner):
+    def _organize_instruction_prompt(self) -> str:
+        """
+        Organize instruction prompt
+        """
+        if self.app_config.agent is None:
+            raise ValueError("Agent configuration is not set")
+        prompt_entity = self.app_config.agent.prompt
+        if prompt_entity is None:
+            raise ValueError("prompt entity is not set")
+        first_prompt = prompt_entity.first_prompt
+
+        system_prompt = (
+            first_prompt.replace("{{instruction}}", self._instruction)
+            .replace("{{tools}}", json.dumps(jsonable_encoder(self._prompt_messages_tools)))
+            .replace("{{tool_names}}", ", ".join([tool.name for tool in self._prompt_messages_tools]))
+        )
+
+        return system_prompt
+
+    def _organize_historic_prompt(self, current_session_messages: Optional[list[PromptMessage]] = None) -> str:
+        """
+        Organize historic prompt
+        """
+        historic_prompt_messages = self._organize_historic_prompt_messages(current_session_messages)
+        historic_prompt = ""
+
+        for message in historic_prompt_messages:
+            if isinstance(message, UserPromptMessage):
+                historic_prompt += f"Question: {message.content}\n\n"
+            elif isinstance(message, AssistantPromptMessage):
+                if isinstance(message.content, str):
+                    historic_prompt += message.content + "\n\n"
+                elif isinstance(message.content, list):
+                    for content in message.content:
+                        if not isinstance(content, TextPromptMessageContent):
+                            continue
+                        historic_prompt += content.data
+
+        return historic_prompt
+
+    def _organize_prompt_messages(self) -> list[PromptMessage]:
+        """
+        Organize prompt messages
+        """
+        # organize system prompt
+        system_prompt = self._organize_instruction_prompt()
+
+        # organize historic prompt messages
+        historic_prompt = self._organize_historic_prompt()
+
+        # organize current assistant messages
+        agent_scratchpad = self._agent_scratchpad
+        assistant_prompt = ""
+        for unit in agent_scratchpad or []:
+            if unit.is_final():
+                assistant_prompt += f"Final Answer: {unit.agent_response}"
+            else:
+                assistant_prompt += f"Thought: {unit.thought}\n\n"
+                if unit.action_str:
+                    assistant_prompt += f"Action: {unit.action_str}\n\n"
+                if unit.observation:
+                    assistant_prompt += f"Observation: {unit.observation}\n\n"
+
+        # query messages
+        query_prompt = f"Question: {self._query}"
+
+        # join all messages
+        prompt = (
+            system_prompt.replace("{{historic_messages}}", historic_prompt)
+            .replace("{{agent_scratchpad}}", assistant_prompt)
+            .replace("{{query}}", query_prompt)
+        )
+
+        return [UserPromptMessage(content=prompt)]

api/core/agent/entities.py

@@ -0,0 +1,82 @@
+from enum import Enum
+from typing import Any, Literal, Optional, Union
+
+from pydantic import BaseModel
+
+
+class AgentToolEntity(BaseModel):
+    """
+    Agent Tool Entity.
+    """
+
+    provider_type: Literal["builtin", "api", "workflow"]
+    provider_id: str
+    tool_name: str
+    tool_parameters: dict[str, Any] = {}
+
+
+class AgentPromptEntity(BaseModel):
+    """
+    Agent Prompt Entity.
+    """
+
+    first_prompt: str
+    next_iteration: str
+
+
+class AgentScratchpadUnit(BaseModel):
+    """
+    Agent First Prompt Entity.
+    """
+
+    class Action(BaseModel):
+        """
+        Action Entity.
+        """
+
+        action_name: str
+        action_input: Union[dict, str]
+
+        def to_dict(self) -> dict:
+            """
+            Convert to dictionary.
+            """
+            return {
+                "action": self.action_name,
+                "action_input": self.action_input,
+            }
+
+    agent_response: Optional[str] = None
+    thought: Optional[str] = None
+    action_str: Optional[str] = None
+    observation: Optional[str] = None
+    action: Optional[Action] = None
+
+    def is_final(self) -> bool:
+        """
+        Check if the scratchpad unit is final.
+        """
+        return self.action is None or (
+            "final" in self.action.action_name.lower() and "answer" in self.action.action_name.lower()
+        )
+
+
+class AgentEntity(BaseModel):
+    """
+    Agent Entity.
+    """
+
+    class Strategy(Enum):
+        """
+        Agent Strategy.
+        """
+
+        CHAIN_OF_THOUGHT = "chain-of-thought"
+        FUNCTION_CALLING = "function-calling"
+
+    provider: str
+    model: str
+    strategy: Strategy
+    prompt: Optional[AgentPromptEntity] = None
+    tools: list[AgentToolEntity] | None = None
+    max_iteration: int = 5

api/core/agent/fc_agent_runner.py

@@ -0,0 +1,473 @@
+import json
+import logging
+from collections.abc import Generator
+from copy import deepcopy
+from typing import Any, Optional, Union
+
+from core.agent.base_agent_runner import BaseAgentRunner
+from core.app.apps.base_app_queue_manager import PublishFrom
+from core.app.entities.queue_entities import QueueAgentThoughtEvent, QueueMessageEndEvent, QueueMessageFileEvent
+from core.file import file_manager
+from core.model_runtime.entities import (
+    AssistantPromptMessage,
+    LLMResult,
+    LLMResultChunk,
+    LLMResultChunkDelta,
+    LLMUsage,
+    PromptMessage,
+    PromptMessageContent,
+    PromptMessageContentType,
+    SystemPromptMessage,
+    TextPromptMessageContent,
+    ToolPromptMessage,
+    UserPromptMessage,
+)
+from core.model_runtime.entities.message_entities import ImagePromptMessageContent
+from core.prompt.agent_history_prompt_transform import AgentHistoryPromptTransform
+from core.tools.entities.tool_entities import ToolInvokeMeta
+from core.tools.tool_engine import ToolEngine
+from models.model import Message
+
+logger = logging.getLogger(__name__)
+
+
+class FunctionCallAgentRunner(BaseAgentRunner):
+    def run(self, message: Message, query: str, **kwargs: Any) -> Generator[LLMResultChunk, None, None]:
+        """
+        Run FunctionCall agent application
+        """
+        self.query = query
+        app_generate_entity = self.application_generate_entity
+
+        app_config = self.app_config
+        assert app_config is not None, "app_config is required"
+        assert app_config.agent is not None, "app_config.agent is required"
+
+        # convert tools into ModelRuntime Tool format
+        tool_instances, prompt_messages_tools = self._init_prompt_tools()
+
+        iteration_step = 1
+        max_iteration_steps = min(app_config.agent.max_iteration, 5) + 1
+
+        # continue to run until there is not any tool call
+        function_call_state = True
+        llm_usage: dict[str, LLMUsage] = {"usage": LLMUsage.empty_usage()}
+        final_answer = ""
+
+        # get tracing instance
+        trace_manager = app_generate_entity.trace_manager
+
+        def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage):
+            if not final_llm_usage_dict["usage"]:
+                final_llm_usage_dict["usage"] = usage
+            else:
+                llm_usage = final_llm_usage_dict["usage"]
+                llm_usage.prompt_tokens += usage.prompt_tokens
+                llm_usage.completion_tokens += usage.completion_tokens
+                llm_usage.prompt_price += usage.prompt_price
+                llm_usage.completion_price += usage.completion_price
+                llm_usage.total_price += usage.total_price
+
+        model_instance = self.model_instance
+
+        while function_call_state and iteration_step <= max_iteration_steps:
+            function_call_state = False
+
+            if iteration_step == max_iteration_steps:
+                # the last iteration, remove all tools
+                prompt_messages_tools = []
+
+            message_file_ids: list[str] = []
+            agent_thought = self.create_agent_thought(
+                message_id=message.id, message="", tool_name="", tool_input="", messages_ids=message_file_ids
+            )
+
+            # recalc llm max tokens
+            prompt_messages = self._organize_prompt_messages()
+            self.recalc_llm_max_tokens(self.model_config, prompt_messages)
+            # invoke model
+            chunks: Union[Generator[LLMResultChunk, None, None], LLMResult] = model_instance.invoke_llm(
+                prompt_messages=prompt_messages,
+                model_parameters=app_generate_entity.model_conf.parameters,
+                tools=prompt_messages_tools,
+                stop=app_generate_entity.model_conf.stop,
+                stream=self.stream_tool_call,
+                user=self.user_id,
+                callbacks=[],
+            )
+
+            tool_calls: list[tuple[str, str, dict[str, Any]]] = []
+
+            # save full response
+            response = ""
+
+            # save tool call names and inputs
+            tool_call_names = ""
+            tool_call_inputs = ""
+
+            current_llm_usage = None
+
+            if self.stream_tool_call and isinstance(chunks, Generator):
+                is_first_chunk = True
+                for chunk in chunks:
+                    if is_first_chunk:
+                        self.queue_manager.publish(
+                            QueueAgentThoughtEvent(agent_thought_id=agent_thought.id), PublishFrom.APPLICATION_MANAGER
+                        )
+                        is_first_chunk = False
+                    # check if there is any tool call
+                    if self.check_tool_calls(chunk):
+                        function_call_state = True
+                        tool_calls.extend(self.extract_tool_calls(chunk) or [])
+                        tool_call_names = ";".join([tool_call[1] for tool_call in tool_calls])
+                        try:
+                            tool_call_inputs = json.dumps(
+                                {tool_call[1]: tool_call[2] for tool_call in tool_calls}, ensure_ascii=False
+                            )
+                        except json.JSONDecodeError as e:
+                            # ensure ascii to avoid encoding error
+                            tool_call_inputs = json.dumps({tool_call[1]: tool_call[2] for tool_call in tool_calls})
+
+                    if chunk.delta.message and chunk.delta.message.content:
+                        if isinstance(chunk.delta.message.content, list):
+                            for content in chunk.delta.message.content:
+                                response += content.data
+                        else:
+                            response += str(chunk.delta.message.content)
+
+                    if chunk.delta.usage:
+                        increase_usage(llm_usage, chunk.delta.usage)
+                        current_llm_usage = chunk.delta.usage
+
+                    yield chunk
+            elif not self.stream_tool_call and isinstance(chunks, LLMResult):
+                result = chunks
+                # check if there is any tool call
+                if self.check_blocking_tool_calls(result):
+                    function_call_state = True
+                    tool_calls.extend(self.extract_blocking_tool_calls(result) or [])
+                    tool_call_names = ";".join([tool_call[1] for tool_call in tool_calls])
+                    try:
+                        tool_call_inputs = json.dumps(
+                            {tool_call[1]: tool_call[2] for tool_call in tool_calls}, ensure_ascii=False
+                        )
+                    except json.JSONDecodeError as e:
+                        # ensure ascii to avoid encoding error
+                        tool_call_inputs = json.dumps({tool_call[1]: tool_call[2] for tool_call in tool_calls})
+
+                if result.usage:
+                    increase_usage(llm_usage, result.usage)
+                    current_llm_usage = result.usage
+
+                if result.message and result.message.content:
+                    if isinstance(result.message.content, list):
+                        for content in result.message.content:
+                            response += content.data
+                    else:
+                        response += str(result.message.content)
+
+                if not result.message.content:
+                    result.message.content = ""
+
+                self.queue_manager.publish(
+                    QueueAgentThoughtEvent(agent_thought_id=agent_thought.id), PublishFrom.APPLICATION_MANAGER
+                )
+
+                yield LLMResultChunk(
+                    model=model_instance.model,
+                    prompt_messages=result.prompt_messages,
+                    system_fingerprint=result.system_fingerprint,
+                    delta=LLMResultChunkDelta(
+                        index=0,
+                        message=result.message,
+                        usage=result.usage,
+                    ),
+                )
+            else:
+                raise RuntimeError(f"invalid chunks type: {type(chunks)}")
+
+            assistant_message = AssistantPromptMessage(content="", tool_calls=[])
+            if tool_calls:
+                assistant_message.tool_calls = [
+                    AssistantPromptMessage.ToolCall(
+                        id=tool_call[0],
+                        type="function",
+                        function=AssistantPromptMessage.ToolCall.ToolCallFunction(
+                            name=tool_call[1], arguments=json.dumps(tool_call[2], ensure_ascii=False)
+                        ),
+                    )
+                    for tool_call in tool_calls
+                ]
+            else:
+                assistant_message.content = response
+
+            self._current_thoughts.append(assistant_message)
+
+            # save thought
+            self.save_agent_thought(
+                agent_thought=agent_thought,
+                tool_name=tool_call_names,
+                tool_input=tool_call_inputs,
+                thought=response,
+                tool_invoke_meta=None,
+                observation=None,
+                answer=response,
+                messages_ids=[],
+                llm_usage=current_llm_usage,
+            )
+            self.queue_manager.publish(
+                QueueAgentThoughtEvent(agent_thought_id=agent_thought.id), PublishFrom.APPLICATION_MANAGER
+            )
+
+            final_answer += response + "\n"
+
+            # call tools
+            tool_responses = []
+            for tool_call_id, tool_call_name, tool_call_args in tool_calls:
+                tool_instance = tool_instances.get(tool_call_name)
+                if not tool_instance:
+                    tool_response = {
+                        "tool_call_id": tool_call_id,
+                        "tool_call_name": tool_call_name,
+                        "tool_response": f"there is not a tool named {tool_call_name}",
+                        "meta": ToolInvokeMeta.error_instance(f"there is not a tool named {tool_call_name}").to_dict(),
+                    }
+                else:
+                    # invoke tool
+                    tool_invoke_response, message_files, tool_invoke_meta = ToolEngine.agent_invoke(
+                        tool=tool_instance,
+                        tool_parameters=tool_call_args,
+                        user_id=self.user_id,
+                        tenant_id=self.tenant_id,
+                        message=self.message,
+                        invoke_from=self.application_generate_entity.invoke_from,
+                        agent_tool_callback=self.agent_callback,
+                        trace_manager=trace_manager,
+                    )
+                    # publish files
+                    for message_file_id, save_as in message_files:
+                        if save_as:
+                            if self.variables_pool:
+                                self.variables_pool.set_file(
+                                    tool_name=tool_call_name, value=message_file_id, name=save_as
+                                )
+
+                        # publish message file
+                        self.queue_manager.publish(
+                            QueueMessageFileEvent(message_file_id=message_file_id), PublishFrom.APPLICATION_MANAGER
+                        )
+                        # add message file ids
+                        message_file_ids.append(message_file_id)
+
+                    tool_response = {
+                        "tool_call_id": tool_call_id,
+                        "tool_call_name": tool_call_name,
+                        "tool_response": tool_invoke_response,
+                        "meta": tool_invoke_meta.to_dict(),
+                    }
+
+                tool_responses.append(tool_response)
+                if tool_response["tool_response"] is not None:
+                    self._current_thoughts.append(
+                        ToolPromptMessage(
+                            content=str(tool_response["tool_response"]),
+                            tool_call_id=tool_call_id,
+                            name=tool_call_name,
+                        )
+                    )
+
+            if len(tool_responses) > 0:
+                # save agent thought
+                self.save_agent_thought(
+                    agent_thought=agent_thought,
+                    tool_name="",
+                    tool_input="",
+                    thought="",
+                    tool_invoke_meta={
+                        tool_response["tool_call_name"]: tool_response["meta"] for tool_response in tool_responses
+                    },
+                    observation={
+                        tool_response["tool_call_name"]: tool_response["tool_response"]
+                        for tool_response in tool_responses
+                    },
+                    answer="",
+                    messages_ids=message_file_ids,
+                )
+                self.queue_manager.publish(
+                    QueueAgentThoughtEvent(agent_thought_id=agent_thought.id), PublishFrom.APPLICATION_MANAGER
+                )
+
+            # update prompt tool
+            for prompt_tool in prompt_messages_tools:
+                self.update_prompt_message_tool(tool_instances[prompt_tool.name], prompt_tool)
+
+            iteration_step += 1
+
+        if self.variables_pool and self.db_variables_pool:
+            self.update_db_variables(self.variables_pool, self.db_variables_pool)
+        # publish end event
+        self.queue_manager.publish(
+            QueueMessageEndEvent(
+                llm_result=LLMResult(
+                    model=model_instance.model,
+                    prompt_messages=prompt_messages,
+                    message=AssistantPromptMessage(content=final_answer),
+                    usage=llm_usage["usage"] or LLMUsage.empty_usage(),
+                    system_fingerprint="",
+                )
+            ),
+            PublishFrom.APPLICATION_MANAGER,
+        )
+
+    def check_tool_calls(self, llm_result_chunk: LLMResultChunk) -> bool:
+        """
+        Check if there is any tool call in llm result chunk
+        """
+        if llm_result_chunk.delta.message.tool_calls:
+            return True
+        return False
+
+    def check_blocking_tool_calls(self, llm_result: LLMResult) -> bool:
+        """
+        Check if there is any blocking tool call in llm result
+        """
+        if llm_result.message.tool_calls:
+            return True
+        return False
+
+    def extract_tool_calls(
+        self, llm_result_chunk: LLMResultChunk
+    ) -> Union[None, list[tuple[str, str, dict[str, Any]]]]:
+        """
+        Extract tool calls from llm result chunk
+
+        Returns:
+            List[Tuple[str, str, Dict[str, Any]]]: [(tool_call_id, tool_call_name, tool_call_args)]
+        """
+        tool_calls = []
+        for prompt_message in llm_result_chunk.delta.message.tool_calls:
+            args = {}
+            if prompt_message.function.arguments != "":
+                args = json.loads(prompt_message.function.arguments)
+
+            tool_calls.append(
+                (
+                    prompt_message.id,
+                    prompt_message.function.name,
+                    args,
+                )
+            )
+
+        return tool_calls
+
+    def extract_blocking_tool_calls(self, llm_result: LLMResult) -> Union[None, list[tuple[str, str, dict[str, Any]]]]:
+        """
+        Extract blocking tool calls from llm result
+
+        Returns:
+            List[Tuple[str, str, Dict[str, Any]]]: [(tool_call_id, tool_call_name, tool_call_args)]
+        """
+        tool_calls = []
+        for prompt_message in llm_result.message.tool_calls:
+            args = {}
+            if prompt_message.function.arguments != "":
+                args = json.loads(prompt_message.function.arguments)
+
+            tool_calls.append(
+                (
+                    prompt_message.id,
+                    prompt_message.function.name,
+                    args,
+                )
+            )
+
+        return tool_calls
+
+    def _init_system_message(
+        self, prompt_template: str, prompt_messages: Optional[list[PromptMessage]] = None
+    ) -> list[PromptMessage]:
+        """
+        Initialize system message
+        """
+        if not prompt_messages and prompt_template:
+            return [
+                SystemPromptMessage(content=prompt_template),
+            ]
+
+        if prompt_messages and not isinstance(prompt_messages[0], SystemPromptMessage) and prompt_template:
+            prompt_messages.insert(0, SystemPromptMessage(content=prompt_template))
+
+        return prompt_messages or []
+
+    def _organize_user_query(self, query: str, prompt_messages: list[PromptMessage]) -> list[PromptMessage]:
+        """
+        Organize user query
+        """
+        if self.files:
+            prompt_message_contents: list[PromptMessageContent] = []
+            prompt_message_contents.append(TextPromptMessageContent(data=query))
+
+            # get image detail config
+            image_detail_config = (
+                self.application_generate_entity.file_upload_config.image_config.detail
+                if (
+                    self.application_generate_entity.file_upload_config
+                    and self.application_generate_entity.file_upload_config.image_config
+                )
+                else None
+            )
+            image_detail_config = image_detail_config or ImagePromptMessageContent.DETAIL.LOW
+            for file in self.files:
+                prompt_message_contents.append(
+                    file_manager.to_prompt_message_content(
+                        file,
+                        image_detail_config=image_detail_config,
+                    )
+                )
+
+            prompt_messages.append(UserPromptMessage(content=prompt_message_contents))
+        else:
+            prompt_messages.append(UserPromptMessage(content=query))
+
+        return prompt_messages
+
+    def _clear_user_prompt_image_messages(self, prompt_messages: list[PromptMessage]) -> list[PromptMessage]:
+        """
+        As for now, gpt supports both fc and vision at the first iteration.
+        We need to remove the image messages from the prompt messages at the first iteration.
+        """
+        prompt_messages = deepcopy(prompt_messages)
+
+        for prompt_message in prompt_messages:
+            if isinstance(prompt_message, UserPromptMessage):
+                if isinstance(prompt_message.content, list):
+                    prompt_message.content = "\n".join(
+                        [
+                            content.data
+                            if content.type == PromptMessageContentType.TEXT
+                            else "[image]"
+                            if content.type == PromptMessageContentType.IMAGE
+                            else "[file]"
+                            for content in prompt_message.content
+                        ]
+                    )
+
+        return prompt_messages
+
+    def _organize_prompt_messages(self):
+        prompt_template = self.app_config.prompt_template.simple_prompt_template or ""
+        self.history_prompt_messages = self._init_system_message(prompt_template, self.history_prompt_messages)
+        query_prompt_messages = self._organize_user_query(self.query or "", [])
+
+        self.history_prompt_messages = AgentHistoryPromptTransform(
+            model_config=self.model_config,
+            prompt_messages=[*query_prompt_messages, *self._current_thoughts],
+            history_messages=self.history_prompt_messages,
+            memory=self.memory,
+        ).get_prompt()
+
+        prompt_messages = [*self.history_prompt_messages, *query_prompt_messages, *self._current_thoughts]
+        if len(self._current_thoughts) != 0:
+            # clear messages after the first iteration
+            prompt_messages = self._clear_user_prompt_image_messages(prompt_messages)
+        return prompt_messages

api/core/agent/output_parser/cot_output_parser.py

@@ -0,0 +1,208 @@
+import json
+import re
+from collections.abc import Generator
+from typing import Union
+
+from core.agent.entities import AgentScratchpadUnit
+from core.model_runtime.entities.llm_entities import LLMResultChunk
+
+
+class CotAgentOutputParser:
+    @classmethod
+    def handle_react_stream_output(
+        cls, llm_response: Generator[LLMResultChunk, None, None], usage_dict: dict
+    ) -> Generator[Union[str, AgentScratchpadUnit.Action], None, None]:
+        def parse_action(json_str):
+            try:
+                action = json.loads(json_str, strict=False)
+                action_name = None
+                action_input = None
+
+                # cohere always returns a list
+                if isinstance(action, list) and len(action) == 1:
+                    action = action[0]
+
+                for key, value in action.items():
+                    if "input" in key.lower():
+                        action_input = value
+                    else:
+                        action_name = value
+
+                if action_name is not None and action_input is not None:
+                    return AgentScratchpadUnit.Action(
+                        action_name=action_name,
+                        action_input=action_input,
+                    )
+                else:
+                    return json_str or ""
+            except:
+                return json_str or ""
+
+        def extra_json_from_code_block(code_block) -> Generator[Union[str, AgentScratchpadUnit.Action], None, None]:
+            code_blocks = re.findall(r"```(.*?)```", code_block, re.DOTALL)
+            if not code_blocks:
+                return
+            for block in code_blocks:
+                json_text = re.sub(r"^[a-zA-Z]+\n", "", block.strip(), flags=re.MULTILINE)
+                yield parse_action(json_text)
+
+        code_block_cache = ""
+        code_block_delimiter_count = 0
+        in_code_block = False
+        json_cache = ""
+        json_quote_count = 0
+        in_json = False
+        got_json = False
+
+        action_cache = ""
+        action_str = "action:"
+        action_idx = 0
+
+        thought_cache = ""
+        thought_str = "thought:"
+        thought_idx = 0
+
+        last_character = ""
+
+        for response in llm_response:
+            if response.delta.usage:
+                usage_dict["usage"] = response.delta.usage
+            response_content = response.delta.message.content
+            if not isinstance(response_content, str):
+                continue
+
+            # stream
+            index = 0
+            while index < len(response_content):
+                steps = 1
+                delta = response_content[index : index + steps]
+                yield_delta = False
+
+                if delta == "`":
+                    last_character = delta
+                    code_block_cache += delta
+                    code_block_delimiter_count += 1
+                else:
+                    if not in_code_block:
+                        if code_block_delimiter_count > 0:
+                            last_character = delta
+                            yield code_block_cache
+                        code_block_cache = ""
+                    else:
+                        last_character = delta
+                        code_block_cache += delta
+                    code_block_delimiter_count = 0
+
+                if not in_code_block and not in_json:
+                    if delta.lower() == action_str[action_idx] and action_idx == 0:
+                        if last_character not in {"\n", " ", ""}:
+                            yield_delta = True
+                        else:
+                            last_character = delta
+                            action_cache += delta
+                            action_idx += 1
+                            if action_idx == len(action_str):
+                                action_cache = ""
+                                action_idx = 0
+                            index += steps
+                            continue
+                    elif delta.lower() == action_str[action_idx] and action_idx > 0:
+                        last_character = delta
+                        action_cache += delta
+                        action_idx += 1
+                        if action_idx == len(action_str):
+                            action_cache = ""
+                            action_idx = 0
+                        index += steps
+                        continue
+                    else:
+                        if action_cache:
+                            last_character = delta
+                            yield action_cache
+                            action_cache = ""
+                            action_idx = 0
+
+                    if delta.lower() == thought_str[thought_idx] and thought_idx == 0:
+                        if last_character not in {"\n", " ", ""}:
+                            yield_delta = True
+                        else:
+                            last_character = delta
+                            thought_cache += delta
+                            thought_idx += 1
+                            if thought_idx == len(thought_str):
+                                thought_cache = ""
+                                thought_idx = 0
+                            index += steps
+                            continue
+                    elif delta.lower() == thought_str[thought_idx] and thought_idx > 0:
+                        last_character = delta
+                        thought_cache += delta
+                        thought_idx += 1
+                        if thought_idx == len(thought_str):
+                            thought_cache = ""
+                            thought_idx = 0
+                        index += steps
+                        continue
+                    else:
+                        if thought_cache:
+                            last_character = delta
+                            yield thought_cache
+                            thought_cache = ""
+                            thought_idx = 0
+
+                    if yield_delta:
+                        index += steps
+                        last_character = delta
+                        yield delta
+                        continue
+
+                if code_block_delimiter_count == 3:
+                    if in_code_block:
+                        last_character = delta
+                        yield from extra_json_from_code_block(code_block_cache)
+                        code_block_cache = ""
+
+                    in_code_block = not in_code_block
+                    code_block_delimiter_count = 0
+
+                if not in_code_block:
+                    # handle single json
+                    if delta == "{":
+                        json_quote_count += 1
+                        in_json = True
+                        last_character = delta
+                        json_cache += delta
+                    elif delta == "}":
+                        last_character = delta
+                        json_cache += delta
+                        if json_quote_count > 0:
+                            json_quote_count -= 1
+                            if json_quote_count == 0:
+                                in_json = False
+                                got_json = True
+                                index += steps
+                                continue
+                    else:
+                        if in_json:
+                            last_character = delta
+                            json_cache += delta
+
+                    if got_json:
+                        got_json = False
+                        last_character = delta
+                        yield parse_action(json_cache)
+                        json_cache = ""
+                        json_quote_count = 0
+                        in_json = False
+
+                if not in_code_block and not in_json:
+                    last_character = delta
+                    yield delta.replace("`", "")
+
+                index += steps
+
+        if code_block_cache:
+            yield code_block_cache
+
+        if json_cache:
+            yield parse_action(json_cache)

api/core/agent/prompt/template.py

@@ -0,0 +1,106 @@
+ENGLISH_REACT_COMPLETION_PROMPT_TEMPLATES = """Respond to the human as helpfully and accurately as possible. 
+
+{{instruction}}
+
+You have access to the following tools:
+
+{{tools}}
+
+Use a json blob to specify a tool by providing an action key (tool name) and an action_input key (tool input).
+Valid "action" values: "Final Answer" or {{tool_names}}
+
+Provide only ONE action per $JSON_BLOB, as shown:
+
+```
+{
+  "action": $TOOL_NAME,
+  "action_input": $ACTION_INPUT
+}
+```
+
+Follow this format:
+
+Question: input question to answer
+Thought: consider previous and subsequent steps
+Action:
+```
+$JSON_BLOB
+```
+Observation: action result
+... (repeat Thought/Action/Observation N times)
+Thought: I know what to respond
+Action:
+```
+{
+  "action": "Final Answer",
+  "action_input": "Final response to human"
+}
+```
+
+Begin! Reminder to ALWAYS respond with a valid json blob of a single action. Use tools if necessary. Respond directly if appropriate. Format is Action:```$JSON_BLOB```then Observation:.
+{{historic_messages}}
+Question: {{query}}
+{{agent_scratchpad}}
+Thought:"""  # noqa: E501
+
+
+ENGLISH_REACT_COMPLETION_AGENT_SCRATCHPAD_TEMPLATES = """Observation: {{observation}}
+Thought:"""
+
+ENGLISH_REACT_CHAT_PROMPT_TEMPLATES = """Respond to the human as helpfully and accurately as possible. 
+
+{{instruction}}
+
+You have access to the following tools:
+
+{{tools}}
+
+Use a json blob to specify a tool by providing an action key (tool name) and an action_input key (tool input).
+Valid "action" values: "Final Answer" or {{tool_names}}
+
+Provide only ONE action per $JSON_BLOB, as shown:
+
+```
+{
+  "action": $TOOL_NAME,
+  "action_input": $ACTION_INPUT
+}
+```
+
+Follow this format:
+
+Question: input question to answer
+Thought: consider previous and subsequent steps
+Action:
+```
+$JSON_BLOB
+```
+Observation: action result
+... (repeat Thought/Action/Observation N times)
+Thought: I know what to respond
+Action:
+```
+{
+  "action": "Final Answer",
+  "action_input": "Final response to human"
+}
+```
+
+Begin! Reminder to ALWAYS respond with a valid json blob of a single action. Use tools if necessary. Respond directly if appropriate. Format is Action:```$JSON_BLOB```then Observation:.
+"""  # noqa: E501
+
+
+ENGLISH_REACT_CHAT_AGENT_SCRATCHPAD_TEMPLATES = ""
+
+REACT_PROMPT_TEMPLATES = {
+    "english": {
+        "chat": {
+            "prompt": ENGLISH_REACT_CHAT_PROMPT_TEMPLATES,
+            "agent_scratchpad": ENGLISH_REACT_CHAT_AGENT_SCRATCHPAD_TEMPLATES,
+        },
+        "completion": {
+            "prompt": ENGLISH_REACT_COMPLETION_PROMPT_TEMPLATES,
+            "agent_scratchpad": ENGLISH_REACT_COMPLETION_AGENT_SCRATCHPAD_TEMPLATES,
+        },
+    }
+}

api/core/app/app_config/base_app_config_manager.py

@@ -0,0 +1,49 @@
+from collections.abc import Mapping
+from typing import Any
+
+from core.app.app_config.entities import AppAdditionalFeatures
+from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
+from core.app.app_config.features.more_like_this.manager import MoreLikeThisConfigManager
+from core.app.app_config.features.opening_statement.manager import OpeningStatementConfigManager
+from core.app.app_config.features.retrieval_resource.manager import RetrievalResourceConfigManager
+from core.app.app_config.features.speech_to_text.manager import SpeechToTextConfigManager
+from core.app.app_config.features.suggested_questions_after_answer.manager import (
+    SuggestedQuestionsAfterAnswerConfigManager,
+)
+from core.app.app_config.features.text_to_speech.manager import TextToSpeechConfigManager
+from models.model import AppMode
+
+
+class BaseAppConfigManager:
+    @classmethod
+    def convert_features(cls, config_dict: Mapping[str, Any], app_mode: AppMode) -> AppAdditionalFeatures:
+        """
+        Convert app config to app model config
+
+        :param config_dict: app config
+        :param app_mode: app mode
+        """
+        config_dict = dict(config_dict.items())
+
+        additional_features = AppAdditionalFeatures()
+        additional_features.show_retrieve_source = RetrievalResourceConfigManager.convert(config=config_dict)
+
+        additional_features.file_upload = FileUploadConfigManager.convert(
+            config=config_dict, is_vision=app_mode in {AppMode.CHAT, AppMode.COMPLETION, AppMode.AGENT_CHAT}
+        )
+
+        additional_features.opening_statement, additional_features.suggested_questions = (
+            OpeningStatementConfigManager.convert(config=config_dict)
+        )
+
+        additional_features.suggested_questions_after_answer = SuggestedQuestionsAfterAnswerConfigManager.convert(
+            config=config_dict
+        )
+
+        additional_features.more_like_this = MoreLikeThisConfigManager.convert(config=config_dict)
+
+        additional_features.speech_to_text = SpeechToTextConfigManager.convert(config=config_dict)
+
+        additional_features.text_to_speech = TextToSpeechConfigManager.convert(config=config_dict)
+
+        return additional_features

api/core/app/app_config/common/sensitive_word_avoidance/manager.py

@@ -0,0 +1,45 @@
+from typing import Optional
+
+from core.app.app_config.entities import SensitiveWordAvoidanceEntity
+from core.moderation.factory import ModerationFactory
+
+
+class SensitiveWordAvoidanceConfigManager:
+    @classmethod
+    def convert(cls, config: dict) -> Optional[SensitiveWordAvoidanceEntity]:
+        sensitive_word_avoidance_dict = config.get("sensitive_word_avoidance")
+        if not sensitive_word_avoidance_dict:
+            return None
+
+        if sensitive_word_avoidance_dict.get("enabled"):
+            return SensitiveWordAvoidanceEntity(
+                type=sensitive_word_avoidance_dict.get("type"),
+                config=sensitive_word_avoidance_dict.get("config"),
+            )
+        else:
+            return None
+
+    @classmethod
+    def validate_and_set_defaults(
+        cls, tenant_id, config: dict, only_structure_validate: bool = False
+    ) -> tuple[dict, list[str]]:
+        if not config.get("sensitive_word_avoidance"):
+            config["sensitive_word_avoidance"] = {"enabled": False}
+
+        if not isinstance(config["sensitive_word_avoidance"], dict):
+            raise ValueError("sensitive_word_avoidance must be of dict type")
+
+        if "enabled" not in config["sensitive_word_avoidance"] or not config["sensitive_word_avoidance"]["enabled"]:
+            config["sensitive_word_avoidance"]["enabled"] = False
+
+        if config["sensitive_word_avoidance"]["enabled"]:
+            if not config["sensitive_word_avoidance"].get("type"):
+                raise ValueError("sensitive_word_avoidance.type is required")
+
+            if not only_structure_validate:
+                typ = config["sensitive_word_avoidance"]["type"]
+                sensitive_word_avoidance_config = config["sensitive_word_avoidance"]["config"]
+
+                ModerationFactory.validate_config(name=typ, tenant_id=tenant_id, config=sensitive_word_avoidance_config)
+
+        return config, ["sensitive_word_avoidance"]

api/core/app/app_config/easy_ui_based_app/agent/manager.py

@@ -0,0 +1,81 @@
+from typing import Optional
+
+from core.agent.entities import AgentEntity, AgentPromptEntity, AgentToolEntity
+from core.agent.prompt.template import REACT_PROMPT_TEMPLATES
+
+
+class AgentConfigManager:
+    @classmethod
+    def convert(cls, config: dict) -> Optional[AgentEntity]:
+        """
+        Convert model config to model config
+
+        :param config: model config args
+        """
+        if "agent_mode" in config and config["agent_mode"] and "enabled" in config["agent_mode"]:
+            agent_dict = config.get("agent_mode", {})
+            agent_strategy = agent_dict.get("strategy", "cot")
+
+            if agent_strategy == "function_call":
+                strategy = AgentEntity.Strategy.FUNCTION_CALLING
+            elif agent_strategy in {"cot", "react"}:
+                strategy = AgentEntity.Strategy.CHAIN_OF_THOUGHT
+            else:
+                # old configs, try to detect default strategy
+                if config["model"]["provider"] == "openai":
+                    strategy = AgentEntity.Strategy.FUNCTION_CALLING
+                else:
+                    strategy = AgentEntity.Strategy.CHAIN_OF_THOUGHT
+
+            agent_tools = []
+            for tool in agent_dict.get("tools", []):
+                keys = tool.keys()
+                if len(keys) >= 4:
+                    if "enabled" not in tool or not tool["enabled"]:
+                        continue
+
+                    agent_tool_properties = {
+                        "provider_type": tool["provider_type"],
+                        "provider_id": tool["provider_id"],
+                        "tool_name": tool["tool_name"],
+                        "tool_parameters": tool.get("tool_parameters", {}),
+                    }
+
+                    agent_tools.append(AgentToolEntity(**agent_tool_properties))
+
+            if "strategy" in config["agent_mode"] and config["agent_mode"]["strategy"] not in {
+                "react_router",
+                "router",
+            }:
+                agent_prompt = agent_dict.get("prompt", None) or {}
+                # check model mode
+                model_mode = config.get("model", {}).get("mode", "completion")
+                if model_mode == "completion":
+                    agent_prompt_entity = AgentPromptEntity(
+                        first_prompt=agent_prompt.get(
+                            "first_prompt", REACT_PROMPT_TEMPLATES["english"]["completion"]["prompt"]
+                        ),
+                        next_iteration=agent_prompt.get(
+                            "next_iteration", REACT_PROMPT_TEMPLATES["english"]["completion"]["agent_scratchpad"]
+                        ),
+                    )
+                else:
+                    agent_prompt_entity = AgentPromptEntity(
+                        first_prompt=agent_prompt.get(
+                            "first_prompt", REACT_PROMPT_TEMPLATES["english"]["chat"]["prompt"]
+                        ),
+                        next_iteration=agent_prompt.get(
+                            "next_iteration", REACT_PROMPT_TEMPLATES["english"]["chat"]["agent_scratchpad"]
+                        ),
+                    )
+
+                return AgentEntity(
+                    provider=config["model"]["provider"],
+                    model=config["model"]["name"],
+                    strategy=strategy,
+                    prompt=agent_prompt_entity,
+                    tools=agent_tools,
+                    max_iteration=agent_dict.get("max_iteration", 5),
+                )
+
+        return None

api/core/app/app_config/easy_ui_based_app/dataset/manager.py

@@ -0,0 +1,221 @@
+import uuid
+from typing import Optional
+
+from core.app.app_config.entities import DatasetEntity, DatasetRetrieveConfigEntity
+from core.entities.agent_entities import PlanningStrategy
+from models.model import AppMode
+from services.dataset_service import DatasetService
+
+
+class DatasetConfigManager:
+    @classmethod
+    def convert(cls, config: dict) -> Optional[DatasetEntity]:
+        """
+        Convert model config to model config
+
+        :param config: model config args
+        """
+        dataset_ids = []
+        if "datasets" in config.get("dataset_configs", {}):
+            datasets = config.get("dataset_configs", {}).get("datasets", {"strategy": "router", "datasets": []})
+
+            for dataset in datasets.get("datasets", []):
+                keys = list(dataset.keys())
+                if len(keys) == 0 or keys[0] != "dataset":
+                    continue
+
+                dataset = dataset["dataset"]
+
+                if "enabled" not in dataset or not dataset["enabled"]:
+                    continue
+
+                dataset_id = dataset.get("id", None)
+                if dataset_id:
+                    dataset_ids.append(dataset_id)
+
+        if (
+            "agent_mode" in config
+            and config["agent_mode"]
+            and "enabled" in config["agent_mode"]
+            and config["agent_mode"]["enabled"]
+        ):
+            agent_dict = config.get("agent_mode", {})
+
+            for tool in agent_dict.get("tools", []):
+                keys = tool.keys()
+                if len(keys) == 1:
+                    # old standard
+                    key = list(tool.keys())[0]
+
+                    if key != "dataset":
+                        continue
+
+                    tool_item = tool[key]
+
+                    if "enabled" not in tool_item or not tool_item["enabled"]:
+                        continue
+
+                    dataset_id = tool_item["id"]
+                    dataset_ids.append(dataset_id)
+
+        if len(dataset_ids) == 0:
+            return None
+
+        # dataset configs
+        if "dataset_configs" in config and config.get("dataset_configs"):
+            dataset_configs = config.get("dataset_configs")
+        else:
+            dataset_configs = {"retrieval_model": "multiple"}
+        if dataset_configs is None:
+            return None
+        query_variable = config.get("dataset_query_variable")
+
+        if dataset_configs["retrieval_model"] == "single":
+            return DatasetEntity(
+                dataset_ids=dataset_ids,
+                retrieve_config=DatasetRetrieveConfigEntity(
+                    query_variable=query_variable,
+                    retrieve_strategy=DatasetRetrieveConfigEntity.RetrieveStrategy.value_of(
+                        dataset_configs["retrieval_model"]
+                    ),
+                ),
+            )
+        else:
+            return DatasetEntity(
+                dataset_ids=dataset_ids,
+                retrieve_config=DatasetRetrieveConfigEntity(
+                    query_variable=query_variable,
+                    retrieve_strategy=DatasetRetrieveConfigEntity.RetrieveStrategy.value_of(
+                        dataset_configs["retrieval_model"]
+                    ),
+                    top_k=dataset_configs.get("top_k", 4),
+                    score_threshold=dataset_configs.get("score_threshold"),
+                    reranking_model=dataset_configs.get("reranking_model"),
+                    weights=dataset_configs.get("weights"),
+                    reranking_enabled=dataset_configs.get("reranking_enabled", True),
+                    rerank_mode=dataset_configs.get("reranking_mode", "reranking_model"),
+                ),
+            )
+
+    @classmethod
+    def validate_and_set_defaults(cls, tenant_id: str, app_mode: AppMode, config: dict) -> tuple[dict, list[str]]:
+        """
+        Validate and set defaults for dataset feature
+
+        :param tenant_id: tenant ID
+        :param app_mode: app mode
+        :param config: app model config args
+        """
+        # Extract dataset config for legacy compatibility
+        config = cls.extract_dataset_config_for_legacy_compatibility(tenant_id, app_mode, config)
+
+        # dataset_configs
+        if not config.get("dataset_configs"):
+            config["dataset_configs"] = {"retrieval_model": "single"}
+
+        if not config["dataset_configs"].get("datasets"):
+            config["dataset_configs"]["datasets"] = {"strategy": "router", "datasets": []}
+
+        if not isinstance(config["dataset_configs"], dict):
+            raise ValueError("dataset_configs must be of object type")
+
+        if not isinstance(config["dataset_configs"], dict):
+            raise ValueError("dataset_configs must be of object type")
+
+        need_manual_query_datasets = config.get("dataset_configs") and config["dataset_configs"].get(
+            "datasets", {}
+        ).get("datasets")
+
+        if need_manual_query_datasets and app_mode == AppMode.COMPLETION:
+            # Only check when mode is completion
+            dataset_query_variable = config.get("dataset_query_variable")
+
+            if not dataset_query_variable:
+                raise ValueError("Dataset query variable is required when dataset is exist")
+
+        return config, ["agent_mode", "dataset_configs", "dataset_query_variable"]
+
+    @classmethod
+    def extract_dataset_config_for_legacy_compatibility(cls, tenant_id: str, app_mode: AppMode, config: dict) -> dict:
+        """
+        Extract dataset config for legacy compatibility
+
+        :param tenant_id: tenant ID
+        :param app_mode: app mode
+        :param config: app model config args
+        """
+        # Extract dataset config for legacy compatibility
+        if not config.get("agent_mode"):
+            config["agent_mode"] = {"enabled": False, "tools": []}
+
+        if not isinstance(config["agent_mode"], dict):
+            raise ValueError("agent_mode must be of object type")
+
+        # enabled
+        if "enabled" not in config["agent_mode"] or not config["agent_mode"]["enabled"]:
+            config["agent_mode"]["enabled"] = False
+
+        if not isinstance(config["agent_mode"]["enabled"], bool):
+            raise ValueError("enabled in agent_mode must be of boolean type")
+
+        # tools
+        if not config["agent_mode"].get("tools"):
+            config["agent_mode"]["tools"] = []
+
+        if not isinstance(config["agent_mode"]["tools"], list):
+            raise ValueError("tools in agent_mode must be a list of objects")
+
+        # strategy
+        if not config["agent_mode"].get("strategy"):
+            config["agent_mode"]["strategy"] = PlanningStrategy.ROUTER.value
+
+        has_datasets = False
+        if config["agent_mode"]["strategy"] in {PlanningStrategy.ROUTER.value, PlanningStrategy.REACT_ROUTER.value}:
+            for tool in config["agent_mode"]["tools"]:
+                key = list(tool.keys())[0]
+                if key == "dataset":
+                    # old style, use tool name as key
+                    tool_item = tool[key]
+
+                    if "enabled" not in tool_item or not tool_item["enabled"]:
+                        tool_item["enabled"] = False
+
+                    if not isinstance(tool_item["enabled"], bool):
+                        raise ValueError("enabled in agent_mode.tools must be of boolean type")
+
+                    if "id" not in tool_item:
+                        raise ValueError("id is required in dataset")
+
+                    try:
+                        uuid.UUID(tool_item["id"])
+                    except ValueError:
+                        raise ValueError("id in dataset must be of UUID type")
+
+                    if not cls.is_dataset_exists(tenant_id, tool_item["id"]):
+                        raise ValueError("Dataset ID does not exist, please check your permission.")
+
+                    has_datasets = True
+
+        need_manual_query_datasets = has_datasets and config["agent_mode"]["enabled"]
+
+        if need_manual_query_datasets and app_mode == AppMode.COMPLETION:
+            # Only check when mode is completion
+            dataset_query_variable = config.get("dataset_query_variable")
+
+            if not dataset_query_variable:
+                raise ValueError("Dataset query variable is required when dataset is exist")
+
+        return config
+
+    @classmethod
+    def is_dataset_exists(cls, tenant_id: str, dataset_id: str) -> bool:
+        # verify if the dataset ID exists
+        dataset = DatasetService.get_dataset(dataset_id)
+
+        if not dataset:
+            return False
+
+        if dataset.tenant_id != tenant_id:
+            return False
+
+        return True

api/core/app/app_config/easy_ui_based_app/model_config/converter.py

@@ -0,0 +1,87 @@
+from typing import cast
+
+from core.app.app_config.entities import EasyUIBasedAppConfig
+from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
+from core.entities.model_entities import ModelStatus
+from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
+from core.model_runtime.entities.model_entities import ModelType
+from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
+from core.provider_manager import ProviderManager
+
+
+class ModelConfigConverter:
+    @classmethod
+    def convert(cls, app_config: EasyUIBasedAppConfig) -> ModelConfigWithCredentialsEntity:
+        """
+        Convert app model config dict to entity.
+        :param app_config: app config
+        :param skip_check: skip check
+        :raises ProviderTokenNotInitError: provider token not init error
+        :return: app orchestration config entity
+        """
+        model_config = app_config.model
+
+        provider_manager = ProviderManager()
+        provider_model_bundle = provider_manager.get_provider_model_bundle(
+            tenant_id=app_config.tenant_id, provider=model_config.provider, model_type=ModelType.LLM
+        )
+
+        provider_name = provider_model_bundle.configuration.provider.provider
+        model_name = model_config.model
+
+        model_type_instance = provider_model_bundle.model_type_instance
+        model_type_instance = cast(LargeLanguageModel, model_type_instance)
+
+        # check model credentials
+        model_credentials = provider_model_bundle.configuration.get_current_credentials(
+            model_type=ModelType.LLM, model=model_config.model
+        )
+
+        if model_credentials is None:
+            raise ProviderTokenNotInitError(f"Model {model_name} credentials is not initialized.")
+
+        # check model
+        provider_model = provider_model_bundle.configuration.get_provider_model(
+            model=model_config.model, model_type=ModelType.LLM
+        )
+
+        if provider_model is None:
+            model_name = model_config.model
+            raise ValueError(f"Model {model_name} not exist.")
+
+        if provider_model.status == ModelStatus.NO_CONFIGURE:
+            raise ProviderTokenNotInitError(f"Model {model_name} credentials is not initialized.")
+        elif provider_model.status == ModelStatus.NO_PERMISSION:
+            raise ModelCurrentlyNotSupportError(f"Dify Hosted OpenAI {model_name} currently not support.")
+        elif provider_model.status == ModelStatus.QUOTA_EXCEEDED:
+            raise QuotaExceededError(f"Model provider {provider_name} quota exceeded.")
+
+        # model config
+        completion_params = model_config.parameters
+        stop = []
+        if "stop" in completion_params:
+            stop = completion_params["stop"]
+            del completion_params["stop"]
+
+        # get model mode
+        model_mode = model_config.mode
+        if not model_mode:
+            mode_enum = model_type_instance.get_model_mode(model=model_config.model, credentials=model_credentials)
+
+            model_mode = mode_enum.value
+
+        model_schema = model_type_instance.get_model_schema(model_config.model, model_credentials)
+
+        if not model_schema:
+            raise ValueError(f"Model {model_name} not exist.")
+
+        return ModelConfigWithCredentialsEntity(
+            provider=model_config.provider,
+            model=model_config.model,
+            model_schema=model_schema,
+            mode=model_mode,
+            provider_model_bundle=provider_model_bundle,
+            credentials=model_credentials,
+            parameters=completion_params,
+            stop=stop,
+        )

api/core/app/app_config/easy_ui_based_app/model_config/manager.py

@@ -0,0 +1,114 @@
+from collections.abc import Mapping
+from typing import Any
+
+from core.app.app_config.entities import ModelConfigEntity
+from core.model_runtime.entities.model_entities import ModelPropertyKey, ModelType
+from core.model_runtime.model_providers import model_provider_factory
+from core.provider_manager import ProviderManager
+
+
+class ModelConfigManager:
+    @classmethod
+    def convert(cls, config: dict) -> ModelConfigEntity:
+        """
+        Convert model config to model config
+
+        :param config: model config args
+        """
+        # model config
+        model_config = config.get("model")
+
+        if not model_config:
+            raise ValueError("model is required")
+
+        completion_params = model_config.get("completion_params")
+        stop = []
+        if "stop" in completion_params:
+            stop = completion_params["stop"]
+            del completion_params["stop"]
+
+        # get model mode
+        model_mode = model_config.get("mode")
+
+        return ModelConfigEntity(
+            provider=config["model"]["provider"],
+            model=config["model"]["name"],
+            mode=model_mode,
+            parameters=completion_params,
+            stop=stop,
+        )
+
+    @classmethod
+    def validate_and_set_defaults(cls, tenant_id: str, config: Mapping[str, Any]) -> tuple[dict, list[str]]:
+        """
+        Validate and set defaults for model config
+
+        :param tenant_id: tenant id
+        :param config: app model config args
+        """
+        if "model" not in config:
+            raise ValueError("model is required")
+
+        if not isinstance(config["model"], dict):
+            raise ValueError("model must be of object type")
+
+        # model.provider
+        provider_entities = model_provider_factory.get_providers()
+        model_provider_names = [provider.provider for provider in provider_entities]
+        if "provider" not in config["model"] or config["model"]["provider"] not in model_provider_names:
+            raise ValueError(f"model.provider is required and must be in {str(model_provider_names)}")
+
+        # model.name
+        if "name" not in config["model"]:
+            raise ValueError("model.name is required")
+
+        provider_manager = ProviderManager()
+        models = provider_manager.get_configurations(tenant_id).get_models(
+            provider=config["model"]["provider"], model_type=ModelType.LLM
+        )
+
+        if not models:
+            raise ValueError("model.name must be in the specified model list")
+
+        model_ids = [m.model for m in models]
+        if config["model"]["name"] not in model_ids:
+            raise ValueError("model.name must be in the specified model list")
+
+        model_mode = None
+        for model in models:
+            if model.model == config["model"]["name"]:
+                model_mode = model.model_properties.get(ModelPropertyKey.MODE)
+                break
+
+        # model.mode
+        if model_mode:
+            config["model"]["mode"] = model_mode
+        else:
+            config["model"]["mode"] = "completion"
+
+        # model.completion_params
+        if "completion_params" not in config["model"]:
+            raise ValueError("model.completion_params is required")
+
+        config["model"]["completion_params"] = cls.validate_model_completion_params(
+            config["model"]["completion_params"]
+        )
+
+        return dict(config), ["model"]
+
+    @classmethod
+    def validate_model_completion_params(cls, cp: dict) -> dict:
+        # model.completion_params
+        if not isinstance(cp, dict):
+            raise ValueError("model.completion_params must be of object type")
+
+        # stop
+        if "stop" not in cp:
+            cp["stop"] = []
+        elif not isinstance(cp["stop"], list):
+            raise ValueError("stop in model.completion_params must be of list type")
+
+        if len(cp["stop"]) > 4:
+            raise ValueError("stop sequences must be less than 4")
+
+        return cp

api/core/app/app_config/easy_ui_based_app/prompt_template/manager.py

@@ -0,0 +1,138 @@
+from core.app.app_config.entities import (
+    AdvancedChatMessageEntity,
+    AdvancedChatPromptTemplateEntity,
+    AdvancedCompletionPromptTemplateEntity,
+    PromptTemplateEntity,
+)
+from core.model_runtime.entities.message_entities import PromptMessageRole
+from core.prompt.simple_prompt_transform import ModelMode
+from models.model import AppMode
+
+
+class PromptTemplateConfigManager:
+    @classmethod
+    def convert(cls, config: dict) -> PromptTemplateEntity:
+        if not config.get("prompt_type"):
+            raise ValueError("prompt_type is required")
+
+        prompt_type = PromptTemplateEntity.PromptType.value_of(config["prompt_type"])
+        if prompt_type == PromptTemplateEntity.PromptType.SIMPLE:
+            simple_prompt_template = config.get("pre_prompt", "")
+            return PromptTemplateEntity(prompt_type=prompt_type, simple_prompt_template=simple_prompt_template)
+        else:
+            advanced_chat_prompt_template = None
+            chat_prompt_config = config.get("chat_prompt_config", {})
+            if chat_prompt_config:
+                chat_prompt_messages = []
+                for message in chat_prompt_config.get("prompt", []):
+                    chat_prompt_messages.append(
+                        AdvancedChatMessageEntity(
+                            **{"text": message["text"], "role": PromptMessageRole.value_of(message["role"])}
+                        )
+                    )
+
+                advanced_chat_prompt_template = AdvancedChatPromptTemplateEntity(messages=chat_prompt_messages)
+
+            advanced_completion_prompt_template = None
+            completion_prompt_config = config.get("completion_prompt_config", {})
+            if completion_prompt_config:
+                completion_prompt_template_params = {
+                    "prompt": completion_prompt_config["prompt"]["text"],
+                }
+
+                if "conversation_histories_role" in completion_prompt_config:
+                    completion_prompt_template_params["role_prefix"] = {
+                        "user": completion_prompt_config["conversation_histories_role"]["user_prefix"],
+                        "assistant": completion_prompt_config["conversation_histories_role"]["assistant_prefix"],
+                    }
+
+                advanced_completion_prompt_template = AdvancedCompletionPromptTemplateEntity(
+                    **completion_prompt_template_params
+                )
+
+            return PromptTemplateEntity(
+                prompt_type=prompt_type,
+                advanced_chat_prompt_template=advanced_chat_prompt_template,
+                advanced_completion_prompt_template=advanced_completion_prompt_template,
+            )
+
+    @classmethod
+    def validate_and_set_defaults(cls, app_mode: AppMode, config: dict) -> tuple[dict, list[str]]:
+        """
+        Validate pre_prompt and set defaults for prompt feature
+        depending on the config['model']
+
+        :param app_mode: app mode
+        :param config: app model config args
+        """
+        if not config.get("prompt_type"):
+            config["prompt_type"] = PromptTemplateEntity.PromptType.SIMPLE.value
+
+        prompt_type_vals = [typ.value for typ in PromptTemplateEntity.PromptType]
+        if config["prompt_type"] not in prompt_type_vals:
+            raise ValueError(f"prompt_type must be in {prompt_type_vals}")
+
+        # chat_prompt_config
+        if not config.get("chat_prompt_config"):
+            config["chat_prompt_config"] = {}
+
+        if not isinstance(config["chat_prompt_config"], dict):
+            raise ValueError("chat_prompt_config must be of object type")
+
+        # completion_prompt_config
+        if not config.get("completion_prompt_config"):
+            config["completion_prompt_config"] = {}
+
+        if not isinstance(config["completion_prompt_config"], dict):
+            raise ValueError("completion_prompt_config must be of object type")
+
+        if config["prompt_type"] == PromptTemplateEntity.PromptType.ADVANCED.value:
+            if not config["chat_prompt_config"] and not config["completion_prompt_config"]:
+                raise ValueError(
+                    "chat_prompt_config or completion_prompt_config is required when prompt_type is advanced"
+                )
+
+            model_mode_vals = [mode.value for mode in ModelMode]
+            if config["model"]["mode"] not in model_mode_vals:
+                raise ValueError(f"model.mode must be in {model_mode_vals} when prompt_type is advanced")
+
+            if app_mode == AppMode.CHAT and config["model"]["mode"] == ModelMode.COMPLETION.value:
+                user_prefix = config["completion_prompt_config"]["conversation_histories_role"]["user_prefix"]
+                assistant_prefix = config["completion_prompt_config"]["conversation_histories_role"]["assistant_prefix"]
+
+                if not user_prefix:
+                    config["completion_prompt_config"]["conversation_histories_role"]["user_prefix"] = "Human"
+
+                if not assistant_prefix:
+                    config["completion_prompt_config"]["conversation_histories_role"]["assistant_prefix"] = "Assistant"
+
+            if config["model"]["mode"] == ModelMode.CHAT.value:
+                prompt_list = config["chat_prompt_config"]["prompt"]
+
+                if len(prompt_list) > 10:
+                    raise ValueError("prompt messages must be less than 10")
+        else:
+            # pre_prompt, for simple mode
+            if not config.get("pre_prompt"):
+                config["pre_prompt"] = ""
+
+            if not isinstance(config["pre_prompt"], str):
+                raise ValueError("pre_prompt must be of string type")
+
+        return config, ["prompt_type", "pre_prompt", "chat_prompt_config", "completion_prompt_config"]
+
+    @classmethod
+    def validate_post_prompt_and_set_defaults(cls, config: dict) -> dict:
+        """
+        Validate post_prompt and set defaults for prompt feature
+
+        :param config: app model config args
+        """
+        # post_prompt
+        if not config.get("post_prompt"):
+            config["post_prompt"] = ""
+
+        if not isinstance(config["post_prompt"], str):
+            raise ValueError("post_prompt must be of string type")
+
+        return config

api/core/app/app_config/easy_ui_based_app/variables/manager.py

@@ -0,0 +1,168 @@
+import re
+
+from core.app.app_config.entities import ExternalDataVariableEntity, VariableEntity, VariableEntityType
+from core.external_data_tool.factory import ExternalDataToolFactory
+
+
+class BasicVariablesConfigManager:
+    @classmethod
+    def convert(cls, config: dict) -> tuple[list[VariableEntity], list[ExternalDataVariableEntity]]:
+        """
+        Convert model config to model config
+
+        :param config: model config args
+        """
+        external_data_variables = []
+        variable_entities = []
+
+        # old external_data_tools
+        external_data_tools = config.get("external_data_tools", [])
+        for external_data_tool in external_data_tools:
+            if "enabled" not in external_data_tool or not external_data_tool["enabled"]:
+                continue
+
+            external_data_variables.append(
+                ExternalDataVariableEntity(
+                    variable=external_data_tool["variable"],
+                    type=external_data_tool["type"],
+                    config=external_data_tool["config"],
+                )
+            )
+
+        # variables and external_data_tools
+        for variables in config.get("user_input_form", []):
+            variable_type = list(variables.keys())[0]
+            if variable_type == VariableEntityType.EXTERNAL_DATA_TOOL:
+                variable = variables[variable_type]
+                if "config" not in variable:
+                    continue
+
+                external_data_variables.append(
+                    ExternalDataVariableEntity(
+                        variable=variable["variable"], type=variable["type"], config=variable["config"]
+                    )
+                )
+            elif variable_type in {
+                VariableEntityType.TEXT_INPUT,
+                VariableEntityType.PARAGRAPH,
+                VariableEntityType.NUMBER,
+                VariableEntityType.SELECT,
+            }:
+                variable = variables[variable_type]
+                variable_entities.append(
+                    VariableEntity(
+                        type=variable_type,
+                        variable=variable.get("variable"),
+                        description=variable.get("description") or "",
+                        label=variable.get("label"),
+                        required=variable.get("required", False),
+                        max_length=variable.get("max_length"),
+                        options=variable.get("options") or [],
+                    )
+                )
+
+        return variable_entities, external_data_variables
+
+    @classmethod
+    def validate_and_set_defaults(cls, tenant_id: str, config: dict) -> tuple[dict, list[str]]:
+        """
+        Validate and set defaults for user input form
+
+        :param tenant_id: workspace id
+        :param config: app model config args
+        """
+        related_config_keys = []
+        config, current_related_config_keys = cls.validate_variables_and_set_defaults(config)
+        related_config_keys.extend(current_related_config_keys)
+
+        config, current_related_config_keys = cls.validate_external_data_tools_and_set_defaults(tenant_id, config)
+        related_config_keys.extend(current_related_config_keys)
+
+        return config, related_config_keys
+
+    @classmethod
+    def validate_variables_and_set_defaults(cls, config: dict) -> tuple[dict, list[str]]:
+        """
+        Validate and set defaults for user input form
+
+        :param config: app model config args
+        """
+        if not config.get("user_input_form"):
+            config["user_input_form"] = []
+
+        if not isinstance(config["user_input_form"], list):
+            raise ValueError("user_input_form must be a list of objects")
+
+        variables = []
+        for item in config["user_input_form"]:
+            key = list(item.keys())[0]
+            if key not in {"text-input", "select", "paragraph", "number", "external_data_tool"}:
+                raise ValueError("Keys in user_input_form list can only be 'text-input', 'paragraph'  or 'select'")
+
+            form_item = item[key]
+            if "label" not in form_item:
+                raise ValueError("label is required in user_input_form")
+
+            if not isinstance(form_item["label"], str):
+                raise ValueError("label in user_input_form must be of string type")
+
+            if "variable" not in form_item:
+                raise ValueError("variable is required in user_input_form")
+
+            if not isinstance(form_item["variable"], str):
+                raise ValueError("variable in user_input_form must be of string type")
+
+            pattern = re.compile(r"^(?!\d)[\u4e00-\u9fa5A-Za-z0-9_\U0001F300-\U0001F64F\U0001F680-\U0001F6FF]{1,100}$")
+            if pattern.match(form_item["variable"]) is None:
+                raise ValueError("variable in user_input_form must be a string, and cannot start with a number")
+
+            variables.append(form_item["variable"])
+
+            if "required" not in form_item or not form_item["required"]:
+                form_item["required"] = False
+
+            if not isinstance(form_item["required"], bool):
+                raise ValueError("required in user_input_form must be of boolean type")
+
+            if key == "select":
+                if "options" not in form_item or not form_item["options"]:
+                    form_item["options"] = []
+
+                if not isinstance(form_item["options"], list):
+                    raise ValueError("options in user_input_form must be a list of strings")
+
+                if "default" in form_item and form_item["default"] and form_item["default"] not in form_item["options"]:
+                    raise ValueError("default value in user_input_form must be in the options list")
+
+        return config, ["user_input_form"]
+
+    @classmethod
+    def validate_external_data_tools_and_set_defaults(cls, tenant_id: str, config: dict) -> tuple[dict, list[str]]:
+        """
+        Validate and set defaults for external data fetch feature
+
+        :param tenant_id: workspace id
+        :param config: app model config args
+        """
+        if not config.get("external_data_tools"):
+            config["external_data_tools"] = []
+
+        if not isinstance(config["external_data_tools"], list):
+            raise ValueError("external_data_tools must be of list type")
+
+        for tool in config["external_data_tools"]:
+            if "enabled" not in tool or not tool["enabled"]:
+                tool["enabled"] = False
+
+            if not tool["enabled"]:
+                continue
+
+            if "type" not in tool or not tool["type"]:
+                raise ValueError("external_data_tools[].type is required")
+
+            typ = tool["type"]
+            config = tool["config"]
+
+            ExternalDataToolFactory.validate_config(name=typ, tenant_id=tenant_id, config=config)
+
+        return config, ["external_data_tools"]

api/core/app/app_config/entities.py

@@ -0,0 +1,267 @@
+from collections.abc import Sequence
+from enum import Enum, StrEnum
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field, field_validator
+
+from core.file import FileTransferMethod, FileType, FileUploadConfig
+from core.model_runtime.entities.message_entities import PromptMessageRole
+from models.model import AppMode
+
+
+class ModelConfigEntity(BaseModel):
+    """
+    Model Config Entity.
+    """
+
+    provider: str
+    model: str
+    mode: Optional[str] = None
+    parameters: dict[str, Any] = {}
+    stop: list[str] = []
+
+
+class AdvancedChatMessageEntity(BaseModel):
+    """
+    Advanced Chat Message Entity.
+    """
+
+    text: str
+    role: PromptMessageRole
+
+
+class AdvancedChatPromptTemplateEntity(BaseModel):
+    """
+    Advanced Chat Prompt Template Entity.
+    """
+
+    messages: list[AdvancedChatMessageEntity]
+
+
+class AdvancedCompletionPromptTemplateEntity(BaseModel):
+    """
+    Advanced Completion Prompt Template Entity.
+    """
+
+    class RolePrefixEntity(BaseModel):
+        """
+        Role Prefix Entity.
+        """
+
+        user: str
+        assistant: str
+
+    prompt: str
+    role_prefix: Optional[RolePrefixEntity] = None
+
+
+class PromptTemplateEntity(BaseModel):
+    """
+    Prompt Template Entity.
+    """
+
+    class PromptType(Enum):
+        """
+        Prompt Type.
+        'simple', 'advanced'
+        """
+
+        SIMPLE = "simple"
+        ADVANCED = "advanced"
+
+        @classmethod
+        def value_of(cls, value: str):
+            """
+            Get value of given mode.
+
+            :param value: mode value
+            :return: mode
+            """
+            for mode in cls:
+                if mode.value == value:
+                    return mode
+            raise ValueError(f"invalid prompt type value {value}")
+
+    prompt_type: PromptType
+    simple_prompt_template: Optional[str] = None
+    advanced_chat_prompt_template: Optional[AdvancedChatPromptTemplateEntity] = None
+    advanced_completion_prompt_template: Optional[AdvancedCompletionPromptTemplateEntity] = None
+
+
+class VariableEntityType(StrEnum):
+    TEXT_INPUT = "text-input"
+    SELECT = "select"
+    PARAGRAPH = "paragraph"
+    NUMBER = "number"
+    EXTERNAL_DATA_TOOL = "external_data_tool"
+    FILE = "file"
+    FILE_LIST = "file-list"
+
+
+class VariableEntity(BaseModel):
+    """
+    Variable Entity.
+    """
+
+    variable: str
+    label: str
+    description: str = ""
+    type: VariableEntityType
+    required: bool = False
+    max_length: Optional[int] = None
+    options: Sequence[str] = Field(default_factory=list)
+    allowed_file_types: Sequence[FileType] = Field(default_factory=list)
+    allowed_file_extensions: Sequence[str] = Field(default_factory=list)
+    allowed_file_upload_methods: Sequence[FileTransferMethod] = Field(default_factory=list)
+
+    @field_validator("description", mode="before")
+    @classmethod
+    def convert_none_description(cls, v: Any) -> str:
+        return v or ""
+
+    @field_validator("options", mode="before")
+    @classmethod
+    def convert_none_options(cls, v: Any) -> Sequence[str]:
+        return v or []
+
+
+class ExternalDataVariableEntity(BaseModel):
+    """
+    External Data Variable Entity.
+    """
+
+    variable: str
+    type: str
+    config: dict[str, Any] = {}
+
+
+class DatasetRetrieveConfigEntity(BaseModel):
+    """
+    Dataset Retrieve Config Entity.
+    """
+
+    class RetrieveStrategy(Enum):
+        """
+        Dataset Retrieve Strategy.
+        'single' or 'multiple'
+        """
+
+        SINGLE = "single"
+        MULTIPLE = "multiple"
+
+        @classmethod
+        def value_of(cls, value: str):
+            """
+            Get value of given mode.
+
+            :param value: mode value
+            :return: mode
+            """
+            for mode in cls:
+                if mode.value == value:
+                    return mode
+            raise ValueError(f"invalid retrieve strategy value {value}")
+
+    query_variable: Optional[str] = None  # Only when app mode is completion
+
+    retrieve_strategy: RetrieveStrategy
+    top_k: Optional[int] = None
+    score_threshold: Optional[float] = 0.0
+    rerank_mode: Optional[str] = "reranking_model"
+    reranking_model: Optional[dict] = None
+    weights: Optional[dict] = None
+    reranking_enabled: Optional[bool] = True
+
+
+class DatasetEntity(BaseModel):
+    """
+    Dataset Config Entity.
+    """
+
+    dataset_ids: list[str]
+    retrieve_config: DatasetRetrieveConfigEntity
+
+
+class SensitiveWordAvoidanceEntity(BaseModel):
+    """
+    Sensitive Word Avoidance Entity.
+    """
+
+    type: str
+    config: dict[str, Any] = {}
+
+
+class TextToSpeechEntity(BaseModel):
+    """
+    Sensitive Word Avoidance Entity.
+    """
+
+    enabled: bool
+    voice: Optional[str] = None
+    language: Optional[str] = None
+
+
+class TracingConfigEntity(BaseModel):
+    """
+    Tracing Config Entity.
+    """
+
+    enabled: bool
+    tracing_provider: str
+
+
+class AppAdditionalFeatures(BaseModel):
+    file_upload: Optional[FileUploadConfig] = None
+    opening_statement: Optional[str] = None
+    suggested_questions: list[str] = []
+    suggested_questions_after_answer: bool = False
+    show_retrieve_source: bool = False
+    more_like_this: bool = False
+    speech_to_text: bool = False
+    text_to_speech: Optional[TextToSpeechEntity] = None
+    trace_config: Optional[TracingConfigEntity] = None
+
+
+class AppConfig(BaseModel):
+    """
+    Application Config Entity.
+    """
+
+    tenant_id: str
+    app_id: str
+    app_mode: AppMode
+    additional_features: AppAdditionalFeatures
+    variables: list[VariableEntity] = []
+    sensitive_word_avoidance: Optional[SensitiveWordAvoidanceEntity] = None
+
+
+class EasyUIBasedAppModelConfigFrom(Enum):
+    """
+    App Model Config From.
+    """
+
+    ARGS = "args"
+    APP_LATEST_CONFIG = "app-latest-config"
+    CONVERSATION_SPECIFIC_CONFIG = "conversation-specific-config"
+
+
+class EasyUIBasedAppConfig(AppConfig):
+    """
+    Easy UI Based App Config Entity.
+    """
+
+    app_model_config_from: EasyUIBasedAppModelConfigFrom
+    app_model_config_id: str
+    app_model_config_dict: dict
+    model: ModelConfigEntity
+    prompt_template: PromptTemplateEntity
+    dataset: Optional[DatasetEntity] = None
+    external_data_variables: list[ExternalDataVariableEntity] = []
+
+
+class WorkflowUIBasedAppConfig(AppConfig):
+    """
+    Workflow UI Based App Config Entity.
+    """
+
+    workflow_id: str

api/core/app/app_config/features/file_upload/manager.py

@@ -0,0 +1,44 @@
+from collections.abc import Mapping
+from typing import Any
+
+from core.file import FileUploadConfig
+
+
+class FileUploadConfigManager:
+    @classmethod
+    def convert(cls, config: Mapping[str, Any], is_vision: bool = True):
+        """
+        Convert model config to model config
+
+        :param config: model config args
+        :param is_vision: if True, the feature is vision feature
+        """
+        file_upload_dict = config.get("file_upload")
+        if file_upload_dict:
+            if file_upload_dict.get("enabled"):
+                transform_methods = file_upload_dict.get("allowed_file_upload_methods", [])
+                data = {
+                    "image_config": {
+                        "number_limits": file_upload_dict["number_limits"],
+                        "transfer_methods": transform_methods,
+                    }
+                }
+
+                if is_vision:
+                    data["image_config"]["detail"] = file_upload_dict.get("image", {}).get("detail", "low")
+
+                return FileUploadConfig.model_validate(data)
+
+    @classmethod
+    def validate_and_set_defaults(cls, config: dict) -> tuple[dict, list[str]]:
+        """
+        Validate and set defaults for file upload feature
+
+        :param config: app model config args
+        """
+        if not config.get("file_upload"):
+            config["file_upload"] = {}
+        else:
+            FileUploadConfig.model_validate(config["file_upload"])
+
+        return config, ["file_upload"]

api/core/app/app_config/features/more_like_this/manager.py

@@ -0,0 +1,36 @@
+class MoreLikeThisConfigManager:
+    @classmethod
+    def convert(cls, config: dict) -> bool:
+        """
+        Convert model config to model config
+
+        :param config: model config args
+        """
+        more_like_this = False
+        more_like_this_dict = config.get("more_like_this")
+        if more_like_this_dict:
+            if more_like_this_dict.get("enabled"):
+                more_like_this = True
+
+        return more_like_this
+
+    @classmethod
+    def validate_and_set_defaults(cls, config: dict) -> tuple[dict, list[str]]:
+        """
+        Validate and set defaults for more like this feature
+
+        :param config: app model config args
+        """
+        if not config.get("more_like_this"):
+            config["more_like_this"] = {"enabled": False}
+
+        if not isinstance(config["more_like_this"], dict):
+            raise ValueError("more_like_this must be of dict type")
+
+        if "enabled" not in config["more_like_this"] or not config["more_like_this"]["enabled"]:
+            config["more_like_this"]["enabled"] = False
+
+        if not isinstance(config["more_like_this"]["enabled"], bool):
+            raise ValueError("enabled in more_like_this must be of boolean type")
+
+        return config, ["more_like_this"]

api/core/app/app_config/features/opening_statement/manager.py

@@ -0,0 +1,41 @@
+class OpeningStatementConfigManager:
+    @classmethod
+    def convert(cls, config: dict) -> tuple[str, list]:
+        """
+        Convert model config to model config
+
+        :param config: model config args
+        """
+        # opening statement
+        opening_statement = config.get("opening_statement", "")
+
+        # suggested questions
+        suggested_questions_list = config.get("suggested_questions", [])
+
+        return opening_statement, suggested_questions_list
+
+    @classmethod
+    def validate_and_set_defaults(cls, config: dict) -> tuple[dict, list[str]]:
+        """
+        Validate and set defaults for opening statement feature
+
+        :param config: app model config args
+        """
+        if not config.get("opening_statement"):
+            config["opening_statement"] = ""
+
+        if not isinstance(config["opening_statement"], str):
+            raise ValueError("opening_statement must be of string type")
+
+        # suggested_questions
+        if not config.get("suggested_questions"):
+            config["suggested_questions"] = []
+
+        if not isinstance(config["suggested_questions"], list):
+            raise ValueError("suggested_questions must be of list type")
+
+        for question in config["suggested_questions"]:
+            if not isinstance(question, str):
+                raise ValueError("Elements in suggested_questions list must be of string type")
+
+        return config, ["opening_statement", "suggested_questions"]

api/core/app/app_config/features/retrieval_resource/manager.py

@@ -0,0 +1,31 @@
+class RetrievalResourceConfigManager:
+    @classmethod
+    def convert(cls, config: dict) -> bool:
+        show_retrieve_source = False
+        retriever_resource_dict = config.get("retriever_resource")
+        if retriever_resource_dict:
+            if retriever_resource_dict.get("enabled"):
+                show_retrieve_source = True
+
+        return show_retrieve_source
+
+    @classmethod
+    def validate_and_set_defaults(cls, config: dict) -> tuple[dict, list[str]]:
+        """
+        Validate and set defaults for retriever resource feature
+
+        :param config: app model config args
+        """
+        if not config.get("retriever_resource"):
+            config["retriever_resource"] = {"enabled": False}
+
+        if not isinstance(config["retriever_resource"], dict):
+            raise ValueError("retriever_resource must be of dict type")
+
+        if "enabled" not in config["retriever_resource"] or not config["retriever_resource"]["enabled"]:
+            config["retriever_resource"]["enabled"] = False
+
+        if not isinstance(config["retriever_resource"]["enabled"], bool):
+            raise ValueError("enabled in retriever_resource must be of boolean type")
+
+        return config, ["retriever_resource"]

api/core/app/app_config/features/speech_to_text/manager.py

@@ -0,0 +1,36 @@
+class SpeechToTextConfigManager:
+    @classmethod
+    def convert(cls, config: dict) -> bool:
+        """
+        Convert model config to model config
+
+        :param config: model config args
+        """
+        speech_to_text = False
+        speech_to_text_dict = config.get("speech_to_text")
+        if speech_to_text_dict:
+            if speech_to_text_dict.get("enabled"):
+                speech_to_text = True
+
+        return speech_to_text
+
+    @classmethod
+    def validate_and_set_defaults(cls, config: dict) -> tuple[dict, list[str]]:
+        """
+        Validate and set defaults for speech to text feature
+
+        :param config: app model config args
+        """
+        if not config.get("speech_to_text"):
+            config["speech_to_text"] = {"enabled": False}
+
+        if not isinstance(config["speech_to_text"], dict):
+            raise ValueError("speech_to_text must be of dict type")
+
+        if "enabled" not in config["speech_to_text"] or not config["speech_to_text"]["enabled"]:
+            config["speech_to_text"]["enabled"] = False
+
+        if not isinstance(config["speech_to_text"]["enabled"], bool):
+            raise ValueError("enabled in speech_to_text must be of boolean type")
+
+        return config, ["speech_to_text"]

api/core/app/app_config/features/suggested_questions_after_answer/manager.py

@@ -0,0 +1,39 @@
+class SuggestedQuestionsAfterAnswerConfigManager:
+    @classmethod
+    def convert(cls, config: dict) -> bool:
+        """
+        Convert model config to model config
+
+        :param config: model config args
+        """
+        suggested_questions_after_answer = False
+        suggested_questions_after_answer_dict = config.get("suggested_questions_after_answer")
+        if suggested_questions_after_answer_dict:
+            if suggested_questions_after_answer_dict.get("enabled"):
+                suggested_questions_after_answer = True
+
+        return suggested_questions_after_answer
+
+    @classmethod
+    def validate_and_set_defaults(cls, config: dict) -> tuple[dict, list[str]]:
+        """
+        Validate and set defaults for suggested questions feature
+
+        :param config: app model config args
+        """
+        if not config.get("suggested_questions_after_answer"):
+            config["suggested_questions_after_answer"] = {"enabled": False}
+
+        if not isinstance(config["suggested_questions_after_answer"], dict):
+            raise ValueError("suggested_questions_after_answer must be of dict type")
+
+        if (
+            "enabled" not in config["suggested_questions_after_answer"]
+            or not config["suggested_questions_after_answer"]["enabled"]
+        ):
+            config["suggested_questions_after_answer"]["enabled"] = False
+
+        if not isinstance(config["suggested_questions_after_answer"]["enabled"], bool):
+            raise ValueError("enabled in suggested_questions_after_answer must be of boolean type")
+
+        return config, ["suggested_questions_after_answer"]

api/core/app/app_config/features/text_to_speech/manager.py

@@ -0,0 +1,45 @@
+from core.app.app_config.entities import TextToSpeechEntity
+
+
+class TextToSpeechConfigManager:
+    @classmethod
+    def convert(cls, config: dict):
+        """
+        Convert model config to model config
+
+        :param config: model config args
+        """
+        text_to_speech = None
+        text_to_speech_dict = config.get("text_to_speech")
+        if text_to_speech_dict:
+            if text_to_speech_dict.get("enabled"):
+                text_to_speech = TextToSpeechEntity(
+                    enabled=text_to_speech_dict.get("enabled"),
+                    voice=text_to_speech_dict.get("voice"),
+                    language=text_to_speech_dict.get("language"),
+                )
+
+        return text_to_speech
+
+    @classmethod
+    def validate_and_set_defaults(cls, config: dict) -> tuple[dict, list[str]]:
+        """
+        Validate and set defaults for text to speech feature
+
+        :param config: app model config args
+        """
+        if not config.get("text_to_speech"):
+            config["text_to_speech"] = {"enabled": False, "voice": "", "language": ""}
+
+        if not isinstance(config["text_to_speech"], dict):
+            raise ValueError("text_to_speech must be of dict type")
+
+        if "enabled" not in config["text_to_speech"] or not config["text_to_speech"]["enabled"]:
+            config["text_to_speech"]["enabled"] = False
+            config["text_to_speech"]["voice"] = ""
+            config["text_to_speech"]["language"] = ""
+
+        if not isinstance(config["text_to_speech"]["enabled"], bool):
+            raise ValueError("enabled in text_to_speech must be of boolean type")
+
+        return config, ["text_to_speech"]

api/core/app/app_config/workflow_ui_based_app/variables/manager.py

@@ -0,0 +1,22 @@
+from core.app.app_config.entities import VariableEntity
+from models.workflow import Workflow
+
+
+class WorkflowVariablesConfigManager:
+    @classmethod
+    def convert(cls, workflow: Workflow) -> list[VariableEntity]:
+        """
+        Convert workflow start variables to variables
+
+        :param workflow: workflow instance
+        """
+        variables = []
+
+        # find start node
+        user_input_form = workflow.user_input_form()
+
+        # variables
+        for variable in user_input_form:
+            variables.append(VariableEntity.model_validate(variable))
+
+        return variables

api/core/app/apps/README.md

@@ -0,0 +1,48 @@
+## Guidelines for Database Connection Management in App Runner and Task Pipeline
+
+Due to the presence of tasks in App Runner that require long execution times, such as LLM generation and external requests, Flask-Sqlalchemy's strategy for database connection pooling is to allocate one connection (transaction) per request. This approach keeps a connection occupied even during non-DB tasks, leading to the inability to acquire new connections during high concurrency requests due to multiple long-running tasks.
+
+Therefore, the database operations in App Runner and Task Pipeline must ensure connections are closed immediately after use, and it's better to pass IDs rather than Model objects to avoid detach errors.
+
+Examples:
+
+1. Creating a new record:
+
+   ```python
+   app = App(id=1)
+   db.session.add(app)
+   db.session.commit()
+   db.session.refresh(app)  # Retrieve table default values, like created_at, cached in the app object, won't affect after close
+   
+   # Handle non-long-running tasks or store the content of the App instance in memory (via variable assignment).
+   
+   db.session.close()
+   
+   return app.id
+   ```
+
+2. Fetching a record from the table:
+
+   ```python
+   app = db.session.query(App).filter(App.id == app_id).first()
+    
+   created_at = app.created_at
+    
+   db.session.close()
+   
+   # Handle tasks (include long-running).
+   
+   ```
+
+3. Updating a table field:
+
+   ```python
+   app = db.session.query(App).filter(App.id == app_id).first()
+
+   app.updated_at = time.utcnow()
+   db.session.commit()
+   db.session.close()
+
+   return app_id
+   ```
+   

api/core/app/apps/advanced_chat/app_config_manager.py

@@ -0,0 +1,91 @@
+from core.app.app_config.base_app_config_manager import BaseAppConfigManager
+from core.app.app_config.common.sensitive_word_avoidance.manager import SensitiveWordAvoidanceConfigManager
+from core.app.app_config.entities import WorkflowUIBasedAppConfig
+from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
+from core.app.app_config.features.opening_statement.manager import OpeningStatementConfigManager
+from core.app.app_config.features.retrieval_resource.manager import RetrievalResourceConfigManager
+from core.app.app_config.features.speech_to_text.manager import SpeechToTextConfigManager
+from core.app.app_config.features.suggested_questions_after_answer.manager import (
+    SuggestedQuestionsAfterAnswerConfigManager,
+)
+from core.app.app_config.features.text_to_speech.manager import TextToSpeechConfigManager
+from core.app.app_config.workflow_ui_based_app.variables.manager import WorkflowVariablesConfigManager
+from models.model import App, AppMode
+from models.workflow import Workflow
+
+
+class AdvancedChatAppConfig(WorkflowUIBasedAppConfig):
+    """
+    Advanced Chatbot App Config Entity.
+    """
+
+    pass
+
+
+class AdvancedChatAppConfigManager(BaseAppConfigManager):
+    @classmethod
+    def get_app_config(cls, app_model: App, workflow: Workflow) -> AdvancedChatAppConfig:
+        features_dict = workflow.features_dict
+
+        app_mode = AppMode.value_of(app_model.mode)
+        app_config = AdvancedChatAppConfig(
+            tenant_id=app_model.tenant_id,
+            app_id=app_model.id,
+            app_mode=app_mode,
+            workflow_id=workflow.id,
+            sensitive_word_avoidance=SensitiveWordAvoidanceConfigManager.convert(config=features_dict),
+            variables=WorkflowVariablesConfigManager.convert(workflow=workflow),
+            additional_features=cls.convert_features(features_dict, app_mode),
+        )
+
+        return app_config
+
+    @classmethod
+    def config_validate(cls, tenant_id: str, config: dict, only_structure_validate: bool = False) -> dict:
+        """
+        Validate for advanced chat app model config
+
+        :param tenant_id: tenant id
+        :param config: app model config args
+        :param only_structure_validate: if True, only structure validation will be performed
+        """
+        related_config_keys = []
+
+        # file upload validation
+        config, current_related_config_keys = FileUploadConfigManager.validate_and_set_defaults(config=config)
+        related_config_keys.extend(current_related_config_keys)
+
+        # opening_statement
+        config, current_related_config_keys = OpeningStatementConfigManager.validate_and_set_defaults(config)
+        related_config_keys.extend(current_related_config_keys)
+
+        # suggested_questions_after_answer
+        config, current_related_config_keys = SuggestedQuestionsAfterAnswerConfigManager.validate_and_set_defaults(
+            config
+        )
+        related_config_keys.extend(current_related_config_keys)
+
+        # speech_to_text
+        config, current_related_config_keys = SpeechToTextConfigManager.validate_and_set_defaults(config)
+        related_config_keys.extend(current_related_config_keys)
+
+        # text_to_speech
+        config, current_related_config_keys = TextToSpeechConfigManager.validate_and_set_defaults(config)
+        related_config_keys.extend(current_related_config_keys)
+
+        # return retriever resource
+        config, current_related_config_keys = RetrievalResourceConfigManager.validate_and_set_defaults(config)
+        related_config_keys.extend(current_related_config_keys)
+
+        # moderation validation
+        config, current_related_config_keys = SensitiveWordAvoidanceConfigManager.validate_and_set_defaults(
+            tenant_id=tenant_id, config=config, only_structure_validate=only_structure_validate
+        )
+        related_config_keys.extend(current_related_config_keys)
+
+        related_config_keys = list(set(related_config_keys))
+
+        # Filter out extra parameters
+        filtered_config = {key: config.get(key) for key in related_config_keys}
+
+        return filtered_config