Add Graph Baidusearch and dsl_example (#1378)

### What problem does this PR solve?

#918

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

graph/component/__init__.py

@@ -8,6 +8,8 @@ from .switch import Switch, SwitchParam
 from .relevant import Relevant, RelevantParam
 from .message import Message, MessageParam
 from .rewrite import RewriteQuestion, RewriteQuestionParam
+from .keyword import KeywordExtract, KeywordExtractParam
+from .baidu import Baidu, BaiduParam
 
 
 def component_class(class_name):

graph/component/baidu.py

@@ -0,0 +1,62 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import random
+from abc import ABC
+from functools import partial
+import pandas as pd
+import requests
+import re
+
+from graph.component.base import ComponentBase, ComponentParamBase
+
+
+class BaiduParam(ComponentParamBase):
+    """
+    Define the Baidu component parameters.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.top_n = 10
+
+    def check(self):
+        self.check_positive_integer(self.top_n, "Top N")
+
+
+class Baidu(ComponentBase, ABC):
+    component_name = "Baidu"
+
+    def _run(self, history, **kwargs):
+        ans = self.get_input()
+        ans = " - ".join(ans["content"]) if "content" in ans else ""
+        if not ans:
+            return Baidu.be_output(self._param.no)
+
+        url = 'https://www.baidu.com/s?wd=' + ans + '&rn=' + str(self._param.top_n)
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36'}
+        response = requests.get(url=url, headers=headers)
+
+        baidu_res = re.findall(r'"contentText":"(.*?)"', response.text)
+        url_res = re.findall(r"'url': \\\"(.*?)\\\"}", response.text)
+        for i in range(min(len(baidu_res), len(url_res))):
+            baidu_res[i] += '<a>' + url_res[i] + '</a>'
+
+        del url_res
+
+        br = pd.DataFrame(baidu_res, columns=['content'])
+        print(">>>>>>>>>>>>>>>>>>>>>>>>>>\n", br)
+        return br

graph/component/keyword.py

@@ -22,18 +22,17 @@ from graph.settings import DEBUG
 
 
 class KeywordExtractParam(GenerateParam):
-
     """
     Define the KeywordExtract component parameters.
     """
+
     def __init__(self):
         super().__init__()
-        self.temperature = 0.5
-        self.prompt = ""
-        self.topn = 1
+        self.top_n = 1
 
     def check(self):
         super().check()
+        self.check_positive_integer(self.top_n, "Top N")
 
     def get_prompt(self):
         self.prompt = """
@@ -43,12 +42,12 @@ class KeywordExtractParam(GenerateParam):
   - Use comma as a delimiter to separate keywords/phrases.
 - Answer format: (in language of user's question)
   - keyword: 
-"""%self.topn
+""" % self.top_n
         return self.prompt
 
 
 class KeywordExtract(Generate, ABC):
-    component_name = "RewriteQuestion"
+    component_name = "KeywordExtract"
 
     def _run(self, history, **kwargs):
         q = ""
@@ -64,5 +63,3 @@ class KeywordExtract(Generate, ABC):
         ans = re.sub(r".*keyword:", "", ans).strip()
         if DEBUG: print(ans, ":::::::::::::::::::::::::::::::::")
         return KeywordExtract.be_output(ans)
-
-

graph/test/dsl_examples/retrieval_relevant_keyword_baidu_and_generate.json

@@ -0,0 +1,103 @@
+{
+  "components": {
+            "begin": {
+                "obj":{
+                    "component_name": "Begin",
+                    "params": {
+                      "prologue": "Hi there!"
+                    }
+                },
+                "downstream": ["answer:0"],
+                "upstream": []
+            },
+            "answer:0": {
+                "obj": {
+                    "component_name": "Answer",
+                    "params": {}
+                },
+                "downstream": ["retrieval:0"],
+                "upstream": ["begin"]
+            },
+            "retrieval:0": {
+                "obj": {
+                    "component_name": "Retrieval",
+                    "params": {
+                      "similarity_threshold": 0.2,
+                      "keywords_similarity_weight": 0.3,
+                      "top_n": 6,
+                      "top_k": 1024,
+                      "rerank_id": "BAAI/bge-reranker-v2-m3",
+                      "kb_ids": ["21ca4e6a2c8911ef8b1e0242ac120006"],
+                      "empty_response": "Sorry, knowledge base has noting related information."
+                    }
+                },
+                "downstream": ["relevant:0"],
+                "upstream": ["answer:0"]
+            },
+            "relevant:0": {
+                "obj": {
+                    "component_name": "Relevant",
+                    "params": {
+                      "llm_id": "deepseek-chat",
+                      "temperature": 0.02,
+                      "yes": "generate:0",
+                      "no": "keyword:0"
+                    }
+                },
+                "downstream": ["keyword:0", "generate:0"],
+                "upstream": ["retrieval:0"]
+            },
+            "generate:0": {
+                "obj": {
+                    "component_name": "Generate",
+                    "params": {
+                      "llm_id": "deepseek-chat",
+                      "prompt": "You are an intelligent assistant. Please answer the question based on content of knowledge base. When all knowledge base content is irrelevant to the question, your answer must include the sentence \"The answer you are looking for is not found in the knowledge base!\". Answers need to consider chat history.\n      Knowledge base content is as following:\n      {input}\n      The above is the content of knowledge base.",
+                      "temperature": 0.2
+                    }
+                },
+                "downstream": ["answer:0"],
+                "upstream": ["relevant:0"]
+            },
+            "keyword:0": {
+                "obj": {
+                    "component_name": "KeywordExtract",
+                    "params": {
+                      "llm_id": "deepseek-chat",
+                      "prompt": "- Role: You're a question analyzer.\n    - Requirements:\n     - Summarize user's question, and give top %s important keyword/phrase.\n    - Use comma as a delimiter to separate keywords/phrases.\n    - Answer format: (in language of user's question)\n    - keyword: ",
+                      "temperature": 0.2,
+                      "top_n": 1
+                    }
+                },
+                "downstream": ["baidu:0"],
+                "upstream": ["relevant:0"]
+            },
+            "baidu:0": {
+                "obj":{
+                    "component_name": "Baidu",
+                    "params": {
+                      "top_n": 10
+                    }
+                },
+                "downstream": ["generate:1"],
+                "upstream": ["keyword:0"]
+            },
+            "generate:1": {
+                "obj": {
+                    "component_name": "Generate",
+                    "params": {
+                      "llm_id": "deepseek-chat",
+                      "prompt": "You are an intelligent assistant. Please answer the question based on content searched from Baidu. When the answer from a Baidu search is incomplete, you need to output the URL link of the corresponding content as well. When all the content searched from Baidu is irrelevant to the question, your answer must include the sentence, \"The answer you are looking for is not found in the Baidu search!\". Answers need to consider chat history.\n       The content of Baidu search is as follows:\n    {input}\n     The above is the content of Baidu search.",
+                      "temperature": 0.2
+                    }
+                },
+                "downstream": ["answer:0"],
+                "upstream": ["baidu:0"]
+            }
+  },
+  "history": [],
+  "path": [],
+  "messages": [],
+  "reference": {},
+  "answer": []
+}