feat: Added auto_keywords and auto_questions fields to the parsing configuration page #2687 (#2987)

### What problem does this PR solve?

feat: Added auto_keywords and auto_questions fields to the parsing
configuration page #2687

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

web/src/components/auto-keywords-item.tsx

@@ -0,0 +1,36 @@
+import { useTranslate } from '@/hooks/common-hooks';
+import { Form, InputNumber } from 'antd';
+
+const style = {
+  width: '100%',
+};
+
+export const AutoKeywordsItem = () => {
+  const { t } = useTranslate('knowledgeDetails');
+
+  return (
+    <Form.Item
+      label={t('autoKeywords')}
+      name={['parser_config', 'auto_keywords']}
+      tooltip={t('autoKeywordsTip')}
+      initialValue={0}
+    >
+      <InputNumber style={style}></InputNumber>
+    </Form.Item>
+  );
+};
+
+export const AutoQuestionsItem = () => {
+  const { t } = useTranslate('knowledgeDetails');
+
+  return (
+    <Form.Item
+      label={t('autoQuestions')}
+      name={['parser_config', 'auto_questions']}
+      tooltip={t('autoQuestionsTip')}
+      initialValue={0}
+    >
+      <InputNumber style={style}></InputNumber>
+    </Form.Item>
+  );
+};

web/src/components/chunk-method-modal/hooks.ts

@@ -1,7 +1,7 @@
 import { useHandleChunkMethodSelectChange } from '@/hooks/logic-hooks';
 import { useSelectParserList } from '@/hooks/user-setting-hooks';
 import { FormInstance } from 'antd';
-import { useEffect, useMemo, useState } from 'react';
+import { useCallback, useEffect, useMemo, useState } from 'react';
 
 const ParserListMap = new Map([
   [
@@ -118,3 +118,13 @@ export const useFetchParserListOnMount = (
 
   return { parserList: nextParserList, handleChange, selectedTag };
 };
+
+const hideAutoKeywords = ['qa', 'table', 'resume', 'knowledge_graph'];
+
+export const useShowAutoKeywords = () => {
+  const showAutoKeywords = useCallback((selectedTag: string) => {
+    return hideAutoKeywords.every((x) => selectedTag !== x);
+  }, []);
+
+  return showAutoKeywords;
+};

web/src/components/chunk-method-modal/index.tsx

@@ -17,11 +17,12 @@ import {
 } from 'antd';
 import omit from 'lodash/omit';
 import React, { useEffect, useMemo } from 'react';
-import { useFetchParserListOnMount } from './hooks';
+import { useFetchParserListOnMount, useShowAutoKeywords } from './hooks';
 
 import { useTranslate } from '@/hooks/common-hooks';
 import { IParserConfig } from '@/interfaces/database/document';
 import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
+import { AutoKeywordsItem, AutoQuestionsItem } from '../auto-keywords-item';
 import Delimiter from '../delimiter';
 import EntityTypesItem from '../entity-types-item';
 import ExcelToHtml from '../excel-to-html';
@@ -108,6 +109,8 @@ const ChunkMethodModal: React.FC<IProps> = ({
   const showExcelToHtml =
     selectedTag === 'naive' && documentExtension === 'xlsx';
 
+  const showAutoKeywords = useShowAutoKeywords();
+
   const afterClose = () => {
     form.resetFields();
   };
@@ -283,6 +286,12 @@ const ChunkMethodModal: React.FC<IProps> = ({
             <Delimiter></Delimiter>
           </>
         )}
+        {showAutoKeywords(selectedTag) && (
+          <>
+            <AutoKeywordsItem></AutoKeywordsItem>
+            <AutoQuestionsItem></AutoQuestionsItem>
+          </>
+        )}
         {showExcelToHtml && <ExcelToHtml></ExcelToHtml>}
         {showRaptorParseConfiguration(selectedTag) && (
           <ParseConfiguration></ParseConfiguration>

web/src/locales/en.ts

@@ -154,6 +154,10 @@ export default {
       delimiter: `Delimiter`,
       html4excel: 'Excel to HTML',
       html4excelTip: `Excel will be parsed into HTML table or not. If it's FALSE, every row in Excel will be formed as a chunk.`,
+      autoKeywords: 'Auto keywords',
+      autoKeywordsTip: `Extract N keywords for every chunk to boost their rank score while querying such keywords. Extra tokens will be comsumed for LLM that you set in 'System model settings'. You can check the result in the chunk list.`,
+      autoQuestions: 'Auto questions',
+      autoQuestionsTip: `Extract N questions for every chunk to boost their rank score while querying such questions. Extra tokens will be comsumed for LLM that you set in 'System model settings'. You can check the result in the chunk list. This function will not destroy the entire chunking process if errors occur except adding empty result to the original chunk.`,
     },
     knowledgeConfiguration: {
       titleDescription:

web/src/locales/zh-traditional.ts

@@ -150,6 +150,10 @@ export default {
       delimiter: `分段標識符`,
       html4excel: '表格轉HTML',
       html4excelTip: `Excel 是否會被解析為 HTML 表格。如果為 FALSE，Excel 中的每一行都會形成一個區塊。`,
+      autoKeywords: '自動關鍵字',
+      autoKeywordsTip: `在查詢此類關鍵字時，為每個區塊提取 N 個關鍵字以提高其排名分數。在「系統模型設定」中設定的 LLM 將消耗額外的 token。您可以在區塊清單中查看結果。 `,
+      autoQuestions: '自動問題',
+      autoQuestionsTip: `在查詢此類問題時，為每個區塊提取 N 個問題以提高其排名分數。在「系統模型設定」中設定的 LLM 將消耗額外的 token。您可以在區塊清單中查看結果。如果發生錯誤，此功能不會破壞整個分塊過程，除了將空結果新增至原始區塊。 `,
     },
     knowledgeConfiguration: {
       titleDescription: '在這裡更新您的知識庫詳細信息，尤其是解析方法。',

web/src/locales/zh.ts

@@ -151,6 +151,10 @@ export default {
       delimiter: `分段标识符`,
       html4excel: '表格转HTML',
       html4excelTip: `Excel 是否将被解析为 HTML 表。如果为 FALSE，Excel 中的每一行都将形成一个块。`,
+      autoKeywords: '自动关键词',
+      autoKeywordsTip: `在查询此类关键词时，为每个块提取 N 个关键词以提高其排名得分。在“系统模型设置”中设置的 LLM 将消耗额外的 token。您可以在块列表中查看结果。`,
+      autoQuestions: '自动问题',
+      autoQuestionsTip: `在查询此类问题时，为每个块提取 N 个问题以提高其排名得分。在“系统模型设置”中设置的 LLM 将消耗额外的 token。您可以在块列表中查看结果。如果发生错误，此功能不会破坏整个分块过程，除了将空结果添加到原始块。`,
     },
     knowledgeConfiguration: {
       titleDescription: '在这里更新您的知识库详细信息，尤其是解析方法。',

web/src/pages/add-knowledge/components/knowledge-setting/configuration.tsx

@@ -1,3 +1,8 @@
+import {
+  AutoKeywordsItem,
+  AutoQuestionsItem,
+} from '@/components/auto-keywords-item';
+import { useShowAutoKeywords } from '@/components/chunk-method-modal/hooks';
 import Delimiter from '@/components/delimiter';
 import EntityTypesItem from '@/components/entity-types-item';
 import ExcelToHtml from '@/components/excel-to-html';
@@ -27,6 +32,7 @@ const ConfigurationForm = ({ form }: { form: FormInstance }) => {
     useFetchKnowledgeConfigurationOnMount(form);
   const { t } = useTranslate('knowledgeConfiguration');
   const handleChunkMethodSelectChange = useHandleChunkMethodSelectChange(form);
+  const showAutoKeywords = useShowAutoKeywords();
 
   return (
     <Form form={form} name="validateOnly" layout="vertical" autoComplete="off">
@@ -120,6 +126,12 @@ const ConfigurationForm = ({ form }: { form: FormInstance }) => {
                   <Delimiter></Delimiter>
                 </>
               )}
+              {showAutoKeywords(parserId) && (
+                <>
+                  <AutoKeywordsItem></AutoKeywordsItem>
+                  <AutoQuestionsItem></AutoQuestionsItem>
+                </>
+              )}
               {parserId === 'naive' && (
                 <>
                   <MaxTokenNumber></MaxTokenNumber>
@@ -128,6 +140,7 @@ const ConfigurationForm = ({ form }: { form: FormInstance }) => {
                   <ExcelToHtml></ExcelToHtml>
                 </>
               )}
+
               {showRaptorParseConfiguration(parserId) && (
                 <ParseConfiguration></ParseConfiguration>
               )}