const getImageName = (prefix: string, length: number) => new Array(length) .fill(0) .map((x, idx) => `chunk-method/${prefix}-0${idx + 1}`); export const ImageMap = { book: getImageName('book', 4), laws: getImageName('law', 2), manual: getImageName('manual', 4), picture: getImageName('media', 2), naive: getImageName('naive', 2), paper: getImageName('paper', 2), presentation: getImageName('presentation', 2), qa: getImageName('qa', 2), resume: getImageName('resume', 2), table: getImageName('table', 2), one: getImageName('one', 2), knowledge_graph: getImageName('knowledge-graph', 2), }; export const TextMap = { book: { title: '', description: `
Supported file formats are DOCX, PDF, TXT.
Since a book is long and not all the parts are useful, if it's a PDF, please setup the page ranges for every book in order eliminate negative effects and save computing time for analyzing.
`, }, laws: { title: '', description: `Supported file formats are DOCX, PDF, TXT.
Legal documents have a very rigorous writing format. We use text feature to detect split point.
The chunk granularity is consistent with 'ARTICLE', and all the upper level text will be included in the chunk.
`, }, manual: { title: '', description: `Only PDF is supported.
We assume manual has hierarchical section structure. We use the lowest section titles as pivots to slice documents. So, the figures and tables in the same section will not be sliced apart, and chunk size might be large.
`, }, naive: { title: '', description: `Supported file formats are DOCX, EXCEL, PPT, IMAGE, PDF, TXT.
This method apply the naive ways to chunk files:
Only PDF file is supported.
If our model works well, the paper will be sliced by it's sections, like abstract, 1.1, 1.2, etc.
The benefit of doing this is that LLM can better summarize the content of relevant sections in the paper, resulting in more comprehensive answers that help readers better understand the paper. The downside is that it increases the context of the LLM conversation and adds computational cost, so during the conversation, you can consider reducing the ‘topN’ setting.
`, }, presentation: { title: '', description: `The supported file formats are PDF, PPTX.
Every page will be treated as a chunk. And the thumbnail of every page will be stored.
All the PPT files you uploaded will be chunked by using this method automatically, setting-up for every PPT file is not necessary.
`, }, qa: { title: '', description: `EXCEL and CSV/TXT files are supported.
If the file is in excel format, there should be 2 columns question and answer without header. And question column is ahead of answer column. And it's O.K if it has multiple sheets as long as the columns are rightly composed.
If it's in csv format, it should be UTF-8 encoded. Use TAB as delimiter to separate question and answer.
All the deformed lines will be ignored. Every pair of Q&A will be treated as a chunk.
`, }, resume: { title: '', description: `The supported file formats are DOCX, PDF, TXT.
The résumé comes in a variety of formats, just like a person’s personality, but we often have to organize them into structured data that makes it easy to search.
Instead of chunking the résumé, we parse the résumé into structured data. As a HR, you can dump all the résumé you have, the you can list all the candidates that match the qualifications just by talk with 'RAGFlow'.
`, }, table: { title: '', description: `EXCEL and CSV/TXT format files are supported.
Here're some tips:
Here are some examples for headers:
Image files are supported. Video is coming soon.
If the picture has text in it, OCR is applied to extract the text as its text description.
If the text extracted by OCR is not enough, visual LLM is used to get the descriptions.
`, }, one: { title: '', description: `Supported file formats are DOCX, EXCEL, PDF, TXT.
For a document, it will be treated as an entire chunk, no split at all.
If you want to summarize something that needs all the context of an article and the selected LLM's context length covers the document length, you can try this method.
`, }, };