cecilia-uu
commited on
Commit
·
4b9f80a
1
Parent(s):
9ae711a
API: show status of parsing (#1504)
Browse files### What problem does this PR solve?
show status of parsing.
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
- api/apps/dataset_api.py +20 -0
- sdk/python/ragflow/ragflow.py +4 -0
- sdk/python/test/test_document.py +45 -0
api/apps/dataset_api.py
CHANGED
@@ -750,7 +750,27 @@ def get_message_during_parsing_document(id, message):
|
|
750 |
# ----------------------------stop parsing-----------------------------------------------------
|
751 |
|
752 |
# ----------------------------show the status of the file-----------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
753 |
|
|
|
|
|
|
|
754 |
# ----------------------------list the chunks of the file-----------------------------------------------------
|
755 |
|
756 |
# -- --------------------------delete the chunk-----------------------------------------------------
|
|
|
750 |
# ----------------------------stop parsing-----------------------------------------------------
|
751 |
|
752 |
# ----------------------------show the status of the file-----------------------------------------------------
|
753 |
+
@manager.route("/<dataset_id>/documents/<document_id>/status", methods=["GET"])
|
754 |
+
@login_required
|
755 |
+
def show_parsing_status(dataset_id, document_id):
|
756 |
+
try:
|
757 |
+
# valid dataset
|
758 |
+
exist, _ = KnowledgebaseService.get_by_id(dataset_id)
|
759 |
+
if not exist:
|
760 |
+
return construct_json_result(code=RetCode.DATA_ERROR,
|
761 |
+
message=f"This dataset: '{dataset_id}' cannot be found!")
|
762 |
+
# valid document
|
763 |
+
exist, _ = DocumentService.get_by_id(document_id)
|
764 |
+
if not exist:
|
765 |
+
return construct_json_result(code=RetCode.DATA_ERROR,
|
766 |
+
message=f"This document: '{document_id}' is not a valid document.")
|
767 |
+
|
768 |
+
_, doc = DocumentService.get_by_id(document_id) # get doc object
|
769 |
+
doc_attributes = doc.to_dict()
|
770 |
|
771 |
+
return construct_json_result(data={"progress": doc_attributes["progress"], "status": doc_attributes["status"]}, code=RetCode.SUCCESS)
|
772 |
+
except Exception as e:
|
773 |
+
return construct_error_response(e)
|
774 |
# ----------------------------list the chunks of the file-----------------------------------------------------
|
775 |
|
776 |
# -- --------------------------delete the chunk-----------------------------------------------------
|
sdk/python/ragflow/ragflow.py
CHANGED
@@ -159,7 +159,11 @@ class RAGFlow:
|
|
159 |
# ----------------------------stop parsing-----------------------------------------------------
|
160 |
|
161 |
# ----------------------------show the status of the file-----------------------------------------------------
|
|
|
|
|
|
|
162 |
|
|
|
163 |
# ----------------------------list the chunks of the file-----------------------------------------------------
|
164 |
|
165 |
# ----------------------------delete the chunk-----------------------------------------------------
|
|
|
159 |
# ----------------------------stop parsing-----------------------------------------------------
|
160 |
|
161 |
# ----------------------------show the status of the file-----------------------------------------------------
|
162 |
+
def show_parsing_status(self, dataset_id, document_id):
|
163 |
+
endpoint = f"{self.dataset_url}/{dataset_id}/documents/{document_id}/status"
|
164 |
+
res = requests.get(endpoint, headers=self.authorization_header)
|
165 |
|
166 |
+
return res.json()
|
167 |
# ----------------------------list the chunks of the file-----------------------------------------------------
|
168 |
|
169 |
# ----------------------------delete the chunk-----------------------------------------------------
|
sdk/python/test/test_document.py
CHANGED
@@ -953,7 +953,52 @@ class TestFile(TestSdk):
|
|
953 |
# ----------------------------stop parsing-----------------------------------------------------
|
954 |
|
955 |
# ----------------------------show the status of the file-----------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
956 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
957 |
# ----------------------------list the chunks of the file-----------------------------------------------------
|
958 |
|
959 |
# ----------------------------delete the chunk-----------------------------------------------------
|
|
|
953 |
# ----------------------------stop parsing-----------------------------------------------------
|
954 |
|
955 |
# ----------------------------show the status of the file-----------------------------------------------------
|
956 |
+
def test_show_status_with_success(self):
|
957 |
+
# create a dataset
|
958 |
+
ragflow = RAGFlow(API_KEY, HOST_ADDRESS)
|
959 |
+
created_res = ragflow.create_dataset("test_show_status_with_success")
|
960 |
+
created_res_id = created_res["data"]["dataset_id"]
|
961 |
+
# upload files
|
962 |
+
file_paths = ["test_data/lol.txt"]
|
963 |
+
uploading_res = ragflow.upload_local_file(created_res_id, file_paths)
|
964 |
+
# get the doc_id
|
965 |
+
data = uploading_res["data"][0]
|
966 |
+
doc_id = data["id"]
|
967 |
+
# parse file
|
968 |
+
res = ragflow.start_parsing_document(created_res_id, doc_id)
|
969 |
+
assert res["code"] == RetCode.SUCCESS and res["message"] == ""
|
970 |
+
# show status
|
971 |
+
status_res = ragflow.show_parsing_status(created_res_id, doc_id)
|
972 |
+
assert status_res["code"] == RetCode.SUCCESS and status_res["data"]["status"] == "1"
|
973 |
+
|
974 |
+
def test_show_status_nonexistent_document(self):
|
975 |
+
"""
|
976 |
+
Test showing the status of a document which does not exist.
|
977 |
+
"""
|
978 |
+
# create a dataset
|
979 |
+
ragflow = RAGFlow(API_KEY, HOST_ADDRESS)
|
980 |
+
created_res = ragflow.create_dataset("test_show_status_nonexistent_document")
|
981 |
+
created_res_id = created_res["data"]["dataset_id"]
|
982 |
+
res = ragflow.show_parsing_status(created_res_id, "imagination")
|
983 |
+
assert res["code"] == RetCode.DATA_ERROR and res["message"] == "This document: 'imagination' is not a valid document."
|
984 |
|
985 |
+
def test_show_status_document_in_nonexistent_dataset(self):
|
986 |
+
"""
|
987 |
+
Test showing the status of a document whose dataset is nonexistent.
|
988 |
+
"""
|
989 |
+
# create a dataset
|
990 |
+
ragflow = RAGFlow(API_KEY, HOST_ADDRESS)
|
991 |
+
created_res = ragflow.create_dataset("test_show_status_document_in_nonexistent_dataset")
|
992 |
+
created_res_id = created_res["data"]["dataset_id"]
|
993 |
+
# upload files
|
994 |
+
file_paths = ["test_data/test.txt"]
|
995 |
+
uploading_res = ragflow.upload_local_file(created_res_id, file_paths)
|
996 |
+
# get the doc_id
|
997 |
+
data = uploading_res["data"][0]
|
998 |
+
doc_id = data["id"]
|
999 |
+
# parse
|
1000 |
+
res = ragflow.show_parsing_status("imagination", doc_id)
|
1001 |
+
assert res["code"] == RetCode.DATA_ERROR and res["message"] == "This dataset: 'imagination' cannot be found!"
|
1002 |
# ----------------------------list the chunks of the file-----------------------------------------------------
|
1003 |
|
1004 |
# ----------------------------delete the chunk-----------------------------------------------------
|