Use consistent log file names, introduced initLogger (#3403)

### What problem does this PR solve?

Use consistent log file names, introduced initLogger

### Type of change

- [ ] Bug Fix (non-breaking change which fixes an issue)
- [ ] New Feature (non-breaking change which adds functionality)
- [ ] Documentation Update
- [x] Refactoring
- [ ] Performance Improvement
- [ ] Other (please describe):

agent/canvas.py

@@ -13,13 +13,13 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import json
 from abc import ABC
 from copy import deepcopy
 from functools import partial
 from agent.component import component_class
 from agent.component.base import ComponentBase
-from api.utils.log_utils import logger
 
 class Canvas(ABC):
     """
@@ -187,7 +187,7 @@ class Canvas(ABC):
                 if cpn.component_name == "Answer":
                     self.answer.append(c)
                 else:
-                    logger.debug(f"Canvas.prepare2run: {c}")
+                    logging.debug(f"Canvas.prepare2run: {c}")
                     cpids = cpn.get_dependent_components()
                     if any([c not in self.path[-1] for c in cpids]):
                         continue
@@ -197,7 +197,7 @@ class Canvas(ABC):
 
         prepare2run(self.components[self.path[-2][-1]]["downstream"])
         while 0 <= ran < len(self.path[-1]):
-            logger.debug(f"Canvas.run: {ran} {self.path}")
+            logging.debug(f"Canvas.run: {ran} {self.path}")
             cpn_id = self.path[-1][ran]
             cpn = self.get_component(cpn_id)
             if not cpn["downstream"]: break
@@ -217,7 +217,7 @@ class Canvas(ABC):
                             self.get_component(p)["obj"].set_exception(e)
                             prepare2run([p])
                             break
-                    logger.exception("Canvas.run got exception")
+                    logging.exception("Canvas.run got exception")
                     break
                 continue
 
@@ -229,7 +229,7 @@ class Canvas(ABC):
                         self.get_component(p)["obj"].set_exception(e)
                         prepare2run([p])
                         break
-                logger.exception("Canvas.run got exception")
+                logging.exception("Canvas.run got exception")
                 break
 
         if self.answer:

agent/component/arxiv.py

@@ -13,11 +13,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 from abc import ABC
 import arxiv
 import pandas as pd
 from agent.component.base import ComponentBase, ComponentParamBase
-from api.utils.log_utils import logger
 
 class ArXivParam(ComponentParamBase):
     """
@@ -64,5 +64,5 @@ class ArXiv(ComponentBase, ABC):
             return ArXiv.be_output("")
 
         df = pd.DataFrame(arxiv_res)
-        logger.debug(f"df: {str(df)}")
+        logging.debug(f"df: {str(df)}")
         return df

agent/component/baidu.py

@@ -13,12 +13,12 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 from abc import ABC
 import pandas as pd
 import requests
 import re
 from agent.component.base import ComponentBase, ComponentParamBase
-from api.utils.log_utils import logger
 
 
 class BaiduParam(ComponentParamBase):
@@ -62,6 +62,6 @@ class Baidu(ComponentBase, ABC):
             return Baidu.be_output("")
 
         df = pd.DataFrame(baidu_res)
-        logger.debug(f"df: {str(df)}")
+        logging.debug(f"df: {str(df)}")
         return df
 

agent/component/base.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 from abc import ABC
 import builtins
 import json
@@ -23,7 +24,6 @@ from typing import Tuple, Union
 import pandas as pd
 
 from agent import settings
-from api.utils.log_utils import logger
 
 
 _FEEDED_DEPRECATED_PARAMS = "_feeded_deprecated_params"
@@ -361,13 +361,13 @@ class ComponentParamBase(ABC):
 
     def _warn_deprecated_param(self, param_name, descr):
         if self._deprecated_params_set.get(param_name):
-            logger.warning(
+            logging.warning(
                 f"{descr} {param_name} is deprecated and ignored in this version."
             )
 
     def _warn_to_deprecate_param(self, param_name, descr, new_param):
         if self._deprecated_params_set.get(param_name):
-            logger.warning(
+            logging.warning(
                 f"{descr} {param_name} will be deprecated in future release; "
                 f"please use {new_param} instead."
             )
@@ -403,7 +403,7 @@ class ComponentBase(ABC):
         return cpnts
 
     def run(self, history, **kwargs):
-        logger.info("{}, history: {}, kwargs: {}".format(self, json.dumps(history, ensure_ascii=False),
+        logging.debug("{}, history: {}, kwargs: {}".format(self, json.dumps(history, ensure_ascii=False),
                                                               json.dumps(kwargs, ensure_ascii=False)))
         try:
             res = self._run(history, **kwargs)
@@ -476,7 +476,7 @@ class ComponentBase(ABC):
             reversed_cpnts.extend(self._canvas.path[-2])
         reversed_cpnts.extend(self._canvas.path[-1])
 
-        logger.debug(f"{self.component_name} {reversed_cpnts[::-1]}")
+        logging.debug(f"{self.component_name} {reversed_cpnts[::-1]}")
         for u in reversed_cpnts[::-1]:
             if self.get_component_name(u) in ["switch", "concentrator"]: continue
             if self.component_name.lower() == "generate" and self.get_component_name(u) == "retrieval":

agent/component/bing.py

@@ -13,11 +13,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 from abc import ABC
 import requests
 import pandas as pd
 from agent.component.base import ComponentBase, ComponentParamBase
-from api.utils.log_utils import logger
 
 class BingParam(ComponentParamBase):
     """
@@ -80,5 +80,5 @@ class Bing(ComponentBase, ABC):
             return Bing.be_output("")
 
         df = pd.DataFrame(bing_res)
-        logger.debug(f"df: {str(df)}")
+        logging.debug(f"df: {str(df)}")
         return df

agent/component/categorize.py

@@ -13,11 +13,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 from abc import ABC
 from api.db import LLMType
 from api.db.services.llm_service import LLMBundle
 from agent.component import GenerateParam, Generate
-from api.utils.log_utils import logger
 
 
 class CategorizeParam(GenerateParam):
@@ -77,7 +77,7 @@ class Categorize(Generate, ABC):
         chat_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.CHAT, self._param.llm_id)
         ans = chat_mdl.chat(self._param.get_prompt(), [{"role": "user", "content": input}],
                             self._param.gen_conf())
-        logger.debug(f"input: {input}, answer: {str(ans)}")
+        logging.debug(f"input: {input}, answer: {str(ans)}")
         for c in self._param.category_description.keys():
             if ans.lower().find(c.lower()) >= 0:
                 return Categorize.be_output(self._param.category_description[c]["to"])

agent/component/duckduckgo.py

@@ -13,11 +13,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 from abc import ABC
 from duckduckgo_search import DDGS
 import pandas as pd
 from agent.component.base import ComponentBase, ComponentParamBase
-from api.utils.log_utils import logger
 
 
 class DuckDuckGoParam(ComponentParamBase):
@@ -62,5 +62,5 @@ class DuckDuckGo(ComponentBase, ABC):
             return DuckDuckGo.be_output("")
 
         df = pd.DataFrame(duck_res)
-        logger.debug("df: {df}")
+        logging.debug("df: {df}")
         return df

agent/component/github.py

@@ -13,11 +13,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 from abc import ABC
 import pandas as pd
 import requests
 from agent.component.base import ComponentBase, ComponentParamBase
-from api.utils.log_utils import logger
 
 
 class GitHubParam(ComponentParamBase):
@@ -57,5 +57,5 @@ class GitHub(ComponentBase, ABC):
             return GitHub.be_output("")
 
         df = pd.DataFrame(github_res)
-        logger.debug(f"df: {df}")
+        logging.debug(f"df: {df}")
         return df

agent/component/google.py

@@ -13,11 +13,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 from abc import ABC
 from serpapi import GoogleSearch
 import pandas as pd
 from agent.component.base import ComponentBase, ComponentParamBase
-from api.utils.log_utils import logger
 
 
 class GoogleParam(ComponentParamBase):
@@ -92,5 +92,5 @@ class Google(ComponentBase, ABC):
             return Google.be_output("")
 
         df = pd.DataFrame(google_res)
-        logger.debug(f"df: {df}")
+        logging.debug(f"df: {df}")
         return df

agent/component/googlescholar.py

@@ -13,11 +13,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 from abc import ABC
 import pandas as pd
 from agent.component.base import ComponentBase, ComponentParamBase
 from scholarly import scholarly
-from api.utils.log_utils import logger
 
 
 class GoogleScholarParam(ComponentParamBase):
@@ -59,12 +59,12 @@ class GoogleScholar(ComponentBase, ABC):
                                                    'bib'].get('abstract', 'no abstract')})
 
             except StopIteration or Exception:
-                logger.exception("GoogleScholar")
+                logging.exception("GoogleScholar")
                 break
 
         if not scholar_res:
             return GoogleScholar.be_output("")
 
         df = pd.DataFrame(scholar_res)
-        logger.debug(f"df: {df}")
+        logging.debug(f"df: {df}")
         return df

agent/component/keyword.py

@@ -13,12 +13,12 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import re
 from abc import ABC
 from api.db import LLMType
 from api.db.services.llm_service import LLMBundle
 from agent.component import GenerateParam, Generate
-from api.utils.log_utils import logger
 
 
 class KeywordExtractParam(GenerateParam):
@@ -58,5 +58,5 @@ class KeywordExtract(Generate, ABC):
                             self._param.gen_conf())
 
         ans = re.sub(r".*keyword:", "", ans).strip()
-        logger.info(f"ans: {ans}")
+        logging.debug(f"ans: {ans}")
         return KeywordExtract.be_output(ans)

agent/component/pubmed.py

@@ -13,13 +13,13 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 from abc import ABC
 from Bio import Entrez
 import re
 import pandas as pd
 import xml.etree.ElementTree as ET
 from agent.component.base import ComponentBase, ComponentParamBase
-from api.utils.log_utils import logger
 
 
 class PubMedParam(ComponentParamBase):
@@ -65,5 +65,5 @@ class PubMed(ComponentBase, ABC):
             return PubMed.be_output("")
 
         df = pd.DataFrame(pubmed_res)
-        logger.debug(f"df: {df}")
+        logging.debug(f"df: {df}")
         return df

agent/component/relevant.py

@@ -13,12 +13,12 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 from abc import ABC
 from api.db import LLMType
 from api.db.services.llm_service import LLMBundle
 from agent.component import GenerateParam, Generate
 from rag.utils import num_tokens_from_string, encoder
-from api.utils.log_utils import logger
 
 
 class RelevantParam(GenerateParam):
@@ -71,7 +71,7 @@ class Relevant(Generate, ABC):
         ans = chat_mdl.chat(self._param.get_prompt(), [{"role": "user", "content": ans}],
                             self._param.gen_conf())
 
-        logger.info(ans)
+        logging.debug(ans)
         if ans.lower().find("yes") >= 0:
             return Relevant.be_output(self._param.yes)
         if ans.lower().find("no") >= 0:

agent/component/retrieval.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 from abc import ABC
 
 import pandas as pd
@@ -22,7 +23,6 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle
 from api.settings import retrievaler
 from agent.component.base import ComponentBase, ComponentParamBase
-from api.utils.log_utils import logger
 
 
 class RetrievalParam(ComponentParamBase):
@@ -81,7 +81,7 @@ class Retrieval(ComponentBase, ABC):
         df = pd.DataFrame(kbinfos["chunks"])
         df["content"] = df["content_with_weight"]
         del df["content_with_weight"]
-        logger.debug("{} {}".format(query, df))
+        logging.debug("{} {}".format(query, df))
         return df
 
 

agent/component/rewrite.py

@@ -13,11 +13,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 from abc import ABC
 from api.db import LLMType
 from api.db.services.llm_service import LLMBundle
 from agent.component import GenerateParam, Generate
-from api.utils.log_utils import logger
 
 
 class RewriteQuestionParam(GenerateParam):
@@ -105,7 +105,7 @@ class RewriteQuestion(Generate, ABC):
         self._canvas.history.pop()
         self._canvas.history.append(("user", ans))
 
-        logger.info(ans)
+        logging.debug(ans)
         return RewriteQuestion.be_output(ans)
 
 

agent/component/wikipedia.py

@@ -13,11 +13,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 from abc import ABC
 import wikipedia
 import pandas as pd
 from agent.component.base import ComponentBase, ComponentParamBase
-from api.utils.log_utils import logger
 
 
 class WikipediaParam(ComponentParamBase):
@@ -63,5 +63,5 @@ class Wikipedia(ComponentBase, ABC):
             return Wikipedia.be_output("")
 
         df = pd.DataFrame(wiki_res)
-        logger.debug(f"df: {df}")
+        logging.debug(f"df: {df}")
         return df

agent/component/yahoofinance.py

@@ -13,11 +13,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 from abc import ABC
 import pandas as pd
 from agent.component.base import ComponentBase, ComponentParamBase
 import yfinance as yf
-from api.utils.log_utils import logger
 
 
 class YahooFinanceParam(ComponentParamBase):
@@ -76,7 +76,7 @@ class YahooFinance(ComponentBase, ABC):
             if self._param.news:
                 yohoo_res.append({"content": "news:\n" + pd.DataFrame(msft.news).to_markdown() + "\n"})
         except Exception:
-            logger.exception("YahooFinance got exception")
+            logging.exception("YahooFinance got exception")
 
         if not yohoo_res:
             return YahooFinance.be_output("")

api/apps/__init__.py

@@ -15,6 +15,7 @@
 #
 import os
 import sys
+import logging
 from importlib.util import module_from_spec, spec_from_file_location
 from pathlib import Path
 from flask import Blueprint, Flask
@@ -32,7 +33,6 @@ from flask_login import LoginManager
 from api.settings import SECRET_KEY
 from api.settings import API_VERSION
 from api.utils.api_utils import server_error_response
-from api.utils.log_utils import logger
 from itsdangerous.url_safe import URLSafeTimedSerializer as Serializer
 
 __all__ = ["app"]
@@ -154,7 +154,7 @@ def load_user(web_request):
             else:
                 return None
         except Exception:
-            logger.exception("load_user got exception")
+            logging.exception("load_user got exception")
             return None
     else:
         return None

api/apps/canvas_app.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import json
 from functools import partial
 from flask import request, Response
@@ -23,7 +24,6 @@ from api.utils import get_uuid
 from api.utils.api_utils import get_json_result, server_error_response, validate_request, get_data_error_result
 from agent.canvas import Canvas
 from peewee import MySQLDatabase, PostgresqlDatabase
-from api.utils.log_utils import logger
 
 
 @manager.route('/templates', methods=['GET'])
@@ -115,7 +115,7 @@ def run():
                 pass
             canvas.add_user_input(req["message"])
         answer = canvas.run(stream=stream)
-        logger.info(canvas)
+        logging.debug(canvas)
     except Exception as e:
         return server_error_response(e)
 

api/apps/llm_app.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import json
 
 from flask import request
@@ -25,7 +26,6 @@ from api.db.db_models import TenantLLM
 from api.utils.api_utils import get_json_result
 from rag.llm import EmbeddingModel, ChatModel, RerankModel, CvModel, TTSModel
 import requests
-from api.utils.log_utils import logger
 
 
 @manager.route('/factories', methods=['GET'])
@@ -90,7 +90,7 @@ def set_api_key():
                 if len(arr) == 0 or tc == 0:
                     raise Exception("Fail")
                 rerank_passed = True
-                logger.info(f'passed model rerank {llm.llm_name}')
+                logging.debug(f'passed model rerank {llm.llm_name}')
             except Exception as e:
                 msg += f"\nFail to access model({llm.llm_name}) using this api key." + str(
                     e)

api/apps/user_app.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import json
 import re
 from datetime import datetime
@@ -54,7 +55,6 @@ from api.settings import (
 from api.db.services.user_service import UserService, TenantService, UserTenantService
 from api.db.services.file_service import FileService
 from api.utils.api_utils import get_json_result, construct_response
-from api.utils.log_utils import logger
 
 
 @manager.route("/login", methods=["POST", "GET"])
@@ -177,7 +177,7 @@ def github_callback():
             try:
                 avatar = download_img(user_info["avatar_url"])
             except Exception as e:
-                logger.exception(e)
+                logging.exception(e)
                 avatar = ""
             users = user_register(
                 user_id,
@@ -202,7 +202,7 @@ def github_callback():
             return redirect("/?auth=%s" % user.get_id())
         except Exception as e:
             rollback_user_registration(user_id)
-            logger.exception(e)
+            logging.exception(e)
             return redirect("/?error=%s" % str(e))
 
     # User has already registered, try to log in
@@ -279,7 +279,7 @@ def feishu_callback():
             try:
                 avatar = download_img(user_info["avatar_url"])
             except Exception as e:
-                logger.exception(e)
+                logging.exception(e)
                 avatar = ""
             users = user_register(
                 user_id,
@@ -304,7 +304,7 @@ def feishu_callback():
             return redirect("/?auth=%s" % user.get_id())
         except Exception as e:
             rollback_user_registration(user_id)
-            logger.exception(e)
+            logging.exception(e)
             return redirect("/?error=%s" % str(e))
 
     # User has already registered, try to log in
@@ -436,7 +436,7 @@ def setting_user():
         UserService.update_by_id(current_user.id, update_dict)
         return get_json_result(data=True)
     except Exception as e:
-        logger.exception(e)
+        logging.exception(e)
         return get_json_result(
             data=False, message="Update failure!", code=RetCode.EXCEPTION_ERROR
         )
@@ -621,7 +621,7 @@ def user_add():
         )
     except Exception as e:
         rollback_user_registration(user_id)
-        logger.exception(e)
+        logging.exception(e)
         return get_json_result(
             data=False,
             message=f"User registration failure, error: {str(e)}",

api/db/db_models.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import inspect
 import os
 import sys
@@ -32,7 +33,6 @@ from playhouse.pool import PooledMySQLDatabase, PooledPostgresqlDatabase
 from api.db import SerializedType, ParserType
 from api.settings import DATABASE, SECRET_KEY, DATABASE_TYPE
 from api import utils
-from api.utils.log_utils import logger
 
 def singleton(cls, *args, **kw):
     instances = {}
@@ -285,7 +285,7 @@ class BaseDataBase:
         database_config = DATABASE.copy()
         db_name = database_config.pop("name")
         self.database_connection = PooledDatabase[DATABASE_TYPE.upper()].value(db_name, **database_config)
-        logger.info('init database on cluster mode successfully')
+        logging.info('init database on cluster mode successfully')
 
 class PostgresDatabaseLock:
     def __init__(self, lock_name, timeout=10, db=None):
@@ -393,7 +393,7 @@ def close_connection():
         if DB:
             DB.close_stale(age=30)
     except Exception as e:
-        logger.exception(e)
+        logging.exception(e)
 
 
 class DataBaseModel(BaseModel):
@@ -409,15 +409,15 @@ def init_database_tables(alter_fields=[]):
     for name, obj in members:
         if obj != DataBaseModel and issubclass(obj, DataBaseModel):
             table_objs.append(obj)
-            logger.info(f"start create table {obj.__name__}")
+            logging.debug(f"start create table {obj.__name__}")
             try:
                 obj.create_table()
-                logger.info(f"create table success: {obj.__name__}")
+                logging.debug(f"create table success: {obj.__name__}")
             except Exception as e:
-                logger.exception(e)
+                logging.exception(e)
                 create_failed_list.append(obj.__name__)
     if create_failed_list:
-        logger.info(f"create tables failed: {create_failed_list}")
+        logging.error(f"create tables failed: {create_failed_list}")
         raise Exception(f"create tables failed: {create_failed_list}")
     migrate_db()
 

api/db/init_data.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import base64
 import json
 import os
@@ -30,7 +31,6 @@ from api.db.services.llm_service import LLMFactoriesService, LLMService, TenantL
 from api.db.services.user_service import TenantService, UserTenantService
 from api.settings import CHAT_MDL, EMBEDDING_MDL, ASR_MDL, IMAGE2TEXT_MDL, PARSERS, LLM_FACTORY, API_KEY, LLM_BASE_URL
 from api.utils.file_utils import get_project_base_directory
-from api.utils.log_utils import logger
 
 
 def encode_to_base64(input_string):
@@ -70,26 +70,26 @@ def init_superuser():
              "api_key": API_KEY, "api_base": LLM_BASE_URL})
 
     if not UserService.save(**user_info):
-        logger.info("can't init admin.")
+        logging.error("can't init admin.")
         return
     TenantService.insert(**tenant)
     UserTenantService.insert(**usr_tenant)
     TenantLLMService.insert_many(tenant_llm)
-    logger.info(
-        "Super user initialized. email: [email protected], password: admin. Changing the password after logining is strongly recomanded.")
+    logging.info(
+        "Super user initialized. email: [email protected], password: admin. Changing the password after login is strongly recommended.")
 
     chat_mdl = LLMBundle(tenant["id"], LLMType.CHAT, tenant["llm_id"])
     msg = chat_mdl.chat(system="", history=[
                         {"role": "user", "content": "Hello!"}], gen_conf={})
     if msg.find("ERROR: ") == 0:
-        logger.error(
+        logging.error(
             "'{}' dosen't work. {}".format(
                 tenant["llm_id"],
                 msg))
     embd_mdl = LLMBundle(tenant["id"], LLMType.EMBEDDING, tenant["embd_id"])
     v, c = embd_mdl.encode(["Hello!"])
     if c == 0:
-        logger.error(
+        logging.error(
             "'{}' dosen't work!".format(
                 tenant["embd_id"]))
 
@@ -172,7 +172,7 @@ def add_graph_templates():
             except:
                 CanvasTemplateService.update_by_id(cnvs["id"], cnvs)
         except Exception:
-            logger.exception("Add graph templates error: ")
+            logging.exception("Add graph templates error: ")
 
 
 def init_web_data():
@@ -183,7 +183,7 @@ def init_web_data():
     #    init_superuser()
 
     add_graph_templates()
-    logger.info("init web data success:{}".format(time.time() - start_time))
+    logging.info("init web data success:{}".format(time.time() - start_time))
 
 
 if __name__ == '__main__':

api/db/services/dialog_service.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import binascii
 import os
 import json
@@ -31,7 +32,6 @@ from rag.app.resume import forbidden_select_fields4resume
 from rag.nlp.search import index_name
 from rag.utils import rmSpace, num_tokens_from_string, encoder
 from api.utils.file_utils import get_project_base_directory
-from api.utils.log_utils import logger
 
 
 class DialogService(CommonService):
@@ -178,7 +178,7 @@ def chat(dialog, messages, stream=True, **kwargs):
         tts_mdl = LLMBundle(dialog.tenant_id, LLMType.TTS)
     # try to use sql if field mapping is good to go
     if field_map:
-        logger.info("Use SQL to retrieval:{}".format(questions[-1]))
+        logging.debug("Use SQL to retrieval:{}".format(questions[-1]))
         ans = use_sql(questions[-1], field_map, dialog.tenant_id, chat_mdl, prompt_config.get("quote", True))
         if ans:
             yield ans
@@ -220,7 +220,7 @@ def chat(dialog, messages, stream=True, **kwargs):
                                         doc_ids=attachments,
                                         top=dialog.top_k, aggs=False, rerank_mdl=rerank_mdl)
     knowledges = [ck["content_with_weight"] for ck in kbinfos["chunks"]]
-    logger.info(
+    logging.debug(
         "{}->{}".format(" ".join(questions), "\n->".join(knowledges)))
     retrieval_tm = timer()
 
@@ -292,7 +292,7 @@ def chat(dialog, messages, stream=True, **kwargs):
         yield decorate_answer(answer)
     else:
         answer = chat_mdl.chat(prompt, msg[1:], gen_conf)
-        logger.info("User: {}|Assistant: {}".format(
+        logging.debug("User: {}|Assistant: {}".format(
             msg[-1]["content"], answer))
         res = decorate_answer(answer)
         res["audio_binary"] = tts(tts_mdl, answer)
@@ -320,7 +320,7 @@ def use_sql(question, field_map, tenant_id, chat_mdl, quota=True):
         nonlocal sys_prompt, user_promt, question, tried_times
         sql = chat_mdl.chat(sys_prompt, [{"role": "user", "content": user_promt}], {
             "temperature": 0.06})
-        logger.info(f"{question} ==> {user_promt} get SQL: {sql}")
+        logging.debug(f"{question} ==> {user_promt} get SQL: {sql}")
         sql = re.sub(r"[\r\n]+", " ", sql.lower())
         sql = re.sub(r".*select ", "select ", sql.lower())
         sql = re.sub(r" +", " ", sql)
@@ -340,7 +340,7 @@ def use_sql(question, field_map, tenant_id, chat_mdl, quota=True):
                     flds.append(k)
                 sql = "select doc_id,docnm_kwd," + ",".join(flds) + sql[8:]
 
-        logger.info(f"{question} get SQL(refined): {sql}")
+        logging.debug(f"{question} get SQL(refined): {sql}")
         tried_times += 1
         return retrievaler.sql_retrieval(sql, format="json"), sql
 
@@ -369,9 +369,9 @@ def use_sql(question, field_map, tenant_id, chat_mdl, quota=True):
             question, sql, tbl["error"]
         )
         tbl, sql = get_table()
-        logger.info("TRY it again: {}".format(sql))
+        logging.debug("TRY it again: {}".format(sql))
 
-    logger.info("GET table: {}".format(tbl))
+    logging.debug("GET table: {}".format(tbl))
     if tbl.get("error") or len(tbl["rows"]) == 0:
         return None
 
@@ -401,7 +401,7 @@ def use_sql(question, field_map, tenant_id, chat_mdl, quota=True):
     rows = re.sub(r"T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+Z)?\|", "|", rows)
 
     if not docid_idx or not docnm_idx:
-        logger.warning("SQL missing field: " + sql)
+        logging.warning("SQL missing field: " + sql)
         return {
             "answer": "\n".join([clmns, line, rows]),
             "reference": {"chunks": [], "doc_aggs": []},

api/db/services/document_service.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import hashlib
 import json
 import random
@@ -39,7 +40,6 @@ from api.db.services.common_service import CommonService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db import StatusEnum
 from rag.utils.redis_conn import REDIS_CONN
-from api.utils.log_utils import logger
 
 
 class DocumentService(CommonService):
@@ -387,7 +387,7 @@ class DocumentService(CommonService):
                 cls.update_by_id(d["id"], info)
             except Exception as e:
                 if str(e).find("'0'") < 0:
-                    logger.exception("fetch task exception")
+                    logging.exception("fetch task exception")
 
     @classmethod
     @DB.connection_context()
@@ -544,7 +544,7 @@ def doc_upload_and_parse(conversation_id, file_objs, user_id):
                     "knowledge_graph_kwd": "mind_map"
                 })
             except Exception as e:
-                logger.exception("Mind map generation error")
+                logging.exception("Mind map generation error")
 
         vects = embedding(doc_id, [c["content_with_weight"] for c in cks])
         assert len(cks) == len(vects)

api/db/services/file_service.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import re
 import os
 from concurrent.futures import ThreadPoolExecutor
@@ -30,7 +31,6 @@ from api.db.services.file2document_service import File2DocumentService
 from api.utils import get_uuid
 from api.utils.file_utils import filename_type, thumbnail_img
 from rag.utils.storage_factory import STORAGE_IMPL
-from api.utils.log_utils import logger
 
 
 class FileService(CommonService):
@@ -276,7 +276,7 @@ class FileService(CommonService):
             return cls.model.delete().where((cls.model.tenant_id == user_id)
                                             & (cls.model.id == folder_id)).execute(),
         except Exception:
-            logger.exception("delete_folder_by_pf_id")
+            logging.exception("delete_folder_by_pf_id")
             raise RuntimeError("Database error (File retrieval)!")
 
     @classmethod
@@ -325,7 +325,7 @@ class FileService(CommonService):
         try:
             cls.filter_update((cls.model.id << file_ids, ), { 'parent_id': folder_id })
         except Exception:
-            logger.exception("move_file")
+            logging.exception("move_file")
             raise RuntimeError("Database error (File move)!")
 
     @classmethod

api/db/services/llm_service.py

@@ -13,13 +13,13 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 from api.db.services.user_service import TenantService
 from rag.llm import EmbeddingModel, CvModel, ChatModel, RerankModel, Seq2txtModel, TTSModel
 from api.db import LLMType
 from api.db.db_models import DB
 from api.db.db_models import LLMFactories, LLM, TenantLLM
 from api.db.services.common_service import CommonService
-from api.utils.log_utils import logger
 
 
 class LLMFactoriesService(CommonService):
@@ -209,7 +209,7 @@ class LLMBundle(object):
         emd, used_tokens = self.mdl.encode(texts, batch_size)
         if not TenantLLMService.increase_usage(
                 self.tenant_id, self.llm_type, used_tokens):
-            logger.error(
+            logging.error(
                 "LLMBundle.encode can't update token usage for {}/EMBEDDING used_tokens: {}".format(self.tenant_id, used_tokens))
         return emd, used_tokens
 
@@ -217,7 +217,7 @@ class LLMBundle(object):
         emd, used_tokens = self.mdl.encode_queries(query)
         if not TenantLLMService.increase_usage(
                 self.tenant_id, self.llm_type, used_tokens):
-            logger.error(
+            logging.error(
                 "LLMBundle.encode_queries can't update token usage for {}/EMBEDDING used_tokens: {}".format(self.tenant_id, used_tokens))
         return emd, used_tokens
 
@@ -225,7 +225,7 @@ class LLMBundle(object):
         sim, used_tokens = self.mdl.similarity(query, texts)
         if not TenantLLMService.increase_usage(
                 self.tenant_id, self.llm_type, used_tokens):
-            logger.error(
+            logging.error(
                 "LLMBundle.similarity can't update token usage for {}/RERANK used_tokens: {}".format(self.tenant_id, used_tokens))
         return sim, used_tokens
 
@@ -233,7 +233,7 @@ class LLMBundle(object):
         txt, used_tokens = self.mdl.describe(image, max_tokens)
         if not TenantLLMService.increase_usage(
                 self.tenant_id, self.llm_type, used_tokens):
-            logger.error(
+            logging.error(
                 "LLMBundle.describe can't update token usage for {}/IMAGE2TEXT used_tokens: {}".format(self.tenant_id, used_tokens))
         return txt
 
@@ -241,7 +241,7 @@ class LLMBundle(object):
         txt, used_tokens = self.mdl.transcription(audio)
         if not TenantLLMService.increase_usage(
                 self.tenant_id, self.llm_type, used_tokens):
-            logger.error(
+            logging.error(
                 "LLMBundle.transcription can't update token usage for {}/SEQUENCE2TXT used_tokens: {}".format(self.tenant_id, used_tokens))
         return txt
 
@@ -250,7 +250,7 @@ class LLMBundle(object):
             if isinstance(chunk,int):
                 if not TenantLLMService.increase_usage(
                     self.tenant_id, self.llm_type, chunk, self.llm_name):
-                        logger.error(
+                        logging.error(
                             "LLMBundle.tts can't update token usage for {}/TTS".format(self.tenant_id))
                 return
             yield chunk     
@@ -259,7 +259,7 @@ class LLMBundle(object):
         txt, used_tokens = self.mdl.chat(system, history, gen_conf)
         if isinstance(txt, int) and not TenantLLMService.increase_usage(
                 self.tenant_id, self.llm_type, used_tokens, self.llm_name):
-            logger.error(
+            logging.error(
                 "LLMBundle.chat can't update token usage for {}/CHAT llm_name: {}, used_tokens: {}".format(self.tenant_id, self.llm_name, used_tokens))
         return txt
 
@@ -268,7 +268,7 @@ class LLMBundle(object):
             if isinstance(txt, int):
                 if not TenantLLMService.increase_usage(
                         self.tenant_id, self.llm_type, txt, self.llm_name):
-                    logger.error(
+                    logging.error(
                         "LLMBundle.chat_streamly can't update token usage for {}/CHAT llm_name: {}, content: {}".format(self.tenant_id, self.llm_name, txt))
                 return
             yield txt

api/ragflow_server.py

@@ -15,6 +15,17 @@
 #
 
 import logging
+import inspect
+from api.utils.log_utils import initRootLogger
+initRootLogger(inspect.getfile(inspect.currentframe()))
+for module in ["pdfminer"]:
+    module_logger = logging.getLogger(module)
+    module_logger.setLevel(logging.WARNING)
+for module in ["peewee"]:
+    module_logger = logging.getLogger(module)
+    module_logger.handlers.clear()
+    module_logger.propagate = True
+
 import os
 import signal
 import sys
@@ -22,7 +33,6 @@ import time
 import traceback
 from concurrent.futures import ThreadPoolExecutor
 
-import validation
 from werkzeug.serving import run_simple
 from api.apps import app
 from api.db.runtime_config import RuntimeConfig
@@ -31,7 +41,6 @@ from api.settings import (
     HOST, HTTP_PORT
 )
 from api import utils
-from api.utils.log_utils import logger
 
 from api.db.db_models import init_database_tables as init_web_db
 from api.db.init_data import init_web_data
@@ -44,11 +53,11 @@ def update_progress():
         try:
             DocumentService.update_progress()
         except Exception:
-            logger.exception("update_progress exception")
+            logging.exception("update_progress exception")
 
 
 if __name__ == '__main__':
-    logger.info(r"""
+    logging.info(r"""
         ____   ___    ______ ______ __               
        / __ \ /   |  / ____// ____// /____  _      __
       / /_/ // /| | / / __ / /_   / // __ \| | /| / /
@@ -56,10 +65,10 @@ if __name__ == '__main__':
     /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/                             
 
     """)
-    logger.info(
+    logging.info(
         f'RAGFlow version: {RAGFLOW_VERSION_INFO}'
     )
-    logger.info(
+    logging.info(
         f'project base: {utils.file_utils.get_project_base_directory()}'
     )
 
@@ -83,26 +92,18 @@ if __name__ == '__main__':
 
     RuntimeConfig.DEBUG = args.debug
     if RuntimeConfig.DEBUG:
-        logger.info("run on debug mode")
+        logging.info("run on debug mode")
 
     RuntimeConfig.init_env()
     RuntimeConfig.init_config(JOB_SERVER_HOST=HOST, HTTP_PORT=HTTP_PORT)
 
-    peewee_logger = logging.getLogger("peewee")
-    peewee_logger.propagate = False
-    # rag_arch.common.log.ROpenHandler
-    peewee_logger.addHandler(logger.handlers[0])
-    peewee_logger.setLevel(logger.handlers[0].level)
 
     thr = ThreadPoolExecutor(max_workers=1)
     thr.submit(update_progress)
 
     # start http server
     try:
-        logger.info("RAG Flow http server start...")
-        werkzeug_logger = logging.getLogger("werkzeug")
-        for h in logger.handlers:
-            werkzeug_logger.addHandler(h)
+        logging.info("RAG Flow http server start...")
         run_simple(
             hostname=HOST,
             port=HTTP_PORT,

api/utils/api_utils.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import functools
 import json
 import random
@@ -40,7 +41,6 @@ from api.settings import (
 from api.settings import RetCode
 from api.utils import CustomJSONEncoder, get_uuid
 from api.utils import json_dumps
-from api.utils.log_utils import logger
 
 requests.models.complexjson.dumps = functools.partial(
     json.dumps, cls=CustomJSONEncoder)
@@ -118,7 +118,7 @@ def get_data_error_result(code=RetCode.DATA_ERROR,
 
 
 def server_error_response(e):
-    logger.exception(e)
+    logging.exception(e)
     try:
         if e.code == 401:
             return get_json_result(code=401, message=repr(e))
@@ -259,7 +259,7 @@ def construct_json_result(code=RetCode.SUCCESS, message='success', data=None):
 
 
 def construct_error_response(e):
-    logger.exception(e)
+    logging.exception(e)
     try:
         if e.code == 401:
             return construct_json_result(code=RetCode.UNAUTHORIZED, message=repr(e))

api/utils/log_utils.py

@@ -14,38 +14,41 @@
 #  limitations under the License.
 #
 import os
+import os.path
 import logging
 from logging.handlers import RotatingFileHandler
 
-from api.utils.file_utils import get_project_base_directory
-
-LOG_LEVEL = logging.INFO
-LOG_FILE = os.path.abspath(os.path.join(get_project_base_directory(), "logs", f"ragflow_{os.getpid()}.log"))
-LOG_FORMAT = "%(asctime)-15s %(levelname)-8s %(process)d %(message)s"
-logger = None
-
-def getLogger():
-    global logger
-    if logger is not None:
-        return logger
-
-    print(f"log file path: {LOG_FILE}")
-    os.makedirs(os.path.dirname(LOG_FILE), exist_ok=True)
-    logger = logging.getLogger("ragflow")
-    logger.setLevel(LOG_LEVEL)
-
-    handler1 = RotatingFileHandler(LOG_FILE, maxBytes=10*1024*1024, backupCount=5)
-    handler1.setLevel(LOG_LEVEL)
-    formatter1 = logging.Formatter(LOG_FORMAT)
-    handler1.setFormatter(formatter1)
+def get_project_base_directory():
+    PROJECT_BASE = os.path.abspath(
+        os.path.join(
+            os.path.dirname(os.path.realpath(__file__)),
+            os.pardir,
+            os.pardir,
+        )
+    )
+    return PROJECT_BASE
+
+def initRootLogger(script_path: str, log_level: int = logging.INFO, log_format: str = "%(asctime)-15s %(levelname)-8s %(process)d %(message)s"):
+    logger = logging.getLogger()
+    if logger.hasHandlers():
+        return
+
+    script_name = os.path.basename(script_path)
+    log_path = os.path.abspath(os.path.join(get_project_base_directory(), "logs", f"{os.path.splitext(script_name)[0]}.log"))
+
+    os.makedirs(os.path.dirname(log_path), exist_ok=True)
+    logger.setLevel(log_level)
+    formatter = logging.Formatter(log_format)
+
+    handler1 = RotatingFileHandler(log_path, maxBytes=10*1024*1024, backupCount=5)
+    handler1.setLevel(log_level)
+    handler1.setFormatter(formatter)
     logger.addHandler(handler1)
 
     handler2 = logging.StreamHandler()
-    handler2.setLevel(LOG_LEVEL)
-    formatter2 = logging.Formatter(LOG_FORMAT)
-    handler2.setFormatter(formatter2)
+    handler2.setLevel(log_level)
+    handler2.setFormatter(formatter)
     logger.addHandler(handler2)
 
-    return logger
-
-logger = getLogger()
+    msg = f"{script_name} log path: {log_path}"
+    logger.info(msg)

api/validation.py

@@ -14,20 +14,20 @@
 #  limitations under the License.
 #
 
+import logging
 import sys
-from api.utils.log_utils import logger
 
 
 def python_version_validation():
     # Check python version
     required_python_version = (3, 10)
     if sys.version_info < required_python_version:
-        logger.info(
+        logging.info(
             f"Required Python: >= {required_python_version[0]}.{required_python_version[1]}. Current Python version: {sys.version_info[0]}.{sys.version_info[1]}."
         )
         sys.exit(1)
     else:
-        logger.info(f"Python version: {sys.version_info[0]}.{sys.version_info[1]}")
+        logging.info(f"Python version: {sys.version_info[0]}.{sys.version_info[1]}")
 
 
 python_version_validation()

deepdoc/parser/pdf_parser.py

@@ -11,6 +11,7 @@
 #  limitations under the License.
 #
 
+import logging
 import os
 import random
 
@@ -18,7 +19,6 @@ import xgboost as xgb
 from io import BytesIO
 import re
 import pdfplumber
-import logging
 from PIL import Image
 import numpy as np
 from timeit import default_timer as timer
@@ -26,15 +26,11 @@ from pypdf import PdfReader as pdf2_read
 
 from api.settings import LIGHTEN
 from api.utils.file_utils import get_project_base_directory
-from api.utils.log_utils import logger
 from deepdoc.vision import OCR, Recognizer, LayoutRecognizer, TableStructureRecognizer
 from rag.nlp import rag_tokenizer
 from copy import deepcopy
 from huggingface_hub import snapshot_download
 
-logging.getLogger("pdfminer").setLevel(logging.WARNING)
-
-
 class RAGFlowPdfParser:
     def __init__(self):
         self.ocr = OCR()
@@ -51,7 +47,7 @@ class RAGFlowPdfParser:
                 if torch.cuda.is_available():
                     self.updown_cnt_mdl.set_param({"device": "cuda"})
             except Exception:
-                logger.exception("RAGFlowPdfParser __init__")
+                logging.exception("RAGFlowPdfParser __init__")
         try:
             model_dir = os.path.join(
                 get_project_base_directory(),
@@ -188,7 +184,7 @@ class RAGFlowPdfParser:
         return True
 
     def _table_transformer_job(self, ZM):
-        logger.info("Table processing...")
+        logging.debug("Table processing...")
         imgs, pos = [], []
         tbcnt = [0]
         MARGIN = 10
@@ -426,7 +422,7 @@ class RAGFlowPdfParser:
             detach_feats = [b["x1"] < b_["x0"],
                             b["x0"] > b_["x1"]]
             if (any(feats) and not any(concatting_feats)) or any(detach_feats):
-                logger.info("{} {} {} {}".format(
+                logging.debug("{} {} {} {}".format(
                     b["text"],
                     b_["text"],
                     any(feats),
@@ -727,14 +723,14 @@ class RAGFlowPdfParser:
             #    continue
             if tv < fv and tk:
                 tables[tk].insert(0, c)
-                logger.debug(
+                logging.debug(
                     "TABLE:" +
                     self.boxes[i]["text"] +
                     "; Cap: " +
                     tk)
             elif fk:
                 figures[fk].insert(0, c)
-                logger.debug(
+                logging.debug(
                     "FIGURE:" +
                     self.boxes[i]["text"] +
                     "; Cap: " +
@@ -761,7 +757,7 @@ class RAGFlowPdfParser:
                 if ii is not None:
                     b = louts[ii]
                 else:
-                    logger.warn(
+                    logging.warn(
                         f"Missing layout match: {pn + 1},%s" %
                         (bxs[0].get(
                             "layoutno", "")))
@@ -919,7 +915,7 @@ class RAGFlowPdfParser:
                 if usefull(boxes[0]):
                     dfs(boxes[0], 0)
                 else:
-                    logger.debug("WASTE: " + boxes[0]["text"])
+                    logging.debug("WASTE: " + boxes[0]["text"])
             except Exception:
                 pass
             boxes.pop(0)
@@ -928,7 +924,7 @@ class RAGFlowPdfParser:
                 res.append(
                     "\n".join([c["text"] + self._line_tag(c, ZM) for c in lines]))
             else:
-                logger.debug("REMOVED: " +
+                logging.debug("REMOVED: " +
                               "<<".join([c["text"] for c in lines]))
 
         return "\n\n".join(res)
@@ -940,7 +936,7 @@ class RAGFlowPdfParser:
                 fnm) if not binary else pdfplumber.open(BytesIO(binary))
             return len(pdf.pages)
         except Exception:
-            logger.exception("total_page_number")
+            logging.exception("total_page_number")
 
     def __images__(self, fnm, zoomin=3, page_from=0,
                    page_to=299, callback=None):
@@ -964,7 +960,7 @@ class RAGFlowPdfParser:
                                self.pdf.pages[page_from:page_to]]
             self.total_page = len(self.pdf.pages)
         except Exception:
-            logger.exception("RAGFlowPdfParser __images__")
+            logging.exception("RAGFlowPdfParser __images__")
 
         self.outlines = []
         try:
@@ -980,11 +976,11 @@ class RAGFlowPdfParser:
 
             dfs(outlines, 0)
         except Exception as e:
-            logger.warning(f"Outlines exception: {e}")
+            logging.warning(f"Outlines exception: {e}")
         if not self.outlines:
-            logger.warning("Miss outlines")
+            logging.warning("Miss outlines")
 
-        logger.info("Images converted.")
+        logging.debug("Images converted.")
         self.is_english = [re.search(r"[a-zA-Z0-9,/¸;:'\[\]\(\)!@#$%^&*\"?<>._-]{30,}", "".join(
             random.choices([c["text"] for c in self.page_chars[i]], k=min(100, len(self.page_chars[i]))))) for i in
                            range(len(self.page_chars))]
@@ -1024,7 +1020,7 @@ class RAGFlowPdfParser:
             self.is_english = re.search(r"[\na-zA-Z0-9,/¸;:'\[\]\(\)!@#$%^&*\"?<>._-]{30,}",
                                         "".join([b["text"] for b in random.choices(bxes, k=min(30, len(bxes)))]))
 
-        logger.info("Is it English:", self.is_english)
+        logging.debug("Is it English:", self.is_english)
 
         self.page_cum_height = np.cumsum(self.page_cum_height)
         assert len(self.page_cum_height) == len(self.page_images) + 1
@@ -1164,9 +1160,9 @@ class PlainParser(object):
 
             dfs(outlines, 0)
         except Exception:
-            logger.exception("Outlines exception")
+            logging.exception("Outlines exception")
         if not self.outlines:
-            logger.warning("Miss outlines")
+            logging.warning("Miss outlines")
 
         return [(l, "") for l in lines], []
 

deepdoc/parser/resume/entities/corporations.py

@@ -11,13 +11,13 @@
 #  limitations under the License.
 #
 
+import logging
 import re
 import json
 import os
 import pandas as pd
 from rag.nlp import rag_tokenizer
 from . import regions
-from api.utils.log_utils import logger
 
 
 current_file_path = os.path.dirname(os.path.abspath(__file__))
@@ -71,7 +71,7 @@ GOOD_CORP = set([corpNorm(rmNoise(c), False) for c in GOOD_CORP])
 for c,v in CORP_TAG.items():
     cc = corpNorm(rmNoise(c), False)
     if not cc:
-        logger.info(c)
+        logging.debug(c)
 CORP_TAG = {corpNorm(rmNoise(c), False):v for c,v in CORP_TAG.items()}
 
 def is_good(nm):

deepdoc/parser/resume/step_two.py

@@ -10,7 +10,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-
+import logging
 import re
 import copy
 import time
@@ -23,7 +23,6 @@ from deepdoc.parser.resume.entities import degrees, schools, corporations
 from rag.nlp import rag_tokenizer, surname
 from xpinyin import Pinyin
 from contextlib import contextmanager
-from api.utils.log_utils import logger
 
 
 class TimeoutException(Exception): pass
@@ -164,7 +163,7 @@ def forEdu(cv):
             y, m, d = getYMD(edu_end_dt)
             cv["work_exp_flt"] = min(int(str(datetime.date.today())[0:4]) - int(y), cv.get("work_exp_flt", 1000))
         except Exception as e:
-            logger.exception("forEdu {} {} {}".format(e, edu_end_dt, cv.get("work_exp_flt")))
+            logging.exception("forEdu {} {} {}".format(e, edu_end_dt, cv.get("work_exp_flt")))
     if sch:
         cv["school_name_kwd"] = sch
         if (len(cv.get("degree_kwd", [])) >= 1 and "本科" in cv["degree_kwd"]) \
@@ -276,7 +275,7 @@ def forWork(cv):
         try:
             duas.append((datetime.datetime.strptime(ed, "%Y-%m-%d") - datetime.datetime.strptime(st, "%Y-%m-%d")).days)
         except Exception:
-            logger.exception("forWork {} {}".format(n.get("start_time"), n.get("end_time")))
+            logging.exception("forWork {} {}".format(n.get("start_time"), n.get("end_time")))
 
         if n.get("scale"):
             r = re.search(r"^([0-9]+)", str(n["scale"]))
@@ -333,7 +332,7 @@ def forWork(cv):
             y, m, d = getYMD(work_st_tm)
             cv["work_exp_flt"] = min(int(str(datetime.date.today())[0:4]) - int(y), cv.get("work_exp_flt", 1000))
         except Exception as e:
-            logger.exception("forWork {} {} {}".format(e, work_st_tm, cv.get("work_exp_flt")))
+            logging.exception("forWork {} {} {}".format(e, work_st_tm, cv.get("work_exp_flt")))
 
     cv["job_num_int"] = 0
     if duas:
@@ -464,7 +463,7 @@ def parse(cv):
                     cv[f"{t}_kwd"] = nms
                     cv[f"{t}_tks"] = rag_tokenizer.tokenize(" ".join(nms))
             except Exception:
-                logger.exception("parse {} {}".format(str(traceback.format_exc()), cv[k]))
+                logging.exception("parse {} {}".format(str(traceback.format_exc()), cv[k]))
                 cv[k] = []
 
         # tokenize fields
@@ -565,7 +564,7 @@ def parse(cv):
                 cv["work_start_dt"] = "%s-%02d-%02d 00:00:00" % (y, int(m), int(d))
                 cv["work_exp_flt"] = int(str(datetime.date.today())[0:4]) - int(y)
     except Exception as e:
-        logger.exception("parse {} ==> {}".format(e, cv.get("work_start_time")))
+        logging.exception("parse {} ==> {}".format(e, cv.get("work_start_time")))
     if "work_exp_flt" not in cv and cv.get("work_experience", 0): cv["work_exp_flt"] = int(cv["work_experience"]) / 12.
 
     keys = list(cv.keys())
@@ -580,7 +579,7 @@ def parse(cv):
 
     cv["tob_resume_id"] = str(cv["tob_resume_id"])
     cv["id"] = cv["tob_resume_id"]
-    logger.info("CCCCCCCCCCCCCCC")
+    logging.debug("CCCCCCCCCCCCCCC")
 
     return dealWithInt64(cv)
 

deepdoc/vision/operators.py

@@ -14,13 +14,13 @@
 #  limitations under the License.
 #
 
+import logging
 import sys
 import six
 import cv2
 import numpy as np
 import math
 from PIL import Image
-from api.utils.log_utils import logger
 
 
 class DecodeImage(object):
@@ -403,7 +403,7 @@ class DetResizeForTest(object):
                 return None, (None, None)
             img = cv2.resize(img, (int(resize_w), int(resize_h)))
         except BaseException:
-            logger.exception("{} {} {}".format(img.shape, resize_w, resize_h))
+            logging.exception("{} {} {}".format(img.shape, resize_w, resize_h))
             sys.exit(0)
         ratio_h = resize_h / float(h)
         ratio_w = resize_w / float(w)

deepdoc/vision/recognizer.py

@@ -11,6 +11,7 @@
 #  limitations under the License.
 #
 
+import logging
 import os
 from copy import deepcopy
 
@@ -19,7 +20,6 @@ from huggingface_hub import snapshot_download
 
 from api.utils.file_utils import get_project_base_directory
 from .operators import *
-from api.utils.log_utils import logger
 
 
 class Recognizer(object):
@@ -440,7 +440,7 @@ class Recognizer(object):
             end_index = min((i + 1) * batch_size, len(imgs))
             batch_image_list = imgs[start_index:end_index]
             inputs = self.preprocess(batch_image_list)
-            logger.info("preprocess")
+            logging.debug("preprocess")
             for ins in inputs:
                 bb = self.postprocess(self.ort_sess.run(None, {k:v for k,v in ins.items() if k in self.input_names})[0], ins, thr)
                 res.append(bb)

deepdoc/vision/seeit.py

@@ -11,10 +11,10 @@
 #  limitations under the License.
 #
 
+import logging
 import os
 import PIL
 from PIL import ImageDraw
-from api.utils.log_utils import logger
 
 
 def save_results(image_list, results, labels, output_dir='output/', threshold=0.5):
@@ -25,7 +25,7 @@ def save_results(image_list, results, labels, output_dir='output/', threshold=0.
 
         out_path = os.path.join(output_dir, f"{idx}.jpg")
         im.save(out_path, quality=95)
-        logger.info("save result to: " + out_path)
+        logging.debug("save result to: " + out_path)
 
 
 def draw_box(im, result, lables, threshold=0.5):

deepdoc/vision/t_recognizer.py

@@ -10,9 +10,9 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import os
 import sys
-from api.utils.log_utils import logger
 
 sys.path.insert(
     0,
@@ -59,7 +59,7 @@ def main(args):
             } for t in lyt]
         img = draw_box(images[i], lyt, labels, float(args.threshold))
         img.save(outputs[i], quality=95)
-        logger.info("save result to: " + outputs[i])
+        logging.info("save result to: " + outputs[i])
 
 
 def get_table_html(img, tb_cpns, ocr):

deepdoc/vision/table_structure_recognizer.py

@@ -38,7 +38,7 @@ class TableStructureRecognizer(Recognizer):
             super().__init__(self.labels, "tsr", os.path.join(
                     get_project_base_directory(),
                     "rag/res/deepdoc"))
-        except Exception as e:
+        except Exception:
             super().__init__(self.labels, "tsr", snapshot_download(repo_id="InfiniFlow/deepdoc",
                                               local_dir=os.path.join(get_project_base_directory(), "rag/res/deepdoc"),
                                               local_dir_use_symlinks=False))

graphrag/claim_extractor.py

@@ -5,6 +5,7 @@ Reference:
  - [graphrag](https://github.com/microsoft/graphrag)
 """
 
+import logging
 import argparse
 import json
 import re
@@ -17,7 +18,6 @@ import tiktoken
 from graphrag.claim_prompt import CLAIM_EXTRACTION_PROMPT, CONTINUE_PROMPT, LOOP_PROMPT
 from rag.llm.chat_model import Base as CompletionLLM
 from graphrag.utils import ErrorHandlerFn, perform_variable_replacements
-from api.utils.log_utils import logger
 
 DEFAULT_TUPLE_DELIMITER = "<|>"
 DEFAULT_RECORD_DELIMITER = "##"
@@ -126,7 +126,7 @@ class ClaimExtractor:
                 ]
                 source_doc_map[document_id] = text
             except Exception as e:
-                logger.exception("error extracting claim")
+                logging.exception("error extracting claim")
                 self._on_error(
                     e,
                     traceback.format_exc(),
@@ -265,4 +265,4 @@ if __name__ == "__main__":
         "claim_description": ""
     }
     claim = ex(info)
-    logger.info(json.dumps(claim.output, ensure_ascii=False, indent=2))
+    logging.info(json.dumps(claim.output, ensure_ascii=False, indent=2))

graphrag/community_reports_extractor.py

@@ -5,6 +5,7 @@ Reference:
  - [graphrag](https://github.com/microsoft/graphrag)
 """
 
+import logging
 import json
 import re
 import traceback
@@ -19,7 +20,6 @@ from rag.llm.chat_model import Base as CompletionLLM
 from graphrag.utils import ErrorHandlerFn, perform_variable_replacements, dict_has_keys_with_types
 from rag.utils import num_tokens_from_string
 from timeit import default_timer as timer
-from api.utils.log_utils import logger
 
 
 @dataclass
@@ -80,7 +80,7 @@ class CommunityReportsExtractor:
                     response = re.sub(r"[^\}]*$", "", response)
                     response = re.sub(r"\{\{", "{", response)
                     response = re.sub(r"\}\}", "}", response)
-                    logger.info(response)
+                    logging.debug(response)
                     response = json.loads(response)
                     if not dict_has_keys_with_types(response, [
                                 ("title", str),
@@ -92,7 +92,7 @@ class CommunityReportsExtractor:
                     response["weight"] = weight
                     response["entities"] = ents
                 except Exception as e:
-                    logger.exception("CommunityReportsExtractor got exception")
+                    logging.exception("CommunityReportsExtractor got exception")
                     self._on_error(e, traceback.format_exc(), None)
                     continue
 

graphrag/description_summary.py

@@ -5,19 +5,11 @@ Reference:
  - [graphrag](https://github.com/microsoft/graphrag)
 """
 
-import argparse
-import html
 import json
-import logging
-import numbers
-import re
-import traceback
-from collections.abc import Callable
 from dataclasses import dataclass
 
 from graphrag.utils import ErrorHandlerFn, perform_variable_replacements
 from rag.llm.chat_model import Base as CompletionLLM
-import networkx as nx
 
 from rag.utils import num_tokens_from_string
 

graphrag/entity_resolution.py

@@ -13,8 +13,8 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import itertools
 import logging
+import itertools
 import re
 import traceback
 from dataclasses import dataclass

graphrag/index.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import os
 from concurrent.futures import ThreadPoolExecutor
 import json
@@ -28,7 +29,6 @@ from graphrag.graph_extractor import GraphExtractor, DEFAULT_ENTITY_TYPES
 from graphrag.mind_map_extractor import MindMapExtractor
 from rag.nlp import rag_tokenizer
 from rag.utils import num_tokens_from_string
-from api.utils.log_utils import logger
 
 
 def graph_merge(g1, g2):
@@ -95,7 +95,7 @@ def build_knowledge_graph_chunks(tenant_id: str, chunks: List[str], callback, en
     chunks = []
     for n, attr in graph.nodes(data=True):
         if attr.get("rank", 0) == 0:
-            logger.info(f"Ignore entity: {n}")
+            logging.debug(f"Ignore entity: {n}")
             continue
         chunk = {
             "name_kwd": n,
@@ -137,7 +137,7 @@ def build_knowledge_graph_chunks(tenant_id: str, chunks: List[str], callback, en
     mg = mindmap(_chunks).output
     if not len(mg.keys()): return chunks
 
-    logger.info(json.dumps(mg, ensure_ascii=False, indent=2))
+    logging.debug(json.dumps(mg, ensure_ascii=False, indent=2))
     chunks.append(
         {
             "content_with_weight": json.dumps(mg, ensure_ascii=False, indent=2),

graphrag/leiden.py

@@ -14,8 +14,6 @@ from graspologic.utils import largest_connected_component
 import networkx as nx
 from networkx import is_empty
 
-log = logging.getLogger(__name__)
-
 
 def _stabilize_graph(graph: nx.Graph) -> nx.Graph:
     """Ensure an undirected graph with the same relationships will always be read the same way."""
@@ -99,7 +97,7 @@ def run(graph: nx.Graph, args: dict[str, Any]) -> dict[int, dict[str, dict]]:
     max_cluster_size = args.get("max_cluster_size", 12)
     use_lcc = args.get("use_lcc", True)
     if args.get("verbose", False):
-        log.info(
+        logging.debug(
             "Running leiden with max_cluster_size=%s, lcc=%s", max_cluster_size, use_lcc
         )
     if not graph.nodes(): return {}

graphrag/mind_map_extractor.py

@@ -14,8 +14,8 @@
 #  limitations under the License.
 #
 
-import collections
 import logging
+import collections
 import os
 import re
 import traceback
@@ -29,7 +29,6 @@ from rag.llm.chat_model import Base as CompletionLLM
 import markdown_to_json
 from functools import reduce
 from rag.utils import num_tokens_from_string
-from api.utils.log_utils import logger
 
 
 @dataclass
@@ -193,6 +192,6 @@ class MindMapExtractor:
         gen_conf = {"temperature": 0.5}
         response = self._llm.chat(text, [{"role": "user", "content": "Output:"}], gen_conf)
         response = re.sub(r"```[^\n]*", "", response)
-        logger.info(response)
-        logger.info(self._todict(markdown_to_json.dictify(response)))
+        logging.debug(response)
+        logging.debug(self._todict(markdown_to_json.dictify(response)))
         return self._todict(markdown_to_json.dictify(response))

intergrations/chatgpt-on-wechat/plugins/ragflow_chat.py

@@ -1,8 +1,8 @@
+import logging
 import requests
 from bridge.context import ContextType  # Import Context, ContextType
 from bridge.reply import Reply, ReplyType  # Import Reply, ReplyType
 from bridge import *
-from api.utils.log_utils import logger
 from plugins import Plugin, register  # Import Plugin and register
 from plugins.event import Event, EventContext, EventAction  # Import event-related classes
 
@@ -16,7 +16,7 @@ class RAGFlowChat(Plugin):
         self.handlers[Event.ON_HANDLE_CONTEXT] = self.on_handle_context
         # Store conversation_id for each user
         self.conversations = {}
-        logger.info("[RAGFlowChat] Plugin initialized")
+        logging.info("[RAGFlowChat] Plugin initialized")
 
     def on_handle_context(self, e_context: EventContext):
         context = e_context['context']
@@ -45,7 +45,7 @@ class RAGFlowChat(Plugin):
         user_id = session_id  # Use session_id as user_id
 
         if not api_key or not host_address:
-            logger.error("[RAGFlowChat] Missing configuration")
+            logging.error("[RAGFlowChat] Missing configuration")
             return "The plugin configuration is incomplete. Please check the configuration."
 
         headers = {
@@ -63,20 +63,20 @@ class RAGFlowChat(Plugin):
             }
             try:
                 response = requests.get(url_new_conversation, headers=headers, params=params_new_conversation)
-                logger.debug(f"[RAGFlowChat] New conversation response: {response.text}")
+                logging.debug(f"[RAGFlowChat] New conversation response: {response.text}")
                 if response.status_code == 200:
                     data = response.json()
                     if data.get("code") == 0:
                         conversation_id = data["data"]["id"]
                         self.conversations[user_id] = conversation_id
                     else:
-                        logger.error(f"[RAGFlowChat] Failed to create conversation: {data.get('message')}")
+                        logging.error(f"[RAGFlowChat] Failed to create conversation: {data.get('message')}")
                         return f"Sorry, unable to create a conversation: {data.get('message')}"
                 else:
-                    logger.error(f"[RAGFlowChat] HTTP error when creating conversation: {response.status_code}")
+                    logging.error(f"[RAGFlowChat] HTTP error when creating conversation: {response.status_code}")
                     return f"Sorry, unable to connect to RAGFlow API (create conversation). HTTP status code: {response.status_code}"
             except Exception as e:
-                logger.exception("[RAGFlowChat] Exception when creating conversation")
+                logging.exception("[RAGFlowChat] Exception when creating conversation")
                 return f"Sorry, an internal error occurred: {str(e)}"
 
         # Step 2: Send the message and get a reply
@@ -95,18 +95,18 @@ class RAGFlowChat(Plugin):
 
         try:
             response = requests.post(url_completion, headers=headers, json=payload_completion)
-            logger.debug(f"[RAGFlowChat] Completion response: {response.text}")
+            logging.debug(f"[RAGFlowChat] Completion response: {response.text}")
             if response.status_code == 200:
                 data = response.json()
                 if data.get("code") == 0:
                     answer = data["data"]["answer"]
                     return answer
                 else:
-                    logger.error(f"[RAGFlowChat] Failed to get answer: {data.get('message')}")
+                    logging.error(f"[RAGFlowChat] Failed to get answer: {data.get('message')}")
                     return f"Sorry, unable to get a reply: {data.get('message')}"
             else:
-                logger.error(f"[RAGFlowChat] HTTP error when getting answer: {response.status_code}")
+                logging.error(f"[RAGFlowChat] HTTP error when getting answer: {response.status_code}")
                 return f"Sorry, unable to connect to RAGFlow API (get reply). HTTP status code: {response.status_code}"
         except Exception as e:
-            logger.exception("[RAGFlowChat] Exception when getting answer")
+            logging.exception("[RAGFlowChat] Exception when getting answer")
             return f"Sorry, an internal error occurred: {str(e)}"

rag/app/book.py

@@ -10,6 +10,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 from tika import parser
 import re
 from io import BytesIO
@@ -20,7 +21,6 @@ from rag.nlp import bullets_category, is_english,remove_contents_table, \
     tokenize_chunks
 from rag.nlp import rag_tokenizer
 from deepdoc.parser import PdfParser, DocxParser, PlainParser, HtmlParser
-from api.utils.log_utils import logger
 
 
 class Pdf(PdfParser):
@@ -39,7 +39,7 @@ class Pdf(PdfParser):
         start = timer()
         self._layouts_rec(zoomin)
         callback(0.67, "Layout analysis finished")
-        logger.info("layouts: {}".format(timer() - start))
+        logging.debug("layouts: {}".format(timer() - start))
         self._table_transformer_job(zoomin)
         callback(0.68, "Table analysis finished")
         self._text_merge()

rag/app/email.py

@@ -11,6 +11,7 @@
 #  limitations under the License.
 #
 
+import logging
 from email import policy
 from email.parser import BytesParser
 from rag.app.naive import chunk as naive_chunk
@@ -18,7 +19,6 @@ import re
 from rag.nlp import rag_tokenizer, naive_merge, tokenize_chunks
 from deepdoc.parser import HtmlParser, TxtParser
 from timeit import default_timer as timer
-from api.utils.log_utils import logger
 import io
 
 
@@ -86,7 +86,7 @@ def chunk(
     )
 
     main_res.extend(tokenize_chunks(chunks, doc, eng, None))
-    logger.info("naive_merge({}): {}".format(filename, timer() - st))
+    logging.debug("naive_merge({}): {}".format(filename, timer() - st))
     # get the attachment info
     for part in msg.iter_attachments():
         content_disposition = part.get("Content-Disposition")

rag/app/laws.py

@@ -21,7 +21,6 @@ from rag.nlp import bullets_category, remove_contents_table, hierarchical_merge,
     make_colon_as_title, tokenize_chunks, docx_question_level
 from rag.nlp import rag_tokenizer
 from deepdoc.parser import PdfParser, DocxParser, PlainParser, HtmlParser
-from api.utils.log_utils import logger
 
 
 class Docx(DocxParser):
@@ -122,7 +121,7 @@ class Pdf(PdfParser):
         start = timer()
         self._layouts_rec(zoomin)
         callback(0.67, "Layout analysis finished")
-        logger.info("layouts:".format(
+        logging.debug("layouts:".format(
             ))
         self._naive_vertical_merge()