Update response_processor.py

response_processor.py

@@ -1,3 +1,4 @@
+# %%writefile response_processor.py
 import re
 import logging
 import traceback
@@ -195,8 +196,8 @@ class ResponseProcessor:
             raise ResponseProcessingError("Empty response provided for cleaning")
 
         try:
-            # 調試：記錄清理前的原始回應
-            self.logger.info(f"DEBUG: Response before cleaning: {response}")
+            # 記錄清理前的最初回應
+            # self.logger.info(f"DEBUG: Response before cleaning: {response}")
 
             self.logger.debug(f"Starting response cleaning (original length: {len(response)})")
 
@@ -209,8 +210,8 @@ class ResponseProcessor:
             else:
                 cleaned_response = self._clean_general_response(response)
 
-            # 調試：記錄清理後的回應
-            self.logger.info(f"DEBUG: Response after cleaning: {cleaned_response}")
+            # 記錄清理後的回應
+            # self.logger.info(f"DEBUG: Response after cleaning: {cleaned_response}")
 
             # 如果清理後內容過短，嘗試從原始回應中恢復
             if len(cleaned_response.strip()) < 40:
@@ -315,80 +316,20 @@ class ResponseProcessor:
     def _critical_format_preprocess(self, response: str) -> str:
         """
         關鍵格式預處理，處理最常見的格式問題
-    
+
         Args:
             response: 原始回應
-    
+
         Returns:
             str: 預處理後的回應
         """
         if not response:
             return response
-    
+
         try:
             import re
-            # 移除各種形式的 confirmed
-            confirmed_patterns = [
-                r'\bconfirmed\s+',  # "confirmed cars" -> "cars"
-                r'\b(\d+)\s+confirmed\s+([a-zA-Z\s]+)',  # "12 confirmed cars" -> "12 cars"
-                r'\b(one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)\s+confirmed\s+([a-zA-Z\s]+)',  # "twelve confirmed cars" -> "twelve cars"
-            ]
-            
-            for pattern in confirmed_patterns:
-                if pattern == r'\bconfirmed\s+':
-                    response = re.sub(pattern, '', response, flags=re.IGNORECASE)
-                else:
-                    response = re.sub(pattern, r'\1 \2', response, flags=re.IGNORECASE)
-    
-            # 數字轉文字的完整字典
-            number_conversions = {
-                '0': 'zero', '1': 'one', '2': 'two', '3': 'three', '4': 'four', '5': 'five',
-                '6': 'six', '7': 'seven', '8': 'eight', '9': 'nine', '10': 'ten',
-                '11': 'eleven', '12': 'twelve', '13': 'thirteen', '14': 'fourteen', '15': 'fifteen',
-                '16': 'sixteen', '17': 'seventeen', '18': 'eighteen', '19': 'nineteen', '20': 'twenty'
-            }
-    
-            # 強化數字替換邏輯 - 處理各種語法結構
-            for digit, word in number_conversions.items():
-                # 模式1: 數字 + 名詞 (如 "3 cars", "12 people")
-                pattern1 = rf'\b{digit}\s+([a-zA-Z]+(?:\s+[a-zA-Z]+)*)\b'
-                response = re.sub(pattern1, rf'{word} \1', response)
-    
-                # 模式2: 數字 + visible/present + 名詞 (如 "3 visible traffic lights")
-                pattern2 = rf'\b{digit}\s+(visible|present|apparent|evident)\s+([a-zA-Z]+(?:\s+[a-zA-Z]+)*)\b'
-                response = re.sub(pattern2, rf'{word} \1 \2', response, flags=re.IGNORECASE)
-    
-                # 模式3: 介詞 + 數字 + 名詞 (如 "against a backdrop of 3 visible traffic lights")
-                pattern3 = rf'\b(against|with|featuring|including|containing)\s+(?:a\s+backdrop\s+of\s+)?{digit}\s+([a-zA-Z]+(?:\s+[a-zA-Z]+)*)\b'
-                response = re.sub(pattern3, rf'\1 {word} \2', response, flags=re.IGNORECASE)
-    
-                # 模式4: 複合描述中的數字 (如 "featuring twelve confirmed cars and 3 confirmed persons")
-                pattern4 = rf'\b(and|,)\s+{digit}\s+([a-zA-Z]+(?:\s+[a-zA-Z]+)*)\b'
-                response = re.sub(pattern4, rf'\1 {word} \2', response, flags=re.IGNORECASE)
-                
-            grammar_fixes = [
-                # persons -> people 的全面修正
-                (r'\b(\d+|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|twenty)\s+persons\b', r'\1 people'),
-                (r'\bmultiple\s+persons\b', 'multiple people'),
-                (r'\bseveral\s+persons\b', 'several people'),
-                (r'\bmany\s+persons\b', 'many people'),
-                (r'\ba\s+few\s+persons\b', 'a few people'),
-                (r'\bsome\s+persons\b', 'some people'),
-                (r'\bvarious\s+persons\b', 'various people'),
-                (r'\bnumerous\s+persons\b', 'numerous people'),
-                
-                # 修正語法結構問題
-                (r'\bvisible\s+traffic\s+lights\b', 'traffic lights visible'),
-                (r'\bpresent\s+traffic\s+lights\b', 'traffic lights present'),
-                (r'\bapparent\s+traffic\s+lights\b', 'traffic lights apparent'),
-                
-                # 修正重複的形容詞結構
-                (r'\b(visible|present|apparent|evident)\s+(visible|present|apparent|evident)\s+', r'\1 '),
-            ]
-    
-            for pattern, replacement in grammar_fixes:
-                response = re.sub(pattern, replacement, response, flags=re.IGNORECASE)
 
+            # 第一優先級：處理斜線問題
             # 首先處理已知的斜線組合，使用形容詞替換
             for slash_combo, replacement in self.slash_replacements.items():
                 if slash_combo.lower() in response.lower():
@@ -399,11 +340,13 @@ class ResponseProcessor:
                         replacement_formatted = replacement.title()
                     else:
                         replacement_formatted = replacement
-    
+
                     # 執行替換（不區分大小寫）
                     response = re.sub(re.escape(slash_combo), replacement_formatted, response, flags=re.IGNORECASE)
-    
+                    self.logger.debug(f"Replaced slash pattern '{slash_combo}' with '{replacement_formatted}'")
+
             # 處理其他未預定義的斜線模式
+            # 標準斜線模式：word/word
             slash_pattern = r'\b([a-zA-Z]+)/([a-zA-Z]+)\b'
             matches = list(re.finditer(slash_pattern, response))
             for match in reversed(matches):  # 從後往前處理避免位置偏移
@@ -414,19 +357,22 @@ class ResponseProcessor:
                 else:
                     replacement = word2
                 response = response[:match.start()] + replacement + response[match.end():]
-    
+                self.logger.debug(f"Replaced general slash pattern '{match.group(0)}' with '{replacement}'")
+
+            # 第二優先級：處理底線格式
             # 首先處理已知的底線組合
             for underscore_combo, replacement in self.underscore_replacements.items():
                 if underscore_combo in response:
                     response = response.replace(underscore_combo, replacement)
-    
+                    self.logger.debug(f"Replaced underscore pattern '{underscore_combo}' with '{replacement}'")
+
             # 處理三個詞的底線組合：word_word_word → word word word
             response = re.sub(r'\b([a-z]+)_([a-z]+)_([a-z]+)\b', r'\1 \2 \3', response)
-    
+
             # 處理任何剩餘的底線模式：word_word → word word
             response = re.sub(r'\b([a-zA-Z]+)_([a-zA-Z]+)\b', r'\1 \2', response)
-    
-            # 確保句子的完整性
+
+            # 第三優先級：修正不完整句子
             incomplete_sentence_fixes = [
                 (r'\bIn\s*,\s*', 'Throughout the area, '),
                 (r'\bOverall,\s+exudes\b', 'Overall, the scene exudes'),
@@ -434,15 +380,35 @@ class ResponseProcessor:
                 (r'\bwith its lights turned illuminating\b', 'with its lights illuminating'),
                 (r'\bwhere it stands as\b', 'where it stands as'),
             ]
-    
+
             for pattern, replacement in incomplete_sentence_fixes:
                 response = re.sub(pattern, replacement, response, flags=re.IGNORECASE)
-    
-            # 清理多餘空格並確保格式一致性 
-            response = re.sub(r'\s+', ' ', response).strip()
-    
+
+            # 第四優先級：語法修正處理(像是person and people)
+            grammar_fixes = [
+                (r'\b(\d+)\s+persons\b', r'\1 people'),
+                (r'\bone\s+persons\b', 'one person'),
+                (r'\btwo\s+persons\b', 'two people'),
+                (r'\bthree\s+persons\b', 'three people'),
+                (r'\bfour\s+persons\b', 'four people'),
+                (r'\bfive\s+persons\b', 'five people'),
+                (r'\bsix\s+persons\b', 'six people'),
+                (r'\bseven\s+persons\b', 'seven people'),
+                (r'\beight\s+persons\b', 'eight people'),
+                (r'\bnine\s+persons\b', 'nine people'),
+                (r'\bten\s+persons\b', 'ten people'),
+                (r'\bmultiple\s+persons\b', 'multiple people'),
+                (r'\bseveral\s+persons\b', 'several people'),
+                (r'\bmany\s+persons\b', 'many people'),
+                (r'\ba\s+few\s+persons\b', 'a few people'),
+                (r'\bsome\s+persons\b', 'some people')
+            ]
+
+            for pattern, replacement in grammar_fixes:
+                response = re.sub(pattern, replacement, response, flags=re.IGNORECASE)
+
             return response
-    
+
         except Exception as e:
             self.logger.warning(f"Error in critical format preprocessing: {str(e)}")
             return response
@@ -669,88 +635,50 @@ class ResponseProcessor:
             return response # 發生錯誤時返回原始回應
 
     def _handle_repetitive_vocabulary(self, response: str) -> str:
-        """處理重複詞彙，使用改進的檢測和替換機制"""
+        """處理重複詞彙，使用 re.sub 和可呼叫的替換函數以提高效率和準確性。"""
         try:
-            # 先進行重複模式���測（記錄但不直接處理）
+            # 檢測重複模式 (僅警告)
             if hasattr(self, 'repetitive_patterns'):
                 for pattern, issue in self.repetitive_patterns:
-                    matches = list(re.finditer(pattern, response, re.IGNORECASE | re.DOTALL))
-                    if matches:
+                    if re.search(pattern, response, re.IGNORECASE | re.DOTALL):
                         self.logger.warning(f"Text quality issue detected: {issue} in response: \"{response[:100]}...\"")
-    
+
             if not hasattr(self, 'replacement_alternatives') or not self.replacement_alternatives:
                 return response
-    
+
             processed_response = response
-    
-            # 強化的重複詞彙處理
+
             for word_to_replace, alternatives in self.replacement_alternatives.items():
-                if not alternatives:
+                if not alternatives:  # 如果沒有可用的替代詞，則跳過
                     continue
-    
-                # 創建更精確的詞彙匹配模式
-                word_pattern = re.compile(r'\b' + re.escape(word_to_replace) + r'\b', re.IGNORECASE)
-                matches = list(word_pattern.finditer(processed_response))
-                
-                if len(matches) <= 1:
-                    continue  # 如果只出現一次或沒有出現，跳過
-    
-                # 對於多次出現的情況，進行智能替換
-                replacement_count = 0
-                alternative_index = 0
-                
-                def smart_replacer(match_obj):
-                    nonlocal replacement_count, alternative_index
-                    replacement_count += 1
-                    original_word = match_obj.group(0)
-                    
-                    # 第一次出現保持原樣，後續出現進行替換
-                    if replacement_count == 1:
-                        return original_word
-                    
-                    # 選擇適當的替代詞
-                    replacement = alternatives[alternative_index % len(alternatives)]
-                    alternative_index += 1
-                    
-                    # 保持原始大小寫格式
-                    if original_word.isupper():
-                        return replacement.upper()
-                    elif original_word.istitle():
-                        return replacement.capitalize()
-                    return replacement
-    
-                processed_response = word_pattern.sub(smart_replacer, processed_response)
-    
-            # === 新增：專門處理 "positioned" 的特殊邏輯 ===
-            # 由於 "positioned" 經常出現問題，給予特別處理
-            positioned_pattern = r'\b(positioned)\b'
-            positioned_matches = re.findall(positioned_pattern, processed_response, re.IGNORECASE)
-            
-            if len(positioned_matches) > 1:
-                # 替換除了第一個以外的所有 "positioned"
-                positioned_alternatives = ['arranged', 'placed', 'set', 'located', 'situated']
-                replacement_counter = 0
-                
-                def positioned_replacer(match):
-                    nonlocal replacement_counter
-                    if replacement_counter == 0:
-                        replacement_counter += 1
-                        return match.group(0)  # 保持第一個不變
-                    else:
-                        alt_index = (replacement_counter - 1) % len(positioned_alternatives)
-                        replacement_counter += 1
-                        original = match.group(0)
-                        new_word = positioned_alternatives[alt_index]
-                        
-                        # 保持大小寫格式
-                        if original.isupper():
-                            return new_word.upper()
-                        elif original.istitle():
-                            return new_word.capitalize()
-                        return new_word
-                
-                processed_response = re.sub(positioned_pattern, positioned_replacer, processed_response, flags=re.IGNORECASE)
-    
+
+                # 為每個詞創建一個獨立的計數器和替代索引
+                # 使用閉包或一個小類來封裝狀態
+                class WordReplacer:
+                    def __init__(self, alternatives_list):
+                        self.count = 0
+                        self.alternative_idx = 0
+                        self.alternatives_list = alternatives_list
+
+                    def __call__(self, match_obj):
+                        self.count += 1
+                        original_word = match_obj.group(0)
+                        if self.count > 1:  # 從第二次出現開始替換
+                            replacement = self.alternatives_list[self.alternative_idx % len(self.alternatives_list)]
+                            self.alternative_idx += 1
+                            # 保持原始大小寫格式
+                            if original_word.isupper():
+                                return replacement.upper()
+                            elif original_word.istitle():
+                                return replacement.capitalize()
+                            return replacement
+                        return original_word # 因為第一次出現, 就不用替換
+
+                replacer_instance = WordReplacer(alternatives)
+                # 使用 \b 確保匹配的是整個單詞
+                pattern = re.compile(r'\b' + re.escape(word_to_replace) + r'\b', re.IGNORECASE)
+                processed_response = pattern.sub(replacer_instance, processed_response)
+
             # 移除 identical 等重複性描述詞彙
             identical_cleanup_patterns = [
                 (r'\b(\d+)\s+identical\s+([a-zA-Z\s]+)', r'\1 \2'),
@@ -760,35 +688,41 @@ class ResponseProcessor:
                 (r'\bcomprehensive view featuring\b', 'scene featuring'),
                 (r'\bcomprehensive display of\b', 'display of'),
             ]
-    
+
             for pattern, replacement in identical_cleanup_patterns:
                 processed_response = re.sub(pattern, replacement, processed_response, flags=re.IGNORECASE)
-    
-            # 數字到文字轉換（保持原有邏輯）
+
+            # 數字到文字
             number_conversions = {
                 '2': 'two', '3': 'three', '4': 'four', '5': 'five', '6': 'six',
                 '7': 'seven', '8': 'eight', '9': 'nine', '10': 'ten',
                 '11': 'eleven', '12': 'twelve'
             }
-    
+
+            # 處理各種語法結構中的數字
             for digit, word in number_conversions.items():
-                # 各種數字模式的處理
-                patterns_to_fix = [
-                    (rf'\b{digit}\s+([a-zA-Z]+s)\b', rf'{word} \1'),
-                    (rf'\b{digit}\s+(more|additional|other|identical)\s+([a-zA-Z]+s)\b', rf'{word} \1 \2'),
-                    (rf'\b{digit}\s+([a-zA-Z]+)\s+([a-zA-Z]+s)\b', rf'{word} \1 \2'),
-                    (rf'\b(around|approximately|about)\s+{digit}\s+([a-zA-Z]+s)\b', rf'\1 {word} \2'),
-                ]
-                
-                for pattern, replacement in patterns_to_fix:
-                    processed_response = re.sub(pattern, replacement, processed_response, flags=re.IGNORECASE)
-    
+                # 模式1: 數字 + 單一複數詞 (如 "7 chairs")
+                pattern1 = rf'\b{digit}\s+([a-zA-Z]+s)\b'
+                processed_response = re.sub(pattern1, rf'{word} \1', processed_response)
+
+                # 模式2: 數字 + 修飾詞 + 複數詞 (如 "7 more chairs")
+                pattern2 = rf'\b{digit}\s+(more|additional|other|identical)\s+([a-zA-Z]+s)\b'
+                processed_response = re.sub(pattern2, rf'{word} \1 \2', processed_response, flags=re.IGNORECASE)
+
+                # 模式3: 數字 + 形容詞 + 複數詞 (如 "2 dining tables")
+                pattern3 = rf'\b{digit}\s+([a-zA-Z]+)\s+([a-zA-Z]+s)\b'
+                processed_response = re.sub(pattern3, rf'{word} \1 \2', processed_response)
+
+                # 模式4: 介詞片語中的數字 (如 "around 2 tables")
+                pattern4 = rf'\b(around|approximately|about)\s+{digit}\s+([a-zA-Z]+s)\b'
+                processed_response = re.sub(pattern4, rf'\1 {word} \2', processed_response, flags=re.IGNORECASE)
+
             return processed_response
-    
+
         except Exception as e:
             self.logger.error(f"Error in _handle_repetitive_vocabulary: {str(e)}")
             self.logger.error(traceback.format_exc())
-            return response
+            return response # 發生錯誤時返回原始回應
 
     def _ensure_grammatical_completeness(self, response: str) -> str:
         """
@@ -1055,75 +989,19 @@ class ResponseProcessor:
         return response.strip()
 
     def _control_word_length(self, response: str) -> str:
-        """控制文字長度在合理範圍內，確保句子完整性"""
+        """控制文字長度在合理範圍內"""
         words = response.split()
-        
-        # 提高基礎限制，給予更多彈性
-        base_limit = 220
-        extended_limit = 250
-        
-        if len(words) <= base_limit:
-            return response
-        
-        # 首先嘗試在基礎限制內找到完整句子
-        truncated = ' '.join(words[:base_limit])
-        last_period = max(truncated.rfind('.'), truncated.rfind('!'), truncated.rfind('?'))
-        
-        # 如果在基礎限制內找到了適當的句子結尾
-        if last_period > len(truncated) * 0.8:  # 確保截斷點不會太早
-            result = truncated[:last_period+1]
-            self.logger.info(f"Text truncated at {base_limit} words with proper sentence ending")
-            return result
-        
-        # 如果基礎限制內沒有找到合適結尾，擴展搜尋範圍
-        if len(words) > extended_limit:
-            extended_truncated = ' '.join(words[:extended_limit])
-            extended_last_period = max(
-                extended_truncated.rfind('.'), 
-                extended_truncated.rfind('!'), 
-                extended_truncated.rfind('?')
-            )
-            
-            # 在擴展範圍內找到合適的結尾
-            if extended_last_period > len(extended_truncated) * 0.7:
-                result = extended_truncated[:extended_last_period+1]
-                self.logger.info(f"Text truncated at extended limit with proper sentence ending")
-                return result
-        
-        # 如果仍然找不到合適的結尾，使用智能截斷
-        # 尋找最後一個完整的句子或子句
-        final_truncated = ' '.join(words[:base_limit])
-        
-        # 尋找可能的子句結尾（逗號後的位置）
-        last_comma = final_truncated.rfind(',')
-        last_semicolon = final_truncated.rfind(';')
-        
-        # 選擇最佳截斷點
-        best_cutoff = max(last_period, last_comma, last_semicolon)
-        
-        if best_cutoff > len(final_truncated) * 0.6:
-            # 如果是逗號或分號結尾，改為句號
-            result = final_truncated[:best_cutoff]
-            if result.endswith(',') or result.endswith(';'):
-                result = result[:-1] + '.'
-            elif not result.endswith(('.', '!', '?')):
-                result += '.'
-            
-            self.logger.warning(f"Text truncated with intelligent cutoff at position {best_cutoff}")
-            return result
-        
-        # 移除可能不完整的最後一個句子
-        # 找到倒數第二個句子的結尾
-        second_last_period = final_truncated.rfind('.', 0, last_period)
-        if second_last_period > 0:
-            result = final_truncated[:second_last_period+1]
-            self.logger.warning("Text truncated by removing incomplete final sentence")
-            return result
-        
-        # 如果所有方法都失敗，添加適合的結尾
-        result = final_truncated.rstrip() + "."
-        self.logger.warning("Text truncated with forced period ending")
-        return result
+        if len(words) > 200:
+            # 找到接近字數限制的句子結束處
+            truncated = ' '.join(words[:200])
+            last_period = max(truncated.rfind('.'), truncated.rfind('!'), truncated.rfind('?'))
+
+            if last_period > 0:
+                response = truncated[:last_period+1]
+            else:
+                response = truncated + "."
+
+        return response
 
     def _final_formatting(self, response: str) -> str:
         """最終格式化處理"""