Upload 30 files

bert_gen.py

@@ -1,17 +1,16 @@
-import argparse
-from multiprocessing import Pool, cpu_count
-
 import torch
-import torch.multiprocessing as mp
-from tqdm import tqdm
-
+from multiprocessing import Pool
 import commons
 import utils
+from tqdm import tqdm
+from text import check_bert_models, cleaned_text_to_sequence, get_bert
+import argparse
+import torch.multiprocessing as mp
 from config import config
-from text import cleaned_text_to_sequence, get_bert
 
 
-def process_line(line):
+def process_line(x):
+    line, add_blank = x
     device = config.bert_gen_config.device
     if config.bert_gen_config.use_multi_device:
         rank = mp.current_process()._identity
@@ -28,18 +27,19 @@ def process_line(line):
     word2ph = [i for i in word2ph]
     phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
 
-    phone = commons.intersperse(phone, 0)
-    tone = commons.intersperse(tone, 0)
-    language = commons.intersperse(language, 0)
-    for i in range(len(word2ph)):
-        word2ph[i] = word2ph[i] * 2
-    word2ph[0] += 1
+    if add_blank:
+        phone = commons.intersperse(phone, 0)
+        tone = commons.intersperse(tone, 0)
+        language = commons.intersperse(language, 0)
+        for i in range(len(word2ph)):
+            word2ph[i] = word2ph[i] * 2
+        word2ph[0] += 1
 
     bert_path = wav_path.replace(".WAV", ".wav").replace(".wav", ".bert.pt")
 
     try:
         bert = torch.load(bert_path)
-        assert bert.shape[-1] == len(phone)
+        assert bert.shape[0] == 2048
     except Exception:
         bert = get_bert(text, word2ph, language_str, device)
         assert bert.shape[-1] == len(phone)
@@ -59,16 +59,23 @@ if __name__ == "__main__":
     args, _ = parser.parse_known_args()
     config_path = args.config
     hps = utils.get_hparams_from_file(config_path)
+    check_bert_models()
     lines = []
     with open(hps.data.training_files, encoding="utf-8") as f:
         lines.extend(f.readlines())
 
     with open(hps.data.validation_files, encoding="utf-8") as f:
         lines.extend(f.readlines())
+    add_blank = [hps.data.add_blank] * len(lines)
+
     if len(lines) != 0:
-        num_processes = min(args.num_processes, cpu_count())
+        num_processes = args.num_processes
         with Pool(processes=num_processes) as pool:
-            for _ in tqdm(pool.imap_unordered(process_line, lines), total=len(lines)):
-                pass
+            for _ in tqdm(
+                pool.imap_unordered(process_line, zip(lines, add_blank)),
+                total=len(lines),
+            ):
+                # 这里是缩进的代码块，表示循环体
+                pass  # 使用pass语句作为占位符
 
     print(f"bert生成完毕!, 共有{len(lines)}个bert.pt生成!")

config.yml

@@ -83,7 +83,7 @@ train_ms:
   base:
     use_base_model: false
     repo_id: "Stardust_minus/Bert-VITS2"
-    model_image: "Bert-VITS2_2.2-Clap底模" # openi网页的模型名
+    model_image: "Bert-VITS2_2.3底模" # openi网页的模型名
   # 训练模型存储目录：与旧版本的区别，原先数据集是存放在logs/model_name下的，现在改为统一存放在Data/你的数据集/models下
   model: "models"
   # 配置文件路径
@@ -172,6 +172,6 @@ server:
 # 请不要在github等网站公开分享你的app id 与 key
 translate:
   # 你的APPID
-  "app_key": ""
+  "app_key": "20231117001883321"
   # 你的密钥
-  "secret_key": ""
+  "secret_key": "lMQbvZHeJveDceLof2wf"

data_utils.py

@@ -3,7 +3,6 @@ import random
 import torch
 import torch.utils.data
 from tqdm import tqdm
-import numpy as np
 from tools.log import logger
 import commons
 from mel_processing import spectrogram_torch, mel_spectrogram_torch
@@ -44,10 +43,6 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
         self.min_text_len = getattr(hparams, "min_text_len", 1)
         self.max_text_len = getattr(hparams, "max_text_len", 384)
 
-        self.empty_emo = torch.squeeze(
-            torch.load("empty_emo.npy", map_location="cpu"), dim=1
-        )
-
         random.seed(1234)
         random.shuffle(self.audiopaths_sid_text)
         self._filter()
@@ -98,14 +93,7 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
         spec, wav = self.get_audio(audiopath)
         sid = torch.LongTensor([int(self.spk_map[sid])])
 
-        if np.random.rand() > 0.1:
-            emo = torch.squeeze(
-                torch.load(audiopath.replace(".wav", ".emo.npy"), map_location="cpu"),
-                dim=1,
-            )
-        else:
-            emo = self.empty_emo
-        return (phones, spec, wav, sid, tone, language, bert, ja_bert, en_bert, emo)
+        return (phones, spec, wav, sid, tone, language, bert, ja_bert, en_bert)
 
     def get_audio(self, filename):
         audio, sampling_rate = load_wav_to_torch(filename)
@@ -168,15 +156,15 @@ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
 
         if language_str == "ZH":
             bert = bert_ori
-            ja_bert = torch.rand(1024, len(phone))
-            en_bert = torch.rand(1024, len(phone))
+            ja_bert = torch.randn(1024, len(phone))
+            en_bert = torch.randn(1024, len(phone))
         elif language_str == "JP":
-            bert = torch.rand(1024, len(phone))
+            bert = torch.randn(1024, len(phone))
             ja_bert = bert_ori
-            en_bert = torch.rand(1024, len(phone))
+            en_bert = torch.randn(1024, len(phone))
         elif language_str == "EN":
-            bert = torch.rand(1024, len(phone))
-            ja_bert = torch.rand(1024, len(phone))
+            bert = torch.randn(1024, len(phone))
+            ja_bert = torch.randn(1024, len(phone))
             en_bert = bert_ori
         phone = torch.LongTensor(phone)
         tone = torch.LongTensor(tone)
@@ -226,7 +214,6 @@ class TextAudioSpeakerCollate:
         bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len)
         ja_bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len)
         en_bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len)
-        emo = torch.FloatTensor(len(batch), 512)
 
         spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len)
         wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len)
@@ -238,7 +225,6 @@ class TextAudioSpeakerCollate:
         bert_padded.zero_()
         ja_bert_padded.zero_()
         en_bert_padded.zero_()
-        emo.zero_()
 
         for i in range(len(ids_sorted_decreasing)):
             row = batch[ids_sorted_decreasing[i]]
@@ -272,8 +258,6 @@ class TextAudioSpeakerCollate:
             en_bert = row[8]
             en_bert_padded[i, :, : en_bert.size(1)] = en_bert
 
-            emo[i, :] = row[9]
-
         return (
             text_padded,
             text_lengths,
@@ -287,7 +271,6 @@ class TextAudioSpeakerCollate:
             bert_padded,
             ja_bert_padded,
             en_bert_padded,
-            emo,
         )
 
 

export_onnx.py

@@ -5,8 +5,10 @@ if __name__ == "__main__":
     export_path = "BertVits2.2PT"
     model_path = "model\\G_0.pth"
     config_path = "model\\config.json"
+    novq = False
+    dev = False
     if not os.path.exists("onnx"):
         os.makedirs("onnx")
     if not os.path.exists(f"onnx/{export_path}"):
         os.makedirs(f"onnx/{export_path}")
-    export_onnx(export_path, model_path, config_path)
+    export_onnx(export_path, model_path, config_path, novq, dev)

hiyoriUI.py

@@ -0,0 +1,725 @@
+"""
+api服务，网页后端 多版本多模型 fastapi实现
+原 server_fastapi
+"""
+import logging
+import gc
+import random
+import librosa
+import gradio
+import numpy as np
+import utils
+from fastapi import FastAPI, Query, Request, File, UploadFile, Form
+from fastapi.responses import Response, FileResponse
+from fastapi.staticfiles import StaticFiles
+from io import BytesIO
+from scipy.io import wavfile
+import uvicorn
+import torch
+import webbrowser
+import psutil
+import GPUtil
+from typing import Dict, Optional, List, Set, Union, Tuple
+import os
+from tools.log import logger
+from urllib.parse import unquote
+
+from infer import infer, get_net_g, latest_version
+import tools.translate as trans
+from tools.sentence import split_by_language
+from re_matching import cut_sent
+
+
+from config import config
+
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+
+class Model:
+    """模型封装类"""
+
+    def __init__(self, config_path: str, model_path: str, device: str, language: str):
+        self.config_path: str = os.path.normpath(config_path)
+        self.model_path: str = os.path.normpath(model_path)
+        self.device: str = device
+        self.language: str = language
+        self.hps = utils.get_hparams_from_file(config_path)
+        self.spk2id: Dict[str, int] = self.hps.data.spk2id  # spk - id 映射字典
+        self.id2spk: Dict[int, str] = dict()  # id - spk 映射字典
+        for speaker, speaker_id in self.hps.data.spk2id.items():
+            self.id2spk[speaker_id] = speaker
+        self.version: str = (
+            self.hps.version if hasattr(self.hps, "version") else latest_version
+        )
+        self.net_g = get_net_g(
+            model_path=model_path,
+            version=self.version,
+            device=device,
+            hps=self.hps,
+        )
+
+    def to_dict(self) -> Dict[str, any]:
+        return {
+            "config_path": self.config_path,
+            "model_path": self.model_path,
+            "device": self.device,
+            "language": self.language,
+            "spk2id": self.spk2id,
+            "id2spk": self.id2spk,
+            "version": self.version,
+        }
+
+
+class Models:
+    def __init__(self):
+        self.models: Dict[int, Model] = dict()
+        self.num = 0
+        # spkInfo[角色名][模型id] = 角色id
+        self.spk_info: Dict[str, Dict[int, int]] = dict()
+        self.path2ids: Dict[str, Set[int]] = dict()  # 路径指向的model的id
+
+    def init_model(
+        self, config_path: str, model_path: str, device: str, language: str
+    ) -> int:
+        """
+        初始化并添加一个模型
+
+        :param config_path: 模型config.json路径
+        :param model_path: 模型路径
+        :param device: 模型推理使用设备
+        :param language: 模型推理默认语言
+        """
+        # 若文件不存在则不进行加载
+        if not os.path.isfile(model_path):
+            if model_path != "":
+                logger.warning(f"模型文件{model_path} 不存在，不进行初始化")
+            return self.num
+        if not os.path.isfile(config_path):
+            if config_path != "":
+                logger.warning(f"配置文件{config_path} 不存在，不进行初始化")
+            return self.num
+
+        # 若路径中的模型已存在，则不添加模型，若不存在，则进行初始化。
+        model_path = os.path.realpath(model_path)
+        if model_path not in self.path2ids.keys():
+            self.path2ids[model_path] = {self.num}
+            self.models[self.num] = Model(
+                config_path=config_path,
+                model_path=model_path,
+                device=device,
+                language=language,
+            )
+            logger.success(f"添加模型{model_path}，使用配置文件{os.path.realpath(config_path)}")
+        else:
+            # 获取一个指向id
+            m_id = next(iter(self.path2ids[model_path]))
+            self.models[self.num] = self.models[m_id]
+            self.path2ids[model_path].add(self.num)
+            logger.success("模型已存在，添加模型引用。")
+        # 添加角色信息
+        for speaker, speaker_id in self.models[self.num].spk2id.items():
+            if speaker not in self.spk_info.keys():
+                self.spk_info[speaker] = {self.num: speaker_id}
+            else:
+                self.spk_info[speaker][self.num] = speaker_id
+        # 修改计数
+        self.num += 1
+        return self.num - 1
+
+    def del_model(self, index: int) -> Optional[int]:
+        """删除对应序号的模型，若不存在则返回None"""
+        if index not in self.models.keys():
+            return None
+        # 删除角色信息
+        for speaker, speaker_id in self.models[index].spk2id.items():
+            self.spk_info[speaker].pop(index)
+            if len(self.spk_info[speaker]) == 0:
+                # 若对应角色的所有模型都被删除，则清除该角色信息
+                self.spk_info.pop(speaker)
+        # 删除路径信息
+        model_path = os.path.realpath(self.models[index].model_path)
+        self.path2ids[model_path].remove(index)
+        if len(self.path2ids[model_path]) == 0:
+            self.path2ids.pop(model_path)
+            logger.success(f"删除模型{model_path}, id = {index}")
+        else:
+            logger.success(f"删除模型引用{model_path}, id = {index}")
+        # 删除模型
+        self.models.pop(index)
+        gc.collect()
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        return index
+
+    def get_models(self):
+        """获取所有模型"""
+        return self.models
+
+
+if __name__ == "__main__":
+    app = FastAPI()
+    app.logger = logger
+    # 挂载静态文件
+    logger.info("开始挂载网页页面")
+    StaticDir: str = "./Web"
+    if not os.path.isdir(StaticDir):
+        logger.warning(
+            "缺少网页资源，无法开启网页页面，如有需要请在 https://github.com/jiangyuxiaoxiao/Bert-VITS2-UI 或者Bert-VITS对应版本的release页面下载"
+        )
+    else:
+        dirs = [fir.name for fir in os.scandir(StaticDir) if fir.is_dir()]
+        files = [fir.name for fir in os.scandir(StaticDir) if fir.is_dir()]
+        for dirName in dirs:
+            app.mount(
+                f"/{dirName}",
+                StaticFiles(directory=f"./{StaticDir}/{dirName}"),
+                name=dirName,
+            )
+    loaded_models = Models()
+    # 加载模型
+    logger.info("开始加载模型")
+    models_info = config.server_config.models
+    for model_info in models_info:
+        loaded_models.init_model(
+            config_path=model_info["config"],
+            model_path=model_info["model"],
+            device=model_info["device"],
+            language=model_info["language"],
+        )
+
+    @app.get("/")
+    async def index():
+        return FileResponse("./Web/index.html")
+
+    async def _voice(
+        text: str,
+        model_id: int,
+        speaker_name: str,
+        speaker_id: int,
+        sdp_ratio: float,
+        noise: float,
+        noisew: float,
+        length: float,
+        language: str,
+        auto_translate: bool,
+        auto_split: bool,
+        emotion: Optional[Union[int, str]] = None,
+        reference_audio=None,
+        style_text: Optional[str] = None,
+        style_weight: float = 0.7,
+    ) -> Union[Response, Dict[str, any]]:
+        """TTS实现函数"""
+
+        # 检查
+        # 检查模型是否存在
+        if model_id not in loaded_models.models.keys():
+            logger.error(f"/voice 请求错误：模型model_id={model_id}未加载")
+            return {"status": 10, "detail": f"模型model_id={model_id}未加载"}
+        # 检查是否提供speaker
+        if speaker_name is None and speaker_id is None:
+            logger.error("/voice 请求错误：推理请求未提供speaker_name或speaker_id")
+            return {"status": 11, "detail": "请提供speaker_name或speaker_id"}
+        elif speaker_name is None:
+            # 检查speaker_id是否存在
+            if speaker_id not in loaded_models.models[model_id].id2spk.keys():
+                logger.error(f"/voice 请求错误：角色speaker_id={speaker_id}不存在")
+                return {"status": 12, "detail": f"角色speaker_id={speaker_id}不存在"}
+            speaker_name = loaded_models.models[model_id].id2spk[speaker_id]
+        # 检查speaker_name是否存在
+        if speaker_name not in loaded_models.models[model_id].spk2id.keys():
+            logger.error(f"/voice 请求错误：角色speaker_name={speaker_name}不存在")
+            return {"status": 13, "detail": f"角色speaker_name={speaker_name}不存在"}
+        # 未传入则使用默认语言
+        if language is None:
+            language = loaded_models.models[model_id].language
+        # 翻译会破坏mix结构，auto也会变得无意义。不要在这两个模式下使用
+        if auto_translate:
+            if language == "auto" or language == "mix":
+                logger.error(
+                    f"/voice 请求错误：请勿同时使用language = {language}与auto_translate模式"
+                )
+                return {
+                    "status": 20,
+                    "detail": f"请勿同时使用language = {language}与auto_translate模式",
+                }
+            text = trans.translate(Sentence=text, to_Language=language.lower())
+        if reference_audio is not None:
+            ref_audio = BytesIO(await reference_audio.read())
+            # 2.2 适配
+            if loaded_models.models[model_id].version == "2.2":
+                ref_audio, _ = librosa.load(ref_audio, 48000)
+        else:
+            ref_audio = reference_audio
+
+        # 改动：增加使用 || 对文本进行主动切分
+        # 切分优先级： || → auto/mix → auto_split
+        text2 = text.replace("\n", "").lstrip()
+        texts: List[str] = text2.split("||")
+
+        # 对于mix和auto的说明：出于版本兼容性���考虑，暂时无法使用multilang的方式进行推理
+        if language == "MIX":
+            text_language_speakers: List[Tuple[str, str, str]] = []
+            for _text in texts:
+                speaker_pieces = _text.split("[")  # 按说话人分割多块
+                for speaker_piece in speaker_pieces:
+                    if speaker_piece == "":
+                        continue
+                    speaker_piece2 = speaker_piece.split("]")
+                    if len(speaker_piece2) != 2:
+                        return {
+                            "status": 21,
+                            "detail": "MIX语法错误",
+                        }
+                    speaker = speaker_piece2[0].strip()
+                    lang_pieces = speaker_piece2[1].split("<")
+                    for lang_piece in lang_pieces:
+                        if lang_piece == "":
+                            continue
+                        lang_piece2 = lang_piece.split(">")
+                        if len(lang_piece2) != 2:
+                            return {
+                                "status": 21,
+                                "detail": "MIX语法错误",
+                            }
+                        lang = lang_piece2[0].strip()
+                        if lang.upper() not in ["ZH", "EN", "JP"]:
+                            return {
+                                "status": 21,
+                                "detail": "MIX语法错误",
+                            }
+                        t = lang_piece2[1]
+                        text_language_speakers.append((t, lang.upper(), speaker))
+
+        elif language == "AUTO":
+            text_language_speakers: List[Tuple[str, str, str]] = [
+                (final_text, language.upper().replace("JA", "JP"), speaker_name)
+                for sub_list in [
+                    split_by_language(_text, target_languages=["zh", "ja", "en"])
+                    for _text in texts
+                    if _text != ""
+                ]
+                for final_text, language in sub_list
+                if final_text != ""
+            ]
+        else:
+            text_language_speakers: List[Tuple[str, str, str]] = [
+                (_text, language, speaker_name) for _text in texts if _text != ""
+            ]
+
+        if auto_split:
+            text_language_speakers: List[Tuple[str, str, str]] = [
+                (final_text, lang, speaker)
+                for _text, lang, speaker in text_language_speakers
+                for final_text in cut_sent(_text)
+            ]
+
+        audios = []
+        with torch.no_grad():
+            for _text, lang, speaker in text_language_speakers:
+                audios.append(
+                    infer(
+                        text=_text,
+                        sdp_ratio=sdp_ratio,
+                        noise_scale=noise,
+                        noise_scale_w=noisew,
+                        length_scale=length,
+                        sid=speaker,
+                        language=lang,
+                        hps=loaded_models.models[model_id].hps,
+                        net_g=loaded_models.models[model_id].net_g,
+                        device=loaded_models.models[model_id].device,
+                        emotion=emotion,
+                        reference_audio=ref_audio,
+                        style_text=style_text,
+                        style_weight=style_weight,
+                    )
+                )
+                # audios.append(np.zeros(int(44100 * 0.2)))
+            # audios.pop()
+            audio = np.concatenate(audios)
+            audio = gradio.processing_utils.convert_to_16_bit_wav(audio)
+        with BytesIO() as wavContent:
+            wavfile.write(
+                wavContent, loaded_models.models[model_id].hps.data.sampling_rate, audio
+            )
+            response = Response(content=wavContent.getvalue(), media_type="audio/wav")
+            return response
+
+    @app.post("/voice")
+    async def voice(
+        request: Request,  # fastapi自动注入
+        text: str = Form(...),
+        model_id: int = Query(..., description="模型ID"),  # 模型序号
+        speaker_name: str = Query(
+            None, description="说话人名"
+        ),  # speaker_name与 speaker_id二者选其一
+        speaker_id: int = Query(None, description="说话人id，与speaker_name二选一"),
+        sdp_ratio: float = Query(0.2, description="SDP/DP混合比"),
+        noise: float = Query(0.2, description="感情"),
+        noisew: float = Query(0.9, description="音素长度"),
+        length: float = Query(1, description="语速"),
+        language: str = Query(None, description="语言"),  # 若不指定使用语言则使用默认值
+        auto_translate: bool = Query(False, description="自动翻译"),
+        auto_split: bool = Query(False, description="自动切分"),
+        emotion: Optional[Union[int, str]] = Query(None, description="emo"),
+        reference_audio: UploadFile = File(None),
+        style_text: Optional[str] = Form(None, description="风格文本"),
+        style_weight: float = Query(0.7, description="风格权重"),
+    ):
+        """语音接口，若需要上传参考音频请仅使用post请求"""
+        logger.info(
+            f"{request.client.host}:{request.client.port}/voice  { unquote(str(request.query_params) )} text={text}"
+        )
+        return await _voice(
+            text=text,
+            model_id=model_id,
+            speaker_name=speaker_name,
+            speaker_id=speaker_id,
+            sdp_ratio=sdp_ratio,
+            noise=noise,
+            noisew=noisew,
+            length=length,
+            language=language,
+            auto_translate=auto_translate,
+            auto_split=auto_split,
+            emotion=emotion,
+            reference_audio=reference_audio,
+            style_text=style_text,
+            style_weight=style_weight,
+        )
+
+    @app.get("/voice")
+    async def voice(
+        request: Request,  # fastapi自动注入
+        text: str = Query(..., description="输入文字"),
+        model_id: int = Query(..., description="模型ID"),  # 模型序号
+        speaker_name: str = Query(
+            None, description="说话人名"
+        ),  # speaker_name与 speaker_id二者选其一
+        speaker_id: int = Query(None, description="说话人id，与speaker_name二选一"),
+        sdp_ratio: float = Query(0.2, description="SDP/DP混合比"),
+        noise: float = Query(0.2, description="感情"),
+        noisew: float = Query(0.9, description="音素长度"),
+        length: float = Query(1, description="语速"),
+        language: str = Query(None, description="语言"),  # 若不指定使用语言则使用默认值
+        auto_translate: bool = Query(False, description="自动翻译"),
+        auto_split: bool = Query(False, description="自动切分"),
+        emotion: Optional[Union[int, str]] = Query(None, description="emo"),
+        style_text: Optional[str] = Query(None, description="风格文本"),
+        style_weight: float = Query(0.7, description="风格权重"),
+    ):
+        """语音接口，不建议使用"""
+        logger.info(
+            f"{request.client.host}:{request.client.port}/voice  { unquote(str(request.query_params) )}"
+        )
+        return await _voice(
+            text=text,
+            model_id=model_id,
+            speaker_name=speaker_name,
+            speaker_id=speaker_id,
+            sdp_ratio=sdp_ratio,
+            noise=noise,
+            noisew=noisew,
+            length=length,
+            language=language,
+            auto_translate=auto_translate,
+            auto_split=auto_split,
+            emotion=emotion,
+            style_text=style_text,
+            style_weight=style_weight,
+        )
+
+    @app.get("/models/info")
+    def get_loaded_models_info(request: Request):
+        """获取已加载模型信息"""
+
+        result: Dict[str, Dict] = dict()
+        for key, model in loaded_models.models.items():
+            result[str(key)] = model.to_dict()
+        return result
+
+    @app.get("/models/delete")
+    def delete_model(
+        request: Request, model_id: int = Query(..., description="删除模型id")
+    ):
+        """删除指定模型"""
+        logger.info(
+            f"{request.client.host}:{request.client.port}/models/delete  { unquote(str(request.query_params) )}"
+        )
+        result = loaded_models.del_model(model_id)
+        if result is None:
+            logger.error(f"/models/delete 模型删除错误：模型{model_id}不存在，删除失败")
+            return {"status": 14, "detail": f"模型{model_id}不存在，删除失败"}
+
+        return {"status": 0, "detail": "删除成功"}
+
+    @app.get("/models/add")
+    def add_model(
+        request: Request,
+        model_path: str = Query(..., description="添加模型路径"),
+        config_path: str = Query(
+            None, description="添加模型配置文件路径，不填则使用./config.json或../config.json"
+        ),
+        device: str = Query("cuda", description="推理使用设备"),
+        language: str = Query("ZH", description="模型默认语言"),
+    ):
+        """添加指定模型：允许重复添加相同路径模型，且不重复占用内存"""
+        logger.info(
+            f"{request.client.host}:{request.client.port}/models/add  { unquote(str(request.query_params) )}"
+        )
+        if config_path is None:
+            model_dir = os.path.dirname(model_path)
+            if os.path.isfile(os.path.join(model_dir, "config.json")):
+                config_path = os.path.join(model_dir, "config.json")
+            elif os.path.isfile(os.path.join(model_dir, "../config.json")):
+                config_path = os.path.join(model_dir, "../config.json")
+            else:
+                logger.error("/models/add 模型添加失败：未在模型所在目录以及上级目录找到config.json文件")
+                return {
+                    "status": 15,
+                    "detail": "查询未传���配置文件路径，同时默认路径./与../中不存在配置文件config.json。",
+                }
+        try:
+            model_id = loaded_models.init_model(
+                config_path=config_path,
+                model_path=model_path,
+                device=device,
+                language=language,
+            )
+        except Exception:
+            logging.exception("模型加载出错")
+            return {
+                "status": 16,
+                "detail": "模型加载出错，详细查看日志",
+            }
+        return {
+            "status": 0,
+            "detail": "模型添加成功",
+            "Data": {
+                "model_id": model_id,
+                "model_info": loaded_models.models[model_id].to_dict(),
+            },
+        }
+
+    def _get_all_models(root_dir: str = "Data", only_unloaded: bool = False):
+        """从root_dir搜索获取所有可用模型"""
+        result: Dict[str, List[str]] = dict()
+        files = os.listdir(root_dir) + ["."]
+        for file in files:
+            if os.path.isdir(os.path.join(root_dir, file)):
+                sub_dir = os.path.join(root_dir, file)
+                # 搜索 "sub_dir" 、 "sub_dir/models" 两个路径
+                result[file] = list()
+                sub_files = os.listdir(sub_dir)
+                model_files = []
+                for sub_file in sub_files:
+                    relpath = os.path.realpath(os.path.join(sub_dir, sub_file))
+                    if only_unloaded and relpath in loaded_models.path2ids.keys():
+                        continue
+                    if sub_file.endswith(".pth") and sub_file.startswith("G_"):
+                        if os.path.isfile(relpath):
+                            model_files.append(sub_file)
+                # 对模型文件按步数排序
+                model_files = sorted(
+                    model_files,
+                    key=lambda pth: int(pth.lstrip("G_").rstrip(".pth"))
+                    if pth.lstrip("G_").rstrip(".pth").isdigit()
+                    else 10**10,
+                )
+                result[file] = model_files
+                models_dir = os.path.join(sub_dir, "models")
+                model_files = []
+                if os.path.isdir(models_dir):
+                    sub_files = os.listdir(models_dir)
+                    for sub_file in sub_files:
+                        relpath = os.path.realpath(os.path.join(models_dir, sub_file))
+                        if only_unloaded and relpath in loaded_models.path2ids.keys():
+                            continue
+                        if sub_file.endswith(".pth") and sub_file.startswith("G_"):
+                            if os.path.isfile(os.path.join(models_dir, sub_file)):
+                                model_files.append(f"models/{sub_file}")
+                    # 对模型文件按步数排序
+                    model_files = sorted(
+                        model_files,
+                        key=lambda pth: int(pth.lstrip("models/G_").rstrip(".pth"))
+                        if pth.lstrip("models/G_").rstrip(".pth").isdigit()
+                        else 10**10,
+                    )
+                    result[file] += model_files
+                if len(result[file]) == 0:
+                    result.pop(file)
+
+        return result
+
+    @app.get("/models/get_unloaded")
+    def get_unloaded_models_info(
+        request: Request, root_dir: str = Query("Data", description="搜索根目录")
+    ):
+        """获取未加载模型"""
+        logger.info(
+            f"{request.client.host}:{request.client.port}/models/get_unloaded  { unquote(str(request.query_params) )}"
+        )
+        return _get_all_models(root_dir, only_unloaded=True)
+
+    @app.get("/models/get_local")
+    def get_local_models_info(
+        request: Request, root_dir: str = Query("Data", description="搜索根目录")
+    ):
+        """获取全部本地模型"""
+        logger.info(
+            f"{request.client.host}:{request.client.port}/models/get_local  { unquote(str(request.query_params) )}"
+        )
+        return _get_all_models(root_dir, only_unloaded=False)
+
+    @app.get("/status")
+    def get_status():
+        """获取电脑运行状态"""
+        cpu_percent = psutil.cpu_percent(interval=1)
+        memory_info = psutil.virtual_memory()
+        memory_total = memory_info.total
+        memory_available = memory_info.available
+        memory_used = memory_info.used
+        memory_percent = memory_info.percent
+        gpuInfo = []
+        devices = ["cpu"]
+        for i in range(torch.cuda.device_count()):
+            devices.append(f"cuda:{i}")
+        gpus = GPUtil.getGPUs()
+        for gpu in gpus:
+            gpuInfo.append(
+                {
+                    "gpu_id": gpu.id,
+                    "gpu_load": gpu.load,
+                    "gpu_memory": {
+                        "total": gpu.memoryTotal,
+                        "used": gpu.memoryUsed,
+                        "free": gpu.memoryFree,
+                    },
+                }
+            )
+        return {
+            "devices": devices,
+            "cpu_percent": cpu_percent,
+            "memory_total": memory_total,
+            "memory_available": memory_available,
+            "memory_used": memory_used,
+            "memory_percent": memory_percent,
+            "gpu": gpuInfo,
+        }
+
+    @app.get("/tools/translate")
+    def translate(
+        request: Request,
+        texts: str = Query(..., description="待翻译文本"),
+        to_language: str = Query(..., description="翻译目标语言"),
+    ):
+        """翻译"""
+        logger.info(
+            f"{request.client.host}:{request.client.port}/tools/translate  { unquote(str(request.query_params) )}"
+        )
+        return {"texts": trans.translate(Sentence=texts, to_Language=to_language)}
+
+    all_examples: Dict[str, Dict[str, List]] = dict()  # 存放示例
+
+    @app.get("/tools/random_example")
+    def random_example(
+        request: Request,
+        language: str = Query(None, description="指定语言，未指定则随机返回"),
+        root_dir: str = Query("Data", description="搜索根目录"),
+    ):
+        """
+        获取一个随机音频+文本，用于对比，音频会从本地目录随机选择。
+        """
+        logger.info(
+            f"{request.client.host}:{request.client.port}/tools/random_example  { unquote(str(request.query_params) )}"
+        )
+        global all_examples
+        # 数据初始化
+        if root_dir not in all_examples.keys():
+            all_examples[root_dir] = {"ZH": [], "JP": [], "EN": []}
+
+            examples = all_examples[root_dir]
+
+            # 从项目Data目录中搜索train/val.list
+            for root, directories, _files in os.walk(root_dir):
+                for file in _files:
+                    if file in ["train.list", "val.list"]:
+                        with open(
+                            os.path.join(root, file), mode="r", encoding="utf-8"
+                        ) as f:
+                            lines = f.readlines()
+                            for line in lines:
+                                data = line.split("|")
+                                if len(data) != 7:
+                                    continue
+                                # 音频存在 且语言为ZH/EN/JP
+                                if os.path.isfile(data[0]) and data[2] in [
+                                    "ZH",
+                                    "JP",
+                                    "EN",
+                                ]:
+                                    examples[data[2]].append(
+                                        {
+                                            "text": data[3],
+                                            "audio": data[0],
+                                            "speaker": data[1],
+                                        }
+                                    )
+
+        examples = all_examples[root_dir]
+        if language is None:
+            if len(examples["ZH"]) + len(examples["JP"]) + len(examples["EN"]) == 0:
+                return {"status": 17, "detail": "没有加载任何示例数据"}
+            else:
+                # 随机选一个
+                rand_num = random.randint(
+                    0,
+                    len(examples["ZH"]) + len(examples["JP"]) + len(examples["EN"]) - 1,
+                )
+                # ZH
+                if rand_num < len(examples["ZH"]):
+                    return {"status": 0, "Data": examples["ZH"][rand_num]}
+                # JP
+                if rand_num < len(examples["ZH"]) + len(examples["JP"]):
+                    return {
+                        "status": 0,
+                        "Data": examples["JP"][rand_num - len(examples["ZH"])],
+                    }
+                # EN
+                return {
+                    "status": 0,
+                    "Data": examples["EN"][
+                        rand_num - len(examples["ZH"]) - len(examples["JP"])
+                    ],
+                }
+
+        else:
+            if len(examples[language]) == 0:
+                return {"status": 17, "detail": f"没有加载任何{language}数据"}
+            return {
+                "status": 0,
+                "Data": examples[language][
+                    random.randint(0, len(examples[language]) - 1)
+                ],
+            }
+
+    @app.get("/tools/get_audio")
+    def get_audio(request: Request, path: str = Query(..., description="本地音频路径")):
+        logger.info(
+            f"{request.client.host}:{request.client.port}/tools/get_audio  { unquote(str(request.query_params) )}"
+        )
+        if not os.path.isfile(path):
+            logger.error(f"/tools/get_audio 获取音频错误：指定音频{path}不存在")
+            return {"status": 18, "detail": "指定音频不存在"}
+        if not path.lower().endswith(".wav"):
+            logger.error(f"/tools/get_audio 获取音频错误：音频{path}非wav文件")
+            return {"status": 19, "detail": "非wav格式文件"}
+        return FileResponse(path=path)
+
+    logger.warning("本地服务，请勿将服务端口暴露于外网")
+    logger.info(f"api文档地址 http://127.0.0.1:{config.server_config.port}/docs")
+    if os.path.isdir(StaticDir):
+        webbrowser.open(f"http://127.0.0.1:{config.server_config.port}")
+    uvicorn.run(
+        app, port=config.server_config.port, host="0.0.0.0", log_level="warning"
+    )

infer.py

@@ -5,19 +5,22 @@
     2. 请在模型的config.json中显示声明版本号，添加一个字段"version" : "你的版本号"
 特殊版本说明：
     1.1.1-fix： 1.1.1版本训练的模型，但是在推理时使用dev的日语修复
-    2.2：当前版本
+    2.3：当前版本
 """
 import torch
 import commons
 from text import cleaned_text_to_sequence, get_bert
-from clap_wrapper import get_clap_audio_feature, get_clap_text_feature
+
+# from clap_wrapper import get_clap_audio_feature, get_clap_text_feature
+from typing import Union
 from text.cleaner import clean_text
 import utils
-import numpy as np
 
 from models import SynthesizerTrn
 from text.symbols import symbols
 
+from oldVersion.V220.models import SynthesizerTrn as V220SynthesizerTrn
+from oldVersion.V220.text import symbols as V220symbols
 from oldVersion.V210.models import SynthesizerTrn as V210SynthesizerTrn
 from oldVersion.V210.text import symbols as V210symbols
 from oldVersion.V200.models import SynthesizerTrn as V200SynthesizerTrn
@@ -29,13 +32,14 @@ from oldVersion.V110.text import symbols as V110symbols
 from oldVersion.V101.models import SynthesizerTrn as V101SynthesizerTrn
 from oldVersion.V101.text import symbols as V101symbols
 
-from oldVersion import V111, V110, V101, V200, V210
+from oldVersion import V111, V110, V101, V200, V210, V220
 
 # 当前版本信息
-latest_version = "2.2"
+latest_version = "2.3"
 
 # 版本兼容
 SynthesizerTrnMap = {
+    "2.2": V220SynthesizerTrn,
     "2.1": V210SynthesizerTrn,
     "2.0.2-fix": V200SynthesizerTrn,
     "2.0.1": V200SynthesizerTrn,
@@ -50,6 +54,7 @@ SynthesizerTrnMap = {
 }
 
 symbolsMap = {
+    "2.2": V220symbols,
     "2.1": V210symbols,
     "2.0.2-fix": V200symbols,
     "2.0.1": V200symbols,
@@ -98,7 +103,8 @@ def get_net_g(model_path: str, version: str, device: str, hps):
     return net_g
 
 
-def get_text(text, language_str, hps, device):
+def get_text(text, language_str, hps, device, style_text=None, style_weight=0.7):
+    style_text = None if style_text == "" else style_text
     # 在此处实现当前版本的get_text
     norm_text, phone, tone, word2ph = clean_text(text, language_str)
     phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
@@ -110,21 +116,23 @@ def get_text(text, language_str, hps, device):
         for i in range(len(word2ph)):
             word2ph[i] = word2ph[i] * 2
         word2ph[0] += 1
-    bert_ori = get_bert(norm_text, word2ph, language_str, device)
+    bert_ori = get_bert(
+        norm_text, word2ph, language_str, device, style_text, style_weight
+    )
     del word2ph
     assert bert_ori.shape[-1] == len(phone), phone
 
     if language_str == "ZH":
         bert = bert_ori
-        ja_bert = torch.rand(1024, len(phone))
-        en_bert = torch.rand(1024, len(phone))
+        ja_bert = torch.randn(1024, len(phone))
+        en_bert = torch.randn(1024, len(phone))
     elif language_str == "JP":
-        bert = torch.rand(1024, len(phone))
+        bert = torch.randn(1024, len(phone))
         ja_bert = bert_ori
-        en_bert = torch.rand(1024, len(phone))
+        en_bert = torch.randn(1024, len(phone))
     elif language_str == "EN":
-        bert = torch.rand(1024, len(phone))
-        ja_bert = torch.rand(1024, len(phone))
+        bert = torch.randn(1024, len(phone))
+        ja_bert = torch.randn(1024, len(phone))
         en_bert = bert_ori
     else:
         raise ValueError("language_str should be ZH, JP or EN")
@@ -141,7 +149,7 @@ def get_text(text, language_str, hps, device):
 
 def infer(
     text,
-    emotion,
+    emotion: Union[int, str],
     sdp_ratio,
     noise_scale,
     noise_scale_w,
@@ -154,8 +162,13 @@ def infer(
     reference_audio=None,
     skip_start=False,
     skip_end=False,
+    style_text=None,
+    style_weight=0.7,
 ):
     # 2.2版本参数位置变了
+    inferMap_V4 = {
+        "2.2": V220.infer,
+    }
     # 2.1 参数新增 emotion reference_audio skip_start skip_end
     inferMap_V3 = {
         "2.1": V210.infer,
@@ -180,6 +193,25 @@ def infer(
     version = hps.version if hasattr(hps, "version") else latest_version
     # 非当前版本，根据版本号选择合适的infer
     if version != latest_version:
+        if version in inferMap_V4.keys():
+            return inferMap_V4[version](
+                text,
+                emotion,
+                sdp_ratio,
+                noise_scale,
+                noise_scale_w,
+                length_scale,
+                sid,
+                language,
+                hps,
+                net_g,
+                device,
+                reference_audio,
+                skip_start,
+                skip_end,
+                style_text,
+                style_weight,
+            )
         if version in inferMap_V3.keys():
             return inferMap_V3[version](
                 text,
@@ -196,6 +228,8 @@ def infer(
                 emotion,
                 skip_start,
                 skip_end,
+                style_text,
+                style_weight,
             )
         if version in inferMap_V2.keys():
             return inferMap_V2[version](
@@ -224,14 +258,19 @@ def infer(
             )
     # 在此处实现当前版本的推理
     # emo = get_emo_(reference_audio, emotion, sid)
-    if isinstance(reference_audio, np.ndarray):
-        emo = get_clap_audio_feature(reference_audio, device)
-    else:
-        emo = get_clap_text_feature(emotion, device)
-    emo = torch.squeeze(emo, dim=1)
+    # if isinstance(reference_audio, np.ndarray):
+    #     emo = get_clap_audio_feature(reference_audio, device)
+    # else:
+    #     emo = get_clap_text_feature(emotion, device)
+    # emo = torch.squeeze(emo, dim=1)
 
     bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
-        text, language, hps, device
+        text,
+        language,
+        hps,
+        device,
+        style_text=style_text,
+        style_weight=style_weight,
     )
     if skip_start:
         phones = phones[3:]
@@ -255,7 +294,7 @@ def infer(
         ja_bert = ja_bert.to(device).unsqueeze(0)
         en_bert = en_bert.to(device).unsqueeze(0)
         x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
-        emo = emo.to(device).unsqueeze(0)
+        # emo = emo.to(device).unsqueeze(0)
         del phones
         speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
         audio = (
@@ -268,7 +307,6 @@ def infer(
                 bert,
                 ja_bert,
                 en_bert,
-                emo,
                 sdp_ratio=sdp_ratio,
                 noise_scale=noise_scale,
                 noise_scale_w=noise_scale_w,
@@ -278,7 +316,16 @@ def infer(
             .float()
             .numpy()
         )
-        del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers, ja_bert, en_bert, emo
+        del (
+            x_tst,
+            tones,
+            lang_ids,
+            bert,
+            x_tst_lengths,
+            speakers,
+            ja_bert,
+            en_bert,
+        )  # , emo
         if torch.cuda.is_available():
             torch.cuda.empty_cache()
         return audio
@@ -302,14 +349,14 @@ def infer_multilang(
 ):
     bert, ja_bert, en_bert, phones, tones, lang_ids = [], [], [], [], [], []
     # emo = get_emo_(reference_audio, emotion, sid)
-    if isinstance(reference_audio, np.ndarray):
-        emo = get_clap_audio_feature(reference_audio, device)
-    else:
-        emo = get_clap_text_feature(emotion, device)
-    emo = torch.squeeze(emo, dim=1)
+    # if isinstance(reference_audio, np.ndarray):
+    #     emo = get_clap_audio_feature(reference_audio, device)
+    # else:
+    #     emo = get_clap_text_feature(emotion, device)
+    # emo = torch.squeeze(emo, dim=1)
     for idx, (txt, lang) in enumerate(zip(text, language)):
-        skip_start = (idx != 0) or (skip_start and idx == 0)
-        skip_end = (idx != len(text) - 1) or (skip_end and idx == len(text) - 1)
+        _skip_start = (idx != 0) or (skip_start and idx == 0)
+        _skip_end = (idx != len(language) - 1) or skip_end
         (
             temp_bert,
             temp_ja_bert,
@@ -318,14 +365,14 @@ def infer_multilang(
             temp_tones,
             temp_lang_ids,
         ) = get_text(txt, lang, hps, device)
-        if skip_start:
+        if _skip_start:
             temp_bert = temp_bert[:, 3:]
             temp_ja_bert = temp_ja_bert[:, 3:]
             temp_en_bert = temp_en_bert[:, 3:]
             temp_phones = temp_phones[3:]
             temp_tones = temp_tones[3:]
             temp_lang_ids = temp_lang_ids[3:]
-        if skip_end:
+        if _skip_end:
             temp_bert = temp_bert[:, :-2]
             temp_ja_bert = temp_ja_bert[:, :-2]
             temp_en_bert = temp_en_bert[:, :-2]
@@ -351,7 +398,7 @@ def infer_multilang(
         bert = bert.to(device).unsqueeze(0)
         ja_bert = ja_bert.to(device).unsqueeze(0)
         en_bert = en_bert.to(device).unsqueeze(0)
-        emo = emo.to(device).unsqueeze(0)
+        # emo = emo.to(device).unsqueeze(0)
         x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
         del phones
         speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
@@ -365,7 +412,6 @@ def infer_multilang(
                 bert,
                 ja_bert,
                 en_bert,
-                emo,
                 sdp_ratio=sdp_ratio,
                 noise_scale=noise_scale,
                 noise_scale_w=noise_scale_w,
@@ -375,7 +421,16 @@ def infer_multilang(
             .float()
             .numpy()
         )
-        del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers, ja_bert, en_bert, emo
+        del (
+            x_tst,
+            tones,
+            lang_ids,
+            bert,
+            x_tst_lengths,
+            speakers,
+            ja_bert,
+            en_bert,
+        )  # , emo
         if torch.cuda.is_available():
             torch.cuda.empty_cache()
         return audio

losses.py

@@ -1,4 +1,6 @@
 import torch
+import torchaudio
+from transformers import AutoModel
 
 
 def feature_loss(fmap_r, fmap_g):
@@ -56,3 +58,96 @@ def kl_loss(z_p, logs_q, m_p, logs_p, z_mask):
     kl = torch.sum(kl * z_mask)
     l = kl / torch.sum(z_mask)
     return l
+
+
+class WavLMLoss(torch.nn.Module):
+    def __init__(self, model, wd, model_sr, slm_sr=16000):
+        super(WavLMLoss, self).__init__()
+        self.wavlm = AutoModel.from_pretrained(model)
+        self.wd = wd
+        self.resample = torchaudio.transforms.Resample(model_sr, slm_sr)
+        self.wavlm.eval()
+        for param in self.wavlm.parameters():
+            param.requires_grad = False
+
+    def forward(self, wav, y_rec):
+        with torch.no_grad():
+            wav_16 = self.resample(wav)
+            wav_embeddings = self.wavlm(
+                input_values=wav_16, output_hidden_states=True
+            ).hidden_states
+        y_rec_16 = self.resample(y_rec)
+        y_rec_embeddings = self.wavlm(
+            input_values=y_rec_16.squeeze(), output_hidden_states=True
+        ).hidden_states
+
+        floss = 0
+        for er, eg in zip(wav_embeddings, y_rec_embeddings):
+            floss += torch.mean(torch.abs(er - eg))
+
+        return floss.mean()
+
+    def generator(self, y_rec):
+        y_rec_16 = self.resample(y_rec)
+        y_rec_embeddings = self.wavlm(
+            input_values=y_rec_16, output_hidden_states=True
+        ).hidden_states
+        y_rec_embeddings = (
+            torch.stack(y_rec_embeddings, dim=1)
+            .transpose(-1, -2)
+            .flatten(start_dim=1, end_dim=2)
+        )
+        y_df_hat_g = self.wd(y_rec_embeddings)
+        loss_gen = torch.mean((1 - y_df_hat_g) ** 2)
+
+        return loss_gen
+
+    def discriminator(self, wav, y_rec):
+        with torch.no_grad():
+            wav_16 = self.resample(wav)
+            wav_embeddings = self.wavlm(
+                input_values=wav_16, output_hidden_states=True
+            ).hidden_states
+            y_rec_16 = self.resample(y_rec)
+            y_rec_embeddings = self.wavlm(
+                input_values=y_rec_16, output_hidden_states=True
+            ).hidden_states
+
+            y_embeddings = (
+                torch.stack(wav_embeddings, dim=1)
+                .transpose(-1, -2)
+                .flatten(start_dim=1, end_dim=2)
+            )
+            y_rec_embeddings = (
+                torch.stack(y_rec_embeddings, dim=1)
+                .transpose(-1, -2)
+                .flatten(start_dim=1, end_dim=2)
+            )
+
+        y_d_rs = self.wd(y_embeddings)
+        y_d_gs = self.wd(y_rec_embeddings)
+
+        y_df_hat_r, y_df_hat_g = y_d_rs, y_d_gs
+
+        r_loss = torch.mean((1 - y_df_hat_r) ** 2)
+        g_loss = torch.mean((y_df_hat_g) ** 2)
+
+        loss_disc_f = r_loss + g_loss
+
+        return loss_disc_f.mean()
+
+    def discriminator_forward(self, wav):
+        with torch.no_grad():
+            wav_16 = self.resample(wav)
+            wav_embeddings = self.wavlm(
+                input_values=wav_16, output_hidden_states=True
+            ).hidden_states
+            y_embeddings = (
+                torch.stack(wav_embeddings, dim=1)
+                .transpose(-1, -2)
+                .flatten(start_dim=1, end_dim=2)
+            )
+
+        y_d_rs = self.wd(y_embeddings)
+
+        return y_d_rs

models.py

@@ -14,8 +14,6 @@ from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
 from commons import init_weights, get_padding
 from text import symbols, num_tones, num_languages
 
-from vector_quantize_pytorch import VectorQuantize
-
 
 class DurationDiscriminator(nn.Module):  # vits2
     def __init__(
@@ -40,33 +38,22 @@ class DurationDiscriminator(nn.Module):  # vits2
         self.norm_2 = modules.LayerNorm(filter_channels)
         self.dur_proj = nn.Conv1d(1, filter_channels, 1)
 
-        self.pre_out_conv_1 = nn.Conv1d(
-            2 * filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
-        )
-        self.pre_out_norm_1 = modules.LayerNorm(filter_channels)
-        self.pre_out_conv_2 = nn.Conv1d(
-            filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
+        self.LSTM = nn.LSTM(
+            2 * filter_channels, filter_channels, batch_first=True, bidirectional=True
         )
-        self.pre_out_norm_2 = modules.LayerNorm(filter_channels)
 
         if gin_channels != 0:
             self.cond = nn.Conv1d(gin_channels, in_channels, 1)
 
-        self.output_layer = nn.Sequential(nn.Linear(filter_channels, 1), nn.Sigmoid())
+        self.output_layer = nn.Sequential(
+            nn.Linear(2 * filter_channels, 1), nn.Sigmoid()
+        )
 
-    def forward_probability(self, x, x_mask, dur, g=None):
+    def forward_probability(self, x, dur):
         dur = self.dur_proj(dur)
         x = torch.cat([x, dur], dim=1)
-        x = self.pre_out_conv_1(x * x_mask)
-        x = torch.relu(x)
-        x = self.pre_out_norm_1(x)
-        x = self.drop(x)
-        x = self.pre_out_conv_2(x * x_mask)
-        x = torch.relu(x)
-        x = self.pre_out_norm_2(x)
-        x = self.drop(x)
-        x = x * x_mask
         x = x.transpose(1, 2)
+        x, _ = self.LSTM(x)
         output_prob = self.output_layer(x)
         return output_prob
 
@@ -86,7 +73,7 @@ class DurationDiscriminator(nn.Module):  # vits2
 
         output_probs = []
         for dur in [dur_r, dur_hat]:
-            output_prob = self.forward_probability(x, x_mask, dur, g)
+            output_prob = self.forward_probability(x, dur)
             output_probs.append(output_prob)
 
         return output_probs
@@ -354,7 +341,6 @@ class TextEncoder(nn.Module):
         n_layers,
         kernel_size,
         p_dropout,
-        n_speakers,
         gin_channels=0,
     ):
         super().__init__()
@@ -376,31 +362,6 @@ class TextEncoder(nn.Module):
         self.bert_proj = nn.Conv1d(1024, hidden_channels, 1)
         self.ja_bert_proj = nn.Conv1d(1024, hidden_channels, 1)
         self.en_bert_proj = nn.Conv1d(1024, hidden_channels, 1)
-        # self.emo_proj = nn.Linear(512, hidden_channels)
-        self.in_feature_net = nn.Sequential(
-            # input is assumed to an already normalized embedding
-            nn.Linear(512, 1028, bias=False),
-            nn.GELU(),
-            nn.LayerNorm(1028),
-            *[Block(1028, 512) for _ in range(1)],
-            nn.Linear(1028, 512, bias=False),
-            # normalize before passing to VQ?
-            # nn.GELU(),
-            # nn.LayerNorm(512),
-        )
-        self.emo_vq = VectorQuantize(
-            dim=512,
-            codebook_size=64,
-            codebook_dim=32,
-            commitment_weight=0.1,
-            decay=0.85,
-            heads=32,
-            kmeans_iters=20,
-            separate_codebook_per_head=True,
-            stochastic_sample_codes=True,
-            threshold_ema_dead_code=2,
-        )
-        self.out_feature_net = nn.Linear(512, hidden_channels)
 
         self.encoder = attentions.Encoder(
             hidden_channels,
@@ -413,18 +374,10 @@ class TextEncoder(nn.Module):
         )
         self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
 
-    def forward(
-        self, x, x_lengths, tone, language, bert, ja_bert, en_bert, emo, sid, g=None
-    ):
-        sid = sid.cpu()
+    def forward(self, x, x_lengths, tone, language, bert, ja_bert, en_bert, g=None):
         bert_emb = self.bert_proj(bert).transpose(1, 2)
         ja_bert_emb = self.ja_bert_proj(ja_bert).transpose(1, 2)
         en_bert_emb = self.en_bert_proj(en_bert).transpose(1, 2)
-        emo_emb = self.in_feature_net(emo)
-        emo_emb, _, loss_commit = self.emo_vq(emo_emb.unsqueeze(1))
-        loss_commit = loss_commit.mean()
-        emo_emb = self.out_feature_net(emo_emb)
-        # emo_emb = self.emo_proj(emo.unsqueeze(1))
         x = (
             self.emb(x)
             + self.tone_emb(tone)
@@ -432,7 +385,6 @@ class TextEncoder(nn.Module):
             + bert_emb
             + ja_bert_emb
             + en_bert_emb
-            + emo_emb
         ) * math.sqrt(
             self.hidden_channels
         )  # [b, t, h]
@@ -445,7 +397,7 @@ class TextEncoder(nn.Module):
         stats = self.proj(x) * x_mask
 
         m, logs = torch.split(stats, self.out_channels, dim=1)
-        return x, m, logs, x_mask, loss_commit
+        return x, m, logs, x_mask
 
 
 class ResidualCouplingBlock(nn.Module):
@@ -748,6 +700,55 @@ class MultiPeriodDiscriminator(torch.nn.Module):
         return y_d_rs, y_d_gs, fmap_rs, fmap_gs
 
 
+class WavLMDiscriminator(nn.Module):
+    """docstring for Discriminator."""
+
+    def __init__(
+        self, slm_hidden=768, slm_layers=13, initial_channel=64, use_spectral_norm=False
+    ):
+        super(WavLMDiscriminator, self).__init__()
+        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
+        self.pre = norm_f(
+            Conv1d(slm_hidden * slm_layers, initial_channel, 1, 1, padding=0)
+        )
+
+        self.convs = nn.ModuleList(
+            [
+                norm_f(
+                    nn.Conv1d(
+                        initial_channel, initial_channel * 2, kernel_size=5, padding=2
+                    )
+                ),
+                norm_f(
+                    nn.Conv1d(
+                        initial_channel * 2,
+                        initial_channel * 4,
+                        kernel_size=5,
+                        padding=2,
+                    )
+                ),
+                norm_f(
+                    nn.Conv1d(initial_channel * 4, initial_channel * 4, 5, 1, padding=2)
+                ),
+            ]
+        )
+
+        self.conv_post = norm_f(Conv1d(initial_channel * 4, 1, 3, 1, padding=1))
+
+    def forward(self, x):
+        x = self.pre(x)
+
+        fmap = []
+        for l in self.convs:
+            x = l(x)
+            x = F.leaky_relu(x, modules.LRELU_SLOPE)
+            fmap.append(x)
+        x = self.conv_post(x)
+        x = torch.flatten(x, 1, -1)
+
+        return x
+
+
 class ReferenceEncoder(nn.Module):
     """
     inputs --- [N, Ty/r, n_mels*r]  mels
@@ -878,7 +879,6 @@ class SynthesizerTrn(nn.Module):
             n_layers,
             kernel_size,
             p_dropout,
-            self.n_speakers,
             gin_channels=self.enc_gin_channels,
         )
         self.dec = Generator(
@@ -946,14 +946,13 @@ class SynthesizerTrn(nn.Module):
         bert,
         ja_bert,
         en_bert,
-        emo=None,
     ):
         if self.n_speakers > 0:
             g = self.emb_g(sid).unsqueeze(-1)  # [b, h, 1]
         else:
             g = self.ref_enc(y.transpose(1, 2)).unsqueeze(-1)
-        x, m_p, logs_p, x_mask, loss_commit = self.enc_p(
-            x, x_lengths, tone, language, bert, ja_bert, en_bert, emo, sid, g=g
+        x, m_p, logs_p, x_mask = self.enc_p(
+            x, x_lengths, tone, language, bert, ja_bert, en_bert, g=g
         )
         z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g)
         z_p = self.flow(z, y_mask, g=g)
@@ -996,9 +995,11 @@ class SynthesizerTrn(nn.Module):
 
         logw_ = torch.log(w + 1e-6) * x_mask
         logw = self.dp(x, x_mask, g=g)
+        logw_sdp = self.sdp(x, x_mask, g=g, reverse=True, noise_scale=1.0)
         l_length_dp = torch.sum((logw - logw_) ** 2, [1, 2]) / torch.sum(
             x_mask
         )  # for averaging
+        l_length_sdp += torch.sum((logw_sdp - logw_) ** 2, [1, 2]) / torch.sum(x_mask)
 
         l_length = l_length_dp + l_length_sdp
 
@@ -1018,9 +1019,8 @@ class SynthesizerTrn(nn.Module):
             x_mask,
             y_mask,
             (z, z_p, m_p, logs_p, m_q, logs_q),
-            (x, logw, logw_),
+            (x, logw, logw_, logw_sdp),
             g,
-            loss_commit,
         )
 
     def infer(
@@ -1033,7 +1033,6 @@ class SynthesizerTrn(nn.Module):
         bert,
         ja_bert,
         en_bert,
-        emo=None,
         noise_scale=0.667,
         length_scale=1,
         noise_scale_w=0.8,
@@ -1047,8 +1046,8 @@ class SynthesizerTrn(nn.Module):
             g = self.emb_g(sid).unsqueeze(-1)  # [b, h, 1]
         else:
             g = self.ref_enc(y.transpose(1, 2)).unsqueeze(-1)
-        x, m_p, logs_p, x_mask, _ = self.enc_p(
-            x, x_lengths, tone, language, bert, ja_bert, en_bert, emo, sid, g=g
+        x, m_p, logs_p, x_mask = self.enc_p(
+            x, x_lengths, tone, language, bert, ja_bert, en_bert, g=g
         )
         logw = self.sdp(x, x_mask, g=g, reverse=True, noise_scale=noise_scale_w) * (
             sdp_ratio

modules.py

@@ -83,7 +83,7 @@ class ConvReluNorm(nn.Module):
 
 class DDSConv(nn.Module):
     """
-    Dialted and Depth-Separable Convolution
+    Dilated and Depth-Separable Convolution
     """
 
     def __init__(self, channels, kernel_size, n_layers, p_dropout=0.0):

onnx_infer.py

@@ -0,0 +1,60 @@
+from onnx_modules.V220_OnnxInference import OnnxInferenceSession
+import numpy as np
+
+Session = OnnxInferenceSession(
+    {
+        "enc": "onnx/BertVits2.2PT/BertVits2.2PT_enc_p.onnx",
+        "emb_g": "onnx/BertVits2.2PT/BertVits2.2PT_emb.onnx",
+        "dp": "onnx/BertVits2.2PT/BertVits2.2PT_dp.onnx",
+        "sdp": "onnx/BertVits2.2PT/BertVits2.2PT_sdp.onnx",
+        "flow": "onnx/BertVits2.2PT/BertVits2.2PT_flow.onnx",
+        "dec": "onnx/BertVits2.2PT/BertVits2.2PT_dec.onnx",
+    },
+    Providers=["CPUExecutionProvider"],
+)
+
+# 这里的输入和原版是一样的，只需要在原版预处理结果出来之后加上.numpy()即可
+x = np.array(
+    [
+        0,
+        97,
+        0,
+        8,
+        0,
+        78,
+        0,
+        8,
+        0,
+        76,
+        0,
+        37,
+        0,
+        40,
+        0,
+        97,
+        0,
+        8,
+        0,
+        23,
+        0,
+        8,
+        0,
+        74,
+        0,
+        26,
+        0,
+        104,
+        0,
+    ]
+)
+tone = np.zeros_like(x)
+language = np.zeros_like(x)
+sid = np.array([0])
+bert = np.random.randn(x.shape[0], 1024)
+ja_bert = np.random.randn(x.shape[0], 1024)
+en_bert = np.random.randn(x.shape[0], 1024)
+emo = np.random.randn(512, 1)
+
+audio = Session(x, tone, language, bert, ja_bert, en_bert, emo, sid)
+
+print(audio)

re_matching.py

@@ -44,7 +44,6 @@ def text_matching(text: str) -> list:
     result = []
     for speaker, dialogue in matches:
         result.append(extract_language_and_text_updated(speaker, dialogue))
-    print(result)
     return result
 
 

resample.py

@@ -10,11 +10,11 @@ from config import config
 
 
 def process(item):
-    wav_name, args = item
-    wav_path = os.path.join(args.in_dir, wav_name)
+    spkdir, wav_name, args = item
+    wav_path = os.path.join(args.in_dir, spkdir, wav_name)
     if os.path.exists(wav_path) and wav_path.lower().endswith(".wav"):
         wav, sr = librosa.load(wav_path, sr=args.sr)
-        soundfile.write(os.path.join(args.out_dir, wav_name), wav, sr)
+        soundfile.write(os.path.join(args.out_dir, spkdir, wav_name), wav, sr)
 
 
 if __name__ == "__main__":
@@ -54,11 +54,15 @@ if __name__ == "__main__":
     tasks = []
 
     for dirpath, _, filenames in os.walk(args.in_dir):
-        if not os.path.isdir(args.out_dir):
-            os.makedirs(args.out_dir, exist_ok=True)
+        # 子级目录
+        spk_dir = os.path.relpath(dirpath, args.in_dir)
+        spk_dir_out = os.path.join(args.out_dir, spk_dir)
+        if not os.path.isdir(spk_dir_out):
+            os.makedirs(spk_dir_out, exist_ok=True)
         for filename in filenames:
             if filename.lower().endswith(".wav"):
-                tasks.append((filename, args))
+                twople = (spk_dir, filename, args)
+                tasks.append(twople)
 
     for _ in tqdm(
         pool.imap_unordered(process, tasks),

resample_legacy.py

@@ -0,0 +1,71 @@
+import os
+import argparse
+import librosa
+from multiprocessing import Pool, cpu_count
+
+import soundfile
+from tqdm import tqdm
+
+from config import config
+
+
+def process(item):
+    wav_name, args = item
+    wav_path = os.path.join(args.in_dir, wav_name)
+    if os.path.exists(wav_path) and wav_path.lower().endswith(".wav"):
+        wav, sr = librosa.load(wav_path, sr=args.sr)
+        soundfile.write(os.path.join(args.out_dir, wav_name), wav, sr)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--sr",
+        type=int,
+        default=config.resample_config.sampling_rate,
+        help="sampling rate",
+    )
+    parser.add_argument(
+        "--in_dir",
+        type=str,
+        default=config.resample_config.in_dir,
+        help="path to source dir",
+    )
+    parser.add_argument(
+        "--out_dir",
+        type=str,
+        default=config.resample_config.out_dir,
+        help="path to target dir",
+    )
+    parser.add_argument(
+        "--processes",
+        type=int,
+        default=0,
+        help="cpu_processes",
+    )
+    args, _ = parser.parse_known_args()
+    # autodl 无卡模式会识别出46个cpu
+    if args.processes == 0:
+        processes = cpu_count() - 2 if cpu_count() > 4 else 1
+    else:
+        processes = args.processes
+    pool = Pool(processes=processes)
+
+    tasks = []
+
+    for dirpath, _, filenames in os.walk(args.in_dir):
+        if not os.path.isdir(args.out_dir):
+            os.makedirs(args.out_dir, exist_ok=True)
+        for filename in filenames:
+            if filename.lower().endswith(".wav"):
+                tasks.append((filename, args))
+
+    for _ in tqdm(
+        pool.imap_unordered(process, tasks),
+    ):
+        pass
+
+    pool.close()
+    pool.join()
+
+    print("音频重采样完毕!")

server.py

@@ -3,10 +3,8 @@ import os
 from pathlib import Path
 
 import logging
-import re_matching
 import uuid
-from flask import Flask, request, jsonify, render_template_string
-from flask_cors import CORS
+import re_matching
 
 logging.getLogger("numba").setLevel(logging.WARNING)
 logging.getLogger("markdown_it").setLevel(logging.WARNING)
@@ -18,6 +16,8 @@ logging.basicConfig(
 )
 
 logger = logging.getLogger(__name__)
+import shutil
+from scipy.io.wavfile import write
 import librosa
 import numpy as np
 import torch
@@ -26,24 +26,44 @@ from torch.utils.data import Dataset
 from torch.utils.data import DataLoader, Dataset
 from tqdm import tqdm
 
+import gradio as gr
+
 import utils
 from config import config
-import requests
+
 import torch
 import commons
 from text import cleaned_text_to_sequence, get_bert
-from clap_wrapper import get_clap_audio_feature, get_clap_text_feature
-
 from text.cleaner import clean_text
 import utils
 
 from models import SynthesizerTrn
 from text.symbols import symbols
 import sys
+import re
 
-from scipy.io.wavfile import write
+import random
+import hashlib
 
+from fugashi import Tagger
+import jaconv
+import unidic
+import subprocess
+
+import requests
+
+from ebooklib import epub
+import PyPDF2
+from PyPDF2 import PdfReader
+from bs4 import BeautifulSoup
+import jieba 
+import romajitable
+
+from flask import Flask, request, jsonify, render_template_string, send_file
+from flask_cors import CORS
+from scipy.io.wavfile import write
 net_g = None
+
 device = (
         "cuda:0"
         if torch.cuda.is_available()
@@ -54,7 +74,375 @@ device = (
         )
     )
 
-#device = 'cpu'
+#device = "cpu"
+BandList = {
+        "PoppinParty":["香澄","有咲","たえ","りみ","沙綾"],
+        "Afterglow":["蘭","モカ","ひまり","巴","つぐみ"],
+        "HelloHappyWorld":["こころ","美咲","薫","花音","はぐみ"],
+        "PastelPalettes":["彩","日菜","千聖","イヴ","麻弥"],
+        "Roselia":["友希那","紗夜","リサ","燐子","あこ"],
+        "RaiseASuilen":["レイヤ","ロック","ますき","チュチュ","パレオ"],
+        "Morfonica":["ましろ","瑠唯","つくし","七深","透子"],
+        "MyGo":["燈","愛音","そよ","立希","楽奈"],
+        "AveMujica":["祥子","睦","海鈴","にゃむ","初華"],
+        "圣翔音乐学园":["華戀","光","香子","雙葉","真晝","純那","克洛迪娜","真矢","奈奈"],
+        "凛明馆女子学校":["珠緒","壘","文","悠悠子","一愛"],
+        "弗隆提亚艺术学校":["艾露","艾露露","菈樂菲","司","靜羽"],
+        "西克菲尔特音乐学院":["晶","未知留","八千代","栞","美帆"]
+}
+
+webBase = 'https://mahiruoshi-bangdream-bert-vits2.hf.space/'
+
+port = 8080
+
+languages = [ "Auto", "ZH", "JP"]
+modelPaths = []
+modes = ['pyopenjtalk-V2.3-Katakana','fugashi-V2.3-Katakana','pyopenjtalk-V2.3-Katakana-Katakana','fugashi-V2.3-Katakana-Katakana','onnx-V2.3']
+sentence_modes = ['sentence','paragraph']
+for dirpath, dirnames, filenames in os.walk('Data/BangDream/models/'):
+    for filename in filenames:
+        modelPaths.append(os.path.join(dirpath, filename))
+hps = utils.get_hparams_from_file('Data/BangDream/config.json')
+
+def translate(Sentence: str, to_Language: str = "jp", from_Language: str = ""):
+    """
+    :param Sentence: 待翻译语句
+    :param from_Language: 待翻译语句语言
+    :param to_Language: 目标语言
+    :return: 翻译后语句 出错时返回None
+
+    常见语言代码：中文 zh 英语 en 日语 jp
+    """
+    appid = "20231117001883321"
+    key = "lMQbvZHeJveDceLof2wf"
+    if appid == "" or key == "":
+        return "请开发者在config.yml中配置app_key与secret_key"
+    url = "https://fanyi-api.baidu.com/api/trans/vip/translate"
+    texts = Sentence.splitlines()
+    outTexts = []
+    for t in texts:
+        if t != "":
+            # 签名计算 参考文档 https://api.fanyi.baidu.com/product/113
+            salt = str(random.randint(1, 100000))
+            signString = appid + t + salt + key
+            hs = hashlib.md5()
+            hs.update(signString.encode("utf-8"))
+            signString = hs.hexdigest()
+            if from_Language == "":
+                from_Language = "auto"
+            headers = {"Content-Type": "application/x-www-form-urlencoded"}
+            payload = {
+                "q": t,
+                "from": from_Language,
+                "to": to_Language,
+                "appid": appid,
+                "salt": salt,
+                "sign": signString,
+            }
+            # 发送请求
+            try:
+                response = requests.post(
+                    url=url, data=payload, headers=headers, timeout=3
+                )
+                response = response.json()
+                if "trans_result" in response.keys():
+                    result = response["trans_result"][0]
+                    if "dst" in result.keys():
+                        dst = result["dst"]
+                        outTexts.append(dst)
+            except Exception:
+                return Sentence
+        else:
+            outTexts.append(t)
+    return "\n".join(outTexts)
+
+#文本清洗工具
+def is_japanese(string):
+        for ch in string:
+            if ord(ch) > 0x3040 and ord(ch) < 0x30FF:
+                return True
+        return False
+
+def is_chinese(string):
+    for ch in string:
+        if '\u4e00' <= ch <= '\u9fff':
+            return True
+    return False
+
+def is_single_language(sentence):
+    # 检查句子是否为单一语言
+    contains_chinese = re.search(r'[\u4e00-\u9fff]', sentence) is not None
+    contains_japanese = re.search(r'[\u3040-\u30ff\u31f0-\u31ff]', sentence) is not None
+    contains_english = re.search(r'[a-zA-Z]', sentence) is not None
+    language_count = sum([contains_chinese, contains_japanese, contains_english])
+    return language_count == 1
+
+def merge_scattered_parts(sentences):
+    """合并零散的部分到相邻的句子中，并确保单一语言性"""
+    merged_sentences = []
+    buffer_sentence = ""
+
+    for sentence in sentences:
+        # 检查是否是单一语言或者太短（可能是标点或单个词）
+        if is_single_language(sentence) and len(sentence) > 1:
+            # 如果缓冲区有内容，先将缓冲区的内容添加到列表
+            if buffer_sentence:
+                merged_sentences.append(buffer_sentence)
+                buffer_sentence = ""
+            merged_sentences.append(sentence)
+        else:
+            # 如果是零散的部分，将其添加到缓冲区
+            buffer_sentence += sentence
+
+    # 确保最后的缓冲区内容被添加
+    if buffer_sentence:
+        merged_sentences.append(buffer_sentence)
+
+    return merged_sentences
+
+def is_only_punctuation(s):
+    """检查字符串是否只包含标点符号"""
+    # 此处列出中文、日文、英文常见标点符号
+    punctuation_pattern = re.compile(r'^[\s。*；，：“”（）、！？《》\u3000\.,;:"\'?!()]+$')
+    return punctuation_pattern.match(s) is not None
+
+def split_mixed_language(sentence):
+    # 分割混合语言句子
+    # 逐字符检查，分割不同语言部分
+    sub_sentences = []
+    current_language = None
+    current_part = ""
+
+    for char in sentence:
+        if re.match(r'[\u4e00-\u9fff]', char):  # Chinese character
+            if current_language != 'chinese':
+                if current_part:
+                    sub_sentences.append(current_part)
+                current_part = char
+                current_language = 'chinese'
+            else:
+                current_part += char
+        elif re.match(r'[\u3040-\u30ff\u31f0-\u31ff]', char):  # Japanese character
+            if current_language != 'japanese':
+                if current_part:
+                    sub_sentences.append(current_part)
+                current_part = char
+                current_language = 'japanese'
+            else:
+                current_part += char
+        elif re.match(r'[a-zA-Z]', char):  # English character
+            if current_language != 'english':
+                if current_part:
+                    sub_sentences.append(current_part)
+                current_part = char
+                current_language = 'english'
+            else:
+                current_part += char
+        else:
+            current_part += char  # For punctuation and other characters
+    
+    if current_part:
+        sub_sentences.append(current_part)
+
+    return sub_sentences
+
+def replace_quotes(text):
+    # 替换中文、日文引号为英文引号
+    text = re.sub(r'[“”‘’『』「」（）()]', '"', text)
+    return text
+
+def remove_numeric_annotations(text):
+    # 定义用于匹配数字注释的正则表达式
+    # 包括 “”、【】和〔〕包裹的数字
+    pattern = r'“\d+”|【\d+】|〔\d+〕'
+    # 使用正则表达式替换掉这些注释
+    cleaned_text = re.sub(pattern, '', text)
+    return cleaned_text
+
+def merge_adjacent_japanese(sentences):
+    """合并相邻且都只包含日语的句子"""
+    merged_sentences = []
+    i = 0
+    while i < len(sentences):
+        current_sentence = sentences[i]
+        if i + 1 < len(sentences) and is_japanese(current_sentence) and is_japanese(sentences[i + 1]):
+            # 当前句子和下一句都是日语，合并它们
+            while i + 1 < len(sentences) and is_japanese(sentences[i + 1]):
+                current_sentence += sentences[i + 1]
+                i += 1
+        merged_sentences.append(current_sentence)
+        i += 1
+    return merged_sentences
+
+def extrac(text):
+    text = replace_quotes(remove_numeric_annotations(text))  # 替换引号
+    text = re.sub("<[^>]*>", "", text)  # 移除 HTML 标签
+    # 使用换行符和标点符号进行初步分割
+    preliminary_sentences = re.split(r'([\n。；！？\.\?!])', text)
+    final_sentences = []
+
+    preliminary_sentences = re.split(r'([\n。；！？\.\?!])', text)
+
+    for piece in preliminary_sentences:
+        if is_single_language(piece):
+            final_sentences.append(piece)
+        else:
+            sub_sentences = split_mixed_language(piece)
+            final_sentences.extend(sub_sentences)
+    
+    # 处理长句子，使用jieba进行分词
+    split_sentences = []
+    for sentence in final_sentences:
+        split_sentences.extend(split_long_sentences(sentence))
+
+    # 合并相邻的日语句子
+    merged_japanese_sentences = merge_adjacent_japanese(split_sentences)
+
+    # 剔除只包含标点符号的元素
+    clean_sentences = [s for s in merged_japanese_sentences if not is_only_punctuation(s)]
+
+    # 移除空字符串并去除多余引号
+    return [s.replace('"','').strip() for s in clean_sentences if s]
+
+
+
+  # 移除空字符串
+
+def is_mixed_language(sentence):
+    contains_chinese = re.search(r'[\u4e00-\u9fff]', sentence) is not None
+    contains_japanese = re.search(r'[\u3040-\u30ff\u31f0-\u31ff]', sentence) is not None
+    contains_english = re.search(r'[a-zA-Z]', sentence) is not None
+    languages_count = sum([contains_chinese, contains_japanese, contains_english])
+    return languages_count > 1
+
+def split_mixed_language(sentence):
+    # 分割混合语言句子
+    sub_sentences = re.split(r'(?<=[。！？\.\?!])(?=")|(?<=")(?=[\u4e00-\u9fff\u3040-\u30ff\u31f0-\u31ff]|[a-zA-Z])', sentence)
+    return [s.strip() for s in sub_sentences if s.strip()]
+
+def seconds_to_ass_time(seconds):
+    """将秒数转换为ASS时间格式"""
+    hours = int(seconds / 3600)
+    minutes = int((seconds % 3600) / 60)
+    seconds = int(seconds) % 60
+    milliseconds = int((seconds - int(seconds)) * 1000)
+    return "{:01d}:{:02d}:{:02d}.{:02d}".format(hours, minutes, seconds, int(milliseconds / 10))
+
+def extract_text_from_epub(file_path):
+    book = epub.read_epub(file_path)
+    content = []
+    for item in book.items:
+        if isinstance(item, epub.EpubHtml):
+            soup = BeautifulSoup(item.content, 'html.parser')
+            content.append(soup.get_text())
+    return '\n'.join(content)
+
+def extract_text_from_pdf(file_path):
+    with open(file_path, 'rb') as file:
+        reader = PdfReader(file)
+        content = [page.extract_text() for page in reader.pages]
+    return '\n'.join(content)
+
+def remove_annotations(text):
+    # 移除方括号、尖括号和中文方括号中的内容
+    text = re.sub(r'\[.*?\]', '', text)
+    text = re.sub(r'\<.*?\>', '', text)
+    text = re.sub(r'&#8203;``【oaicite:1】``&#8203;', '', text)
+    return text
+
+def extract_text_from_file(inputFile):
+    file_extension = os.path.splitext(inputFile)[1].lower()
+    if file_extension == ".epub":
+        return extract_text_from_epub(inputFile)
+    elif file_extension == ".pdf":
+        return extract_text_from_pdf(inputFile)
+    elif file_extension == ".txt":
+        with open(inputFile, 'r', encoding='utf-8') as f:
+            return f.read()
+    else:
+        raise ValueError(f"Unsupported file format: {file_extension}")
+
+def split_by_punctuation(sentence):
+    """按照中文次级标点符号分割句子"""
+    # 常见的中文次级分隔符号：逗号、分号等
+    parts = re.split(r'([，,；;])', sentence)
+    # 将标点符号与前面的词语合并，避免单独标点符号成为一个部分
+    merged_parts = []
+    for part in parts:
+        if part and not part in '，,；;':
+            merged_parts.append(part)
+        elif merged_parts:
+            merged_parts[-1] += part
+    return merged_parts
+
+def split_long_sentences(sentence, max_length=30):
+    """如果中文句子太长，先按标点分割，必要时使用jieba进行分词并分割"""
+    if len(sentence) > max_length and is_chinese(sentence):
+        # 首先尝试按照次级标点符号分割
+        preliminary_parts = split_by_punctuation(sentence)
+        new_sentences = []
+
+        for part in preliminary_parts:
+            # 如果部分仍然太长，使用jieba进行分词
+            if len(part) > max_length:
+                words = jieba.lcut(part)
+                current_sentence = ""
+                for word in words:
+                    if len(current_sentence) + len(word) > max_length:
+                        new_sentences.append(current_sentence)
+                        current_sentence = word
+                    else:
+                        current_sentence += word
+                if current_sentence:
+                    new_sentences.append(current_sentence)
+            else:
+                new_sentences.append(part)
+
+        return new_sentences
+    return [sentence]  # 如果句子不长或不是中文，直接返回
+
+def extract_and_convert(text):
+
+    # 使用正则表达式找出所有英文单词
+    english_parts = re.findall(r'\b[A-Za-z]+\b', text)  # \b为单词边界标识
+    
+    # 对每个英文单词进行片假名转换
+    kana_parts = ['\n{}\n'.format(romajitable.to_kana(word).katakana) for word in english_parts]
+
+    # 替换原文本中的英文部分
+    for eng, kana in zip(english_parts, kana_parts):
+        text = text.replace(eng, kana, 1)  # 限制每次只替换一个实例
+    
+    return text
+# 推理工具
+def download_unidic():
+    try:
+        Tagger()
+        print("Tagger launch successfully.")
+    except Exception as e:
+        print("UNIDIC dictionary not found, downloading...")
+        subprocess.run([sys.executable, "-m", "unidic", "download"])
+        print("Download completed.")
+
+def kanji_to_hiragana(text):
+    global tagger
+    output = ""
+
+    # 更新正则表达式以更准确地区分文本和标点符号
+    segments = re.findall(r'[一-龥ぁ-んァ-ン\w]+|[^\一-龥ぁ-んァ-ン\w\s]', text, re.UNICODE)
+
+    for segment in segments:
+        if re.match(r'[一-龥ぁ-んァ-ン\w]+', segment):
+            # 如果是单词或汉字，转换为平假名
+            for word in tagger(segment):
+                kana = word.feature.kana or word.surface
+                hiragana = jaconv.kata2hira(kana)  # 将片假名转换为平假名
+                output += hiragana
+        else:
+            # 如果是标点符号，保持不变
+            output += segment
+
+    return output
 
 def get_net_g(model_path: str,  device: str, hps):
     net_g = SynthesizerTrn(
@@ -68,11 +456,11 @@ def get_net_g(model_path: str,  device: str, hps):
     _ = utils.load_checkpoint(model_path, net_g, None, skip_optimizer=True)
     return net_g
 
-
-def get_text(text, language_str, hps, device):
+def get_text(text, language_str, hps, device, style_text=None, style_weight=0.7):
+    style_text = None if style_text == "" else style_text
     norm_text, phone, tone, word2ph = clean_text(text, language_str)
     phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
-    #print(text)
+
     if hps.data.add_blank:
         phone = commons.intersperse(phone, 0)
         tone = commons.intersperse(tone, 0)
@@ -80,18 +468,24 @@ def get_text(text, language_str, hps, device):
         for i in range(len(word2ph)):
             word2ph[i] = word2ph[i] * 2
         word2ph[0] += 1
-    bert_ori = get_bert(norm_text, word2ph, language_str, device)
+    bert_ori = get_bert(
+        norm_text, word2ph, language_str, device, style_text, style_weight
+    )
     del word2ph
     assert bert_ori.shape[-1] == len(phone), phone
 
     if language_str == "ZH":
         bert = bert_ori
-        ja_bert = torch.zeros(1024, len(phone))
-        en_bert = torch.zeros(1024, len(phone))
+        ja_bert = torch.randn(1024, len(phone))
+        en_bert = torch.randn(1024, len(phone))
     elif language_str == "JP":
-        bert = torch.zeros(1024, len(phone))
+        bert = torch.randn(1024, len(phone))
         ja_bert = bert_ori
-        en_bert = torch.zeros(1024, len(phone))
+        en_bert = torch.randn(1024, len(phone))
+    elif language_str == "EN":
+        bert = torch.randn(1024, len(phone))
+        ja_bert = torch.randn(1024, len(phone))
+        en_bert = bert_ori
     else:
         raise ValueError("language_str should be ZH, JP or EN")
 
@@ -111,19 +505,47 @@ def infer(
     noise_scale_w,
     length_scale,
     sid,
-    reference_audio=None,
-    emotion='Happy',
+    style_text=None,
+    style_weight=0.7,
+    language = "Auto",
+    mode = 'pyopenjtalk-V2.3-Katakana',
+    skip_start=False,
+    skip_end=False,
 ):
-
-    language= 'JP' if is_japanese(text) else 'ZH'
-    if isinstance(reference_audio, np.ndarray):
-        emo = get_clap_audio_feature(reference_audio, device)
-    else:
-        emo = get_clap_text_feature(emotion, device)
-    emo = torch.squeeze(emo, dim=1)
+    if style_text == None:
+        style_text = ""
+        style_weight=0,
+    if mode == 'fugashi-V2.3-Katakana':
+        text = kanji_to_hiragana(text) if is_japanese(text) else text
+    if language == "JP":
+        text = translate(text,"jp")
+    if language == "ZH":
+        text = translate(text,"zh")
+    if language == "Auto":
+        language= 'JP' if is_japanese(text) else 'ZH'
+    #print(f'{text}:{sdp_ratio}:{noise_scale}:{noise_scale_w}:{length_scale}:{length_scale}:{sid}:{language}:{mode}:{skip_start}:{skip_end}')            
     bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
-        text, language, hps, device
+        text,
+        language,
+        hps,
+        device,
+        style_text=style_text,
+        style_weight=style_weight,
     )
+    if skip_start:
+        phones = phones[3:]
+        tones = tones[3:]
+        lang_ids = lang_ids[3:]
+        bert = bert[:, 3:]
+        ja_bert = ja_bert[:, 3:]
+        en_bert = en_bert[:, 3:]
+    if skip_end:
+        phones = phones[:-2]
+        tones = tones[:-2]
+        lang_ids = lang_ids[:-2]
+        bert = bert[:, :-2]
+        ja_bert = ja_bert[:, :-2]
+        en_bert = en_bert[:, :-2]
     with torch.no_grad():
         x_tst = phones.to(device).unsqueeze(0)
         tones = tones.to(device).unsqueeze(0)
@@ -132,7 +554,7 @@ def infer(
         ja_bert = ja_bert.to(device).unsqueeze(0)
         en_bert = en_bert.to(device).unsqueeze(0)
         x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
-        emo = emo.to(device).unsqueeze(0)
+        # emo = emo.to(device).unsqueeze(0)
         del phones
         speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
         audio = (
@@ -145,7 +567,6 @@ def infer(
                 bert,
                 ja_bert,
                 en_bert,
-                emo,
                 sdp_ratio=sdp_ratio,
                 noise_scale=noise_scale,
                 noise_scale_w=noise_scale_w,
@@ -155,79 +576,292 @@ def infer(
             .float()
             .numpy()
         )
-        del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers, ja_bert, en_bert, emo
+        del (
+            x_tst,
+            tones,
+            lang_ids,
+            bert,
+            x_tst_lengths,
+            speakers,
+            ja_bert,
+            en_bert,
+        )  # , emo
         if torch.cuda.is_available():
             torch.cuda.empty_cache()
-        unique_filename = f"temp{uuid.uuid4()}.wav"
-        write(unique_filename, 44100, audio)
-        return unique_filename
-
-def is_japanese(string):
-        for ch in string:
-            if ord(ch) > 0x3040 and ord(ch) < 0x30FF:
-                return True
-        return False
+    print("Success.")
+    return audio
 
 def loadmodel(model):
-    try:
-        _ = net_g.eval()
-        _ = utils.load_checkpoint(model, net_g, None, skip_optimizer=True)
-        return "success"
-    except:
-        return "error"
-
-def send_audio_to_server(audio_path,text):
-    url="http://127.0.0.1:3000/response"
-    files = {'file': open(audio_path, 'rb')}
-    data = {'text': text}
-    try:
-        response = requests.post(url, files=files,data=data)
-        return response.status_code, response.text
-    except Exception as e:
-        return 500, str(e)
+    _ = net_g.eval()
+    _ = utils.load_checkpoint(model, net_g, None, skip_optimizer=True)
+    return "success"
+
+def generate_audio_and_srt_for_group(
+    group,
+    outputPath,
+    group_index,
+    sampling_rate,
+    speaker,
+    sdp_ratio,
+    noise_scale,
+    noise_scale_w,
+    length_scale,
+    speakerList,
+    silenceTime,
+    language,
+    mode,
+    skip_start,
+    skip_end,
+    style_text,
+    style_weight,
+    ):
+    audio_fin = []
+    ass_entries = []
+    start_time = 0
+    #speaker = random.choice(cara_list)
+    ass_header = """[Script Info]
+        ; 我没意见
+        Title: Audiobook
+        ScriptType: v4.00+
+        WrapStyle: 0
+        PlayResX: 640
+        PlayResY: 360
+        ScaledBorderAndShadow: yes
+        [V4+ Styles]
+        Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
+        Style: Default,Arial,20,&H00FFFFFF,&H000000FF,&H00000000,&H00000000,0,0,0,0,100,100,0,0,1,1,1,2,10,10,10,1
+        [Events]
+        Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
+        """
+
+    for sentence in group:
+
+            if len(sentence) > 1:
+                FakeSpeaker = sentence.split("|")[0]
+                print(FakeSpeaker)
+                SpeakersList = re.split('\n', speakerList)
+                if FakeSpeaker in list(hps.data.spk2id.keys()):
+                    speaker = FakeSpeaker
+                for i in SpeakersList:
+                    if FakeSpeaker == i.split("|")[1]:
+                        speaker = i.split("|")[0]
+                if sentence != '\n':
+                    text = (remove_annotations(sentence.split("|")[-1]).replace(" ","")+"。").replace("，。","。")
+                    if mode == 'pyopenjtalk-V2.3-Katakana' or mode == 'fugashi-V2.3-Katakana':
+                        #print(f'{text}:{sdp_ratio}:{noise_scale}:{noise_scale_w}:{length_scale}:{length_scale}:{speaker}:{language}:{mode}:{skip_start}:{skip_end}')
+                        audio = infer(
+                            text,
+                            sdp_ratio,
+                            noise_scale,
+                            noise_scale_w,
+                            length_scale,
+                            speaker,
+                            style_text,
+                            style_weight,
+                            language,
+                            mode,
+                            skip_start,
+                            skip_end,
+                        )
+                    silence_frames = int(silenceTime * 44010) if is_chinese(sentence) else int(silenceTime * 44010)
+                    silence_data = np.zeros((silence_frames,), dtype=audio.dtype)
+                    audio_fin.append(audio)
+                    audio_fin.append(silence_data)
+                    duration = len(audio) / sampling_rate
+                    print(duration)
+                    end_time = start_time + duration + silenceTime
+                    ass_entries.append("Dialogue: 0,{},{},".format(seconds_to_ass_time(start_time), seconds_to_ass_time(end_time)) + "Default,,0,0,0,,{}".format(sentence.replace("|","：")))
+                    start_time = end_time
+
+    wav_filename = os.path.join(outputPath, f'audiobook_part_{group_index}.wav')
+    ass_filename = os.path.join(outputPath, f'audiobook_part_{group_index}.ass')
+    write(wav_filename, sampling_rate, gr.processing_utils.convert_to_16_bit_wav(np.concatenate(audio_fin)))
 
-app = Flask(__name__)
-CORS(app)
-@app.route('/')
+    with open(ass_filename, 'w', encoding='utf-8') as f:
+        f.write(ass_header + '\n'.join(ass_entries))
+    return (hps.data.sampling_rate, gr.processing_utils.convert_to_16_bit_wav(np.concatenate(audio_fin)))
+
+def generate_audio(
+    inputFile,
+    groupsize,
+    filepath,
+    silenceTime,
+    speakerList,
+    text,
+    sdp_ratio,
+    noise_scale,
+    noise_scale_w,
+    length_scale,
+    sid,
+    style_text=None,
+    style_weight=0.7,
+    language = "Auto",
+    mode = 'pyopenjtalk-V2.3-Katakana',
+    sentence_mode = 'sentence',
+    skip_start=False,
+    skip_end=False,
+):
+    if mode == 'pyopenjtalk-V2.3-Katakana' or mode == 'fugashi-V2.3-Katakana':
+        if sentence_mode == 'sentence':
+            audio = infer(
+                        text,
+                        sdp_ratio,
+                        noise_scale,
+                        noise_scale_w,
+                        length_scale,
+                        sid,
+                        style_text,
+                        style_weight,
+                        language,
+                        mode,
+                        skip_start,
+                        skip_end,
+                    )
+            return (hps.data.sampling_rate,gr.processing_utils.convert_to_16_bit_wav(audio))
+        if sentence_mode == 'paragraph':
+            GROUP_SIZE = groupsize
+            directory_path = filepath if torch.cuda.is_available() else "books"
+            if os.path.exists(directory_path):
+                shutil.rmtree(directory_path)
+            os.makedirs(directory_path)
+            if inputFile:
+                text = extract_text_from_file(inputFile.name)
+            if language == 'Auto':
+                sentences = extrac(extract_and_convert(text))
+            else:
+                sentences = extrac(text)
+            for i in range(0, len(sentences), GROUP_SIZE):
+                group = sentences[i:i+GROUP_SIZE]
+                if speakerList == "":
+                    speakerList = "无"
+                result = generate_audio_and_srt_for_group(
+                    group,
+                    directory_path,
+                    i//GROUP_SIZE + 1,
+                    44100,
+                    sid,
+                    sdp_ratio,
+                    noise_scale,
+                    noise_scale_w,
+                    length_scale,
+                    speakerList,
+                    silenceTime,
+                    language,
+                    mode,
+                    skip_start,
+                    skip_end,
+                    style_text,
+                    style_weight,       
+                    )
+                if not torch.cuda.is_available():
+                    return result
+            return result
+
+Flaskapp = Flask(__name__)
+CORS(Flaskapp)
+@Flaskapp.route('/', methods=['GET', 'POST'])
 
 def tts():
-    global last_text, last_model
-    speaker = request.args.get('speaker')
-    sdp_ratio = float(request.args.get('sdp_ratio', 0.2))
-    noise_scale = float(request.args.get('noise_scale', 0.6))
-    noise_scale_w = float(request.args.get('noise_scale_w', 0.8))
-    length_scale = float(request.args.get('length_scale', 1))
-    emotion = request.args.get('emotion', 'happy')
-    text = request.args.get('text')
-    is_chat = request.args.get('is_chat', 'false').lower() == 'true'
-    model = request.args.get('model',modelPaths[-1])
-    
-    if not speaker or not text:
-        return render_template_string("""
-            <!DOCTYPE html>
-            <html>
-            <head>
-                <title>TTS API Documentation</title>
-            </head>
-            <body>
-                <iframe src="http://love.soyorin.top" style="width:100%; height:100vh; border:none;"></iframe>
-            </body>
-            </html>
-        """)
-    
-    if model != last_model:
-        unique_filename  = loadmodel(model)
-        last_model = model
-    if is_chat and text == last_text:
-        # Generate 1 second of silence and return
-        unique_filename = 'blank.wav'
-        silence = np.zeros(44100, dtype=np.int16)
-        write(unique_filename , 44100, silence)
-    else:
-        last_text = text
-        unique_filename  = infer(text, sdp_ratio=sdp_ratio, noise_scale=noise_scale, noise_scale_w=noise_scale_w, length_scale=length_scale,sid = speaker, reference_audio=None, emotion=emotion)
-        status_code, response_text = send_audio_to_server(unique_filename,text)
-        print(f"Response from server: {response_text} (Status code: {status_code})")
+    if request.method == 'POST':
+        input = request.json 
+        inputFile = None
+        filepath = input['filepath']
+        groupSize = input['groupSize']
+        text = input['text']
+        sdp_ratio = input['sdp_ratio']
+        noise_scale = input['noise_scale']
+        noise_scale_w = input['noise_scale_w']
+        length_scale = input['length_scale']
+        sid = input['speaker']
+        style_text = input['style_text']
+        style_weight = input['style_weight']
+        language = input['language']
+        mode = input['mode']
+        sentence_mode = input['sentence_mode']
+        skip_start = input['skip_start']
+        skip_end = input['skip_end']
+        speakerList = input['speakerList']
+        silenceTime = input['silenceTime']
+        samplerate, audio = generate_audio(
+                inputFile,
+                groupSize,
+                filepath,
+                silenceTime,
+                speakerList,
+                text,
+                sdp_ratio,
+                noise_scale,
+                noise_scale_w,
+                length_scale,
+                sid,
+                style_text,
+                style_weight,
+                language,
+                mode,
+                sentence_mode,
+                skip_start,
+                skip_end,
+            )
+        unique_filename = f"temp{uuid.uuid4()}.wav"
+        write(unique_filename, samplerate, audio)
+        with open(unique_filename ,'rb') as bit:
+            wav_bytes = bit.read()
+        os.remove(unique_filename)
+        headers = {
+                'Content-Type': 'audio/wav',
+                'Text': unique_filename .encode('utf-8')}
+        return wav_bytes, 200, headers
+    groupSize = request.args.get('groupSize', default = 50, type = int)
+    text = request.args.get('text', default = '', type = str)
+    sdp_ratio = request.args.get('sdp_ratio', default = 0.5, type = float)
+    noise_scale = request.args.get('noise_scale', default = 0.6, type = float)
+    noise_scale_w = request.args.get('noise_scale_w', default = 0.667, type = float)
+    length_scale = request.args.get('length_scale', default = 1, type = float)
+    sid = request.args.get('speaker', default = '八千代', type = str)
+    style_text = request.args.get('style_text', default = '', type = str)
+    style_weight = request.args.get('style_weight', default = 0.7, type = float)
+    language = request.args.get('language', default = 'Auto', type = str)
+    mode = request.args.get('mode', default = 'pyopenjtalk-V2.3-Katakana', type = str)
+    sentence_mode = request.args.get('sentence_mode', default = 'sentence', type = str)
+    skip_start = request.args.get('skip_start', default = False, type = bool)
+    skip_end = request.args.get('skip_end', default = False, type = bool)
+    speakerList = request.args.get('speakerList', default = '', type = str)
+    silenceTime = request.args.get('silenceTime', default = 0.1, type = float)
+    inputFile = None
+    if not sid or not text:
+        return render_template_string(f"""
+                            <!DOCTYPE html>
+                            <html>
+                            <head>
+                                <title>TTS API Documentation</title>
+                            </head>
+                            <body>
+                                <iframe src={webBase} style="width:100%; height:100vh; border:none;"></iframe>
+                            </body>
+                            </html>
+                        """)
+    samplerate, audio = generate_audio(
+                inputFile,
+                groupSize,
+                None,
+                silenceTime,
+                speakerList,
+                text,
+                sdp_ratio,
+                noise_scale,
+                noise_scale_w,
+                length_scale,
+                sid,
+                style_text,
+                style_weight,
+                language,
+                mode,
+                sentence_mode,
+                skip_start,
+                skip_end,
+            )
+    unique_filename = f"temp{uuid.uuid4()}.wav"
+    write(unique_filename, samplerate, audio)
     with open(unique_filename ,'rb') as bit:
         wav_bytes = bit.read()
     os.remove(unique_filename)
@@ -238,17 +872,13 @@ def tts():
 
 
 if __name__ == "__main__":
-    languages = [ "Auto", "ZH", "JP"]
-    modelPaths = []
-    for dirpath, dirnames, filenames in os.walk("Data/BangDreamV22/models/"):
-        for filename in filenames:
-            modelPaths.append(os.path.join(dirpath, filename))
-    hps = utils.get_hparams_from_file('Data/BangDreamV22/configs/config.json')
+    download_unidic()
+    tagger = Tagger()
     net_g = get_net_g(
         model_path=modelPaths[-1], device=device, hps=hps
     )
     speaker_ids = hps.data.spk2id
     speakers = list(speaker_ids.keys())
-    last_text = ""
-    last_model = modelPaths[-1]
-    app.run(host="0.0.0.0", port=5000)
+    
+    print("推理页面已开启!")
+    Flaskapp.run(host="0.0.0.0", port=8080,debug=True)

test.py

@@ -0,0 +1,36 @@
+import re
+from fugashi import Tagger
+import jaconv
+
+def kanji_to_hiragana(text):
+    tagger = Tagger()
+    output = ""
+
+    # 更新正则表达式以更准确地区分文本和标点符号
+    segments = re.findall(r'[一-龥ぁ-んァ-ン\w]+|[^\一-龥ぁ-んァ-ン\w\s]', text, re.UNICODE)
+
+    for segment in segments:
+        if re.match(r'[一-龥ぁ-んァ-ン\w]+', segment):
+            # 如果是单词或汉字，转换为平假名
+            for word in tagger(segment):
+                kana = word.feature.kana or word.surface
+                hiragana = jaconv.kata2hira(kana)  # 将片假名转换为平假名
+                output += hiragana
+        else:
+            # 如果是标点符号，保持不变
+            output += segment
+
+    return output
+
+
+text = "私は学生です。"
+tagger = Tagger()
+
+for word in tagger(text):
+    print(word.surface, word.feature.pos1)  
+
+
+# 示例文本
+text = "業火とはね、どんな人でも彼女が築いた悪業は、いつの日か、彼女を少しも残さず焼き払うことになる……"
+converted_text = kanji_to_hiragana(text)
+print(converted_text)

train_ms.py

@@ -13,7 +13,6 @@ import logging
 from config import config
 import argparse
 import datetime
-import gc
 
 logging.getLogger("numba").setLevel(logging.WARNING)
 import commons
@@ -27,14 +26,21 @@ from models import (
     SynthesizerTrn,
     MultiPeriodDiscriminator,
     DurationDiscriminator,
+    WavLMDiscriminator,
+)
+from losses import (
+    generator_loss,
+    discriminator_loss,
+    feature_loss,
+    kl_loss,
+    WavLMLoss,
 )
-from losses import generator_loss, discriminator_loss, feature_loss, kl_loss
 from mel_processing import mel_spectrogram_torch, spec_to_mel_torch
 from text.symbols import symbols
 
 torch.backends.cuda.matmul.allow_tf32 = True
 torch.backends.cudnn.allow_tf32 = (
-    True  # If encontered training problem,please try to disable TF32.
+    True  # If encountered training problem,please try to disable TF32.
 )
 torch.set_float32_matmul_precision("medium")
 torch.backends.cuda.sdp_kernel("flash")
@@ -42,7 +48,6 @@ torch.backends.cuda.enable_flash_sdp(True)
 torch.backends.cuda.enable_mem_efficient_sdp(
     True
 )  # Not available if torch version is lower than 2.0
-torch.backends.cuda.enable_math_sdp(True)
 global_step = 0
 
 
@@ -97,7 +102,7 @@ def run():
     args = parser.parse_args()
     model_dir = os.path.join(args.model, config.train_ms_config.model)
     if not os.path.exists(model_dir):
-        os.makedirs(model_dir)
+        os.makedirs(model_dir, exist_ok=True)
     hps = utils.get_hparams_from_file(args.config)
     hps.model_dir = model_dir
     # 比较路径是否相同
@@ -173,6 +178,8 @@ def run():
             0.1,
             gin_channels=hps.model.gin_channels if hps.data.n_speakers != 0 else 0,
         ).cuda(local_rank)
+    else:
+        net_dur_disc = None
     if (
         "use_spk_conditioned_encoder" in hps.model.keys()
         and hps.model.use_spk_conditioned_encoder is True
@@ -210,6 +217,9 @@ def run():
             param.requires_grad = False
 
     net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm).cuda(local_rank)
+    net_wd = WavLMDiscriminator(
+        hps.model.slm.hidden, hps.model.slm.nlayers, hps.model.slm.initial_channel
+    ).cuda(local_rank)
     optim_g = torch.optim.AdamW(
         filter(lambda p: p.requires_grad, net_g.parameters()),
         hps.train.learning_rate,
@@ -222,6 +232,12 @@ def run():
         betas=hps.train.betas,
         eps=hps.train.eps,
     )
+    optim_wd = torch.optim.AdamW(
+        net_wd.parameters(),
+        hps.train.learning_rate,
+        betas=hps.train.betas,
+        eps=hps.train.eps,
+    )
     if net_dur_disc is not None:
         optim_dur_disc = torch.optim.AdamW(
             net_dur_disc.parameters(),
@@ -233,12 +249,11 @@ def run():
         optim_dur_disc = None
     net_g = DDP(net_g, device_ids=[local_rank], bucket_cap_mb=512)
     net_d = DDP(net_d, device_ids=[local_rank], bucket_cap_mb=512)
-    dur_resume_lr = None
+    net_wd = DDP(net_wd, device_ids=[local_rank], bucket_cap_mb=512)
     if net_dur_disc is not None:
         net_dur_disc = DDP(
             net_dur_disc,
             device_ids=[local_rank],
-            find_unused_parameters=True,
             bucket_cap_mb=512,
         )
 
@@ -250,9 +265,10 @@ def run():
             token=config.openi_token,
             mirror=config.mirror,
         )
-
-    try:
-        if net_dur_disc is not None:
+    dur_resume_lr = hps.train.learning_rate
+    wd_resume_lr = hps.train.learning_rate
+    if net_dur_disc is not None:
+        try:
             _, _, dur_resume_lr, epoch_str = utils.load_checkpoint(
                 utils.latest_checkpoint_path(hps.model_dir, "DUR_*.pth"),
                 net_dur_disc,
@@ -261,28 +277,32 @@ def run():
                 if "skip_optimizer" in hps.train
                 else True,
             )
-            _, optim_g, g_resume_lr, epoch_str = utils.load_checkpoint(
-                utils.latest_checkpoint_path(hps.model_dir, "G_*.pth"),
-                net_g,
-                optim_g,
-                skip_optimizer=hps.train.skip_optimizer
-                if "skip_optimizer" in hps.train
-                else True,
-            )
-            _, optim_d, d_resume_lr, epoch_str = utils.load_checkpoint(
-                utils.latest_checkpoint_path(hps.model_dir, "D_*.pth"),
-                net_d,
-                optim_d,
-                skip_optimizer=hps.train.skip_optimizer
-                if "skip_optimizer" in hps.train
-                else True,
-            )
-            if not optim_g.param_groups[0].get("initial_lr"):
-                optim_g.param_groups[0]["initial_lr"] = g_resume_lr
-            if not optim_d.param_groups[0].get("initial_lr"):
-                optim_d.param_groups[0]["initial_lr"] = d_resume_lr
             if not optim_dur_disc.param_groups[0].get("initial_lr"):
                 optim_dur_disc.param_groups[0]["initial_lr"] = dur_resume_lr
+        except:
+            print("Initialize dur_disc")
+
+    try:
+        _, optim_g, g_resume_lr, epoch_str = utils.load_checkpoint(
+            utils.latest_checkpoint_path(hps.model_dir, "G_*.pth"),
+            net_g,
+            optim_g,
+            skip_optimizer=hps.train.skip_optimizer
+            if "skip_optimizer" in hps.train
+            else True,
+        )
+        _, optim_d, d_resume_lr, epoch_str = utils.load_checkpoint(
+            utils.latest_checkpoint_path(hps.model_dir, "D_*.pth"),
+            net_d,
+            optim_d,
+            skip_optimizer=hps.train.skip_optimizer
+            if "skip_optimizer" in hps.train
+            else True,
+        )
+        if not optim_g.param_groups[0].get("initial_lr"):
+            optim_g.param_groups[0]["initial_lr"] = g_resume_lr
+        if not optim_d.param_groups[0].get("initial_lr"):
+            optim_d.param_groups[0]["initial_lr"] = d_resume_lr
 
         epoch_str = max(epoch_str, 1)
         # global_step = (epoch_str - 1) * len(train_loader)
@@ -297,21 +317,43 @@ def run():
         epoch_str = 1
         global_step = 0
 
+    try:
+        _, optim_wd, wd_resume_lr, epoch_str = utils.load_checkpoint(
+            utils.latest_checkpoint_path(hps.model_dir, "WD_*.pth"),
+            net_wd,
+            optim_wd,
+            skip_optimizer=hps.train.skip_optimizer
+            if "skip_optimizer" in hps.train
+            else True,
+        )
+        if not optim_wd.param_groups[0].get("initial_lr"):
+            optim_wd.param_groups[0]["initial_lr"] = wd_resume_lr
+    except Exception as e:
+        print(e)
+
     scheduler_g = torch.optim.lr_scheduler.ExponentialLR(
         optim_g, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2
     )
     scheduler_d = torch.optim.lr_scheduler.ExponentialLR(
         optim_d, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2
     )
+    scheduler_wd = torch.optim.lr_scheduler.ExponentialLR(
+        optim_wd, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2
+    )
     if net_dur_disc is not None:
-        if not optim_dur_disc.param_groups[0].get("initial_lr"):
-            optim_dur_disc.param_groups[0]["initial_lr"] = dur_resume_lr
         scheduler_dur_disc = torch.optim.lr_scheduler.ExponentialLR(
             optim_dur_disc, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2
         )
     else:
         scheduler_dur_disc = None
-    scaler = GradScaler(enabled=hps.train.fp16_run)
+    scaler = GradScaler(enabled=hps.train.bf16_run)
+
+    wl = WavLMLoss(
+        hps.model.slm.model,
+        net_wd,
+        hps.data.sampling_rate,
+        hps.model.slm.sr,
+    ).to(local_rank)
 
     for epoch in range(epoch_str, hps.train.epochs + 1):
         if rank == 0:
@@ -320,9 +362,9 @@ def run():
                 local_rank,
                 epoch,
                 hps,
-                [net_g, net_d, net_dur_disc],
-                [optim_g, optim_d, optim_dur_disc],
-                [scheduler_g, scheduler_d, scheduler_dur_disc],
+                [net_g, net_d, net_dur_disc, net_wd, wl],
+                [optim_g, optim_d, optim_dur_disc, optim_wd],
+                [scheduler_g, scheduler_d, scheduler_dur_disc, scheduler_wd],
                 scaler,
                 [train_loader, eval_loader],
                 logger,
@@ -334,9 +376,9 @@ def run():
                 local_rank,
                 epoch,
                 hps,
-                [net_g, net_d, net_dur_disc],
-                [optim_g, optim_d, optim_dur_disc],
-                [scheduler_g, scheduler_d, scheduler_dur_disc],
+                [net_g, net_d, net_dur_disc, net_wd, wl],
+                [optim_g, optim_d, optim_dur_disc, optim_wd],
+                [scheduler_g, scheduler_d, scheduler_dur_disc, scheduler_wd],
                 scaler,
                 [train_loader, None],
                 None,
@@ -344,6 +386,7 @@ def run():
             )
         scheduler_g.step()
         scheduler_d.step()
+        scheduler_wd.step()
         if net_dur_disc is not None:
             scheduler_dur_disc.step()
 
@@ -361,9 +404,9 @@ def train_and_evaluate(
     logger,
     writers,
 ):
-    net_g, net_d, net_dur_disc = nets
-    optim_g, optim_d, optim_dur_disc = optims
-    scheduler_g, scheduler_d, scheduler_dur_disc = schedulers
+    net_g, net_d, net_dur_disc, net_wd, wl = nets
+    optim_g, optim_d, optim_dur_disc, optim_wd = optims
+    scheduler_g, scheduler_d, scheduler_dur_disc, scheduler_wd = schedulers
     train_loader, eval_loader = loaders
     if writers is not None:
         writer, writer_eval = writers
@@ -373,6 +416,7 @@ def train_and_evaluate(
 
     net_g.train()
     net_d.train()
+    net_wd.train()
     if net_dur_disc is not None:
         net_dur_disc.train()
     for batch_idx, (
@@ -388,7 +432,6 @@ def train_and_evaluate(
         bert,
         ja_bert,
         en_bert,
-        emo,
     ) in enumerate(tqdm(train_loader)):
         if net_g.module.use_noise_scaled_mas:
             current_mas_noise_scale = (
@@ -411,9 +454,8 @@ def train_and_evaluate(
         bert = bert.cuda(local_rank, non_blocking=True)
         ja_bert = ja_bert.cuda(local_rank, non_blocking=True)
         en_bert = en_bert.cuda(local_rank, non_blocking=True)
-        emo = emo.cuda(local_rank, non_blocking=True)
 
-        with autocast(enabled=hps.train.fp16_run):
+        with autocast(enabled=hps.train.bf16_run, dtype=torch.bfloat16):
             (
                 y_hat,
                 l_length,
@@ -422,9 +464,8 @@ def train_and_evaluate(
                 x_mask,
                 z_mask,
                 (z, z_p, m_p, logs_p, m_q, logs_q),
-                (hidden_x, logw, logw_),
+                (hidden_x, logw, logw_, logw_sdp),
                 g,
-                loss_commit,
             ) = net_g(
                 x,
                 x_lengths,
@@ -436,7 +477,6 @@ def train_and_evaluate(
                 bert,
                 ja_bert,
                 en_bert,
-                emo,
             )
             mel = spec_to_mel_torch(
                 spec,
@@ -450,7 +490,7 @@ def train_and_evaluate(
                 mel, ids_slice, hps.train.segment_size // hps.data.hop_length
             )
             y_hat_mel = mel_spectrogram_torch(
-                y_hat.squeeze(1),
+                y_hat.squeeze(1).float(),
                 hps.data.filter_length,
                 hps.data.n_mel_channels,
                 hps.data.sampling_rate,
@@ -466,7 +506,7 @@ def train_and_evaluate(
 
             # Discriminator
             y_d_hat_r, y_d_hat_g, _, _ = net_d(y, y_hat.detach())
-            with autocast(enabled=False):
+            with autocast(enabled=hps.train.bf16_run, dtype=torch.bfloat16):
                 loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(
                     y_d_hat_r, y_d_hat_g
                 )
@@ -475,11 +515,20 @@ def train_and_evaluate(
                 y_dur_hat_r, y_dur_hat_g = net_dur_disc(
                     hidden_x.detach(),
                     x_mask.detach(),
+                    logw_.detach(),
                     logw.detach(),
+                    g.detach(),
+                )
+                y_dur_hat_r_sdp, y_dur_hat_g_sdp = net_dur_disc(
+                    hidden_x.detach(),
+                    x_mask.detach(),
                     logw_.detach(),
+                    logw_sdp.detach(),
                     g.detach(),
                 )
-                with autocast(enabled=False):
+                y_dur_hat_r = y_dur_hat_r + y_dur_hat_r_sdp
+                y_dur_hat_g = y_dur_hat_g + y_dur_hat_g_sdp
+                with autocast(enabled=hps.train.bf16_run, dtype=torch.bfloat16):
                     # TODO: I think need to mean using the mask, but for now, just mean all
                     (
                         loss_dur_disc,
@@ -490,31 +539,60 @@ def train_and_evaluate(
                 optim_dur_disc.zero_grad()
                 scaler.scale(loss_dur_disc_all).backward()
                 scaler.unscale_(optim_dur_disc)
-                commons.clip_grad_value_(net_dur_disc.parameters(), None)
+                # torch.nn.utils.clip_grad_norm_(
+                #     parameters=net_dur_disc.parameters(), max_norm=100
+                # )
+                grad_norm_dur = commons.clip_grad_value_(
+                    net_dur_disc.parameters(), None
+                )
                 scaler.step(optim_dur_disc)
 
         optim_d.zero_grad()
         scaler.scale(loss_disc_all).backward()
         scaler.unscale_(optim_d)
+        if getattr(hps.train, "bf16_run", False):
+            torch.nn.utils.clip_grad_norm_(parameters=net_d.parameters(), max_norm=200)
         grad_norm_d = commons.clip_grad_value_(net_d.parameters(), None)
         scaler.step(optim_d)
 
-        with autocast(enabled=hps.train.fp16_run):
+        with autocast(enabled=hps.train.bf16_run, dtype=torch.bfloat16):
+            loss_slm = wl.discriminator(
+                y.detach().squeeze(), y_hat.detach().squeeze()
+            ).mean()
+
+        optim_wd.zero_grad()
+        scaler.scale(loss_slm).backward()
+        scaler.unscale_(optim_wd)
+        # torch.nn.utils.clip_grad_norm_(parameters=net_wd.parameters(), max_norm=200)
+        grad_norm_wd = commons.clip_grad_value_(net_wd.parameters(), None)
+        scaler.step(optim_wd)
+
+        with autocast(enabled=hps.train.bf16_run, dtype=torch.bfloat16):
             # Generator
             y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = net_d(y, y_hat)
             if net_dur_disc is not None:
-                y_dur_hat_r, y_dur_hat_g = net_dur_disc(
-                    hidden_x, x_mask, logw, logw_, g
-                )
-            with autocast(enabled=False):
+                _, y_dur_hat_g = net_dur_disc(hidden_x, x_mask, logw_, logw, g)
+                _, y_dur_hat_g_sdp = net_dur_disc(hidden_x, x_mask, logw_, logw_sdp, g)
+                y_dur_hat_g = y_dur_hat_g + y_dur_hat_g_sdp
+            with autocast(enabled=hps.train.bf16_run, dtype=torch.bfloat16):
                 loss_dur = torch.sum(l_length.float())
                 loss_mel = F.l1_loss(y_mel, y_hat_mel) * hps.train.c_mel
                 loss_kl = kl_loss(z_p, logs_q, m_p, logs_p, z_mask) * hps.train.c_kl
 
                 loss_fm = feature_loss(fmap_r, fmap_g)
                 loss_gen, losses_gen = generator_loss(y_d_hat_g)
+
+                loss_lm = wl(y.detach().squeeze(), y_hat.squeeze()).mean()
+                loss_lm_gen = wl.generator(y_hat.squeeze())
+
                 loss_gen_all = (
-                    loss_gen + loss_fm + loss_mel + loss_dur + loss_kl + loss_commit
+                    loss_gen
+                    + loss_fm
+                    + loss_mel
+                    + loss_dur
+                    + loss_kl
+                    + loss_lm
+                    + loss_lm_gen
                 )
                 if net_dur_disc is not None:
                     loss_dur_gen, losses_dur_gen = generator_loss(y_dur_hat_g)
@@ -522,6 +600,8 @@ def train_and_evaluate(
         optim_g.zero_grad()
         scaler.scale(loss_gen_all).backward()
         scaler.unscale_(optim_g)
+        if getattr(hps.train, "bf16_run", False):
+            torch.nn.utils.clip_grad_norm_(parameters=net_g.parameters(), max_norm=500)
         grad_norm_g = commons.clip_grad_value_(net_g.parameters(), None)
         scaler.step(optim_g)
         scaler.update()
@@ -540,9 +620,12 @@ def train_and_evaluate(
                 scalar_dict = {
                     "loss/g/total": loss_gen_all,
                     "loss/d/total": loss_disc_all,
+                    "loss/wd/total": loss_slm,
                     "learning_rate": lr,
                     "grad_norm_d": grad_norm_d,
                     "grad_norm_g": grad_norm_g,
+                    "grad_norm_dur": grad_norm_dur,
+                    "grad_norm_wd": grad_norm_wd,
                 }
                 scalar_dict.update(
                     {
@@ -550,6 +633,8 @@ def train_and_evaluate(
                         "loss/g/mel": loss_mel,
                         "loss/g/dur": loss_dur,
                         "loss/g/kl": loss_kl,
+                        "loss/g/lm": loss_lm,
+                        "loss/g/lm_gen": loss_lm_gen,
                     }
                 )
                 scalar_dict.update(
@@ -562,6 +647,30 @@ def train_and_evaluate(
                     {"loss/d_g/{}".format(i): v for i, v in enumerate(losses_disc_g)}
                 )
 
+                if net_dur_disc is not None:
+                    scalar_dict.update({"loss/dur_disc/total": loss_dur_disc_all})
+
+                    scalar_dict.update(
+                        {
+                            "loss/dur_disc_g/{}".format(i): v
+                            for i, v in enumerate(losses_dur_disc_g)
+                        }
+                    )
+                    scalar_dict.update(
+                        {
+                            "loss/dur_disc_r/{}".format(i): v
+                            for i, v in enumerate(losses_dur_disc_r)
+                        }
+                    )
+
+                    scalar_dict.update({"loss/g/dur_gen": loss_dur_gen})
+                    scalar_dict.update(
+                        {
+                            "loss/g/dur_gen_{}".format(i): v
+                            for i, v in enumerate(losses_dur_gen)
+                        }
+                    )
+
                 image_dict = {
                     "slice/mel_org": utils.plot_spectrogram_to_numpy(
                         y_mel[0].data.cpu().numpy()
@@ -599,6 +708,13 @@ def train_and_evaluate(
                     epoch,
                     os.path.join(hps.model_dir, "D_{}.pth".format(global_step)),
                 )
+                utils.save_checkpoint(
+                    net_wd,
+                    optim_wd,
+                    hps.train.learning_rate,
+                    epoch,
+                    os.path.join(hps.model_dir, "WD_{}.pth".format(global_step)),
+                )
                 if net_dur_disc is not None:
                     utils.save_checkpoint(
                         net_dur_disc,
@@ -617,8 +733,8 @@ def train_and_evaluate(
 
         global_step += 1
 
-    gc.collect()
-    torch.cuda.empty_cache()
+    # gc.collect()
+    # torch.cuda.empty_cache()
     if rank == 0:
         logger.info("====> Epoch: {}".format(epoch))
 
@@ -642,7 +758,6 @@ def evaluate(hps, generator, eval_loader, writer_eval):
             bert,
             ja_bert,
             en_bert,
-            emo,
         ) in enumerate(eval_loader):
             x, x_lengths = x.cuda(), x_lengths.cuda()
             spec, spec_lengths = spec.cuda(), spec_lengths.cuda()
@@ -653,7 +768,6 @@ def evaluate(hps, generator, eval_loader, writer_eval):
             en_bert = en_bert.cuda()
             tone = tone.cuda()
             language = language.cuda()
-            emo = emo.cuda()
             for use_sdp in [True, False]:
                 y_hat, attn, mask, *_ = generator.module.infer(
                     x,
@@ -664,7 +778,6 @@ def evaluate(hps, generator, eval_loader, writer_eval):
                     bert,
                     ja_bert,
                     en_bert,
-                    emo,
                     y=spec,
                     max_len=1000,
                     sdp_ratio=0.0 if not use_sdp else 1.0,

utils.py

@@ -301,7 +301,11 @@ def clean_checkpoints(path_to_models="logs/44k/", n_ckpts_to_keep=2, sort_by_tim
 
     to_del = [
         os.path.join(path_to_models, fn)
-        for fn in (x_sorted("G")[:-n_ckpts_to_keep] + x_sorted("D")[:-n_ckpts_to_keep])
+        for fn in (
+            x_sorted("G")[:-n_ckpts_to_keep]
+            + x_sorted("D")[:-n_ckpts_to_keep]
+            + x_sorted("WD")[:-n_ckpts_to_keep]
+        )
     ]
 
     def del_info(fn):

webui.py

@@ -1,4 +1,5 @@
 # flake8: noqa: E402
+import gc
 import os
 import logging
 import re_matching
@@ -32,6 +33,14 @@ if device == "mps":
     os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
 
 
+def free_up_memory():
+    # Prior inference run might have large variables not cleaned up due to exception during the run.
+    # Free up as much memory as possible to allow this run to be successful.
+    gc.collect()
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
+
+
 def generate_audio(
     slices,
     sdp_ratio,
@@ -42,15 +51,20 @@ def generate_audio(
     language,
     reference_audio,
     emotion,
+    style_text,
+    style_weight,
     skip_start=False,
     skip_end=False,
 ):
     audio_list = []
     # silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
+
+    free_up_memory()
+
     with torch.no_grad():
         for idx, piece in enumerate(slices):
-            skip_start = (idx != 0) and skip_start
-            skip_end = (idx != len(slices) - 1) and skip_end
+            skip_start = idx != 0
+            skip_end = idx != len(slices) - 1
             audio = infer(
                 piece,
                 reference_audio=reference_audio,
@@ -66,10 +80,11 @@ def generate_audio(
                 device=device,
                 skip_start=skip_start,
                 skip_end=skip_end,
+                style_text=style_text,
+                style_weight=style_weight,
             )
             audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
             audio_list.append(audio16bit)
-            # audio_list.append(silence)  # 将静音添加到列表中
     return audio_list
 
 
@@ -88,10 +103,13 @@ def generate_audio_multilang(
 ):
     audio_list = []
     # silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
+
+    free_up_memory()
+
     with torch.no_grad():
         for idx, piece in enumerate(slices):
-            skip_start = (idx != 0) and skip_start
-            skip_end = (idx != len(slices) - 1) and skip_end
+            skip_start = idx != 0
+            skip_end = idx != len(slices) - 1
             audio = infer_multilang(
                 piece,
                 reference_audio=reference_audio,
@@ -110,7 +128,6 @@ def generate_audio_multilang(
             )
             audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
             audio_list.append(audio16bit)
-            # audio_list.append(silence)  # 将静音添加到列表中
     return audio_list
 
 
@@ -127,63 +144,50 @@ def tts_split(
     interval_between_sent,
     reference_audio,
     emotion,
+    style_text,
+    style_weight,
 ):
-    if language == "mix":
-        return ("invalid", None)
     while text.find("\n\n") != -1:
         text = text.replace("\n\n", "\n")
+    text = text.replace("|", "")
     para_list = re_matching.cut_para(text)
+    para_list = [p for p in para_list if p != ""]
     audio_list = []
-    if not cut_by_sent:
-        for idx, p in enumerate(para_list):
-            skip_start = idx != 0
-            skip_end = idx != len(para_list) - 1
-            audio = infer(
+    for p in para_list:
+        if not cut_by_sent:
+            audio_list += process_text(
                 p,
-                reference_audio=reference_audio,
-                emotion=emotion,
-                sdp_ratio=sdp_ratio,
-                noise_scale=noise_scale,
-                noise_scale_w=noise_scale_w,
-                length_scale=length_scale,
-                sid=speaker,
-                language=language,
-                hps=hps,
-                net_g=net_g,
-                device=device,
-                skip_start=skip_start,
-                skip_end=skip_end,
+                speaker,
+                sdp_ratio,
+                noise_scale,
+                noise_scale_w,
+                length_scale,
+                language,
+                reference_audio,
+                emotion,
+                style_text,
+                style_weight,
             )
-            audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
-            audio_list.append(audio16bit)
             silence = np.zeros((int)(44100 * interval_between_para), dtype=np.int16)
             audio_list.append(silence)
-    else:
-        for idx, p in enumerate(para_list):
-            skip_start = idx != 0
-            skip_end = idx != len(para_list) - 1
+        else:
             audio_list_sent = []
             sent_list = re_matching.cut_sent(p)
-            for idx, s in enumerate(sent_list):
-                skip_start = (idx != 0) and skip_start
-                skip_end = (idx != len(sent_list) - 1) and skip_end
-                audio = infer(
+            sent_list = [s for s in sent_list if s != ""]
+            for s in sent_list:
+                audio_list_sent += process_text(
                     s,
-                    reference_audio=reference_audio,
-                    emotion=emotion,
-                    sdp_ratio=sdp_ratio,
-                    noise_scale=noise_scale,
-                    noise_scale_w=noise_scale_w,
-                    length_scale=length_scale,
-                    sid=speaker,
-                    language=language,
-                    hps=hps,
-                    net_g=net_g,
-                    device=device,
-                    skip_start=skip_start,
-                    skip_end=skip_end,
+                    speaker,
+                    sdp_ratio,
+                    noise_scale,
+                    noise_scale_w,
+                    length_scale,
+                    language,
+                    reference_audio,
+                    emotion,
+                    style_text,
+                    style_weight,
                 )
-                audio_list_sent.append(audio)
                 silence = np.zeros((int)(44100 * interval_between_sent))
                 audio_list_sent.append(silence)
             if (interval_between_para - interval_between_sent) > 0:
@@ -196,10 +200,49 @@ def tts_split(
             )  # 对完整句子做音量归一
             audio_list.append(audio16bit)
     audio_concat = np.concatenate(audio_list)
-    return ("Success", (44100, audio_concat))
+    return ("Success", (hps.data.sampling_rate, audio_concat))
 
 
-def tts_fn(
+def process_mix(slice):
+    _speaker = slice.pop()
+    _text, _lang = [], []
+    for lang, content in slice:
+        content = content.split("|")
+        content = [part for part in content if part != ""]
+        if len(content) == 0:
+            continue
+        if len(_text) == 0:
+            _text = [[part] for part in content]
+            _lang = [[lang] for part in content]
+        else:
+            _text[-1].append(content[0])
+            _lang[-1].append(lang)
+            if len(content) > 1:
+                _text += [[part] for part in content[1:]]
+                _lang += [[lang] for part in content[1:]]
+    return _text, _lang, _speaker
+
+
+def process_auto(text):
+    _text, _lang = [], []
+    for slice in text.split("|"):
+        if slice == "":
+            continue
+        temp_text, temp_lang = [], []
+        sentences_list = split_by_language(slice, target_languages=["zh", "ja", "en"])
+        for sentence, lang in sentences_list:
+            if sentence == "":
+                continue
+            temp_text.append(sentence)
+            if lang == "ja":
+                lang = "jp"
+            temp_lang.append(lang.upper())
+        _text.append(temp_text)
+        _lang.append(temp_lang)
+    return _text, _lang
+
+
+def process_text(
     text: str,
     speaker,
     sdp_ratio,
@@ -209,15 +252,9 @@ def tts_fn(
     language,
     reference_audio,
     emotion,
-    prompt_mode,
+    style_text=None,
+    style_weight=0,
 ):
-    if prompt_mode == "Audio prompt":
-        if reference_audio == None:
-            return ("Invalid audio prompt", None)
-        else:
-            reference_audio = load_audio(reference_audio)[1]
-    else:
-        reference_audio = None
     audio_list = []
     if language == "mix":
         bool_valid, str_valid = re_matching.validate_text(text)
@@ -226,120 +263,40 @@ def tts_fn(
                 hps.data.sampling_rate,
                 np.concatenate([np.zeros(hps.data.sampling_rate // 2)]),
             )
-        result = []
         for slice in re_matching.text_matching(text):
-            _speaker = slice.pop()
-            temp_contant = []
-            temp_lang = []
-            for lang, content in slice:
-                if "|" in content:
-                    temp = []
-                    temp_ = []
-                    for i in content.split("|"):
-                        if i != "":
-                            temp.append([i])
-                            temp_.append([lang])
-                        else:
-                            temp.append([])
-                            temp_.append([])
-                    temp_contant += temp
-                    temp_lang += temp_
-                else:
-                    if len(temp_contant) == 0:
-                        temp_contant.append([])
-                        temp_lang.append([])
-                    temp_contant[-1].append(content)
-                    temp_lang[-1].append(lang)
-            for i, j in zip(temp_lang, temp_contant):
-                result.append([*zip(i, j), _speaker])
-        for i, one in enumerate(result):
-            skip_start = i != 0
-            skip_end = i != len(result) - 1
-            _speaker = one.pop()
-            idx = 0
-            while idx < len(one):
-                text_to_generate = []
-                lang_to_generate = []
-                while True:
-                    lang, content = one[idx]
-                    temp_text = [content]
-                    if len(text_to_generate) > 0:
-                        text_to_generate[-1] += [temp_text.pop(0)]
-                        lang_to_generate[-1] += [lang]
-                    if len(temp_text) > 0:
-                        text_to_generate += [[i] for i in temp_text]
-                        lang_to_generate += [[lang]] * len(temp_text)
-                    if idx + 1 < len(one):
-                        idx += 1
-                    else:
-                        break
-                skip_start = (idx != 0) and skip_start
-                skip_end = (idx != len(one) - 1) and skip_end
-                print(text_to_generate, lang_to_generate)
-                audio_list.extend(
-                    generate_audio_multilang(
-                        text_to_generate,
-                        sdp_ratio,
-                        noise_scale,
-                        noise_scale_w,
-                        length_scale,
-                        _speaker,
-                        lang_to_generate,
-                        reference_audio,
-                        emotion,
-                        skip_start,
-                        skip_end,
-                    )
+            _text, _lang, _speaker = process_mix(slice)
+            if _speaker is None:
+                continue
+            print(f"Text: {_text}\nLang: {_lang}")
+            audio_list.extend(
+                generate_audio_multilang(
+                    _text,
+                    sdp_ratio,
+                    noise_scale,
+                    noise_scale_w,
+                    length_scale,
+                    _speaker,
+                    _lang,
+                    reference_audio,
+                    emotion,
                 )
-                idx += 1
+            )
     elif language.lower() == "auto":
-        for idx, slice in enumerate(text.split("|")):
-            if slice == "":
-                continue
-            skip_start = idx != 0
-            skip_end = idx != len(text.split("|")) - 1
-            sentences_list = split_by_language(
-                slice, target_languages=["zh", "ja", "en"]
+        _text, _lang = process_auto(text)
+        print(f"Text: {_text}\nLang: {_lang}")
+        audio_list.extend(
+            generate_audio_multilang(
+                _text,
+                sdp_ratio,
+                noise_scale,
+                noise_scale_w,
+                length_scale,
+                speaker,
+                _lang,
+                reference_audio,
+                emotion,
             )
-            idx = 0
-            while idx < len(sentences_list):
-                text_to_generate = []
-                lang_to_generate = []
-                while True:
-                    content, lang = sentences_list[idx]
-                    temp_text = [content]
-                    lang = lang.upper()
-                    if lang == "JA":
-                        lang = "JP"
-                    if len(text_to_generate) > 0:
-                        text_to_generate[-1] += [temp_text.pop(0)]
-                        lang_to_generate[-1] += [lang]
-                    if len(temp_text) > 0:
-                        text_to_generate += [[i] for i in temp_text]
-                        lang_to_generate += [[lang]] * len(temp_text)
-                    if idx + 1 < len(sentences_list):
-                        idx += 1
-                    else:
-                        break
-                skip_start = (idx != 0) and skip_start
-                skip_end = (idx != len(sentences_list) - 1) and skip_end
-                print(text_to_generate, lang_to_generate)
-                audio_list.extend(
-                    generate_audio_multilang(
-                        text_to_generate,
-                        sdp_ratio,
-                        noise_scale,
-                        noise_scale_w,
-                        length_scale,
-                        speaker,
-                        lang_to_generate,
-                        reference_audio,
-                        emotion,
-                        skip_start,
-                        skip_end,
-                    )
-                )
-                idx += 1
+        )
     else:
         audio_list.extend(
             generate_audio(
@@ -352,13 +309,65 @@ def tts_fn(
                 language,
                 reference_audio,
                 emotion,
+                style_text,
+                style_weight,
             )
         )
+    return audio_list
+
+
+def tts_fn(
+    text: str,
+    speaker,
+    sdp_ratio,
+    noise_scale,
+    noise_scale_w,
+    length_scale,
+    language,
+    reference_audio,
+    emotion,
+    prompt_mode,
+    style_text=None,
+    style_weight=0,
+):
+    if style_text == "":
+        style_text = None
+    if prompt_mode == "Audio prompt":
+        if reference_audio == None:
+            return ("Invalid audio prompt", None)
+        else:
+            reference_audio = load_audio(reference_audio)[1]
+    else:
+        reference_audio = None
+
+    audio_list = process_text(
+        text,
+        speaker,
+        sdp_ratio,
+        noise_scale,
+        noise_scale_w,
+        length_scale,
+        language,
+        reference_audio,
+        emotion,
+        style_text,
+        style_weight,
+    )
 
     audio_concat = np.concatenate(audio_list)
     return "Success", (hps.data.sampling_rate, audio_concat)
 
 
+def format_utils(text, speaker):
+    _text, _lang = process_auto(text)
+    res = f"[{speaker}]"
+    for lang_s, content_s in zip(_lang, _text):
+        for lang, content in zip(lang_s, content_s):
+            res += f"<{lang.lower()}>{content}"
+        res += "|"
+    return "mix", res[:-1]
+
+
 def load_audio(path):
     audio, sr = librosa.load(path, 48000)
     # audio = librosa.resample(audio, 44100, 48000)
@@ -408,34 +417,37 @@ if __name__ == "__main__":
                 )
                 trans = gr.Button("中翻日", variant="primary")
                 slicer = gr.Button("快速切分", variant="primary")
+                formatter = gr.Button("检测语言，并整理为 MIX 格式", variant="primary")
                 speaker = gr.Dropdown(
                     choices=speakers, value=speakers[0], label="Speaker"
                 )
                 _ = gr.Markdown(
-                    value="提示模式（Prompt mode）：可选文字提示或音频提示，用于生成文字或音频指定风格的声音。\n"
+                    value="提示模式（Prompt mode）：可选文字提示或音频提示，用于生成文字或音频指定风格的声音。\n",
+                    visible=False,
                 )
                 prompt_mode = gr.Radio(
                     ["Text prompt", "Audio prompt"],
                     label="Prompt Mode",
                     value="Text prompt",
+                    visible=False,
                 )
                 text_prompt = gr.Textbox(
                     label="Text prompt",
                     placeholder="用文字描述生成风格。如：Happy",
                     value="Happy",
-                    visible=True,
+                    visible=False,
                 )
                 audio_prompt = gr.Audio(
                     label="Audio prompt", type="filepath", visible=False
                 )
                 sdp_ratio = gr.Slider(
-                    minimum=0, maximum=1, value=0.2, step=0.1, label="SDP Ratio"
+                    minimum=0, maximum=1, value=0.5, step=0.1, label="SDP Ratio"
                 )
                 noise_scale = gr.Slider(
                     minimum=0.1, maximum=2, value=0.6, step=0.1, label="Noise"
                 )
                 noise_scale_w = gr.Slider(
-                    minimum=0.1, maximum=2, value=0.8, step=0.1, label="Noise_W"
+                    minimum=0.1, maximum=2, value=0.9, step=0.1, label="Noise_W"
                 )
                 length_scale = gr.Slider(
                     minimum=0.1, maximum=2, value=1.0, step=0.1, label="Length"
@@ -445,6 +457,21 @@ if __name__ == "__main__":
                 )
                 btn = gr.Button("生成音频！", variant="primary")
             with gr.Column():
+                with gr.Accordion("融合文本语义", open=False):
+                    gr.Markdown(
+                        value="使用辅助文本的语意来辅助生成对话（语言保持与主文本相同）\n\n"
+                        "**注意**：不要使用**指令式文本**（如：开心），要使用**带有强烈情感的文本**（如：我好快乐！！！）\n\n"
+                        "效果较不明确，留空即为不使用该功能"
+                    )
+                    style_text = gr.Textbox(label="辅助文本")
+                    style_weight = gr.Slider(
+                        minimum=0,
+                        maximum=1,
+                        value=0.7,
+                        step=0.1,
+                        label="Weight",
+                        info="主文本和辅助文本的bert混合比率，0表示仅主文本，1表示仅辅助文本",
+                    )
                 with gr.Row():
                     with gr.Column():
                         interval_between_sent = gr.Slider(
@@ -487,6 +514,8 @@ if __name__ == "__main__":
                 audio_prompt,
                 text_prompt,
                 prompt_mode,
+                style_text,
+                style_weight,
             ],
             outputs=[text_output, audio_output],
         )
@@ -511,6 +540,8 @@ if __name__ == "__main__":
                 interval_between_sent,
                 audio_prompt,
                 text_prompt,
+                style_text,
+                style_weight,
             ],
             outputs=[text_output, audio_output],
         )
@@ -527,6 +558,12 @@ if __name__ == "__main__":
             outputs=[audio_prompt],
         )
 
+        formatter.click(
+            format_utils,
+            inputs=[text, speaker],
+            outputs=[language, text],
+        )
+
     print("推理页面已开启!")
     webbrowser.open(f"http://127.0.0.1:{config.webui_config.port}")
     app.launch(share=config.webui_config.share, server_port=config.webui_config.port)

webui_preprocess.py

@@ -19,9 +19,9 @@ def generate_config(data_dir, batch_size):
     assert data_dir != "", "数据集名称不能为空"
     start_path, _, train_path, val_path, config_path = get_path(data_dir)
     if os.path.isfile(config_path):
-        config = json.load(open(config_path))
+        config = json.load(open(config_path, "r", encoding="utf-8"))
     else:
-        config = json.load(open("configs/config.json"))
+        config = json.load(open("configs/config.json", "r", encoding="utf-8"))
     config["data"]["training_files"] = train_path
     config["data"]["validation_files"] = val_path
     config["train"]["batch_size"] = batch_size
@@ -44,7 +44,7 @@ def resample(data_dir):
     in_dir = os.path.join(start_path, "raw")
     out_dir = os.path.join(start_path, "wavs")
     subprocess.run(
-        f"python resample.py "
+        f"python resample_legacy.py "
         f"--sr 44100 "
         f"--in_dir {in_dir} "
         f"--out_dir {out_dir} ",
@@ -60,7 +60,9 @@ def preprocess_text(data_dir):
     with open(lbl_path, "w", encoding="utf-8") as f:
         for line in lines:
             path, spk, language, text = line.strip().split("|")
-            path = os.path.join(start_path, "wavs", os.path.basename(path))
+            path = os.path.join(start_path, "wavs", os.path.basename(path)).replace(
+                "\\", "/"
+            )
             f.writelines(f"{path}|{spk}|{language}|{text}\n")
     subprocess.run(
         f"python preprocess_text.py "
@@ -83,16 +85,6 @@ def bert_gen(data_dir):
     return "BERT 特征文件生成完成"
 
 
-def clap_gen(data_dir):
-    assert data_dir != "", "数据集名称不能为空"
-    _, _, _, _, config_path = get_path(data_dir)
-    subprocess.run(
-        f"python clap_gen.py " f"--config {config_path}",
-        shell=True,
-    )
-    return "CLAP 特征文件生成完成"
-
-
 if __name__ == "__main__":
     with gr.Blocks() as app:
         with gr.Row():
@@ -100,13 +92,13 @@ if __name__ == "__main__":
                 _ = gr.Markdown(
                     value="# Bert-VITS2 数据预处理\n"
                     "## 预先准备：\n"
-                    "下载 BERT 和 CLAP 模型：\n"
+                    "下载 BERT 和 WavLM 模型：\n"
                     "- [中文 RoBERTa](https://huggingface.co/hfl/chinese-roberta-wwm-ext-large)\n"
                     "- [日文 DeBERTa](https://huggingface.co/ku-nlp/deberta-v2-large-japanese-char-wwm)\n"
                     "- [英文 DeBERTa](https://huggingface.co/microsoft/deberta-v3-large)\n"
-                    "- [CLAP](https://huggingface.co/laion/clap-htsat-fused)\n"
+                    "- [WavLM](https://huggingface.co/microsoft/wavlm-base-plus)\n"
                     "\n"
-                    "将 BERT 模型放置到 `bert` 文件夹下，CLAP 模型放置到 `emotional` 文件夹下，覆盖同名文件夹。\n"
+                    "将 BERT 模型放置到 `bert` 文件夹下，WavLM 模型放置到 `slm` 文件夹下，覆盖同名文件夹。\n"
                     "\n"
                     "数据准备：\n"
                     "将数据放置在 data 文件夹下，按照如下结构组织：\n"
@@ -156,12 +148,10 @@ if __name__ == "__main__":
                 preprocess_text_btn = gr.Button(value="执行", variant="primary")
                 _ = gr.Markdown(value="## 第四步：生成 BERT 特征文件")
                 bert_gen_btn = gr.Button(value="执行", variant="primary")
-                _ = gr.Markdown(value="## 第五步：生成 CLAP 特征文件")
-                clap_gen_btn = gr.Button(value="执行", variant="primary")
                 _ = gr.Markdown(
                     value="## 训练模型及部署：\n"
                     "修改根目录下的 `config.yml` 中 `dataset_path` 一项为 `data/{你的数据集名称}`\n"
-                    "- 训练：将[预训练模型文件](https://openi.pcl.ac.cn/Stardust_minus/Bert-VITS2/modelmanage/show_model)（`D_0.pth`、`DUR_0.pth` 和 `G_0.pth`）放到 `data/{你的数据集名称}/models` 文件夹下，执行 `torchrun --nproc_per_node=1 train_ms.py` 命令（多卡运行可参考 `run_MnodesAndMgpus.sh` 中的命令。\n"
+                    "- 训练：将[预训练模型文件](https://openi.pcl.ac.cn/Stardust_minus/Bert-VITS2/modelmanage/show_model)（`D_0.pth`、`DUR_0.pth`、`WD_0.pth` 和 `G_0.pth`）放到 `data/{你的数据集名称}/models` 文件夹下，执行 `torchrun --nproc_per_node=1 train_ms.py` 命令（多卡运行可参考 `run_MnodesAndMgpus.sh` 中的命令。\n"
                     "- 部署：修改根目录下的 `config.yml` 中 `webui` 下 `model` 一项为 `models/{权重文件名}.pth` （如 G_10000.pth），然后执行 `python webui.py`"
                 )
 
@@ -171,7 +161,6 @@ if __name__ == "__main__":
         resample_btn.click(resample, inputs=[data_dir], outputs=[info])
         preprocess_text_btn.click(preprocess_text, inputs=[data_dir], outputs=[info])
         bert_gen_btn.click(bert_gen, inputs=[data_dir], outputs=[info])
-        clap_gen_btn.click(clap_gen, inputs=[data_dir], outputs=[info])
 
     webbrowser.open("http://127.0.0.1:7860")
     app.launch(share=False, server_port=7860)