Merge pull request #32 from erew123/main

merge main to dev

README.md

@@ -25,12 +25,9 @@ AllTalk is an updated version of the Coqui_tts extension for Text Generation web
 - 🟠 [API Suite and JSON-CURL](https://github.com/erew123/alltalk_tts?#-api-suite-and-json-curl)
 - 🔴 [Future to-do list & Upcoming updates](https://github.com/erew123/alltalk_tts?#-future-to-do-list)
 
-#### Updates
+## 🔄 Updates
-**Dec 28th**- Finetuning has been updated to compact models and make the final steps easier (buttons basically). Narrator has been updated. If you wish to compact existing finetuned models, pre the update please see [here](https://github.com/erew123/alltalk_tts/issues/28)
+Please check the below link to find a list of all recent updates and changes.
-
+####     🔄 **Updates list & bug fixes list** can be found [here](https://github.com/erew123/alltalk_tts/issues/25)
-**Dec 25th** - Applied a small update to avoid a possible race condition on file naming with small sentences when generating narrator/character speech. Please update if you are experiencing missing small bits of missing audio on narrator/character.
-
-####     🔄 **Minor updates/bug fixes list** can be found [here](https://github.com/erew123/alltalk_tts/issues/25)
 
 ## 🟩 Installation on Text generation web UI
 This has been tested on the current Dec 2023 release of Text generation webUI. If you have not updated it for a while, you may wish to update Text generation webUI, [instructions here](https://github.com/oobabooga/text-generation-webui?tab=readme-ov-file#how-to-install)
@@ -473,7 +470,7 @@ def get_shm_size():
 ## 🟦 Running AllTalk as a standalone app
 AllTalk will run as a standalone app, as long as you install its requirements files into whatever Python environment you are using. You can follow the steps to install the AllTalk's requirements into whatever Python environment you wish. Because I dont know what Python environment you are wanting to use, I can only give you a loose set of installation instructions. 
 
-Please note, at time of writing, the TTS engine requires Python **3.9.x** to **3.11.x** [TTS Engine details here](https://pypi.org/project/TTS/)
+Please note, at time of writing, the TTS engine requires Python **3.9.x** to **3.11.x** [TTS Engine details here](https://pypi.org/project/TTS/). AllTalk and its requirements are tested on Python 3.11.x.
 
 #### 🟦 (Option 1) I already have AllTalk installed as an extension of Text-generation-webui
 If you already have AllTalk as a extension of Text-generation-webui, and wish to run it as standalone, load Text-generation-webui's Python environment `cmd_windows.bat`, `./cmd_linux.sh`, `cmd_macos.sh` or `cmd_wsl.bat`, move into the AllTalk folder `cd extensions` > `cd alltalk_tts` and start AllTalk with `python script.py`. There is nothing beyond this you would need to do.
@@ -537,6 +534,10 @@ Generate a preview of a specified voice with hardcoded settings.
 
    Switch between the 3 models respectively.
 
+   `curl -X POST "http://127.0.0.1:7851/api/reload?tts_method=XTTSv2%20FT"`<br>
+
+   If you have a finetuned model in `/models/trainedmodel/` (will error otherwise)
+
    JSON return `{"status": "model-success"}`
 
 #### 🟠 Switch DeepSpeed Endpoint<br>

script.py

@@ -77,6 +77,8 @@ with open(this_dir / "languages.json", encoding="utf8") as f:
     languages = json.load(f)
 # Create a global lock
 process_lock = threading.Lock()
+# Base setting for a possible FineTuned model existing and the loader being available
+tts_method_xtts_ft = False
 
 # Gather the voice files
 def get_available_voices():
@@ -139,10 +141,23 @@ def check_required_files():
     subprocess.run(["python", str(download_script_path)])
     print(f"[{params['branding']}Startup] All required files are present.")
 
-
 # STARTUP Call Check routine
 check_required_files()
 
+##################################################
+#### Check to see if a finetuned model exists ####
+##################################################
+# Set the path to the directory
+trained_model_directory = this_dir / "models" / "trainedmodel"
+# Check if the directory "trainedmodel" exists
+finetuned_model = trained_model_directory.exists()
+# If the directory exists, check for the existence of the required files
+# If true, this will add a extra option in the Gradio interface for loading Xttsv2 FT
+if finetuned_model:
+    required_files = ["model.pth", "config.json", "vocab.json"]
+    finetuned_model = all((trained_model_directory / file).exists() for file in required_files)
+if finetuned_model:
+    print(f"[{params['branding']}Startup] Finetuned model \033[93mDetected\033[0m")
 
 ####################################################
 #### SET GRADIO BUTTONS BASED ON confignew.json ####
@@ -272,11 +287,13 @@ else:
     # Cleanly kill off this script, but allow text-generation-webui to keep running, albeit without this alltalk_tts
     sys.exit(1)
 
+
 #####################################
 #### MODEL LOADING AND UNLOADING ####
 #####################################
 # MODEL - Swap model based on Gradio selection API TTS, API Local, XTTSv2 Local
 def send_reload_request(tts_method):
+    global tts_method_xtts_ft
     try:
         params["tts_model_loaded"] = False
         url = f"{base_url}/api/reload"
@@ -295,17 +312,26 @@ def send_reload_request(tts_method):
                 params["tts_method_api_tts"] = True
                 params["deepspeed_activate"] = False
                 audio_path = this_dir / "templates" / "apitts.wav"
+                tts_method_xtts_ft = False
             elif tts_method == "API Local":
                 params["tts_method_api_tts"] = False
                 params["tts_method_xtts_local"] = False
                 params["tts_method_api_local"] = True
                 params["deepspeed_activate"] = False
                 audio_path = this_dir / "templates" / "apilocal.wav"
+                tts_method_xtts_ft = False
             elif tts_method == "XTTSv2 Local":
                 params["tts_method_api_tts"] = False
                 params["tts_method_api_local"] = False
                 params["tts_method_xtts_local"] = True
                 audio_path = this_dir / "templates" / "xttslocal.wav"
+                tts_method_xtts_ft = False
+            elif tts_method == "XTTSv2 FT":
+                params["tts_method_api_tts"] = False
+                params["tts_method_api_local"] = False
+                params["tts_method_xtts_local"] = False
+                audio_path = this_dir / "templates" / "xttsfinetuned.wav"
+                tts_method_xtts_ft = True
         return f'<audio src="file/{audio_path}" controls autoplay></audio>'
     except requests.exceptions.RequestException as e:
         # Handle the HTTP request error
@@ -494,7 +520,6 @@ def output_modifier(string, state):
     if not params["activate"]:
         return string
     original_string = string
-    #print("ORGINAL STRING: ", original_string)
     cleaned_string = before_audio_generation(string, params)
     if cleaned_string is None:
         return
@@ -505,7 +530,6 @@ def output_modifier(string, state):
     if process_lock.acquire(blocking=False):
         try:
             if params["narrator_enabled"]:
-                #print(original_string)
                 # Do Some basic stripping and remove any double CR's
                 processed_string = (
                     original_string
@@ -786,8 +810,11 @@ def ui():
             deepspeed_checkbox_play = gr.HTML(visible=False)
 
         with gr.Row():
+            model_loader_choices=["API TTS", "API Local", "XTTSv2 Local"]
+            if finetuned_model:
+                model_loader_choices.append("XTTSv2 FT")
             tts_radio_buttons = gr.Radio(
-                choices=["API TTS", "API Local", "XTTSv2 Local"],
+                choices=model_loader_choices,
                 label="Select TTS Generation Method (Read NOTE)",
                 value=gr_modelchoice,  # Set the default value
             )

tts_server.py

@@ -33,7 +33,8 @@ device = "cuda" if torch.cuda.is_available() else "cpu"
 # STARTUP VARIABLE - Import languges file for Gradio to be able to display them in the interface
 with open(this_dir / "languages.json", encoding="utf8") as f:
     languages = json.load(f)
-
+# Base setting for a possible FineTuned model existing and the loader being available
+tts_method_xtts_ft = False
 
 #################################################################
 #### LOAD PARAMS FROM confignew.json - REQUIRED FOR BRANDING ####
@@ -123,7 +124,7 @@ app = FastAPI(lifespan=startup_shutdown)
 #####################################
 #### MODEL LOADING AND UNLOADING ####
 #####################################
-# MODEL LOADERS Picker For API TTS, API Local, XTTSv2 Local
+# MODEL LOADERS Picker For API TTS, API Local, XTTSv2 Local, XTTSv2 FT
 async def setup():
     global device
     # Set a timer to calculate load times
@@ -150,6 +151,13 @@ async def setup():
             "\033[0m",
         )
         model = await xtts_manual_load_model()
+    elif tts_method_xtts_ft:
+        print(
+            f"[{params['branding']}Model] \033[94mXTTSv2 FT Loading\033[0m /models/fintuned/model.pth \033[94minto\033[93m",
+            device,
+            "\033[0m",
+        )
+        model = await xtts_ft_manual_load_model()
     # Create an end timer for calculating load times
     generate_end_time = time.time()
     # Calculate start time minus end time
@@ -222,6 +230,23 @@ async def xtts_manual_load_model():
     model.to(device)
     return model
 
+# MODEL LOADER For "XTTSv2 FT"
+async def xtts_ft_manual_load_model():
+    global model
+    config = XttsConfig()
+    config_path = this_dir / "models" / "trainedmodel" / "config.json"
+    vocab_path_dir = this_dir / "models" / "trainedmodel" / "vocab.json"
+    checkpoint_dir = this_dir / "models" / "trainedmodel"
+    config.load_json(str(config_path))
+    model = Xtts.init_from_config(config)
+    model.load_checkpoint(
+        config,
+        checkpoint_dir=str(checkpoint_dir),
+        vocab_path=str(vocab_path_dir),
+        use_deepspeed=params["deepspeed_activate"],
+    )
+    model.to(device)
+    return model
 
 # MODEL UNLOADER
 async def unload_model(model):
@@ -236,6 +261,7 @@ async def unload_model(model):
 # MODEL - Swap model based on Gradio selection API TTS, API Local, XTTSv2 Local
 async def handle_tts_method_change(tts_method):
     global model
+    global tts_method_xtts_ft
     # Update the params dictionary based on the selected radio button
     print(
         f"[{params['branding']}Model] \033[94mChanging model \033[92m(Please wait 15 seconds)\033[0m"
@@ -246,15 +272,23 @@ async def handle_tts_method_change(tts_method):
         params["tts_method_xtts_local"] = False
         params["tts_method_api_tts"] = True
         params["deepspeed_activate"] = False
+        tts_method_xtts_ft = False
     elif tts_method == "API Local":
         params["tts_method_api_tts"] = False
         params["tts_method_xtts_local"] = False
         params["tts_method_api_local"] = True
         params["deepspeed_activate"] = False
+        tts_method_xtts_ft = False
     elif tts_method == "XTTSv2 Local":
         params["tts_method_api_tts"] = False
         params["tts_method_api_local"] = False
         params["tts_method_xtts_local"] = True
+        tts_method_xtts_ft = False
+    elif tts_method == "XTTSv2 FT":
+        tts_method_xtts_ft = True
+        params["tts_method_api_tts"] = False
+        params["tts_method_api_local"] = False
+        params["tts_method_xtts_local"] = False
 
     # Unload the current model
     model = await unload_model(model)
@@ -267,7 +301,7 @@ async def handle_tts_method_change(tts_method):
 @app.route("/api/reload", methods=["POST"])
 async def reload(request: Request):
     tts_method = request.query_params.get("tts_method")
-    if tts_method not in ["API TTS", "API Local", "XTTSv2 Local"]:
+    if tts_method not in ["API TTS", "API Local", "XTTSv2 Local", "XTTSv2 FT"]:
         return {"status": "error", "message": "Invalid TTS method specified"}
     await handle_tts_method_change(tts_method)
     return Response(
@@ -424,8 +458,8 @@ async def generate_audio(text, voice, language, output_file):
     if params["low_vram"] and device == "cpu":
         await switch_device()
     generate_start_time = time.time()  # Record the start time of generating TTS
-    # XTTSv2 LOCAL Method
+    # XTTSv2 LOCAL & Xttsv2 FT Method
-    if params["tts_method_xtts_local"]:
+    if params["tts_method_xtts_local"] or tts_method_xtts_ft:
         print(f"[{params['branding']}TTSGen] {text}")
         gpt_cond_latent, speaker_embedding = model.get_conditioning_latents(
             audio_path=[f"{this_dir}/voices/{voice}"],
@@ -805,10 +839,6 @@ async def tts_generate(
     autoplay_volume: float = Form(...),
 ):
     try:
-        #print(f"text_filtering: {text_filtering}")
-        #print(f"narrator_enabled: {narrator_enabled}")
-        #print(f"text_not_inside: {text_not_inside}")
-        #print(f"output_file_timestamp: {output_file_timestamp}")
         json_input_data = {
             "text_input": text_input,
             "text_filtering": text_filtering,
@@ -828,46 +858,38 @@ async def tts_generate(
         else:
             return JSONResponse(content={"error": JSONresult}, status_code=400)
         if narrator_enabled:
-            print("ORIGINAL UNTOUCHED STRING IS:", text_input,"\n")
+            #print("ORIGINAL UNTOUCHED STRING IS:", text_input,"\n")
             if text_filtering in ["standard", "none"]:
                     cleaned_string = (
                         text_input
                         .replace('"', '"<')
                         .replace('*', '<*')
                     )
-                    print("STANDARD FILTERING IS NOW:", cleaned_string,"\n")
-                    #parts = re.split(r'\.(?<!<[*"])', cleaned_string)
                     parts = re.split(r'(?<=<\*\s)|(?<=\.\s)|(?<=\."\<)|(?<=\."<)', cleaned_string)
                     parts = list(filter(lambda x: x.strip(), parts))
             elif text_filtering == "html":
                 cleaned_string = standard_filtering(text_input)
                 cleaned_string = narrator_filtering(cleaned_string)
-                print("HTML FILTERING IS NOW:", cleaned_string,"\n")
                 parts = re.split(r'&quot;|\.\*', cleaned_string)
             audio_files_all_paragraphs = []
             audio_files_paragraph = []
             for i, part in enumerate(parts):
                 if len(part.strip()) <= 1:
                     continue
-                print("THIS IS A PART", part)
                 # Figure out which type of line it is, then replace characters as necessary to avoid TTS trying to pronunce them, htmlunescape after. 
                 # Character will always be a < with a letter immediately after it
                 if '<' in part and '<*' not in part and '< *' not in part and '<  *' not in part and '< ' not in part and '<  ' not in part:
-                    print("IF - Character\n")
                     cleaned_part = html.unescape(part.replace('<', ''))
                     voice_to_use = character_voice_gen
                 #Narrator will always be an * or < with an * a position or two after it.
                 elif '<*' in part or '< *' in part or '<  *' in part or '*' in part:
-                    print("IF - Narrator\n")
                     cleaned_part = html.unescape(part.replace('<*', '').replace('< *', '').replace('<  *', '').replace('*', '').replace('<. ', '')) 
                     voice_to_use = narrator_voice_gen
                 #If the other two dont capture it, aka, the AI gave no * or &quot; on the line, use non_quoted_text_is aka user interface, user can choose Char or Narrator
                 elif text_not_inside == "character":
-                    print("ELSE - CHARACTER\n")
                     cleaned_part = html.unescape(part.replace('< ', '').replace('<  ', '').replace('<  ', ''))
                     voice_to_use = character_voice_gen
                 elif text_not_inside == "narrator":
-                    print("ELSE - NARRATOR\n")
                     cleaned_part = html.unescape(part.replace('< ', '').replace('<  ', '').replace('<  ', ''))
                     voice_to_use = narrator_voice_gen
                 output_file = this_dir / "outputs" / f"{output_file_name}_{uuid.uuid4()}_{int(time.time())}_{i}.wav"