From 5a81e4818ba2d38fa5d59d0eac911a3a3ffb8b85 Mon Sep 17 00:00:00 2001 From: erew123 <35898566+erew123@users.noreply.github.com> Date: Tue, 5 Nov 2024 10:19:45 +0000 Subject: [PATCH] Add support for the E2 model to F5-TTS --- .../tts_engines/f5tts/available_models.json | 12 +- .../tts_engines/f5tts/f5tts_settings_page.py | 8 ++ system/tts_engines/f5tts/model_engine.py | 125 ++++++++++++------ 3 files changed, 107 insertions(+), 38 deletions(-) diff --git a/system/tts_engines/f5tts/available_models.json b/system/tts_engines/f5tts/available_models.json index 464dab0..bd17b89 100644 --- a/system/tts_engines/f5tts/available_models.json +++ b/system/tts_engines/f5tts/available_models.json @@ -10,6 +10,16 @@ "vocos/config.yaml": "https://huggingface.co/charactr/vocos-mel-24khz/resolve/main/config.yaml", "vocos/pytorch_model.bin": "https://huggingface.co/charactr/vocos-mel-24khz/resolve/main/pytorch_model.bin" } - } + }, + { + "model_name": "e2tts_v1", + "folder_path": "e2tts_v1", + "files_to_download": { + "model_1200000.safetensors": "https://huggingface.co/SWivid/E2-TTS/resolve/main/E2TTS_Base/model_1200000.safetensors", + "vocab.txt": "https://raw.githubusercontent.com/SWivid/F5-TTS/refs/heads/main/src/f5_tts/infer/examples/vocab.txt", + "vocos/config.yaml": "https://huggingface.co/charactr/vocos-mel-24khz/resolve/main/config.yaml", + "vocos/pytorch_model.bin": "https://huggingface.co/charactr/vocos-mel-24khz/resolve/main/pytorch_model.bin" + } + } ] } \ No newline at end of file diff --git a/system/tts_engines/f5tts/f5tts_settings_page.py b/system/tts_engines/f5tts/f5tts_settings_page.py index a4e90c5..74a720f 100644 --- a/system/tts_engines/f5tts/f5tts_settings_page.py +++ b/system/tts_engines/f5tts/f5tts_settings_page.py @@ -493,6 +493,14 @@ def f5tts_model_alltalk_settings(model_config_data): ### 🟧 Cloned audio and punctuation. When F5-TTS reproduces audio, punctuation appears to matter greatly. As such `Im` and `I'm` or `I am` will all sound a little different. Additionally it appears to want to spell out text in CAPS. You may want to look at the F5-TTS developers page for more information on this. + + ### 🟧 Difference between F5-TTS and E2-TTS models. + F5-TTS and E2-TTS represent two different approaches to zero-shot text-to-speech: + + - F5-TTS is optimized for voice fidelity and natural speech patterns. It excels at capturing the nuances and characteristics of the reference voice, producing highly natural and faithful speech output. The tradeoff is slightly longer generation times and higher computational requirements.
+ - E2-TTS prioritizes efficiency and ease of use. It offers faster generation speeds and better memory efficiency, making it ideal for longer texts or resource-constrained environments. While its voice reproduction may be slightly less precise than F5-TTS, it still produces high-quality output with good speaker similarity. + + Choose F5-TTS when voice fidelity is paramount, or E2-TTS when generation speed and efficiency are the priority. """) return app diff --git a/system/tts_engines/f5tts/model_engine.py b/system/tts_engines/f5tts/model_engine.py index 1795c49..e628fc2 100644 --- a/system/tts_engines/f5tts/model_engine.py +++ b/system/tts_engines/f5tts/model_engine.py @@ -27,6 +27,7 @@ deepspeed_available = False # In this section you will import any imports that your specific TTS Engine will use. You will provide any # start-up errors for those bits, as if you were starting up a normal Python script. Note the logging.disable # a few lines up from here, you may want to # that out while debugging! +from vocos import Vocos import soundfile as sf import tempfile from pydub import AudioSegment, silence @@ -38,22 +39,9 @@ import sys def install_and_restart(): try: - print("#################################################################") - print("Installing required packages for F5-TTS... This may take a while.") - print("################################################################") - - # Install vocos first - print("Installing vocos...") - subprocess.check_call([ - sys.executable, - "-m", - "pip", - "install", - "vocos" - ]) - - # Then install F5-TTS - print("Installing F5-TTS...") + print("##########################################") + print("F5-TTS not found. Attempting to install...") + print("##########################################") subprocess.check_call([ sys.executable, "-m", @@ -61,10 +49,9 @@ def install_and_restart(): "install", "git+https://github.com/SWivid/F5-TTS.git" ]) - - print("##############################################################") - print("All packages installed successfully! Restarting application...") - print("##############################################################") + print("########################################################") + print("F5-TTS installed successfully! Restarting application...") + print("########################################################") # Get the current script's path script_path = sys.argv[0] @@ -74,17 +61,16 @@ def install_and_restart(): except subprocess.CalledProcessError as e: print("########################################################") - print(f"Failed to install required packages: {str(e)}") + print(f"Failed to install F5-TTS: {str(e)}") print("########################################################") - raise ImportError("Could not install required packages") + raise ImportError("Could not install required package F5-TTS") try: - from f5_tts.model import CFM, DiT + from f5_tts.model import CFM, DiT, UNetT from f5_tts.model.utils import ( get_tokenizer, convert_char_to_pinyin, ) - from vocos import Vocos except ImportError: install_and_restart() @@ -190,14 +176,21 @@ class tts_class: self.sway_sampling_coef = -1.0 self.speed = 1.0 # F5-TTS model configuration - self.model_cfg = dict( + self.f5_model_cfg = dict( dim=1024, depth=22, heads=16, ff_mult=2, text_dim=512, conv_layers=4 - ) + ) + # Add E2-TTS model configuration + self.e2_model_cfg = dict( + dim=1024, + depth=24, + heads=16, + ff_mult=4 + ) ################################################################ # DONT CHANGE # Print out Python, CUDA, DeepSpeed versions #### ################################################################ @@ -387,7 +380,7 @@ class tts_class: return self.available_models # Look for model directories that match the pattern f5tts_v* - for model_dir in models_dir.glob("f5tts_v*"): + for model_dir in models_dir.glob("*tts_v*"): if model_dir.is_dir(): # First try to find model_*.safetensors files model_files = list(model_dir.glob("model_*.safetensors")) @@ -507,7 +500,10 @@ class tts_class: self.vocoder.load_state_dict(vocoder_state) self.vocoder = self.vocoder.eval() - # Initialize F5-TTS model with same mel_spec_kwargs as original + # Determine if this is an E2-TTS model from the folder name + is_e2_model = "e2tts" in model_folder.lower() + + # Initialize model with appropriate architecture vocab_char_map, vocab_size = get_tokenizer(str(vocab_path), "custom") mel_spec_kwargs = dict( @@ -519,14 +515,34 @@ class tts_class: mel_spec_type=self.mel_spec_type, ) - self.model = CFM( - transformer=DiT(**self.model_cfg, text_num_embeds=vocab_size, mel_dim=self.n_mel_channels), - mel_spec_kwargs=mel_spec_kwargs, - odeint_kwargs=dict( - method=self.ode_method, - ), - vocab_char_map=vocab_char_map - ).float() # Ensure float32 + if is_e2_model: + print(f"[{self.branding}ENG] Initializing E2-TTS model") if self.debug_tts else None + self.model = CFM( + transformer=UNetT( + **self.e2_model_cfg, + text_num_embeds=vocab_size, + mel_dim=self.n_mel_channels + ), + mel_spec_kwargs=mel_spec_kwargs, + odeint_kwargs=dict( + method=self.ode_method, + ), + vocab_char_map=vocab_char_map + ).float() # Ensure float32 + else: + print(f"[{self.branding}ENG] Initializing F5-TTS model") if self.debug_tts else None + self.model = CFM( + transformer=DiT( + **self.f5_model_cfg, + text_num_embeds=vocab_size, + mel_dim=self.n_mel_channels + ), + mel_spec_kwargs=mel_spec_kwargs, + odeint_kwargs=dict( + method=self.ode_method, + ), + vocab_char_map=vocab_char_map + ).float() # Ensure float32 # Load model weights if str(model_path).endswith('.safetensors'): @@ -572,7 +588,8 @@ class tts_class: self.vocoder = self.vocoder.to(self.device) self.is_tts_model_loaded = True - print(f"[{self.branding}ENG] F5-TTS model loaded successfully") if self.debug_tts else None + model_type = "E2-TTS" if is_e2_model else "F5-TTS" + print(f"[{self.branding}ENG] {model_type} model loaded successfully") if self.debug_tts else None except Exception as e: print(f"[{self.branding}ENG] \033[91mError loading model: {str(e)}\033[0m") @@ -895,6 +912,38 @@ class tts_class: return chunks + ############################################################################################################################# + ############################################################################################################################# + # DONT CHANGE ME # De-Capitalise words (Not used currently) ################################################################# + ############################################################################################################################# + ############################################################################################################################# + def process_text_for_tts(self, text): + """ + Process text by converting fully capitalized words to lowercase, + while preserving other words' capitalization. + + Args: + self: The class instance + text (str): Input text string + + Returns: + str: Processed text with fully capitalized words converted to lowercase + """ + # Split text into words + words = text.split() + + # Process each word + processed_words = [] + for word in words: + # Check if word is fully uppercase and longer than 1 character + if word.isupper() and len(word) > 1: + processed_words.append(word.lower()) + else: + processed_words.append(word) + + # Join words back together + return ' '.join(processed_words) + ########################################################################################################################################## ########################################################################################################################################## # CHANGE ME # Model changing. Unload out old model and load in a new one ################################################################# @@ -981,6 +1030,8 @@ class tts_class: str(ref_audio_path), ref_text ) + text = self.process_text_for_tts(text) + # Generate the audio using infer_process final_wave, final_sample_rate, _ = await self.infer_process( ref_audio,