From 5124c6c6b3a169084a1d53e9e6a60f6afc97698b Mon Sep 17 00:00:00 2001 From: erew123 <35898566+erew123@users.noreply.github.com> Date: Sun, 10 Dec 2023 10:48:48 +0000 Subject: [PATCH] Test around cuda for non-cuda users - made logic for model loading on non-cuda systems - split requirements files into nvidia and other - updated other files as necessary --- modeldownload.py | 6 +-- requirements_nvidia.txt | 12 ++++++ requirements_other.txt | 12 ++++++ script.py | 1 - tts_server.py | 81 ++++++++++++++++++++++------------------- 5 files changed, 71 insertions(+), 41 deletions(-) create mode 100644 requirements_nvidia.txt create mode 100644 requirements_other.txt diff --git a/modeldownload.py b/modeldownload.py index 7ddc851..cb078ee 100644 --- a/modeldownload.py +++ b/modeldownload.py @@ -44,8 +44,8 @@ else: f"[{params['branding']}Startup] \033[91mWarning\033[0m API Local and XTTSv2 Local will error unless this is corrected." ) -# Read the version specifier from requirements.txt -with open(this_dir / "requirements.txt", "r") as req_file: +# Read the version specifier from requirements_nvidia.txt +with open(this_dir / "requirements_nvidia.txt", "r") as req_file: requirements = req_file.readlines() tts_version_required = None @@ -56,7 +56,7 @@ for req in requirements: if tts_version_required is None: raise ValueError( - f"[{params['branding']}Startup] \033[91mWarning\033[0m Could not find TTS version specifier in requirements.txt" + f"[{params['branding']}Startup] \033[91mWarning\033[0m Could not find TTS version specifier in requirements_nvidia.txt" ) diff --git a/requirements_nvidia.txt b/requirements_nvidia.txt new file mode 100644 index 0000000..a278f80 --- /dev/null +++ b/requirements_nvidia.txt @@ -0,0 +1,12 @@ +numpy>=1.24.4 +soundfile>=0.12.1 +uvicorn>=0.24.0.post1 +TTS>=0.21.3 +torch>=2.1.0+cu118 +torchaudio>=2.1.0+cu118 +fastapi>=0.104.1 +Jinja2>=3.1.2 +requests>=2.31.0 +tqdm>=4.66.1 +importlib-metadata>=4.8.1 +packaging>=23.2 diff --git a/requirements_other.txt b/requirements_other.txt new file mode 100644 index 0000000..6eb4b14 --- /dev/null +++ b/requirements_other.txt @@ -0,0 +1,12 @@ +numpy>=1.24.4 +soundfile>=0.12.1 +uvicorn>=0.24.0.post1 +TTS>=0.21.3 +torch>=2.1.0 +torchaudio>=2.1.0 +fastapi>=0.104.1 +Jinja2>=3.1.2 +requests>=2.31.0 +tqdm>=4.66.1 +importlib-metadata>=4.8.1 +packaging>=23.2 diff --git a/script.py b/script.py index f731610..f139cee 100644 --- a/script.py +++ b/script.py @@ -404,7 +404,6 @@ def combine(audio_files, output_folder, state): ) sf.write(output_file_path, audio, samplerate=sample_rate) print(f"[{params['branding']}TTSGen] Narrated audio generated") - # Clean up unnecessary files for audio_file in audio_files: os.remove(audio_file) diff --git a/tts_server.py b/tts_server.py index 2db791d..b1b23df 100644 --- a/tts_server.py +++ b/tts_server.py @@ -24,7 +24,6 @@ from fastapi.responses import JSONResponse, HTMLResponse, RedirectResponse from fastapi.templating import Jinja2Templates from contextlib import asynccontextmanager - ########################### #### STARTUP VARIABLES #### ########################### @@ -36,24 +35,22 @@ device = "cuda" if torch.cuda.is_available() else "cpu" with open(this_dir / "languages.json", encoding="utf8") as f: languages = json.load(f) - ############################################################## #### LOAD PARAMS FROM CONFIG.JSON - REQUIRED FOR BRANDING #### ############################################################## - - # Load config file and get settings def load_config(file_path): with open(file_path, "r") as configfile_path: configfile_data = json.load(configfile_path) return configfile_data - # Define the path to the config.json file configfile_path = this_dir / "config.json" # Load config.json and assign it to a different variable (config_data) params = load_config(configfile_path) +# check someone hasnt enabled lowvram on a system thats not cuda enabled +params["low_vram"] = "false" if not torch.cuda.is_available() else params["low_vram"] # Define the path to the JSON file config_file_path = this_dir / "modeldownload.json" @@ -215,7 +212,6 @@ async def xtts_manual_load_model(): checkpoint_dir=str(checkpoint_dir), use_deepspeed=params["deepspeed_activate"], ) - model.cuda() model.to(device) return model @@ -224,7 +220,8 @@ async def xtts_manual_load_model(): async def unload_model(model): print(f"[{params['branding']}Model] \033[94mUnloading model \033[0m") del model - torch.cuda.empty_cache() + if torch.cuda.is_available(): + torch.cuda.empty_cache() params["model_loaded"] = False return None @@ -258,7 +255,6 @@ async def handle_tts_method_change(tts_method): # Load the correct model based on the updated params await setup() - # MODEL WEBSERVER- API Swap Between Models @app.route("/api/reload", methods=["POST"]) async def reload(request: Request): @@ -270,22 +266,22 @@ async def reload(request: Request): content=json.dumps({"status": "model-success"}), media_type="application/json" ) - ################## #### LOW VRAM #### ################## # LOW VRAM - MODEL MOVER VRAM(cuda)<>System RAM(cpu) for Low VRAM setting async def switch_device(): global model, device - if device == "cuda": - device = "cpu" - model.to(device) - torch.cuda.empty_cache() - else: - device == "cpu" - device = "cuda" - model.to(device) - + # Check if CUDA is available before performing GPU-related operations + if torch.cuda.is_available(): + if device == "cuda": + device = "cpu" + model.to(device) + torch.cuda.empty_cache() + else: + device == "cpu" + device = "cuda" + model.to(device) @app.post("/api/lowvramsetting") async def set_low_vram(request: Request, new_low_vram_value: bool): @@ -293,6 +289,7 @@ async def set_low_vram(request: Request, new_low_vram_value: bool): try: if new_low_vram_value is None: raise ValueError("Missing 'low_vram' parameter") + if params["low_vram"] == new_low_vram_value: return Response( content=json.dumps( @@ -305,29 +302,38 @@ async def set_low_vram(request: Request, new_low_vram_value: bool): params["low_vram"] = new_low_vram_value if params["low_vram"]: await unload_model(model) - device = "cpu" - print( - f"[{params['branding']}Model] \033[94mChanging model \033[92m(Please wait 15 seconds)\033[0m" - ) - print( - f"[{params['branding']}Model] \033[94mLowVRAM Enabled.\033[0m Model will move between \033[93mVRAM(cuda) <> System RAM(cpu)\033[0m" - ) - await setup() + if torch.cuda.is_available(): + device = "cpu" + print( + f"[{params['branding']}Model] \033[94mChanging model \033[92m(Please wait 15 seconds)\033[0m" + ) + print( + f"[{params['branding']}Model] \033[94mLowVRAM Enabled.\033[0m Model will move between \033[93mVRAM(cuda) <> System RAM(cpu)\033[0m" + ) + await setup() + else: + # Handle the case where CUDA is not available + print(f"[{params['branding']}Model] \033[91mError:\033[0m Nvidia CUDA is not available on this system. Unable to use LowVRAM mode.") + params["low_vram"] = False else: await unload_model(model) - device = "cuda" - print( - f"[{params['branding']}Model] \033[94mChanging model \033[92m(Please wait 15 seconds)\033[0m" - ) - print( - f"[{params['branding']}Model] \033[94mLowVRAM Disabled.\033[0m Model will stay in \033[93mVRAM(cuda)\033[0m" - ) - await setup() + if torch.cuda.is_available(): + device = "cuda" + print( + f"[{params['branding']}Model] \033[94mChanging model \033[92m(Please wait 15 seconds)\033[0m" + ) + print( + f"[{params['branding']}Model] \033[94mLowVRAM Disabled.\033[0m Model will stay in \033[93mVRAM(cuda)\033[0m" + ) + await setup() + else: + # Handle the case where CUDA is not available + print(f"[{params['branding']}Model] \033[91mError:\033[0m Nvidia CUDA is not available on this system. Unable to use LowVRAM mode.") + params["low_vram"] = False return Response(content=json.dumps({"status": "lowvram-success"})) except Exception as e: return Response(content=json.dumps({"status": "error", "message": str(e)})) - ################### #### DeepSpeed #### ################### @@ -393,7 +399,6 @@ async def deepspeed(request: Request, new_deepspeed_value: bool): except Exception as e: return Response(content=json.dumps({"status": "error", "message": str(e)})) - ######################## #### TTS GENERATION #### ######################## @@ -718,6 +723,7 @@ simple_webpage = """

Low VRAM Overview:

The Low VRAM option is a crucial feature designed to enhance performance under constrained Video Random Access Memory (VRAM) conditions, as the TTS models require 2GB-3GB of VRAM to run effectively. This feature strategically manages the relocation of the Text-to-Speech (TTS) model between your system's Random Access Memory (RAM) and VRAM, moving it between the two on the fly.

+

Note: An Nvidia Graphics card is required for LowVRAM, as you will be using system memory for the models otherwise.

How It Works:

The Low VRAM option intelligently orchestrates the relocation of the entire TTS model. When the TTS engine requires VRAM for processing, the entire model seamlessly moves into VRAM, causing your LLM to unload/displace some layers, ensuring optimal performance of the TTS engine.

@@ -730,6 +736,7 @@ simple_webpage = """

What's DeepSpeed?

DeepSpeed, developed by Microsoft, is like a speed boost for Text-to-Speech (TTS) tasks. It's all about making TTS happen faster and more efficiently.

+

Note: An Nvidia Graphics card is required for DeepSpeed

How Does It Speed Things Up?