From 1bd8f6aed9cd0bc4c95d751332be6eddc0e3aa5a Mon Sep 17 00:00:00 2001
From: erew123 <35898566+erew123@users.noreply.github.com>
Date: Fri, 15 Nov 2024 00:21:40 +0000
Subject: [PATCH] New Finetune for XTTS
Interface cleaned
Documentation improved & expanded
Dataset generation improved:
- Better VAD detection
- Better debugging/information
- Better editing/managing transcriptions
- Basic audio editing
- Re-transcription on detections of duplicate audio when splitting large files.
- Control over model precision fp16/fp32/mixed
- Migrate to openai-whisper (also saving 1.5GB on installation requirements for cublas)
Testing stage allows you to choose what minimum length audio
Model export exports all audio with text file descriptions of too long/too short
Misc other improvements too numerous to mention
---
finetune.py | 3888 ++++++++++++++++++++++++++++++------------
training_metrics.png | Bin 0 -> 48489 bytes
2 files changed, 2818 insertions(+), 1070 deletions(-)
create mode 100644 training_metrics.png
diff --git a/finetune.py b/finetune.py
index f085dbc..bb8068c 100644
--- a/finetune.py
+++ b/finetune.py
@@ -8,7 +8,6 @@ import glob
import json
import site
import datetime
-
import torch
import signal
import random
@@ -19,16 +18,22 @@ import tempfile
import platform
import argparse
import torchaudio
+import torch
import traceback
+import numpy as np
import gradio as gr
import pandas as pd
from word2number import w2n
from tqdm import tqdm
+from pynvml import nvmlInit, nvmlShutdown, nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo
from pathlib import Path
-from faster_whisper import WhisperModel
+from trainer import TrainerArgs, Trainer
+from trainer_alltalk.trainer import Trainer
+from TTS.config.shared_configs import BaseDatasetConfig
+from TTS.tts.datasets import load_tts_samples
+from TTS.tts.layers.xtts.trainer.gpt_trainer import GPTArgs, GPTTrainer, GPTTrainerConfig, XttsAudioConfig
from TTS.tts.configs.xtts_config import XttsConfig
from TTS.tts.models.xtts import Xtts
-
from metrics_logger import MetricsLogger
# Use a local Tokenizer to resolve Japanese support
# from TTS.tts.layers.xtts.tokenizer import multilingual_cleaners
@@ -37,6 +42,29 @@ import importlib.metadata as metadata
from packaging import version
from tokenizers import ByteLevelBPETokenizer
from tokenizers.pre_tokenizers import Whitespace
+########################################################################################
+# START-UP # Silence RVC warning about torch.nn.utils.weight_norm even though not used #
+########################################################################################
+import warnings
+# Filter Flash Attention warning
+warnings.filterwarnings("ignore", message="1Torch was not compiled with flash attention")
+warnings.filterwarnings("ignore", message="Failed to launch Triton kernels, likely due to missing CUDA toolkit")
+os.environ["GRADIO_ANALYTICS_ENABLED"] = "False"
+
+# Try to import whisper, install if not found
+try:
+ import whisper
+except ImportError:
+ print("[FINETUNE] OpenAI Whisper not found. Attempting to install...")
+ try:
+ subprocess.check_call([sys.executable, "-m", "pip", "install", "openai-whisper"])
+ import whisper
+ print("[FINETUNE] Successfully installed OpenAI Whisper!. Continuing.")
+ except Exception as e:
+ print("[FINETUNE] Failed to install OpenAI Whisper:")
+ print(f"[FINETUNE] Error: {str(e)}")
+ print("[FINETUNE] Please try manually installing with: pip install openai-whisper")
+ sys.exit(1)
# STARTUP VARIABLES
this_dir = Path(__file__).parent.resolve()
@@ -62,6 +90,88 @@ validate_audio_folder = None
validate_whisper_model = None
validate_target_language = None
+##############################
+#### Debugging management ####
+##############################
+class DebugLevels:
+ GPU_MEMORY = False # GPU memory and CUDA related debugging
+ MODEL_OPS = True # Model loading, transcription, cleanup operations
+ DATA_PROCESS = False # Data processing, words, sentences
+ GENERAL = True # General flow, file operations, metadata
+ AUDIO = False # Audio processing statistics and info
+ SEGMENTS = False # Detailed segment information
+ DUPLICATES = False # Duplicate handling information
+
+def debug_print(message, level, is_error=False, is_warning=False):
+ """Enhanced debug printing with categorization and formatting"""
+ prefix = "[FINETUNE]"
+ if is_error:
+ prefix += " ERROR:" # Was red colored emoji
+ elif is_warning:
+ prefix += " WARNING:" # Was yellow colored emoji
+
+ if level == "GPU_MEMORY" and DebugLevels.GPU_MEMORY:
+ print(f"{prefix} [GPU] {message}")
+ elif level == "MODEL_OPS" and DebugLevels.MODEL_OPS:
+ print(f"{prefix} [MODEL] {message}")
+ elif level == "DATA_PROCESS" and DebugLevels.DATA_PROCESS:
+ print(f"{prefix} [DATA] {message}")
+ elif level == "GENERAL" and DebugLevels.GENERAL:
+ print(f"{prefix} [INFO] {message}")
+ elif level == "AUDIO" and DebugLevels.AUDIO:
+ print(f"{prefix} [AUDIO] {message}")
+ elif level == "SEGMENTS" and DebugLevels.SEGMENTS:
+ print(f"{prefix} [SEG] {message}")
+ elif level == "DUPLICATES" and DebugLevels.DUPLICATES:
+ print(f"{prefix} [DUP] {message}")
+
+class AudioStats:
+ """Track audio processing statistics"""
+ def __init__(self):
+ self.total_segments = 0
+ self.segments_under_min = 0
+ self.segments_over_max = 0
+ self.total_duration = 0
+ self.segment_durations = []
+
+ def add_segment(self, duration):
+ self.total_segments += 1
+ self.total_duration += duration
+ self.segment_durations.append(duration)
+
+ def print_stats(self):
+ if not self.segment_durations:
+ return
+
+ avg_duration = self.total_duration / self.total_segments
+ debug_print(f"Audio Processing Statistics:", "AUDIO")
+ debug_print(f"Total segments: {self.total_segments}", "AUDIO")
+ debug_print(f"Average duration: {avg_duration:.2f}s", "AUDIO")
+ debug_print(f"Segments under minimum: {self.segments_under_min}", "AUDIO")
+ debug_print(f"Segments over maximum: {self.segments_over_max}", "AUDIO")
+
+ if self.segments_under_min > 0:
+ debug_print(f"{self.segments_under_min} segments are under minimum duration!", "AUDIO", is_warning=True)
+
+def get_gpu_memory():
+ """Enhanced GPU memory reporting"""
+ if DebugLevels.GPU_MEMORY:
+ try:
+ nvmlInit()
+ handle = nvmlDeviceGetHandleByIndex(0)
+ info = nvmlDeviceGetMemoryInfo(handle)
+ debug_print("GPU Memory Status:", "GPU_MEMORY")
+ debug_print(f"Total: {info.total / 1024**2:.2f} MB", "GPU_MEMORY")
+ debug_print(f"Used: {info.used / 1024**2:.2f} MB", "GPU_MEMORY")
+ debug_print(f"Free: {info.free / 1024**2:.2f} MB", "GPU_MEMORY")
+
+ # Add warning if memory is low
+ if info.free / info.total < 0.1: # Less than 10% free
+ debug_print("Low GPU memory available!", "GPU_MEMORY", is_warning=True)
+ except Exception as e:
+ debug_print(f"NVML not available: {e}", "GPU_MEMORY", is_error=True)
+
+
########################
#### Find All Model ####
########################
@@ -95,205 +205,226 @@ def scan_models_folder():
# Get available models
available_models = scan_models_folder()
-#######################
-#### DIAGS for PFC ####
-#######################
+#################################################
+#### Pre-Flight Checklist Functions & Gradio ####
+#################################################
-def check_disk_space():
- global pfc_status
- # Get the current working directory
- current_directory = os.getcwd()
- # Get the disk usage statistics for the current directory's disk
- disk_usage = shutil.disk_usage(current_directory)
- # Convert the free space to GB (1GB = 1 << 30 bytes)
- free_space_gb = disk_usage.free / (1 << 30)
- # Check if the free space is more than 18GB
- is_more_than_18gb = free_space_gb > 18
- disk_space_icon = "✅"
- if not is_more_than_18gb:
- disk_space_icon ="❌"
- pfc_status = "fail" # Update global status if disk space check fails
- # Generating the markdown text for disk space check
- disk_space_markdown = f"""
- ### 🟩 Disk Space Check
- {disk_space_icon} **Disk Space (> 18 GB):** {'' if is_more_than_18gb else 'You have less than 18GB on this disk '} {free_space_gb:.2f} GB
- """
- return disk_space_markdown
-
-def test_cuda():
- global pfc_status
- cuda_home = os.environ.get('CUDA_HOME', 'N/A')
- cuda_available = torch.cuda.is_available()
- if cuda_available:
+class SystemChecks:
+ """Centralized system check management"""
+ def __init__(self):
+ self.status = {
+ "overall": True,
+ "disk_space": True,
+ "ram": True,
+ "gpu": True,
+ "cuda": True,
+ "pytorch": True,
+ "tts": True,
+ "base_model": True
+ }
+ self.results = {}
+
+ def check_disk_space(self, required_gb=18):
+ """Check available disk space"""
try:
- # Attempt to create a tensor on GPU
- torch.tensor([1.0, 2.0]).cuda()
- cuda_status = "CUDA is available and working."
- cuda_icon = "✅"
+ disk_usage = shutil.disk_usage(os.getcwd())
+ free_space_gb = disk_usage.free / (1 << 30)
+
+ status = free_space_gb > required_gb
+ self.status["disk_space"] = status
+
+ self.results["disk_space"] = {
+ "status": "✅ Pass" if status else "❌ Fail",
+ "details": f"{free_space_gb:.2f} GB available",
+ "icon": "✅" if status else "❌",
+ "message": ("Sufficient disk space available" if status
+ else f"Insufficient disk space. Need {required_gb}GB, have {free_space_gb:.2f}GB")
+ }
except Exception as e:
- cuda_status = f"CUDA is available but not working. Error: {e}"
- cuda_icon = "❌"
- pfc_status = "fail" # Update global status
- else:
- cuda_status = "CUDA is not available."
- pfc_status = "fail" # Update global status
- return cuda_status, cuda_icon, cuda_home
+ self.handle_check_error("disk_space", str(e))
+
+ def check_system_ram(self):
+ """Check system RAM and GPU memory with refined thresholds"""
+ try:
+ vm = psutil.virtual_memory()
+ total_ram_gb = vm.total / (1024 ** 3)
+ available_ram_gb = vm.available / (1024 ** 3)
+
+ # RAM check
+ ram_status = available_ram_gb >= 8
+ self.status["ram"] = ram_status
+
+ # GPU memory check
+ if torch.cuda.is_available():
+ gpu_id = torch.cuda.current_device()
+ gpu_name = torch.cuda.get_device_name(gpu_id)
+ total_gpu_mem = torch.cuda.get_device_properties(gpu_id).total_memory / (1024 ** 3)
+ used_gpu_mem = torch.cuda.memory_reserved(gpu_id) / (1024 ** 3)
+ available_gpu_mem = total_gpu_mem - used_gpu_mem
+
+ # New GPU status logic with better thresholds
+ if total_gpu_mem >= 13:
+ gpu_status = "pass" # More than 13GB - Clear pass
+ elif total_gpu_mem >= 11.5:
+ gpu_status = "warning" # Between 11.5GB and 13GB - Warning
+ else:
+ gpu_status = "fail" # Less than 11.5GB - Fail
+
+ self.status["gpu"] = gpu_status in ["pass", "warning"] # Both pass and warning are considered "okay"
+ else:
+ gpu_name = "No GPU detected"
+ total_gpu_mem = 0
+ available_gpu_mem = 0
+ gpu_status = "fail"
+ self.status["gpu"] = False
+
+ self.results["memory"] = {
+ "ram_status": "✅ Pass" if ram_status else "❌ Fail",
+ "ram_details": f"{available_ram_gb:.2f}GB available of {total_ram_gb:.2f}GB",
+ "gpu_status": {
+ "pass": "✅ Pass",
+ "warning": "⚠️ Warning",
+ "fail": "❌ Fail"
+ }[gpu_status],
+ "gpu_details": f"{gpu_name}: {available_gpu_mem:.2f}GB available of {total_gpu_mem:.2f}GB"
+ }
+ except Exception as e:
+ self.handle_check_error("memory", str(e))
+
+ def check_cuda_pytorch(self):
+ """Check CUDA and PyTorch setup"""
+ try:
+ cuda_available = torch.cuda.is_available()
+ pytorch_version = torch.__version__
+
+ if cuda_available:
+ cuda_version = torch.version.cuda
+
+ # Check if CUDA is actually working
+ try:
+ torch.tensor([1.0, 2.0]).cuda()
+ cuda_working = True
+ except:
+ cuda_working = False
+
+ pytorch_cuda_status = cuda_version in ['11.8', '12.1']
+ else:
+ cuda_working = False
+ cuda_version = "N/A"
+ pytorch_cuda_status = False
+
+ self.status["cuda"] = cuda_working
+ self.status["pytorch"] = pytorch_cuda_status
+
+ # Separate results for CUDA and PyTorch
+ self.results["cuda"] = {
+ "status": "✅ Pass" if cuda_working else "❌ Fail",
+ "details": f"CUDA {cuda_version}" if cuda_working else "CUDA not working"
+ }
+
+ self.results["pytorch"] = {
+ "status": "✅ Pass" if pytorch_cuda_status else "❌ Fail",
+ "details": f"PyTorch {pytorch_version} with CUDA {cuda_version}",
+ }
+ except Exception as e:
+ self.handle_check_error("cuda_pytorch", str(e))
+
+ def check_tts_version(self, required_version="0.24.0"):
+ """Check TTS version"""
+ try:
+ installed_version = metadata.version("coqui-tts")
+ status = version.parse(installed_version) >= version.parse(required_version)
+
+ self.status["tts"] = status
+ self.results["tts"] = {
+ "status": "✅ Pass" if status else "❌ Fail",
+ "details": f"TTS version {installed_version} installed",
+ "meets_requirement": status
+ }
+ except Exception as e:
+ self.handle_check_error("tts", str(e))
+
+ def check_base_model(self):
+ """Check XTTS base model"""
+ try:
+ base_model_detected = any(available_models.values()) if available_models else False
+
+ self.status["base_model"] = base_model_detected
+ self.results["base_model"] = {
+ "status": "✅ Pass" if base_model_detected else "❌ Fail",
+ "details": "Base model detected" if base_model_detected else "No base model found"
+ }
+ except Exception as e:
+ self.handle_check_error("base_model", str(e))
+
+ def handle_check_error(self, check_name: str, error_msg: str):
+ """Handle errors in checks"""
+ self.status[check_name] = False
+ self.status["overall"] = False
+ self.results[check_name] = {
+ "status": "❌ Error",
+ "details": f"Check failed: {error_msg}"
+ }
+
+ def run_all_checks(self):
+ """Run all system checks"""
+ self.check_disk_space()
+ self.check_system_ram()
+ self.check_cuda_pytorch()
+ self.check_tts_version()
+ self.check_base_model()
+
+ # Update overall status
+ self.status["overall"] = all(self.status.values())
+ return self.status["overall"]
+
+ def get_markdown_report(self):
+ """Generate markdown report of all checks"""
+ report = []
+
+ # Overall Status
+ overall_icon = "✅" if self.status["overall"] else "❌"
+ report.append(f"## System Check Results {overall_icon}\n")
+
+ # Individual Checks
+ if "disk_space" in self.results:
+ report.append(f"### Storage\n{self.results['disk_space']['status']} {self.results['disk_space']['details']}\n")
+
+ if "memory" in self.results:
+ report.append("### Memory")
+ report.append(f"- RAM: {self.results['memory']['ram_status']} {self.results['memory']['ram_details']}")
+ report.append(f"- GPU: {self.results['memory']['gpu_status']} {self.results['memory']['gpu_details']}\n")
+
+ if "cuda_pytorch" in self.results:
+ report.append(f"### CUDA & PyTorch\n{self.results['cuda_pytorch']['status']} {self.results['cuda_pytorch']['details']}\n")
+
+ if "tts" in self.results:
+ report.append(f"### TTS\n{self.results['tts']['status']} {self.results['tts']['details']}\n")
+
+ if "base_model" in self.results:
+ report.append(f"### Base Model\n{self.results['base_model']['status']} {self.results['base_model']['details']}\n")
+
+ return "\n".join(report)
-def find_files_in_path_with_wildcard(pattern):
- # Get the site-packages directory of the current Python environment
- site_packages_path = site.getsitepackages()
- found_paths = []
- # Adjust the sub-directory based on the operating system
- sub_directory = "nvidia/cublas"
- if platform.system() == "Linux":
- sub_directory = os.path.join(sub_directory, "lib")
- else:
- sub_directory = os.path.join(sub_directory, "bin")
- # Iterate over each site-packages directory (there can be more than one)
- for directory in site_packages_path:
- # Construct the search directory path
- search_directory = os.path.join(directory, sub_directory)
- # Use glob to find all files matching the pattern in this directory
- for file_path in glob.glob(os.path.join(search_directory, pattern)):
- if os.path.isfile(file_path): # Ensure it's a file
- found_paths.append(file_path)
- return found_paths
+ def get_ui_updates(self, components):
+ """Map check results to UI components"""
+ # Get status for all checks
+ status_updates = []
+
+ # Overall status first
+ overall_status = "✅ All Systems Go!" if self.status["overall"] else "⚠️ Some Checks Failed"
+ status_updates.append(overall_status)
+
+ # Add status for each check in the order they appear in CHECK_REQUIREMENTS
+ for check_id in CHECK_REQUIREMENTS.keys():
+ if check_id in self.results:
+ status_updates.append(self.results[check_id]["status"])
+ else:
+ status_updates.append("❌ Check not run")
+
+ return status_updates
-def test_cuda():
- if torch.cuda.is_available():
- pytorch_cuda_version = torch.version.cuda
- return f'CUDA {pytorch_cuda_version} is available in this Python environment', '✅', torch.cuda.get_device_name(0)
- else:
- return 'CUDA is not available in this Python environment', '❌', 'None'
-
-def generate_cuda_markdown():
- global pfc_status
- pfc_status = "pass" # Initialize global status as pass
-
- # Check CUDA availability and get CUDA home path
- cuda_status, cuda_icon, cuda_home = test_cuda()
-
- # Check for specific CUDA library files
- file_name = 'cublas64_11.*' if platform.system() == "Windows" else 'libcublas.so.11*'
- found_paths = find_files_in_path_with_wildcard(file_name)
- if found_paths:
- found_paths_str = ' '.join(str(path) for path in found_paths)
- found_path_icon = '✅'
- else:
- found_paths_str = "cublas64_11 is not accessible."
- found_path_icon = '❌'
- pfc_status = "fail" # Update global status
-
- # Check PyTorch version and CUDA version
- pytorch_version = torch.__version__
- pytorch_cuda_version = torch.version.cuda
-
- if pytorch_cuda_version in ['11.8', '12.1']:
- pytorch_cuda_version_status = ''
- pytorch_icon = '✅'
- else:
- pytorch_cuda_version_status = 'Pytorch CUDA version problem '
- pytorch_icon = '❌'
- pfc_status = "fail" # Update global status
-
- cuda_markdown = f"""
- ### 🟨 CUDA Information
- {found_path_icon} **Cublas64_11 found:** {found_paths_str}
- {pytorch_icon} **CUDA_HOME path:** {cuda_home} **(Will list your GPU if no specifc path set, which is ok)**
- """
- pytorch_markdown = f"""
- ### 🟦 Python & Pytorch Information
- {pytorch_icon} **PyTorch Version:** {pytorch_cuda_version_status} {torch.__version__}
- {cuda_icon} **CUDA is working:** {cuda_status}
- """
- return cuda_markdown, pytorch_markdown
-
-def get_system_ram_markdown():
- global pfc_status
- virtual_memory = psutil.virtual_memory()
- total_ram_gb = virtual_memory.total / (1024 ** 3)
- available_ram_gb = virtual_memory.available / (1024 ** 3)
- used_ram_percentage = virtual_memory.percent
-
- # Check if the available RAM is less than 8GB
- warning_if_low_ram = available_ram_gb < 8
-
- # Decide the message based on the available RAM
- ram_status_message = "Warning" if warning_if_low_ram else ""
- ram_status_icon = "⚠️" if warning_if_low_ram else "✅"
-
- if torch.cuda.is_available():
- gpu_device_id = torch.cuda.current_device()
- gpu_device_name = torch.cuda.get_device_name(gpu_device_id)
- # Get the total and available memory in bytes, then convert to GB
- gpu_total_mem_gb = torch.cuda.get_device_properties(gpu_device_id).total_memory / (1024 ** 3)
- # gpu_available_mem_gb = (torch.cuda.get_device_properties(gpu_device_id).total_memory - torch.cuda.memory_allocated(gpu_device_id)) / (1024 ** 3)
- # gpu_available_mem_gb = (torch.cuda.get_device_properties(gpu_device_id).total_memory - torch.cuda.memory_reserved(gpu_device_id)) / (1024 ** 3)
- gpu_reserved_mem_gb = torch.cuda.memory_reserved(gpu_device_id) / (1024 ** 3)
- gpu_available_mem_gb = gpu_total_mem_gb - gpu_reserved_mem_gb
- # Check if total or available memory is less than 11 GB and set icons
- gpu_total_status_icon = "⚠️" if gpu_total_mem_gb < 12 else "✅"
- gpu_available_status_icon = "⚠️" if gpu_available_mem_gb < 12 else "✅"
- gpu_status_icon = "✅"
- else:
- gpu_status_icon = "⚠️"
- gpu_device_name = "Cannot detect a CUDA card"
- gpu_total_mem_gb = "Cannot detect a CUDA card"
- gpu_available_mem_gb = "Cannot detect a CUDA card"
- gpu_total_status_icon = gpu_status_icon
- gpu_available_status_icon = gpu_status_icon
-
- system_ram_markdown = f"""
- ### 🟪 System RAM and VRAM Information
- {ram_status_icon} **Total RAM:** {total_ram_gb:.2f} GB
- {ram_status_icon} **Available RAM:** {ram_status_message + ' - Available RAM is less than 8 GB. You have ' if warning_if_low_ram else ''} {available_ram_gb:.2f} GB available ({used_ram_percentage:.2f}% used)
- {gpu_status_icon} **GPU Name:** {gpu_device_name}
- {gpu_total_status_icon} **GPU Total RAM:** {gpu_total_mem_gb:.2f} GB
- {gpu_available_status_icon} **GPU Available RAM:** {gpu_available_mem_gb:.2f} GB
- """
- return system_ram_markdown
-
-def generate_base_model_markdown(base_model_detected):
- global pfc_status
- base_model_status = 'Base model detected' if base_model_detected else 'Base model not detected'
- base_model_icon = '✅' if base_model_detected else '❌'
- base_model_markdown = f"""
- ### ⬛ XTTS Base Model Detection
- {base_model_icon} **Base XTTS Model Status:** {base_model_status}
- """
- return base_model_markdown
-
-def check_tts_version(required_version="0.24.0"):
- global pfc_status
- try:
- # Get the installed version of TTS
- installed_version = metadata.version("coqui-tts")
- # Check if the installed version meets the required version
- if version.parse(installed_version) >= version.parse(required_version):
- tts_status = f"TTS version {installed_version} is installed and meets the requirement."
- tts_status_icon = "✅"
- else:
- tts_status = f"❌ Fail - TTS version {installed_version} is installed but does not meet the required version {required_version}."
- tts_status_icon = "❌"
- pfc_status = "fail" # Update global status
- except metadata.PackageNotFoundError:
- # If TTS is not installed
- tts_status = "TTS is not installed."
- pfc_status = "fail" # Update global status
- tts_markdown = f"""
- ### 🟥 TTS Information
- {tts_status_icon} **TTS Version:** {tts_status}
- """
- return tts_markdown
-
-# Disk space check results to append to the Markdown
-disk_space_results = check_disk_space()
-cuda_results, pytorch_results = generate_cuda_markdown()
-system_ram_results = get_system_ram_markdown()
-base_model_results = generate_base_model_markdown(base_model_detected)
-tts_version_status = check_tts_version()
-
def pfc_check_fail():
global pfc_status
if pfc_status == "fail":
@@ -304,43 +435,295 @@ def pfc_check_fail():
print("[FINETUNE]")
return
-#####################
-#### STEP 1 BITS ####
-#####################
+# PFC Help Content and Requirements
+HELP_CONTENT = {
+ "disk_space_details": """
+### Disk Space Requirements
-def create_temp_folder():
- temp_folder = os.path.join(os.path.dirname(__file__), 'temp_files')
- os.makedirs(temp_folder, exist_ok=True)
- return temp_folder
+#### Minimum Requirements
+- **18GB** free disk space for training
+- SSD recommended for better performance
-def create_temporary_file(folder, suffix=".wav"):
- unique_filename = f"custom_tempfile_{int(time.time())}_{random.randint(1, 1000)}{suffix}"
- return os.path.join(folder, unique_filename)
+#### Space Usage Breakdown
+- Model files: ~4GB
+- Training data: ~5-10GB
+- Temporary files: ~4GB
+- Safety margin: ~2GB
+
+#### Troubleshooting
+1. **Insufficient Space**
+ - Clear space on the current partition
+ - Run fine-tuning on another partition with more space
+ - Keep at least 20% free space on drive
+
+2. **Performance Issues**
+ - Mechanical HDDs will be significantly slower
+ - Consider using an SSD for training
+""",
+
+ "gpu_requirements": """
+### GPU Memory Requirements
+
+#### System Requirements
+- **NVIDIA GPU Required**
+- **12GB+ VRAM** recommended for optimal performance
+- **8GB+ VRAM** minimum requirement
+
+#### Platform Differences
+- **Windows Systems:**
+ * Can use system RAM as extended VRAM
+ * 12GB VRAM or less requires 24GB+ System RAM and may fail.
+ * 12GB VRAM cards will have a warning because of the above.
+ * Performance may vary when using extended VRAM
+
+- **Linux Systems:**
+ * Cannot use extended VRAM
+ * Limited to physical GPU VRAM only
+ * Minimum 12GB VRAM recommended
+""",
+
+ "pytorch_cuda": """
+### PyTorch and CUDA
+
+- CUDA-enabled PyTorch installation required
+- Compatible with CUDA 11.8/12.1
+- Current PyTorch version recommended
+""",
+
+ "model_setup": """
+### Base Model Requirements
+
+#### Model Installation
+- Download XTTS v2.0.3 model (recommended)
+- BPE Tokenize needs a v2.0.3 model
+- Use AllTalk's main interface:
+ * TTS Engine Settings > XTTS > Model/Voices Download
+
+#### Model Location
+- Place in `/models/xtts/{modelname}/`
+- Required files:
+ * model.pth
+ * config.json
+ * vocab.json
+ * dvae.pth
+ * mel_stats.pth
+ * speakers_xtts.pth
+"""
+}
+
+CHECK_REQUIREMENTS = {
+ "disk_space": {
+ "label": "Disk Space",
+ "requirement": "18GB+ Required",
+ "description": "Required for temporary files and model storage",
+ "priority": "high",
+ "check_function": "check_disk_space"
+ },
+ "ram": {
+ "label": "System RAM",
+ "requirement": "16GB+ Recommended",
+ "description": "Affects overall processing speed and stability",
+ "priority": "medium",
+ "check_function": "check_system_ram"
+ },
+ "vram": {
+ "label": "GPU VRAM",
+ "requirement": "12GB+ Recommended",
+ "description": "Crucial for model training performance",
+ "priority": "high",
+ "check_function": "check_gpu_memory"
+ },
+ "cuda": {
+ "label": "CUDA Support",
+ "requirement": "CUDA 11.8/12.1",
+ "description": "Required for GPU acceleration",
+ "priority": "high",
+ "check_function": "check_cuda_pytorch"
+ },
+ "pytorch": {
+ "label": "PyTorch",
+ "requirement": "CUDA enabled",
+ "description": "Must be CUDA compatible version",
+ "priority": "high",
+ "check_function": "check_cuda_pytorch"
+ },
+ "tts": {
+ "label": "TTS Version",
+ "requirement": "0.24.0+",
+ "description": "Required for fine-tuning",
+ "priority": "high",
+ "check_function": "check_tts_version"
+ },
+ "base_model": {
+ "label": "Base Model",
+ "requirement": "Must be present",
+ "description": "XTTS base model required",
+ "priority": "high",
+ "check_function": "check_base_model"
+ }
+}
+
+# Display categories for organizing checks
+CHECK_CATEGORIES = {
+ "hardware": {
+ "title": "💻 Hardware Requirements",
+ "description": "System hardware capabilities",
+ "checks": ["disk_space", "ram", "vram"]
+ },
+ "cuda": {
+ "title": "🔧 CUDA Setup",
+ "description": "CUDA and PyTorch configuration",
+ "checks": ["cuda", "pytorch"]
+ },
+ "software": {
+ "title": "🤖 Software Setup",
+ "description": "Required software and models",
+ "checks": ["tts", "base_model"]
+ }
+}
+
+class PFCComponents:
+ """Store all PFC UI components"""
+ def __init__(self):
+ self.overall_status = None
+ self.status_boxes = {}
+ self.accordions = {}
+
+ def create_status_box(self, check_id):
+ """Create a status box for a specific check"""
+ check_info = CHECK_REQUIREMENTS[check_id]
+ with gr.Group():
+ status = gr.Label(
+ label=check_info["label"],
+ value="Ready to Check",
+ elem_classes="status-indicator"
+ )
+ gr.Markdown(f"*Required: {check_info['requirement']}*")
+ self.status_boxes[check_id] = status
+ return status
+
+def create_pfc_interface():
+ components = PFCComponents()
+ system_checks = SystemChecks()
+
+ with gr.Column():
+ # Status and button at top
+ with gr.Row():
+ components.overall_status = gr.Label(
+ value="Click 'Run System Checks' to begin",
+ label="System Status",
+ elem_classes="status-label",
+ scale=2
+ )
+ with gr.Row():
+ refresh_btn = gr.Button("Run System Checks", scale=1)
+
+ # Checks in 3x2 grid
+ with gr.Row():
+ for check_id in CHECK_REQUIREMENTS.keys():
+ with gr.Column(scale=1):
+ components.create_status_box(check_id)
+
+ # Help sections in 2x2 grid
+ with gr.Group("💡 Help & Troubleshooting"):
+ with gr.Row():
+ with gr.Column():
+ with gr.Accordion("💻 Hardware Requirements", open=False):
+ gr.Markdown(HELP_CONTENT["gpu_requirements"])
+
+ with gr.Column():
+ with gr.Accordion("🖥️ Storage Requirements", open=False):
+ gr.Markdown(HELP_CONTENT["disk_space_details"])
+
+ with gr.Row():
+ with gr.Column():
+ with gr.Accordion("🤖 PyTorch & CUDA", open=False):
+ gr.Markdown(HELP_CONTENT["pytorch_cuda"])
+
+ with gr.Column():
+ with gr.Accordion("📦 Model Setup", open=False):
+ gr.Markdown(HELP_CONTENT["model_setup"])
+
+ def run_checks():
+ system_checks.run_all_checks()
+
+ # Prepare updates for all components
+ updates = []
+
+ # Overall status
+ updates.append("✅ All Systems Go!" if system_checks.status["overall"] else "⚠️ Some Checks Failed")
+
+ # Add updates for each category's checks
+ for category in CHECK_CATEGORIES.values():
+ for check_id in category["checks"]:
+ if check_id == "ram":
+ updates.append(system_checks.results["memory"]["ram_status"])
+ elif check_id == "vram":
+ updates.append(system_checks.results["memory"]["gpu_status"])
+ elif check_id == "cuda":
+ updates.append(system_checks.results["cuda"]["status"])
+ elif check_id == "pytorch":
+ updates.append(system_checks.results["pytorch"]["status"])
+ else:
+ updates.append(system_checks.results.get(check_id, {}).get("status", "❌ Check not run"))
+
+ return updates
+
+ refresh_btn.click(
+ fn=run_checks,
+ outputs=[components.overall_status] + list(components.status_boxes.values())
+ )
+
+ return components
+###########################################
+#### STEP 1 Dataset Creation Functions ####
+###########################################
-def format_audio_list(target_language, whisper_model, max_sample_length, eval_split_number, speaker_name_input, create_bpe_tokenizer, gradio_progress=gr.Progress()):
- global validate_train_metadata_path, validate_eval_metadata_path, validate_audio_folder, validate_whisper_model, validate_target_language
+def format_audio_list(target_language, whisper_model, max_sample_length, eval_split_number, speaker_name_input, create_bpe_tokenizer, gradio_progress=gr.Progress(), use_vad=True, precision="mixed", min_sample_length=6):
+ """
+ Enhanced audio processing function with VAD and precision options
+ Args:
+ precision: One of "mixed" (default), "float16", or "float32"
+ use_vad: Whether to use SileroVAD for better speech detection
+ min_sample_length: Minimum audio segment length in seconds (default 6s)
+ """
+ global validate_train_metadata_path, validate_eval_metadata_path, validate_audio_folder
+ global validate_whisper_model, validate_target_language, out_path, torch, whisper
+
+ # Clear down the finetune.log file
+ Logger().clear_log()
+
+ # Initialize statistics tracker
+ stats = AudioStats()
+
+ # Basic setup
buffer = 0.3
- max_duration = float(max_sample_length) # Ensure max_duration is a float
+ max_duration = float(max_sample_length)
+ min_duration = float(min_sample_length)
eval_percentage = eval_split_number / 100.0
speaker_name = speaker_name_input
audio_total_size = 0
- global out_path
+ metadata = {"audio_file": [], "text": [], "speaker_name": []}
+ too_long_files = []
+
+ # Initialize directories
if speaker_name and speaker_name != 'personsname':
out_path = this_dir / "finetune" / speaker_name
else:
out_path = default_path
+ debug_print(f"Initializing output directory: {out_path}", "GENERAL")
+
os.makedirs(out_path, exist_ok=True)
temp_folder = os.path.join(out_path, "temp")
os.makedirs(temp_folder, exist_ok=True)
audio_folder = os.path.join(out_path, "wavs")
os.makedirs(audio_folder, exist_ok=True)
original_samples_folder = os.path.join(out_path, "..", "put-voice-samples-in-here")
- print("[FINETUNE] Preparing Audio/Generating the dataset")
- # Write the target language to lang.txt in the output directory
+ # Handle language file
lang_file_path = os.path.join(out_path, "lang.txt")
current_language = None
if os.path.exists(lang_file_path):
@@ -350,65 +733,95 @@ def format_audio_list(target_language, whisper_model, max_sample_length, eval_sp
if current_language != target_language:
with open(lang_file_path, 'w', encoding='utf-8') as lang_file:
lang_file.write(target_language + '\n')
- print("[FINETUNE] Updated lang.txt with the target language.")
+ debug_print(f"Updated language to: {target_language}", "GENERAL")
else:
- print("[FINETUNE] The existing language matches the target language")
+ debug_print("Using existing language setting", "GENERAL")
- gradio_progress((1, 4), desc="Loading in Whisper")
+ # Load Whisper model with specified precision
+ gradio_progress((1, 10), desc="Loading Whisper Model")
device = "cuda" if torch.cuda.is_available() else "cpu"
- asr_model = WhisperModel(whisper_model, device=device, compute_type="float32")
+ debug_print(f"Using device: {device}", "MODEL_OPS")
+
+ # Monitor GPU memory before model loading
+ get_gpu_memory()
+
+ debug_print(f"Loading Whisper model: {whisper_model}", "MODEL_OPS")
+ asr_model = whisper.load_model(whisper_model, device=device)
+
+ if precision == "float16" and device == "cuda":
+ debug_print("Using FP16 precision", "MODEL_OPS")
+ asr_model = asr_model.half()
+ elif precision == "mixed" and device == "cuda":
+ debug_print("Using mixed precision", "MODEL_OPS")
+ else:
+ debug_print("Using FP32 precision", "MODEL_OPS")
- metadata = {"audio_file": [], "text": [], "speaker_name": []}
+ # Initialize SileroVAD if requested
+ vad_model = None
+ if use_vad:
+ gradio_progress((2, 10), desc="Loading VAD Model")
+ debug_print("Initializing Silero VAD", "MODEL_OPS")
+ model, utils = torch.hub.load(repo_or_dir='snakers4/silero-vad',
+ model='silero_vad',
+ force_reload=False)
+ vad_model = model.to(device)
+ get_speech_timestamps, collect_chunks = utils[0], utils[4]
- existing_metadata = {'train': None, 'eval': None}
+ # Monitor GPU memory after model loading
+ get_gpu_memory()
+
+ # Load existing metadata
+ gradio_progress((3, 10), desc="Checking for Existing Metadata")
train_metadata_path = os.path.join(out_path, "metadata_train.csv")
eval_metadata_path = os.path.join(out_path, "metadata_eval.csv")
+ existing_metadata = {'train': None, 'eval': None}
if os.path.exists(train_metadata_path):
existing_metadata['train'] = pd.read_csv(train_metadata_path, sep="|")
- print("[FINETUNE] Existing training metadata found and loaded.")
+ debug_print("Loaded existing training metadata", "DATA_PROCESS")
if os.path.exists(eval_metadata_path):
existing_metadata['eval'] = pd.read_csv(eval_metadata_path, sep="|")
- print("[FINETUNE] Existing evaluation metadata found and loaded.")
+ debug_print("Loaded existing evaluation metadata", "DATA_PROCESS")
- # List to store information about files that are too long
- too_long_files = []
-
- # Process original voice samples to create WAV files
- original_audio_files = [os.path.join(original_samples_folder, file) for file in os.listdir(original_samples_folder) if file.endswith(('.mp3', '.flac', '.wav'))]
+ # Get audio files list
+ original_audio_files = [os.path.join(original_samples_folder, file)
+ for file in os.listdir(original_samples_folder)
+ if file.endswith(('.mp3', '.flac', '.wav'))]
+ gradio_progress((4, 10), desc="Scanning for Audio Files")
if not original_audio_files:
- print(f"[FINETUNE] No audio files found in {original_samples_folder}. Skipping processing.")
+ debug_print(f"No audio files found in {original_samples_folder}", "AUDIO", is_error=True)
return None, None, 0
+ debug_print(f"Found {len(original_audio_files)} audio files to process", "AUDIO")
+ # Initialize processing
whisper_words = []
audio_steps = (0, len(original_audio_files))
gradio_progress_duration = 0
gradio_progress(audio_steps, desc="Processing Audio Files", unit="files")
- for audio_path in tqdm(original_audio_files): # No start argument for tqdm
+ for audio_path in original_audio_files:
start = datetime.datetime.now()
audio_file_name_without_ext, _ = os.path.splitext(os.path.basename(audio_path))
temp_audio_path = os.path.join(temp_folder, f"{audio_file_name_without_ext}.wav")
try:
shutil.copy2(audio_path, temp_audio_path)
+ gradio_progress(audio_steps, desc=f"Processing {audio_file_name_without_ext}", unit="files")
+ debug_print(f"Processing: {audio_file_name_without_ext}", "GENERAL")
except Exception as e:
- print(f"[FINETUNE] Error copying file: {e}")
+ debug_print(f"Error copying file {audio_path}: {str(e)}", "GENERAL", is_error=True)
continue
- wav, sr = torchaudio.load(temp_audio_path, format="wav")
- wav = torch.as_tensor(wav).clone().detach().t().to(torch.float32), sr
-
+ # Check if already processed
prefix_check = f"wavs/{audio_file_name_without_ext}_"
skip_processing = False
-
for key in ['train', 'eval']:
if existing_metadata[key] is not None:
mask = existing_metadata[key]['audio_file'].str.startswith(prefix_check)
if mask.any():
- print(f"\n[FINETUNE] Segments from {audio_file_name_without_ext} have been previously processed; skipping...")
+ debug_print(f"Skipping previously processed file: {audio_file_name_without_ext}", "GENERAL")
skip_processing = True
audio_total_size = 121
break
@@ -416,94 +829,115 @@ def format_audio_list(target_language, whisper_model, max_sample_length, eval_sp
if skip_processing:
continue
+ # Load and process audio
wav, sr = torchaudio.load(audio_path)
if wav.size(0) != 1:
wav = torch.mean(wav, dim=0, keepdim=True)
-
wav = wav.squeeze()
audio_total_size += (wav.size(-1) / sr)
+
+ original_duration = wav.size(-1) / sr
+ debug_print(f"Original audio duration: {original_duration:.2f}s", "AUDIO")
- segments, _ = asr_model.transcribe(audio_path, vad_filter=True, word_timestamps=True, language=target_language)
- segments = list(segments)
- i = 1 # Start the index at 1
- sentence = ""
- sentence_start = None
- first_word = True
- words_list = []
- for _, segment in enumerate(segments):
- words = list(segment.words)
- words_list.extend(words)
-
- for word_idx, word in enumerate(words_list):
- if create_bpe_tokenizer:
- whisper_words.append(word.word)
-
- if first_word:
- sentence_start = word.start
- if word_idx == 0:
- sentence_start = max(sentence_start - buffer, 0)
- else:
- previous_word_end = words_list[word_idx - 1].end
- sentence_start = max(sentence_start - buffer, (previous_word_end + sentence_start) / 2)
- sentence = word.word
- first_word = False
- else:
- sentence += " " + word.word
-
- # Split segment if it ends with punctuation or exceeds max_duration
- if word.word[-1] in ["!", ".", "?"] or (word.end - sentence_start) > max_duration:
- sentence = sentence.strip()
- # Clean and normalize the sentence
- sentence = multilingual_cleaners(sentence, target_language)
- audio_file_name, _ = os.path.splitext(os.path.basename(audio_path))
- audio_file = f"{audio_file_name}_{str(i).zfill(8)}.wav"
-
- if word_idx + 1 < len(words_list):
- next_word_start = words_list[word_idx + 1].start
- else:
- next_word_start = (wav.shape[0] - 1) / sr
-
- word_end = min((word.end + next_word_start) / 2, word.end + buffer)
-
- absolute_path = os.path.join(audio_folder, audio_file) # Save to the audio_folder
- os.makedirs(os.path.dirname(absolute_path), exist_ok=True)
- first_word = True
-
- audio = wav[int(sr * sentence_start):int(sr * word_end)].unsqueeze(0)
+ # Process with VAD if enabled
+ if use_vad and vad_model is not None:
+ debug_print("Processing with VAD", "AUDIO")
+ # Get VAD segments with resampling
+ vad_segments = process_audio_with_vad(wav, sr, vad_model, get_speech_timestamps)
+
+ # Group short segments that are close together
+ merged_segments = merge_short_segments(vad_segments, min_duration, max_gap=0.3)
+ debug_print(f"Merged {len(vad_segments)-len(merged_segments)} short segments", "SEGMENTS")
+
+ # Convert VAD segments to audio chunks
+ speech_chunks = []
+ for segment in merged_segments:
+ chunk = wav[segment['start']:segment['end']]
+ duration = chunk.size(-1) / sr
+ if duration < min_duration:
+ debug_print(f"Segment too short ({duration:.2f}s), attempting to extend", "SEGMENTS", is_warning=True)
+ # Try to extend segment if possible
+ chunk = extend_segment(wav, segment['start'], segment['end'], sr, min_duration)
+ duration = chunk.size(-1) / sr
- # If the audio is too long, split it into smaller chunks
- if audio.size(-1) > max_duration * sr:
- gradio_progress((3,4), "Splitting overly large audio")
- too_long_files.append((audio_file, audio.size(-1) / sr))
- while audio.size(-1) > max_duration * sr:
- split_audio = audio[:, :int(max_duration * sr)]
- audio = audio[:, int(max_duration * sr):]
- # Normalize the file path
- split_file_name = f"{audio_file_name}_{str(i).zfill(8)}.wav"
- split_relative_path = os.path.join(split_file_name)
- split_absolute_path = os.path.normpath(os.path.join(audio_folder, split_relative_path))
- # Ensure the directory exists
- os.makedirs(os.path.dirname(split_absolute_path), exist_ok=True)
- # Save the split audio
- torchaudio.save(split_absolute_path, split_audio, sr)
- # Update metadata
- metadata["audio_file"].append(f"wavs/{split_relative_path}")
- metadata["text"].append(sentence)
- metadata["speaker_name"].append(speaker_name)
- i += 1
+ if chunk.numel() > 0:
+ speech_chunks.append((chunk, duration))
+ stats.add_segment(duration)
- if audio.size(-1) >= sr: # if the remaining audio is at least half a second long
- torchaudio.save(absolute_path, audio, sr)
+ # Process each speech chunk
+ for chunk_idx, (chunk, duration) in enumerate(speech_chunks):
+ if duration < min_duration:
+ stats.segments_under_min += 1
+ debug_print(f"Short segment: {duration:.2f}s", "SEGMENTS", is_warning=True)
+ elif duration > max_duration:
+ stats.segments_over_max += 1
+ debug_print(f"Long segment: {duration:.2f}s", "SEGMENTS", is_warning=True)
+
+ chunk_path = os.path.join(temp_folder, f"{audio_file_name_without_ext}_chunk_{chunk_idx}.wav")
+ torchaudio.save(chunk_path, chunk.unsqueeze(0), sr)
+
+ # Transcribe with appropriate precision
+ if precision == "mixed" and device == "cuda":
+ with torch.cuda.amp.autocast():
+ gradio_progress((5, 10), desc="Transcribing Audio")
+ result = asr_model.transcribe(
+ chunk_path,
+ language=target_language,
+ word_timestamps=True,
+ verbose=None
+ )
else:
+ gradio_progress((5, 10), desc="Transcribing Audio")
+ result = asr_model.transcribe(
+ chunk_path,
+ language=target_language,
+ word_timestamps=True,
+ verbose=None
+ )
+
+ if not result.get("text", "").strip():
+ debug_print(f"Empty transcription for chunk {chunk_idx}", "DATA_PROCESS", is_warning=True)
continue
- metadata["audio_file"].append(f"wavs/{audio_file_name}_{str(i).zfill(8)}.wav")
- metadata["text"].append(sentence)
- metadata["speaker_name"].append(speaker_name)
- i += 1 # Increment the index after saving
+ # Process transcription result
+ process_transcription_result(
+ result, chunk, sr, chunk_idx, audio_file_name_without_ext,
+ metadata, whisper_words, max_duration, buffer, speaker_name,
+ audio_folder, too_long_files, create_bpe_tokenizer, target_language
+ )
+
+ os.remove(chunk_path)
+ debug_print(f"Processed chunk {chunk_idx} ({duration:.2f}s)", "SEGMENTS")
+
+ else:
+ # Regular processing without VAD
+ debug_print("Processing without VAD", "AUDIO")
+ if precision == "mixed" and device == "cuda":
+ with torch.cuda.amp.autocast():
+ result = asr_model.transcribe(
+ audio_path,
+ language=target_language,
+ word_timestamps=True,
+ verbose=None
+ )
+ else:
+ result = asr_model.transcribe(
+ audio_path,
+ language=target_language,
+ word_timestamps=True,
+ verbose=None
+ )
+
+ # Process transcription result
+ process_transcription_result(
+ result, wav, sr, 0, audio_file_name_without_ext,
+ metadata, whisper_words, max_duration, buffer, speaker_name,
+ audio_folder, too_long_files, create_bpe_tokenizer, target_language
+ )
os.remove(temp_audio_path)
+ # Update progress
end = datetime.datetime.now()
gradio_progress_duration += (end - start).total_seconds()
audio_steps = (audio_steps[0] + 1, audio_steps[1])
@@ -512,95 +946,528 @@ def format_audio_list(target_language, whisper_model, max_sample_length, eval_sp
gradio_estimated_duration = (avg_duration * additional_data_points_needed)
gradio_progress(audio_steps, desc=f"Processing. Estimated Completion: {c_logger.format_duration(gradio_estimated_duration)}", unit="files")
- # Check if the WAV files folder contains files after processing
- audio_files = [os.path.join(audio_folder, file) for file in os.listdir(audio_folder) if file.endswith('.wav')]
+ # Print final statistics
+ stats.print_stats()
+ # Verify processed files exist
+ audio_files = [os.path.join(audio_folder, file) for file in os.listdir(audio_folder) if file.endswith('.wav')]
if not audio_files:
- print(f"[FINETUNE] No processed audio files found in {audio_folder}. Skipping processing.")
+ debug_print("No processed audio files found", "AUDIO", is_error=True)
return None, None, 0
+ # Final statistics before metadata handling
+ stats.print_stats()
+
+ # Handle existing metadata case
if os.path.exists(train_metadata_path) and os.path.exists(eval_metadata_path):
- validate_train_metadata_path = train_metadata_path
- validate_eval_metadata_path = eval_metadata_path
- validate_audio_folder = audio_folder
- validate_whisper_model = whisper_model
- validate_target_language = target_language
-
- gradio_progress((4,4), "Finalizing")
-
- del asr_model, existing_metadata
- gc.collect()
- if torch.cuda.is_available():
- torch.cuda.empty_cache()
-
+ debug_print("Using existing metadata files", "DATA_PROCESS")
+ _set_validation_paths(
+ train_metadata_path, eval_metadata_path, audio_folder,
+ whisper_model, target_language
+ )
+ _cleanup_resources(asr_model, existing_metadata)
return train_metadata_path, eval_metadata_path, audio_total_size
- # Check if there are any new audio files to process
+ # Check for new metadata
if not metadata["audio_file"]:
- print("[FINETUNE] No new audio files to process. Skipping processing.")
- validate_train_metadata_path = train_metadata_path
- validate_eval_metadata_path = eval_metadata_path
- validate_audio_folder = audio_folder
- validate_whisper_model = whisper_model
- validate_target_language = target_language
-
- gradio_progress((4,4), "Finalizing")
-
- del asr_model, existing_metadata
- gc.collect()
- if torch.cuda.is_available():
- torch.cuda.empty_cache()
-
+ debug_print("No new audio files to process", "DATA_PROCESS", is_warning=True)
+ _set_validation_paths(
+ train_metadata_path, eval_metadata_path, audio_folder,
+ whisper_model, target_language
+ )
+ _cleanup_resources(asr_model, existing_metadata)
return train_metadata_path, eval_metadata_path, audio_total_size
+ # Process metadata and handle duplicates
+ debug_print("Processing metadata and handling duplicates", "DATA_PROCESS")
new_data_df = pd.DataFrame(metadata)
- combined_train_df_shuffled = new_data_df.sample(frac=1)
- num_val_samples = int(len(combined_train_df_shuffled) * eval_percentage)
-
- final_eval_set = combined_train_df_shuffled[:num_val_samples]
- final_training_set = combined_train_df_shuffled[num_val_samples:]
-
- final_training_set.sort_values('audio_file').to_csv(train_metadata_path, sep='|', index=False)
- final_eval_set.sort_values('audio_file').to_csv(eval_metadata_path, sep='|', index=False)
+
+ # Duplicate detection and handling
+ duplicate_files = new_data_df['audio_file'].value_counts()
+ duplicates_found = duplicate_files[duplicate_files > 1]
+
+ gradio_progress((6, 10), desc="Handling Duplicate Transcriptions")
+ if not duplicates_found.empty:
+ debug_print(f"Found {len(duplicates_found)} files with multiple transcriptions", "DUPLICATES")
+ for file, count in duplicates_found.items():
+ debug_print(f"{file}: {count} occurrences", "DUPLICATES")
+
+ # Re-transcribe duplicates
+ best_transcriptions = handle_duplicates(duplicates_found.index, audio_folder, target_language, whisper_model)
+
+ # Update transcriptions and remove duplicates
+ for file_path, trans_info in best_transcriptions.items():
+ new_data_df.loc[new_data_df['audio_file'] == file_path, 'text'] = trans_info['text']
+ debug_print(f"Updated transcription for {file_path}", "DUPLICATES")
+
+ new_data_df = new_data_df.drop_duplicates(subset='audio_file', keep='first')
+ debug_print(f"Cleaned up {len(duplicates_found)} duplicate entries", "DUPLICATES")
+
+ # Handle evaluation split
+ debug_print("Creating train/eval split", "DATA_PROCESS")
+ eval_percentage = _adjust_eval_percentage(eval_split_number)
+
+ # Create and validate splits
+ gradio_progress((7, 10), desc="Creating Train/Eval Split")
+ train_eval_split = create_dataset_splits(
+ new_data_df,
+ eval_percentage,
+ random_seed=42
+ )
+
+ if train_eval_split is None:
+ debug_print("Failed to create valid dataset splits", "DATA_PROCESS", is_error=True)
+ return None, None, 0
+
+ final_training_set, final_eval_set = train_eval_split
+
+ # Write metadata files
+ debug_print(f"Writing {len(final_training_set)} training and {len(final_eval_set)} eval samples", "DATA_PROCESS")
+ try:
+ gradio_progress((8, 10), desc="Saving Metadata Files")
+ _write_metadata_files(
+ final_training_set, final_eval_set,
+ train_metadata_path, eval_metadata_path
+ )
+ except Exception as e:
+ debug_print(f"Error writing metadata: {str(e)}", "DATA_PROCESS", is_error=True)
+ raise
+ # Handle BPE tokenizer
if create_bpe_tokenizer:
- print(f"Training BPE Tokenizer")
- tokenizer = ByteLevelBPETokenizer(str(this_dir / base_path / "xttsv2_2.0.3" / "vocab.json"))
+ gradio_progress((9, 10), desc="Training BPE Tokenizer")
+ debug_print("Training BPE Tokenizer", "MODEL_OPS")
+ _create_bpe_tokenizer(whisper_words, out_path, base_path)
+
+ debug_print("Finalizing processing", "GENERAL")
+ gradio_progress((10, 10), desc="Finalizing Process")
+
+ # Cleanup and set validation paths
+ _cleanup_resources(asr_model, final_eval_set, final_training_set, new_data_df, existing_metadata)
+ _set_validation_paths(
+ train_metadata_path, eval_metadata_path, audio_folder,
+ whisper_model, target_language
+ )
+
+ # Log final statistics
+ if too_long_files:
+ debug_print("Files that were split due to length:", "SEGMENTS")
+ for file_name, length in too_long_files:
+ debug_print(f" {file_name}: {length:.2f} seconds", "SEGMENTS")
+
+ return train_metadata_path, eval_metadata_path, audio_total_size
+
+def _create_bpe_tokenizer(whisper_words, out_path, base_path):
+ """Create and train BPE tokenizer"""
+ vocab_path = base_path / "xttsv2_2.0.3" / "vocab.json"
+ if not vocab_path.exists():
+ debug_print("BPE tokenizer will not be created for this dataset creation run.", "GENERAL", is_warning=True)
+ debug_print("XTTS v2.0.3 base model not found for BPE Tokenizer. Please download it first using AllTalk's", "GENERAL", is_warning=True)
+ debug_print("main Gradio interface > TTS Engine Settings > XTTS > Model/Voices Downloads.", "GENERAL", is_warning=True)
+ debug_print("Dataset creation will continue without the BPE tokenizer.", "GENERAL", is_warning=True)
+ raise FileNotFoundError(f"Missing required file: {vocab_path}")
+
+ try:
+ debug_print("Initializing BPE tokenizer training", "MODEL_OPS")
+ tokenizer = ByteLevelBPETokenizer(str(vocab_path))
tokenizer.pre_tokenizer = Whitespace()
- tokenizer.add_tokens(["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]", "[STOP]", "[SAPCE]"])
- tokenizer.train_from_iterator(whisper_words, vocab_size=30000, show_progress=True, min_frequency=2, special_tokens=[
- "[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]", "[STOP]", "[SPACE]",
- ])
- # Save the tokenizer model
- tokenizer.save(path=str(out_path / "bpe_tokenizer-vocab.json"), pretty=True)
+
+ # Add special tokens
+ special_tokens = [
+ "[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]", "[STOP]", "[SPACE]"
+ ]
+ tokenizer.add_tokens(special_tokens)
+
+ debug_print(f"Training tokenizer on {len(whisper_words)} words", "MODEL_OPS")
+ tokenizer.train_from_iterator(
+ whisper_words,
+ vocab_size=30000,
+ show_progress=True,
+ min_frequency=2,
+ special_tokens=special_tokens
+ )
+
+ # Save tokenizer
+ tokenizer_path = str(out_path / "bpe_tokenizer-vocab.json")
+ tokenizer.save(path=tokenizer_path, pretty=True)
+ debug_print(f"Saved BPE tokenizer to {tokenizer_path}", "MODEL_OPS")
+
+ except Exception as e:
+ debug_print(f"Failed to create BPE tokenizer: {str(e)}", "MODEL_OPS", is_error=True)
+ raise
- gradio_progress((4,4), "Finalizing")
+def merge_short_segments(segments, min_duration, max_gap=0.5):
+ """
+ More aggressive merge strategy for short segments
+ - Increases max_gap to 0.5s (from 0.3s)
+ - Looks ahead multiple segments for potential merges
+ - Considers surrounding context
+ """
+ if not segments:
+ return segments
- del asr_model, final_eval_set, final_training_set, new_data_df, existing_metadata
+ merged = []
+ current_group = []
+
+ for i, segment in enumerate(segments):
+ current_duration = sum(s['end'] - s['start'] for s in current_group) if current_group else 0
+
+ # If this is a continuation of current group
+ if current_group and (segment['start'] - current_group[-1]['end']) <= max_gap:
+ current_group.append(segment)
+ # If starting new group but previous was too short
+ elif current_group and current_duration < min_duration:
+ # Look ahead for close segments
+ look_ahead = min(3, len(segments) - i) # Look up to 3 segments ahead
+ for j in range(i, i + look_ahead):
+ if j < len(segments) and (segments[j]['start'] - current_group[-1]['end']) <= max_gap * 2:
+ current_group.append(segments[j])
+ else:
+ break
+ else:
+ # Save previous group if it exists
+ if current_group:
+ merged_segment = {
+ 'start': current_group[0]['start'],
+ 'end': current_group[-1]['end']
+ }
+ merged.append(merged_segment)
+ current_group = [segment]
+
+ # Handle last group
+ if current_group:
+ merged_segment = {
+ 'start': current_group[0]['start'],
+ 'end': current_group[-1]['end']
+ }
+ merged.append(merged_segment)
+
+ debug_print(f"Merged {len(segments) - len(merged)} segments into {len(merged)} longer segments", "SEGMENTS")
+ return merged
+
+def extend_segment(wav, start, end, sr, min_duration, context_window=1.0):
+ """
+ Improved segment extension with better context handling
+ - Adds context_window parameter for smoother extensions
+ - More balanced extension on both sides
+ - Checks audio content when extending
+ """
+ current_duration = (end - start) / sr
+ if current_duration >= min_duration:
+ return wav[start:end]
+
+ samples_needed = int((min_duration - current_duration) * sr)
+
+ # Try to extend equally on both sides
+ extend_left = samples_needed // 2
+ extend_right = samples_needed - extend_left
+
+ # Add some context window
+ context_samples = int(context_window * sr)
+ new_start = max(0, start - extend_left - context_samples)
+ new_end = min(wav.size(-1), end + extend_right + context_samples)
+
+ # Check if we got enough duration
+ if (new_end - new_start) / sr < min_duration:
+ # If still too short, try to compensate from the other side
+ if new_start == 0:
+ new_end = min(wav.size(-1), end + samples_needed + context_samples)
+ elif new_end == wav.size(-1):
+ new_start = max(0, start - samples_needed - context_samples)
+
+ debug_print(f"Extended segment from {current_duration:.2f}s to {(new_end - new_start) / sr:.2f}s", "SEGMENTS")
+ return wav[new_start:new_end]
+
+# Helper functions for better organization
+def _adjust_eval_percentage(eval_split_number):
+ """Adjust evaluation percentage to be within acceptable bounds"""
+ eval_percentage = eval_split_number / 100.0
+ min_eval_percentage = 0.1
+ max_eval_percentage = 0.3
+
+ if eval_percentage < min_eval_percentage:
+ debug_print(f"Adjusting eval split from {eval_percentage:.1%} to {min_eval_percentage:.1%}", "DATA_PROCESS", is_warning=True)
+ return min_eval_percentage
+ elif eval_percentage > max_eval_percentage:
+ debug_print(f"Adjusting eval split from {eval_percentage:.1%} to {max_eval_percentage:.1%}", "DATA_PROCESS", is_warning=True)
+ return max_eval_percentage
+ return eval_percentage
+
+def _set_validation_paths(train_path, eval_path, audio_folder, whisper_model, target_language):
+ """Set global validation paths"""
+ global validate_train_metadata_path, validate_eval_metadata_path, validate_audio_folder
+ global validate_whisper_model, validate_target_language
+
+ validate_train_metadata_path = train_path
+ validate_eval_metadata_path = eval_path
+ validate_audio_folder = audio_folder
+ validate_whisper_model = whisper_model
+ validate_target_language = target_language
+
+def _cleanup_resources(*resources):
+ """Clean up resources and free memory"""
+ for resource in resources:
+ del resource
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
- # Print or log the files that were too long
- if too_long_files:
- print(f"[FINETUNE] The following files were too long and were split into smaller chunks:")
- print(f"[FINETUNE]")
- for file_name, length in too_long_files:
- print(f"[FINETUNE] File: {file_name}, Length: {length:.2f} seconds")
- print(f"[FINETUNE]")
+def _write_metadata_files(train_set, eval_set, train_path, eval_path):
+ """Write metadata files with error handling"""
+ try:
+ train_set.sort_values('audio_file').to_csv(train_path, sep='|', index=False)
+ eval_set.sort_values('audio_file').to_csv(eval_path, sep='|', index=False)
+ debug_print("Successfully wrote metadata files", "DATA_PROCESS")
+ except Exception as e:
+ debug_print(f"Failed to write metadata files: {str(e)}", "DATA_PROCESS", is_error=True)
+ raise
+
+def create_dataset_splits(df, eval_percentage, random_seed=42):
+ """Create training and evaluation splits with validation"""
+ if df.empty:
+ debug_print("No data available for splitting", "DATA_PROCESS", is_error=True)
+ return None
+
+ shuffled_df = df.sample(frac=1, random_state=random_seed)
+ num_val_samples = max(1, int(len(shuffled_df) * eval_percentage))
- validate_train_metadata_path = train_metadata_path
- validate_eval_metadata_path = eval_metadata_path
- validate_audio_folder = audio_folder
- validate_whisper_model = whisper_model
- validate_target_language = target_language
-
- return train_metadata_path, eval_metadata_path, audio_total_size
+ if num_val_samples >= len(shuffled_df):
+ debug_print("Not enough samples for valid split", "DATA_PROCESS", is_error=True)
+ return None
+
+ return (
+ shuffled_df[num_val_samples:], # training
+ shuffled_df[:num_val_samples] # eval
+ )
-#######################
-# STEP 1 # Validation #
-#######################
+def save_audio_segment(audio, sr, start_time, end_time, sentence, audio_file_name_without_ext,
+ segment_idx, speaker_name, audio_folder, metadata, max_duration, buffer,
+ too_long_files, target_language):
+ """Helper function to save audio segments and update metadata"""
+ sentence = sentence.strip()
+ sentence = multilingual_cleaners(sentence, target_language)
+ audio_file_name = f"{audio_file_name_without_ext}_{str(segment_idx).zfill(8)}.wav"
+
+ absolute_path = os.path.join(audio_folder, audio_file_name)
+ os.makedirs(os.path.dirname(absolute_path), exist_ok=True)
+
+ # Extract audio segment
+ audio_start = int(sr * start_time)
+ audio_end = int(sr * end_time)
+ audio_segment = audio[audio_start:audio_end].unsqueeze(0)
+
+ # Handle long audio segments
+ if audio_segment.size(-1) > max_duration * sr:
+ too_long_files.append((audio_file_name, audio_segment.size(-1) / sr))
+
+ while audio_segment.size(-1) > max_duration * sr:
+ split_audio = audio_segment[:, :int(max_duration * sr)]
+ audio_segment = audio_segment[:, int(max_duration * sr):]
+ split_file_name = f"{audio_file_name_without_ext}_{str(segment_idx).zfill(8)}.wav"
+ split_relative_path = os.path.join(split_file_name)
+ split_absolute_path = os.path.normpath(os.path.join(audio_folder, split_relative_path))
+
+ os.makedirs(os.path.dirname(split_absolute_path), exist_ok=True)
+ torchaudio.save(split_absolute_path, split_audio, sr)
+
+ metadata["audio_file"].append(f"wavs/{split_relative_path}")
+ metadata["text"].append(sentence)
+ metadata["speaker_name"].append(speaker_name)
+ segment_idx += 1
+
+ if audio_segment.size(-1) >= sr: # Only save if segment is at least 1 second
+ torchaudio.save(absolute_path, audio_segment, sr)
+ metadata["audio_file"].append(f"wavs/{audio_file_name}")
+ metadata["text"].append(sentence)
+ metadata["speaker_name"].append(speaker_name)
+
+def process_transcription_result(result, audio, sr, segment_idx, audio_file_name_without_ext,
+ metadata, whisper_words, max_duration, buffer, speaker_name,
+ audio_folder, too_long_files, create_bpe_tokenizer, target_language):
+ """Helper function to process transcription results and save audio segments"""
+ i = segment_idx + 1
+ sentence = ""
+ sentence_start = None
+ first_word = True
+ current_words = []
+
+ for segment in result["segments"]:
+ if "words" not in segment:
+ continue
+
+ for word_info in segment["words"]:
+ word = word_info.get("word", "").strip()
+ if not word:
+ continue
+
+ start_time = word_info.get("start", 0)
+ end_time = word_info.get("end", 0)
+
+ if create_bpe_tokenizer:
+ whisper_words.append(word)
+
+ if first_word:
+ sentence_start = start_time
+ if len(current_words) == 0:
+ sentence_start = max(sentence_start - buffer, 0)
+ else:
+ previous_end = current_words[-1].get("end", 0) if current_words else 0
+ sentence_start = max(sentence_start - buffer, (previous_end + start_time) / 2)
+ sentence = word
+ first_word = False
+ else:
+ sentence += " " + word
+
+ current_words.append({
+ "word": word,
+ "start": start_time,
+ "end": end_time
+ })
+
+ # Handle sentence splitting and audio saving
+ if word[-1] in ["!", ".", "?"] or (end_time - sentence_start) > max_duration:
+ save_audio_segment(
+ audio, sr, sentence_start, end_time, sentence,
+ audio_file_name_without_ext, i, speaker_name,
+ audio_folder, metadata, max_duration, buffer,
+ too_long_files, target_language
+ )
+ i += 1
+ first_word = True
+ current_words = []
+ sentence = ""
+
+def process_audio_with_vad(wav, sr, vad_model, get_speech_timestamps):
+ """
+ Enhanced VAD processing with better end-of-speech detection
+ """
+ import torchaudio.transforms as T
+
+ device = "cuda" if torch.cuda.is_available() else "cpu"
+ wav = wav.to(device)
+
+ resampler = T.Resample(sr, 16000).to(device)
+ wav_16k = resampler(wav)
+
+ # Adjusted VAD parameters
+ vad_segments = get_speech_timestamps(
+ wav_16k,
+ vad_model,
+ sampling_rate=16000,
+ threshold=0.2, # Lower threshold to be more sensitive to speech
+ min_speech_duration_ms=200, # Shorter to catch brief utterances
+ max_speech_duration_s=float('inf'),
+ min_silence_duration_ms=300,# Shorter silence duration
+ window_size_samples=1024, # Smaller window for more precise detection
+ speech_pad_ms=300 # Add padding to end of speech segments
+ )
+
+ # Scale timestamps back to original sample rate
+ scale_factor = sr / 16000
+ for segment in vad_segments:
+ segment['start'] = int(segment['start'] * scale_factor)
+ # Add extra padding at the end
+ segment['end'] = int(segment['end'] * scale_factor) + int(0.2 * sr) # Add 200ms padding
+
+ merged_segments = merge_short_segments(vad_segments, min_duration=6.0, max_gap=0.5)
+
+ debug_print(f"VAD processing: {len(vad_segments)} original segments, {len(merged_segments)} after merging", "SEGMENTS")
+ return merged_segments
+
+def process_audio_segments(result, wav, sr, audio_file_name_without_ext, metadata, min_duration=6.0, max_duration=None):
+ """
+ Process audio into segments with minimum and maximum duration constraints
+ Returns list of segments with their transcriptions
+ """
+ segments = []
+ current_segment = {
+ 'start': None,
+ 'end': None,
+ 'text': [],
+ 'words': []
+ }
+
+ for segment in result["segments"]:
+ if "words" not in segment:
+ continue
+
+ for word_info in segment["words"]:
+ word = word_info.get("word", "").strip()
+ if not word:
+ continue
+
+ start_time = word_info.get("start", 0)
+ end_time = word_info.get("end", 0)
+
+ if current_segment['start'] is None:
+ current_segment['start'] = start_time
+
+ current_segment['end'] = end_time
+ current_segment['text'].append(word)
+ current_segment['words'].append(word_info)
+
+ # Check if we should split (on punctuation or max length)
+ if (word[-1] in ["!", ".", "?"] or
+ (max_duration and end_time - current_segment['start'] > max_duration)):
+
+ segment_duration = current_segment['end'] - current_segment['start']
+
+ # If segment is too short, try to extend it unless it's the last segment
+ if segment_duration < min_duration and len(result["segments"]) > 0:
+ continue
+
+ # Save the segment if it's valid
+ if segment_duration >= min_duration or word[-1] in ["!", ".", "?"]:
+ segments.append(current_segment)
+ current_segment = {
+ 'start': None,
+ 'end': None,
+ 'text': [],
+ 'words': []
+ }
+
+ # Handle any remaining segment
+ if current_segment['start'] is not None:
+ segment_duration = current_segment['end'] - current_segment['start']
+ if segment_duration >= min_duration:
+ segments.append(current_segment)
+
+ return segments
+
+def handle_duplicates(duplicate_files, audio_folder, target_language, whisper_model):
+ """Re-transcribe duplicate files to get best transcription"""
+ print("[FINETUNE] [INFO] Re-transcribing duplicate files to get best transcription")
+
+ best_transcriptions = {}
+ device = "cuda" if torch.cuda.is_available() else "cpu"
+ asr_model = whisper.load_model(whisper_model, device=device)
+
+ for file_path in duplicate_files:
+ print(f"[FINETUNE] [INFO] Re-transcribing {file_path}")
+
+ # Get full path
+ full_path = os.path.join(audio_folder, os.path.basename(file_path.replace("wavs/", "")))
+
+ # Re-transcribe with highest quality settings
+ result = asr_model.transcribe(
+ full_path,
+ language=target_language,
+ word_timestamps=True,
+ verbose=None
+ )
+
+ # Store the new transcription
+ best_transcriptions[file_path] = {
+ 'text': result["text"].strip(),
+ 'confidence': sum(s.get('confidence', 0) for s in result["segments"]) / len(result["segments"])
+ }
+
+ return best_transcriptions
+
+#############################################
+#### STEP 1 Dataset Validation Functions ####
+#############################################
def normalize_text(text):
# Convert text to lowercase
text = text.lower()
@@ -631,18 +1498,21 @@ def get_audio_file_list(mismatches):
def load_and_display_mismatches():
def validate_audio_transcriptions(csv_paths, audio_folder, whisper_model, target_language, progress=None):
+ # Load and combine metadata from CSV files
metadata_dfs = []
for csv_path in csv_paths:
metadata_df = pd.read_csv(csv_path, sep='|')
- # Add a column to track which CSV file each row came from
+ # Add source CSV tracking
metadata_df['source_csv'] = csv_path
- # Add a column for the row index in original CSV
metadata_df['row_index'] = metadata_df.index
metadata_dfs.append(metadata_df)
metadata_df = pd.concat(metadata_dfs, ignore_index=True)
+
+ # Load Whisper model
device = "cuda" if torch.cuda.is_available() else "cpu"
- asr_model = WhisperModel(whisper_model, device=device, compute_type="float32")
+ asr_model = whisper.load_model(whisper_model, device=device)
+
mismatches = []
missing_files = []
total_files = metadata_df.shape[0]
@@ -662,8 +1532,18 @@ def load_and_display_mismatches():
progress((index + 1, total_files), desc="Processing files")
continue
- segments, _ = asr_model.transcribe(audio_path, vad_filter=True, word_timestamps=True, language=target_language)
- transcribed_text = " ".join([segment.text for segment in segments]).strip()
+ # Transcribe with OpenAI Whisper
+ result = asr_model.transcribe(
+ audio_path,
+ language=target_language,
+ word_timestamps=True,
+ verbose=None
+ )
+
+ # Get the full transcription from the result
+ transcribed_text = result["text"].strip()
+
+ # Normalize and compare texts
normalized_expected_text = normalize_text(expected_text)
normalized_transcribed_text = normalize_text(transcribed_text)
@@ -737,14 +1617,9 @@ def save_correction_to_csv(csv_path, row_index, new_text):
new_text = str(new_text.iloc[0])
else:
new_text = str(new_text)
-
- # print(f"[FINETUNE] Attempting to save correction to {csv_path}")
- # print(f"[FINETUNE] Row index type: {type(row_index)}, value: {row_index}")
- # print(f"[FINETUNE] New text type: {type(new_text)}, value: {new_text}")
-
+
# Read the CSV file
df = pd.read_csv(csv_path, sep='|')
- # print(f"[FINETUNE] Original text at index {row_index}: {df.loc[row_index, 'text']}")
# Update the text
df.loc[row_index, 'text'] = new_text
@@ -755,7 +1630,6 @@ def save_correction_to_csv(csv_path, row_index, new_text):
# Verify the save
df_check = pd.read_csv(csv_path, sep='|')
if df_check.loc[row_index, 'text'] == new_text:
- print(f"[FINETUNE] Successfully verified save. New text at index {row_index}: {df_check.loc[row_index, 'text']}")
return f"Successfully updated transcription in {os.path.basename(csv_path)}"
else:
print(f"[FINETUNE] Save verification failed. Text mismatch.")
@@ -769,93 +1643,89 @@ def save_correction_to_csv(csv_path, row_index, new_text):
traceback.print_exc()
return f"Error updating CSV: {str(e)}"
-def save_selected_text(choice, manual_text, df, current_idx):
+def save_audio_and_correction(choice, manual_text, audio_data, df, current_idx):
+ """Handle both audio and transcription saves"""
if current_idx is None:
return {
mismatch_table: df[['expected_text', 'transcribed_text', 'filename']],
current_expected: "",
- save_status: "Please select a row first"
+ save_status: "Please select a row first",
+ audio_player: None
}
try:
- # Debug the current_idx type
- # print(f"[FINETUNE] Current index type: {type(current_idx)}, value: {current_idx}")
-
- # Ensure current_idx is a single integer
+ # Handle current_idx coming as a list from Gradio
if isinstance(current_idx, list):
current_idx = current_idx[0]
- current_idx = int(current_idx)
- row = df.iloc[current_idx]
- # print(f"[FINETUNE] Selected row for saving: {row}")
+ row = df.iloc[int(current_idx)]
+ audio_path = str(row['full_path']).strip()
- # Get single values for each field
- source_csv = row['source_csv']
- if isinstance(source_csv, pd.Series):
- source_csv = source_csv.iloc[0]
-
- row_index = row['row_index']
- if isinstance(row_index, pd.Series):
- row_index = row_index.iloc[0]
- row_index = int(row_index) # Ensure it's an integer
-
- # Determine which text to use
+ debug_print(f"Processing file: {audio_path}", "DATA_PROCESS")
+ save_status_msg = []
+
+ # Handle audio save if audio was edited
+ if audio_data is not None and isinstance(audio_data, tuple) and len(audio_data) == 2:
+ try:
+ sr, audio = audio_data
+ debug_print(f"Saving edited audio: {sr}Hz, length: {len(audio)}", "DATA_PROCESS")
+
+ import torch
+ import torchaudio
+ audio_tensor = torch.tensor(audio).unsqueeze(0)
+ torchaudio.save(audio_path, audio_tensor, sr)
+ save_status_msg.append("Audio saved successfully")
+ debug_print(f"Saved edited audio to {audio_path}", "DATA_PROCESS")
+ except Exception as e:
+ save_status_msg.append(f"Error saving audio: {str(e)}")
+ debug_print(f"Error saving audio: {str(e)}", "DATA_PROCESS", is_error=True)
+
+ # Handle text correction
if choice == "Use Original":
- new_text = str(row['expected_text']) # Ensure it's a string
+ new_text = str(row['expected_text'])
elif choice == "Use Whisper":
- new_text = str(row['transcribed_text']) # Ensure it's a string
+ new_text = str(row['transcribed_text'])
elif choice == "Edit Manually":
- new_text = str(manual_text) # Ensure it's a string
- else:
- return {
- mismatch_table: df[['expected_text', 'transcribed_text', 'filename']],
- current_expected: row['expected_text'],
- save_status: f"Invalid choice: {choice}"
- }
+ new_text = str(manual_text)
- # print(f"[FINETUNE] Choice: {choice}")
- # print(f"[FINETUNE] New text to save: {new_text}")
- # print(f"[FINETUNE] CSV file: {source_csv}")
- # print(f"[FINETUNE] Row index: {row_index}")
-
- # Save the correction
- result = save_correction_to_csv(source_csv, row_index, new_text)
- # print(f"[FINETUNE] Save result: {result}")
+ # Save text correction to CSV
+ result = save_correction_to_csv(
+ str(row['source_csv']),
+ int(row['row_index']),
+ new_text
+ )
+ save_status_msg.append(result)
+ # Update both text and expected_text in DataFrame
if "Successfully" in result:
- # Update the DataFrame with the new text
+ df.loc[current_idx, 'text'] = new_text
df.loc[current_idx, 'expected_text'] = new_text
-
+ debug_print(f"Updated DataFrame with new text: {new_text}", "DATA_PROCESS")
+
+ # Create updated display DataFrame
+ display_df = df[['expected_text', 'transcribed_text', 'filename']].copy()
+
return {
- mismatch_table: df[['expected_text', 'transcribed_text', 'filename']],
+ mismatch_table: display_df,
current_expected: new_text if "Successfully" in result else row['expected_text'],
- save_status: result
+ save_status: " | ".join(save_status_msg),
+ audio_player: audio_path
}
except Exception as e:
error_msg = f"Error saving correction: {str(e)}"
- print(f"[FINETUNE] {error_msg}")
- print(f"[FINETUNE] Full error traceback:")
+ debug_print(error_msg, "DATA_PROCESS", is_error=True)
traceback.print_exc()
return {
mismatch_table: df[['expected_text', 'transcribed_text', 'filename']],
current_expected: row['expected_text'] if 'row' in locals() else "",
- save_status: error_msg
+ save_status: error_msg,
+ audio_player: None
}
-######################
-#### STEP 2 BITS #####
-######################
-
-from trainer import TrainerArgs, Trainer
-from trainer_alltalk.trainer import Trainer
-
-from TTS.config.shared_configs import BaseDatasetConfig
-from TTS.tts.datasets import load_tts_samples
-from TTS.tts.layers.xtts.trainer.gpt_trainer import GPTArgs, GPTTrainer, GPTTrainerConfig, XttsAudioConfig
-from TTS.utils.manage import ModelManager
-
-
+#########################################
+#### STEP 2 Model Training Functions ####
+#########################################
def basemodel_or_finetunedmodel_choice(value):
global basemodel_or_finetunedmodel
if value == "Base Model":
@@ -867,9 +1737,10 @@ def train_gpt(language, num_epochs, batch_size, grad_acumm, train_csv, eval_csv,
pfc_check_fail()
if "No Models Available" in model_to_train:
- print(f"[FINETUNE] \033[91mError\033[0m: Cannot train. You have not selected a model. Please download a model.")
- print(f"[FINETUNE] \033[91mError\033[0m: into a sub folder within the correct models folder.")
- return
+ print("[FINETUNE] Error: No XTTS model selected for training.")
+ print("[FINETUNE] Please download a model using AllTalk's main interface > TTS Engine Settings > XTTS > Model/Voices Download")
+ return
+
# Logging parameters
RUN_NAME = "XTTS_FT"
PROJECT_NAME = "XTTS_trainer"
@@ -879,29 +1750,40 @@ def train_gpt(language, num_epochs, batch_size, grad_acumm, train_csv, eval_csv,
# Set here the path that the checkpoints will be saved. Default: ./training/
OUT_PATH = os.path.join(out_path, "training")
- print("[FINETUNE] \033[94mStarting Step 2\033[0m - Fine-tuning the XTTS Encoder")
- print(f"[FINETUNE] \033[94mLanguage: \033[92m{language} \033[94mEpochs: \033[92m{num_epochs} \033[94mBatch size: \033[92m{batch_size}\033[0m \033[94mGrad accumulation steps: \033[92m{grad_acumm}\033[0m")
- print(f"[FINETUNE] \033[94mTraining : \033[92m{train_csv}\033[0m")
- print(f"[FINETUNE] \033[94mEvaluation : \033[92m{eval_csv}\033[0m")
- print(f"[FINETUNE] \033[94mModel used : \033[92m{model_path}\033[0m")
+ print(f"[FINETUNE] [INFO] Starting Step 2 - Fine-tuning the XTTS Model")
+ print(f"[FINETUNE] [INFO] Configuration Summary:")
+ print(f"[FINETUNE] [INFO] - Language: {language}")
+ print(f"[FINETUNE] [INFO] - Training Epochs: {num_epochs}")
+ print(f"[FINETUNE] [INFO] - Batch Size: {batch_size}")
+ print(f"[FINETUNE] [INFO] - Gradient Accumulation Steps: {grad_acumm}")
+ print(f"[FINETUNE] [INFO] File Paths:")
+ print(f"[FINETUNE] [INFO] - Training Data: {train_csv}")
+ print(f"[FINETUNE] [INFO] - Evaluation Data: {eval_csv}")
+ print(f"[FINETUNE] [INFO] - Base Model: {model_path}")
if torch.cuda.is_available():
torch.cuda.empty_cache()
- # Get the current device ID
+ # Get the current device ID
gpu_device_id = torch.cuda.current_device()
gpu_available_mem_gb = (torch.cuda.get_device_properties(gpu_device_id).total_memory - torch.cuda.memory_allocated(gpu_device_id)) / (1024 ** 3)
- print(f"[FINETUNE] \033[94mAvailable VRAM: \033[92m{gpu_available_mem_gb:.2f} GB\033[0m")
+ print(f"[FINETUNE] [INFO] GPU Memory Status:")
+ print(f"[FINETUNE] [INFO] - Available VRAM: {gpu_available_mem_gb:.2f} GB")
+
if gpu_available_mem_gb < 12:
- print(f"[FINETUNE]")
- print(f"[FINETUNE] \033[91m****** WARNING PRE-FLIGHT CHECKS FAILED ******* WARNING PRE-FLIGHT CHECKS FAILED *****\033[0m")
- print(f"[FINETUNE] \033[94mAvailable VRAM: \033[92m{gpu_available_mem_gb:.2f} GB\033[0m")
- print(f"[FINETUNE] \033[94mIf you are running on a Linux system and you have 12GB's or less of VRAM, this step\033[0m")
- print(f"[FINETUNE] \033[94mmay fail, due to not enough GPU VRAM. Windows systems will use system RAM as extended\033[0m")
- print(f"[FINETUNE] \033[94mVRAM and so should work ok. However, Windows machines will need enough System RAM\033[0m")
- print(f"[FINETUNE] \033[94mavailable. Please read the PFC help section available on the first tab of the web\033[0m")
- print(f"[FINETUNE] \033[94minterface for more information.\033[0m")
- print(f"[FINETUNE] \033[91m****** WARNING PRE-FLIGHT CHECKS FAILED ******* WARNING PRE-FLIGHT CHECKS FAILED *****\033[0m")
- print(f"[FINETUNE]")
+ print("[FINETUNE] [WARNING] *** IMPORTANT MEMORY CONSIDERATION ***")
+ print("[FINETUNE] [WARNING] Your available VRAM is below the recommended 12GB threshold.")
+ print("[FINETUNE] [WARNING]")
+ print("[FINETUNE] [WARNING] System-Specific Considerations:")
+ print("[FINETUNE] [WARNING] - Windows: Will utilize system RAM as extended VRAM")
+ print("[FINETUNE] [WARNING] * Ensure sufficient system RAM is available")
+ print("[FINETUNE] [WARNING] * Recommended minimum: 24GB system RAM")
+ print("[FINETUNE] [WARNING] - Linux: Limited to physical VRAM only")
+ print("[FINETUNE] [WARNING] * Training may fail with insufficient VRAM")
+ print("[FINETUNE] [WARNING] * Consider reducing batch size or using gradient accumulation")
+ print("[FINETUNE] [WARNING]")
+ print("[FINETUNE] [WARNING] For detailed memory management strategies and optimization tips:")
+ print("[FINETUNE] [WARNING] 1. Refer to the 'Memory Management' section in the Training Guide")
+ print("[FINETUNE] [WARNING] 2. Review the Pre-flight Check tab for system requirements")
# Create the directory
os.makedirs(OUT_PATH, exist_ok=True)
@@ -933,7 +1815,7 @@ def train_gpt(language, num_epochs, batch_size, grad_acumm, train_csv, eval_csv,
training_assets = None
if (out_path / "bpe_tokenizer-vocab.json").exists():
- print("[FINETUNE] Using custom BPE tokenizer")
+ print("[FINETUNE] [INFO] Using custom BPE tokenizer")
training_assets = {
'Tokenizer': str(out_path / "bpe_tokenizer-vocab.json")
}
@@ -1076,7 +1958,7 @@ def train_gpt(language, num_epochs, batch_size, grad_acumm, train_csv, eval_csv,
# These are Hardcoded learning rate schedulers
- print(f"[FINETUNE] Learning Scheduler {lr_scheduler}, params {lr_scheduler_params}")
+ print(f"[FINETUNE] [INFO] Learning Scheduler {lr_scheduler}, params {lr_scheduler_params}")
# training parameters config
config = GPTTrainerConfig(
@@ -1143,12 +2025,13 @@ def train_gpt(language, num_epochs, batch_size, grad_acumm, train_csv, eval_csv,
training_assets=training_assets,
c_logger=c_logger,
warmup=warm_up,
-
)
+
+ c_logger.update_model_path(OUT_PATH)
if(disable_shared_memory):
# Limit training to GPU memory instead of shared memory
- print("[FINETUNE] Limiting GPU memory to 95% to prevent spillover")
+ print("[FINETUNE] [INFO] Limiting GPU memory to 95% to prevent spillover")
torch.cuda.set_per_process_memory_fraction(0.95)
trainer.fit()
@@ -1206,7 +2089,7 @@ def load_model(xtts_checkpoint, xtts_config, xtts_vocab):
if torch.cuda.is_available():
XTTS_MODEL.cuda()
- print("[FINETUNE] Model Loaded!")
+ print("[FINETUNE] [INFO] Model Loaded!")
return "Model Loaded!"
def run_tts(lang, tts_text, speaker_audio_file):
@@ -1239,15 +2122,37 @@ def run_tts(lang, tts_text, speaker_audio_file):
return "Speech generated !", out_path, speaker_audio_file
-def get_available_voices(minimum_size_kb=1200, speaker_name=None):
+def get_available_voices(min_duration_seconds=6, speaker_name=None):
+ """Get available voice files based on minimum duration.
+
+ Args:
+ min_duration_seconds (int): Minimum duration in seconds
+ speaker_name (str, optional): Project name for custom directory
+
+ Returns:
+ list: Sorted list of audio file paths meeting duration requirement
+ """
directory = out_path
if speaker_name and speaker_name != 'personsname':
directory = this_dir / "finetune" / speaker_name
- voice_files = [
- voice for voice in Path(f"{directory}/wavs").glob("*.wav")
- if voice.stat().st_size > minimum_size_kb * 1200 # Convert KB to bytes
- ]
- return sorted([str(file) for file in voice_files]) # Return full path as string
+
+ valid_files = []
+ wav_files = Path(f"{directory}/wavs").glob("*.wav")
+
+ for voice_file in wav_files:
+ try:
+ # Load audio file and get duration
+ waveform, sample_rate = torchaudio.load(str(voice_file))
+ duration = waveform.size(1) / sample_rate # Duration in seconds
+
+ if duration >= float(min_duration_seconds):
+ valid_files.append(str(voice_file))
+
+ except Exception as e:
+ print(f"[FINETUNE] Error processing {voice_file}: {str(e)}")
+ continue
+
+ return sorted(valid_files)
def find_best_models(directory, speaker_name=None):
if speaker_name and speaker_name != 'personsname':
@@ -1317,7 +2222,8 @@ def compact_custom_model(xtts_checkpoint_copy, folder_path, overwrite_existing):
# Get the source directory (either tmp-trn or custom named directory)
source_dir = xtts_checkpoint_copy.parent.parent.parent # Go up to the base directory
- print(f"[FINETUNE] Source directory: {source_dir}")
+ print(f"[FINETUNE] === File Copy Operations ===")
+ print(f"[FINETUNE] Source base directory: {source_dir}")
print(f"[FINETUNE] Target directory: {target_dir}")
try:
@@ -1334,28 +2240,38 @@ def compact_custom_model(xtts_checkpoint_copy, folder_path, overwrite_existing):
if "dvae" in key:
del checkpoint["model"][key]
+ print(f"[FINETUNE] Processing model.pth:")
+ print(f"[FINETUNE] From: {xtts_checkpoint_copy}")
+ print(f"[FINETUNE] To: {target_dir / 'model.pth'}")
torch.save(checkpoint, target_dir / "model.pth")
- def copy_files(src_folder, dest_folder, files):
- for file_name in files:
- src_path = src_folder / file_name
- dest_path = dest_folder / file_name
- if src_path.exists():
- shutil.copy2(src_path, dest_path)
- print(f"[FINETUNE] Copied {src_path} to {dest_path}")
- else:
- print(f"[FINETUNE] Warning: {src_path} does not exist and will not be copied.")
-
# Copy first set of files
folder_path_new = xtts_checkpoint_copy.parent
- files_to_copy = ["vocab.json", "config.json"]
- copy_files(folder_path_new, target_dir, files_to_copy)
+ print(f"[FINETUNE] Copying config files:")
+ for file_name in ["vocab.json", "config.json"]:
+ src_path = folder_path_new / file_name
+ dest_path = target_dir / file_name
+ if src_path.exists():
+ shutil.copy2(src_path, dest_path)
+ print(f"[FINETUNE] {file_name}:")
+ print(f"[FINETUNE] From: {src_path}")
+ print(f"[FINETUNE] To: {dest_path}")
+ else:
+ print(f"[FINETUNE] Warning: {src_path} not found")
# Copy second set of files from chkptandnorm directory
chkptandnorm_path = source_dir / "chkptandnorm"
- print(f"[FINETUNE] Looking for support files in: {chkptandnorm_path}")
- files_to_copy2 = ["speakers_xtts.pth", "mel_stats.pth", "dvae.pth"]
- copy_files(chkptandnorm_path, target_dir, files_to_copy2)
+ print(f"[FINETUNE] Copying support files:")
+ for file_name in ["speakers_xtts.pth", "mel_stats.pth", "dvae.pth"]:
+ src_path = chkptandnorm_path / file_name
+ dest_path = target_dir / file_name
+ if src_path.exists():
+ shutil.copy2(src_path, dest_path)
+ print(f"[FINETUNE] {file_name}:")
+ print(f"[FINETUNE] From: {src_path}")
+ print(f"[FINETUNE] To: {dest_path}")
+ else:
+ print(f"[FINETUNE] Warning: {src_path} not found")
# Create directories for different categories of WAV files
target_wavs_dir = target_dir / "wavs"
@@ -1370,7 +2286,9 @@ def compact_custom_model(xtts_checkpoint_copy, folder_path, overwrite_existing):
# Process WAV files
source_wavs_dir = source_dir / "wavs"
- print(f"[FINETUNE] Processing WAV files from: {source_wavs_dir}")
+ print(f"[FINETUNE] Processing WAV files:")
+ print(f"[FINETUNE] From: {source_wavs_dir}")
+ print(f"[FINETUNE] To: {target_wavs_dir}")
file_stats = {
"too_short": [],
@@ -1412,6 +2330,12 @@ def compact_custom_model(xtts_checkpoint_copy, folder_path, overwrite_existing):
except Exception as e:
print(f"[FINETUNE] Error processing {file_path.name}: {str(e)}")
+ print(f"[FINETUNE] WAV File Statistics:")
+ print(f"[FINETUNE] Suitable files (6-30s): {len(file_stats['suitable'])}")
+ print(f"[FINETUNE] Too short files (<6s): {len(file_stats['too_short'])}")
+ print(f"[FINETUNE] Too long files (>30s): {len(file_stats['too_long'])}")
+ print(f"[FINETUNE] === File Copy Operations Complete ===")
+
# Create report file
report_content = """WAV Files Processing Report
===========================
@@ -1499,11 +2423,11 @@ def delete_training_data():
except PermissionError:
print(f"[FINETUNE] PermissionError: Could not delete {item}. Skipping.")
- print(f"[FINETUNE] Folder {folder_to_delete} contents (excluding trainer_0_log.txt) deleted successfully.")
- return "Folder '/finetune/tmp-trn/' contents (excluding trainer_0_log.txt) deleted successfully."
+ print(f"[FINETUNE] Project Name folder > {folder_to_delete} < was deleted successfully.")
+ return "Specified Project Name folder & tmp data was deleted successfully."
else:
- print(f"[FINETUNE] Folder {folder_to_delete} does not exist.")
- return "Folder '/finetune/tmp-trn/' does not exist."
+ print(f"[FINETUNE] Project Name folder > {folder_to_delete} < does not exist.")
+ return "Specified Project Name folder could not be found."
def clear_folder_contents(folder_path):
# Check if the folder exists before clearing its contents
@@ -1538,39 +2462,82 @@ def delete_voice_sample_contents():
#### OTHER Generic ####
#######################
# define a logger to redirect
+import logging
class Logger:
- def __init__(self, filename="finetune.log"):
- self.log_file = filename
- self.terminal = sys.stdout
- self.log = open(self.log_file, "w")
+ _instance = None
+
+ def __new__(cls, *args, **kwargs):
+ if cls._instance is None:
+ cls._instance = super(Logger, cls).__new__(cls)
+ cls._instance.log_file = "finetune.log"
+ cls._instance.terminal = sys.stdout
+ cls._instance.current_model_path = None # To store current training path
+
+ # Open in append mode
+ cls._instance.log = open(cls._instance.log_file, "a")
+ return cls._instance
+
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def set_model_path(self, path):
+ self.current_model_path = path
def write(self, message):
- # Filter out problematic whitespace characters
filtered_message = ''.join(char for char in message
if char.isprintable() or char in '\n\r\t')
self.terminal.write(filtered_message)
- self.log.write(filtered_message)
+ try:
+ self.log.write(filtered_message)
+ self.log.flush()
+ except:
+ pass
def flush(self):
self.terminal.flush()
- self.log.flush()
+ try:
+ self.log.flush()
+ except:
+ pass
def isatty(self):
return False
+
+ def clear_log(self):
+ """Delete and recreate the log file to clear its contents."""
+ try:
+ # Close the current append-only file handle
+ self.log.close()
+ # Delete the log file
+ os.remove(self.log_file)
+ # Reopen the file in append mode for further logging
+ self.log = open(self.log_file, "a")
+ except Exception as e:
+ print(f"Failed to delete and recreate log file: {e}")
-# redirect stdout and stderr to a file
-sys.stdout = Logger()
-sys.stderr = sys.stdout
+_logging_setup_done = False
-# logging.basicConfig(stream=sys.stdout, level=logging.INFO)
-import logging
-logging.basicConfig(
- level=logging.INFO,
- format="[FINETUNE] %(message)s",
- handlers=[
- logging.StreamHandler(sys.stdout)
- ]
-)
+def setup_logging():
+ global _logging_setup_done
+ if _logging_setup_done:
+ return
+
+ # redirect stdout and stderr to a file
+ sys.stdout = Logger()
+ sys.stderr = sys.stdout
+
+ logging.basicConfig(
+ level=logging.INFO,
+ format="[FINETUNE] %(message)s",
+ handlers=[
+ logging.StreamHandler(sys.stdout)
+ ]
+ )
+
+ _logging_setup_done = True
+
+# Call setup at module level
+setup_logging()
c_logger = MetricsLogger()
def load_metrics():
@@ -1589,12 +2556,12 @@ def cleanup_before_exit(signum, frame):
sys.exit(0)
def create_refresh_button(refresh_components, refresh_methods, elem_class, interactive=True):
- def refresh(speaker_name):
+ def refresh(speaker_name, min_duration_seconds):
updates = {}
for component, method in zip(refresh_components, refresh_methods):
- args = method(speaker_name=speaker_name) if callable(method) else method
+ # Pass both speaker_name and min_duration_seconds to the method
+ args = method(speaker_name=speaker_name, min_duration_seconds=min_duration_seconds) if callable(method) else method
if args and 'choices' in args:
- # Select the most recent file (last in the sorted list)
args['value'] = args['choices'][-1] if args['choices'] else ""
for k, v in args.items():
setattr(component, k, v)
@@ -1604,7 +2571,7 @@ def create_refresh_button(refresh_components, refresh_methods, elem_class, inter
refresh_button = gr.Button("Refresh Dropdowns", elem_classes=elem_class, interactive=interactive)
refresh_button.click(
fn=refresh,
- inputs=[speaker_name_input_testing],
+ inputs=[speaker_name_input_testing, min_audio_length],
outputs=refresh_components
)
@@ -1618,13 +2585,10 @@ def create_refresh_button_next(refresh_components, refresh_methods, elem_class,
else:
out_path = this_dir / "finetune" / "tmp-trn"
- print(f"[FINETUNE] Refresh using directory: {out_path}")
-
updates = {}
for component, method in zip(refresh_components, refresh_methods):
args = method(speaker_name=speaker_name) if callable(method) else method
if args and 'choices' in args:
- print(f"[FINETUNE] Found files: {args['choices']}")
args['value'] = args['choices'][-1] if args['choices'] else ""
for k, v in args.items():
setattr(component, k, v)
@@ -1639,27 +2603,70 @@ def create_refresh_button_next(refresh_components, refresh_methods, elem_class,
)
return refresh_button
-
-pfc_markdown = f"""
- ### 🚀 Pre-flight Checklist for Fine-tuning
- ◽ Ensure each criterion is marked with a green check mark ✅ and a Pass status. Finetuning will fail otherwise.
- ◽ The help tabs along the top will assist in resolving issues and you can also find additional help guides on the AllTalk [GitHub repository](https://github.com/erew123/alltalk_tts#-finetuning-a-model).
- ◽ For an overview of fine-tuning procedures, please refer to the "General Finetuning info" tab or visit the AllTalk [GitHub repository](https://github.com/erew123/alltalk_tts#-finetuning-a-model).
- """
-
+################################
+#### Gradio CSS for Styling ####
+################################
custom_css = """
-body {
- font-size: 16px; /* Adjust the base font size as needed */
+/* Status indicators and labels */
+.status-label {
+ font-size: 1.2em;
+ font-weight: bold;
+ padding: 10px;
+ border-radius: 5px;
+ margin: 10px 0;
}
-h1, h2, h3, h4, h5, h6 {
- font-size: 1.25em; /* Adjust heading sizes relative to the base size */
+
+.status-indicator {
+ padding: 5px 10px;
+ border-radius: 3px;
+ font-weight: bold;
}
-p {
- font-size: 1.1em; /* Paragraph font size, relative to the base size */
- margin-bottom: 10px; /* Adjust paragraph spacing */
+
+/* Layout components */
+.status-box {
+ display: flex;
+ align-items: center;
+ padding: 0.5rem;
+ margin-bottom: 0.5rem;
+ border-radius: 4px;
+ background: var(--background-fill-secondary);
}
-.gradio_container {
- zoom: 1.1; /* Adjust the zoom to scale the entire container */
+
+.status-group {
+ margin-bottom: 1rem;
+ padding: 1rem;
+ border-radius: 8px;
+ background: var(--background-fill-primary);
+}
+
+/* Help section styling */
+.help-section {
+ margin-top: 2rem;
+ border-top: 1px solid var(--border-color-primary);
+ padding-top: 1rem;
+}
+
+.help-accordion {
+ margin-bottom: 0.5rem;
+}
+
+/* Ensure consistent font size in accordions */
+.gradio-container .prose,
+.gradio-container .prose p,
+.gradio-container .prose ul,
+.gradio-container .prose ol,
+.gradio-container .prose ul li,
+.gradio-container .prose ol li {
+ font-size: 1.02em !important; /* Adjust to the preferred size, e.g., 1em or 14px */
+ line-height: 1.3; /* Improves readability */
+}
+
+/* Grid layout */
+.check-grid {
+ display: grid;
+ grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+ gap: 1rem;
+ margin-bottom: 1rem;
}
"""
@@ -1726,129 +2733,14 @@ if __name__ == "__main__":
// localStorage.setItem('darkMode', 'enabled');
}
}""", show_api=False)
- with gr.Tab("🚀 PFC"):
- with gr.Tab("🚀 Pre-Flight Checklist"):
- gr.Markdown(
- f"""
- {pfc_markdown}
- {disk_space_results}
- {system_ram_results}
- {cuda_results}
- {pytorch_results}
- {base_model_results}
- {tts_version_status}
- """
- )
- with gr.Tab("🟩 Disks Help"):
- gr.Markdown(
- f"""
- {disk_space_results}
- ◽ During actual training (Step 2) Finetuning will require approximately 18GB's of free disk space while performing training and will fail or perform badly if there is any less disk space. The majority of this disk space is used temporarily and will be cleared when you reach Step 4 and move & compact the model then delete the training data.
- ◽ Because lots of data is being copied around, mechanical hard disks will be slow.
- """
- )
- with gr.Tab("🟪 RAM & VRAM Help"):
- gr.Markdown(
- f"""
- {system_ram_results}
- ◽ During actual training (Step 2) Finetuning will use around **14GB's** of VRAM. If your GPU doesnt have 14GB's of VRAM:
- - **Windows** systems will attempt to extend VRAM into your System RAM, and so should work.
- - **Linux** systems can only use the available VRAM, so may fail on 12GB VRAM or smaller GPU's.
- ◽ For **Windows** users with 12GB or less, if you also have very low or slow System RAM you can expect bad performance or the training to fail.
- - 12GB cards may need 2GB System RAM.
- - 8GB cards may need 6GB System RAM.
- - 6GB cards may need 8GB System RAM.
- ◽ Its hard to estimate what performance impact this could will have, due to different memory speeds, PCI speeds, GPU speeds etc.
- ◽ If you have a low VRAM scenario and also are attempting to run on a mechanical hard drive, Finetuning could take ??? amount of time.
- ◽ On Windows machines, please ensure you have **not** disabled System Memory Fallback for Stable Diffusion link here
- """
- )
- with gr.Tab("🟨 CUDA & Cublas Help"):
- gr.Markdown(
- f"""
- {cuda_results}
- ◽ It DOESNT matter what version of CUDA you have installed within Python either, CUDA 11.8, CUDA 12.1 etc. The NVIDIA CUDA Development Toolkit is a completly different and seperate thing from Python/PyTorch.
- ◽ Finetuning simply wants to access a tool within the CUDA Development Toolkit 11.8 called Cublas64_11.
- ◽ If you dont have the toolkit installed, the idea is just to install the smallest bit possible and this will not affect or impact other things on your system.
- ◽ You will need to download the Nvidia Cuda Toolkit 11.8 network install from link here
- ◽ 1) Run the installer and select Custom Advanced Uncheck everything at the top then expand CUDA, Development > Compiler > and select nvcc then expand Libraries and select CUBLAS.
- ◽ 2) Back at the top of CUDA, expand Runtime > Libraries and select CUBLAS. Click Next, accept the default path (taking a note of its location) and let the install run.
- ◽ 3) You should be able to drop to your terminal or command prompt and type nvcc --version and have it report Cuda compilation tools, release 11.8. If it does you are good to go. If it doesn't > Step 4.
- ◽ 4) Linux users, you can temporarily add these paths on your current terminal window with (you may need to confirm these are correct for your flavour of Linux):
- export LD_LIBRARY_PATH=/usr/local/cuda-11.8/lib64:${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} (Add it to your ~/.bashrc if you want this to be permanent)
- export LD_LIBRARY_PATH=/usr/local/cuda-11.8/bin
- Windows users need the add the following to the PATH environment variable. Start menu and search for "Environment Variables" or "Edit the system environment variables.".
- Find and select the "Path" variable, then click on the "Edit...". Click on the "New" button and add:
- C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\bin.
- ◽ 5) Once you have these set correctly, you should be able to open a new command prompt/terminal and nvcc --version at the command prompt/terminal, resulting in Cuda compilation tools, release 11.8.
- ◽ 6) If the nvcc command doesn't work OR it reports a version different from 11.8, finetuning wont work, so you will to double check your environment variables and get them working correctly.
- """
- )
- with gr.Tab("🟦 Python & PyTorch Help"):
- gr.Markdown(
- f"""
- {pytorch_results}
- ◽ On the PyTorch version the:
- - first few digits are the version of PyTorch e.g. 2.1.0 is PyTorch 2.1.0
- - last few digits refer to the CUDA version e.g. cu118 is Cuda 11.8. cu121 is Cuda 12.1.
- ◽ Ensure you have started your Python envuronment before running finetuning otherwise you will have failures on the above checks.
- ◽ If PyTorch does not show a CUDA version, then PyTorch will need reinstalling with CUDA. I would suggest running pip cache purge before installing PyTorch again.
- ◽ It DOESNT matter what version of PyTorch and CUDA you have installed within Python, CUDA 11.8, CUDA 12.1 etc. The NVIDIA CUDA Development Toolkit is a completly different and seperate thing.
- ◽ Finetuning simply wants to access a tool within the CUDA Development Toolkit called Cublas64_11.
- ◽ If you dont have the toolkit installed, the idea is just to install the smallest bit possible and this will not affect or impact other things on your system.
- """
- )
- with gr.Tab("⬛ XTTS Base Model Help"):
- gr.Markdown(
- f"""
- {base_model_results}
- ◽ If your basemodel is not being detected, please ensure that finetune.py is being run from the AllTalk main folder.
- ◽ Ensure you have started AllTalk normally at least once and downlaoded a XTTS model.
- ◽ Check that there is an XTTS model within the models folder e.g. /models/xtts/model-folder-here/
- ◽ The files required inside the folder are "model.pth", "vocab.json", "config.json", "dvae.pth", "mel_stats.pth", "speakers_xtts.pth".
- """
- )
- with gr.Tab("🟥 TTS Version Help"):
- gr.Markdown(
- f"""
- {tts_version_status}
- ◽ If your TTS version is showing as the incorrect version, please reinstall the Finetuning requirements at the command prompt/terminal.
- ◽ pip install -r requirements_finetune.txt
- """
- )
-
- with gr.Tab("ℹ️ General Finetuning info"):
- gr.Markdown(
- f"""
- ### 🟥 Important Note
- ◽ finetune.py needs to be run from the /alltalk_tts/ folder. Don't move the location of this script.
- ### 🟦 What you need to run finetuning
- ◽ An Nvidia GPU.
- ◽ If you have multiple Nvidia GPU's in your system, please see the Github Help section [Multiple GPU's](https://github.com/erew123/alltalk_tts#performance-and-compatibility-issues).
- ◽ Some decent quality audio, multiple files if you like. Minimum of 2 minutes and Ive tested up to 20 minutes of audio.
- ◽ There is no major need to chop down your audio files into small slices as Step 1 will do that for you automatically and prepare the training set. But this can be helpful in cases where:
- - Whisper doesnt correctly detect and split down audio files.
- - You do not get a lot of "Speaker Reference Audio" files at the end of training.
- ◽ This process will need access to all your GPU and VRAM, so close any other software that's using your GPU currently.
- ### 🟨 What do I do from here?
- ◽ Proceed through Step 1, 2, 3 and onto "What to do next".
- ### 🟩 Additional Information
- ◽ Guidance is provided on each step of the process however, if you are after more detailed information please visit:
- ◽ [AllTalk Github Finetuning](https://github.com/erew123/alltalk_tts#-finetuning-a-model)
- ◽ [Coqui XTTS Documentation](https://docs.coqui.ai/en/latest/index.html)
- """
- )
+ with gr.Tab("🚀 Pre-flight Checklist"):
+ create_pfc_interface()
#######################
#### GRADIO STEP 1 ####
#######################
with gr.Tab("📁 Step 1 - Generating the dataset"):
with gr.Tab("Generate Dataset"):
- # out_path = gr.Textbox(
- # label="Output path (where data and checkpoints will be saved):",
- # value=out_path,
- # visible=False,
- # )
# Define directories
this_dir = Path(__file__).parent.resolve()
voice_samples_dir = this_dir / "finetune" / "put-voice-samples-in-here"
@@ -1899,7 +2791,7 @@ if __name__ == "__main__":
- If you wish to delete previously uploaded audio samples files then use 'Delete Existing Audio Samples'.
- If you wish to delete previously generated training datasets, please use 'Delete Existing Training Dataset'.
- - If you wish to re-use your previously created training data, just click 'Create Dataset'.
+ - If you wish to re-use your previously created training data, fill in the `Training Project Name` corectly and click 'Create Dataset'.
""")
with gr.Row():
output_text = gr.Textbox(label="Audio File Management Result", interactive=False)
@@ -1908,6 +2800,18 @@ if __name__ == "__main__":
audio_upload_button.click(upload_audio, inputs=audio_files_upload, outputs=output_text)
delete_audio_button.click(delete_existing_audio, outputs=output_text)
delete_dataset_button.click(delete_existing_training_data, outputs=output_text)
+
+ def update_language_options(model):
+ # English-only models
+ if model in ["tiny.en", "base.en", "small.en", "medium.en"]:
+ languages = ["en"]
+ else:
+ # Multilingual models
+ languages = [
+ "en", "es", "fr", "de", "it", "pt", "pl", "tr", "ru", "nl", "cs",
+ "ar", "zh", "hu", "ko", "ja"
+ ]
+ return gr.Dropdown(choices=languages, value=languages[0])
with gr.Row():
speaker_name_input = gr.Textbox(
@@ -1918,19 +2822,25 @@ if __name__ == "__main__":
)
whisper_model = gr.Dropdown(
label="Whisper Model",
- value="distil-large-v3",
+ value="large-v3",
choices=[
- ("large-v3", "large-v3"),
+ ("tiny.en", "tiny.en"),
+ ("tiny", "tiny"),
+ ("base.en", "base.en"),
+ ("base", "base"),
+ ("small.en", "small.en"),
+ ("small", "small"),
+ ("medium.en", "medium.en"),
+ ("medium", "medium"),
+ ("large-v1", "large-v1"),
("large-v2", "large-v2"),
+ ("large-v3", "large-v3"),
("large", "large"),
- ("medium", "large"),
- ("small", "large"),
- ("distil-large-v3 (en only)", "distil-large-v3"),
- ("distil-large-v2 (en only)", "distil-large-v2"),
+ ("large-v3-turbo", "large-v3-turbo"),
+ ("turbo", "turbo")
],
scale=2,
)
-
lang = gr.Dropdown(
label="Dataset Language",
value="en",
@@ -1973,6 +2883,130 @@ if __name__ == "__main__":
value=False,
info='Custom Tokenizer for training'
)
+ use_vad = gr.Checkbox(
+ label="VAD",
+ value=True,
+ info="Enable Silero VAD for better speech detection"
+ )
+ precision = gr.Dropdown(
+ label="Model Precision",
+ value="mixed",
+ choices=[
+ ("Mixed", "mixed"),
+ ("FP16", "float16"),
+ ("FP32", "float32")
+ ],
+ )
+
+
+ with gr.Accordion("🔍 Dataset Creation Debug Settings", open=False):
+ def update_debug_levels(gpu, model, data, general, audio, segments, duplicates):
+ DebugLevels.GPU_MEMORY = gpu
+ DebugLevels.MODEL_OPS = model
+ DebugLevels.DATA_PROCESS = data
+ DebugLevels.GENERAL = general
+ DebugLevels.AUDIO = audio
+ DebugLevels.SEGMENTS = segments
+ DebugLevels.DUPLICATES = duplicates
+ return "Debug settings updated"
+
+ def select_all_debug():
+ return {
+ debug_gpu: True,
+ debug_model: True,
+ debug_data: True,
+ debug_general: True,
+ debug_audio: True,
+ debug_segments: True,
+ debug_duplicates: True
+ }
+
+ def clear_all_debug():
+ return {
+ debug_gpu: False,
+ debug_model: False,
+ debug_data: False,
+ debug_general: False,
+ debug_audio: False,
+ debug_segments: False,
+ debug_duplicates: False
+ }
+
+ with gr.Row():
+ gr.Markdown("""
+ Enable or disable different types of debug messages during dataset creation.
+ These settings will apply to the current session only.
+ """)
+
+ with gr.Row():
+ with gr.Column(scale=1):
+ debug_gpu = gr.Checkbox(
+ label="GPU Memory",
+ value=DebugLevels.GPU_MEMORY,
+ info="GPU memory and CUDA related debugging"
+ )
+ debug_model = gr.Checkbox(
+ label="Model Operations",
+ value=DebugLevels.MODEL_OPS,
+ info="Model loading, transcription, cleanup operations"
+ )
+ debug_data = gr.Checkbox(
+ label="Data Processing",
+ value=DebugLevels.DATA_PROCESS,
+ info="Data processing, words, sentences"
+ )
+
+ with gr.Column(scale=1):
+ debug_general = gr.Checkbox(
+ label="General",
+ value=DebugLevels.GENERAL,
+ info="General flow, file operations, metadata"
+ )
+ debug_audio = gr.Checkbox(
+ label="Audio",
+ value=DebugLevels.AUDIO,
+ info="Audio processing statistics and info"
+ )
+ debug_segments = gr.Checkbox(
+ label="Segments",
+ value=DebugLevels.SEGMENTS,
+ info="Detailed segment information"
+ )
+ debug_duplicates = gr.Checkbox(
+ label="Duplicates",
+ value=DebugLevels.DUPLICATES,
+ info="Duplicate handling information"
+ )
+
+ with gr.Row():
+ debug_select_all = gr.Button("Select All")
+ debug_clear_all = gr.Button("Clear All")
+
+ # Debug update functions
+ for checkbox in [debug_gpu, debug_model, debug_data, debug_general,
+ debug_audio, debug_segments, debug_duplicates]:
+ checkbox.change(
+ fn=update_debug_levels,
+ inputs=[debug_gpu, debug_model, debug_data, debug_general,
+ debug_audio, debug_segments, debug_duplicates],
+ outputs=[gr.Textbox(visible=False)] # Hidden status output
+ )
+
+ debug_select_all.click(
+ fn=select_all_debug,
+ inputs=[],
+ outputs=[debug_gpu, debug_model, debug_data, debug_general,
+ debug_audio, debug_segments, debug_duplicates]
+ )
+
+ debug_clear_all.click(
+ fn=clear_all_debug,
+ inputs=[],
+ outputs=[debug_gpu, debug_model, debug_data, debug_general,
+ debug_audio, debug_segments, debug_duplicates]
+ )
+
+
progress_data = gr.Label(
label="Progress:"
)
@@ -1982,18 +3016,31 @@ if __name__ == "__main__":
lines=10,
)
demo.load(read_logs, None, logs, every=1)
-
+ # Update `lang` options when the `whisper_model` changes
+ whisper_model.change(fn=update_language_options, inputs=whisper_model, outputs=lang)
prompt_compute_btn = gr.Button(value="Step 1 - Create dataset")
- def preprocess_dataset(language, whisper_model, max_sample_length, eval_split_number, speaker_name_input, create_bpe_tokenizer, progress=gr.Progress()):
+ def preprocess_dataset(language, whisper_model, max_sample_length, eval_split_number,
+ speaker_name_input, create_bpe_tokenizer, use_vad, precision,
+ progress=gr.Progress()):
clear_gpu_cache()
- test_for_audio_files = [file for file in os.listdir(audio_folder) if any(file.lower().endswith(ext) for ext in ['.wav', '.mp3', '.flac'])]
+ test_for_audio_files = [file for file in os.listdir(audio_folder)
+ if any(file.lower().endswith(ext) for ext in ['.wav', '.mp3', '.flac'])]
if not test_for_audio_files:
return "I cannot find any mp3, wav or flac files in the folder called 'put-voice-samples-in-here'", "", ""
else:
try:
-
- train_meta, eval_meta, audio_total_size = format_audio_list(target_language=language, whisper_model=whisper_model, max_sample_length=max_sample_length, eval_split_number=eval_split_number, speaker_name_input=speaker_name_input, create_bpe_tokenizer=create_bpe_tokenizer, gradio_progress=progress)
+ train_meta, eval_meta, audio_total_size = format_audio_list(
+ target_language=language,
+ whisper_model=whisper_model,
+ max_sample_length=max_sample_length,
+ eval_split_number=eval_split_number,
+ speaker_name_input=speaker_name_input,
+ create_bpe_tokenizer=create_bpe_tokenizer,
+ gradio_progress=progress,
+ use_vad=use_vad,
+ precision=precision
+ )
except:
traceback.print_exc()
error = traceback.format_exc()
@@ -2001,205 +3048,493 @@ if __name__ == "__main__":
clear_gpu_cache()
- # if audio total len is less than 2 minutes raise an error
if audio_total_size < 120:
message = "The total duration of the audio file or files you provided, was less than 2 minutes in length. Please add more audio samples."
- print("[FINETUNE] ",message)
+ print("[FINETUNE] ", message)
return message, "", ""
- print("[FINETUNE] Dataset Generated. Either run Dataset Validation or move to Step 2")
- return "Dataset Generated. Either run Dataset Validation or move to Step 2", train_meta, eval_meta, speaker_name_input
+ # Final GPU after full cleanup
+ get_gpu_memory()
+
+ print("[FINETUNE] [INFO] Dataset Generated. Either run Dataset Validation or move to Step 2")
+ # Return 3x speaker_name_input's at the end to satisfy `prompt_compute_btn.click` populating all the `personname` Project Name boxes across the Gradio interface
+ return "Dataset Generated. Either run Dataset Validation or move to Step 2", train_meta, eval_meta, speaker_name_input, speaker_name_input, speaker_name_input
- def play_selected_audio(selected_file):
- if selected_file and selected_file != "Please Generate Your Dataset":
- return selected_file
- return None
-
- with gr.TabItem("Dataset Validation"):
- with gr.Row():
- gr.Markdown("""# Audio Transcription Validation
-
- This tool helps validate and correct transcriptions in your dataset by comparing:
- - The original transcriptions stored in your CSV files
- - New Whisper transcriptions of the same audio files
-
- Only mismatched transcriptions are shown here for review.
-
- ### Understanding the Display:
- - **Original Text**: Current transcription from metadata_eval.csv/metadata_train.csv
- - **Whisper Text**: New transcription from Whisper
- - **Filename**: The audio file being transcribed""")
- gr.Markdown("""
- ### How to Use:
- 1. Click any row in the table to:
- - Listen to the audio
- - View both transcriptions
- 2. Choose how to correct the transcription:
- - **Use Original**: No need to update if you are happy with the current transcription.
- - **Use Whisper**: Update with the new Whisper transcription
- - **Edit Manually**: Write your own corrected transcription
-
- When you save a correction, it will update the appropriate CSV file and refresh the display. This helps ensure your training dataset has the most accurate transcriptions possible.""")
+
+ with gr.Tab("Dataset Generation Guide"):
+ gr.Markdown("""
+ # Dataset Generation Guide
+ Below you'll find comprehensive instructions and information about generating your dataset.
+ Click each section to expand its contents.
+ """)
- with gr.Row():
- load_button = gr.Button("Run Validation")
- progress_box = gr.Textbox(label="Progress", interactive=False)
-
- with gr.Row():
- with gr.Column(scale=2):
- # Store full DataFrame in state
- state = gr.State()
- # Display DataFrame shows only visible columns
- mismatch_table = gr.DataFrame(
- headers=["Original Text", "Whisper Text", "Filename"],
- datatype=["str", "str", "str"],
- interactive=False,
- wrap=True
- )
+ with gr.Accordion("🎯 Quick Start Guide", open=False):
+ gr.Markdown(f"""
+ 1. Place audio files (MP3, WAV, or FLAC) in folder `/alltalk_tts/finetune/put-voice-samples-in-here` or upload through the interface
+ 2. Provide 2+ minutes of clear audio (more is better and 5+ recommended). Variation in cadence and speech patterns recommended
+ 3. Set your project name and adjust settings as needed
+ 4. Generate the dataset
+ 5. When your dataset it generated, you can either validate the dataset to confirm transcriptions or move to Step 2
- with gr.Column(scale=1):
- audio_player = gr.Audio(label="Audio Player", interactive=False)
- current_expected = gr.Textbox(label="Original Text", interactive=False)
- current_transcribed = gr.Textbox(label="Whisper Text", interactive=False)
- text_choice = gr.Radio(
- choices=["Use Original", "Use Whisper", "Edit Manually"],
- label="Choose Transcription",
- value="Use Original"
- )
- manual_edit = gr.Textbox(
- label="Manual Edit",
- interactive=True,
- visible=False
- )
- current_index = gr.Number(visible=False)
- save_button = gr.Button("Save Correction")
- save_status = gr.Textbox(label="Save Status", interactive=False)
+ **Minimum Requirements:**
+ - 2 minutes of clear audio
+ - Consistent audio quality
+ - Minimal background noise
+ """)
+
+ with gr.Accordion("📋 Detailed Instructions", open=False):
+ gr.Markdown(f"""
+ #### Step-by-Step Process
+ 1. Review [Coqui's guide on creating a good dataset](https://docs.coqui.ai/en/latest/what_makes_a_good_dataset.html)
+ 2. Place your audio files in `{str(audio_folder)}` or upload through interface
+ 3. Ensure at least 2 minutes of audio (5+ minutes recommended)
+ 4. Set Maximum Audio Length (controls segment size per epoch)
+ 5. Enter unique Project Name for dedicated training folder
+ 6. Select appropriate Whisper model (see Model Selection guide below)
+ 7. Choose Dataset Language (en Wisper models only support English)
+ 8. Set Evaluation Split percentage (default 15% recommended)
+ 9. Consider enabling BPE Tokenizer for unique speech patterns
+ """)
+
+ with gr.Accordion("🔧 Process Overview", open=False):
+ gr.Markdown("""
+ #### What Happens During Generation
+ 1. Whisper transcribes and segments your audio
+ 2. System creates smaller clips with transcriptions
+ 3. Generates training and evaluation datasets
+ 4. Organizes files in project directory
+
+ #### Generated Files:
+ - `/finetune/[project_name]/lang.txt`: Language code
+ - `/finetune/[project_name]/metadata_train.csv`: Training data
+ - `/finetune/[project_name]/metadata_eval.csv`: Evaluation data
+ - `/finetune/[project_name]/wavs/`: Processed audio clips
+ """)
+
+ with gr.Accordion("🔍 Whisper Model Selection", open=False):
+ gr.Markdown("""
+ ### Choose Your Model
+
+ #### Quick Selection Guide:
+ - **English Only, Limited Resources:** `tiny.en` or `base.en`
+ - **Multilingual, Limited Resources:** `tiny` or `base`
+ - **Balanced Performance:** `small` or `medium`
+ - **Highest Quality:** `large-v3`
+ - **Speed Priority:** `large-v3-turbo` or `turbo`
+
+ ### Detailed Model Comparison
+ | Model | Language | Speed | VRAM | Best For |
+ |------------------|----------|---------------|-------|-----------------------------|
+ | `tiny.en` | English | ⚡️⚡️⚡️⚡️⚡️ | 0.5GB | Quick English tests |
+ | `tiny` | All | ⚡️⚡️⚡️⚡️⚡️ | 0.5GB | Quick multilingual tests |
+ | `base.en` | English | ⚡️⚡️⚡️⚡️ | 1GB | Basic English use |
+ | `base` | All | ⚡️⚡️⚡️⚡️ | 1GB | Basic multilingual use |
+ | `small.en` | English | ⚡️⚡️⚡️ | 2GB | Daily English use |
+ | `small` | All | ⚡️⚡️⚡️ | 2GB | Daily multilingual use |
+ | `medium.en` | English | ⚡️⚡️ | 4GB | Better English quality |
+ | `medium` | All | ⚡️⚡️ | 4GB | Better multilingual quality |
+ | `large-v1` | All | ⚡️ | 8GB | Legacy support |
+ | `large-v2` | All | ⚡️ | 8GB | Improved legacy |
+ | `large-v3` | All | ⚡️ | 10GB | Best overall quality |
+ | `large` | All | ⚡️ | 10GB | Standard high quality |
+ | `large-v3-turbo` | All | ⚡️⚡️⚡️ | 10GB | Fast high quality |
+ | `turbo` | All | ⚡️⚡️⚡️⚡️ | 10GB | Fastest high quality |
+ """)
+
+ with gr.Accordion("⚙️ Advanced Settings", open=False):
+ gr.Markdown("""
+ ### Voice Activity Detection (VAD)
+ - Recommended: Enabled
+ - Detects and isolates speech segments
+ - Removes silence and non-speech audio
+ - Improves accuracy and processing speed
+
+ ### Model Precision
+ #### Mixed Precision (Recommended for Modern GPUs)
+ - Balances speed and accuracy
+ - Best for NVIDIA GPUs Pascal architecture (GTX 10 series) and newer
+ - Automatically switches between FP16 and FP32 as needed
+ - Can significantly reduce VRAM usage
- def update_audio_player(evt: gr.SelectData, df):
- try:
- selected_row = df.iloc[evt.index]
- audio_path = selected_row['full_path']
-
- # If it's a Series, get the first value
- if isinstance(audio_path, pd.Series):
- audio_path = audio_path.iloc[0]
-
- # Clean the path string
- audio_path = str(audio_path).strip()
- print(f"[FINETUNE] Selected audio path: {audio_path}")
-
- # Get the text values directly from the row, ensuring we get strings not Series
- expected = str(selected_row['expected_text']).strip() if isinstance(selected_row['expected_text'], str) else selected_row['expected_text'].iloc[0]
- transcribed = str(selected_row['transcribed_text']).strip() if isinstance(selected_row['transcribed_text'], str) else selected_row['transcribed_text'].iloc[0]
-
- if not os.path.exists(audio_path):
- print(f"[FINETUNE] Warning: Audio file not found: {audio_path}")
- return {
- audio_player: None,
- current_expected: "",
- current_transcribed: "",
- current_index: None,
- save_status: f"Error: Audio file not found"
- }
-
- return {
- audio_player: audio_path,
- current_expected: expected,
- current_transcribed: transcribed,
- current_index: evt.index,
- save_status: "Ready to save correction"
- }
- except Exception as e:
- print(f"[FINETUNE] Error in update_audio_player: {str(e)}")
- print(f"[FINETUNE] Selected row data: {selected_row if 'selected_row' in locals() else 'Not available'}")
+ #### FP16 (Half Precision)
+ - Fastest processing and lowest memory usage
+ - Only supported on NVIDIA GPUs Pascal architecture (GTX 10 series) and newer
+ - May have slightly lower accuracy
+ - Not recommended for older GPUs as they lack hardware support
+
+ #### FP32 (Full Precision)
+ - Universal compatibility - works on all GPU models
+ - Required for older NVIDIA GPUs (pre-GTX 10 series)
+ - Default option for CPU processing
+ - Highest accuracy but uses more VRAM
+ - Slower processing speed compared to FP16/Mixed
+ - Use if experiencing stability issues with other precision modes
+
+ ### GPU Compatibility Guide
+
+ | Precision Mode | Description | Supported GPU Architectures | Examples of Compatible GPUs |
+ |------------------|-----------------------------------------------|--------------------------------------------------|----------------------------------------------------|
+ | **FP32 (Full Precision)** | Universal compatibility, highest accuracy | All GPUs, including older models | GTX 900 series and older, all AMD GPUs, all laptop GPUs before 2016 |
+ | **Mixed Precision (Recommended)** | Balances speed and accuracy, auto-switches between FP16 and FP32 | NVIDIA Turing (RTX 20 series) and newer | RTX 20 series, RTX 30 series, RTX 40 series, Tesla V100, A100 |
+ | **FP16 (Half Precision)** | Fastest processing, lowest memory usage | NVIDIA Turing and newer (with Tensor Cores) | RTX 20 series, RTX 30 series, RTX 40 series, Tesla V100, A100 |
+ """)
+
+ with gr.Accordion("🔍 Dataset Creation Debug Settings", open=False):
+ gr.Markdown("""
+ Each debug category serves a specific purpose in helping you understand what's happening during the dataset creation process:
+
+ #### GPU Memory Monitoring
+ - **What it shows:** Detailed GPU VRAM usage and allocation
+ - **When it's useful:**
+ * Troubleshooting out-of-memory errors
+ * Monitoring VRAM consumption during processing
+ * Understanding memory spikes during model loading
+ * Tracking memory cleanup effectiveness
+ - **Example messages:**
+ * "GPU Memory Status: Total: 8192 MB, Used: 3584 MB, Free: 4608 MB"
+ * "Low GPU memory available!"
+
+ #### Model Operations
+ - **What it shows:** Model loading, inference, and cleanup steps
+ - **When it's useful:**
+ * Tracking model initialization progress
+ * Monitoring transcription operations
+ * Debugging model-related errors
+ * Verifying proper model cleanup
+ - **Example messages:**
+ * "Loading Whisper model: large-v3"
+ * "Using FP16 precision"
+ * "Model successfully unloaded from GPU"
+
+ #### Data Processing
+ - **What it shows:** Text and dataset handling operations
+ - **When it's useful:**
+ * Understanding how text is being processed
+ * Tracking dataset creation progress
+ * Monitoring CSV file operations
+ * Debugging transcription issues
+ - **Example messages:**
+ * "Loaded existing training metadata"
+ * "Processing metadata and handling duplicates"
+ * "Created train/eval split with ratio 0.15"
+
+ #### General Flow
+ - **What it shows:** Overall process status and file operations
+ - **When it's useful:**
+ * Following the general progress
+ * Tracking file system operations
+ * Understanding process flow
+ * Identifying process bottlenecks
+ - **Example messages:**
+ * "Initializing output directory"
+ * "Dataset Generated. Move to Dataset Validation"
+ * "Successfully wrote metadata files"
+
+ #### Audio Processing
+ - **What it shows:** Audio file statistics and processing details
+ - **When it's useful:**
+ * Understanding audio segmentation
+ * Tracking audio duration statistics
+ * Identifying problematic audio files
+ * Monitoring audio quality issues
+ - **Example messages:**
+ * "Total segments: 157"
+ * "Average duration: 8.3s"
+ * "Found 3 segments under minimum duration"
+
+ #### Segments
+ - **What it shows:** Detailed information about audio segmentation
+ - **When it's useful:**
+ * Understanding how audio is being split
+ * Debugging segment length issues
+ * Tracking segment merging operations
+ * Identifying problematic splits
+ - **Example messages:**
+ * "Extended segment from 4.2s to 6.0s"
+ * "Merged 5 segments into 2 longer segments"
+ * "Segment too short (2.1s)"
+
+ #### Duplicates
+ - **What it shows:** Information about duplicate handling
+ - **When it's useful:**
+ * Tracking duplicate detection
+ * Understanding how duplicates are resolved
+ * Monitoring transcription quality
+ * Verifying duplicate cleanup
+ - **Example messages:**
+ * "Found 2 files with multiple transcriptions"
+ * "Re-transcribing duplicate files"
+ * "Updated transcription for file.wav"
+
+ ### Usage Tips
+ - Enable specific categories based on what you're trying to troubleshoot
+ - Use GPU Memory when dealing with VRAM issues
+ - Enable Segments when audio splitting seems incorrect
+ - Turn on Data Process when checking transcription quality
+ - Enable Duplicates when investigating repeated content
+ - Use General for overall process monitoring
+ - Enable Audio when investigating quality issues
+ - Turn on Model Ops when dealing with model-related problems
+ """)
+
+ with gr.Accordion("❗ Troubleshooting", open=False):
+ gr.Markdown("""
+ ### Common Issues
+
+ #### Windows Users
+ - If you see `UserWarning: huggingface_hub cache-system uses symlinks` you can safely ignore it or alternatively:
+ 1. Close the application
+ 2. Restart command prompt as Administrator
+ 3. Relaunch the finetuning process
+
+ ### Managing Whisper Model Files
+
+ #### Why Delete Model Files?
+ - Free up disk space (each model is ~3GB)
+ - Troubleshoot model-related issues
+ - Switch to different model versions
+ - Clear corrupted downloads
+
+ #### How to Find and Delete Whisper Models
+
+ Each Whisper model is downloaded automatically when first used and stored in your system's cache. Here's how to manage them:
+
+ **Windows:**
+ 1. Open File Explorer
+ 2. Copy and paste this path into the address bar:
+ ```
+ C:\\Users\\\\.cache\\whisper
+ ```
+ (Replace `` with your actual Windows username)
+ 3. You'll see folders named after each model you've used (e.g., "base", "large-v3")
+
+ **Linux:**
+ 1. Open your terminal or file manager
+ 2. Navigate to:
+ ```
+ /home//.cache/whisper
+ ```
+ (Replace `` with your Linux username)
+ 3. Model files will be in this directory
+
+ **macOS:**
+ 1. Open Finder
+ 2. Press `Cmd+Shift+G` to open "Go to Folder"
+ 3. Enter this path:
+ ```
+ /Users//.cache/whisper
+ ```
+ (Replace `` with your macOS username)
+
+ #### To Delete Models:
+ 1. **Individual Models:** Delete specific model folders to free up space
+ 2. **All Models:** Delete the entire `whisper` folder to remove all cached models
+ 3. Don't worry - models will automatically redownload when needed
+
+ #### Important Notes:
+ - Deleting models won't affect your training data or results
+ - New model downloads will occur next time you use a model
+ - Consider keeping frequently used models to avoid redownloading
+ """)
+
+ with gr.TabItem("📊 Dataset Validation"):
+ with gr.Row():
+ gr.Markdown("""# Audio Transcription Validation""")
+ with gr.Row():
+ with gr.Accordion("🎯 Audio Transcription Validation Help", open=False):
+ with gr.Row():
+ with gr.Column():
+ gr.Markdown("""
+ # Dataset Validation and Correction Tool
+
+ This tool helps you ensure the quality of your training data by comparing:
+ - Original transcriptions from your CSV dataset files
+ - Running new Whisper transcriptions of the same audio files
+
+ Only mismatched transcriptions are shown for review, helping you focus on potential problems.
+
+ ### Understanding the Display
+
+ #### Table Contents:
+ - **Original Text**: Current transcription from metadata files
+ - **Whisper Text**: New transcription from Whisper
+ - **Filename**: The audio file being transcribed
+
+ ### Common Mismatch Causes:
+ - Background noise interference
+ - Multiple speakers in audio
+ - Unclear pronunciation
+ - Audio quality issues
+ - Whisper confidence variations
+ - Punctuation differences
+
+ ### Interactive Audio Editor
+
+ The built-in audio editor provides several powerful features:
+
+ #### Playback Controls:
+ - Play/Pause: Standard audio playback
+ - Seek Bar: Navigate through audio
+ - Volume Control: Adjust playback volume
+ - Speed Control: Adjust playback speed
+
+ #### Audio Editing Features:
+ - **Trim Start/End**: Remove silence or unwanted audio
+ * Drag handles on waveform edges
+ * Fine-tune start/end points
+ * Preview trimmed audio
+ """)
+
+ with gr.Column():
+ gr.Markdown("""
+ ...continued
+
+ #### Tips for Audio Editing:
+ - Trim silence from start/end
+ - Remove background noises
+ - Keep only clear speech
+ - Maintain natural pauses
+ - Ensure clean cut points
+
+ ### Correction Workflow:
+
+ 1. **Select an Entry**:
+ * Click any row in the mismatch table
+ * Audio loads automatically
+ * Both transcriptions display
+
+ 2. **Review Audio**:
+ * Listen to original
+ * Edit if necessary
+ * Preview changes
+
+ 3. **Choose Text Correction Method**:
+ * **Use Original**: Keep current transcription
+ * **Use Whisper**: Accept new transcription
+ * **Edit Manually**: Write custom transcription
+
+ 4. **Save Changes**:
+ * Click "Save Audio and Correction"
+ * Both audio and text will be saved
+ * The table will reflect the change
+
+ ### Best Practices:
+ - Listen before editing
+ - Make minimal necessary cuts
+ - Verify transcription accuracy
+ - Check edited audio playback
+
+ Changes automatically update the appropriate CSV file and refresh the display.
+ """)
+
+ with gr.Row():
+ progress_box = gr.Textbox(label="Progress", interactive=False)
+ load_button = gr.Button("Run Validation")
+
+ with gr.Row():
+ with gr.Column(scale=2):
+ # Store full DataFrame in state
+ state = gr.State()
+ # Display DataFrame shows only visible columns
+ mismatch_table = gr.DataFrame(
+ headers=["Original Text", "Whisper Text", "Filename"],
+ datatype=["str", "str", "str"],
+ interactive=False,
+ wrap=True
+ )
+
+ with gr.Column(scale=1):
+ audio_player = gr.Audio(
+ label="Audio Player (Edit enabled)",
+ interactive=True
+ )
+ current_expected = gr.Textbox(label="Original Text", interactive=False)
+ current_transcribed = gr.Textbox(label="Whisper Text", interactive=False)
+ text_choice = gr.Radio(
+ choices=["Use Original", "Use Whisper", "Edit Manually"],
+ label="Choose Transcription",
+ value="Use Original"
+ )
+ manual_edit = gr.Textbox(
+ label="Manual Edit",
+ interactive=True,
+ visible=False
+ )
+ current_index = gr.Number(visible=False)
+ save_button = gr.Button("Save Audio and Correction")
+ save_status = gr.Textbox(label="Save Status", interactive=False)
+
+ def update_audio_player(evt: gr.SelectData, df):
+ try:
+ selected_row = df.iloc[evt.index]
+ audio_path = selected_row['full_path']
+
+ # If it's a Series, get the first value
+ if isinstance(audio_path, pd.Series):
+ audio_path = audio_path.iloc[0]
+
+ # Clean the path string
+ audio_path = str(audio_path).strip()
+ # print(f"[FINETUNE] Selected audio path: {audio_path}")
+
+ # Get the text values directly from the row, ensuring we get strings not Series
+ expected = str(selected_row['expected_text']).strip() if isinstance(selected_row['expected_text'], str) else selected_row['expected_text'].iloc[0]
+ transcribed = str(selected_row['transcribed_text']).strip() if isinstance(selected_row['transcribed_text'], str) else selected_row['transcribed_text'].iloc[0]
+
+ if not os.path.exists(audio_path):
+ print(f"[FINETUNE] Warning: Audio file not found: {audio_path}")
return {
audio_player: None,
current_expected: "",
current_transcribed: "",
current_index: None,
- save_status: f"Error: {str(e)}"
+ save_status: f"Error: Audio file not found"
}
+
+ return {
+ audio_player: audio_path,
+ current_expected: expected,
+ current_transcribed: transcribed,
+ current_index: evt.index,
+ save_status: "Ready to save correction"
+ }
+ except Exception as e:
+ print(f"[FINETUNE] Error in update_audio_player: {str(e)}")
+ print(f"[FINETUNE] Selected row data: {selected_row if 'selected_row' in locals() else 'Not available'}")
+ return {
+ audio_player: None,
+ current_expected: "",
+ current_transcribed: "",
+ current_index: None,
+ save_status: f"Error: {str(e)}"
+ }
- # Event handlers
- text_choice.change(
- lambda x: gr.update(visible=x=="Edit Manually"),
- text_choice,
- manual_edit
- )
-
- mismatch_table.select(
- update_audio_player,
- [state], # Use full DataFrame from state
- [audio_player, current_expected, current_transcribed, current_index, save_status]
- )
-
- save_button.click(
- save_selected_text,
- inputs=[text_choice, manual_edit, state, current_index],
- outputs=[mismatch_table, current_expected, save_status] # Add mismatch_table and current_expected to outputs
- )
-
- # Store both display and full DataFrame
- load_button.click(
- load_and_display_mismatches,
- outputs=[state, mismatch_table, progress_box] # state gets full df, mismatch_table gets display_df
- )
+ # Event handlers
+ text_choice.change(
+ lambda x: gr.update(visible=x=="Edit Manually"),
+ text_choice,
+ manual_edit
+ )
- with gr.Tab("Generate Dataset Instructions"):
- gr.Markdown(
- f"""
- #### 🟦 What You Need to Do
- 1. Read [Coqui's guide on creating a good dataset](https://docs.coqui.ai/en/latest/what_makes_a_good_dataset.html).
- 2. Place your audio files (MP3, WAV, or FLAC) in `{str(audio_folder)}` Or Upload them through the interface.
- 3. Provide at least 2 minutes of audio, preferably 5-10 minutes for better results.
- 4. Maximum Audio Length will ensure the audio samples do not exceed the set value in seconds. The AI model can only process so much audio per audio sample, per epoch.
- 5. Enter a unique Project Name if you wish to create a dedicated training folder. You will need to specify this name as you move through the interfaces and click the Refresh buttons where needed.
- 6. Select the Whisper model (distil-large-v3 recommended for English use cases).
- 7. Choose your Dataset Language.
- 8. Adjust the Evaluation Split percentage (default 15% is suitable for most cases).
- 9. Optionally enable the BPE Tokenizer for custom tokenization, which can improve the model's ability to handle unique words, accents, or languages by creating a vocabulary tailored to your specific dataset. This is especially useful for non-standard speech patterns or languages with complex morphology.
-
- #### 🟨 What This Step Does
- 1. Uses Whisper to transcribe and segment your audio files.
- 2. Creates smaller audio clips and corresponding transcriptions.
- 3. Generates two CSV files: `metadata_train.csv` and `metadata_eval.csv`.
- 4. Saves processed audio in `/finetune/[project_name]/wavs/`.
- 5. Stores CSV files and metadata in `/finetune/[project_name]/`.
-
- Key files generated:
- ◽ `/finetune/[project_name]/lang.txt`: Contains the two-letter language code.
- ◽ `/finetune/[project_name]/metadata_train.csv`: List of audio files and transcriptions for training.
- ◽ `/finetune/[project_name]/metadata_eval.csv`: Evaluation data for assessing training quality.
-
- #### Important Considerations
- ◽ For multi-speaker audio, manually edit CSV files to remove other speakers.
- ◽ You can manually edit CSV files to improve transcription accuracy & use the `Dataset Validation` to help identify possible bad transcriptions.
- ◽ The Whisper 2 model may provide better results in audio splitting and dataset creation for English languages, however please see the Whisper site for more details on other languages.
- ◽ Each Whisper model is about 3GB. It's recommended to stick with one model to save disk space.
- """
- )
-
- with gr.Tab("Generate Dataset Notes"):
- gr.Markdown(
- f"""
- #### 🟥 Important Notes
- ◽ **Windows Users:** If you encounter a `UserWarning: huggingface_hub cache-system uses symlinks` error, restart your Windows command prompt with `Run as Administrator` and relaunch the finetuning process. This typically only occurs during the first Whisper model download.
- ◽ **Language Support:** Whisper's performance varies across languages. The Large-v3 model may offer better results for certain languages. For problematic languages, consider manual dataset creation.
-
- #### 🟩 Processing Time
- ◽ First run: Requires downloading the 3GB Whisper model.
- ◽ Subsequent runs: A few minutes on an average 3-4 year old system.
-
- After processing, use the 'Dataset Validation' tab to check transcription accuracy and make necessary corrections.
-
- For more details on Whisper models or manual dataset creation, refer to:
- ◽ [Whisper Models Information](https://github.com/openai/whisper?tab=readme-ov-file#available-models-and-languages)
- ◽ [Manual Dataset Creation Guide](https://docs.coqui.ai/en/latest/formatting_your_dataset.html)
- """
- )
-
-
+ mismatch_table.select(
+ update_audio_player,
+ [state], # Use full DataFrame from state
+ [audio_player, current_expected, current_transcribed, current_index, save_status]
+ )
+ save_button.click(
+ save_audio_and_correction,
+ inputs=[text_choice, manual_edit, audio_player, state, current_index],
+ outputs=[mismatch_table, current_expected, save_status, audio_player]
+ )
+
+ # Store both display and full DataFrame
+ load_button.click(
+ load_and_display_mismatches,
+ outputs=[state, mismatch_table, progress_box] # state gets full df, mismatch_table gets display_df
+ )
+
+
#######################
#### GRADIO STEP 2 ####
#######################
@@ -2227,7 +3562,7 @@ if __name__ == "__main__":
)
warm_up = gr.Checkbox(
value=False,
- label="Perform Small a Warmup Learning",
+ label="Perform Warmup Learning",
scale=1,
)
train_csv = gr.Textbox(
@@ -2399,164 +3734,463 @@ if __name__ == "__main__":
print(f"[FINETUNE] Error: {error_message}")
return f"Training error: {error_message}", "", "", "", "", ""
- with gr.Tab("Training Instructions"):
- gr.Markdown(
- f"""
- ### 🟥 Important Note - Language support.
- ◽ If this step is failing/erroring you may wish to check your training data was created correctly (Detailed in Step 1), confirming that wav files have been generated and your `metadata_train.csv` and `metadata_eval.csv` files have been populated.
- ◽ Ignore 'fatal: not a git repository (or any of the parent directories): .git' as this is a legacy training script issue.
- ### 🟦 What you need to do
- ◽ The Train CSV and Eval CSV should already be populated. If not, just go back to Step 1 and click "Create Dataset" again.
- ◽ The default settings below are the suggested settings for most purposes, however you may choose to alter them depending on your specific use case.
- ◽ There are no absolute correct settings for training. It will vary based on:
- - What you are training (A human voice, A cartoon voice, A new language entirely etc)
- - How much audio you have (you may want less or more eval or epochs)
- ◽ The key indicator of sufficient model training is whether it sounds right to you. Coqui suggests beginning with the base settings for training. If the resulting audio doesn't meet your expectations, additional training sessions may be necessary.
- #### 🟪 Continue Previous Project (CURRENTLY REMOVED AND NOT AVAILABLE)
- This option allows you to resume training from where it left off if your previous session was interrupted (e.g., due to a power outage or system crash). When enabled:
- ◽ The system will attempt to load the most recent checkpoint from your previous training session.
- ◽ It will continue training from that point, preserving your progress and saving time.
- ◽ This is particularly useful for long training sessions or when you want to extend training on a previously trained model.
- ### 🟨 What this step is doing
- ◽ Very simply put, it's taking all our wav files generated in Step 1, along with our recorded speech in out excel documents and its training the model on that voice e.g. listen to this audio file and this is what is being said in it, so learn to reproduce it.
- ### 🟩 How long will this take?
- ◽ On a RTX 4070 with factory settings (as below) and 20 minutes of audio, this took 20 minutes. Again, it will vary by system and how much audio you are throwing at it. You have time to go make a coffee.
- ◽ Look at your command prompt/terminal window if you want to see what it is doing.
- """
- )
-
- with gr.Tab("Memory Optimization"):
- gr.Markdown(
- f"""
- ### 🟦 Memory Optimization Strategies
-
- Adjusting these parameters can help manage memory usage during training:
-
- ◽ **Batch Size**: Smaller batch sizes use less memory. If you encounter out-of-memory errors, try reducing the batch size.
- ◽ **Gradient Accumulation**: Increasing grad accumulation steps allows you to simulate larger batch sizes without increasing memory usage.
- ◽ **Workers/Threads**: Fewer workers generally use less memory. If you're experiencing memory issues, try reducing the number of workers.
- ◽ **Max Audio Length**: Limiting the maximum audio length reduces memory usage by preventing very long audio clips from being loaded.
- ◽ **Learning Rate and Optimizer**: While these don't directly affect memory usage, they can impact training efficiency. A well-tuned learning rate and appropriate optimizer can lead to faster convergence, potentially reducing overall training time and resource usage.
+ with gr.Tab("Training Guide"):
+ gr.Markdown("""
+ # Comprehensive Training Guide
+ Below you'll find detailed explanations of all training parameters and processes.
+ Each section contains both conceptual explanations and specific configuration guidance.
+ """)
+
+ with gr.Accordion("🎯 Quick Start Training Guide", open=False):
+ gr.Markdown("""
+ Before diving into specific settings, it's important to understand the basic workflow of training your model.
+ This section provides an overview of the essential parameters you'll need to get started with training.
- #### 🟪 Disable Shared Memory
- This option affects how the system manages memory during training:
- ◽ When enabled, it limits GPU memory usage to 95% of its capacity.
- ◽ This prevents the training process from spilling over into your system's shared memory.
- ◽ You're more likely to get a clear "out of memory" error if your GPU runs out of memory, rather than experiencing slower performance due to memory spillover.
- ◽ This option is useful if you prefer to keep training confined to GPU memory for better performance and clearer error reporting.
+ The training process involves feeding your prepared dataset through the model multiple times (epochs) while adjusting
+ the model's parameters using specific optimization strategies. The goal is to teach the model to replicate your voice
+ accurately while maintaining natural speech patterns.
- Important Note for Windows Users with Low VRAM GPUs:
- ◽ If you're using Windows and have a GPU with lower VRAM (e.g., 8GB or less), it's recommended to leave this option disabled.
- ◽ Windows can utilize system RAM as extended VRAM, which can be crucial for completing the training process on lower VRAM GPUs.
- ◽ Disabling this option allows Windows to manage memory allocation more flexibly, potentially preventing out-of-memory errors on GPUs with limited VRAM.
+ #### Required Settings:
+ - **Project Name:** Name for your training session
+ - **Train/Eval CSV:** Should be auto-populated from Step 1
+ - **Model Selection:** Choose your base model
+ - **Basic Parameters:**
+ * Epochs: How many training cycles (default: 10)
+ * Batch Size: Samples per training step (default: 4)
+ * Grad Accumulation: Steps between updates (default: 1)
+ * Learning Rate: Training speed (default: 5e-6)
+
+ #### Learning Settings
+ - **Learning Rate:** 5e-6 (recommended start)
+ - **Optimizer:** AdamW (default)
+ - **Scheduler:** CosineAnnealingWarmRestarts (default)
+ These default settings are carefully chosen to provide a good starting point for most training scenarios. They
+ balance training speed with stability and are suitable for most consumer-grade GPUs.
+ """)
- Remember, the goal is to find a balance between memory usage and training effectiveness. Start with lower values and gradually increase them while monitoring your system's memory usage.
- """
- )
-
- with gr.Tab("Info - Grad Accumulation"):
- gr.Markdown(
- f"""
- ### 🟩 Grad Accumulation
- ◽ Gradient accumulation is a technique that allows you to simulate a larger batch size without actually increasing the memory consumption. Here's how it works:
- ◽ Instead of updating the model's parameters after every batch, gradient accumulation enables you to process multiple batches and accumulate their gradients.
- ◽ By setting steps greater than 1, you can process multiple batches before a single optimization step e.g if steps is set to 4, the gradients will be accumulated over 4 batches before updating the model's parameters.
- ◽ This means that you can effectively use a larger batch size while keeping the actual batch size per iteration smaller, thereby reducing the memory footprint.
- ◽ Increasing the steps allows you to find a balance between memory consumption and computational efficiency. You can process more examples per optimization step without exceeding the available memory.
- ◽ However, it's important to note that increasing the gradient accumulation steps does have an impact on the training process:
- ◽ Since the model's parameters are updated less frequently (every grad_accum_steps batches), the training dynamics may be slightly different compared to updating the model after every batch.
- ◽ You may need to adjust the learning rate and other hyperparameters accordingly to compensate for the less frequent updates. Typically, you can increase the learning rate slightly when using gradient accumulation.
- ◽ The training progress will be slower in terms of the number of optimization steps per epoch, as the model updates occur less frequently.
- ◽ However, the overall training time may still be reduced compared to using a smaller batch size without gradient accumulation, as it allows for better utilization of GPU resources.
- ◽ To start, you can try setting steps to a value like 4 and see if it resolves the OOM error. If the error persists, you can experiment with higher values until you find a balance that works for your specific setup.
- """
- )
- with gr.Tab("Info - Batch Size"):
- gr.Markdown(
- f"""
- ### 🟩 Batch Size
- ◽ Batch size refers to the number of training examples processed in a single iteration (forward and backward pass) before updating the model's parameters.
- ◽ A larger batch size allows for more efficient computation and utilization of hardware resources, especially when using GPUs.
- ◽ However, increasing the batch size also increases the memory consumption during training, as more examples need to be stored in memory.
- ◽ If the batch size is too large, it can lead to out-of-memory (OOM) errors, especially when training on GPUs with limited memory.
- ◽ On the other hand, using a smaller batch size can result in slower training and may require more iterations to converge.
- ◽ The optimal batch size depends on various factors, such as the model architecture, available hardware resources, and the specific problem being solved.
- ◽ It's common to experiment with different batch sizes to find a balance between training speed and memory efficiency.
- ◽ If you encounter OOM errors, try reducing the batch size until the training can proceed without memory issues.
- ◽ Keep in mind that the batch size also affects the model's generalization and convergence behavior, so it's important to monitor the model's performance while adjusting the batch size.
- """
- )
- with gr.Tab("Info - Learning Rate & Schedulers"):
- gr.Markdown(
- f"""
- ### 🟩 Learning Rate
- ◽ **1e-6:** Very small learning rate, slow but stable learning.
- ◽ **5e-6:** Small learning rate, slow but stable learning.
- ◽ **1e-5:** Moderate learning rate, balanced between stability and convergence speed.
- ◽ **5e-5:** Higher learning rate, faster convergence but potential instability.
- ◽ **1e-4:** High learning rate, faster convergence but increased risk of instability.
- ◽ **5e-4:** Very high learning rate, fast convergence but higher risk of instability.
- ◽ **1e-3:** Extremely high learning rate, very fast convergence but high risk of instability.
- The optimal learning rate depends on the model architecture, dataset, and other hyperparameters.
+ with gr.Accordion("📁 Using Your Own Training Dataset", open=False):
+ gr.Markdown("""
+ If you are **not** using a dataset built on Step 1 and wish to use your own custom pre-built dataset, you can do so with the following instructions.
+
+ To train the model with your custom dataset, follow these steps carefully to ensure your files and folder structure are set up correctly.
- ### 🟩 Learning Rate Schedulers
- ◽ **None:** Learning rate remains unchanged throughout the training.
- ◽ **Step:** Decreases the learning rate by a factor at specified intervals, offering a straightforward method for controlling learning rate decay.
- ◽ **MultiStep:** Decreases the learning rate by a factor at specified milestones, useful for gradually reducing the learning rate during training.
- ◽ **Exponential:** Decreases the learning rate exponentially over epochs, providing a simple decay mechanism for training.
- ◽ **CosineAnnealing:** Adjusts the learning rate following a cosine annealing schedule, facilitating smoother optimization and potentially better convergence.
- ◽ **Reduce On Plateau:** Reduces the learning rate when a monitored metric has stopped improving, helping to fine-tune learning rates during training.
- ◽ **Cyclic:** Cycles the learning rate between two boundary values, allowing for dynamic learning rate schedules during training.
- ◽ **One Cycle (disabled):** Sets the learning rate using a cyclical learning rate policy and varies momentum inversely, potentially accelerating training and improving performance.
- ◽ **Cosine Annealing Warm Restarts:** Adjusts the learning rate following a cosine annealing schedule with warm restarts, offering a balance between exploration and exploitation during optimization.
- The choice of learning rate scheduler depends on the specific characteristics of the model, dataset, and training objectives.
- """
- )
- with gr.Tab("Info - Optimizers"):
- gr.Markdown(
- f"""
- ### 🟩 Optimizers
- ◽ **SGD with Momentum:** Good for convergence, requires careful tuning of the learning rate and momentum.
- ◽ **RMSprop:** Adaptively adjusts learning rates, good for varying gradient magnitudes.
- ◽ **Adam:** Widely used, performs well with default parameters.
- ◽ **Adagrad:** Good for sparse data, adapts learning rates based on historical gradients.
- ◽ **AdamW:** Similar to Adam but with decoupled weight decay for better regularization.
- When picking an optimizer, you need to experiment with different options to find the one that achieves the best performance and convergence for your task.
- AdamW has been set to default as its a sensible pick that will handle most needs.
- """
- )
- with gr.Tab("Info - Epochs"):
- gr.Markdown(
- f"""
- ### 🟩 Epochs
- ◽ An epoch represents a single pass through the entire training dataset during the training process.
- ◽ In each epoch, the model sees and learns from all the training examples once.
- ◽ The number of epochs determines how many times the model will iterate over the complete training dataset.
- ◽ Increasing the number of epochs allows the model to learn more from the data and potentially improve its performance.
- ◽ However, training for too many epochs can lead to overfitting, where the model becomes too specialized to the training data and fails to generalize well to unseen data.
- ◽ The optimal number of epochs depends on factors such as the complexity of the problem, the size of the dataset, and the model's capacity.
- ◽ It's common to use techniques like early stopping or validation monitoring to determine the appropriate number of epochs.
- ◽ Early stopping involves monitoring the model's performance on a validation set and stopping the training when the performance starts to degrade, indicating potential overfitting.
- ◽ Validation monitoring involves evaluating the model's performance on a separate validation set after each epoch and selecting the model checkpoint that performs best on the validation set.
- ◽ It's recommended to start with a reasonable number of epochs and monitor the model's performance to determine if more epochs are needed or if early stopping should be applied
- """
- )
- with gr.Tab("Info - Max Audio"):
- gr.Markdown(
- f"""
- ### 🟩 Max Permitted Audio Size (in seconds)
- ◽ The max permitted audio size determines the maximum duration of audio files that can be used as input to the model during training and inference.
- ◽ It is specified in seconds and represents the longest audio clip that the model can process.
- ◽ Limiting the audio size helps to control the memory usage and computational requirements of the model.
- ◽ If the audio files in your dataset vary in length, it's important to consider the max permitted audio size to ensure that all files can be processed effectively.
- ◽ Audio files longer than the max permitted size will typically be truncated or split into smaller segments before being fed into the model.
- ◽ The choice of max permitted audio size depends on the specific requirements of your application and the available hardware resources.
- ◽ Increasing the max permitted audio size allows the model to handle longer audio clips but also increases the memory consumption and computational burden.
- ◽ If you encounter memory issues or slow processing times, you may need to reduce the max permitted audio size to find a suitable balance.
- ◽ It's important to consider the nature of your audio data and choose a max permitted audio size that captures the relevant information while being computationally feasible.
- """
- )
+ #### Folder and File Structure:
+ 1. **Create a project folder:** Navigate to `/alltalk_tts/finetune/` and create a new folder with the name of your project (e.g., `myproject`).
+ 2. **Prepare Required Files:**
+ - Inside your project folder, add a file named `lang.txt` containing the two-letter language code for your dataset (e.g. `en` for English, `de` for German).
+ - Place your `metadata_train.csv` and `metadata_eval.csv` files in the project folder. These files must follow the dataset formatting guidelines provided by [Coqui AI documentation](https://docs.coqui.ai/en/latest/formatting_your_dataset.html).
+ 3. **Add Your Audio Files:**
+ - Within your project folder, create a subfolder named `wavs` and add your audio files here. The audio files should be in `.wav` format.
+
+ #### CSV Files: Training vs. Evaluation
+ - **Training Data (`metadata_train.csv`):** This CSV file contains the majority of your dataset. The model uses this data to learn and adapt to the patterns in your audio. It’s recommended to have about 85% of your audio files in the training dataset, though this ratio can be adjusted based on your needs.
+ - **Evaluation Data (`metadata_eval.csv`):** This CSV file contains a smaller subset of your audio (typically around 15%). During training, the model periodically tests itself on this evaluation data to gauge how well it is learning. This evaluation step helps prevent overfitting, where the model might learn training data too closely and struggle with new data.
+
+ **Guideline:** The `metadata_train.csv` and `metadata_eval.csv` files must follow the formatting guidelines provided by [Coqui AI documentation](https://docs.coqui.ai/en/latest/formatting_your_dataset.html).
+
+ #### Updating the Gradio Interface:
+ - In the `Project Name` field, enter the name of your project folder (e.g., `myproject`).
+ - Populate the `Train CSV file path` and `Eval CSV file path` fields with the full paths to your `metadata_train.csv` and `metadata_eval.csv` files, respectively.
+ - Once set up, you are ready to begin training with your custom dataset.
+
+ #### Directory Structure Example:
+ ```plaintext
+ finetune/
+ └── {projectfolder}/
+ ├── metadata_train.csv
+ ├── metadata_eval.csv
+ ├── lang.txt
+ └── wavs/
+ ├── audio1.wav
+ ├── audio2.wav
+ └── ...
+ ```
+ """)
+
+ with gr.Accordion("📊 Training Metrics and Logs", open=False):
+ gr.Markdown("""
+ ### Understanding Training Metrics
+
+ The training progress is visualized through six different graphs, each showing different aspects of the training process:
+
+ #### 1. Epoch Metrics (Avg Losses)
+ - Shows average losses across full epochs
+ - Lower values indicate better model performance
+ - Three metrics tracked:
+ * Avg Loss (Green): Overall model loss (may be hidden behind the blue line)
+ * Avg Loss Text CE (Red): Text cross-entropy loss
+ * Avg Loss MEL CE (Blue): Audio MEL spectrogram loss
+
+ #### 2. Step-wise Loss Metrics
+ - Shows individual training step losses
+ - More granular view than epoch metrics
+ - Same color coding as epoch metrics
+ - Helps identify specific problematic training steps
+
+ #### 3. Learning Rate Schedule
+ - Shows learning rate changes over training steps
+ - Yellow line indicates current learning rate
+ - Useful for verifying scheduler behavior
+ - Cosine annealing shows cyclical pattern
+
+ #### 4. Gradient Norm over Steps
+ - Indicates how dramatically weights are changing
+ - Helps monitor training stability
+ - Sudden spikes may indicate training issues
+ - Should generally remain stable
+
+ #### 5. Step and Loader Times
+ - Orange: Time taken for each training step
+ - Cyan: Time taken to load data
+ - Helps identify performance bottlenecks
+ - Higher loader times may indicate I/O issues
+
+ #### 6. Training vs Validation Loss
+ - Compares training and validation performance
+ - Green: Training loss
+ - Red: Validation loss
+ - Growing gap may indicate overfitting
+
+ ### Detailed Training Logs
+
+ A complete record of your training run is stored in `trainer_0_log.txt`, located in your training folder:
+ ```
+ finetune/[project_name]/training/XTTS_FT-[timestamp]/trainer_0_log.txt
+ ```
+
+ This log contains:
+ - All training metrics
+ - Detailed step information
+ - Model configuration
+ - Performance statistics
+ - Error messages (if any)
+
+ **Note:** This log file will be deleted when you use the "Delete Training Data" option in the Model Export step. If you want to keep a record of your training run, make sure to save a copy of this file before deleting the training data.
+
+ ### Reading the Graphs
+
+ - **Downward Trends** in loss values indicate improving model performance
+ - **Smooth Lines** suggest stable training
+ - **Jagged Lines** might indicate instability or need for hyperparameter adjustment
+ - **Plateaus** in loss curves might indicate learning has stalled
+ - **Diverging** validation and training losses might indicate overfitting
+
+ ### Common Patterns
+
+ - **Good Training Run:**
+ * Steadily decreasing losses
+ * Stable gradient norms
+ * Consistent step times
+ * Close training/validation losses
+
+ - **Potential Issues:**
+ * Rapidly increasing losses
+ * Highly erratic gradient norms
+ * Growing gap between training/validation loss
+ * Inconsistent step times
+ """)
+
+ with gr.Accordion("💾 Memory Management", open=False):
+ gr.Markdown("""
+ Memory management is crucial for successful training. Understanding how different parameters affect memory usage
+ can help you optimize your training process and avoid out-of-memory errors. The two main types of memory to
+ consider are GPU VRAM (Video RAM) and system RAM.
+
+ Your GPU's VRAM is the primary resource that will limit what you can do during training. The more VRAM you have,
+ the larger batches you can process and the faster your training can progress. However, even with limited VRAM,
+ there are several strategies you can use to successfully train your model.
+
+ #### VRAM Requirements:
+ - **12GB+ VRAM (Recommended)**
+ * Optimal performance
+ * Fewer memory constraints
+ * Better batch size options
+ * Suitable for larger models
+ - **8GB VRAM (Minimum)**
+ * Windows: Requires 24GB+ System RAM
+ * Linux: May encounter limitations
+ * Requires careful parameter tuning
+ * May need smaller batch sizes
+
+ #### Memory Optimization Strategies:
+ - **Batch Size Reduction**
+ * Smaller batches use less VRAM
+ * May increase training time
+ * Start small, increase if stable
+ * Monitor memory usage while adjusting
+
+ - **Gradient Accumulation**
+ * Simulates larger batches
+ * Uses less peak memory
+ * Helps with limited VRAM
+ * Maintains training stability
+
+ - **Precision Settings**
+ * Mixed precision for modern GPUs
+ * FP32 for older GPUs
+ * FP16 for maximum efficiency
+ * Balance accuracy vs. memory
+
+ - **Worker Threads**
+ * Fewer workers = less memory
+ * Balance between speed and RAM usage
+ * Adjust based on system resources
+ * Start with 4-8 workers
+
+ #### Shared Memory Option:
+ The shared memory option affects how your system handles memory overflow:
+
+ - **Disable Shared Memory Use:**
+ * Limits GPU to 95% VRAM usage
+ * Prevents system RAM spillover
+ * Recommended for high-VRAM GPUs
+ * Clearer error messages
+
+ - **Keep Enabled for:**
+ * Windows with low VRAM GPUs
+ * Systems with plenty of RAM
+ * When encountering OOM errors
+ * Better fallback options
+ """)
+
+ with gr.Accordion("⚙️ Batch Size & Gradient Accumulation", open=False):
+ gr.Markdown("""
+ Understanding batch size and gradient accumulation is crucial for successful training. These two parameters work
+ together to determine how your model processes data and updates its weights during training.
+
+ #### Batch Size Explained
+ Batch size determines how many audio samples are processed simultaneously during training. Think of it like
+ cooking multiple meals at once - while it's more efficient to cook several dishes simultaneously, your kitchen
+ (GPU memory) needs to be large enough to handle it.
+
+ - **What It Controls:**
+ * Number of samples processed together
+ * Memory usage per step
+ * Training stability
+ * Learning effectiveness
+
+ - **Guidelines:**
+ * Larger = faster but more VRAM
+ * Smaller = stable but slower
+ * Find balance for your GPU
+ * Monitor training stability
+
+ - **Recommended Ranges:**
+ * 12GB+ VRAM: 4-8 batch size
+ * 8GB VRAM: 2-4 batch size
+ * Adjust based on performance
+ * Start small and increase if stable
+
+ #### Gradient Accumulation Deep Dive
+ Gradient accumulation is like keeping a running tally of changes before updating your recipe. Instead of
+ adjusting your cooking technique after each meal, you gather feedback from several attempts before making
+ changes.
+
+ - **Purpose:**
+ * Simulates larger batch sizes
+ * Reduces memory usage
+ * Maintains training stability
+ * Improves learning quality
+
+ - **How It Works:**
+ * Accumulates gradients over steps
+ * Updates model less frequently
+ * Allows larger effective batches
+ * Maintains learning quality
+
+ - **Setting Guidelines:**
+ * Start with 1 (default)
+ * Increase if OOM errors occur
+ * Multiply with batch size for true batch
+ * Monitor training progress
+
+ - **Common Values:**
+ * 1-4: Standard usage
+ * 4-8: Memory constrained
+ * 8+: Severe memory limitations
+ * Adjust based on stability
+
+ By carefully balancing batch size and gradient accumulation, you can achieve good training results even
+ with limited GPU resources. Start conservative and adjust based on your system's performance.
+ """)
+
+ with gr.Accordion("📊 Learning Rate & Schedulers", open=False):
+ gr.Markdown("""
+ The learning rate and its scheduling are perhaps the most critical aspects of training. Think of the learning
+ rate as the size of the steps your model takes when learning - too large and it might overshoot the best
+ solution, too small and it might take too long to get there.
+
+ #### Understanding Learning Rates
+ The learning rate determines how much the model changes in response to each batch of training data. Finding
+ the right learning rate is crucial for successful training.
+
+ #### Learning Rate Options:
+ | Learning Rate Range | Description | Best Use Cases | Additional Notes |
+ |----------------------------|-------------------------------------------|-------------------------------|------------------------------------|
+ | **1e-6 (Very Conservative)** | Extremely stable, very slow progress | Fine adjustments | Use when stability is crucial |
+ | **5e-6 (Recommended Start)** | Good balance of speed and stability | Most training scenarios | Best starting point, stable |
+ | **1e-5 to 5e-5 (Moderate)** | Faster learning, can be unstable | Experienced users | Requires monitoring |
+ | **1e-4 to 1e-3 (Aggressive)**| Very fast, high risk of instability | Advanced users | Caution required, may diverge |
+ | **5e-4 to 1e-3 (Very Aggressive)** | Rapid rate, often too large | Experimental only | High risk of divergence, high regularization needed |
+
+ #### Learning Rate Schedulers Explained
+ Schedulers automatically adjust the learning rate during training. This can help fine-tune the learning
+ process and avoid getting stuck in suboptimal solutions.
+
+ #### Scheduler Types:
+ | Scheduler | Description | Best Use Cases | Additional Notes |
+ |---------------------------------|------------------------------------------------|--------------------------------------------|--------------------------------|
+ | **None** | Fixed learning rate throughout | Testing | Simple, predictable behavior; may miss optimal points |
+ | **CosineAnnealingWarmRestarts** | Cyclical learning rate changes | General training, exploration | Requires 4+ epochs; best all-around choice |
+ | **CosineAnnealingLR** | Smooth cosine decay, no restarts | Continuous decay needs, stable convergence | Suitable for gradual reduction |
+ | **StepLR** | Step-wise reduction | Planned schedules, predictable decay | Allows manual step control |
+ | **MultiStepLR** | Step reductions at multiple intervals | Customized training schedules | Requires precise tuning |
+ | **ReduceLROnPlateau** | Adapts to training performance | Avoiding plateaus, unknown scenarios | Reduces rate when improvement slows |
+ | **ExponentialLR** | Exponential decay by a fixed factor | Rapid convergence | Requires monitoring |
+ | **CyclicLR** | Cycles between min and max rates | Early-stage convergence, high variability | Advanced usage |
+ | **OneCycleLR** | Peaks and returns over single cycle | Large datasets, fast single-cycle training | Efficient but requires setup |
+
+ The choice of scheduler can significantly impact your training results. The recommended
+ CosineAnnealingWarmRestarts provides a good balance of exploration and exploitation during training.
+
+ #### Perform Warmup Learning
+ The warmup learning phase is a useful feature for stabilizing early training. During warmup, the learning rate gradually increases from a low initial value to the target learning rate over a set number of steps or epochs. This helps the model adjust gradually, especially beneficial for high learning rates or large batch sizes.
+
+ - **What Warmup Does:**
+ * Prevents large early adjustments
+ * Improves initial stability
+ * Gradually increases learning rate
+ * Helps avoid gradient explosion
+
+ - **When to Use:**
+ * Recommended with high learning rates
+ * Useful for models that diverge early
+ * Ideal for large batch sizes
+ * Suitable for most cases
+
+ - **Guidelines:**
+ * Enable to improve training stability
+ * Monitor initial learning curve
+ * Adjust warmup steps as needed
+
+ The warmup feature, labeled as **Perform Warmup Learning**, provides a smoother start to the training process, making it ideal for ensuring stability and allowing the model to settle into a balanced learning pattern.
+ """)
+
+ with gr.Accordion("🔧 Optimizers", open=False):
+ gr.Markdown("""
+ Optimizers are the algorithms that actually update your model's weights during training. Different optimizers
+ have different characteristics and can be better suited for certain types of training tasks.
+
+ Think of optimizers as different strategies for adjusting your model's parameters - some are more aggressive,
+ some more conservative, and some try to be more adaptive to the specific characteristics of your training data.
+
+ #### Available Optimizers:
+ | Optimizer | Description | Best Use Cases | Additional Notes |
+ |---------------------------|-------------------------------------------------|-------------------------------------|----------------------------------------------|
+ | **AdamW (Recommended)** | Best all-around performance with weight decay | General training, modern standard | Stable convergence, handles most scenarios well |
+ | **Adam** | Similar to AdamW, less regularization | Small datasets, testing | Reliable, well-established optimizer |
+ | **SGD with Momentum** | Traditional approach with momentum | Learning optimization | Requires more tuning, good for manual control |
+ | **RMSprop** | Adaptive learning rates, good for RNNs | Specialized use cases | Handles varying gradients, alternative to Adam |
+ | **RAdam** | Rectified Adam with stable start | Large batches, advanced setups | Improved convergence, newer variant |
+ | **Adagrad** | Adjusts learning rates for sparse data | Tasks with rare features | Slower convergence, handles infrequent updates well |
+ | **Step Wise (StepLR)** | Reduces learning rate in predefined steps | Long training schedules | Requires careful setup, effective for staged training |
+ | **Noam (NoamLR)** | Uses warmup and decay, designed for NLP | Transformer architectures | Good for gradual adaptation, specialized for certain tasks |
+
+ For most users, AdamW provides the best combination of performance and stability. It includes good defaults
+ that work well across a wide range of scenarios and includes proper weight decay regularization.
+ """)
+
+ with gr.Accordion("🔄 Training Epochs", open=False):
+ gr.Markdown("""
+ An epoch represents one complete pass through your entire training dataset. Understanding how many epochs to
+ use and how to monitor training progress is crucial for achieving good results.
+
+ The number of epochs needed can vary significantly depending on your dataset size, model complexity, and
+ desired output quality. Too few epochs might result in underfitting, while too many could lead to overfitting.
+
+ #### Understanding Epochs:
+ - **What Is An Epoch**
+ * Complete dataset pass
+ * Training milestone
+ * Progress measurement
+ * Learning cycle
+ * Quality checkpoint
+ * Resource consideration
+
+ - **How Many to Use:**
+ * Minimum: 4-5 epochs
+ * Standard: 10-15 epochs
+ * Maximum: Based on monitoring
+ * Dataset dependent
+ * Quality dependent
+ * Resource dependent
+
+ - **Monitoring Factors:**
+ * Loss curves
+ * Evaluation metrics
+ * Generated audio quality
+ * Training stability
+ * Resource usage
+ * Time constraints
+
+ - **Warning Signs:**
+ * Increasing eval loss
+ * Degrading quality
+ * Plateauing metrics
+ * Unstable metrics
+ * Resource exhaustion
+ * Diminishing returns
+
+ Training for the right number of epochs is crucial. Start with the standard range of 10-15 epochs and adjust
+ based on your results. Pay attention to both quantitative metrics and qualitative audio output quality.
+ """)
+
+ with gr.Accordion("📈 Max Audio Length", open=False):
+ gr.Markdown("""
+ The maximum audio length setting controls how long individual audio segments can be during training. This
+ setting has important implications for both memory usage and training effectiveness.
+
+ This parameter needs to be balanced carefully - longer segments can capture more context but require more
+ memory and processing power. Shorter segments are more memory-efficient but might miss important long-term
+ patterns in speech.
+
+ #### Configuration Impact:
+ - **Purpose:**
+ * Controls maximum clip length
+ * Affects memory usage
+ * Influences training stability
+ * Determines context window
+ * Impacts training speed
+ * Memory management tool
+
+ - **Setting Guidelines:**
+ * Default: 11 seconds
+ * Minimum: 2 seconds
+ * Maximum: 20 seconds
+ * System dependent
+ * Quality dependent
+ * Resource balanced
+
+ - **Considerations:**
+ * Longer = more VRAM needed
+ * Shorter = more segments
+ * Balance quality vs resources
+ * Context preservation
+ * Memory limitations
+ * Training stability
+
+ #### Optimization Tips:
+ - Start with default (11s)
+ - Reduce if encountering OOM
+ - Increase if audio quality suffers
+ - Monitor system resources
+ - Check segment distribution
+ - Balance with batch size
+
+ The default setting of 11 seconds works well for most cases, providing a good balance between context
+ preservation and resource usage. Adjust based on your specific needs and system capabilities.
+ """)
#######################
@@ -2600,19 +4234,26 @@ if __name__ == "__main__":
# Create Dropdown for speaker reference audio
speaker_reference_audio = gr.Dropdown(
available_speaker_audios,
- label="Speaker reference audio:",
+ label="Speaker reference audio (Press Refresh Dropdowns):",
value="", # Set the default value if needed
allow_custom_value=True, # Allow custom values
- scale=1,
+ scale=2,
)
+ min_audio_length = gr.Dropdown(
+ label="Min Audio Length (seconds)",
+ value="6",
+ choices=["3", "4", "5", "6", "7", "8", "9", "10"],
+ type="value",
+ allow_custom_value=False,
+ scale=1,
+ )
+ with gr.Row():
speaker_name_input_testing = gr.Textbox(
- label="Project Name (Refresh Dropdowns on change)",
+ label="Project Name",
value="personsname",
visible=True,
scale=1,
- )
-
- with gr.Row():
+ )
tts_language = gr.Dropdown(
label="Language",
value="en",
@@ -2638,12 +4279,12 @@ if __name__ == "__main__":
# Create refresh button
refresh_button = create_refresh_button(
[xtts_checkpoint, xtts_config, xtts_vocab, speaker_reference_audio,
- speaker_name_input_testing],
+ speaker_name_input_testing],
[
- lambda speaker_name: {"choices": find_best_models(out_path, speaker_name=speaker_name), "value": ""},
- lambda speaker_name: {"choices": find_jsons(out_path, "config.json", speaker_name=speaker_name), "value": ""},
- lambda speaker_name: {"choices": find_jsons(out_path, "vocab.json", speaker_name=speaker_name),"value": ""},
- lambda speaker_name: {"choices": get_available_voices(speaker_name=speaker_name),"value": ""},
+ lambda speaker_name, min_duration_seconds: {"choices": find_best_models(out_path, speaker_name=speaker_name), "value": ""},
+ lambda speaker_name, min_duration_seconds: {"choices": find_jsons(out_path, "config.json", speaker_name=speaker_name), "value": ""},
+ lambda speaker_name, min_duration_seconds: {"choices": find_jsons(out_path, "vocab.json", speaker_name=speaker_name), "value": ""},
+ lambda speaker_name, min_duration_seconds: {"choices": get_available_voices(min_duration_seconds=int(min_duration_seconds), speaker_name=speaker_name), "value": ""},
],
elem_class="refresh-button-class"
)
@@ -2662,45 +4303,148 @@ if __name__ == "__main__":
tts_output_audio = gr.Audio(label="TTS Generated Speech.")
reference_audio = gr.Audio(label="Speaker Reference Audio Sample.")
- with gr.Tab("Testing Instructions"):
- gr.Markdown(
- f"""
- ### ✅ Testing
- ### 🟥 Important Note
- ◽ If you dont believe the dropdown lists are populated correctly, you can use the "Refresh Dropdowns" button to search the finetuning folder path and update the lists. You can also manually edit the path to the listed files.
- ◽ You can use any config.json or vocab.json, as long as they match the model version you trained on. Only the XTTS checkpoint matters.
- ◽ Upon successful processing of your speech data, expect to find multiple speaker reference files if your input consisted of over 3 minutes of speech.
- ◽ Audio clips shorter than 7 seconds are automatically excluded from the reference list, as they generally do not provide sufficient length for effective TTS generation.
- ◽ Should you find a lower number of reference files than anticipated, it may be due to the initial segmentation performed by Whisper (Step 1), which can occasionally result in clips that are too brief for our criteria or indeed too long. In such cases, consider manually segmenting your audio to give Whisper a better chance at generating training data and repeating the process.
- ### 🟦 What you need to do
- ◽ **Click** the **Refresh Dropdowns** button to correctly populate the **Speaker Reference Audio** with all the available WAV samples.
- ◽ The model is now trained and you are at the testing stage. Hopefully all the dropdowns should be pre-populated now.
- ◽ You need to Load Fine-tuned XTTS model and then select your Speaker Reference Audio. You can choose various Speaker Reference Audios to see which works best.
- ◽ All the Speaker Reference Audios in the dropdown are ones that are 8 seconds long or more. You can use one of these later for your voice sample in All Talk, so remember the one you like.
- ◽ Some Speaker Reference Audios are a bit long and may cause issues generating the TTS, so try different ones.
- ◽ Once you are happy, move to the next tab for instruction on cleaning out the training data and how to copy your newly trained model over your current model.
- ### 🟨 What this step is doing
- ◽ Its loading the finetuned model into memory, loading a voice sample and generating TTS, so that you can test out how well fine tuning worked.
- """
- )
-
+ with gr.Tab("Testing Guide"):
+ gr.Markdown("""
+ # Testing Your Fine-tuned Model
+ This section guides you through evaluating the quality of your fine-tuned model using various reference audios. Testing ensures your model produces high-quality TTS that matches your target speaker characteristics. Follow the steps below for effective testing and validation.
+ """)
- with gr.Tab("🔜 What to do next"):
- gr.Markdown(
- f"""
- ### 🔜 What to do next
- ### 🟦 What you need to do
- You have a few options below:
- ◽ **Compact & move model:** This will compress the raw finetuned model and move it, along with any large enough wav files created to the folder name of your choice.
- ◽ **Delete Generated Training data:** This will delete the finetuned model and any other training data generated during this process. You will more than likely want to do this after you have Compacted & moved the model.
- ◽ **Delete original voice samples:** This will delete the voice samples you placed in put-voice-samples-in-here (if you no longer have a use for them).
- ◽ **Note:** Your sample wav files will be copied into the model folder and organized into 'suitable', 'too_short', and 'too_long' directories. Read the **audio_report.txt** file in that folder. You will need to **MANUALLY** copy/move the WAV file(s) you want to use from the 'suitable' directory to the AllTalk **voices** folder to make them available within the interface.
- ### 🟨 Clearing up disk space
- ◽ If you are not going to train anything again, you can delete the whisper model from inside of your huggingface cache (3GB approx)
- ◽ **Linux:** ~/.cache/huggingface/hub/(folder-here)
- ◽ **Windows:** C:\users(your-username)\.cache\huggingface\hub\(folder-here).
- """
- )
+ with gr.Accordion("🎯 Testing Overview", open=False):
+ gr.Markdown("""
+ The testing phase allows you to assess your model's TTS performance by loading a fine-tuned XTTS model, selecting a speaker reference audio, and generating sample speech. This helps verify that your model closely matches the desired speaker's voice, tone, and inflection.
+
+ - **Select a Speaker Reference**: Reference audios 8 seconds or longer are available for testing to ensure a meaningful TTS quality check.
+ - **Testing Multiple References**: Use various speaker reference audios to identify which one provides the most accurate TTS results for your intended use.
+ - **Exporting Your Model**: Once satisfied with your model's performance, you can proceed to export it. All reference audio files will be packaged with your model export, along with a text document explaining the available audio files, their durations, and the relevance of each for TTS generation.
+
+ After testing, you'll be ready to export your model and prepare it for integration into your TTS applications.
+ """)
+
+ with gr.Accordion("⚠️ Important Notes", open=False):
+ gr.Markdown("""
+ Before beginning the testing phase, please review these important considerations to ensure a smooth experience.
+
+ - **Refreshing Dropdowns**: If dropdown lists do not seem fully populated or correct, use the **Refresh Dropdowns** button. This rescans the finetuning folder path and repopulates the lists with updated reference files. You can also manually edit paths if necessary.
+ - **Compatible Configurations**: You may use any `config.json` or `vocab.json` files as long as they match the model version used during training. The key file is the XTTS checkpoint, which is essential for model loading.
+ - **Reference File Length Requirement**: Any audio clips under 7 seconds are excluded from the reference list by default, as shorter clips generally do not provide sufficient information for quality TTS generation.
+ - **Unexpected Reference Counts**: If the number of reference files is lower than anticipated, it may be due to Whisper's initial segmentation step, which sometimes results in shorter or longer-than-ideal clips. If needed, try manually segmenting your audio files to ensure optimal segments for Whisper processing and training.
+ """)
+
+ with gr.Accordion("📝 Testing Instructions", open=False):
+ gr.Markdown("""
+ Follow these instructions to test and validate your fine-tuned model effectively.
+
+ 1. **Refresh Dropdowns**: Begin by clicking **Refresh Dropdowns** to ensure all speaker reference WAV files are listed correctly in the dropdown. This step ensures that any new files added are included.
+ 2. **Load the Fine-tuned Model**: Load the XTTS model from your training. The dropdowns should automatically populate with available reference options.
+ 3. **Select a Speaker Reference Audio**: Choose from the available speaker references in the dropdown list.
+ - **Experiment with Multiple References**: Test with different reference audios to identify which best captures your target speaker's characteristics.
+ - **Reference Length Considerations**: Reference audios longer than 8 seconds are prioritized here, as they generally provide more reliable TTS quality.
+ 4. **Troubleshooting Long Reference Files**: Some references may be too lengthy, potentially causing issues in generating TTS output. If this occurs, try a different reference sample from the list.
+ 5. **Evaluate Results**: Listen to the generated TTS outputs. Consider both the voice quality and consistency with your intended speaker's tone, as well as clarity and naturalness of speech.
+
+ Once satisfied with the TTS performance, you can move to the next stage for exporting your model and managing your training data.
+ """)
+
+ with gr.Accordion("🔍 What the Testing Step Does", open=False):
+ gr.Markdown("""
+ The testing step serves as the final stage for validating your fine-tuned model's performance.
+
+ - **Load Model and Configuration**: This step loads the fine-tuned XTTS model into memory, along with any compatible configuration files specified.
+ - **Generate TTS Output**: Using the selected speaker reference audio, the model generates TTS to assess its quality and voice accuracy. This allows you to evaluate how well the model aligns with the desired voice characteristics and whether further refinement is needed.
+ - **Evaluate Model Effectiveness**: Listen carefully to the TTS samples generated. This step is essential to confirm that the model is effectively fine-tuned and ready for deployment.
+
+ The testing stage allows you to confirm that your model accurately matches the intended voice characteristics before final export.
+ """)
+
+ with gr.Tab("📦 Model Export"):
+ with gr.Accordion("🎯 Export Overview", open=False):
+ gr.Markdown("""
+ The model export process compresses your trained model, organizes your voice samples, and prepares everything for production use. This phase helps you:
+
+ - Package your fine-tuned model efficiently
+ - Organize voice samples by quality
+ - Clean up training data
+ - Manage disk space effectively
+
+ Before proceeding with export, ensure you've completed testing and are satisfied with your model's performance.
+
+ Your model will be moved to `/alltalk_tts/models/xtts/{your-chosen-folder-name}`
+ """)
+
+ with gr.Accordion("📊 Voice Sample Organization", open=False):
+ gr.Markdown("""
+ During export, your voice samples are automatically analyzed and organized into three categories:
+
+ #### Directory Structure
+ - **suitable/**: Contains files between 6-30 seconds - ideal for voice cloning
+ - **too_short/**: Files under 6 seconds - may lack sufficient voice characteristics
+ - **too_long/**: Files over 30 seconds - may cause processing issues
+
+ #### Audio Report
+ The system generates an `audio_report.txt` file in your export folder `/alltalk_tts/models/xtts/{your-chosen-folder-name}/wavs/` containing:
+ - Complete list of processed audio files
+ - Duration of each sample
+ - Quality categorization
+ - Recommendations for best samples to use
+ - Usage guidelines for each category
+
+ #### Using Voice Samples
+ To use your exported voice samples:
+ 1. Review the `audio_report.txt` file in your export folder
+ 2. Select preferred samples from the 'suitable' directory
+ 3. Manually copy chosen samples to the AllTalk 'voices' folder
+ 4. Use these samples for TTS generation in the main interface
+ """)
+
+ with gr.Accordion("🔄 Export Options", open=False):
+ gr.Markdown("""
+ The system provides several export and cleanup options:
+
+ #### Compact & Move Model
+ - Compresses the fine-tuned model for efficient storage
+ - Organizes voice samples into quality-based directories
+ - Creates detailed audio report
+ - Moves everything to your specified folder in the models directory
+ - Preserves all necessary files for future use
+
+ #### Cleanup Options
+ 1. **Delete Generated Training Data**
+ - Removes intermediate training files
+ - Clears temporary data
+ - Frees up significant disk space
+ - Should be done after successful model export
+
+ 2. **Delete Original Voice Samples**
+ - Removes files from 'put-voice-samples-in-here' folder
+ - Optional cleanup step
+ - Consider keeping originals for future training
+ """)
+
+ with gr.Accordion("💾 Storage Management", open=False):
+ gr.Markdown("""
+ After export, you can free up additional disk space:
+
+ #### Whisper Model Cleanup
+ If you don't plan to train more models, you can remove the Whisper model from your cache (approximately 3GB):
+
+ **Linux Users:**
+ ```
+ ~/.cache/huggingface/hub/(whisper-folder)
+ ```
+
+ **Windows Users:**
+ ```
+ C:\\Users\\(your-username)\\.cache\\huggingface\\hub\\(whisper-folder)
+ ```
+
+ #### Storage Tips
+ - Keep exported models and selected voice samples
+ - Remove training data after successful export
+ - Consider archiving original audio files
+ - Maintain backup of successful exports
+
+ Remember to test your exported model and voice samples in the main AllTalk interface before removing any files.
+ """)
final_progress_data = gr.Label(
label="Progress:"
)
@@ -2730,29 +4474,33 @@ if __name__ == "__main__":
folder_path = gr.Textbox(label="Enter a new folder name (will be sub the models folder)", lines=1, value="mycustomfolder")
compact_custom_btn = gr.Button(value="Compact and move model to a folder name of your choosing")
with gr.Row():
- gr.Textbox(value="This will DELETE your training data and the raw finetuned model from /finetune/tmp-trn", scale=2, show_label=False, interactive=False)
+ gr.Textbox(value="This will DELETE your training data and the raw finetuned model from the specified Project Name (above)", scale=2, show_label=False, interactive=False)
delete_training_btn = gr.Button(value="Delete generated training data")
with gr.Row():
gr.Textbox(value="This will DELETE your original voice samples from /finetune/put-voice-samples-in-here/.", scale=2, show_label=False, interactive=False)
delete_voicesamples_btn = gr.Button(value="Delete original voice samples")
- prompt_compute_btn.click(
- fn=preprocess_dataset,
- inputs=[
- lang,
- whisper_model,
- max_sample_length,
- eval_split_number,
- speaker_name_input,
- create_bpe_tokenizer
- ],
- outputs=[
- progress_data,
- train_csv,
- eval_csv,
- speaker_name_input_training
- ],
- )
+ prompt_compute_btn.click(
+ fn=preprocess_dataset,
+ inputs=[
+ lang,
+ whisper_model,
+ max_sample_length,
+ eval_split_number,
+ speaker_name_input,
+ create_bpe_tokenizer,
+ use_vad,
+ precision
+ ],
+ outputs=[
+ progress_data,
+ train_csv,
+ eval_csv,
+ speaker_name_input_training,
+ speaker_name_input_testing,
+ speaker_name_input_export
+ ],
+ )
train_btn.click(
diff --git a/training_metrics.png b/training_metrics.png
new file mode 100644
index 0000000000000000000000000000000000000000..529c4fc649a115c7c15d8ab3f59472a50e7e9440
GIT binary patch
literal 48489
zcmeFa2|Sf~|30p18Z#}7NxP-dQ<7p#3lT??Qd$V55+;%)M4@mTP0bKRO|%JZ6vB|L
za
zP@gb%`dB_bz6onqt7-G`eWk_6_eIB-U*KO%9_~4Xr=c^*avi
z+iQJbuetF|dsEx}=GImVWtPf(KY!-#0|)l)SCo~t9R2|rYg;qfZ#RrC!a=^;w|eV-
zK0d)6^k>u!)^&3}zPtHr)RyZWyx36Ybo|Vl^Q?Ck2X}liIrwGNc(+M{hYJ_2((;sE
z((2ljmN#`=?B&g;RujL8OXYpf<8)FZ
zyb@N9JN}vN$@@E2{nFj8&Ut!9RI7Yx2VZ;j>Fvk%vv|t9-gDdE?R%rd(^XTW56(PX
z$G6Vy8v2HhZ>7+s&xXE>o~=4&=#Pwp<`Jt@}T
zxKYSkxowMcqlJ;DQ`2q9V9h`=b+JP`mvuMXvMIGm_jMAc!+dYDgg&CGUj5dlU+|c!
zuB@C?-x9NX%`SK1B?%-Ibu?F1l{r+Lz^h-gw0!u>gs9!wR}$Vm
z74%RG{P1s*ru}1+GjD{195d>z(`(G1F@3s(gz@uB%cG1_J?smlrKPWihVF@vT_LEv
zXo*#6p)x;x)Jy(#b#+@3?Jcv@YreZBMF$+M=a3n!lE=3xP=$Tv_U+qcU2TmeB@w*G
z$EPhcZfhtiJG2Nt#@BZL<^3aqgR27H0D7!eY6-g})U>o33panOqpA9U^R!?sdYbr*cYmaq+emo0&?E
zUwd5bzqAEUdC8pqEwOtx?Jc_#TXXfmxguc^k!2HAqgrMPtM@Obr?5mb<~1*0{&vrv
zJ;mX?Z6*3Y+{DS=wryK+xOGwV>?PLar9oz&t85L?^{_07OMUyyPgz8D)bJsmZ{4M{
zZ{NFhe5pA{v_yLO@w9JWeX}`ttAT;R@#%}_;bE^YuIzYVZ%N#Oy83$7vnu-ZCxw`k
z#n*PNLo_03tMrdc^O7z-r_$8(Utm>XUyT{W7{&#tXGZJcv$O=G^i
zb-9S1;UUxQhu-*#94dpBLU_d5rtE@>sz)_bD`Q(6m*#08aGwAT46F5afEUj<;{EoYME
z+`t_tlQ1*D`TgS?H*X#cfk8p`2b%AP7K=OEvTO~VpGaL#
zIr`N^{-{K7018F+bkAqTHTseZkM~DwnIW$HJoO
zl)Bh4fDfJEb;tO{oxjUmT77X#+}`*Pj;2jn&&GKBLj5k+#d|`zF-k?%rMC@71t>yy?4Qa
z1=%?{IaW!rv3cU_v`+dfIn*tmdvf@bRyGK-vD`OQ;v(Se4_*X?vN8wdCx>hRq{fS@
zEDH?_b6|i}NvrbGhTk4u(Tg;6>R}&E%tG5
zOiAJNw{y|~T$P+;>EI$MPlM^XM!>XpZ1S6ov*}k>dKzb5Tx#?BlzQ3|Uul!dlw$-<
z>2#FTPoBT!YqN_st8{gB)lvdg(wZ`rc*m_d-6c^6YIEp*#gL?*951T4NaptIUn7#F
z>F5HAgCNN*`ZHZ2p|b||TTAUrENyCo&Zd3qCh|?>-S-+7IeneYrTeqk4P2gcwNkSi
ztLNL`4gS+_?Rw@B8X9`=jjx_@X@X7jqgmbuI%xaJ_sDG!$b~oXZQ8$o|GGdGWo%%$
z1qBxmy*=-=ozwBkt?HGVxJ>&3X6GR>_)#5|JyhuTnTP7eKeGe-r{>&|7Z^O(|J*Kk
ziPxof^
z9Rwxuc4B+xg~ifhE*-Brs>J)J78r5Zgh`s8NJplCqEI?;kPi2L5aEE@$@J7*`o`zn
zcRPMq=eJ}57=bB5LgOaScb_D^IRzkwWv{guFryN8ri_0{%KJp%3
zqBJi}XSp~x6h_TzO5}0lV&A@fdyd6%xoBT(F1Pd1$)u(vupX7f^zb}svYsdW=~Iq2LPW^`hu=j4@YYFOFt;xpLfu9s&woZORk
zd>Zk6_Vv{l_iw;RIJ1A8>Xf;@8ghGH+`z*OM`$*J8cksHqz5(0SHsIOlV7ohF&^SCK#;qm3uSGWnBVD&Y7
zJC(BjQ%Eb`rYddso;@DV{N&R!fu9oNtjg{HSie5GqD}GI9&__!fcp0J;o;$ZKmPPn
zL+NcD!5skx&>@BQVa!TlL+RwfDA6q%k)
zr9H)2Lq~HF+ehCrH-2Yzpm-Lbc6w6@m+f@XE`O_>LsMx-Vsdw4O-a1&Ksz#}4z0xz|S~tB5()Tt260w!`o+`
z6@6{-QTtv!uq|IQkExJ_Hexxgb9V;>**MSlNtdB&wXa4HR#msYBR|zFpX=3k#JG0(
z-2A@hPP=qe|CaG_;Q3CvRF`{QbCJA!ynaRNL?^5wVBZ#ID$GhdF=NVAb#Ivu?m`Q9
zJ#`0N7_ZL)>bVjX71hAwKd=EMs*P7yt;7SDWHMXik=HQ8V%%C^5Lq(|r9Pp*WxAC8
z=1HsOy?7B*qFK_Iljy*Ut9R0#=Ud6o>R8^>kY9plw~h*>66vyo`x0BhB!b6G
zv~Lt^H?S`tJGsA`mTvpR%?1Xd>Y{eKU~8-*S405m>^3(yk1sDSt`nRy_Yv?=
z>rCO*fMNqU46-o5D7y-mVVN@2h3t`!B=jMDtQ;c=!ox=c*Y5
zd(O#Q8>*@4f-t6%R&7pDlw&K#z*NNpyL(cNdRE?TPMaYl)O?Y?BbFZhj)aJH#UBKpQKWL+fz|TI%
z&$+U93{|^FN_2GM=<(--3L|i4;B1W)xc+}DaQ)9GTmQI3*PFs7Lciy=B-sGS%kU;)0Z4XRp2Dz$+K!G8%>V!!zQaA&FdiIy&CSa&DIt
zM{;`WsVYD_VmVG*9|I@x7Dy2=j
zG<~VTI^CUxFBJQ`IIh>1)aX)ijHHH?{Del`LnvS`tw82yUa
zCA*(riZ1U-n_+!iRLLGpOk+AXB0BneV77r$le;@=VnNy=H`_l~jMSaC(pyX!OS0PV
z${f|^R*seVcLgS=NR({;MlKXaKyi}Zb|HsS8}*ypZ@)J&QUb(0r|e{(*t$P+spRw-
zGuk_q4@UrI8Nax(5f35IA1lTAqpuFvkhWmcE#37cke1rPIj&a#yaw^tW{0fcCu6l1
zFJ9c(mTR9lAU5JEpr}iTGU{M+<+-K%-%4^_sKirX;1J738A_B&!Kl5>gDU8*wHyDP
zb%0zx+Vc(X7Csy$tord|eldQkR*TTnU{`l2gI4B#-
z+}&d>3S27N6ADbwd
z21$|e``3>xBlR)*FWRI{!&S_Vi^J(xXe%_S@=|JYhfq&>1FN?s&VrZJZU~`Qa?MXA
zbho|y(ez`{`Txx0D4_;DOQI5kcdtVuD$$FtF=Wuh!H-Q^xyADGgmg@>HF07;J8u?H
zWYw6(sT9PE@hJ}RHTffa4V@2qjF1r{WW>No`1j~}M_}^6*~N|a$9zqqrW5~!>B8ft
z$Q)nqwKT?sdU|^5YHIy`y%|eY
zp09-Rh_u!K17H6hm~=lBNdqtPJACla=Tia*wTF_{q()D7fyNW==j2RQJq=J!Eb9v0
z6sA)9h~DRc%K)q~Ws<7wZ37jV_xT2YzmS!EIY~%p|J$d>5}WRXF(lQNTf5ee7h9I_
z+hNhy>jZ!JL>FzLCumig5cS(aB8*TY`&1bTRf(S?=yWgk89H^_w~0EvI4>aWexH4Hwvmbsa*h;R2k%hCk5ytdU9$i{wt1
z40N_lv;hC*MTj2QFXL}7)Fg(I2~c)Q^O4di&IcW50&y`TxJh!fQ&S_Y&8p*5v#oej
zxbJ>esJ(W&IG>mxGk?3yJ@v_b7G_NZD_qA9qHY82rLkc4UgYHb+LTb`bpXPe;Oan*
zN^DHbHCdNtPs>QCXUw!U%;TxFv(}p1xy!M=%po_%JU8b>h)yoJRezMg`~_G&@1irc
zwr)MY>)8cgyseDl>ppcCa*k+B(s$GfsnqC~+O)TJd5}kVZxH{pU>|M#qw!k7d7HM^
zvVuGoDBsYHYL#mF@z4yw>>SjWbBkjwbnw^)=18%K$z2tsM8R|FMQtkdHR=SKYn2w}!vbkjl&3)ZE`cT!5OX^yf4`#gU)=
zVPW2!cWL<%V-)*HE=WrKo(40PC#uDF{4~e&pY>{cxpInuFst*iW+jTLjH(`VvKUde
zExd|;9fidr%M?GfzYKH&vYZtp>`=jWkiYnizy|-uWKUVHf+246-Fx>kv)wq^yI-3H
z9NHxxFjYXtzxMvEEES~ym#*3^$u8`@#W7Nt4JRA*cNHXdrtEu1g*)6oN&(pP9PICH
z$Fafjl~(umcDH*|^$;7QDJYXSx8!Q3!!?oEoKlvtcdQ3sX6pBTKUJPvzdY(`@TD!?
zxz6(%Wm{$Y-^+%;EAY%;N!|d$^;+l;R-;;M&TJWJDy;=8w;Fp#tFAP8?x_gecr9%u
z*4L>nDGKuZ=Zdr1{2F#K`~{IlfiCTd6+!w|w`Z>pShkc6$-r8?Tg~MDUY-twf8&y6
zg9Lo)b_@0$GB}XE(j4V}T*M}9OCRvogp3YS%;jLF*1$AljuUSV++JH37P+CnMYG?B
zl&Mo(j5I{&=4FfBefiTkZOAtr`#P(0^Yo1TBkt6Nwlv(z!nnHw#lcv5p`(hRT`^q3
zx5ds=#+C>#HD|-c1WhODdGm6>76zjDe1Y2w@N_e%36B94rLCYHQT~GQ(=OepqOz26OMc`gw|qJx`w7d;&w2ZugR6
zccEE1kP!NCdmF8%xL9k5ewlY(tS9Fz_k~p>>;>zkf)*Wm|M~%MeaPs`in&KPeO>*X
zf&FXb%MP6W)>qq9*R0FBr6xNBvP0G{zIAGL!TkLnSp5sbv5kScH!d8^e9K!u>tJxe
z0|&v~nM<1*VIfQp8|GQ4w)#do&kpj*NwBFZ&Zj#$28DR6gMyP>tlyj|^3hKH_?PeS
zw*@YX0_XK{R;gW6YD$=el~_IG&K<}2R%@3JemI%GjGq==kd$#|!CL&Wi05LvLvLg9
z&?hX_0bMVPSM_%&-`bHRWj9k-ujS=4`HG96rhOgTn=Drc=}yZ#-{%D7UM_?&+Ti>>
zI#4zcX0tRzcjF{&kKVb)u{BSxJ%PLP@vqE>3x6W2wP1U~cZ;!^?^R9+>%*y9x_HTw
zrX{K>Uw^zxbmcx?wB9g}i<+(Oy(zdKNc&2%gOV;Rz1T{nOw=r`;-dD>Y+YgXuyU~M
zeE)|0P_&ZX>AAb_x7yu;P0xnu1p`|Zg+jYHemCqOD|+6_R}q!aSQ770AouFhi}QU-
z|3I{o_~Fq$t=e^hD)^aZm|;|`l3MebUE(Hp89O%A=FL
zTB?zeAk(c65$WAar|pgx2@4P1)pCA68J}N&|8Tf;wT^OP*FI5*Z6{8cFrhKuA$-F0
zMZ3Mb1(aK!*OVo;OE!f^M)DryKDdAVs&az7eEC5W((XX|koekp-*5GM1v7zsT)Imt
z(jNRgF7b`?zVGUH_fMGm-ApfqO82P?cda80flbDO_!K4H-H9YeB_A1lFjuLTb*Hsq
z-*+`nD-Oq(+3OQ@lfz1#(D~L6v<$ZBa#)QIViYA--&6HNt)wF_(k0KTH>=wr)nig%
zTa@C{-%V35@8$b`2}Y+xK7HHduZWjf2}G6HTAt(-Z%|Rfb+37D_>zOUvOkgtfwx(K;!rSq
zonL5XUpwv9kNMeR^L;g%kBIloB<(z}S??FVz?-+SE@M7U?A3MBkeV#BeJYTH1)+dG
zl(p<1hP-;2eZKlRU4{+HtuLl7-lJQhzDRak%v?jK#@LR&_GEF=S+|Q!cXyBIc6i+f
z2HfjAvp8TshC9mRn5?3Hz_fo8z%W4z|ytFXuGFZ6gzHZ*%
zihu5O#O!;Ao2;!aPkN4KfCyZQg;vQGE|BsAVr!;6mJAe0LNO{}#b}}I#r}C|rIN0!4T`hlhogRB?T@8NC%2PvzIhUZvwGmgM)gT9vKpf9SXxEojMeVIh*`2pL#j8O)1yw`A2~
zS?AvZbw8JR4`HV_DB>OFq#T=AVlV>+)9HBKI#h#8iXn)+2s{e)x21)fsQnh^P*@_y
zB?sCLFUZW&{OJTgHh@;;AN6Q$AZ$xh0fT3tfzrhNQ|oEeV_aep0JQ7V*D~#(WgZL*
z%(4k&CXYSmg7xsVB=xqHn77vD)xte1+r#xy)C#?-+)+}|w*>DMHktsExtHz1q&We`RS|d6BP`_Uwac2v(WPrI{#Rc5W9c(GPOD_oE?8>}&-UU@r{4`3o>ld13uA@JJ
zolniQ)vk5(aB+O>$ISswn3aSPWODx*j_Ng3v_c8{4jhI6p-nL*)
zyhihZs8a*Z$!TqVZA@>hOcYsln0K#h)+g4JY>K7+5SDr+i1Bh@XbxPmx9}t#7?p?O
z#!VV*XP_F%b>~jPo^%1e+;g}S&YYRQ^;d4s8(HSeu5N`zU`J$saLMA6#kr^g=Rn$-
zL$&7^==ok07M`O^Tc+}Jt?I|m!+ge%-$wTCQ2D>wD12(q{(qOw*$jy>m))J#P;{&6
zm}G!^dj*H3twHsAc6kK~PUOHcDZ68rU066(lhu3RxX9AzzoSy7)fIchN$KJo=bk2q
zWN8a)0J~*i!kc3w*$bL5j-V)X6gl+zR5KuZ8bmUBO&f^@cmz~K&j#UUw_)6#Zv67z
z;bu6-Q`boP3xjfbufh|B+!(r>IcYGUvwCYafh?de*l1)VPRd@YJiYlVLpa>yeV>P7
z7k?1EyJE{OTC(Vd7sIQ^#ZH*9Bn23t^1(RSD-^0a(CsjQ--}0uB+J;cx
zDeYoQHh>{Xn5U{ZcU(V2W4j1_R9AZ#f=u8-S}A_@;Q;j^u)S=lUzhG^U1(;G0f*Sv1NB)a
zawP}lG7i<*^ty%oU@laCT%kL_HSxzaU0BZ!9TgfJ9Bj`DpCXh!pn>`RchX)eRsgV<
z?cefaB)A>az?OgpFd{cxcKYFlkK&%qV1P{WLN`U()kA!ov*ULbajlyu1kr0k00YOp
zc|6y`()z2YVamib<4ob*L;KGL2zBg)X^j>)xVQewucc}R;Clc5!ciYtl?V63)30~#
zva_=r6h9YQvN5ei!M6@b*1yP`w)v{722NJtX{&Ks6GRlg10^D)?9|6|+m5PF8kVc7
z45L2my>_^XPB!X%qE}hX>hBce{K`~8o(z;+?s4m$N4`sh4BG?>A32l-;WEU|()iKI
zZ-s#oqQO!3Kcm3*vOmg!cYo^{v%Y(g?;gz9vO3jd
zf}4li;o1%9qkJko{2lomu7~ADX(_!J9X(em%u8W4WzPhUxmCG4_Lj70ut43m!kZ;_
z@l5#4&oAPnjKhTVVvcUPon5;>oWH&6X|IZdEh`h*GtcCC
zByu}B{}UC>)=xrXqCXM2?EEA&=J6+}=_-npAlF$HRA183T
z>*+ZmICh#}pVmwRhbEA~18;ZYdZ3GADKo$#Fc?R4P&Db`igW)R88*sdTJ!sz2
z=V{A`fjxodaVbF~+Vs1-V|`2o?R6zIw9fTN=nU(-awrH?5f;IwdT@aN_v%hoc-m7(
zR1hrTQ&%21s^br9c&dw`X(&u`a+tq0`WibNJTCeB;cK!QKkfUS$xf`+5Y1HVXiNA}
zV!|hg0%Qk`()1vk7Kzoz20pLyt
zALt3jXCX-#6dGf4E8|M6Jc*_TrqLL%yRR9nHA(uk2)RIQ0hCy1&gK<^MsG`Wxc}1_
zo4SzeX<+`Vz#eO@)_P3nqJRG&C&lvjbFAQxhKh^IYMvL8YHPpD_04
zA7E)-+rNB6%gomZyy4=heqRmF!ORB>N4;Jz_=8J}_-oW;xQ=()HO6WY=ZM@J#c*Ef
z?OVOAy^z=*!v^^VEaU*B%WybSR}=gr>l=3Pdy<{1pPrxip*o8fqk{S${7LiPOm?->
znk#G!+_7=<=E*p5hs61Ry~_VBt4r{G5dXJEqrVvcsC!6|bGLT+Bk#T=&Un((%nHVv
zfCR&#!!j|6GK2cM`j(L_3+DP1@KCcdqM;FvGG2MNudCjsg5919)BUZAB&vLZ#JS#E
z4=R!dE(!v30GT#FS-@9zNo%g!8Ph1+HyVQcCr2xP*_q8bDT}A+*f^J~L!u(_x97rf
z$tG>r(5NUm)Kf~$lOS)Bkieenw=6cnN|ZE+nCiwh%DBC$j1fU&q)ny&Gk|`1KL=We;GbMGR{R5w&2eA>Abk9)tDbi5gvB
zCWFYMb3PyB0FfdT^|bE%_p=Goi76peF
zwYMl5!%@z%IEJ*)+ig-Ol{x9ri-_0_8pytyJ?4gl1Z5a80yw+8O*Rct>EzR|`gN_$
zu_=ZJU%7U3Klt+)-jt9$Zp4*`4jm#b5~>nY$;NrsnvUsuU=Zn*X$WY6$wp;YR3Ob8u7L5q{=|ayzi|;5Stn^M;W2XrN(`I`EO-A4
z!O(y7KA5*ub<_OGWcbL2vO=8Qvhl@>7o-9ebnHiAO^j4Cixw{?K*E^1ksEaBg$Z4r
zlQV~q(&JH~Zeose!FNsrblZ(8^ejx?m8bw5MzW+m8L+-G$hfDO^IcEDkfSJ
z#JU(LE~BHF4?>Z|R+%@I?0?|d-*$aj+~5qc)1<&
z7C4?x%6|kLb+zTvQmRCq!@E=)!`)w<*;deQ3iz%v@<{&$zg_yWXQ8fAR*BKKPU73l8_0){bAl
zzuG_32v2#rw?cpx0JcS4N~|KmR{~um!{H9q0KDI2+yK+9^N}FX1eZVKTml1GQr$NZZQ&Tc%eSH1#&3Hdyfs1$i507KW6KMySl?KxzmsCLh`p8^|u^e!|2kS1Wg$vkfEd5cCGs|yzqZ?4+<;^U9RXJ($l+s%!3c7o?HcnK
zVVZ`{gzLYbjQc|>Yj*!jVxOYtd!qE4{F<@1UmLWyH*L0ct%J7L1G17d2o|T)b`#4h
zV{t+GfxlN-Cb1#9!!xw;bUD&^cB8@A7^+fK&y$!Iq{<49vi6I(g1E|
zfHdH5Req9@!C1%OD6*@)N9SYcwF9)YUH6b))v3K{3&dO9+t#jKOO||+Ng&YoQM!va
ziu~$O1ylkp$@Iq1Z~q1A20XbkzfC46l=D#Z;{9f*uMfvDDsanQ_R;!;0O)54t@--o
z)Cuxu3TW4cw)u`{ZEYaYKzNS*wu(eDd5vx|x_i*3r%o@~15tNp-ayx&&>nd|c$+R`C8F{7BLRv0{|$LRo;|x2zk$I9Vqlke(^&0efwyE^_R8Wp^YGtAtpBV4O?V2yzRqa11hV31U5=bOJBt|w$1(s@U$OxqQUZ~i
zbahim7Z15mDeq)hY%C>&Duu_##&#b0$~M|8OHBF8r9hQ~;4<*xWgfs~5+WLj&l{xm
zu3ztt(36@{fK~h5tA|&wT)FbFt2|x>k=Z5HTuScTIDc{#_7Ps-e`592QXt7PMmgH1
zId#fe7e^XMzYaPVHRslf-auuwYDPuJD&qSl%NmLZ38fm#QN|!qI?xOZI@^l^d$-<&
zMh(vPhvI$v0!QBz@2yd9Mu?UI7rNDSsMH0TH`CD-E)0z|P<;Ot&;w^XNa4NWaJIv(
zwxNeI0lqC|rc|jxf0?9@Fr1GRYh^-IY2GHiREicwGNspAncIH>fAn*}ps{`UUKP1S
ziAfZ=3W})R>Dibm+xKZzSn#Yz(V_3%A3Sto>yvJJ`vZ?dpJdsO%;iI8@)=>Q#3O9hp>94BC~87iUCS5xwF@vNnt#^km^@G+>i*nfi_N6^l#n2k1v#U!fgCLm
zI%oX_VbT{V-Ry-|mnqeei+k-^nZ^gr3@7vR@b1*%(gkG2HIyeQL9v#@&CkjC20wRV
zf0nX5Y2+)C`W=wa)@8o;U5*R&~)RWbxC6$^j2*Jf{Z|hPAgqp%BO7W`Xj)jl<;tL*}n|%yo;)+
zf|>BRWWYfQ3Yh{IsCa|51^DJFSiD?EeSJUHxtWsS5x`x|`uOpdigbY_Da=5T64lAU
zBBhJt9iF4F?NAytQg0M*dIK@`wvJwkvZ^uiaiTzTcEB7WtYqIXxAAxQI24L%cqc?*
zldv|Dp6n@SSNDS2odAVmm^{C{!X(9ANVLtRD>u-Kwg@1hCUQE@a;8Cg)$~ghRftZG
z9WEe?ztRK5CX*wbOp@8b1b0G8Cpd*{3Np(@E-ETOD|Q;!!1R75nvr&`vsznbBhD}(wLy11A{kWS-4*L
zN2Svv&rg(2|4W1qBM5`W93CSzFUI{#r(%S;`ZqIIe_Zq0!#GHx7$&^}wHT^YXLW6)
z-UhP=9PNt+x06)phbnjlxuwV-g?yN4+4=dCHB~xyq9H_+Yj;;Wk8)gL4fixUPI>v%
zGJ~u>JuTN+ClD|a@qYnv$ZP;1Go?q*A^#H;M^NY(qd660@WL7^_-YA_A%)`MeXk>b
zC={ouXliBh_}AHu7GTiHnydf=uL)GM{H6K`8CJng_TYzS@>SU8CRd?n6Yo#Ujk4H4
zG&V3K+fUW%tz`lODTU^DwT!=021C^1c7VNhllJ_A0^WWOBqZ)SBsd!l-&Gxa>rL
z7IoCqpk^8rH5u&ExfckIhJ-}}`Xcq~hC$o6!_-7M#E>Nk)FZUglEXFv?^s~yv>+2M
z#to4tpdc@I>273%Q*&v+`~{f+cw+dxKZRHP+g$oS1bW}fR19NVBE*^D8H|
z4tSDp5Km}q)>wezo`t35Gis;SfIk`Jq`nEpsr$hc?sJ!CIFn~hx90Dl=UbDn~o9y)e2LeICCHu
zx&4j4+gia3ushK|o)OcXmz=}FKf>JZTtBhqe$En~vuDk~iWWgi6b-fhUI^xk;9x3!
zb%8x0L_%T)f+6D&5rKN3Wf3yK&nd&uR#baEFrd4C=a8FSL&|Qa4{W!?{`R`2dENc}
zw9Q~1CodWvVz3VL-Avg(5{HUz9u;aFHI*WYAj$85bN&ka2(U_Uw3u`a$_hn4n>13j
zv%0dqD-nL#_-f+A?4U5^gA^bwh~Pyg#gvS9aJ8l&xm^^-xTli)_9!@tJgGRssl$8s
znx9b#7a#Ai&V%e)t!^H*WrI|GDK>b<@N_dh(?CW&KO*_d)r`w6j8i^KrF$=V*R)yuubiR
z0@!UuH0KEhB)~PW9N`Nyjdtmr6u%|)=xmWNM1dDHP2JY^!Wz9lzqt4UE{Z8oszDp1
znx-fY7cs)kNKSuWk)Jj9d=k|AMP$=%K=nS8hD8I4{~5??N+{*64Ve{mr}&8963Ztp
zz4bm)+3cN1L{fCpU;$W`g?D~
z45h9is?m?Ib6M1VGmst?upV4wBGN*lIX&r|rxADGKN)^83hsvP^9{w2-;WQ|kW_IyZcamMu(_o(`@xy;b4u2IdaJ5snuYVziED8W;Ot;#@}
zi{WjZ)Vi6@qUb)x*FHZZx&gi&E29~(?8RZf&;Wm&eN%hzi&I)|IY+3Ap;EX3ByjXL
z5hG!nEFa``RMAJQC%3-8n$stl*cewbKE!9DzI4QG2P?D4ZM<9=Vb3-d&8F`Xrh7
zov@1TPmb3nyP)&%Up-_GXTI&J|L%h38#3`n(+;*I)jdIWtvEqAOBA9xc2LHzBTNC6r>KZA6F1OIyGDnHyIZD`YIw-xj2guxFe
zhW*Y>bY?{7$qb3Uj;4u+D!jGTw~1;9a^F9ZE`mZLE~^z?1829xnz=emn_m}qX%>XQ
zCH-Qh`b_iz_-ebUGsV9$T@s|r9sbK@=&Kj6HXX{m?wGoR$&dS8>7=xpi|x0ffGRowByyt4gk`=MUK=0l^%rcPaJ4We+m9(n&vh
zHRp0m1rvMG--~=rDUStzaPawMcFg+P54ujb*Qn)e2(^Jzi
z-A$WXUq>D8FzN3@cgW-OGZb$YAgYb*Mwt7T`rot|ytQRZFDEE@T0+9Up^(Yh1x0YhOkjKrNpvV4*FV2MhEs5J>xVVs
zVHtn$nH2Yari=B+6Rs&pdN?|lQCjLQn<8XTJgz4)qU|M@G~*FTZN=0Ii~3cS*{@Q#
zfZ-E>bIfGvLcrwhFX*+)pfNd#_m@9b;rM+!N#y8?PNm}cHX0hpaN*H
zya#pAiBtqL@@$;1p7ju30g?8
zDikTS?!qD|2+!{HUfdUh|0IA+dCrV$3qlq1NO$Vjpzy{=rzcLy`8eLDB~pfI0gFNx
z34ut<(gZmP2@8m54+a{aMZ;%
z{DYxFwm9dGcjyUH^BcE)fe~W5+iuHdL^Hk9aHOV|Vzia1f>qfIPQao2+sPjdS?Fqz
z^U%3=t&s=R_bo3rzPj-uPps#unaV_z#tdCL-%wv+rKpasV>N*S
zcUFH-7W17w=D%tej)rKa-CPbob-$zcfQZzR0u1t7$hO=CZuoI;IgSzrw%I=UL=EbRJF07tC!M(nGbfhEtdR*!>bk=DF!mD
z79``g$$@{kYCR0I4WL!)N9#X@fjQ2CciV_$ZOhlUHsvMvyDY(H*o)0&5lZ=fw8$zU
ztRQgj&uIyCHJvEpGlhscrB)PC*7DQ!$PklyWzOlsQ(
zzWY}KTa+e3%jI57m_?MG|Ayjvkj*5<6g&wwOiAuoxdhwWy$}}^Aufrq5*q+3s~1*#
z?K&jsJR;XQ1qb2mWT1k33*l?KFw;I$fs@5l=v41Zeiqi<`mfj3c2gr
z_sux%4xF=?DJ_Zsa8FKvw5+|mPUHM~DPQa1zl?^z;)u@q5Yp4H-Vly$;?2;|d6*Pd
zFSq`aU+fH0-)x02mHAmP*u{OBVxq_dGiPz0&vDSuF)48GIGnGt*qM{~x&C(s%)
zyPMn1?R7x+?p>^o`-0JcfM!rE76J6;m`7^Y-NP7FK=t7!;CTtMrQ)!!YJd;vFbGd%D(2wIyh#+(69{0s0xTko3M|v}Y
z9hAj@A}H_o;(?qs%{?Eo(gDRc*8P<_vn<>pWCoePU@-Hl=Q#c$eh9`eab$SOCeYMR{)M6eO*o%#iM14XK`edjxc`;c2F@jkT=3kmE3!D@cAW
z=f*)kP5NBt&I}_jY3kzkn>4wp$zO5jqA@GlVy^eqJx|Of|;l*kKEHR8{0SSKiMwxKs>5KP%{)
zU$c+P3`iLiY+GONt(!REZ@pV=2$(j&U6(BBO|;6r<><2TPH4uJfiP%==++m^Z;{!m
z%QDH`eLxE=`#UEUbqvRh@O+`$n6@yjxgAkKYKXpHFA{)
zYCxa4Jpr5*fU23~%mb=sFRLV#$Ksp8vD{3J|#|!#-Mp{pHuB8%I9D
zO0^U;SeuUixCn8kAQ2TviinfizW*81rv4y|@UtqtSA^0o#pT@*@svR1lcl2~S%n=Y
zmH5y?q&`@mh}M_8cMYZ)*Fgw|%_oU~+ae|5aH2*O?(XVhhyFq@z-}tJWv0nM5d@TQ
z+$s5r8l|ylXMw(}&68{CU5f}05I=Wx&VHtg9HY8)!-c@*q|m+IN}cS{UU&-a^7UN)
zO30=(qIsD|=Z_vE)sT^D$iPVWU%1HsXG_0-sCil5$k=?SbvVTvV{b*tDwNl8#+m81
zo-@?V57q8g1T&^WKw3pP`qLKfQo|N`8Jby9Zpi4S#Fm`3RYbyq>4j<88vuC~3@0Q6
zwL#Ivp|yuD_Fqy4DEOxQL?6%e1@2C^K#_cheX2AYFxN}~K6IY&o_;t;&#kv4xjrS@)q1pSNF-EZjm%UR3b+VLf
zR@gU5NKkGc(?tX+NaRUxaNr^8u?n4&%=_|l8u`-(H1E
zSqoyMEvD`jv@iH`=BPeu_XY3e<~nbtb(Z^}!nk_`ZH=HU8kBtR`im3$r$*Ly3%rc1
z@^nalIcLj8!D}7T>KEpJA*>;}q?P|p&@J#ulZV
zoWAXPx{Tkfu`EW@OqY@y{p5l%A1|Q(
zSY7%zdZ@+;-x&A`bjipI44sOZBd2KSR6N>TG$!=PlP47WftZll2yZ^kln(-yb?&d=
z0N|y8>lJ}crWsX_GzxVikM`|!I>5{o_B!<35*IyAmvOtpHADqNqKjhU;~82Cdeg3_
zPCO_b4x*D+6^hiJ1R5YQ5hC`LRKoghMDfj(9Xi_KH+K)AltUFPN?#_g7TR9p?NOjS
z6xlWIo}o1JJHSIvsmGOM#q}1T<`t?g4Z-x5YQ~we`9nzviuycheMAeOcr_v4JAzzO
zQU7u*bVARHG>V|FB3=@$3DAYXk5RzPR3CgRKSz(%CGv{=>>%kC4Xl9++})x17Hcb~
z7<|%wq!SwfTa!0xU}$P;u^ol)H-SvkkA}j5)VnE9x+Ht3LpMah8LdYOPzSJi)J;nw
zNmGa#qp~DaNSb$16@ume_`#aXZpNqz`g3jH$s;9q6)5U-Z;L&95crIqULm^ks)bw)
zwa|47=JmINV~xMKtngOQ!}N>GTht#m6v20{GEp`A=}h7D9^~cE6b_3ti@f<+9@31i
zpkAHPipClCcU|TV&?aQRrzH7QCQ|$g%Dq%Qp~9rlS>5;w7pGF_3k)KR5Kjr)n0DqY
zzNAaYlt%4pBE0z7<>iJgHZzrWeeGe=ht%J%J?!=+>-;4V2}fq1&Al+~XyrN9+%waT
z8n?Duh2lM+IfNoRwrn|AAJIthCPWUv-G|7angn!rN7fOO(+L|Lg#d`Y?<0P5wBWMm
z%~imc)X9Lj%qYMWTd`3!V-U=!twG5`;sTfWKE8v+2Wp;Q@%~YMjM`D^dAjh+R5^58
z$5UN>neNsxp^&;vMI*Me14#yi^H-F|gT*cctAoDn3k5_Jp4Y|#M1o#Iu}vWH8m=Ch
z{ACJCN=kaYzZz@%^4_S0Uyk`6ZN)zDr{d}2)g4U@!LCP~=eQc5M~k!0BlNVvD67oX
zw!HMk3t`m0U{}wr?Lusnk|E2-N1MVVU=^zZ5nA6uIr|b4#*ja}5uq91_37~7nWu%g
z_!LxnDFs5bfSPKc3_?2~3aEtCe){5h_=nuua-2TWhQTwIRK;|6hmN5cG?zNISegN(
zo_004SVqo6KKA9CoClF8DpFf8+KCaoLw4;{LixEHlolYvqNtfmDPIu7e2oh&S|H}q
z@hSm%z>NEW?CE5xWc+kUe})*vx}|&~?X>o-iym-f%KB^qQl_3Xr3Ba!E1ZtgSs7bk
z=$RptIG=oBv&!SqyDtGT5)_5nY{qFblS4u;#YnKauChE3Bo$o3O8oaXm6l=?0V~Rjn(eK53Paj>@`r+qczds6mVu1N5lo$OMc-|D4g+=^B7v
zIHgVIz^;9KW4`-%bR-K*W`7R2B?0Jh{^G@fN8Q|xR9U7AMbG;T3X1YlQzLIX
zWtozAwDUT%3z#tsy}6?|K}}w2o3TI+!jCP;LKx`ojuoC%{*(m@4?MXDGI>2420ieQ
z!im@(Zl$DCO-@%XBMXD-Oxc;iJYJ^5M<1(;dM|0V&apTyRNE%w8l^X0gw}|x3|{SI
z9g{(#Hq=VULPtA-TP8bN{U{$jmDQWeYJDk0qyN$2k2eo;en5&$0k=g^t(fY5U}J6;ZKAQ
zSLTjT1$24<@kVy?11AE*aD)#UIu#?7+0dEz=YU-}XdNUWNBwyJC)#&ALu5#M&U4ES
zy*Dd&=zMcF9mIhU^%NlUcsVL=OiuxbDS7$#8C>E0aiYtkC?87EOBZvRzbh+16zeAw
z+pM%zH4MU3esfD|
zm!Ei!Pz{;5JjMqN2)pA0s4|39LixSK{j6b|R^E~wIR=_ZqA@@VkX4D7;DWTD#P
zcGf{l*q=>%E2&8qy%&_TygIlb=uqmc>##vdMNQ2|BM7Y;OnWb=W}$W7)bDnK5
zg)_$g5Y^=f?&|O!q%Zq(i<>UN)$din@}?|3I=9q+9E&*nZTBAnA6CVy!-hKZd?kW}
z@zX7@_{RSD3EpxOSnv<<7VK(XD8$;|9*AK=UOM1w70H#UxjWQ4l=2r?&zTQL0sM;j
zZycSR68q(rF9-R9S5f`QChcGJ)W1x4-$_tO#32n*NwxL2XJ}>mp~27;k(k9SSF&aA5nGMR#nOow_Cuo7pa>!`JJ(Xt@j*M%{X$RCf2E8TB
z@Q|??Bj&-sNg1SU6>S(Gc}fxQxi6kVaT*}Lx^2=Qh>Dy^ng3b%=Yp%Wsh7lhC;
zI@S%{x0q9ayot7fw7zLa#eQUS+23xfjqG69sixlsh@w#m?t`s^@
zl1FtQU1u`7q5lxi_+?$4hxdPAN4}akCxtSRDXtmB2v!IH8e^z-s8`4=nNVtNd;_fu
z=;y#Ed?O@8UCyWhfz_SbV3K$XiVai%u$a-g1NJ}%4$~(-g~=Qab>OJlhTSL=cZg;g
z3XryVAG8LAlm62@CIRj^r#qhGNzT_9>de|BIlT7#VVUeb)0!3)8>Iy|RM@<&X8x0i
zCaGqO);rUN;TMZZa)HSJoXp)NaJj4~cR(Psnw?#%S$QJVCM(JZ8@vzEKhS+}FIJ6C
zx`s;B%lk*jcJ+5EQHvUOpeXZY;8&M+e_YS=3Y^ZNvUlW*2_Dq{vj8@`lIE^gO`h&n
zWiy0qV8%#?bGYnH7y4eQ)IXwg)pBLx28GKR4z696QOpYUNZ+WD=#|EzTi0{-Npe?mI=7?MNtwM&=lh$mF=+j$8ykgId^7Nlf(OeW;{`?Z2RE+4vv+s$l1!
z1EvbF(B$=_Qw9-aF;q)IT~6>&g_o$3F#$N9bTxXsQ}xWkqTZGb{ZU`~=TwsVh*CO%
zPE~-&s0KJt3Ka4C!so+VxT|Qaprp2;0P^O-=B~@vd?=1k&ZVbACUX5n1v$nPZ*Km(
zYqj}sC@O>yP)$X*>$-%^VC5*HbFoD}#XLcuhT5GRpo~nr3#s&BXpgVjhLq>+=$YNz
z&gpMgK16zWzVA_v+xzJLv`rSV`E?iFo~kX$xK
z&Kh#_I-+QsOP!dpkMO6)oYczk0OW9I?0x}OJ^)2^5T2u(CY~xkHceP+$u6yH3Fla+
zj7`2+>G3T4`y=)`62|!Q1pZIvt>(4DbIVT|f6za7|GgD(wgwF78bFKd!Mv`4KX^T{
zQ`g%?_NKqd<*V}(c8N^vwX>9KzGVqZhe?9395kE&o
z4kYrXcR|mg%KHuZlI}vRNnr+Uua3Pc%sD-c8pu{bBk4(%HpBr;rMM+)PIn30*;v7z
zf*;6Ge6XwbdR0MkpB+XNI=KdR_mQNZYKMh!mijU134$>(1$FiMhZEZuB%%Rv89KyT
zjrA~*ACnr5ZqTCZyF(7dyyo>a@wA~J+!nKY;~n^-av?wP0Y?l+2yGd0)8~>Ez8r4#
z18{p^qZUL-aA4Nr=3P&-=&P*RIYv2U(@Noo=pw4~08&VRodIC4ebE{n%|Yk)oX3ASe$R0zv(wvSZLJ9^+8kBbS`<|^IUnRX
zr?nwZj$khq`Pyz6L4m0^SYGwbYX&Li~E!S^uaOCeTv
zKl;cb(!B3>Q}~S=t7W+z#t0zS>q73kAI<309u#(P@gTAbzB8-e&=9QVaiYv;bySq!
zH&QYf$*E~p8rKR%R(6L}a^Qcb3IMvju9q)+VG|Cj9}T+oYhjU4tW2UFlL%>Ou1a^<
zPB!`Ccx$7su;i|=2Z8Iav93uj>_CUh@QhgEE^!EGc*k*;oa#Fn@bu;NihkM>)FNQu
zw`<=>%~C#}3<*YXTbn&++T_~hj^7{Ye=pnrE7B+~l1Vj!n!3Zc7le6c;NbsXan~Bt
z#f1A~Fg(?Cc@uvEP^{`%2d<+y*>0;0E&iR)vh)V}TTg2f8%@gD4_;5eFh
z<45JNba#V!JwTr=Aj|~gL@Xa7zi!M6EGXSCxa6cxu*aOZ7bq
zyX6!>V0Vk~VZ+2Z8Xc-v9VCi3K%F621XpJ#Mn+3SXoM=PENj}RvEJe-@f8OIex86{
zL@Z+Dp{g24fYH2^`G3s)7<-YVQS?b8k*uaHTyI37?~WgOIsRQy7=QTed%OGal;e1a
zNzXFM_c%UzJ@2R}YSH23hGsT2(;SA2&~7GgDgi7JSC+yX{?2UHRO?>UmK7*m_qoe2
z?(ygGuyM_*Vdmbd6PfzJo`D_%9lT%!_oQa+QcgCOEm2?%hJH^8^;975U`LWSfID1=
zebdy~-Gr@pE*o(68%h{JgTqy`UQ)_&dMSu3=%^`=pMLWVq%LC_Nu0h
zDn#s?;RUy^j|B^YX|)lf68GF?uUV*h3l%
zy9TPz2zE|_bWr|C`}0V`dqeju1PM3if|U9Rd4^3r-;eJ!23rIFEuti|{vD8b_TiCy
zPI1_^m7tu>4_L0}q>e>Lq9h}I0s=PMGu`%N<_`Oh9+Img|9~>Ystq8#E5wrZ@3ha1
z7;hkmGZR;}@zoP4_wouhR)E$`O{6{5?DbO}IQpHHHhM9FmP&*GRo|Q+5C#1{l4cTk
zIC1VH7y?|w5~hgq?Zsp}+=w1h_cx*5;0qtWj@8lKEbMN7nBAC5a+mx2RtOK{rcB-&
z_JIo|Zv+5Ev2O0^ZR#p(A0FFFH2s8Iw=NzwbNDC?%U1=|XHBtduHDSPjEnZW6CeXb8NBlb0zYRI)
zglu4fg*^-H!BqOTK7zwLc
zvwF4d7~I_HEkoHx)mK4CHs~2Mgnsle3s}HY2U=dCSPk8cOPlSH`Z*wwcYwy23Xpy?-e_B5I*fMM3IPLmVTW)U
z<%k>d9sJ!CRmlxpxeN$A**7L;$>9o6_c8RDBvmt#=^{h9E<=K-8tLb{y-L0m>1~jy
zDAMl{<#!pkk1FSwg@F4{Fthv`OYugkUiLelB`fHI@7U8>JgfS$DQQPuI_-GmZKd@rARrJKP_Z!J$R^@
zp4q^PjVPA?dJ9vVP3B1N9}e%>BTL8PrCcoeX<(ji%?!n=1S-
g$ASM1_I-JJ&a?N+tRLkb)6((9ZwsGSvN`g=-_;NC4FCWD
literal 0
HcmV?d00001