Added debugging to Training section

finetune.py

@@ -960,13 +960,14 @@ def format_audio_list(
         fal_target_language,
         fal_whisper_model,
         fal_max_sample_length,
+        fal_min_sample_length,
         fal_eval_split_number,
         fal_speaker_name_input,
         fal_create_bpe_tokenizer,
         fal_gradio_progress=gr.Progress(),
         fal_use_vad=True,
         fal_precision="mixed",
-        fal_min_sample_length=6):
+        ):
     """
     Process and format audio files for XTTS training. Handles audio segmentation, transcription,
     and metadata creation with optional VAD and precision settings.
@@ -1526,32 +1527,32 @@ def merge_short_segments(segments, min_duration, max_gap=0.5):
 
     merged = []
     current_group = []
+    target_duration = (min_duration + 10.0) / 2  # Target middle of range
 
     for i, segment in enumerate(segments):
         current_duration = sum(s["end"] - s["start"] 
-                               for s in current_group) if current_group else 0
+                             for s in current_group) if current_group else 0
 
         # If this is a continuation of current group
-        if current_group and (
+        if current_group and (segment["start"] - current_group[-1]["end"]) <= max_gap:
-                segment["start"] - current_group[-1]["end"]) <= max_gap:
+            # Check if adding this segment gets us closer to target duration
-            current_group.append(segment)
+            new_duration = current_duration + (segment["end"] - segment["start"])
-        # If starting new group but previous was too short
+            if abs(new_duration - target_duration) < abs(current_duration - target_duration):
-        elif current_group and current_duration < min_duration:
+                current_group.append(segment)
-            # Look ahead for close segments
+            else:
-            # Look up to 3 segments ahead
+                # Save current group and start new one
-            look_ahead = min(3, len(segments) - i)
+                merged_segment = {
-            for j in range(i, i + look_ahead):
+                    "start": current_group[0]["start"],
-                if (j < len(segments) and (
+                    "end": current_group[-1]["end"]
-                        segments[j]["start"] - current_group[-1]["end"]) <= max_gap * 2):
+                }
-                    current_group.append(segments[j])
+                merged.append(merged_segment)
-                else:
+                current_group = [segment]
-                    break
         else:
             # Save previous group if it exists
             if current_group:
                 merged_segment = {
                     "start": current_group[0]["start"],
-                    "end": current_group[-1]["end"],
+                    "end": current_group[-1]["end"]
                 }
                 merged.append(merged_segment)
             current_group = [segment]
@@ -1559,12 +1560,14 @@ def merge_short_segments(segments, min_duration, max_gap=0.5):
     # Handle last group
     if current_group:
         merged_segment = {
-            "start": current_group[0]["start"], "end": current_group[-1]["end"]}
+            "start": current_group[0]["start"],
+            "end": current_group[-1]["end"]
+        }
         merged.append(merged_segment)
 
     debug_print(
-        f"Merged {len(segments) - len(merged)} segments into {len(merged)} longer segments",
+        f"Merged {len(segments) - len(merged)} segments into {len(merged)} segments with mid-range preference",
-        "SEGMENTS",
+        "SEGMENTS"
     )
     return merged
 
@@ -2285,6 +2288,22 @@ def basemodel_or_finetunedmodel_choice(value):
     elif value == "Existing finetuned model":
         basemodel_or_finetunedmodel = False
 
+def check_model_requirements(model_folder):
+    """Check if all required files exist in the model folder"""
+    required_files = {
+        "model.pth": False,
+        "config.json": False,
+        "vocab.json": False,
+        "dvae.pth": False,
+        "mel_stats.pth": False,
+        "speakers_xtts.pth": False
+    }
+    
+    if model_folder.exists():
+        for file in required_files:
+            required_files[file] = (model_folder / file).exists()
+    
+    return required_files
 
 def train_gpt(
         language,
@@ -2303,60 +2322,75 @@ def train_gpt(
         warm_up,
         max_audio_length=255995,
         progress=gr.Progress()):
+    
+    # First check if a model was selected
     if "No Models Available" in model_to_train:
-        print("[FINETUNE] Error: No XTTS model selected for training.")
+        debug_print("No XTTS model selected for training.", "MODEL_OPS", is_error=True)
-        print("[FINETUNE] Please download a model using AllTalk's main interface > TTS Engine Settings > XTTS > Model/Voices Download")
+        debug_print("Please download a model using AllTalk's main interface > TTS Engine Settings > XTTS > Model/Voices Download", "MODEL_OPS", is_info=True)
         return
     
-    #  Logging parameters
+    # Check if selected model exists and has required files
-    project_run_name = "XTTS_FT"
-    project_name = "XTTS_trainer"
-    dashboard_logger = "tensorboard"
-    logger_uri = None
     model_path = this_dir / "models" / "xtts" / model_to_train
+    if not model_path.exists():
+        debug_print(f"Selected model folder not found: {model_path}", "MODEL_OPS", is_error=True)
+        debug_print("Please ensure you have downloaded the model correctly", "MODEL_OPS", is_info=True)
+        return
 
-    # Check for lang.txt in the same directory as train_csv
+    # Check for required files
-    dataset_dir = os.path.dirname(train_csv)
+    files = check_model_requirements(model_path)
-    lang_file = os.path.join(dataset_dir, "lang.txt")
+    missing_files = [file for file, exists in files.items() if not exists]
     
-    if os.path.exists(lang_file):
+    if missing_files:
-        try:
+        debug_print(f"Missing required files in {model_to_train}:", "MODEL_OPS", is_error=True)
-            with open(lang_file, 'r', encoding='utf-8') as f:
+        for file in missing_files:
-                dataset_language = f.read().strip()
+            debug_print(f"❌ {file}", "MODEL_OPS", is_error=True)
-            debug_print(f"Found language file, using language: {dataset_language}", "GENERAL", is_info=True)
+        debug_print("\nPlease redownload the model using AllTalk's interface", "MODEL_OPS", is_info=True)
-            # Override the input language with the one from lang.txt
+        return
-            language = dataset_language
+    
-        except Exception as e:
+    # Confirm all model files found and continue with training
-            debug_print(f"Error reading lang.txt: {str(e)}", "GENERAL", is_warning=True)
+    debug_print(f"✓ All required files found for model: {model_to_train}", "MODEL_OPS", is_info=True)
-            debug_print(f"Falling back to provided language: {language}", "GENERAL", is_warning=True)
+
+    # Dataset validation
+    if not train_csv or not eval_csv:
+        debug_print("Missing training or evaluation CSV files", "MODEL_OPS", is_error=True)
+        debug_print(f"Train CSV: {train_csv}", "MODEL_OPS", is_error=True)
+        debug_print(f"Eval CSV: {eval_csv}", "MODEL_OPS", is_error=True)
+        return
+
+    # Training parameter validation
+    debug_print("***********************", "MODEL_OPS", is_info=True)
+    debug_print("Training Configuration:", "MODEL_OPS", is_info=True)
+    debug_print("***********************", "MODEL_OPS", is_info=True)
+    debug_print(f"- Language: {language}", "MODEL_OPS", is_info=True)
+    debug_print(f"- Epochs: {num_epochs}", "MODEL_OPS", is_info=True)
+    debug_print(f"- Batch Size: {batch_size}", "MODEL_OPS", is_info=True)
+    debug_print(f"- Gradient Accumulation: {grad_acumm}", "MODEL_OPS", is_info=True)
+    debug_print(f"- Learning Rate: {learning_rate}", "MODEL_OPS", is_info=True)
+    debug_print(f"- Learning Rate Scheduler: {learning_rate_scheduler}", "MODEL_OPS", is_info=True)
+    debug_print(f"- Optimizer: {optimizer}", "MODEL_OPS", is_info=True)
+    debug_print(f"- Number of Workers: {num_workers}", "MODEL_OPS", is_info=True)
+    debug_print(f"- Warm Up: {warm_up}", "MODEL_OPS", is_info=True)
+    debug_print(f"- Max Audio Length: {max_audio_length}", "MODEL_OPS", is_info=True)
+
+    # GPU/Memory information
+    if torch.cuda.is_available():
+        gpu_id = torch.cuda.current_device()
+        debug_print("****************", "GPU_MEMORY", is_info=True)
+        debug_print("GPU Information:", "GPU_MEMORY", is_info=True)
+        debug_print("****************", "GPU_MEMORY", is_info=True)
+        debug_print(f"- Device: {torch.cuda.get_device_name(gpu_id)}", "GPU_MEMORY", is_info=True)
+        debug_print(f"- CUDA Version: {torch.version.cuda}", "GPU_MEMORY", is_info=True)
+        total_memory = torch.cuda.get_device_properties(gpu_id).total_memory / (1024**3)
+        free_memory = total_memory - (torch.cuda.memory_allocated(gpu_id) / (1024**3))
+        debug_print(f"- Total VRAM: {total_memory:.2f}GB", "GPU_MEMORY", is_info=True)
+        debug_print(f"- Free VRAM: {free_memory:.2f}GB", "GPU_MEMORY", is_info=True)
+        
+        # Memory warnings
+        if free_memory < 3:
+            debug_print("WARNING: Very low available VRAM!", "GPU_MEMORY", is_warning=True)
+            debug_print("Consider reducing batch size or using gradient accumulation", "GPU_MEMORY", is_warning=True)
     else:
-        debug_print("No lang.txt found, using provided language setting", "GENERAL", is_warning=True)
+        debug_print("No GPU detected - training will be very slow!", "GPU_MEMORY", is_warning=True)
-
-    # Set here the path that the checkpoints will be saved. Default:
-    # ./training/
-    project_path = os.path.join(out_path, "training")
-    debug_print(
-        "Starting Step 2 - Fine-tuning the XTTS Model",
-        level="GENERAL",
-        is_info=True)
-    debug_print("Configuration Summary:", level="GENERAL", is_info=True)
-    debug_print(f"- Language: {language}", level="GENERAL", is_info=True)
-    debug_print(
-        f"- Training Epochs: {num_epochs}",
-        level="GENERAL",
-        is_info=True)
-    debug_print(f"- Batch Size: {batch_size}", level="GENERAL", is_info=True)
-    debug_print(
-        f"- Gradient Accumulation Steps: {grad_acumm}",
-        level="GENERAL",
-        is_info=True)
-    debug_print("File Paths:", level="GENERAL", is_info=True)
-    debug_print(f"- Training Data: {train_csv}", level="GENERAL", is_info=True)
-    debug_print(
-        f"- Evaluation Data: {eval_csv}",
-        level="GENERAL",
-        is_info=True)
-    debug_print(f"- Base Model: {model_path}", level="GENERAL", is_info=True)
 
     if torch.cuda.is_available():
         torch.cuda.empty_cache()
@@ -2364,14 +2398,17 @@ def train_gpt(
         gpu_device_id = torch.cuda.current_device()
         gpu_available_mem_gb = (torch.cuda.get_device_properties(
             gpu_device_id).total_memory - torch.cuda.memory_allocated(gpu_device_id)) / (1024 ** 3)
-        debug_print("GPU Memory Status:", level="GPU_MEMORY", is_info=True)
-        debug_print(
-            f"- Available VRAM: {gpu_available_mem_gb:.2f} GB",
-            level="GPU_MEMORY",
-            is_info=True)
         if gpu_available_mem_gb < 12:
             debug_print(
-                "*** IMPORTANT MEMORY CONSIDERATION ***",
+                "******************************",
+                level="GPU_MEMORY",
+                is_warning=True)            
+            debug_print(
+                "IMPORTANT MEMORY CONSIDERATION",
+                level="GPU_MEMORY",
+                is_warning=True)
+            debug_print(
+                "******************************",
                 level="GPU_MEMORY",
                 is_warning=True)            
             debug_print(
@@ -2423,6 +2460,79 @@ def train_gpt(
                 level="GPU_MEMORY",
                 is_warning=True)
 
+    # Dataset statistics
+    try:
+        train_df = pd.read_csv(train_csv, sep="|")
+        eval_df = pd.read_csv(eval_csv, sep="|")
+        debug_print("*******************", "DATA_PROCESS", is_info=True)
+        debug_print("Dataset Statistics:", "DATA_PROCESS", is_info=True)
+        debug_print("*******************", "DATA_PROCESS", is_info=True)
+        debug_print(f"- Training samples: {len(train_df)}", "DATA_PROCESS", is_info=True)
+        debug_print(f"- Evaluation samples: {len(eval_df)}", "DATA_PROCESS", is_info=True)
+        if (out_path / "bpe_tokenizer-vocab.json").exists():
+            debug_print(
+                "- Using custom BPE tokenizer",
+                level="DATA_PROCESS",
+                is_info=True)
+            training_assets = {
+                'Tokenizer': str(out_path / "bpe_tokenizer-vocab.json")
+            }        
+        # Check for potential issues
+        if len(train_df) < 100:
+            debug_print("Very small training dataset", "DATA_PROCESS", is_warning=True)
+        if len(eval_df) < 10:
+            debug_print("Very small evaluation dataset", "DATA_PROCESS", is_warning=True)
+    except Exception as e:
+        debug_print(f"Error reading dataset files: {str(e)}", "DATA_PROCESS", is_error=True)
+        return
+
+    #  Logging parameters
+    project_run_name = "XTTS_FT"
+    project_name = "XTTS_trainer"
+    dashboard_logger = "tensorboard"
+    logger_uri = None
+
+    # Check for lang.txt in the same directory as train_csv
+    dataset_dir = os.path.dirname(train_csv)
+    lang_file = os.path.join(dataset_dir, "lang.txt")
+    
+    # Set here the path that the checkpoints will be saved. Default:
+    # ./training/
+    project_path = os.path.join(out_path, "training")
+    # Project path information
+    debug_print("**********************", "GENERAL", is_info=True)
+    debug_print("Project Configuration:", "GENERAL", is_info=True)
+    debug_print("**********************", "GENERAL", is_info=True)
+    debug_print(f"- Project Path: {project_path}", "GENERAL", is_info=True)
+    debug_print(f"- Model Path: {model_path}", "GENERAL", is_info=True)
+    debug_print(f"- Training Data: {train_csv}", level="GENERAL", is_info=True)
+    debug_print(
+        f"- Evaluation Data: {eval_csv}",
+        level="GENERAL",
+        is_info=True)  
+    debug_print(f"- Language: {language}", level="GENERAL", is_info=True)
+    if os.path.exists(lang_file):
+        try:
+            with open(lang_file, 'r', encoding='utf-8') as f:
+                dataset_language = f.read().strip()
+            debug_print(f"- Found language file, using language: {dataset_language}", "GENERAL", is_info=True)
+            # Override the input language with the one from lang.txt
+            language = dataset_language
+        except Exception as e:
+            debug_print(f"- Error reading lang.txt: {str(e)}", "GENERAL", is_warning=True)
+            debug_print(f"- Falling back to provided language: {language}", "GENERAL", is_warning=True)
+    else:
+        debug_print("- No lang.txt found, using provided language setting", "GENERAL", is_warning=True)    
+    debug_print(f"- Batch Size: {batch_size}", level="GENERAL", is_info=True)
+    debug_print(
+        f"- Grad Steps: {grad_acumm}",
+        level="GENERAL",
+        is_info=True)        
+    debug_print(
+        f"- Training Epochs: {num_epochs}",
+        level="GENERAL",
+        is_info=True)
+
     # Create the directory
     os.makedirs(project_path, exist_ok=True)
 
@@ -2455,14 +2565,6 @@ def train_gpt(
     dataset_speakers_file = model_path / "speakers_xtts.pth"
 
     training_assets = None
-    if (out_path / "bpe_tokenizer-vocab.json").exists():
-        debug_print(
-            "Using custom BPE tokenizer",
-            level="DATA_PROCESS",
-            is_info=True)
-        training_assets = {
-            'Tokenizer': str(out_path / "bpe_tokenizer-vocab.json")
-        }
 
     continue_path = None
     if continue_run:
@@ -2480,7 +2582,7 @@ def train_gpt(
                         dataset_xtts_checkpoint = None
                         continue_path = last_run
                         print(
-                            f"[FINETUNE] Continuing previous fine tuning {latest_checkpoint}")
+                            f"[FINETUNE] - Continuing previous fine tuning {latest_checkpoint}")
 
     # Copy the supporting files
     destination_dir = out_path / "chkptandnorm"
@@ -2653,8 +2755,14 @@ def train_gpt(
 
     optimizer_params = OPTIMIZER_PARAMS.get(optimizer, {})
 
-    print(
+    debug_print(
-        f"[FINETUNE] [INFO] Learning Scheduler {lr_scheduler}, params {lr_scheduler_params}")
+        f"- Learning Scheduler {lr_scheduler} Parameters",
+        level="GENERAL",
+        is_info=True)
+    debug_print(
+        f"- {lr_scheduler_params}",
+        level="GENERAL",
+        is_info=True)
     
     # training parameters config
     config = GPTTrainerConfig(
@@ -2695,12 +2803,14 @@ def train_gpt(
     # init the model from config
     model = GPTTrainer.init_from_config(config)
     # load training samples
+    debug_print("Loading training samples...", "MODEL_OPS", is_info=True)
     train_samples, eval_samples = load_tts_samples(
         dataset_config_list,
         eval_split=True,
         eval_split_max_size=config.eval_split_max_size,
         eval_split_size=config.eval_split_size,
     )
+    debug_print(f"Loaded {len(train_samples)} training and {len(eval_samples)} eval samples\n", "MODEL_OPS", is_info=True)
 
     global c_logger
     c_logger = MetricsLogger()
@@ -2728,10 +2838,18 @@ def train_gpt(
 
     if disable_shared_memory:
         # Limit training to GPU memory instead of shared memory
-        print("[FINETUNE] [INFO] Limiting GPU memory to 95% to prevent spillover")
+        debug_print("Limiting GPU memory to 95%", "GPU_MEMORY", is_info=True)
         torch.cuda.set_per_process_memory_fraction(0.95)
     
+    print("\n")
+    debug_print("********************************", "MODEL_OPS", is_info=True)
+    debug_print("Starting training the XTTS model", "MODEL_OPS", is_info=True)
+    debug_print("********************************\n", "MODEL_OPS", is_info=True)
     trainer.fit()
+    print("\n")
+    debug_print("********************************", "MODEL_OPS", is_info=True)
+    debug_print("Training completed successfully", "MODEL_OPS", is_info=True)
+    debug_print("********************************", "MODEL_OPS", is_info=True)
 
     # get the longest text audio file to use as speaker reference
     samples_len = [len(item["text"].split(" ")) for item in train_samples]
@@ -2740,6 +2858,10 @@ def train_gpt(
 
     trainer_out_path = trainer.output_path
 
+    # Before cleanup
+    debug_print("GPU Memory Before Cleanup:", "GPU_MEMORY")
+    get_gpu_memory()
+
     # deallocate VRAM and RAM
     del model, trainer, train_samples, eval_samples, config, model_args, config_dataset
     gc.collect()
@@ -2752,8 +2874,9 @@ def train_gpt(
     try:
         return dataset_xtts_config_file, dataset_xtts_checkpoint, dataset_tokenizer_file, trainer_out_path, speaker_ref
     except Exception as e:
-        print(f"Error returning values: {e}")
+        debug_print(f"Error during training: {str(e)}", "MODEL_OPS", is_error=True)
-        return "Error", "Error", "Error", "Error", "Error"
+        debug_print(traceback.format_exc(), "MODEL_OPS", is_error=True)
+        return
 
 ##########################
 #### STEP 3 AND OTHER ####
@@ -2764,6 +2887,8 @@ def clear_gpu_cache():
     """clear the GPU cache"""
     if torch.cuda.is_available():
         torch.cuda.empty_cache()
+        debug_print("GPU Memory After Cleanup:", "GPU_MEMORY")
+        get_gpu_memory()
 
 
 def find_a_speaker_file(folder_path):
@@ -3474,8 +3599,11 @@ if __name__ == "__main__":
                             ("large-v3-turbo", "large-v3-turbo"),
                             ("turbo", "turbo"),
                         ],
-                        scale=2,
+                        scale=1,
                     )
+                    precision = gr.Dropdown(
+                        label="Model Precision", value="mixed", choices=[
+                            ("Mixed", "mixed"), ("FP16", "float16"), ("FP32", "float32")], scale=1)                    
                     lang = gr.Dropdown(
                         label="Dataset Language",
                         value="en",
@@ -3499,48 +3627,43 @@ if __name__ == "__main__":
                         ],
                         scale=1,
                     )
-                    max_sample_length = gr.Dropdown(
-                        label="Max Audio Length (in seconds)",
-                        value="30",
-                        choices=[
-                            "10",
-                            "15",
-                            "20",
-                            "25",
-                            "30",
-                            "35",
-                            "40",
-                            "45",
-                            "50",
-                            "55",
-                            "60",
-                            "65",
-                            "70",
-                            "75",
-                            "80",
-                            "85",
-                            "90",
-                        ],
-                        scale=2,
-                    )
                     eval_split_number = gr.Number(
                         label="Evaluation Data Split",
-                        value=15,  # Default value
+                        value=15,
-                        minimum=5,  # Minimum value
+                        minimum=5,
-                        maximum=95,  # Maximum value
+                        maximum=95,
-                        step=1,  # Increment step
+                        step=1,
                         scale=1,
                     )
-                    create_bpe_tokenizer = gr.Checkbox(
+                with gr.Row():                    
-                        label="BPE Tokenizer", value=False, info="Custom Tokenizer for training")
+                    create_bpe_tokenizer = gr.Dropdown(
-                    use_vad = gr.Checkbox(
+                        label="BPE Tokenizer",
+                        value="False",
+                        choices=[("Enabled", "True"), ("Disabled", "False")],
+                        info="Custom Tokenizer for training",
+                        scale=1,
+                    )
+                    use_vad = gr.Dropdown(
                         label="VAD",
-                        value=True,
+                        value="True",
+                        choices=[("Enabled", "True"), ("Disabled", "False")],
                         info="Enable Silero VAD for better speech detection",
+                        scale=1,
+                    )
+                    min_sample_length = gr.Dropdown(
+                        label="Min Audio Length (seconds)", 
+                        value="2",
+                        choices=["1", "2", "3", "4", "5"],
+                        info="Split large audio into a minimum of",
+                        scale=1,
+                    )                    
+                    max_sample_length = gr.Dropdown(
+                        label="Max Audio Length (seconds)",
+                        value="10", 
+                        choices=["8","9","10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20"],
+                        info="Split large audio into a maximum of",
+                        scale=1,
                     )                    
-                    precision = gr.Dropdown(
-                        label="Model Precision", value="mixed", choices=[
-                            ("Mixed", "mixed"), ("FP16", "float16"), ("FP32", "float32")], )
 
                 with gr.Accordion("🔍 Dataset Creation Debug Settings", open=False):
 
@@ -3712,6 +3835,7 @@ if __name__ == "__main__":
                     pd_language,
                     pd_whisper_model,
                     pd_max_sample_length,
+                    pd_min_sample_length,
                     pd_eval_split_number,
                     pd_speaker_name_input,
                     pd_create_bpe_tokenizer,
@@ -3741,6 +3865,7 @@ if __name__ == "__main__":
                             fal_target_language=pd_language,
                             fal_whisper_model=pd_whisper_model,
                             fal_max_sample_length=pd_max_sample_length,
+                            fal_min_sample_length=pd_min_sample_length,
                             fal_eval_split_number=pd_eval_split_number,
                             fal_speaker_name_input=pd_speaker_name_input,
                             fal_create_bpe_tokenizer=pd_create_bpe_tokenizer,
@@ -4125,6 +4250,85 @@ if __name__ == "__main__":
                         value=args.max_audio_length,
                     )
 
+
+                with gr.Accordion("🔍 Training Debug Settings", open=False):
+                    gr.Markdown("""
+                        Enable or disable different types of debug messages during model training.
+                        These settings will apply to the current training session only.
+                    """)
+
+                    with gr.Row():
+                        with gr.Column(scale=1):
+                            debug_gpu = gr.Checkbox(
+                                label="GPU Memory",
+                                value=DebugLevels.GPU_MEMORY,
+                                info="GPU memory and CUDA related debugging",
+                            )
+                            debug_model = gr.Checkbox(
+                                label="Model Operations",
+                                value=DebugLevels.MODEL_OPS,
+                                info="Model loading, training operations, cleanup",
+                            )
+                        with gr.Column(scale=1):                            
+                            debug_general = gr.Checkbox(
+                                label="General",
+                                value=DebugLevels.GENERAL,
+                                info="Training flow, file operations",
+                            )
+                            debug_data = gr.Checkbox(
+                                label="Data Processing",
+                                value=DebugLevels.DATA_PROCESS,
+                                info="Data processing, files, folders",
+                            )                         
+
+                    with gr.Row():
+                        debug_select_all = gr.Button("Select All")
+                        debug_clear_all = gr.Button("Clear All")
+
+                    # Debug update functions
+                    def train_update_debug_levels(gpu, model, general, data):
+                        DebugLevels.GPU_MEMORY = gpu
+                        DebugLevels.MODEL_OPS = model
+                        DebugLevels.GENERAL = general
+                        DebugLevels.DATA_PROCESS = data
+                        return "Debug settings updated"
+
+                    def train_select_all_debug():
+                        return {
+                            debug_gpu: True,
+                            debug_model: True,
+                            debug_general: True,
+                            debug_data: True,
+                        }
+
+                    def train_clear_all_debug():
+                        return {
+                            debug_gpu: False,
+                            debug_model: False,
+                            debug_general: False,
+                            debug_data: False,
+                        }
+
+                    # Connect the debugging controls
+                    for checkbox in [debug_gpu, debug_model, debug_general, debug_data]:
+                        checkbox.change(
+                            fn=train_update_debug_levels,
+                            inputs=[debug_gpu, debug_model, debug_general, debug_data],
+                            outputs=[gr.Textbox(visible=False)],
+                        )
+
+                    debug_select_all.click(
+                        fn=train_select_all_debug,
+                        inputs=[],
+                        outputs=[debug_gpu, debug_model, debug_general, debug_data],
+                    )
+
+                    debug_clear_all.click(
+                        fn=train_clear_all_debug,
+                        inputs=[],
+                        outputs=[debug_gpu, debug_model, debug_general, debug_data],
+                    )
+                
                 progress_train = gr.Label(label="Progress:")
 
                 with gr.Row():
@@ -4512,6 +4716,7 @@ if __name__ == "__main__":
                         lang,
                         whisper_model,
                         max_sample_length,
+                        min_sample_length,
                         eval_split_number,
                         speaker_name_input,
                         create_bpe_tokenizer,