From e081442bf39394fee2988012e8b875651544a902 Mon Sep 17 00:00:00 2001
From: bmalaski <5288648+bmalaski@users.noreply.github.com>
Date: Tue, 25 Jun 2024 15:49:06 -0400
Subject: [PATCH] Update finetune.py

---
 finetune.py           | 14 +++++++-------
 start_diagnostics.bat |  6 ------
 2 files changed, 7 insertions(+), 13 deletions(-)
 delete mode 100644 start_diagnostics.bat

diff --git a/finetune.py b/finetune.py
index 095afe4..1809a98 100644
--- a/finetune.py
+++ b/finetune.py
@@ -843,12 +843,12 @@ def train_gpt(language, num_epochs, batch_size, grad_acumm, train_csv, eval_csv,
         if lr_scheduler == "StepLR":
             lr_scheduler_params = {'step_size': 30, 'gamma': 0.1, 'last_epoch': -1}
         elif lr_scheduler == "MultiStepLR":
-            exponent = 3 - int(math.log2(num_epochs) / 2)
-            base = 2
-            num_milestones = min(num_epochs, int(math.pow(base, exponent)))
-            milestone_interval = num_epochs // (num_milestones + 1)
-            milestones = [milestone_interval * (i + 1) for i in range(num_milestones)]
-            lr_scheduler_params = {'milestones': milestones, 'gamma': lr_gamma_mapping[learning_rate], 'last_epoch': -1}
+            # exponent = 3 - int(math.log2(num_epochs) / 2)
+            # base = 2
+            # num_milestones = min(num_epochs, int(math.pow(base, exponent)))
+            # milestone_interval = num_epochs // (num_milestones + 1)
+            # milestones = [milestone_interval * (i + 1) for i in range(num_milestones)]
+            lr_scheduler_params = {"milestones": [50000 * 18, 150000 * 18, 300000 * 18], "gamma": 0.5, "last_epoch": -1}
         elif lr_scheduler == "ExponentialLR":
             lr_scheduler_params = {'gamma': 0.5, 'last_epoch': -1}
         elif lr_scheduler == "CosineAnnealingLR":
@@ -1850,7 +1850,7 @@ if __name__ == "__main__":
                     grad_acumm = gr.Slider(
                         label="Grad accumulation steps:",
                         minimum=1,
-                        maximum=32,
+                        maximum=1024,
                         step=1,
                         value=args.grad_acumm,
                     )
diff --git a/start_diagnostics.bat b/start_diagnostics.bat
deleted file mode 100644
index f36efbc..0000000
--- a/start_diagnostics.bat
+++ /dev/null
@@ -1,6 +0,0 @@
-@echo off 
-cd /D "C:\AI\text-generation-webui\extensions\alltalk_tts\" 
-set CONDA_ROOT_PREFIX=C:\AI\text-generation-webui\extensions\alltalk_tts\alltalk_environment\conda 
-set INSTALL_ENV_DIR=C:\AI\text-generation-webui\extensions\alltalk_tts\alltalk_environment\env 
-call "C:\AI\text-generation-webui\extensions\alltalk_tts\alltalk_environment\conda\condabin\conda.bat" activate "C:\AI\text-generation-webui\extensions\alltalk_tts\alltalk_environment\env" 
-call python diagnostics.py