Fixed merge of vocab.json and new tokenizer for custom dataset

Fixed vocab.json merge, new tokenizer for custom dataset, dataset cleaner.

system/ft_tokenizer/compare_and_merge.py

@@ -1,48 +1,54 @@
 import json
 import os
 
+def merge_vocabularies(base_vocab_path, new_vocab_path, output_path):
+    # Load the base model's vocab.json
+    with open(base_vocab_path, 'r') as f:
+        base_data = json.load(f)
+
+    # Load the new bpe_tokenizer.json
+    with open(new_vocab_path, 'r') as f:
+        new_data = json.load(f)
+
+    # Extract the vocabularies
+    base_vocab = base_data['model']['vocab']
+    new_vocab = new_data['model']['vocab']
+
+    # Find the maximum value in the base vocabulary
+    max_value = max(base_vocab.values())
+
+    # Merge the vocabularies
+    for key, value in new_vocab.items():
+        if key not in base_vocab:
+            max_value += 1
+            base_vocab[key] = max_value
+
+    # Update the base data with the merged vocabulary
+    base_data['model']['vocab'] = base_vocab
+
+    # Extract the merges
+    base_merges = base_data['model']['merges']
+    new_merges = new_data['model']['merges']
+
+    # Merge the merges
+    merged_merges = base_merges.copy()
+    for merge in new_merges:
+        if merge not in merged_merges:
+            merged_merges.append(merge)
+
+    # Update the base data with the merged merges
+    base_data['model']['merges'] = merged_merges
+
+    # Write the merged vocabulary and merges to the output file
+    with open(output_path, 'w') as f:
+        json.dump(base_data, f, ensure_ascii=False, indent=2)
+
+    print(f"Merged vocabulary and merges saved to {output_path}")
+
 # Define file paths
-base_path = "/alltalk_tts/system/ft_tokenizer/"
-original_file = os.path.join(base_path, "/alltalk_tts/models/xtts/xttsv2_2.0.3/vocab.json") # base model vocab.json
-new_file = os.path.join(base_path, "/expanded_model/bpe_tokenizer-vocab.json")  # new bpe tokenizer vocab.json
-output_file = os.path.join(base_path, "/expanded_model/expanded_vocab.json")  # output expanded_vocab.json file/path
+base_vocab_path = "/alltalk_tts/models/xtts/xttsv2_2.0.2/originalvocab.json" # base model vocab.json path (2.0.2)
+new_vocab_path = "/alltalk_tts/expanded_models/tortoise_tokenizer2xttsv2.json" # path to the custom dataset vocab.json
+output_path = "/alltalk_tts/expanded_models/combined_vocab.json" # location for combined vocab.json
 
-
-with open(original_file, 'r') as f:
-    original_data = json.load(f)
-
-with open(new_file, 'r') as f:
-    new_data = json.load(f)
-
-# Get the original and new vocabularies
-original_vocab = original_data['model']['vocab']
-new_vocab = new_data['model']['vocab']
-
-# Find the maximum value in the original vocabulary. We do this so we make sure to append and not overwrite any new values
-max_value = max(original_vocab.values())
-
-# merge
-for key, value in new_vocab.items():
-    if key not in original_vocab:
-        max_value += 1
-        original_vocab[key] = max_value
-
-# Update
-original_data['model']['vocab'] = original_vocab
-
-# Get the original and new merges
-original_merges = original_data['model']['merges']
-new_merges = new_data['model']['merges']
-
-# Merge the merges
-merged_merges = original_merges.copy()
-for merge in new_merges:
-    if merge not in merged_merges:
-        merged_merges.append(merge)
-
-# Update
-original_data['model']['merges'] = merged_merges
-with open(output_file, 'w') as f:
-    json.dump(original_data, f, ensure_ascii=False, indent=2)
-
-print(f"Merged vocabulary and merges saved to {output_file}")
+# Merge the vocabularies
+merge_vocabularies(base_vocab_path, new_vocab_path, output_path)