Fixed merge of vocab.json and new tokenizer for custom dataset

Fixed vocab.json merge, new tokenizer for custom dataset, dataset cleaner.

system/ft_tokenizer/compare_and_merge.py

@@ -1,48 +1,54 @@
 import json
 import os
 
+def merge_vocabularies(base_vocab_path, new_vocab_path, output_path):
+    # Load the base model's vocab.json
+    with open(base_vocab_path, 'r') as f:
+        base_data = json.load(f)
+
+    # Load the new bpe_tokenizer.json
+    with open(new_vocab_path, 'r') as f:
+        new_data = json.load(f)
+
+    # Extract the vocabularies
+    base_vocab = base_data['model']['vocab']
+    new_vocab = new_data['model']['vocab']
+
+    # Find the maximum value in the base vocabulary
+    max_value = max(base_vocab.values())
+
+    # Merge the vocabularies
+    for key, value in new_vocab.items():
+        if key not in base_vocab:
+            max_value += 1
+            base_vocab[key] = max_value
+
+    # Update the base data with the merged vocabulary
+    base_data['model']['vocab'] = base_vocab
+
+    # Extract the merges
+    base_merges = base_data['model']['merges']
+    new_merges = new_data['model']['merges']
+
+    # Merge the merges
+    merged_merges = base_merges.copy()
+    for merge in new_merges:
+        if merge not in merged_merges:
+            merged_merges.append(merge)
+
+    # Update the base data with the merged merges
+    base_data['model']['merges'] = merged_merges
+
+    # Write the merged vocabulary and merges to the output file
+    with open(output_path, 'w') as f:
+        json.dump(base_data, f, ensure_ascii=False, indent=2)
+
+    print(f"Merged vocabulary and merges saved to {output_path}")
+
 # Define file paths
-base_path = "/alltalk_tts/system/ft_tokenizer/"
-original_file = os.path.join(base_path, "/alltalk_tts/models/xtts/xttsv2_2.0.3/vocab.json") # base model vocab.json
-new_file = os.path.join(base_path, "/expanded_model/bpe_tokenizer-vocab.json")  # new bpe tokenizer vocab.json
-output_file = os.path.join(base_path, "/expanded_model/expanded_vocab.json")  # output expanded_vocab.json file/path
+base_vocab_path = "/alltalk_tts/models/xtts/xttsv2_2.0.2/originalvocab.json" # base model vocab.json path (2.0.2)
+new_vocab_path = "/alltalk_tts/expanded_models/tortoise_tokenizer2xttsv2.json" # path to the custom dataset vocab.json
+output_path = "/alltalk_tts/expanded_models/combined_vocab.json" # location for combined vocab.json
 
-
-with open(original_file, 'r') as f:
-    original_data = json.load(f)
-
-with open(new_file, 'r') as f:
-    new_data = json.load(f)
-
-# Get the original and new vocabularies
-original_vocab = original_data['model']['vocab']
-new_vocab = new_data['model']['vocab']
-
-# Find the maximum value in the original vocabulary. We do this so we make sure to append and not overwrite any new values
-max_value = max(original_vocab.values())
-
-# merge
-for key, value in new_vocab.items():
-    if key not in original_vocab:
-        max_value += 1
-        original_vocab[key] = max_value
-
-# Update
-original_data['model']['vocab'] = original_vocab
-
-# Get the original and new merges
-original_merges = original_data['model']['merges']
-new_merges = new_data['model']['merges']
-
-# Merge the merges
-merged_merges = original_merges.copy()
-for merge in new_merges:
-    if merge not in merged_merges:
-        merged_merges.append(merge)
-
-# Update
-original_data['model']['merges'] = merged_merges
-with open(output_file, 'w') as f:
-    json.dump(original_data, f, ensure_ascii=False, indent=2)
-
-print(f"Merged vocabulary and merges saved to {output_file}")
+# Merge the vocabularies
+merge_vocabularies(base_vocab_path, new_vocab_path, output_path)

system/ft_tokenizer/custom_tokenizer.py

@@ -0,0 +1,60 @@
+# Credit Jarod Mica https://github.com/JarodMica/tortoise_dataset_tools/blob/master/bpe_tokenizer_tools/train_bpe_tokenizer.py
+# provide a cleaned txt file with only the transcription. Youll get back out the datasets vocab.json
+
+from tokenizers import Tokenizer
+from tokenizers.models import BPE
+from tokenizers.trainers import BpeTrainer
+from tokenizers.pre_tokenizers import Whitespace
+from tkinter import Tk, filedialog
+import json
+import re
+
+def clean_text(input_file_path, output_file_path):
+    # Define the pattern to match numbers, specific symbols, and new lines
+    # add \d to match any digit, and | is used to specify alternatives
+    pattern = r'|�|«|\$|\n'
+
+    with open(input_file_path, 'r', encoding='utf-8') as input_file:
+        text = input_file.read()
+        cleaned_text = re.sub(pattern, '', text)
+
+    with open(output_file_path, 'w', encoding='utf-8') as output_file:
+        output_file.write(cleaned_text)
+
+def train_tokenizer(input_path, tokenizer_path, language, special_tokens=["[STOP]", "[UNK]", "[SPACE]" ], vocab_size=256):
+    # Initialize a tokenizer with the BPE model
+    tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
+    # Use a basic whitespace pre-tokenizer
+    tokenizer.pre_tokenizer = Whitespace()
+
+    # trainer = BpeTrainer(special_tokens=["[STOP]", "[UNK]", "[SPACE]", "0","1","2","3","4","5","6","7","8","9",], vocab_size=256)
+    trainer = BpeTrainer(special_tokens=special_tokens, vocab_size=vocab_size)
+    
+
+    clean_text(input_path, input_path)
+    tokenizer.train([input_path], trainer)
+
+    tokenizer.save(tokenizer_path)
+
+    with open(tokenizer_path, 'r', encoding='utf-8') as f:
+        tokenizer_json = json.load(f)
+
+    # Add language to tokenizer
+    tokenizer_json['model']['language'] = language
+
+    with open(tokenizer_path, 'w', encoding='utf-8') as f:
+        json.dump(tokenizer_json, f, ensure_ascii=False, indent=4)
+        
+def choose_file():
+    root = Tk()
+    root.withdraw()
+    file = filedialog.askopenfilename()
+    root.destroy()
+    return file
+    
+if __name__ == "__main__":
+    input_path = choose_file()
+    tokenizer_path = "/alltalk_tts/expanded_models/tortoise_tokenizer2xttsv2.json" # define path to put the newly create vocab.json
+    special_tokens = ["[STOP]", "[UNK]", "[SPACE]"]
+    vocab_size = 256 # model is stuck at this size
+    train_tokenizer(input_path, tokenizer_path, language='multi', special_tokens=special_tokens, vocab_size=vocab_size)

system/ft_tokenizer/extract_dataset_for_tokenizer.py

@@ -0,0 +1,26 @@
+# Simple script to remove LJ Speech formatting for the tokenizer
+# Combine metadata_train and metadata_eval.csv into a single file then run
+import csv
+
+# Input and output file names
+input_file = '/alltalkbeta/metadata_eval.csv'  # combine metadata_train and metadata_eval.csv
+output_file = '/alltalkbeta/dataset.txt' # this goes to the tokenizer
+
+# Read the input CSV and write to the output file
+with open(input_file, 'r', newline='', encoding='utf-8') as infile, \
+     open(output_file, 'w', newline='', encoding='utf-8') as outfile:
+    
+    # Create CSV reader and writer objects
+    reader = csv.reader(infile, delimiter='|')
+    writer = csv.writer(outfile, delimiter='|')
+    
+    # Skip the header
+    next(reader, None)
+    
+    # Process each row
+    for row in reader:
+        if len(row) >= 2:
+            # Write only the second column (index 1) to the output file
+            writer.writerow([row[1]])
+
+print(f"Processing complete. Output written to {output_file}")