From 44bd45a6241ba07199088a63b823faf239125e12 Mon Sep 17 00:00:00 2001 From: IIEleven11 Date: Fri, 2 Aug 2024 21:39:13 -0700 Subject: [PATCH] Fixed merge of vocab.json and new tokenizer for custom dataset Fixed vocab.json merge, new tokenizer for custom dataset, dataset cleaner. --- finetune.py | 14 +-- system/ft_tokenizer/compare_and_merge.py | 92 ++++++++++--------- system/ft_tokenizer/custom_tokenizer.py | 60 ++++++++++++ .../extract_dataset_for_tokenizer.py | 26 ++++++ 4 files changed, 136 insertions(+), 56 deletions(-) create mode 100644 system/ft_tokenizer/custom_tokenizer.py create mode 100644 system/ft_tokenizer/extract_dataset_for_tokenizer.py diff --git a/finetune.py b/finetune.py index b622baf..96f1b8b 100644 --- a/finetune.py +++ b/finetune.py @@ -918,7 +918,6 @@ def train_gpt(language, num_epochs, batch_size, grad_acumm, train_csv, eval_csv, print(f"[FINETUNE] Learning Scheduler {lr_scheduler}, params {lr_scheduler_params}") - # training parameters config config = GPTTrainerConfig( epochs=num_epochs, @@ -951,8 +950,6 @@ def train_gpt(language, num_epochs, batch_size, grad_acumm, train_csv, eval_csv, lr_scheduler=lr_scheduler, # it was adjusted accordly for the new step scheme lr_scheduler_params=lr_scheduler_params, - test_sentences=[], - ) progress(0, desc="Model is currently training. See console for more information") # init the model from config model = GPTTrainer.init_from_config(config) @@ -2374,13 +2371,4 @@ if __name__ == "__main__": fn=delete_voice_sample_contents, outputs=[final_progress_data], ) - model_to_train.change(basemodel_or_finetunedmodel_choice, model_to_train, None) - - demo.queue().launch( - show_api=False, - inbrowser=True, - share=False, - debug=False, - server_port=7052, - server_name="127.0.0.1", - ) + model_to_train.change(basemodel_or_finetunedmodel_choice, model_to_train, None) \ No newline at end of file diff --git a/system/ft_tokenizer/compare_and_merge.py b/system/ft_tokenizer/compare_and_merge.py index dd1c453..58d5b3a 100644 --- a/system/ft_tokenizer/compare_and_merge.py +++ b/system/ft_tokenizer/compare_and_merge.py @@ -1,48 +1,54 @@ import json import os +def merge_vocabularies(base_vocab_path, new_vocab_path, output_path): + # Load the base model's vocab.json + with open(base_vocab_path, 'r') as f: + base_data = json.load(f) + + # Load the new bpe_tokenizer.json + with open(new_vocab_path, 'r') as f: + new_data = json.load(f) + + # Extract the vocabularies + base_vocab = base_data['model']['vocab'] + new_vocab = new_data['model']['vocab'] + + # Find the maximum value in the base vocabulary + max_value = max(base_vocab.values()) + + # Merge the vocabularies + for key, value in new_vocab.items(): + if key not in base_vocab: + max_value += 1 + base_vocab[key] = max_value + + # Update the base data with the merged vocabulary + base_data['model']['vocab'] = base_vocab + + # Extract the merges + base_merges = base_data['model']['merges'] + new_merges = new_data['model']['merges'] + + # Merge the merges + merged_merges = base_merges.copy() + for merge in new_merges: + if merge not in merged_merges: + merged_merges.append(merge) + + # Update the base data with the merged merges + base_data['model']['merges'] = merged_merges + + # Write the merged vocabulary and merges to the output file + with open(output_path, 'w') as f: + json.dump(base_data, f, ensure_ascii=False, indent=2) + + print(f"Merged vocabulary and merges saved to {output_path}") + # Define file paths -base_path = "/alltalk_tts/system/ft_tokenizer/" -original_file = os.path.join(base_path, "/alltalk_tts/models/xtts/xttsv2_2.0.3/vocab.json") # base model vocab.json -new_file = os.path.join(base_path, "/expanded_model/bpe_tokenizer-vocab.json") # new bpe tokenizer vocab.json -output_file = os.path.join(base_path, "/expanded_model/expanded_vocab.json") # output expanded_vocab.json file/path +base_vocab_path = "/alltalk_tts/models/xtts/xttsv2_2.0.2/originalvocab.json" # base model vocab.json path (2.0.2) +new_vocab_path = "/alltalk_tts/expanded_models/tortoise_tokenizer2xttsv2.json" # path to the custom dataset vocab.json +output_path = "/alltalk_tts/expanded_models/combined_vocab.json" # location for combined vocab.json - -with open(original_file, 'r') as f: - original_data = json.load(f) - -with open(new_file, 'r') as f: - new_data = json.load(f) - -# Get the original and new vocabularies -original_vocab = original_data['model']['vocab'] -new_vocab = new_data['model']['vocab'] - -# Find the maximum value in the original vocabulary. We do this so we make sure to append and not overwrite any new values -max_value = max(original_vocab.values()) - -# merge -for key, value in new_vocab.items(): - if key not in original_vocab: - max_value += 1 - original_vocab[key] = max_value - -# Update -original_data['model']['vocab'] = original_vocab - -# Get the original and new merges -original_merges = original_data['model']['merges'] -new_merges = new_data['model']['merges'] - -# Merge the merges -merged_merges = original_merges.copy() -for merge in new_merges: - if merge not in merged_merges: - merged_merges.append(merge) - -# Update -original_data['model']['merges'] = merged_merges -with open(output_file, 'w') as f: - json.dump(original_data, f, ensure_ascii=False, indent=2) - -print(f"Merged vocabulary and merges saved to {output_file}") +# Merge the vocabularies +merge_vocabularies(base_vocab_path, new_vocab_path, output_path) \ No newline at end of file diff --git a/system/ft_tokenizer/custom_tokenizer.py b/system/ft_tokenizer/custom_tokenizer.py new file mode 100644 index 0000000..4362425 --- /dev/null +++ b/system/ft_tokenizer/custom_tokenizer.py @@ -0,0 +1,60 @@ +# Credit Jarod Mica https://github.com/JarodMica/tortoise_dataset_tools/blob/master/bpe_tokenizer_tools/train_bpe_tokenizer.py +# provide a cleaned txt file with only the transcription. Youll get back out the datasets vocab.json + +from tokenizers import Tokenizer +from tokenizers.models import BPE +from tokenizers.trainers import BpeTrainer +from tokenizers.pre_tokenizers import Whitespace +from tkinter import Tk, filedialog +import json +import re + +def clean_text(input_file_path, output_file_path): + # Define the pattern to match numbers, specific symbols, and new lines + # add \d to match any digit, and | is used to specify alternatives + pattern = r'|�|«|\$|\n' + + with open(input_file_path, 'r', encoding='utf-8') as input_file: + text = input_file.read() + cleaned_text = re.sub(pattern, '', text) + + with open(output_file_path, 'w', encoding='utf-8') as output_file: + output_file.write(cleaned_text) + +def train_tokenizer(input_path, tokenizer_path, language, special_tokens=["[STOP]", "[UNK]", "[SPACE]" ], vocab_size=256): + # Initialize a tokenizer with the BPE model + tokenizer = Tokenizer(BPE(unk_token="[UNK]")) + # Use a basic whitespace pre-tokenizer + tokenizer.pre_tokenizer = Whitespace() + + # trainer = BpeTrainer(special_tokens=["[STOP]", "[UNK]", "[SPACE]", "0","1","2","3","4","5","6","7","8","9",], vocab_size=256) + trainer = BpeTrainer(special_tokens=special_tokens, vocab_size=vocab_size) + + + clean_text(input_path, input_path) + tokenizer.train([input_path], trainer) + + tokenizer.save(tokenizer_path) + + with open(tokenizer_path, 'r', encoding='utf-8') as f: + tokenizer_json = json.load(f) + + # Add language to tokenizer + tokenizer_json['model']['language'] = language + + with open(tokenizer_path, 'w', encoding='utf-8') as f: + json.dump(tokenizer_json, f, ensure_ascii=False, indent=4) + +def choose_file(): + root = Tk() + root.withdraw() + file = filedialog.askopenfilename() + root.destroy() + return file + +if __name__ == "__main__": + input_path = choose_file() + tokenizer_path = "/alltalk_tts/expanded_models/tortoise_tokenizer2xttsv2.json" # define path to put the newly create vocab.json + special_tokens = ["[STOP]", "[UNK]", "[SPACE]"] + vocab_size = 256 # model is stuck at this size + train_tokenizer(input_path, tokenizer_path, language='multi', special_tokens=special_tokens, vocab_size=vocab_size) \ No newline at end of file diff --git a/system/ft_tokenizer/extract_dataset_for_tokenizer.py b/system/ft_tokenizer/extract_dataset_for_tokenizer.py new file mode 100644 index 0000000..36dee0c --- /dev/null +++ b/system/ft_tokenizer/extract_dataset_for_tokenizer.py @@ -0,0 +1,26 @@ +# Simple script to remove LJ Speech formatting for the tokenizer +# Combine metadata_train and metadata_eval.csv into a single file then run +import csv + +# Input and output file names +input_file = '/alltalkbeta/metadata_eval.csv' # combine metadata_train and metadata_eval.csv +output_file = '/alltalkbeta/dataset.txt' # this goes to the tokenizer + +# Read the input CSV and write to the output file +with open(input_file, 'r', newline='', encoding='utf-8') as infile, \ + open(output_file, 'w', newline='', encoding='utf-8') as outfile: + + # Create CSV reader and writer objects + reader = csv.reader(infile, delimiter='|') + writer = csv.writer(outfile, delimiter='|') + + # Skip the header + next(reader, None) + + # Process each row + for row in reader: + if len(row) >= 2: + # Write only the second column (index 1) to the output file + writer.writerow([row[1]]) + +print(f"Processing complete. Output written to {output_file}") \ No newline at end of file