kromskii2/alltalk_tts
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update to use openai-whisper

Browse Source
This commit is contained in:
erew123 authored and GitHub committed 2024-11-15 00:40:02 +00:00
1 parent 5c4726de96
commit b538564da6
1 file changed
+241 -269
Download Patch File Download Diff File Expand all files Collapse all files
system/tts_diff/tts_diff.py
+241 -269
View File
@@ -1,269 +1,241 @@
import json import json
import argparse import argparse
from pathlib import Path from pathlib import Path
import os import os
import glob import sys
import platform import subprocess
import site # version 0.9
import subprocess import warnings
import sys # Filter Flash Attention warning
# version 0.8 warnings.filterwarnings("ignore", message="1Torch was not compiled with flash attention")
os.environ['KMP_DUPLICATE_LIB_OK']='True' warnings.filterwarnings("ignore", message="Failed to launch Triton kernels, likely due to missing CUDA toolkit")
# Try to import whisper, install if not found
def find_files_in_path_with_wildcard(pattern): try:
# Get the site-packages directory of the current Python environment import whisper
site_packages_path = site.getsitepackages() except ImportError:
found_paths = [] print("[AllTalk TTSDiff] OpenAI Whisper not found. Attempting to install...")
# Adjust the sub-directory based on the operating system try:
sub_directory = "nvidia/cublas" subprocess.check_call([sys.executable, "-m", "pip", "install", "openai-whisper"])
if platform.system() == "Linux": import whisper
sub_directory = os.path.join(sub_directory, "lib") print("[AllTalk TTSDiff] Successfully installed OpenAI Whisper!")
else: except Exception as e:
sub_directory = os.path.join(sub_directory, "bin") print("[AllTalk TTSDiff] Failed to install OpenAI Whisper:")
print(f"[AllTalk TTSDiff] Error: {str(e)}")
# Iterate over each site-packages directory (there can be more than one) print("[AllTalk TTSDiff] Please try manually installing with: pip install openai-whisper")
for directory in site_packages_path: sys.exit(1)
# Construct the search directory path
search_directory = os.path.join(directory, sub_directory) parser = argparse.ArgumentParser(description="Compare TTS output with the original text using detailed comparison.")
# Use glob to find all files matching the pattern in this directory parser.add_argument("--threshold", type=int, default=98, help="Similarity threshold for considering a match (default: 98)")
for file_path in glob.glob(os.path.join(search_directory, pattern)): parser.add_argument("--ttslistpath", help="Path to the ttsList.json file")
if os.path.isfile(file_path): # Ensure it's a file parser.add_argument("--wavfilespath", help="Path to the wav outputs folder")
found_paths.append(file_path) args = parser.parse_args()
return found_paths
if not args.ttslistpath:
def detect_cublas(): parser.error("[AllTalk TTSDiff] Please specify the path to the ttslist.json file using the --ttslistpath argument.")
# Use different file patterns based on the operating system if not args.wavfilespath:
file_name_pattern = 'cublas64_11.*' if platform.system() == "Windows" else 'libcublas.so.11*' parser.error("[AllTalk TTSDiff] Please specify the path to the wavs output folder file using the --wavfilespath argument.")
found_paths = find_files_in_path_with_wildcard(file_name_pattern)
if found_paths: json_file_path = Path(args.ttslistpath).resolve()
print("[AllTalk TTSDiff] \033[94mCublas:\033[0m \033[92mDetected\033[0m", found_paths) wav_file_path = Path(args.wavfilespath).resolve()
else: accuracy_threshold = args.threshold
print("[AllTalk TTSDiff] \033[94mCublas:\033[0m \033[91mERROR Not Detected\033[0m")
print("[AllTalk TTSDiff] \033[94mPlease install Cublas from the Nvidia CUDA Toolkit \033[0mhttps://developer.nvidia.com/cuda-downloads") def disclaimer_text():
exit(1) print(f"\nDESCRIPTION: ")
print(f" This script is designed to assist in identifying bad TTS generation ID's from the TTS Generator.\n")
print(f" This will download the Whisper Large-v3 model (if not already downloaded), and use it to compare your text to")
parser = argparse.ArgumentParser(description="Compare TTS output with the original text using detailed comparison.") print(f" to the generated TTS. If it finds an issue, it will flag the ID number that has the issue. This does NOT 100%")
parser.add_argument("--threshold", type=int, default=98, help="Similarity threshold for considering a match (default: 98)") print(f" ensure that all TTS generated is perfect, sounds good etc, but it should flag up glaring issues and help with")
parser.add_argument("--ttslistpath", help="Path to the ttsList.json file") print(f" the process of identifying bad TTS generations.")
parser.add_argument("--wavfilespath", help="Path to the wav outputs folder") print(f"\nNOTES ON USE: ")
args = parser.parse_args() print(f" - Ensure AllTalk is running in its own command prompt/terminal.")
print(f" - This script should be running in its own command prompt/terminal window in the same Python Environment that")
if not args.ttslistpath: print(f" AllTalk is running in.")
parser.error("[AllTalk TTSDiff] Please specify the path to the ttslist.json file using the --ttslistpath argument.") print(f" - After generating your text in the TTS Generator, you need to `Export List to JSON` and save the file in the")
if not args.wavfilespath: print(f" same folder/directory this script is running from.")
parser.error("[AllTalk TTSDiff] Please specify the path to the wavs output folder file using the --wavfilespath argument.") print(f" - You can use `--threshold` with a numerical value from 1 to 100 to set accuracy detection. For example it may")
print(f" detect/transcribe `their` as `there`, which of course, both words sound the same. If you want 100% accuracy")
json_file_path = Path(args.ttslistpath).resolve() print(f" you will set 100 as the threshold. You will more than likely want a number in the high 90's. Default is set")
wav_file_path = Path(args.wavfilespath).resolve() print(f" at 98, which gives a little flexibility.")
accuracy_threshold = args.threshold print(f" - The JSON file must be named \033[93mttsList.json\033[0m")
print(f" - When you have your ID list, go back into the TTS Generator, correct any lines and regenerate them. If you")
def disclaimer_text(): print(f" want to re-test everthing again after re-generating, you will need to export your list again and re-run this")
print(f"\nDESCRIPTION: ") print(f" script again, against your newly exported JSON list.\n")
print(f" This script is designed to assist in identifying bad TTS generation ID's from the TTS Generator.\n") return
print(f" This will download the Whisper Large-v2 model (if not already downloaded), and use it to compare your text to")
print(f" to the generated TTS. If it finds an issue, it will flag the ID number that has the issue. This does NOT 100%") try:
print(f" ensure that all TTS generated is perfect, sounds good etc, but it should flag up glaring issues and help with") import re
print(f" the process of identifying bad TTS generations.") import string
print(f"\nNOTES ON USE: ") import spacy
print(f" - Ensure AllTalk is running in its own command prompt/terminal.") import torch
print(f" - This script should be running in its own command prompt/terminal window in the same Python Environment that") from fuzzywuzzy import fuzz
print(f" AllTalk is running in.") except ImportError as e:
print(f" - After generating your text in the TTS Generator, you need to `Export List to JSON` and save the file in the") print(f"[AllTalk TTSDiff] ERROR STARTING SCRIPT:")
print(f" same folder/directory this script is running from.") print("[AllTalk TTSDiff] An error occurred importing one or more required libraries.")
print(f" - You can use `--threshold` with a numerical value from 1 to 100 to set accuracy detection. For example it may") print("[AllTalk TTSDiff] Please ensure you have \033[94mactivated your Python environment \033[0mand \033[94minstalled all requirements.\033[0m")
print(f" detect/transcribe `their` as `there`, which of course, both words sound the same. If you want 100% accuracy") print("[AllTalk TTSDiff] \033[94mMissing library:\033[93m", str(e), "\033[0m")
print(f" you will set 100 as the threshold. You will more than likely want a number in the high 90's. Default is set") exit(1)
print(f" at 98, which gives a little flexibility.")
print(f" - The JSON file must be named \033[93mttsList.json\033[0m") # Attempt to load the spaCy model
print(f" - When you have your ID list, go back into the TTS Generator, correct any lines and regenerate them. If you") try:
print(f" want to re-test everthing again after re-generating, you will need to export your list again and re-run this") # Load a medium-sized spaCy model, adjust to "en_core_web_md" or "en_core_web_lg" as needed
print(f" script again, against your newly exported JSON list.") nlp = spacy.load("en_core_web_md")
print(f" - This requires access to \033[93mcublas64_11\033[0m, the same as Finetuning.") except Exception as e:
print(f" https://github.com/erew123/alltalk_tts/tree/main?#-important-requirements-cuda-118\n") try:
return print("[AllTalk TTSDiff] SpaCy language model not installed.")
print("[AllTalk TTSDiff] Attempting to automatically download the spaCy language model.")
try: subprocess.run([sys.executable, "-m", "spacy", "download", "en_core_web_md"], check=True)
from faster_whisper import WhisperModel # Try to load the model again after downloading
import re nlp = spacy.load("en_core_web_md")
import string print("[AllTalk TTSDiff] Model downloaded and loaded successfully.")
import spacy except Exception as download_exception:
import torch print("[AllTalk TTSDiff] Automatic download failed.")
from fuzzywuzzy import fuzz print("[AllTalk TTSDiff] Please manually install the spaCy language model by running:")
except ImportError as e: print("[AllTalk TTSDiff] \033[93mpython -m spacy download en_core_web_md\033[0m")
print(f"[AllTalk TTSDiff] ERROR STARTING SCRIPT:") exit(1)
print("[AllTalk TTSDiff] An error occurred importing one or more required libraries.")
print("[AllTalk TTSDiff] Please ensure you have \033[94mactivated your Python environment \033[0mand \033[94minstalled all requirements.\033[0m") def texts_are_similar(text1, text2, threshold=0.8):
print("[AllTalk TTSDiff] \033[94mMissing library:\033[93m", str(e), "\033[0m") # Preliminary check for short texts
exit(1) if len(text1) < 10 or len(text2) < 10:
ratio = fuzz.ratio(text1, text2) / 100.0 # Use FuzzyWuzzy for short texts
# Attempt to load the spaCy model return ratio > threshold
try:
# Load a medium-sized spaCy model, adjust to "en_core_web_md" or "en_core_web_lg" as needed # Process the texts through the NLP model for longer texts
nlp = spacy.load("en_core_web_md") doc1 = nlp(text1)
except Exception as e: doc2 = nlp(text2)
print("[AllTalk TTSDiff] Failed to load the spaCy language model.")
# Compute semantic similarity
# Attempt to download the spaCy model automatically similarity = doc1.similarity(doc2)
try:
print("[AllTalk TTSDiff] Attempting to automatically download the spaCy language model...") return similarity >= threshold
subprocess.run([sys.executable, "-m", "spacy", "download", "en_core_web_md"], check=True)
def normalize_text(text):
# Try to load the model again after downloading # Normalize or remove CRLF and other non-standard whitespaces
nlp = spacy.load("en_core_web_md") text = text.replace("\r\n", " ").replace("\n", " ").replace("\r", " ")
print("[AllTalk TTSDiff] Model downloaded and loaded successfully.")
except Exception as download_exception: # Convert to lowercase
print("[AllTalk TTSDiff] Automatic download failed.") text = text.lower()
print("[AllTalk TTSDiff] Please manually install the spaCy language model by running:")
print("[AllTalk TTSDiff] \033[93mpython -m spacy download en_core_web_md\033[0m") # Standardize and then remove quotation marks
exit(1) text = text.replace(""", '"').replace(""", '"').replace("'", "'").replace("'", "'")
text = text.translate(str.maketrans('', '', '"\''))
def texts_are_similar(text1, text2, threshold=0.8):
# Preliminary check for short texts # Remove all other punctuation except hyphens to preserve compound words
if len(text1) < 10 or len(text2) < 10: text = text.translate(str.maketrans('', '', string.punctuation.replace("-", "")))
ratio = fuzz.ratio(text1, text2) / 100.0 # Use FuzzyWuzzy for short texts
return ratio > threshold # Collapse any sequence of whitespace (including spaces) into a single space
text = re.sub(r'\s+', ' ', text)
# Process the texts through the NLP model for longer texts
doc1 = nlp(text1) return text.strip()
doc2 = nlp(text2)
spoken_punctuation_mapping = {
# Compute semantic similarity "dot": ".",
similarity = doc1.similarity(doc2) "comma": ",",
# Add more mappings as needed
return similarity >= threshold }
def normalize_text(text): def contains_spoken_punctuation(transcribed_text):
# Normalize or remove CRLF and other non-standard whitespaces for spoken, punctuation in spoken_punctuation_mapping.items():
text = text.replace("\r\n", " ").replace("\n", " ").replace("\r", " ") if spoken in transcribed_text:
return True
# Convert to lowercase return False
text = text.lower()
def detailed_comparison(original_text, transcribed_text, threshold=98):
# Standardize and then remove quotation marks original_clean = normalize_text(original_text)
text = text.replace("“", '"').replace("”", '"').replace("‘", "'").replace("’", "'") transcribed_clean = normalize_text(transcribed_text)
text = text.translate(str.maketrans('', '', '"\'')) ratio = fuzz.ratio(original_clean, transcribed_clean)
return ratio > threshold
# Remove all other punctuation except hyphens to preserve compound words
text = text.translate(str.maketrans('', '', string.punctuation.replace("-", ""))) def access_local_file(url):
# Extract the file name from the URL
# Collapse any sequence of whitespace (including spaces) into a single space file_name = url.split('/')[-1]
text = re.sub(r'\s+', ' ', text) # Construct the full path to the file on disk
file_path = wav_file_path / file_name
return text.strip() # Check if the file exists
if not file_path.exists():
spoken_punctuation_mapping = { print(f"[AllTalk TTSDiff] File does not exist: {file_path}")
"dot": ".", return None
"comma": ",", return file_path
# Add more mappings as needed
} def transcribe_and_compare(file_url, original_text, model, item_id, flagged_ids):
audio_file_path = access_local_file(file_url)
def contains_spoken_punctuation(transcribed_text): if audio_file_path is None:
for spoken, punctuation in spoken_punctuation_mapping.items(): print(f"[AllTalk TTSDiff] Could not access local file for URL: {file_url}")
if spoken in transcribed_text: return # Skip this file if it can't be accessed
return True
return False try:
result = model.transcribe(str(audio_file_path))
def detailed_comparison(original_text, transcribed_text, threshold=98): transcribed_text = result["text"]
original_clean = normalize_text(original_text) except Exception as e:
transcribed_clean = normalize_text(transcribed_text) print(f"[AllTalk TTSDiff] Error transcribing file {audio_file_path}: {e}")
ratio = fuzz.ratio(original_clean, transcribed_clean) return # Skip this file if transcription fails
return ratio > threshold
# Normalize texts for comparison
def access_local_file(url): original_text_normalized = normalize_text(original_text)
# Extract the file name from the URL transcribed_text_normalized = normalize_text(transcribed_text)
file_name = url.split('/')[-1]
# Construct the full path to the file on disk # Enhanced comparison using detailed fuzzy matching
file_path = wav_file_path / file_name is_detailed_match = detailed_comparison(original_text_normalized, transcribed_text_normalized, args.threshold)
# Check if the file exists has_spoken_punctuation = contains_spoken_punctuation(transcribed_text_normalized) and not contains_spoken_punctuation(original_text_normalized)
if not file_path.exists():
print(f"[AllTalk TTSDiff] File does not exist: {file_path}") # Adjust is_match based on detailed comparison and spoken punctuation check
return None is_match = is_detailed_match and not has_spoken_punctuation
return file_path
# Only log and flag IDs for review if there's a mismatch or detected issues
def transcribe_and_compare(file_url, original_text, model, item_id, flagged_ids): if not is_match:
audio_file_path = access_local_file(file_url) print(f"[AllTalk TTSDiff] \033[93mText:\033[0m {original_text}")
if audio_file_path is None: print(f"[AllTalk TTSDiff] \033[91mTTS :\033[0m{transcribed_text}")
print(f"[AllTalk TTSDiff] Could not access local file for URL: {file_url}") if has_spoken_punctuation:
return # Skip this file if it can't be accessed print(f"[AllTalk TTSDiff] Note: Potential incorrect spoken punctuation detected.")
flagged_ids.append(item_id) # Track the ID for review
try:
segments, info = model.transcribe(str(audio_file_path), beam_size=5) def main():
transcribed_text = " ".join([segment.text for segment in segments]) print(f"[AllTalk TTSDiff]")
except Exception as e: print(f"[AllTalk TTSDiff]")
print(f"[AllTalk TTSDiff] Error transcribing file {audio_file_path}: {e}") print("[AllTalk TTSDiff] \033[92mStarting Compare of Text vs TTS:\033[0m")
return # Skip this file if transcription fails print("[AllTalk TTSDiff]")
print("[AllTalk TTSDiff] \033[94mJSON file :\033[92m", json_file_path, "\033[0m")
segments, info = model.transcribe(str(audio_file_path), beam_size=5) print("[AllTalk TTSDiff] \033[94mWAV files :\033[92m", wav_file_path, "\033[0m")
transcribed_text = " ".join([segment.text for segment in segments]) print("[AllTalk TTSDiff] \033[94mAccuracy :\033[92m", accuracy_threshold, "%", "\033[0m")
# Normalize texts for comparison device = "cuda" if torch.cuda.is_available() else "cpu"
original_text_normalized = normalize_text(original_text) print("[AllTalk TTSDiff] \033[94mDevice :\033[92m", device, "\033[0m")
transcribed_text_normalized = normalize_text(transcribed_text) print("[AllTalk TTSDiff] Loading Whisper Large-v3 model...")
model = whisper.load_model("large-v3", device=device)
# Enhanced comparison using detailed fuzzy matching
is_detailed_match = detailed_comparison(original_text_normalized, transcribed_text_normalized, args.threshold) flagged_ids = [] # Initialize the list to track IDs needing review
has_spoken_punctuation = contains_spoken_punctuation(transcribed_text_normalized) and not contains_spoken_punctuation(original_text_normalized)
try:
# Adjust is_match based on detailed comparison and spoken punctuation check with open(json_file_path, "r", encoding="utf-8") as f:
is_match = is_detailed_match and not has_spoken_punctuation tts_list = json.load(f)
except FileNotFoundError:
# Only log and flag IDs for review if there's a mismatch or detected issues print(f"[AllTalk TTSDiff] ERROR STARTING SCRIPT:")
if not is_match: print("[AllTalk TTSDiff] Error: \033[93mttsList.json\033[0m file not found.")
#print(f"\033[93mMismatch Detected - ID:\033[0m {item_id}") print("[AllTalk TTSDiff] Please ensure you follow these steps:")
print(f"[AllTalk TTSDiff] \033[93mText:\033[0m {original_text}") print("[AllTalk TTSDiff] After generating your text in the TTS Generator, you need to `Export List to JSON` and save the file in the")
print(f"[AllTalk TTSDiff] \033[91mTTS :\033[0m{transcribed_text}") print("[AllTalk TTSDiff] same folder/directory this script is running from.")
if has_spoken_punctuation: print("[AllTalk TTSDiff] The JSON file must be named \033[93mttsList.json\033[0m")
print(f"[AllTalk TTSDiff] Note: Potential incorrect spoken punctuation detected.") exit(1)
flagged_ids.append(item_id) # Track the ID for review
print(f"[AllTalk TTSDiff]")
def main():
print(f"[AllTalk TTSDiff]") for item in tts_list:
print(f"[AllTalk TTSDiff]") print(f"[AllTalk TTSDiff] Processing ID: {item['id']}")
print("[AllTalk TTSDiff] \033[92mStarting Compare of Text vs TTS:\033[0m") transcribe_and_compare(item['fileUrl'], item['text'], model, item['id'], flagged_ids)
print("[AllTalk TTSDiff]")
print("[AllTalk TTSDiff] \033[94mJSON file :\033[92m", json_file_path, "\033[0m") print(f"[AllTalk TTSDiff]")
print("[AllTalk TTSDiff] \033[94mWAV files :\033[92m", wav_file_path, "\033[0m") # Print summary information at the end
print("[AllTalk TTSDiff] \033[94mAccuracy :\033[92m", accuracy_threshold, "%", "\033[0m") if flagged_ids:
detect_cublas() print(f"[AllTalk TTSDiff] \033[94mIDs needing review:\033[0m", ', '.join(map(str, flagged_ids)))
model_size = "large-v2" print(f"[AllTalk TTSDiff]")
device = "cuda" if torch.cuda.is_available() else "cpu" print(f"[AllTalk TTSDiff] Review ID's and correct any lines by editing & regenerating as necessary.")
model = WhisperModel(model_size, device=device, compute_type="float32") else:
print("[AllTalk TTSDiff] \033[94mIDs needing review:\033[0m No issues detected.")
flagged_ids = [] # Initialize the list to track IDs needing review summary_data = {
"flagged_ids": flagged_ids
try: }
with open(json_file_path, "r", encoding="utf-8") as f: with open(wav_file_path / "analysis_summary.json", "w") as summary_file:
tts_list = json.load(f) json.dump(summary_data, summary_file)
except FileNotFoundError: print("[AllTalk TTSDiff]")
print(f"[AllTalk TTSDiff] ERROR STARTING SCRIPT:")
print("[AllTalk TTSDiff] Error: \033[93mttsList.json\033[0m file not found.") if __name__ == "__main__":
print("[AllTalk TTSDiff] Please ensure you follow these steps:") main()
print("[AllTalk TTSDiff] After generating your text in the TTS Generator, you need to `Export List to JSON` and save the file in the")
print("[AllTalk TTSDiff] same folder/directory this script is running from.")
print("[AllTalk TTSDiff] The JSON file must be named \033[93mttsList.json\033[0m")
exit(1)
print(f"[AllTalk TTSDiff]")
for item in tts_list:
print(f"[AllTalk TTSDiff] Processing ID: {item['id']}")
transcribe_and_compare(item['fileUrl'], item['text'], model, item['id'], flagged_ids)
print(f"[AllTalk TTSDiff]")
# Print summary information at the end
if flagged_ids:
print(f"[AllTalk TTSDiff] \033[94mIDs needing review:\033[0m", ', '.join(map(str, flagged_ids)))
print(f"[AllTalk TTSDiff]")
print(f"[AllTalk TTSDiff] Review ID's and correct any lines by editing & regenerating as necessary.")
else:
print("[AllTalk TTSDiff] \033[94mIDs needing review:\033[0m No issues detected.")
summary_data = {
"flagged_ids": flagged_ids
}
with open(wav_file_path / "analysis_summary.json", "w") as summary_file:
json.dump(summary_data, summary_file)
print("[AllTalk TTSDiff]")
if __name__ == "__main__":
main()