kromskii2/alltalk_tts
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add files via upload

Browse Source
This commit is contained in:
erew123 authored and GitHub committed 2024-01-01 18:10:14 +00:00
1 parent df69fb0b07
commit 2dd303f53c
4 files changed
+1918 -814
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

script.py
+78 -67
View File
@@ -509,11 +509,53 @@ def voice_preview(string):
return f"[{params['branding']}Server] Audio generation failed. Status: {generate_response.get('status')}"
#def replace_asterisk(match):
# return match.group(0) + '&quot;<*'
def process_text(text):
# Normalize HTML encoded quotes
text = html.unescape(text)
def replace_asterisk(match):
return '&quot;<*'
# Replace ellipsis with a single dot
text = re.sub(r'\.{3,}', '.', text)
# Pattern to identify combined narrator and character speech
combined_pattern = r'(\*[^*"]+\*|"[^"*]+")'
# List to hold parts of speech along with their type
ordered_parts = []
# Track the start of the next segment
start = 0
# Find all matches
for match in re.finditer(combined_pattern, text):
# Add the text before the match, if any, as ambiguous
if start < match.start():
ambiguous_text = text[start:match.start()].strip()
if ambiguous_text:
ordered_parts.append(('ambiguous', ambiguous_text))
# Add the matched part as either narrator or character
matched_text = match.group(0)
if matched_text.startswith('*') and matched_text.endswith('*'):
ordered_parts.append(('narrator', matched_text.strip('*').strip()))
elif matched_text.startswith('"') and matched_text.endswith('"'):
ordered_parts.append(('character', matched_text.strip('"').strip()))
else:
# In case of mixed or improperly formatted parts
if '*' in matched_text:
ordered_parts.append(('narrator', matched_text.strip('*').strip('"')))
else:
ordered_parts.append(('character', matched_text.strip('"').strip('*')))
# Update the start of the next segment
start = match.end()
# Add any remaining text after the last match as ambiguous
if start < len(text):
ambiguous_text = text[start:].strip()
if ambiguous_text:
ordered_parts.append(('ambiguous', ambiguous_text))
return ordered_parts
#################################
#### TTS STANDARD GENERATION ####
@@ -534,86 +576,55 @@ def output_modifier(string, state):
if process_lock.acquire(blocking=False):
try:
if params["narrator_enabled"]:
# Do Some basic stripping and remove any double CR's
processed_string = (
original_string
.replace("***", "*")
.replace("**", "*")
.replace("\n\n", "\n")
.replace("\n", "")
)
processed_string = re.sub(r'\.\*\n\*', '. ', processed_string)
processed_string = (processed_string .replace('&quot;\n\n*', '&quot;<*'))
processed_string = (processed_string .replace('&quot;\n', '< '))
processed_string = (processed_string .replace("&#x27;", "'"))
processed_string = (processed_string .replace("\n", " "))
processed_string = (processed_string .replace('&quot;', '&quot;<'))
processed_string = (processed_string .replace('"', '&quot;<'))
#pattern = re.compile(r'(?<=[a-zA-Z!])&quot;<(?=[a-zA-Z])')
pattern = re.compile(r'(?<=[a-zA-Z!?\.])&quot;<(?=[a-zA-Z])')
processed_string = pattern.sub('&quot; ', processed_string)
pattern = re.compile(r'(?<=[a-zA-Z])\*(?=[a-zA-Z])')
processed_string = pattern.sub('*&quot;', processed_string)
processed_string = processed_string.replace('&quot;<. *', '&quot;< *"')
processed_string = processed_string.replace('< *"', '< *')
processed_string = processed_string.replace('. *', '< *')
processed_string = processed_string.replace('! *', '&quot;< *')
processed_string = processed_string.replace('? *', '< *')
processed_string = processed_string.replace('* ', '* &quot;<')
processed_string = processed_string.replace('*. ', '* &quot;<')
processed_string = re.sub(r'\*(?=[a-zA-Z])', replace_asterisk, processed_string)
#print("PROCESSED STRING IS:", processed_string)
# Set up a tracking of the individual wav files.
processed_parts = process_text(original_string)
audio_files_all_paragraphs = []
# Split the line using &quot; and ".* " (so end of sentences, leaving special characters added to the start of all OTHER sentences, bar possibly the first one if its starting with a *
parts = re.split(r'&quot;|\.\*', processed_string)
audio_files_paragraph = []
for i, part in enumerate(parts):
for part_type, part in processed_parts:
# Skip parts that are too short
if len(part.strip()) <= 3:
continue
# Figure out which type of line it is, then replace characters as necessary to avoid TTS trying to pronunce them, htmlunescape after.
# Character will always be a < with a letter immediately after it
#print("PART IS:", part)
if '<' in part and '< *' not in part and '<*' not in part and '< *' not in part and '< ' not in part and '< ' not in part:
cleaned_part = html.unescape(part.replace('<', ''))
voice_to_use = params["voice"]
#Narrator will always be an * or < with an * a position or two after it.
elif '<*' in part or '< *' in part or '< *' in part or '*' in part:
cleaned_part = html.unescape(part.replace('<*', '').replace('< *', '').replace('< *', '').replace('*', ''))
# Determine the voice to use based on the part type
if part_type == 'narrator':
voice_to_use = params["narrator_voice"]
#If the other two dont capture it, aka, the AI gave no * or &quot; on the line, use non_quoted_text_is aka user interface, user can choose Char or Narrator
elif non_quoted_text_is:
#print("HIT ELIF CHAR VOICE")
cleaned_part = html.unescape(part.replace('< ', '').replace('< ', '').replace('< ', ''))
print(f"[{params['branding']}TTSGen] \033[92mNarrator\033[0m") # Green
elif part_type == 'character':
voice_to_use = params["voice"]
print(f"[{params['branding']}TTSGen] \033[36mCharacter\033[0m") # Yellow
else:
#print("HIT ELIF NARRATOR VOICE")
cleaned_part = html.unescape(part.replace('< ', '').replace('< ', '').replace('< ', ''))
voice_to_use = params["narrator_voice"]
# Generate that TTS and output to a file
# Handle ambiguous parts based on user preference
voice_to_use = params["voice"] if non_quoted_text_is else params["narrator_voice"]
voice_description = "\033[36mCharacter (Text-not-inside)\033[0m" if non_quoted_text_is else "\033[92mNarrator (Text-not-inside)\033[0m"
print(f"[{params['branding']}TTSGen] {voice_description}")
# Replace multiple exclamation marks, question marks, or other punctuation with a single instance
cleaned_part = re.sub(r'([!?.])\1+', r'\1', part)
# Further clean to remove any other unwanted characters
cleaned_part = re.sub(r'[^a-zA-Z0-9\s\.,;:!?\-\'"]', '', cleaned_part)
# Remove all newline characters (single or multiple)
cleaned_part = re.sub(r'\n+', ' ', cleaned_part)
# Generate TTS and output to a file
output_filename = get_output_filename(state)
generate_response = send_generate_request(
cleaned_part, voice_to_use, language_code, output_filename
)
audio_path = generate_response.get("data", {}).get("audio_path")
audio_files_paragraph.append(audio_path)
# Accumulate audio files within the paragraph
audio_files_all_paragraphs.extend(audio_files_paragraph)
audio_files_all_paragraphs.append(audio_path)
# Combine audio files across paragraphs
final_output_file = combine(
audio_files_all_paragraphs, params["output_folder_wav"], state
)
else:
processed_string = (
original_string
.replace("***", "")
.replace("**", "")
.replace("*", "")
.replace("\n\n", "\n")
.replace("&#x27;", "'")
)
cleaned_part = html.unescape(processed_string)
# Decode HTML entities first
cleaned_part = html.unescape(original_string)
# Replace multiple instances of certain punctuation marks with a single instance
cleaned_part = re.sub(r'([!?.])\1+', r'\1', cleaned_part)
# Further clean to remove any other unwanted characters
cleaned_part = re.sub(r'[^a-zA-Z0-9\s\.,;:!?\-\'"]', '', cleaned_part)
# Remove all newline characters (single or multiple)
cleaned_part = re.sub(r'\n+', ' ', cleaned_part)
# Process the part and give it a non-character name if being used vai API or standalone.
if "character_menu" in state:
output_file = Path(f'{params["output_folder_wav"]}/{state["character_menu"]}_{int(time.time())}.wav')
templates/TTS_Input_Proof_of_concept.html
+584
View File
@@ -0,0 +1,584 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>AllTalk TTS Generator</title>
<style>
body,
textarea,
select,
input[type="range"],
input[type="number"],
button {
font-family: Arial, sans-serif;
}
body {
margin: 20px;
background-color: #f7f7f7;
color: #333;
}
h1 {
color: #333;
text-align: center;
}
.form-container {
max-width: 80%;
margin: 50px auto;
padding: 20px;
background: #fff;
border-radius: 8px;
box-shadow: 0 2px 5px rgba(0, 0, 0, .2);
}
.settings-section {
display: flex;
flex-wrap: wrap;
justify-content: flex-start;
/* Align items to the start */
align-items: center;
/* Align items vertically */
gap: 20px;
/* This creates a consistent gap between all child elements */
}
.setting {
margin-bottom: 10px;
/* Space from bottom if wrapped */
display: flex;
flex-direction: column;
/* Removed flex property to stop forcing them to grow */
}
label {
font-size: 0.9em;
color: #333;
margin-bottom: 5px;
}
textarea {
resize: vertical;
width: 100%;
padding: 10px;
margin-bottom: 16px;
border: 1px solid #ccc;
border-radius: 4px;
box-sizing: border-box;
}
/* Ensure select and input elements do not exceed the width of their parent */
select,
input[type="range"],
input[type="number"] {
width: 100%;
/* They will fill the width of their .setting container */
padding: 10px;
margin-bottom: 16px;
border: 1px solid #ccc;
border-radius: 4px;
}
@media (min-width: 768px) {
select,
input[type="range"],
input[type="number"] {
width: auto;
/* Only auto width for larger screens */
}
}
button {
background-color: #4caf50;
color: #fff;
padding: 10px 20px;
border: none;
border-radius: 4px;
cursor: pointer;
display: inline-block;
margin: 5px 0;
}
button:hover {
background-color: #45a049;
}
.dark-mode {
background-color: #121212;
color: #ffffff;
/* Text color for dark mode background */
}
.dark-mode .form-container {
background-color: #1e1e1e;
}
/* Modify the text color for elements within dark mode */
.dark-mode label,
.dark-mode textarea,
.dark-mode select,
.dark-mode input[type="range"],
.dark-mode input[type="number"],
.dark-mode button,
.dark-mode .error {
color: #ffffff;
/* Text color for dark mode elements */
}
/* Modify text color for input elements, textarea, and dropdowns in dark mode */
.dark-mode input,
.dark-mode textarea,
.dark-mode select {
color: #000000;
/* Text color for input elements, textarea, and dropdowns in dark mode */
}
/* Set background color for textarea in dark mode */
.dark-mode textarea {
background-color: #C0C0C0;
/* Light grey background color for textarea in dark mode */
}
/* Set background color for option elements within select in dark mode */
.dark-mode select {
background-color: #C0C0C0;
/* Light grey background color for options in dark mode */
}
/* Additional styles if needed for fixed-width dropdowns */
.fixed-size-dropdown {
width: 130px;
/* Adjust the width as needed */
}
.progress-overlay {
position: fixed;
top: 0;
left: 0;
width: 100%;
height: 100%;
background: rgba(0, 0, 0, 0.5);
display: flex;
align-items: center;
justify-content: center;
z-index: 1000;
}
.spinner {
border: 5px solid #f3f3f3;
border-top: 5px solid #3498db;
border-radius: 50%;
width: 50px;
height: 50px;
animation: spin 2s linear infinite;
}
@keyframes spin {
0% {
transform: rotate(0deg);
}
100% {
transform: rotate(360deg);
}
}
.error {
color: #ff0000;
}
/* Add space between sliders */
.slider-setting {
margin-right: 20px;
}
/* Adjust the thumb appearance for a more standard line slider look */
input[type="range"]::-webkit-slider-thumb,
input[type="range"]::-moz-range-thumb,
input[type="range"]::-ms-thumb {
-webkit-appearance: none;
appearance: none;
width: 16px;
/* Smaller width for a more standard thumb size */
height: 16px;
/* Smaller height for a more standard thumb size */
border-radius: 50%;
/* Keep it rounded, or set to 0 for square */
background: #3071a9;
cursor: pointer;
margin-top: -7px;
/* Adjust if needed to align thumb with the track */
}
</style>
</head>
<h1>AllTalk TTS Generator</h1>
<div class="form-container">
<div class="input-section">
<label for="text_input">Text Input:</label>
<textarea id="text_input" rows="8"></textarea> <!-- Removed cols attribute -->
</div>
<div class="settings-section">
<div class="setting voice-setting">
<label for="character_voice_gen">Character Voice:</label>
<select id="character_voice_gen"></select>
</div>
<div class="setting language-setting">
<label for="language">Language:</label>
<select id="language" name="language" class="fixed-size-dropdown">
<option value="ar">Arabic</option>
<option value="zh-cn">Chinese (Simplified)</option>
<option value="cs">Czech</option>
<option value="nl">Dutch</option>
<option value="en" selected>English</option>
<option value="fr">French</option>
<option value="de">German</option>
<option value="hu">Hungarian</option>
<option value="it">Italian</option>
<option value="ja">Japanese</option>
<option value="ko">Korean</option>
<option value="pl">Polish</option>
<option value="pt">Portuguese</option>
<option value="ru">Russian</option>
<option value="es">Spanish</option>
<option value="tr">Turkish</option>
</select>
</div>
<div class="setting play-mode-setting">
<label for="playMode">Playback:</label>
<select id="playMode" name="playMode" class="fixed-size-dropdown">
<option value="local" selected>In Browser</option>
<option value="remote">On Server</option>
</select>
</div>
<div class="setting theme-mode-setting">
<label for="themeMode">Theme:</label>
<select id="themeMode" name="themeMode" class="fixed-size-dropdown">
<option value="light" selected>Light</option>
<option value="dark">Dark</option>
</select>
</div>
<div class="setting chunk-size-setting">
<label for="chunkSize">Sentence Chunk Size:</label>
<input type="range" id="chunkSize" name="chunkSize" class="slider-setting" min="1" max="10" value="4"
oninput="updateChunkSizeValue(this.value)">
<span id="chunkSizeValue">3</span> <!-- Display the value of the slider -->
</div>
<div class="setting volume-setting">
<label for="volume">Volume:</label>
<input type="range" id="volume" name="volume" class="slider-setting" min="0.1" max="1.0" step="0.1"
value="0.8" oninput="updateVolumeValue(this.value)">
<span id="volumeValue">0.8</span>
</div>
</div>
<div class="action-buttons">
<button onclick="generateTTS()">Generate TTS</button>
<button onclick="stopPlayback()">Stop Playback</button>
<button onclick="previewVoice()">Preview Voice</button>
</div>
<div id="progressIndicator" class="progress-overlay" style="display: none;">
<div class="spinner"></div>
</div>
</div>
<script>
async function fetchVoices() {
try {
const response = await fetch("http://127.0.0.1:7851/api/voices");
const result = await response.json();
const characterVoiceSelect = document.getElementById("character_voice_gen");
result.voices.forEach(voice => {
const option = document.createElement("option");
option.value = voice;
option.text = voice;
characterVoiceSelect.add(option);
});
} catch (error) {
console.error(error);
}
}
function updateVolumeValue(value) {
document.getElementById("volumeValue").innerText = value;
// Set the volume for browser playback
audioElement.volume = parseFloat(value);
}
let stopTTSGeneration = false;
let audioQueue = [];
let currentlyPlaying = false;
const audioElement = new Audio();
async function generateTTS() {
try {
// Reset the flag at the start
stopTTSGeneration = false;
const textInput = document.getElementById("text_input").value;
const language = document.getElementById("language").value;
const characterVoice = document.getElementById("character_voice_gen").value;
const volume = parseFloat(document.getElementById("volume").value);
const chunkSize = parseInt(document.getElementById("chunkSize").value);
// Retrieve the value of the selected option for playMode from the dropdown
const playModeValue = document.getElementById("playMode").value;
// Clean up the textInput
let cleanedText = textInput.replace(/ \- | \– /g, ' '); // Replaces " - " and " – " with " "
cleanedText = cleanedText.replace(/\.\s'/g, ".'"); // Correctly remove space between a period and a single quote
cleanedText = cleanedText.replace(/'/g, ""); // Remove all single quotes
// Split text into chunks based on the specified number of sentences
const sentences = cleanedText.match(/("[^"]+"|[^.!?]+[.!?]+)/g);
const chunks = [];
for (let i = 0; i < sentences.length; i += chunkSize) {
let chunk = sentences.slice(i, i + chunkSize).map(s => s.trim()).join(' ');
chunks.push(chunk);
}
// Use playModeValue for the conditional check
if (playModeValue === "remote") {
for (const chunk of chunks) {
if (stopTTSGeneration) break; // Stop if flag is set
await generateAndPlayRemote(chunk, language, characterVoice, volume);
}
} else if (playModeValue === "local") {
generateAndPlayLocal(chunks, 0);
}
} catch (error) {
console.error("Error in generateTTS:", error);
}
}
async function generateAndPlayLocal(chunks, index) {
if (index >= chunks.length || stopTTSGeneration) {
return; // Stop if no more chunks or flag is set
}
if (index >= chunks.length) {
// No more chunks to process
return;
}
try {
const chunk = chunks[index];
const data = new URLSearchParams({
text_input: chunk.trim(),
text_filtering: "standard",
character_voice_gen: document.getElementById("character_voice_gen").value,
narrator_enabled: "false",
narrator_voice_gen: document.getElementById("character_voice_gen").value,
text_not_inside: "character",
language: document.getElementById("language").value,
output_file_name: "myoutputfile",
output_file_timestamp: "true",
autoplay: "false",
autoplay_volume: document.getElementById("volume").value
});
const response = await fetch("http://127.0.0.1:7851/api/tts-generate", {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
},
body: data,
});
const result = await response.json();
if (result.output_file_url) {
audioQueue.push(result.output_file_url);
if (!currentlyPlaying) {
playNextLocalChunk();
}
}
// Recursively call the function with the next index
generateAndPlayLocal(chunks, index + 1);
} catch (error) {
console.error("Error in generateAndPlayLocal with chunk index", index, ":", error);
}
}
async function generateAndPlayRemote(chunk, language, characterVoice, volume) {
try {
if (stopTTSGeneration) {
console.log("TTS generation stopped.");
return; // Early return if stop flag is set
}
const data = new URLSearchParams({
text_input: chunk.trim(),
text_filtering: "standard",
character_voice_gen: characterVoice,
narrator_enabled: "false",
narrator_voice_gen: characterVoice,
text_not_inside: "character",
language: language,
output_file_name: "myoutputfile",
output_file_timestamp: "true",
autoplay: "true",
autoplay_volume: volume.toString(),
});
const response = await fetch("http://127.0.0.1:7851/api/tts-generate", {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
},
body: data,
});
const result = await response.json();
if (!response.ok) {
throw new Error(`API error: ${result.detail || 'Unknown error'}`);
}
console.log("TTS generation complete for chunk:", chunk);
} catch (error) {
console.error("Error in generateAndPlayRemote:", error);
}
}
async function previewVoice() {
const characterVoice = document.getElementById("character_voice_gen").value;
try {
const formData = new FormData();
formData.append("voice", characterVoice);
const response = await fetch("http://127.0.0.1:7851/api/previewvoice/", {
method: "POST",
body: formData,
});
const result = await response.json();
if (result.status === "generate-success") {
// Play the preview audio
const audioPreview = new Audio(result.output_file_url);
audioPreview.play();
} else {
throw new Error('Preview generation failed');
}
} catch (error) {
console.error('Error in previewVoice:', error);
}
}
function playNextLocalChunk() {
if (audioQueue.length > 0) {
currentlyPlaying = true;
const audioUrl = audioQueue.shift();
audioElement.src = audioUrl;
audioElement.play();
audioElement.onended = () => {
currentlyPlaying = false;
playNextLocalChunk();
};
}
}
function toggleThemeMode() {
// Retrieve the value of the selected option for themeMode from the dropdown
const themeModeValue = document.getElementById('themeMode').value;
if (themeModeValue === "dark") {
document.body.classList.add('dark-mode');
localStorage.setItem('darkMode', 'enabled'); // Assuming saveDarkModePreference sets this
} else {
document.body.classList.remove('dark-mode');
localStorage.setItem('darkMode', 'disabled'); // Assuming saveDarkModePreference sets this
}
}
function updateChunkSizeValue(value) {
document.getElementById("chunkSizeValue").innerText = value;
}
// Optionally, save the dark mode preference in localStorage
function saveDarkModePreference(isDarkMode) {
localStorage.setItem('darkMode', isDarkMode ? 'enabled' : 'disabled');
}
function stopPlayback() {
// Stop current audio playback
audioElement.pause();
audioElement.currentTime = 0;
// Clear the audio queue, reset playback flag, and stop further TTS generation
audioQueue = [];
currentlyPlaying = false;
stopTTSGeneration = true; // Set flag to stop TTS generation
}
function showProgress() {
document.getElementById('progressIndicator').style.display = 'flex';
}
function hideProgress() {
document.getElementById('progressIndicator').style.display = 'none';
}
function displayOutput(result) {
const outputContainer = document.getElementById("output");
outputContainer.innerHTML = `<p>Status: ${result.status}</p>`;
if (result.output_file_path) {
outputContainer.innerHTML += `<p>File Path: ${result.output_file_path}</p>`;
}
if (result.output_file_url) {
outputContainer.innerHTML += `<p>File URL: ${result.output_file_url}</p>`;
}
if (result.error) {
outputContainer.innerHTML += `<p class="error">Error: ${result.error}</p>`;
}
}
window.onload = function() {
fetchVoices();
const isDarkModeEnabled = localStorage.getItem('darkMode') === 'enabled';
const themeModeSelect = document.getElementById('themeMode');
if (isDarkModeEnabled) {
document.body.classList.add('dark-mode');
themeModeSelect.value = 'dark';
} else {
themeModeSelect.value = 'light';
}
// Add change event listener to the themeMode dropdown
themeModeSelect.addEventListener('change', toggleThemeMode);
// Initialize other elements if necessary
};
</script>
</body>
</html>
templates/admin.html
+1125 -666
View File
File diff suppressed because it is too large. Load diff
tts_server.py
+131 -81
View File
@@ -18,8 +18,10 @@ from fastapi import (
Request,
Response,
Depends,
HTTPException,
)
from fastapi.responses import JSONResponse, HTMLResponse, RedirectResponse, FileResponse, StreamingResponse
from fastapi.middleware.cors import CORSMiddleware
from fastapi.templating import Jinja2Templates
from fastapi.staticfiles import StaticFiles
from contextlib import asynccontextmanager
@@ -120,7 +122,14 @@ async def startup_shutdown(no_actual_value_it_demanded_something_be_here):
# Create FastAPI app with lifespan
app = FastAPI(lifespan=startup_shutdown)
# Allow all origins, and set other CORS options
app.add_middleware(
CORSMiddleware,
allow_origins=["*"], # Set this to the specific origins you want to allow
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
#####################################
#### MODEL LOADING AND UNLOADING ####
@@ -480,29 +489,32 @@ async def generate_audio_internal(text, voice, language, output_file, streaming)
sound_norm_refs=model.config.sound_norm_refs,
)
# If the user has requested streaming, then use the TTS.inference_stream function,
# otherwise use TTS.inference
inference_func = model.inference_stream if streaming else model.inference
output = inference_func(
text,
language,
gpt_cond_latent=gpt_cond_latent,
speaker_embedding=speaker_embedding,
stream_chunk_size=20 if streaming else None,
temperature=float(params["local_temperature"]),
length_penalty=float(model.config.length_penalty),
repetition_penalty=float(params["local_repetition_penalty"]),
top_k=int(model.config.top_k), # Convert to int if necessary
top_p=float(model.config.top_p),
enable_text_splitting=True,
)
# Common arguments for both functions
common_args = {
"text": text,
"language": language,
"gpt_cond_latent": gpt_cond_latent,
"speaker_embedding": speaker_embedding,
"temperature": float(params["local_temperature"]),
"length_penalty": float(model.config.length_penalty),
"repetition_penalty": float(params["local_repetition_penalty"]),
"top_k": int(model.config.top_k),
"top_p": float(model.config.top_p),
"enable_text_splitting": True
}
# Determine the correct inference function and add streaming specific argument if needed
inference_func = model.inference_stream if streaming else model.inference
if streaming:
# If streaming, then yield each chunk of audio as it is generated. We also collect each
# chunk in a list so that we can save the full audio to a file once the stream is finished.
common_args["stream_chunk_size"] = 20
# Call the appropriate function
output = inference_func(**common_args)
# Process the output based on streaming or non-streaming
if streaming:
# Streaming-specific operations
file_chunks = []
# Create and yield a wav file with no data to start the stream
wav_buf = io.BytesIO()
with wave.open(wav_buf, "wb") as vfout:
vfout.setnchannels(1)
@@ -512,23 +524,19 @@ async def generate_audio_internal(text, voice, language, output_file, streaming)
wav_buf.seek(0)
yield wav_buf.read()
# Yield each chunk of audio as it is generated
for i, chunk in enumerate(output):
# Append the chunk to later save to a file
file_chunks.append(chunk)
# Process the wav data
if isinstance(chunk, list):
chunk = torch.cat(chunk, dim=0)
chunk = chunk.clone().detach().cpu().numpy()
chunk = chunk[None, : int(chunk.shape[0])]
chunk = np.clip(chunk, -1, 1)
chunk = (chunk * 32767).astype(np.int16)
# Then yield the chunk to be sent to the client
yield chunk.tobytes()
else:
# If not streaming, then save to the file on disk
# Non-streaming-specific operation
torchaudio.save(output_file, torch.tensor(output["wav"]).unsqueeze(0), 24000)
# API LOCAL Methods
elif params["tts_method_api_local"]:
@@ -586,7 +594,7 @@ async def generate(request: Request):
voice = data["voice"]
language = data["language"]
output_file = data["output_file"]
streaming = data["streaming"] or False
streaming = False
# Generation logic
response = await generate_audio(text, voice, language, output_file, streaming)
if streaming:
@@ -727,6 +735,22 @@ async def get_audio(filename: str):
audio_path = this_dir / "outputs" / filename
return FileResponse(audio_path)
@app.get("/audiocache/{filename}")
async def get_audio(filename: str):
audio_path = Path("outputs") / filename
if not audio_path.is_file():
raise HTTPException(status_code=404, detail="File not found")
response = FileResponse(
path=audio_path,
media_type='audio/wav',
filename=filename
)
# Set caching headers
response.headers["Cache-Control"] = "public, max-age=604800" # Cache for one week
response.headers["ETag"] = str(audio_path.stat().st_mtime) # Use the file's last modified time as a simple ETag
return response
#########################
#### VOICES LIST API ####
@@ -840,6 +864,45 @@ class TTSGenerator:
except ValidationError as e:
return str(e)
def process_text(text):
# Normalize HTML encoded quotes
text = html.unescape(text)
# Replace ellipsis with a single dot
text = re.sub(r'\.{3,}', '.', text)
# Pattern to identify combined narrator and character speech
combined_pattern = r'(\*[^*"]+\*|"[^"*]+")'
# List to hold parts of speech along with their type
ordered_parts = []
# Track the start of the next segment
start = 0
# Find all matches
for match in re.finditer(combined_pattern, text):
# Add the text before the match, if any, as ambiguous
if start < match.start():
ambiguous_text = text[start:match.start()].strip()
if ambiguous_text:
ordered_parts.append(('ambiguous', ambiguous_text))
# Add the matched part as either narrator or character
matched_text = match.group(0)
if matched_text.startswith('*') and matched_text.endswith('*'):
ordered_parts.append(('narrator', matched_text.strip('*').strip()))
elif matched_text.startswith('"') and matched_text.endswith('"'):
ordered_parts.append(('character', matched_text.strip('"').strip()))
else:
# In case of mixed or improperly formatted parts
if '*' in matched_text:
ordered_parts.append(('narrator', matched_text.strip('*').strip('"')))
else:
ordered_parts.append(('character', matched_text.strip('"').strip('*')))
# Update the start of the next segment
start = match.end()
# Add any remaining text after the last match as ambiguous
if start < len(text):
ambiguous_text = text[start:].strip()
if ambiguous_text:
ordered_parts.append(('ambiguous', ambiguous_text))
return ordered_parts
def standard_filtering(text_input):
text_output = (text_input
.replace("***", "")
@@ -850,20 +913,6 @@ def standard_filtering(text_input):
)
return text_output
def narrator_filtering(narrator_text_input):
processed_string = re.sub(r'\.\*\n\*', '. ', narrator_text_input)
processed_string = (
processed_string
.replace("\n", " ")
.replace('&quot;', '&quot;<')
.replace('"', '&quot;<')
)
processed_string = processed_string.replace('&quot;<. *', '&quot;< *"')
processed_string = processed_string.replace('< *"', '< *')
processed_string = processed_string.replace('. *', '< *')
text_output = html.unescape(processed_string)
return text_output
def combine(output_file_timestamp, output_file_name, audio_files):
audio = np.array([])
sample_rate = None
@@ -884,9 +933,11 @@ def combine(output_file_timestamp, output_file_name, audio_files):
timestamp = int(time.time())
output_file_path = os.path.join(this_dir / "outputs" / f'{output_file_name}_{timestamp}_combined.wav')
output_file_url = f'http://{params["ip_address"]}:{params["port_number"]}/audio/{output_file_name}_{timestamp}_combined.wav'
output_cache_url = f'http://{params["ip_address"]}:{params["port_number"]}/audiocache/{output_file_name}_{timestamp}_combined.wav'
else:
output_file_path = os.path.join(this_dir / "outputs" / f'{output_file_name}_combined.wav')
output_file_url = f'http://{params["ip_address"]}:{params["port_number"]}/audio/{output_file_name}.wav'
output_cache_url = f'http://{params["ip_address"]}:{params["port_number"]}/audiocache/{output_file_name}.wav'
try:
sf.write(output_file_path, audio, samplerate=sample_rate)
# Clean up unnecessary files
@@ -895,7 +946,7 @@ def combine(output_file_timestamp, output_file_name, audio_files):
except Exception as e:
# Handle exceptions (e.g., failed to write output file)
return None, None
return output_file_path, output_file_url
return output_file_path, output_file_url, output_cache_url
# Generation API (separate from text-generation-webui)
@app.post("/api/tts-generate", response_class=JSONResponse)
@@ -934,71 +985,70 @@ async def tts_generate(
else:
return JSONResponse(content={"error": JSONresult}, status_code=400)
if narrator_enabled:
#print("ORIGINAL UNTOUCHED STRING IS:", text_input,"\n")
if text_filtering in ["standard", "none"]:
cleaned_string = (
text_input
.replace('"', '"<')
.replace('*', '<*')
)
parts = re.split(r'(?<=<\*\s)|(?<=\.\s)|(?<=\."\<)|(?<=\."<)', cleaned_string)
parts = list(filter(lambda x: x.strip(), parts))
elif text_filtering == "html":
cleaned_string = standard_filtering(text_input)
cleaned_string = narrator_filtering(cleaned_string)
parts = re.split(r'&quot;|\.\*', cleaned_string)
processed_parts = process_text(text_input)
audio_files_all_paragraphs = []
audio_files_paragraph = []
for i, part in enumerate(parts):
if len(part.strip()) <= 1:
for part_type, part in processed_parts:
# Skip parts that are too short
if len(part.strip()) <= 3:
continue
# Figure out which type of line it is, then replace characters as necessary to avoid TTS trying to pronunce them, htmlunescape after.
# Character will always be a < with a letter immediately after it
if '<' in part and '<*' not in part and '< *' not in part and '< *' not in part and '< ' not in part and '< ' not in part:
cleaned_part = html.unescape(part.replace('<', ''))
voice_to_use = character_voice_gen
#Narrator will always be an * or < with an * a position or two after it.
elif '<*' in part or '< *' in part or '< *' in part or '*' in part:
cleaned_part = html.unescape(part.replace('<*', '').replace('< *', '').replace('< *', '').replace('*', '').replace('<. ', ''))
# Determine the voice to use based on the part type
if part_type == 'narrator':
voice_to_use = narrator_voice_gen
#If the other two dont capture it, aka, the AI gave no * or &quot; on the line, use non_quoted_text_is aka user interface, user can choose Char or Narrator
elif text_not_inside == "character":
cleaned_part = html.unescape(part.replace('< ', '').replace('< ', '').replace('< ', ''))
print(f"[{params['branding']}TTSGen] \033[92mNarrator\033[0m") # Green
elif part_type == 'character':
voice_to_use = character_voice_gen
elif text_not_inside == "narrator":
cleaned_part = html.unescape(part.replace('< ', '').replace('< ', '').replace('< ', ''))
voice_to_use = narrator_voice_gen
output_file = this_dir / "outputs" / f"{output_file_name}_{uuid.uuid4()}_{int(time.time())}_{i}.wav"
print(f"[{params['branding']}TTSGen] \033[36mCharacter\033[0m") # Yellow
else:
# Handle ambiguous parts based on user preference
voice_to_use = character_voice_gen if text_not_inside == "character" else narrator_voice_gen
voice_description = "\033[36mCharacter (Text-not-inside)\033[0m" if text_not_inside == "character" else "\033[92mNarrator (Text-not-inside)\033[0m"
print(f"[{params['branding']}TTSGen] {voice_description}")
# Replace multiple exclamation marks, question marks, or other punctuation with a single instance
cleaned_part = re.sub(r'([!?.])\1+', r'\1', part)
# Further clean to remove any other unwanted characters
cleaned_part = re.sub(r'[^a-zA-Z0-9\s\.,;:!?\-\'"]', '', cleaned_part)
# Remove all newline characters (single or multiple)
cleaned_part = re.sub(r'\n+', ' ', cleaned_part)
output_file = this_dir / "outputs" / f"{output_file_name}_{uuid.uuid4()}_{int(time.time())}.wav"
output_file_str = output_file.as_posix()
response = await generate_audio(cleaned_part, voice_to_use, language, output_file_str, streaming)
audio_path = output_file_str
audio_files_paragraph.append(audio_path)
# Accumulate audio files within the paragraph
audio_files_all_paragraphs.extend(audio_files_paragraph)
audio_files_all_paragraphs.append(audio_path)
# Combine audio files across paragraphs
output_file_path, output_file_url = combine(output_file_timestamp, output_file_name, audio_files_all_paragraphs)
output_file_path, output_file_url, output_cache_url = combine(output_file_timestamp, output_file_name, audio_files_all_paragraphs)
else:
if output_file_timestamp:
timestamp = int(time.time())
output_file_path = this_dir / "outputs" / f"{output_file_name}_{timestamp}.wav"
output_file_url = f'http://{params["ip_address"]}:{params["port_number"]}/audio/{output_file_name}_{timestamp}.wav'
output_cache_url = f'http://{params["ip_address"]}:{params["port_number"]}/audiocache/{output_file_name}_{timestamp}.wav'
else:
output_file_path = this_dir / "outputs" / f"{output_file_name}.wav"
output_file_url = f'http://{params["ip_address"]}:{params["port_number"]}/audio/{output_file_name}.wav'
output_cache_url = f'http://{params["ip_address"]}:{params["port_number"]}/audiocache/{output_file_name}_{timestamp}.wav'
if text_filtering == "html":
cleaned_string = html.unescape(standard_filtering(text_input))
cleaned_string = re.sub(r'([!?.])\1+', r'\1', text_input)
# Further clean to remove any other unwanted characters
cleaned_string = re.sub(r'[^a-zA-Z0-9\s\.,;:!?\-\'"]', '', cleaned_string)
# Remove all newline characters (single or multiple)
cleaned_string = re.sub(r'\n+', ' ', cleaned_string)
elif text_filtering == "standard":
cleaned_string = standard_filtering(text_input)
cleaned_string = re.sub(r'([!?.])\1+', r'\1', text_input)
# Further clean to remove any other unwanted characters
cleaned_string = re.sub(r'[^a-zA-Z0-9\s\.,;:!?\-\'"]', '', cleaned_string)
# Remove all newline characters (single or multiple)
cleaned_string = re.sub(r'\n+', ' ', cleaned_string)
else:
cleaned_string = text_input
response = await generate_audio(cleaned_string, character_voice_gen, language, output_file_path, streaming)
if sounddevice_installed == False or streaming == True:
autoplay = False
if autoplay:
play_audio(output_file_path, autoplay_volume)
play_audio(output_file_path, autoplay_volume)
if streaming:
return StreamingResponse(response, media_type="audio/wav")
return JSONResponse(content={"status": "generate-success", "output_file_path": str(output_file_path), "output_file_url": str(output_file_url)}, status_code=200)
return JSONResponse(content={"status": "generate-success", "output_file_path": str(output_file_path), "output_file_url": str(output_file_url), "output_cache_url": str(output_cache_url)}, status_code=200)
except Exception as e:
return JSONResponse(content={"status": "generate-failure", "error": "An error occurred"}, status_code=500)