New API endpoint

adds the /api/tts-generation endpoint & documentation.
This commit is contained in:
erew123 authored and GitHub committed 2023-12-21 11:32:59 +00:00
1 parent 28a62ed39a
commit 8f318892c2
1 file changed
+320 -9
+320 -9
View File
@@ -22,6 +22,7 @@ from fastapi.templating import Jinja2Templates
from fastapi.staticfiles import StaticFiles
from jinja2 import Template
from contextlib import asynccontextmanager
from pydantic import field_validator
###########################
#### STARTUP VARIABLES ####
@@ -617,6 +618,218 @@ async def get_audio(filename: str):
audio_path = this_dir / "outputs" / filename
return FileResponse(audio_path)
########################
#### GENERATION API ####
########################
import html
import re
import numpy as np
import soundfile as sf
from typing import Union, Dict
from pydantic import BaseModel, ValidationError, Field
class Request(BaseModel):
# Define the structure of the 'Request' class if needed
pass
class JSONInput(BaseModel):
text_input: str = Field(..., max_length=1000, description="text_input needs to be 1000 characters or less.")
text_filtering: str = Field(..., pattern="^(none|standard|html)$", description="text_filtering needs to be 'none', 'standard' or 'html'.")
character_voice_gen: str = Field(..., pattern="^.*\.wav$", description="character_voice_gen needs to be the name of a wav file e.g. mysample.wav.")
narrator_enabled: bool = Field(..., description="narrator_enabled needs to be true or false.")
narrator_voice_gen: str = Field(..., pattern="^.*\.wav$", description="narrator_voice_gen needs to be the name of a wav file e.g. mysample.wav.")
text_not_inside: str = Field(..., pattern="^(character|narrator)$", description="text_not_inside needs to be 'character' or 'narrator'.")
language: str = Field(..., pattern="^(ar|zh-cn|cs|nl|en|fr|de|hu|it|ja|ko|pl|pt|ru|es|tr)$", description="language needs to be one of the following ar|zh-cn|cs|nl|en|fr|de|hu|it|ja|ko|pl|pt|ru|es|tr.")
output_file_name: str = Field(..., pattern="^[a-zA-Z0-9_]+$", description="output_file_name needs to be the name without any special characters or file extension e.g. 'filename'")
output_file_timestamp: bool = Field(..., description="output_file_timestamp needs to be true or false.")
autoplay: bool = Field(..., description="autoplay needs to be a true or false value.")
autoplay_volume: float = Field(..., ge=0.1, le=1.0, description="autoplay_volume needs to be from 0.1 to 1.0")
@field_validator("autoplay_volume")
@classmethod
def validate_autoplay_volume(cls, value):
if not (0.1 <= value <= 1.0):
raise ValidationError("Autoplay volume must be between 0.1 and 1.0")
return value
class TTSGenerator:
@staticmethod
def validate_json_input(json_data: Union[Dict, str]) -> Union[None, str]:
try:
if isinstance(json_data, str):
json_data = json.loads(json_data)
JSONInput(**json_data)
return None # JSON is valid
except ValidationError as e:
return str(e)
def standard_filtering(text_input):
text_output = (text_input
.replace("***", "")
.replace("**", "")
.replace("*", "")
.replace("\n\n", "\n")
.replace("&#x27;", "'")
)
return text_output
def narrator_filtering(narrator_text_input):
processed_string = re.sub(r'\.\*\n\*', '. ', narrator_text_input)
processed_string = (
processed_string
.replace("\n", " ")
.replace('&quot;', '&quot;<')
.replace('"', '&quot;<')
)
processed_string = processed_string.replace('&quot;<. *', '&quot;< *"')
processed_string = processed_string.replace('< *"', '< *')
processed_string = processed_string.replace('. *', '< *')
text_output = html.unescape(processed_string)
return text_output
def combine(output_file_timestamp, output_file_name, audio_files):
audio = np.array([])
sample_rate = None
try:
for audio_file in audio_files:
audio_data, current_sample_rate = sf.read(audio_file)
if audio.size == 0:
audio = audio_data
sample_rate = current_sample_rate
elif sample_rate == current_sample_rate:
audio = np.concatenate((audio, audio_data))
else:
raise ValueError("Sample rates of input files are not consistent.")
except Exception as e:
# Handle exceptions (e.g., file not found, invalid audio format)
return None, None
if output_file_timestamp:
timestamp = int(time.time())
output_file_path = os.path.join(this_dir / "outputs" / f'{output_file_name}_{timestamp}_combined.wav')
output_file_url = f'http://{params["ip_address"]}:{params["port_number"]}/audio/{output_file_name}_{timestamp}_combined.wav'
else:
output_file_path = os.path.join(this_dir / "outputs" / f'{output_file_name}_combined.wav')
output_file_url = f'http://{params["ip_address"]}:{params["port_number"]}/audio/{output_file_name}.wav'
try:
sf.write(output_file_path, audio, samplerate=sample_rate)
# Clean up unnecessary files
for audio_file in audio_files:
os.remove(audio_file)
except Exception as e:
# Handle exceptions (e.g., failed to write output file)
return None, None
return output_file_path, output_file_url
# Generation API (separate from text-generation-webui)
@app.post("/api/tts-generate", response_class=JSONResponse)
async def tts_generate(
text_input: str = Form(...),
text_filtering: str = Form(...),
character_voice_gen: str = Form(...),
narrator_enabled: bool = Form(...),
narrator_voice_gen: str = Form(...),
text_not_inside: str = Form(...),
language: str = Form(...),
output_file_name: str = Form(...),
output_file_timestamp: bool = Form(...),
autoplay: bool = Form(...),
autoplay_volume: float = Form(...),
):
try:
#print(f"text_filtering: {text_filtering}")
#print(f"narrator_enabled: {narrator_enabled}")
#print(f"text_not_inside: {text_not_inside}")
#print(f"output_file_timestamp: {output_file_timestamp}")
json_input_data = {
"text_input": text_input,
"text_filtering": text_filtering,
"character_voice_gen": character_voice_gen,
"narrator_enabled": narrator_enabled,
"narrator_voice_gen": narrator_voice_gen,
"text_not_inside": text_not_inside,
"language": language,
"output_file_name": output_file_name,
"output_file_timestamp": output_file_timestamp,
"autoplay": autoplay,
"autoplay_volume": autoplay_volume,
}
JSONresult = TTSGenerator.validate_json_input(json_input_data)
if JSONresult is None:
pass
else:
return JSONResponse(content={"error": JSONresult}, status_code=400)
if narrator_enabled:
print("ORIGINAL UNTOUCHED STRING IS:", text_input,"\n")
if text_filtering in ["standard", "none"]:
cleaned_string = (
text_input
.replace('"', '"<')
.replace('*', '<*')
)
print("STANDARD FILTERING IS NOW:", cleaned_string,"\n")
#parts = re.split(r'\.(?<!<[*"])', cleaned_string)
parts = re.split(r'(?<=<\*\s)|(?<=\.\s)|(?<=\."\<)|(?<=\."<)', cleaned_string)
parts = list(filter(lambda x: x.strip(), parts))
elif text_filtering == "html":
cleaned_string = standard_filtering(text_input)
cleaned_string = narrator_filtering(cleaned_string)
print("HTML FILTERING IS NOW:", cleaned_string,"\n")
parts = re.split(r'&quot;|\.\*', cleaned_string)
audio_files_all_paragraphs = []
audio_files_paragraph = []
for i, part in enumerate(parts):
if len(part.strip()) <= 1:
continue
print("THIS IS A PART", part)
# Figure out which type of line it is, then replace characters as necessary to avoid TTS trying to pronunce them, htmlunescape after.
# Character will always be a < with a letter immediately after it
if '<' in part and '<*' not in part and '< *' not in part and '< *' not in part and '< ' not in part and '< ' not in part:
print("IF - Character\n")
cleaned_part = html.unescape(part.replace('<', ''))
voice_to_use = character_voice_gen
#Narrator will always be an * or < with an * a position or two after it.
elif '<*' in part or '< *' in part or '< *' in part or '*' in part:
print("IF - Narrator\n")
cleaned_part = html.unescape(part.replace('<*', '').replace('< *', '').replace('< *', '').replace('*', '').replace('<. ', ''))
voice_to_use = narrator_voice_gen
#If the other two dont capture it, aka, the AI gave no * or &quot; on the line, use non_quoted_text_is aka user interface, user can choose Char or Narrator
elif text_not_inside == "character":
print("ELSE - CHARACTER\n")
cleaned_part = html.unescape(part.replace('< ', '').replace('< ', '').replace('< ', ''))
voice_to_use = character_voice_gen
elif text_not_inside == "narrator":
print("ELSE - NARRATOR\n")
cleaned_part = html.unescape(part.replace('< ', '').replace('< ', '').replace('< ', ''))
voice_to_use = narrator_voice_gen
output_file = this_dir / "outputs" / f"{output_file_name}_{int(time.time())}_{i}.wav"
output_file_str = output_file.as_posix()
await generate_audio(cleaned_part, voice_to_use, language, output_file_str)
audio_path = output_file_str
audio_files_paragraph.append(audio_path)
# Accumulate audio files within the paragraph
audio_files_all_paragraphs.extend(audio_files_paragraph)
# Combine audio files across paragraphs
output_file_path, output_file_url = combine(output_file_timestamp, output_file_name, audio_files_all_paragraphs)
else:
if output_file_timestamp:
timestamp = int(time.time())
output_file_path = this_dir / "outputs" / f"{output_file_name}_{timestamp}.wav"
output_file_url = f'http://{params["ip_address"]}:{params["port_number"]}/audio/{output_file_name}_{timestamp}.wav'
else:
output_file_path = this_dir / "outputs" / f"{output_file_name}.wav"
output_file_url = f'http://{params["ip_address"]}:{params["port_number"]}/audio/{output_file_name}.wav'
if text_filtering == "html":
cleaned_string = html.unescape(standard_filtering(text_input))
elif text_filtering == "standard":
cleaned_string = standard_filtering(text_input)
else:
cleaned_string = text_input
await generate_audio(cleaned_string, character_voice_gen, language, output_file_path)
return JSONResponse(content={"status": "generate-success", "output_file_path": str(output_file_path), "output_file_url": str(output_file_url)}, status_code=200)
except Exception as e:
return JSONResponse(content={"status": "generate-failure", "error": "An error occurred"}, status_code=500)
#############################################################
#### DOCUMENTATION - README ETC - PRESENTED AS A WEBPAGE ####
#############################################################
@@ -1258,24 +1471,122 @@ simple_webpage = """
</table>
<h2 id="curl-commands"><strong>JSON calls &amp; CURL Commands</strong></h2>
<h4>Generating TTS - Linux</h4>
<p style="padding-left: 30px;"><span style="color: #3366ff;">curl -X POST -H "Content-Type: application/json" -d '{"text": "<span style="color: #ff9900;">This is text to generate as TTS</span>","voice": "<span style="color: #ff9900;">female_01.wav</span>", "language": "<span style="color: #ff9900;">en</span>", "output_file": "<span style="color: #ff9900;">outputfile.wav</span>"}' "http://127.0.0.1:7851/api/generate"</span></p>
<h4>Generating TTS - Windows</h4>
<p style="padding-left: 30px;"><span style="color: #3366ff;">curl -X POST -H "Content-Type: application/json" -d "{&bsol;"text&bsol;": &bsol;"<span style="color: #ff9900;">This is text to generate as TTS</span>&bsol;", &bsol;"voice&bsol;": &bsol;"<span style="color: #ff9900;">female_01.wav</span>&bsol;", &bsol;"language&bsol;": &bsol;"<span style="color: #ff9900;">en</span>&bsol;", &bsol;"output_file&bsol;": &bsol;"<span style="color: #ff9900;">outputfile.wav</span>&bsol;"}" http://127.0.0.1:7851/api/generate</span></p>
<h4>Generating TTS - Notes</h4>
<p style="padding-left: 30px;">Replace <span style="color: #ff9900;">This is text to generate as TTS</span> with whatever you want it to say. <span style="color: #ff9900;">female_01.wav</span> with the voice sample you want to use. <span style="color: #ff9900;">output_file.wav</span> with the file name you want it to create.</p>
<p style="padding-left: 30px;">JSON return <span style="color: #339966;">{"status":"generate-success","data":{"audio_path":"outputfile.wav"}}</span></p>
<h3>Overview</h3>
<p style="margin-left: 40px; text-align: justify;">The Text-to-Speech (TTS) Generation API allows you to generate speech from text input using various configuration options. This API supports both character and narrator voices, providing flexibility for creating dynamic and engaging audio content.</p>
<h3>TTS Generation Endpoint</h3>
<ul>
<li><strong>URL</strong>: <span style="color: #3366ff;">http://127.0.0.1:7851/api/tts-generate</span></li>
<li><strong>Method</strong>: <span style="color: #3366ff;">POST</span></li>
<li><strong>Content-Type</strong>: <span style="color: #3366ff;">application/x-www-form-urlencoded</span></li>
</ul>
<h3>Example command line</h3>
<p style="padding-left: 30px;">Standard TTS speech Example (standard text) generating a time-stamped file</p>
<p style="padding-left: 30px;"><span style="color: #3366ff;">curl -X POST "http://127.0.0.1:7851/api/tts-generate" -d "text_input=</span><span style="color: #ff9900;">All of this is text spoken by the character. This is text not inside quotes, though that doesnt matter in the slightest</span><span style="color: #3366ff;">" -d "text_filtering=</span><span style="color: #339966;">standard</span><span style="color: #3366ff;">" -d "character_voice_gen=</span><span style="color: #ff9900;">female_01.wav</span><span style="color: #3366ff;">" -d "narrator_enabled=</span><span style="color: #339966;">false</span><span style="color: #3366ff;">" -d "narrator_voice_gen=male_01.wav" -d "text_not_inside=character" -d "language=en" -d "output_file_name=</span><span style="color: #ff9900;">myoutputfile</span><span style="color: #3366ff;">" -d "output_file_timestamp=</span><span style="color: #ff9900;">true</span><span style="color: #3366ff;">" -d "autoplay=true" -d "autoplay_volume=0.8"</span></p>
<p style="padding-left: 30px;">Narrator Example (standard text)&nbsp;generating a time-stamped file</p>
<p style="padding-left: 30px;"><span style="color: #3366ff;">curl -X POST "http://127.0.0.1:7851/api/tts-generate" -d "text_input=</span><span style="color: #ff9900;">*This is text spoken by the narrator* &bsol;"This is text spoken by the character&bsol;". This is text not inside quotes.</span><span style="color: #3366ff;">" -d "text_filtering=</span><span style="color: #339966;">standard</span><span style="color: #3366ff;">" -d "character_voice_gen=</span><span style="color: #ff9900;">female_01.wav</span><span style="color: #3366ff;">" -d "narrator_enabled=</span><span style="color: #339966;">true</span><span style="color: #3366ff;">" -d "narrator_voice_gen=</span><span style="color: #339966;">male_01.wav</span><span style="color: #3366ff;">" -d "text_not_inside=</span><span style="color: #339966;">character</span><span style="color: #3366ff;">" -d "language=en" -d "output_file_name=</span><span style="color: #ff9900;">myoutputfile</span><span style="color: #3366ff;">" -d "output_file_timestamp=</span><span style="color: #ff9900;">true</span><span style="color: #3366ff;">" -d "autoplay=true" -d "autoplay_volume=0.8"</span></p>
<p style="padding-left: 30px; text-align: justify;"><strong><span style="color: #ff0000;">Note</span></strong> that if your text that needs to be generated contains double quotes you will need to escape them with&nbsp;&bsol;" (Please see the narrator example).</p>
<h3>&nbsp;Request Parameters</h3>
<ul>
<li>
<p style="text-align: justify;"><strong>text_input</strong>: The text you want the TTS engine to produce. Use escaped double quotes for character speech and asterisks for narrator speech if using the narrator function. Example:</p>
<p>-<span style="color: #3366ff;">d "text_input=*This is text spoken by the narrator* &bsol;"This is text spoken by the character&bsol;". This is text not inside quotes." </span></p>
</li>
<li>
<p><strong>text_filtering</strong>: Filter for text. Options:</p>
<ul>
<li style="text-align: justify;"><span style="color: #3366ff;">none</span>&nbsp;No filtering. Whatever is sent will go over to the TTS engine as raw text, which may result in some odd sounds with some special characters.</li>
<li><span style="color: #3366ff;">standard</span>&nbsp;Human-readable text and a basic level of filtering, just to clean up some special characters.</li>
<li><span style="color: #3366ff;">html</span>&nbsp;HTML content. Where you are using HTML entity's like &amp;quot;</li>
</ul>
<p><span style="color: #3366ff;">-d "text_filtering=none" </span><br /><span style="color: #3366ff;">-d "text_filtering=standard" </span><br /><span style="color: #3366ff;">-d "text_filtering=html"</span></p>
<p><strong>Example:</strong></p>
<ul>
<li>Standard Example: <span style="color: #3366ff;">*This is text spoken by the narrator* "This is text spoken by the character" This is text not inside quotes.</span></li>
<li>HTML Example:<span style="color: #3366ff;"> *This is text spoken by the narrator*&nbsp;&amp;quot;This is text spoken by the character&amp;quot;&nbsp;This is text not inside quotes.</span></li>
<li>None will just pass whatever characters/text you send at it.</li>
</ul>
</li>
<li>
<p><strong>character_voice_gen</strong>: The WAV file name for the character's voice.</p>
<p><span style="color: #3366ff;">-d "character_voice_gen=female_01.wav" </span></p>
</li>
<li>
<p><strong>narrator_enabled</strong>: Enable or disable the narrator function. If true, minimum text filtering is set to standard. Anything between double quotes is considered the character's speech, and anything between asterisks is considered the narrator's speech.</p>
<p><span style="color: #3366ff;">-d "narrator_enabled=true"<br />-d "narrator_enabled=false" </span><span style="font-family: Verdana, Arial, Helvetica, sans-serif;">&nbsp;</span></p>
</li>
<li>
<p><strong>narrator_voice_gen</strong>: The WAV file name for the narrator's voice.</p>
<p><span style="color: #3366ff;">-d "narrator_voice_gen=male_01.wav"</span></p>
</li>
<li>
<p><strong>text_not_inside</strong>: Specify the handling of lines not inside double quotes or asterisks, for the narrator feature. Options:</p>
<ul>
<li><span style="color: #3366ff;">character</span>: Treat as character speech.</li>
<li><span style="color: #3366ff;">narrator</span>: Treat as narrator speech.</li>
</ul>
<p><span style="color: #3366ff;">-d "text_not_inside=character" </span><br /><span style="color: #3366ff;">-d "text_not_inside=narrator"</span></p>
</li>
<li>
<p><strong>language</strong>: Choose the language for TTS. Options:</p>
<ul>
<li><span style="color: #3366ff;">ar</span>&nbsp;Arabic</li>
<li><span style="color: #3366ff;">zh-cn</span>&nbsp;Chinese (Simplified)</li>
<li><span style="color: #3366ff;">cs</span>&nbsp;Czech</li>
<li><span style="color: #3366ff;">nl</span> Dutch</li>
<li><span style="color: #3366ff;">en</span> English</li>
<li><span style="color: #3366ff;">fr</span> French</li>
<li><span style="color: #3366ff;">de</span> German</li>
<li><span style="color: #3366ff;">hu</span> Hungarian</li>
<li><span style="color: #3366ff;">it</span> Italian</li>
<li><span style="color: #3366ff;">ja</span> Japanese</li>
<li><span style="color: #3366ff;">ko</span> Korean</li>
<li><span style="color: #3366ff;">pl</span> Polish</li>
<li><span style="color: #3366ff;">pt</span> Portuguese</li>
<li><span style="color: #3366ff;">ru </span>Russian</li>
<li><span style="color: #3366ff;">es</span> Spanish</li>
<li><span style="color: #3366ff;">tr</span> Turkish</li>
</ul>
<p><span style="color: #3366ff;">-d "language=en"</span></p>
</li>
<li>
<p><strong>output_file_name</strong>: The name of the output file (excluding the .wav extension).</p>
<p><span style="color: #3366ff;">-d "output_file_name=myoutputfile" </span></p>
</li>
<li>
<p><strong>output_file_timestamp</strong>: Add a timestamp to the output file name. If true, each file will have a unique timestamp; otherwise, the same file name will be overwritten each time you generate TTS.</p>
<p><span style="color: #3366ff;">-d "output_file_timestamp=true" <br />-d "output_file_timestamp=false" </span></p>
</li>
<li>
<p><strong>autoplay</strong>: <span style="color: #ff0000;">Feature not yet available</span>. Enable or disable autoplay. Still needs to be specified in the JSON request.</p>
<p><span style="color: #3366ff;">-d "autoplay=true" <br />-d "autoplay=false"</span></p>
</li>
<li>
<p><strong>autoplay_volume</strong>: <span style="color: #ff0000;">Feature not yet available</span>. Set the autoplay volume. Should be between 0.1 and 1.0.&nbsp;Still needs to be specified in the JSON request.</p>
<p><span style="color: #3366ff;">-d "autoplay_volume=0.8" </span></p>
</li>
</ul>
<h3>TTS Generation Response</h3>
<p style="padding-left: 30px;">The API returns a JSON object with the following properties:</p>
<ul>
<li><span style="color: #3366ff;"><strong>status</strong></span>&nbsp;Indicates whether the generation was successful (<span style="color: #3366ff;">generate-success</span>) or failed (<span style="color: #3366ff;">generate-failure</span>).</li>
<li><span style="color: #3366ff;"><strong>output_file_path</strong></span>&nbsp;The on-disk location of the generated WAV file.</li>
<li><span style="color: #3366ff;"><strong>output_file_url</strong></span>&nbsp;The HTTP location for accessing the generated WAV file.</li>
</ul>
<p><strong>Example JSON TTS Generation Response:</strong></p>
<p style="padding-left: 30px;"><span style="color: #339966;">{"status": "generate-success", "output_file_path": "C:\\text-generation-webui\\extensions\\alltalk_tts\\outputs\\myoutputfile_1703149973.wav", "output_file_url": "http://127.0.0.1:7851/audio/myoutputfile_1703149973.wav"}</span></p>
<h4>Switching Model</h4>
<p style="padding-left: 30px;"><span style="color: #3366ff;">curl -X POST "http://127.0.0.1:7851/api/reload?tts_method=API%20Local"</span><br /><span style="color: #3366ff;"> curl -X POST "http://127.0.0.1:7851/api/reload?tts_method=API%20TTS"</span><br /><span style="color: #3366ff;"> curl -X POST "http://127.0.0.1:7851/api/reload?tts_method=XTTSv2%20Local"</span></p>
<p style="padding-left: 30px;">Switch between the 3 models respectively.</p>
<p style="padding-left: 30px;">JSON return <span style="color: #339966;">{"status": "model-success"}</span></p>
<h4>Switch DeepSpeed</h4>
<p style="padding-left: 30px;"><span style="color: #3366ff;">curl -X POST -H "Content-Type: application/json" "http://127.0.0.1:7851/api/deepspeed?new_deepspeed_value=True"</span></p>
<p style="padding-left: 30px;"><span style="color: #3366ff;">curl -X POST "http://127.0.0.1:7851/api/deepspeed?new_deepspeed_value=True"</span></p>
<p style="padding-left: 30px;">Replace True with False to disable DeepSpeed mode.</p>
<p style="padding-left: 30px;">JSON return <span style="color: #339966;">{"status": "deepspeed-success"}</span></p>
<h4>Switching Low VRAM</h4>
<p style="padding-left: 30px;"><span style="color: #3366ff;">curl -X POST -H "Content-Type: application/json" "http://127.0.0.1:7851/api/lowvramsetting?new_low_vram_value=True"</span></p>
<p style="padding-left: 30px;"><span style="color: #3366ff;">curl -X POST "http://127.0.0.1:7851/api/lowvramsetting?new_low_vram_value=True"</span></p>
<p style="padding-left: 30px;">Replace True with False to disable Low VRAM mode.</p>
<p style="padding-left: 30px;">JSON return <span style="color: #339966;">{"status": "lowvram-success"}</span></p>
<p><a href="#toc">Back to top of page<br /></a></p>