-Text-generation-webui users - Uncheck Enable TTS in Text-generation-webui.
-
-
`;
-
- return html;
- }
-
-
- //#################//
- // Startup ST & AT //
- //#################//
-
- async loadSettings(settings) {
- updateStatus('Offline');
-
- if (Object.keys(settings).length === 0) {
- console.info('Using default AllTalk TTS Provider settings');
- } else {
- // Populate settings with provided values, ignoring server-provided settings
- for (const key in settings) {
- if (key in this.settings) {
- this.settings[key] = settings[key];
- } else {
- console.debug(`Ignoring non-user-configurable setting: ${key}`);
- }
- }
- }
-
- // Update UI elements to reflect the loaded settings
- $('#at_server').val(this.settings.provider_endpoint);
- $('#language_options').val(this.settings.language);
- //$('#voicemap').val(this.settings.voiceMap);
- $('#at_generation_method').val(this.settings.at_generation_method);
- $('#at_narrator_enabled').val(this.settings.narrator_enabled);
- $('#at_narrator_text_not_inside').val(this.settings.at_narrator_text_not_inside);
- $('#narrator_voice').val(this.settings.narrator_voice_gen);
-
- console.debug('AllTalkTTS: Settings loaded');
- try {
- // Check if TTS provider is ready
- await this.checkReady();
- await this.updateSettingsFromServer(); // Fetch dynamic settings from the TTS server
- await this.fetchTtsVoiceObjects(); // Fetch voices only if service is ready
- this.updateNarratorVoicesDropdown();
- this.updateLanguageDropdown();
- this.setupEventListeners();
- this.applySettingsToHTML();
- updateStatus('Ready');
- } catch (error) {
- console.error("Error loading settings:", error);
- updateStatus('Offline');
- }
- }
-
- applySettingsToHTML() {
- // Apply loaded settings or use defaults
- const narratorVoiceSelect = document.getElementById('narrator_voice');
- const atNarratorSelect = document.getElementById('at_narrator_enabled');
- const textNotInsideSelect = document.getElementById('at_narrator_text_not_inside');
- const generationMethodSelect = document.getElementById('at_generation_method');
- this.settings.narrator_voice = this.settings.narrator_voice_gen;
- // Apply settings to Narrator Voice dropdown
- if (narratorVoiceSelect && this.settings.narrator_voice) {
- narratorVoiceSelect.value = this.settings.narrator_voice.replace('.wav', '');
- }
- // Apply settings to AT Narrator Enabled dropdown
- if (atNarratorSelect) {
- // Sync the state with the checkbox in index.js
- const ttsPassAsterisksCheckbox = document.getElementById('tts_pass_asterisks'); // Access the checkbox from index.js
- const ttsNarrateQuotedCheckbox = document.getElementById('tts_narrate_quoted'); // Access the checkbox from index.js
- const ttsNarrateDialoguesCheckbox = document.getElementById('tts_narrate_dialogues'); // Access the checkbox from index.js
- // Sync the state with the checkbox in index.js
- if (this.settings.narrator_enabled) {
- ttsPassAsterisksCheckbox.checked = false;
- $('#tts_pass_asterisks').click(); // Simulate a click event
- $('#tts_pass_asterisks').trigger('change');
- }
- if (!this.settings.narrator_enabled) {
- ttsPassAsterisksCheckbox.checked = true;
- $('#tts_pass_asterisks').click(); // Simulate a click event
- $('#tts_pass_asterisks').trigger('change');
- }
- // Uncheck and set tts_narrate_quoted to false if narrator is enabled
- if (this.settings.narrator_enabledd) {
- ttsNarrateQuotedCheckbox.checked = true;
- ttsNarrateDialoguesCheckbox.checked = true;
- // Trigger click events instead of change events
- $('#tts_narrate_quoted').click();
- $('#tts_narrate_quoted').trigger('change');
- $('#tts_narrate_dialogues').click();
- $('#tts_narrate_dialogues').trigger('change');
- }
- atNarratorSelect.value = this.settings.narrator_enabled.toString();
- this.settings.narrator_enabled = this.settings.narrator_enabled.toString();
- }
- // Apply settings to the Language dropdown
- const languageSelect = document.getElementById('language_options');
- if (languageSelect && this.settings.language) {
- languageSelect.value = this.settings.language;
- }
- // Apply settings to Text Not Inside dropdown
- if (textNotInsideSelect && this.settings.text_not_inside) {
- textNotInsideSelect.value = this.settings.text_not_inside;
- this.settings.at_narrator_text_not_inside = this.settings.text_not_inside;
- }
- // Apply settings to Generation Method dropdown
- if (generationMethodSelect && this.settings.at_generation_method) {
- generationMethodSelect.value = this.settings.at_generation_method;
- }
- // Additional logic to disable/enable dropdowns based on the selected generation method
- const isStreamingEnabled = this.settings.at_generation_method === 'streaming_enabled';
- if (isStreamingEnabled) {
- // Disable certain dropdowns when streaming is enabled
- if (atNarratorSelect) atNarratorSelect.disabled = true;
- if (textNotInsideSelect) textNotInsideSelect.disabled = true;
- if (narratorVoiceSelect) narratorVoiceSelect.disabled = true;
- } else {
- // Enable dropdowns for standard generation
- if (atNarratorSelect) atNarratorSelect.disabled = false;
- if (textNotInsideSelect) textNotInsideSelect.disabled = !this.settings.narrator_enabled;
- if (narratorVoiceSelect) narratorVoiceSelect.disabled = !this.settings.narrator_enabled;
- }
- const modelSelect = document.getElementById('switch_model');
- if (this.settings.finetuned_model === 'true') {
- const ftOption = document.createElement('option');
- ftOption.value = 'XTTSv2 FT';
- ftOption.textContent = 'XTTSv2 FT';
- modelSelect.appendChild(ftOption);
- }
- }
-
- //##############################//
- // Check AT Server is Available //
- //##############################//
-
- async checkReady() {
- try {
- const response = await fetch(`${this.settings.provider_endpoint}/api/ready`);
- // Check if the HTTP request was successful
- if (!response.ok) {
- throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`);
- }
- const statusText = await response.text();
- // Check if the response is 'Ready'
- if (statusText === 'Ready') {
- this.ready = true; // Set the ready flag to true
- console.log('TTS service is ready.');
- } else {
- this.ready = false;
- console.log('TTS service is not ready.');
- }
- } catch (error) {
- console.error('Error checking TTS service readiness:', error);
- this.ready = false; // Ensure ready flag is set to false in case of error
- }
- }
-
- //######################//
- // Get Available Voices //
- //######################//
-
- async fetchTtsVoiceObjects() {
- const response = await fetch(`${this.settings.provider_endpoint}/api/voices`);
- if (!response.ok) {
- const errorText = await response.text();
- throw new Error(`HTTP ${response.status}: ${errorText}`);
- }
- const data = await response.json();
- const voices = data.voices.map(filename => {
- const voiceName = filename.replace('.wav', '');
- return {
- name: voiceName,
- voice_id: voiceName,
- preview_url: null, // Preview URL will be dynamically generated
- lang: 'en' // Default language
- };
- });
- this.voices = voices; // Assign to the class property
- return voices; // Also return this list
- }
-
- //##########################################//
- // Get Current AT Server Config & Update ST //
- //##########################################//
-
- async updateSettingsFromServer() {
- try {
- // Fetch current settings
- const response = await fetch(`${this.settings.provider_endpoint}/api/currentsettings`);
- if (!response.ok) {
- throw new Error(`Failed to fetch current settings: ${response.statusText}`);
- }
- const currentSettings = await response.json();
- // Update internal settings
- this.settings.modelsAvailable = currentSettings.models_available;
- this.settings.currentModel = currentSettings.current_model_loaded;
- this.settings.deepspeed_available = currentSettings.deepspeed_available;
- this.settings.deepSpeedEnabled = currentSettings.deepspeed_status;
- this.settings.lowVramEnabled = currentSettings.low_vram_status;
- this.settings.finetuned_model = currentSettings.finetuned_model
- // Update HTML elements
- this.updateModelDropdown();
- this.updateCheckboxes();
- } catch (error) {
- console.error(`Error updating settings from server: ${error}`);
- }
- }
-
- //###################################################//
- // Get Current AT Server Config & Update ST (Models) //
- //###################################################//
-
- updateModelDropdown() {
- const modelSelect = document.getElementById('switch_model');
- if (modelSelect) {
- modelSelect.innerHTML = ''; // Clear existing options
- this.settings.modelsAvailable.forEach(model => {
- const option = document.createElement('option');
- option.value = model.model_name;
- option.textContent = model.model_name; // Use model_name instead of name
- option.selected = model.model_name === this.settings.currentModel;
- modelSelect.appendChild(option);
- });
- }
- }
-
- //#######################################################//
- // Get Current AT Server Config & Update ST (DS and LVR) //
- //#######################################################//
-
- updateCheckboxes() {
- const deepspeedCheckbox = document.getElementById('deepspeed');
- const lowVramCheckbox = document.getElementById('low_vram');
- if (lowVramCheckbox) lowVramCheckbox.checked = this.settings.lowVramEnabled;
- if (deepspeedCheckbox) {
- deepspeedCheckbox.checked = this.settings.deepSpeedEnabled;
- deepspeedCheckbox.disabled = !this.settings.deepspeed_available; // Disable checkbox if deepspeed is not available
- }
- }
-
- //###############################################################//
- // Get Current AT Server Config & Update ST (AT Narrator Voices) //
- //###############################################################//
-
- updateNarratorVoicesDropdown() {
- const narratorVoiceSelect = document.getElementById('narrator_voice');
- if (narratorVoiceSelect && this.voices) {
- // Clear existing options
- narratorVoiceSelect.innerHTML = '';
- // Add new options
- for (let voice of this.voices) {
- const option = document.createElement('option');
- option.value = voice.voice_id;
- option.textContent = voice.name;
- narratorVoiceSelect.appendChild(option);
- }
- }
- }
-
- //######################################################//
- // Get Current AT Server Config & Update ST (Languages) //
- //######################################################//
-
- updateLanguageDropdown() {
- const languageSelect = document.getElementById('language_options');
- if (languageSelect) {
- // Ensure default language is set
- this.settings.language = this.settings.language;
-
- languageSelect.innerHTML = '';
- for (let language in this.languageLabels) {
- const option = document.createElement('option');
- option.value = this.languageLabels[language];
- option.textContent = language;
- if (this.languageLabels[language] === this.settings.language) {
- option.selected = true;
- }
- languageSelect.appendChild(option);
- }
- }
- }
-
- //########################################//
- // Start AT TTS extenstion page listeners //
- //########################################//
-
- setupEventListeners() {
-
- let debounceTimeout;
- const debounceDelay = 500; // Milliseconds
-
- // Define the event handler function
- const onModelSelectChange = async (event) => {
- console.log("Model select change event triggered"); // Debugging statement
- const selectedModel = event.target.value;
- console.log(`Selected model: ${selectedModel}`); // Debugging statement
- // Set status to Processing
- updateStatus('Processing');
- try {
- const response = await fetch(`${this.settings.provider_endpoint}/api/reload?tts_method=${encodeURIComponent(selectedModel)}`, {
- method: 'POST'
- });
- if (!response.ok) {
- throw new Error(`HTTP Error: ${response.status}`);
- }
- const data = await response.json();
- console.log("POST response data:", data); // Debugging statement
- // Set status to Ready if successful
- updateStatus('Ready');
- } catch (error) {
- console.error("POST request error:", error); // Debugging statement
- // Set status to Error in case of failure
- updateStatus('Error');
- }
-
- // Handle response or error
- };
-
- const debouncedModelSelectChange = (event) => {
- clearTimeout(debounceTimeout);
- debounceTimeout = setTimeout(() => {
- onModelSelectChange(event);
- }, debounceDelay);
- };
-
- // Switch Model Listener
- const modelSelect = document.getElementById('switch_model');
- if (modelSelect) {
- // Remove the event listener if it was previously added
- modelSelect.removeEventListener('change', debouncedModelSelectChange);
- // Add the debounced event listener
- modelSelect.addEventListener('change', debouncedModelSelectChange);
- }
-
- // DeepSpeed Listener
- const deepspeedCheckbox = document.getElementById('deepspeed');
- if (deepspeedCheckbox) {
- deepspeedCheckbox.addEventListener('change', async (event) => {
- const deepSpeedValue = event.target.checked ? 'True' : 'False';
- // Set status to Processing
- updateStatus('Processing');
- try {
- const response = await fetch(`${this.settings.provider_endpoint}/api/deepspeed?new_deepspeed_value=${deepSpeedValue}`, {
- method: 'POST'
- });
- if (!response.ok) {
- throw new Error(`HTTP Error: ${response.status}`);
- }
- const data = await response.json();
- console.log("POST response data:", data); // Debugging statement
- // Set status to Ready if successful
- updateStatus('Ready');
- } catch (error) {
- console.error("POST request error:", error); // Debugging statement
- // Set status to Error in case of failure
- updateStatus('Error');
- }
- });
- }
-
- // Low VRAM Listener
- const lowVramCheckbox = document.getElementById('low_vram');
- if (lowVramCheckbox) {
- lowVramCheckbox.addEventListener('change', async (event) => {
- const lowVramValue = event.target.checked ? 'True' : 'False';
- // Set status to Processing
- updateStatus('Processing');
- try {
- const response = await fetch(`${this.settings.provider_endpoint}/api/lowvramsetting?new_low_vram_value=${lowVramValue}`, {
- method: 'POST'
- });
- if (!response.ok) {
- throw new Error(`HTTP Error: ${response.status}`);
- }
- const data = await response.json();
- console.log("POST response data:", data); // Debugging statement
- // Set status to Ready if successful
- updateStatus('Ready');
- } catch (error) {
- console.error("POST request error:", error); // Debugging statement
- // Set status to Error in case of failure
- updateStatus('Error');
- }
- });
- }
-
- // Narrator Voice Dropdown Listener
- const narratorVoiceSelect = document.getElementById('narrator_voice');
- if (narratorVoiceSelect) {
- narratorVoiceSelect.addEventListener('change', (event) => {
- this.settings.narrator_voice_gen = `${event.target.value}.wav`;
- this.onSettingsChange(); // Save the settings after change
- });
- }
-
- const textNotInsideSelect = document.getElementById('at_narrator_text_not_inside');
- if (textNotInsideSelect) {
- textNotInsideSelect.addEventListener('change', (event) => {
- this.settings.text_not_inside = event.target.value;
- this.onSettingsChange(); // Save the settings after change
- });
- }
-
- // AT Narrator Dropdown Listener
- const atNarratorSelect = document.getElementById('at_narrator_enabled');
- const ttsPassAsterisksCheckbox = document.getElementById('tts_pass_asterisks'); // Access the checkbox from index.js
- const ttsNarrateQuotedCheckbox = document.getElementById('tts_narrate_quoted'); // Access the checkbox from index.js
- const ttsNarrateDialoguesCheckbox = document.getElementById('tts_narrate_dialogues'); // Access the checkbox from index.js
-
- if (atNarratorSelect && textNotInsideSelect && narratorVoiceSelect) {
- atNarratorSelect.addEventListener('change', (event) => {
- const isNarratorEnabled = event.target.value === 'true';
- this.settings.narrator_enabled = isNarratorEnabled; // Update the setting here
- textNotInsideSelect.disabled = !isNarratorEnabled;
- narratorVoiceSelect.disabled = !isNarratorEnabled;
-
- // Sync the state with the checkbox in index.js
- if (isNarratorEnabled) {
- ttsPassAsterisksCheckbox.checked = false;
- $('#tts_pass_asterisks').click(); // Simulate a click event
- $('#tts_pass_asterisks').trigger('change');
- }
- if (!isNarratorEnabled) {
- ttsPassAsterisksCheckbox.checked = true;
- $('#tts_pass_asterisks').click(); // Simulate a click event
- $('#tts_pass_asterisks').trigger('change');
- }
- // Uncheck and set tts_narrate_quoted to false if narrator is enabled
- if (isNarratorEnabled) {
- ttsNarrateQuotedCheckbox.checked = true;
- ttsNarrateDialoguesCheckbox.checked = true;
- // Trigger click events instead of change events
- $('#tts_narrate_quoted').click();
- $('#tts_narrate_quoted').trigger('change');
- $('#tts_narrate_dialogues').click();
- $('#tts_narrate_dialogues').trigger('change');
- }
- this.onSettingsChange(); // Save the settings after change
- });
- }
-
-
- // Event Listener for AT Generation Method Dropdown
- const atGenerationMethodSelect = document.getElementById('at_generation_method');
- const atNarratorEnabledSelect = document.getElementById('at_narrator_enabled');
- if (atGenerationMethodSelect) {
- atGenerationMethodSelect.addEventListener('change', (event) => {
- const selectedMethod = event.target.value;
-
- if (selectedMethod === 'streaming_enabled') {
- // Disable and unselect AT Narrator
- atNarratorEnabledSelect.disabled = true;
- atNarratorEnabledSelect.value = 'false';
- textNotInsideSelect.disabled = true;
- narratorVoiceSelect.disabled = true;
- } else if (selectedMethod === 'standard_generation') {
- // Enable AT Narrator
- atNarratorEnabledSelect.disabled = false;
- }
- this.settings.at_generation_method = selectedMethod; // Update the setting here
- this.onSettingsChange(); // Save the settings after change
- });
- }
-
- // Listener for Language Dropdown
- const languageSelect = document.getElementById('language_options');
- if (languageSelect) {
- languageSelect.addEventListener('change', (event) => {
- this.settings.language = event.target.value;
- this.onSettingsChange(); // Save the settings after change
- });
- }
-
- // Listener for AllTalk Endpoint Input
- const atServerInput = document.getElementById('at_server');
- if (atServerInput) {
- atServerInput.addEventListener('input', (event) => {
- this.settings.provider_endpoint = event.target.value;
- this.onSettingsChange(); // Save the settings after change
- });
- }
-
- }
-
- //#############################//
- // Store ST interface settings //
- //#############################//
-
- onSettingsChange() {
- // Update settings based on the UI elements
- //this.settings.provider_endpoint = $('#at_server').val();
- this.settings.language = $('#language_options').val();
- //this.settings.voiceMap = $('#voicemap').val();
- this.settings.at_generation_method = $('#at_generation_method').val();
- this.settings.narrator_enabled = $('#at_narrator_enabled').val();
- this.settings.at_narrator_text_not_inside = $('#at_narrator_text_not_inside').val();
- this.settings.narrator_voice_gen = $('#narrator_voice').val();
- // Save the updated settings
- saveTtsProviderSettings();
- }
-
- //#########################//
- // ST Handle Reload button //
- //#########################//
-
- async onRefreshClick() {
- await this.checkReady(); // Check if the TTS provider is ready
- await this.loadSettings(this.settings); // Reload the settings
- // Additional actions as needed
- }
-
- //##################//
- // Preview AT Voice //
- //##################//
-
- async previewTtsVoice(voiceName) {
- try {
- // Prepare data for POST request
- const postData = new URLSearchParams();
- postData.append("voice", `${voiceName}.wav`);
- // Making the POST request
- const response = await fetch(`${this.settings.provider_endpoint}/api/previewvoice/`, {
- method: "POST",
- headers: {
- 'Content-Type': 'application/x-www-form-urlencoded'
- },
- body: postData,
- });
- if (!response.ok) {
- const errorText = await response.text();
- console.error(`[previewTtsVoice] Error Response Text:`, errorText);
- throw new Error(`HTTP ${response.status}: ${errorText}`);
- }
- // Assuming the server returns a URL to the .wav file
- const data = await response.json();
- if (data.output_file_url) {
- // Use an audio element to play the .wav file
- const audioElement = new Audio(data.output_file_url);
- audioElement.play().catch(e => console.error("Error playing audio:", e));
- } else {
- console.warn("[previewTtsVoice] No output file URL received in the response");
- throw new Error("No output file URL received in the response");
- }
-
- } catch (error) {
- console.error("[previewTtsVoice] Exception caught during preview generation:", error);
- throw error;
- }
- }
-
- //#####################//
- // Populate ST voices //
- //#####################//
-
- async getVoice(voiceName, generatePreview = false) {
- // Ensure this.voices is populated
- if (this.voices.length === 0) {
- // Fetch voice objects logic
- }
- // Find the object where the name matches voiceName
- const match = this.voices.find(voice => voice.name === voiceName);
- if (!match) {
- // Error handling
- }
- // Generate preview URL only if requested
- if (!match.preview_url && generatePreview) {
- // Generate preview logic
- }
- return match; // Return the found voice object
- }
-
- //##########################################//
- // Generate TTS Streaming or call Standard //
- //##########################################//
-
- async generateTts(inputText, voiceId) {
- try {
- if (this.settings.at_generation_method === 'streaming_enabled') {
- // Construct the streaming URL
- const streamingUrl = `${this.settings.provider_endpoint}/api/tts-generate-streaming?text=${encodeURIComponent(inputText)}&voice=${encodeURIComponent(voiceId)}.wav&language=${encodeURIComponent(this.settings.language)}&output_file=stream_output.wav`;
- console.log("Streaming URL:", streamingUrl);
-
- // Return the streaming URL directly
- return streamingUrl;
- } else {
- // For standard method
- const outputUrl = await this.fetchTtsGeneration(inputText, voiceId);
- const audioResponse = await fetch(outputUrl);
- if (!audioResponse.ok) {
- throw new Error(`HTTP ${audioResponse.status}: Failed to fetch audio data`);
- }
- return audioResponse; // Return the fetch response directly
- }
- } catch (error) {
- console.error("Error in generateTts:", error);
- throw error;
- }
- }
-
-
- //####################//
- // Generate Standard //
- //####################//
-
- async fetchTtsGeneration(inputText, voiceId) {
- // Prepare the request payload
- const requestBody = new URLSearchParams({
- 'text_input': inputText,
- 'text_filtering': "standard",
- 'character_voice_gen': voiceId + ".wav",
- 'narrator_enabled': this.settings.narrator_enabled,
- 'narrator_voice_gen': this.settings.narrator_voice_gen + ".wav",
- 'text_not_inside': this.settings.at_narrator_text_not_inside,
- 'language': this.settings.language,
- 'output_file_name': "st_output",
- 'output_file_timestamp': "true",
- 'autoplay': "false",
- 'autoplay_volume': "0.8"
- }).toString();
-
- try {
- const response = await doExtrasFetch(
- `${this.settings.provider_endpoint}/api/tts-generate`,
- {
- method: 'POST',
- headers: {
- 'Content-Type': 'application/x-www-form-urlencoded',
- 'Cache-Control': 'no-cache',
- },
- body: requestBody
- }
- );
-
- if (!response.ok) {
- const errorText = await response.text();
- console.error(`[fetchTtsGeneration] Error Response Text:`, errorText);
- // toastr.error(response.statusText, 'TTS Generation Failed');
- throw new Error(`HTTP ${response.status}: ${errorText}`);
- }
- const data = await response.json();
- const outputUrl = data.output_file_url;
- return outputUrl; // Return only the output_file_url
- } catch (error) {
- console.error("[fetchTtsGeneration] Exception caught:", error);
- throw error; // Rethrow the error for further handling
- }
- }
-}
-
-//#########################//
-// Update Status Messages //
-//#########################//
-
-function updateStatus(message) {
- const statusElement = document.getElementById('status_info');
- if (statusElement) {
- statusElement.textContent = message;
- switch (message) {
- case 'Offline':
- statusElement.style.color = 'red';
- break;
- case 'Ready':
- statusElement.style.color = 'lightgreen';
- break;
- case 'Processing':
- statusElement.style.color = 'blue';
- break;
- case 'Error':
- statusElement.style.color = 'red';
- break;
- }
- }
-}
\ No newline at end of file
diff --git a/templates/STfiles/index.js b/templates/STfiles/index.js
deleted file mode 100644
index dd84a22..0000000
--- a/templates/STfiles/index.js
+++ /dev/null
@@ -1,1046 +0,0 @@
-import { callPopup, cancelTtsPlay, eventSource, event_types, name2, saveSettingsDebounced } from '../../../script.js';
-import { ModuleWorkerWrapper, doExtrasFetch, extension_settings, getApiUrl, getContext, modules } from '../../extensions.js';
-import { delay, escapeRegex, getBase64Async, getStringHash, onlyUnique } from '../../utils.js';
-import { EdgeTtsProvider } from './edge.js';
-import { ElevenLabsTtsProvider } from './elevenlabs.js';
-import { SileroTtsProvider } from './silerotts.js';
-import { CoquiTtsProvider } from './coqui.js';
-import { SystemTtsProvider } from './system.js';
-import { NovelTtsProvider } from './novel.js';
-import { power_user } from '../../power-user.js';
-import { registerSlashCommand } from '../../slash-commands.js';
-import { OpenAITtsProvider } from './openai.js';
-import { XTTSTtsProvider } from './xtts.js';
-import { AllTalkTtsProvider } from './alltalk.js';
-export { talkingAnimation };
-
-const UPDATE_INTERVAL = 1000;
-
-let voiceMapEntries = [];
-let voiceMap = {}; // {charName:voiceid, charName2:voiceid2}
-let storedvalue = false;
-let lastChatId = null;
-let lastMessageHash = null;
-
-const DEFAULT_VOICE_MARKER = '[Default Voice]';
-const DISABLED_VOICE_MARKER = 'disabled';
-
-export function getPreviewString(lang) {
- const previewStrings = {
- 'en-US': 'The quick brown fox jumps over the lazy dog',
- 'en-GB': 'Sphinx of black quartz, judge my vow',
- 'fr-FR': 'Portez ce vieux whisky au juge blond qui fume',
- 'de-DE': 'Victor jagt zwölf Boxkämpfer quer über den großen Sylter Deich',
- 'it-IT': 'Pranzo d\'acqua fa volti sghembi',
- 'es-ES': 'Quiere la boca exhausta vid, kiwi, piña y fugaz jamón',
- 'es-MX': 'Fabio me exige, sin tapujos, que añada cerveza al whisky',
- 'ru-RU': 'В чащах юга жил бы цитрус? Да, но фальшивый экземпляр!',
- 'pt-BR': 'Vejo xá gritando que fez show sem playback.',
- 'pt-PR': 'Todo pajé vulgar faz boquinha sexy com kiwi.',
- 'uk-UA': 'Фабрикуймо гідність, лящім їжею, ґав хапаймо, з\'єднавці чаш!',
- 'pl-PL': 'Pchnąć w tę łódź jeża lub ośm skrzyń fig',
- 'cs-CZ': 'Příliš žluťoučký kůň úpěl ďábelské ódy',
- 'sk-SK': 'Vyhŕňme si rukávy a vyprážajme čínske ryžové cestoviny',
- 'hu-HU': 'Árvíztűrő tükörfúrógép',
- 'tr-TR': 'Pijamalı hasta yağız şoföre çabucak güvendi',
- 'nl-NL': 'De waard heeft een kalfje en een pinkje opgegeten',
- 'sv-SE': 'Yxskaftbud, ge vårbygd, zinkqvarn',
- 'da-DK': 'Quizdeltagerne spiste jordbær med fløde, mens cirkusklovnen Walther spillede på xylofon',
- 'ja-JP': 'いろはにほへと ちりぬるを わかよたれそ つねならむ うゐのおくやま けふこえて あさきゆめみし ゑひもせす',
- 'ko-KR': '가나다라마바사아자차카타파하',
- 'zh-CN': '我能吞下玻璃而不伤身体',
- 'ro-RO': 'Muzicologă în bej vând whisky și tequila, preț fix',
- 'bg-BG': 'Щъркелите се разпръснаха по цялото небе',
- 'el-GR': 'Ταχίστη αλώπηξ βαφής ψημένη γη, δρασκελίζει υπέρ νωθρού κυνός',
- 'fi-FI': 'Voi veljet, miksi juuri teille myin nämä vehkeet?',
- 'he-IL': 'הקצינים צעקו: "כל הכבוד לצבא הצבאות!"',
- 'id-ID': 'Jangkrik itu memang enak, apalagi kalau digoreng',
- 'ms-MY': 'Muzik penyanyi wanita itu menggambarkan kehidupan yang penuh dengan duka nestapa',
- 'th-TH': 'เป็นไงบ้างครับ ผมชอบกินข้าวผัดกระเพราหมูกรอบ',
- 'vi-VN': 'Cô bé quàng khăn đỏ đang ngồi trên bãi cỏ xanh',
- 'ar-SA': 'أَبْجَدِيَّة عَرَبِيَّة',
- 'hi-IN': 'श्वेता ने श्वेता के श्वेते हाथों में श्वेता का श्वेता चावल पकड़ा',
- };
- const fallbackPreview = 'Neque porro quisquam est qui dolorem ipsum quia dolor sit amet';
-
- return previewStrings[lang] ?? fallbackPreview;
-}
-
-let ttsProviders = {
- ElevenLabs: ElevenLabsTtsProvider,
- Silero: SileroTtsProvider,
- XTTSv2: XTTSTtsProvider,
- System: SystemTtsProvider,
- Coqui: CoquiTtsProvider,
- Edge: EdgeTtsProvider,
- Novel: NovelTtsProvider,
- OpenAI: OpenAITtsProvider,
- AllTalk: AllTalkTtsProvider,
-};
-let ttsProvider;
-let ttsProviderName;
-
-let ttsLastMessage = null;
-
-async function onNarrateOneMessage() {
- audioElement.src = '/sounds/silence.mp3';
- const context = getContext();
- const id = $(this).closest('.mes').attr('mesid');
- const message = context.chat[id];
-
- if (!message) {
- return;
- }
-
- resetTtsPlayback();
- ttsJobQueue.push(message);
- moduleWorker();
-}
-
-async function onNarrateText(args, text) {
- if (!text) {
- return;
- }
-
- audioElement.src = '/sounds/silence.mp3';
-
- // To load all characters in the voice map, set unrestricted to true
- await initVoiceMap(true);
-
- const baseName = args?.voice || name2;
- const name = (baseName === 'SillyTavern System' ? DEFAULT_VOICE_MARKER : baseName) || DEFAULT_VOICE_MARKER;
-
- const voiceMapEntry = voiceMap[name] === DEFAULT_VOICE_MARKER
- ? voiceMap[DEFAULT_VOICE_MARKER]
- : voiceMap[name];
-
- if (!voiceMapEntry || voiceMapEntry === DISABLED_VOICE_MARKER) {
- toastr.info(`Specified voice for ${name} was not found. Check the TTS extension settings.`);
- return;
- }
-
- resetTtsPlayback();
- ttsJobQueue.push({ mes: text, name: name });
- await moduleWorker();
-
- // Return back to the chat voices
- await initVoiceMap(false);
-}
-
-async function moduleWorker() {
- // Primarily determining when to add new chat to the TTS queue
- const enabled = $('#tts_enabled').is(':checked');
- $('body').toggleClass('tts', enabled);
- if (!enabled) {
- return;
- }
-
- const context = getContext();
- const chat = context.chat;
-
- processTtsQueue();
- processAudioJobQueue();
- updateUiAudioPlayState();
-
- // Auto generation is disabled
- if (extension_settings.tts.auto_generation == false) {
- return;
- }
-
- // no characters or group selected
- if (!context.groupId && context.characterId === undefined) {
- return;
- }
-
- // Chat changed
- if (
- context.chatId !== lastChatId
- ) {
- currentMessageNumber = context.chat.length ? context.chat.length : 0;
- saveLastValues();
-
- // Force to speak on the first message in the new chat
- if (context.chat.length === 1) {
- lastMessageHash = -1;
- }
-
- return;
- }
-
- // take the count of messages
- let lastMessageNumber = context.chat.length ? context.chat.length : 0;
-
- // There's no new messages
- let diff = lastMessageNumber - currentMessageNumber;
- let hashNew = getStringHash((chat.length && chat[chat.length - 1].mes) ?? '');
-
- // if messages got deleted, diff will be < 0
- if (diff < 0) {
- // necessary actions will be taken by the onChatDeleted() handler
- return;
- }
-
- // if no new messages, or same message, or same message hash, do nothing
- if (diff == 0 && hashNew === lastMessageHash) {
- return;
- }
-
- // If streaming, wait for streaming to finish before processing new messages
- if (context.streamingProcessor && !context.streamingProcessor.isFinished) {
- return;
- }
-
- // clone message object, as things go haywire if message object is altered below (it's passed by reference)
- const message = structuredClone(chat[chat.length - 1]);
-
- // if last message within current message, message got extended. only send diff to TTS.
- if (ttsLastMessage !== null && message.mes.indexOf(ttsLastMessage) !== -1) {
- let tmp = message.mes;
- message.mes = message.mes.replace(ttsLastMessage, '');
- ttsLastMessage = tmp;
- } else {
- ttsLastMessage = message.mes;
- }
-
- // We're currently swiping. Don't generate voice
- if (!message || message.mes === '...' || message.mes === '') {
- return;
- }
-
- // Don't generate if message doesn't have a display text
- if (extension_settings.tts.narrate_translated_only && !(message?.extra?.display_text)) {
- return;
- }
-
- // Don't generate if message is a user message and user message narration is disabled
- if (message.is_user && !extension_settings.tts.narrate_user) {
- return;
- }
-
- // New messages, add new chat to history
- lastMessageHash = hashNew;
- currentMessageNumber = lastMessageNumber;
-
- console.debug(
- `Adding message from ${message.name} for TTS processing: "${message.mes}"`,
- );
- ttsJobQueue.push(message);
-}
-
-function talkingAnimation(switchValue) {
- if (!modules.includes('talkinghead')) {
- console.debug('Talking Animation module not loaded');
- return;
- }
-
- const apiUrl = getApiUrl();
- const animationType = switchValue ? 'start' : 'stop';
-
- if (switchValue !== storedvalue) {
- try {
- console.log(animationType + ' Talking Animation');
- doExtrasFetch(`${apiUrl}/api/talkinghead/${animationType}_talking`);
- storedvalue = switchValue; // Update the storedvalue to the current switchValue
- } catch (error) {
- // Handle the error here or simply ignore it to prevent logging
- }
- }
- updateUiAudioPlayState();
-}
-
-function resetTtsPlayback() {
- // Stop system TTS utterance
- cancelTtsPlay();
-
- // Clear currently processing jobs
- currentTtsJob = null;
- currentAudioJob = null;
-
- // Reset audio element
- audioElement.currentTime = 0;
- audioElement.src = '';
-
- // Clear any queue items
- ttsJobQueue.splice(0, ttsJobQueue.length);
- audioJobQueue.splice(0, audioJobQueue.length);
-
- // Set audio ready to process again
- audioQueueProcessorReady = true;
-}
-
-function isTtsProcessing() {
- let processing = false;
-
- // Check job queues
- if (ttsJobQueue.length > 0 || audioJobQueue.length > 0) {
- processing = true;
- }
- // Check current jobs
- if (currentTtsJob != null || currentAudioJob != null) {
- processing = true;
- }
- return processing;
-}
-
-function debugTtsPlayback() {
- console.log(JSON.stringify(
- {
- 'ttsProviderName': ttsProviderName,
- 'voiceMap': voiceMap,
- 'currentMessageNumber': currentMessageNumber,
- 'audioPaused': audioPaused,
- 'audioJobQueue': audioJobQueue,
- 'currentAudioJob': currentAudioJob,
- 'audioQueueProcessorReady': audioQueueProcessorReady,
- 'ttsJobQueue': ttsJobQueue,
- 'currentTtsJob': currentTtsJob,
- 'ttsConfig': extension_settings.tts,
- },
- ));
-}
-window.debugTtsPlayback = debugTtsPlayback;
-
-//##################//
-// Audio Control //
-//##################//
-
-let audioElement = new Audio();
-audioElement.id = 'tts_audio';
-audioElement.autoplay = true;
-
-let audioJobQueue = [];
-let currentAudioJob;
-let audioPaused = false;
-let audioQueueProcessorReady = true;
-
-async function playAudioData(audioBlob) {
- // Since current audio job can be cancelled, don't playback if it is null
- if (currentAudioJob == null) {
- console.log('Cancelled TTS playback because currentAudioJob was null');
- }
- if (audioBlob instanceof Blob) {
- const srcUrl = await getBase64Async(audioBlob);
- audioElement.src = srcUrl;
- } else if (typeof audioBlob === 'string') {
- audioElement.src = audioBlob;
- } else {
- throw `TTS received invalid audio data type ${typeof audioBlob}`;
- }
- audioElement.addEventListener('ended', completeCurrentAudioJob);
- audioElement.addEventListener('canplay', () => {
- console.debug('Starting TTS playback');
- audioElement.play();
- });
-}
-
-window['tts_preview'] = function (id) {
- const audio = document.getElementById(id);
-
- if (audio && !$(audio).data('disabled')) {
- audio.play();
- }
- else {
- ttsProvider.previewTtsVoice(id);
- }
-};
-
-async function onTtsVoicesClick() {
- let popupText = '';
-
- try {
- const voiceIds = await ttsProvider.fetchTtsVoiceObjects();
-
- for (const voice of voiceIds) {
- popupText += `
-
- ${voice.lang || ''}
- ${voice.name}
-
-
`;
- if (voice.preview_url) {
- popupText += ``;
- }
- }
- } catch {
- popupText = 'Could not load voices list. Check your API key.';
- }
-
- callPopup(popupText, 'text');
-}
-
-function updateUiAudioPlayState() {
- if (extension_settings.tts.enabled == true) {
- $('#ttsExtensionMenuItem').show();
- let img;
- // Give user feedback that TTS is active by setting the stop icon if processing or playing
- if (!audioElement.paused || isTtsProcessing()) {
- img = 'fa-solid fa-stop-circle extensionsMenuExtensionButton';
- } else {
- img = 'fa-solid fa-circle-play extensionsMenuExtensionButton';
- }
- $('#tts_media_control').attr('class', img);
- } else {
- $('#ttsExtensionMenuItem').hide();
- }
-}
-
-function onAudioControlClicked() {
- audioElement.src = '/sounds/silence.mp3';
- let context = getContext();
- // Not pausing, doing a full stop to anything TTS is doing. Better UX as pause is not as useful
- if (!audioElement.paused || isTtsProcessing()) {
- resetTtsPlayback();
- talkingAnimation(false);
- } else {
- // Default play behavior if not processing or playing is to play the last message.
- ttsJobQueue.push(context.chat[context.chat.length - 1]);
- }
- updateUiAudioPlayState();
-}
-
-function addAudioControl() {
-
- $('#extensionsMenu').prepend(`
-
-
- TTS Playback
- `);
- $('#ttsExtensionMenuItem').attr('title', 'TTS play/pause').on('click', onAudioControlClicked);
- updateUiAudioPlayState();
-}
-
-function completeCurrentAudioJob() {
- audioQueueProcessorReady = true;
- currentAudioJob = null;
- talkingAnimation(false); //stop lip animation
- // updateUiPlayState();
-}
-
-/**
- * Accepts an HTTP response containing audio/mpeg data, and puts the data as a Blob() on the queue for playback
- * @param {Response} response
- */
-async function addAudioJob(response) {
- if (typeof response === 'string') {
- audioJobQueue.push(response);
- } else {
- const audioData = await response.blob();
- if (!audioData.type.startsWith('audio/')) {
- throw `TTS received HTTP response with invalid data format. Expecting audio/*, got ${audioData.type}`;
- }
- audioJobQueue.push(audioData);
- }
- console.debug('Pushed audio job to queue.');
-}
-
-async function processAudioJobQueue() {
- // Nothing to do, audio not completed, or audio paused - stop processing.
- if (audioJobQueue.length == 0 || !audioQueueProcessorReady || audioPaused) {
- return;
- }
- try {
- audioQueueProcessorReady = false;
- currentAudioJob = audioJobQueue.shift();
- playAudioData(currentAudioJob);
- talkingAnimation(true);
- } catch (error) {
- console.error(error);
- audioQueueProcessorReady = true;
- }
-}
-
-//################//
-// TTS Control //
-//################//
-
-let ttsJobQueue = [];
-let currentTtsJob; // Null if nothing is currently being processed
-let currentMessageNumber = 0;
-
-function completeTtsJob() {
- console.info(`Current TTS job for ${currentTtsJob?.name} completed.`);
- currentTtsJob = null;
-}
-
-function saveLastValues() {
- const context = getContext();
- lastChatId = context.chatId;
- lastMessageHash = getStringHash(
- (context.chat.length && context.chat[context.chat.length - 1].mes) ?? '',
- );
-}
-
-async function tts(text, voiceId, char) {
- async function processResponse(response) {
- // RVC injection
- if (extension_settings.rvc.enabled && typeof window['rvcVoiceConversion'] === 'function')
- response = await window['rvcVoiceConversion'](response, char, text);
-
- await addAudioJob(response);
- }
-
- let response = await ttsProvider.generateTts(text, voiceId);
-
- // If async generator, process every chunk as it comes in
- if (typeof response[Symbol.asyncIterator] === 'function') {
- for await (const chunk of response) {
- await processResponse(chunk);
- }
- } else {
- await processResponse(response);
- }
-
- completeTtsJob();
-}
-
-async function processTtsQueue() {
- // Called each moduleWorker iteration to pull chat messages from queue
- if (currentTtsJob || ttsJobQueue.length <= 0 || audioPaused) {
- return;
- }
-
- console.debug('New message found, running TTS');
- currentTtsJob = ttsJobQueue.shift();
- let text = extension_settings.tts.narrate_translated_only ? (currentTtsJob?.extra?.display_text || currentTtsJob.mes) : currentTtsJob.mes;
-
- if (extension_settings.tts.skip_codeblocks) {
- text = text.replace(/^\s{4}.*$/gm, '').trim();
- text = text.replace(/```.*?```/gs, '').trim();
- }
-
- if (!extension_settings.tts.pass_asterisks) {
- text = extension_settings.tts.narrate_dialogues_only
- ? text.replace(/\*[^*]*?(\*|$)/g, '').trim() // remove asterisks content
- : text.replaceAll('*', '').trim(); // remove just the asterisks
- }
-
- if (extension_settings.tts.narrate_quoted_only) {
- const special_quotes = /[“”]/g; // Extend this regex to include other special quotes
- text = text.replace(special_quotes, '"');
- const matches = text.match(/".*?"/g); // Matches text inside double quotes, non-greedily
- const partJoiner = (ttsProvider?.separator || ' ... ');
- text = matches ? matches.join(partJoiner) : text;
- }
-
- if (typeof ttsProvider?.processText === 'function') {
- text = await ttsProvider.processText(text);
- }
-
- // Collapse newlines and spaces into single space
- text = text.replace(/\s+/g, ' ').trim();
-
- console.log(`TTS: ${text}`);
- const char = currentTtsJob.name;
-
- // Remove character name from start of the line if power user setting is disabled
- if (char && !power_user.allow_name2_display) {
- const escapedChar = escapeRegex(char);
- text = text.replace(new RegExp(`^${escapedChar}:`, 'gm'), '');
- }
-
- try {
- if (!text) {
- console.warn('Got empty text in TTS queue job.');
- completeTtsJob();
- return;
- }
-
- const voiceMapEntry = voiceMap[char] === DEFAULT_VOICE_MARKER ? voiceMap[DEFAULT_VOICE_MARKER] : voiceMap[char];
-
- if (!voiceMapEntry || voiceMapEntry === DISABLED_VOICE_MARKER) {
- throw `${char} not in voicemap. Configure character in extension settings voice map`;
- }
- const voice = await ttsProvider.getVoice(voiceMapEntry);
- const voiceId = voice.voice_id;
- if (voiceId == null) {
- toastr.error(`Specified voice for ${char} was not found. Check the TTS extension settings.`);
- throw `Unable to attain voiceId for ${char}`;
- }
- tts(text, voiceId, char);
- } catch (error) {
- console.error(error);
- currentTtsJob = null;
- }
-}
-
-// Secret function for now
-async function playFullConversation() {
- const context = getContext();
- const chat = context.chat;
- ttsJobQueue = chat;
-}
-window.playFullConversation = playFullConversation;
-
-//#############################//
-// Extension UI and Settings //
-//#############################//
-
-function loadSettings() {
- if (Object.keys(extension_settings.tts).length === 0) {
- Object.assign(extension_settings.tts, defaultSettings);
- }
- for (const key in defaultSettings) {
- if (!(key in extension_settings.tts)) {
- extension_settings.tts[key] = defaultSettings[key];
- }
- }
- $('#tts_provider').val(extension_settings.tts.currentProvider);
- $('#tts_enabled').prop(
- 'checked',
- extension_settings.tts.enabled,
- );
- $('#tts_narrate_dialogues').prop('checked', extension_settings.tts.narrate_dialogues_only);
- $('#tts_narrate_quoted').prop('checked', extension_settings.tts.narrate_quoted_only);
- $('#tts_auto_generation').prop('checked', extension_settings.tts.auto_generation);
- $('#tts_narrate_translated_only').prop('checked', extension_settings.tts.narrate_translated_only);
- $('#tts_narrate_user').prop('checked', extension_settings.tts.narrate_user);
- $('#tts_pass_asterisks').prop('checked', extension_settings.tts.pass_asterisks);
- $('body').toggleClass('tts', extension_settings.tts.enabled);
-}
-
-const defaultSettings = {
- voiceMap: '',
- ttsEnabled: false,
- currentProvider: 'ElevenLabs',
- auto_generation: true,
- narrate_user: false,
-};
-
-function setTtsStatus(status, success) {
- $('#tts_status').text(status);
- if (success) {
- $('#tts_status').removeAttr('style');
- } else {
- $('#tts_status').css('color', 'red');
- }
-}
-
-function onRefreshClick() {
- Promise.all([
- ttsProvider.onRefreshClick(),
- // updateVoiceMap()
- ]).then(() => {
- extension_settings.tts[ttsProviderName] = ttsProvider.settings;
- saveSettingsDebounced();
- setTtsStatus('Successfully applied settings', true);
- console.info(`Saved settings ${ttsProviderName} ${JSON.stringify(ttsProvider.settings)}`);
- initVoiceMap();
- updateVoiceMap();
- }).catch(error => {
- console.error(error);
- setTtsStatus(error, false);
- });
-}
-
-function onEnableClick() {
- extension_settings.tts.enabled = $('#tts_enabled').is(
- ':checked',
- );
- updateUiAudioPlayState();
- saveSettingsDebounced();
-}
-
-
-function onAutoGenerationClick() {
- extension_settings.tts.auto_generation = !!$('#tts_auto_generation').prop('checked');
- saveSettingsDebounced();
-}
-
-
-function onNarrateDialoguesClick() {
- extension_settings.tts.narrate_dialogues_only = !!$('#tts_narrate_dialogues').prop('checked');
- saveSettingsDebounced();
-}
-
-function onNarrateUserClick() {
- extension_settings.tts.narrate_user = !!$('#tts_narrate_user').prop('checked');
- saveSettingsDebounced();
-}
-
-function onNarrateQuotedClick() {
- extension_settings.tts.narrate_quoted_only = !!$('#tts_narrate_quoted').prop('checked');
- saveSettingsDebounced();
-}
-
-
-function onNarrateTranslatedOnlyClick() {
- extension_settings.tts.narrate_translated_only = !!$('#tts_narrate_translated_only').prop('checked');
- saveSettingsDebounced();
-}
-
-function onSkipCodeblocksClick() {
- extension_settings.tts.skip_codeblocks = !!$('#tts_skip_codeblocks').prop('checked');
- saveSettingsDebounced();
-}
-
-function onPassAsterisksClick() {
- extension_settings.tts.pass_asterisks = !!$('#tts_pass_asterisks').prop('checked');
- saveSettingsDebounced();
- console.log("setting pass asterisks", extension_settings.tts.pass_asterisks)
-}
-
-//##############//
-// TTS Provider //
-//##############//
-
-async function loadTtsProvider(provider) {
- //Clear the current config and add new config
- $('#tts_provider_settings').html('');
-
- if (!provider) {
- return;
- }
-
- // Init provider references
- extension_settings.tts.currentProvider = provider;
- ttsProviderName = provider;
- ttsProvider = new ttsProviders[provider];
-
- // Init provider settings
- $('#tts_provider_settings').append(ttsProvider.settingsHtml);
- if (!(ttsProviderName in extension_settings.tts)) {
- console.warn(`Provider ${ttsProviderName} not in Extension Settings, initiatilizing provider in settings`);
- extension_settings.tts[ttsProviderName] = {};
- }
- await ttsProvider.loadSettings(extension_settings.tts[ttsProviderName]);
- await initVoiceMap();
-}
-
-function onTtsProviderChange() {
- const ttsProviderSelection = $('#tts_provider').val();
- extension_settings.tts.currentProvider = ttsProviderSelection;
- loadTtsProvider(ttsProviderSelection);
-}
-
-// Ensure that TTS provider settings are saved to extension settings.
-export function saveTtsProviderSettings() {
- extension_settings.tts[ttsProviderName] = ttsProvider.settings;
- updateVoiceMap();
- saveSettingsDebounced();
- console.info(`Saved settings ${ttsProviderName} ${JSON.stringify(ttsProvider.settings)}`);
-}
-
-
-//###################//
-// voiceMap Handling //
-//###################//
-
-async function onChatChanged() {
- await resetTtsPlayback();
- const voiceMapInit = initVoiceMap();
- await Promise.race([voiceMapInit, delay(1000)]);
- ttsLastMessage = null;
-}
-
-async function onChatDeleted() {
- const context = getContext();
-
- // update internal references to new last message
- lastChatId = context.chatId;
- currentMessageNumber = context.chat.length ? context.chat.length : 0;
-
- // compare against lastMessageHash. If it's the same, we did not delete the last chat item, so no need to reset tts queue
- let messageHash = getStringHash((context.chat.length && context.chat[context.chat.length - 1].mes) ?? '');
- if (messageHash === lastMessageHash) {
- return;
- }
- lastMessageHash = messageHash;
- ttsLastMessage = (context.chat.length && context.chat[context.chat.length - 1].mes) ?? '';
-
- // stop any tts playback since message might not exist anymore
- await resetTtsPlayback();
-}
-
-/**
- * Get characters in current chat
- * @param {boolean} unrestricted - If true, will include all characters in voiceMapEntries, even if they are not in the current chat.
- * @returns {string[]} - Array of character names
- */
-function getCharacters(unrestricted) {
- const context = getContext();
-
- if (unrestricted) {
- const names = context.characters.map(char => char.name);
- names.unshift(DEFAULT_VOICE_MARKER);
- return names.filter(onlyUnique);
- }
-
- let characters = [];
- if (context.groupId === null) {
- // Single char chat
- characters.push(DEFAULT_VOICE_MARKER);
- characters.push(context.name1);
- characters.push(context.name2);
- } else {
- // Group chat
- characters.push(DEFAULT_VOICE_MARKER);
- characters.push(context.name1);
- const group = context.groups.find(group => context.groupId == group.id);
- for (let member of group.members) {
- const character = context.characters.find(char => char.avatar == member);
- if (character) {
- characters.push(character.name);
- }
- }
- }
- return characters.filter(onlyUnique);
-}
-
-function sanitizeId(input) {
- // Remove any non-alphanumeric characters except underscore (_) and hyphen (-)
- let sanitized = input.replace(/[^a-zA-Z0-9-_]/g, '');
-
- // Ensure first character is always a letter
- if (!/^[a-zA-Z]/.test(sanitized)) {
- sanitized = 'element_' + sanitized;
- }
-
- return sanitized;
-}
-
-function parseVoiceMap(voiceMapString) {
- let parsedVoiceMap = {};
- for (const [charName, voiceId] of voiceMapString
- .split(',')
- .map(s => s.split(':'))) {
- if (charName && voiceId) {
- parsedVoiceMap[charName.trim()] = voiceId.trim();
- }
- }
- return parsedVoiceMap;
-}
-
-
-
-/**
- * Apply voiceMap based on current voiceMapEntries
- */
-function updateVoiceMap() {
- const tempVoiceMap = {};
- for (const voice of voiceMapEntries) {
- if (voice.voiceId === null) {
- continue;
- }
- tempVoiceMap[voice.name] = voice.voiceId;
- }
- if (Object.keys(tempVoiceMap).length !== 0) {
- voiceMap = tempVoiceMap;
- console.log(`Voicemap updated to ${JSON.stringify(voiceMap)}`);
- }
- if (!extension_settings.tts[ttsProviderName].voiceMap) {
- extension_settings.tts[ttsProviderName].voiceMap = {};
- }
- Object.assign(extension_settings.tts[ttsProviderName].voiceMap, voiceMap);
- saveSettingsDebounced();
-}
-
-class VoiceMapEntry {
- name;
- voiceId;
- selectElement;
- constructor(name, voiceId = DEFAULT_VOICE_MARKER) {
- this.name = name;
- this.voiceId = voiceId;
- this.selectElement = null;
- }
-
- addUI(voiceIds) {
- let sanitizedName = sanitizeId(this.name);
- let defaultOption = this.name === DEFAULT_VOICE_MARKER ?
- `` :
- ``;
- let template = `
-
- ${this.name}
-
-
- `;
- $('#tts_voicemap_block').append(template);
-
- // Populate voice ID select list
- for (const voiceId of voiceIds) {
- const option = document.createElement('option');
- option.innerText = voiceId.name;
- option.value = voiceId.name;
- $(`#tts_voicemap_char_${sanitizedName}_voice`).append(option);
- }
-
- this.selectElement = $(`#tts_voicemap_char_${sanitizedName}_voice`);
- this.selectElement.on('change', args => this.onSelectChange(args));
- this.selectElement.val(this.voiceId);
- }
-
- onSelectChange(args) {
- this.voiceId = this.selectElement.find(':selected').val();
- updateVoiceMap();
- }
-}
-
-/**
- * Init voiceMapEntries for character select list.
- * @param {boolean} unrestricted - If true, will include all characters in voiceMapEntries, even if they are not in the current chat.
- */
-export async function initVoiceMap(unrestricted = false) {
- // Gate initialization if not enabled or TTS Provider not ready. Prevents error popups.
- const enabled = $('#tts_enabled').is(':checked');
- if (!enabled) {
- return;
- }
-
- // Keep errors inside extension UI rather than toastr. Toastr errors for TTS are annoying.
- try {
- await ttsProvider.checkReady();
- } catch (error) {
- const message = `TTS Provider not ready. ${error}`;
- setTtsStatus(message, false);
- return;
- }
-
- setTtsStatus('TTS Provider Loaded', true);
-
- // Clear existing voiceMap state
- $('#tts_voicemap_block').empty();
- voiceMapEntries = [];
-
- // Get characters in current chat
- const characters = getCharacters(unrestricted);
-
- // Get saved voicemap from provider settings, handling new and old representations
- let voiceMapFromSettings = {};
- if ('voiceMap' in extension_settings.tts[ttsProviderName]) {
- // Handle previous representation
- if (typeof extension_settings.tts[ttsProviderName].voiceMap === 'string') {
- voiceMapFromSettings = parseVoiceMap(extension_settings.tts[ttsProviderName].voiceMap);
- // Handle new representation
- } else if (typeof extension_settings.tts[ttsProviderName].voiceMap === 'object') {
- voiceMapFromSettings = extension_settings.tts[ttsProviderName].voiceMap;
- }
- }
-
- // Get voiceIds from provider
- let voiceIdsFromProvider;
- try {
- voiceIdsFromProvider = await ttsProvider.fetchTtsVoiceObjects();
- }
- catch {
- toastr.error('TTS Provider failed to return voice ids.');
- }
-
- // Build UI using VoiceMapEntry objects
- for (const character of characters) {
- if (character === 'SillyTavern System') {
- continue;
- }
- // Check provider settings for voiceIds
- let voiceId;
- if (character in voiceMapFromSettings) {
- voiceId = voiceMapFromSettings[character];
- } else if (character === DEFAULT_VOICE_MARKER) {
- voiceId = DISABLED_VOICE_MARKER;
- } else {
- voiceId = DEFAULT_VOICE_MARKER;
- }
- const voiceMapEntry = new VoiceMapEntry(character, voiceId);
- voiceMapEntry.addUI(voiceIdsFromProvider);
- voiceMapEntries.push(voiceMapEntry);
- }
- updateVoiceMap();
-}
-
-$(document).ready(function () {
- function addExtensionControls() {
- const settingsHtml = `
-
AllTalk is a web interface, based around the Coqui TTS voice
- cloning/speech generation system. To generate TTS, you can use the provided gradio interface or interact with
- the server using JSON/CURL commands.
-
Note: When loading up a new
- character in Text generation webUI it may look like nothing is happening for 20-30 seconds. It's actually
- processing the introduction section (greeting message) of the text and once that is completed, it will appear.
- You can see the activity occurring in the console window. Refreshing the page multiple times will try force the
- TTS engine to keep re-generating the text, so please just wait and check the console if needed.
-
Note: Ensure that your RP
- character card has asterisks around anything for the narration and double quotes around anything spoken. There
- is a complication ONLY with the greeting card so, ensuring it has the correct use of quotes and asterisks will
- help make sure that the greeting card sounds correct. I will aim to address this issue in a future update. In
- Text-generation-webUI parameters menu > character tab > greeting make
- sure that anything in there that is the narrator is in asterisks and anything spoken is in double quotes, then
- hit the save (💾) button.
-
AllTalk Minor Bug Fixes Changelog & Known
- issues
-
If I squash any minor bugs or find any issues, I will try to apply an update asap. If
- you think something isnt working correctly or you have a problem, check these two links below first.
AllTalk TTS Generator is the solution for converting large volumes of text into
- speech using the voice of your choice. Whether you're creating audio content or just want to hear text read
- aloud, the TTS Generator is equipped to handle it all efficiently.
Text Input: Enter the text you wish to convert into speech in the 'Text Input' box.
-
Generate TTS Button: Hit this to start the text-to-speech conversion.
-
Pause/Resume: Used to pause and resume the playback of the initial generation of wavs or
- the stream.
-
Stop Playback: This will stop the current audio playing back. It does not stop the text
- from being generated however.
-
-
Once you have sent text off to be generated, either as a stream or wav file
- generation, the TTS server will remain busy until this process has competed. As such, think carefully as to how
- much you want to send to the server.
-
If you are generating wav files and populating the queue, you can generate one lot of
- text to speech, then input your next lot of text and it will continue adding to the list.
-
TTS Generation Modes
-
-
Wav Chunks: Perfect for creating audio books, or anything you want to keep long term.
- Breaks down your text into manageable wav files and queues them up. Generation begins automatically, and
- playback will start after a few chunks have been prepared ahead. You can set the volume to 0 if you
- don’t want to hear playback. With Wav chunks, you can edit and/or regenerate portions of the TTS as
- needed.
-
Streaming: For immediate playback without the ability to save. Ideal for on-the-fly speech
- generation and listening. This will not generate wav files and it will play back through your browser. You
- cannot stop the server generating the TTS once it has been sent.
-
-
With wav chunks you can either playback “In Browser” which is the web
- page you are on, or “On Server” which is through the console/terminal where AllTalk is running from.
- Only generation “In Browser” can play back smoothly and populate the Generated TTS List. Setting the
- Volume will affect the volume level played back both “In Browser” and “On Server”.
-
Playback and List Management
-
-
Playback Controls: Utilize 'Play List' to start from the beginning or 'Stop Playback' to
- halt at any time.
-
Custom Start: Jump into your list at a specific ID to hear a particular section.
-
Regeneration and Editing: If a chunk isn't quite right, you can opt to regenerate it or
- edit the text directly. Click off the text to save changes and hit regenerate for the specific line.
-
Export/Import List: Save your TTS list as a JSON file or import one. Note: Existing wav
- files are needed for playback. Exporting is handy if you want to take your files away into another program
- and have a list of which wav is which, or if you keep your audio files, but want to come back at a later
- date, edit one or two lines, regenerate the speech and re-combine the wav’s into one new long wav.
-
-
-
Exporting Your Audio
-
-
Export to WAV: Combine all generated TTS from the list, into one single WAV file for easy
- download and distribution.
-
-
Customization and Preferences
-
-
Character Voice: Choose the voice that will read your text.
-
Language: Select the language of your text.
-
Chunk Sizes: Decide the size of text chunks for generation. Smaller sizes are recommended
- for better TTS quality.
-
-
Interface and Accessibility
-
-
Dark/Light Mode: Switch between themes for your visual comfort.
-
Word Count and Generation Queue: Keep track of the word count and the generation progress.
-
-
-
Notes on Usage
-
-
For seamless TTS generation, it's advised to keep text chunks under 250 characters, which you can control
- with the Chunk sizes.
-
Generated audio can be played back from the list, which also highlights the currently playing chunk.
-
The TTS Generator remembers your settings, so you can pick up where you left off even after refreshing the
- page.
Voice samples are stored in /alltalk_tts/voices/
- and should be named using the following format name.wav
-
-
Where are the outputs stored & Automatic output wav file deletion.
-
Voice outputs are stored in /alltalk_tts/outputs/
-
You can configure automatic maintenance deletion of old wav
- files by setting Del WAV's older than in the settings above.
-
When Disabled your output
- wav files will be left untouched. When set to a setting 1 Day or greater,
- your output wav files older than that time period will be automatically deleted on start-up of AllTalk.
-
-
Where are the models stored?
-
This extension will download the 2.0.2 model to /alltalk_tts/models/
-
This TTS engine will also download the latest available model
- and store it wherever your OS normally stores it (Windows/Linux/Mac).
-
-
How do I create a new voice sample?
-
To create a new voice sample, you need to make a wav file that
- is
- 22050Hz, Mono, 16 bit and between 6 to 30 seconds long, though 8 to 10 seconds is usually
- good enough. The model can handle up to 30 second samples, however I've not noticed any improvement in voice
- output from a much longer clips.
-
-
You want to find a nice clear selection of audio, so lets say
- you wanted to clone your favourite celebrity. You may go looking for an interview where they are talking. Pay
- close attention to the audio you are listening to and trying to sample. Are there noises in the background, hiss
- on the soundtrack, a low humm, some quiet music playing or something? The better quality the audio the better
- the final TTS result. Don't forget, the AI that processes the sounds can hear everything in your sample and it
- will use them in the voice its trying to recreate.
-
Try make your clip one of nice flowing speech, like the included
- example files. No big pauses, gaps or other sounds. Preferably a sample that the person you are trying to copy
- will show a little vocal range and emotion in their voice. Also, try to avoid a clip starting or ending with
- breathy sounds (breathing in/out etc).
-
-
Editing your sample!
-
So, you’ve downloaded your favourite celebrity interview off
- YouTube, from here you need to chop it down to 6 to 30 seconds in length and resample it.
-
If you need to clean it up, do audio processing, volume level
- changes etc, do this before down-sampling.
Using the latest version of Audacity select/highlight your 6 to 30 second clip and:
-
-
-
- Make sure the «Project Rate(Hz)» (lower left corner of the interface) is set to 22050 Hz.
-
-
-
- Tracks
- > Resample to 22050Hz then Tracks >
- Mix > Stereo to
- Mono then
- File > Export Audio saving it as a WAV of 22050Hz.
-
-
-
-
-
-
Save your generated wav file in the /alltalk_tts/voices/ folder.
-
Its worth mentioning that using AI generated audio clips may
- introduce unwanted sounds as its already a copy/simulation of a voice.
-
-
Why doesnt it sound like XXX Person?
-
Maybe you might be interested in trying Finetuning of the model. Otherwise, the reasons can be that you:
-
-
Didn't down-sample it as above.
-
Have a bad quality voice sample.
-
Try using the 3x different generation methods: API TTS, API Local, and XTTSv2 Local within
- the web interface, as they generate output in different ways and sound different.
-
-
Some samples just never seem to work correctly, so maybe try a
- different sample. Always remember though, this is an AI model attempting to re-create a voice, so you will never
- get a 100% match.
This only affects the Narrator function. Most AI models should
- be using asterisks or double quotes to differentiate between the Narrator or the Character, however, many models
- sometimes switch between using asterisks and double quotes or sometimes nothing at all for the text it outputs.
-
This leaves a bit of a mess because sometimes un-marked text is narration and sometimes its the
- character talking, leaving no clear way to know where to split sentences and what voice to use. Whilst there is
- no 100% solution at the moment many models lean more one way or the other as to what that unmarked text will be
- (character or narrator).
-
-
As such, the "Text not inside" function at least gives you the
- choice to set how you want the TTS engine to handle situations of un-marked text.
The Low VRAM option is a crucial feature designed to enhance
- performance under constrained (VRAM) conditions, as the TTS models require 2GB-3GB of VRAM to run effectively.
- This feature strategically manages the relocation of the Text-to-Speech (TTS) model between your system's Random
- Access Memory (RAM) and VRAM, moving it between the two on the fly. Obviously, this is very useful for people
- who have smaller graphics cards and will use all their VRAM to load in their LLM.
-
When you don't have enough VRAM free after loading your LLM
- model
- into your VRAM (Normal Mode example below), you can see that with so little working space, your GPU will have to
- swap in and out bits of the TTS model, which causes horrible slowdown.
-
Note: An Nvidia Graphics
- card is required for the LowVRAM option to work, as you will just be using system RAM otherwise.
-
-
How It Works:
-
The Low VRAM mode intelligently orchestrates the relocation of
- the entire TTS model and stores the TTS model in your system RAM. When the TTS engine requires VRAM for
- processing, the entire model seamlessly moves into VRAM, causing your LLM to unload/displace some layers,
- ensuring optimal performance of the TTS engine.
-
Post-TTS processing, the model moves back to system RAM, freeing
- up VRAM space for your Language Model (LLM) to load back in the missing layers. This adds about 1-2 seconds to
- both text generation by the LLM and the TTS engine.
-
By transferring the entire model between RAM and VRAM, the Low
- VRAM option avoids fragmentation, ensuring the TTS model remains cohesive and has all the working space it needs
- in your GPU, without having to just work on small bits of the TTS model at a time (which causes terrible slow
- down).
-
This creates a TTS generation performance Boost for Low VRAM
- Users and is particularly beneficial for users with less than 2GB of free VRAM after loading their LLM,
- delivering a substantial 5-10x improvement in TTS generation speed.
You HAVE to disable
- Enable TTS within the Text-generation-webui AllTalk interface,
- otherwise Text-generation-webui will also generate TTS due to the way it sends out text. You can do this
- each time you start up Text-generation-webui or set it in the start-up settings at the top of this page.
-
-
Quick Tips
-
-
Only change DeepSpeed, Low VRAM or Model one at a time. Wait for it to say Ready before changing something
- else.
-
You can permanently change the AllTalk startup settings or DeepSpeed, Low VRAM and Model at the top of this
- page.
-
Different AI models use quotes and asterisks differently, so you may need to change "Text not inside"
- depending on model.
-
Add new voice samples to the voices folder. You can Finetune a model to make it sound even closer to the
- original sample.
-
DeepSpeed will improve processing time to TTS to be 2-3x faster.
-
Low VRAM can be very beneficial if you don't have much memory left after loading your LLM.
-
-
TTS Generation Methods in SilllyTavern
-
You have 2 types of audio generation options, Streaming and Standard.
-
The Streaming Audio Generation method is designed for speed and
- is best suited for situations where you just want quick audio playback. This method, however, is limited to
- using just one voice per TTS generation request. This means a limitation of the Streaming method is the
- inability to utilize the AllTalk narrator function, making it a straightforward but less nuanced option.
-
On the other hand, the Standard Audio Generation method provides
- a richer auditory experience. It's slightly slower than the Streaming method but compensates for this with its
- ability to split text into multiple voices. This functionality is particularly useful in scenarios where
- differentiating between character dialogues and narration can enhance the storytelling and delivery. The
- inclusion of the AllTalk narrator functionality in the Standard method allows for a more layered and immersive
- experience, making it ideal for content where depth and variety in voice narration add significant value.
-
In summary, the choice between Streaming and Standard methods in
- AllTalk TTS depends on what you want. Streaming is great for quick and simple audio generation, while Standard
- is preferable for a more dynamic and engaging audio experience.
-
-
AllTalk TTS Generation Method:
-
-
Select between Standard and Streaming Audio Generation methods.
-
This setting impacts the AllTalk narrator functionality.
-
-
-
Language Selection:
-
-
Select your preferred TTS generation language from the "Language" dropdown.
-
-
-
Model Switching:
-
-
Switch between different TTS models like API TTS, API Local, XTTSv2 Local, and optionally XTTSv2 FT
- if you have a finetuned model available.
-
Fine-tuned model availability (XTTSv2 FT) will only show when a finetuned model is detected by
- AllTalk.
-
See TTS Models/Methods for more information (though most people will want to stick with XTTSv2
- Local).
-
-
-
DeepSpeed and Low VRAM Options:
-
-
Optimize performance with DeepSpeed and Low VRAM settings.
-
DeepSpeed can offer a 2-3x performance boost on TTS generation. (Requires installation)
-
See the relevant sections in this documentation for details.
-
-
-
-
Changing model or DeepSpeed or Low VRAM each take about 15 seconds so you should only change one at
- a time and wait for Ready before changing the next setting. To set these
- options long term you can apply the settings at the top of this page.
-
AllTalk Narrator
-
Only available on the Standard Audo Generation method.
-
-
Narrator Voice Selection:
-
-
Allows users to choose different narrator voices.
-
Access via the "Narrator Voice" dropdown.
-
-
-
AllTalk Narrator:
-
-
Toggle the AllTalk narrator feature.
-
Access via "AT Narrator" dropdown with Enabled/Disabled options.
-
-
-
Text Outside Asterisks Handling:
-
-
Choose how text outside asterisks is interpreted (as Narrator or Character voice).
-
Managed via the "Text Not Inside * or "" dropdown.
-
Note, only available when the AllTalk Narrator is enabled.
-
-
-
-
Usage Notes:
-
-
On startup of SillyTavern, it will pull your current settings from AllTalk (Current model, DeepSpeed status,
- Low VRAM status and Finetuned model availability).
-
Enabling the narrator automatically unchecks certain checkboxes related to text handling.
-
Changes in model or settings might trigger multiple requests to the server; patience is advised.
-
-
Troubleshooting:
-
-
If experiencing issues, use the Reload button in SillyTavern's TTS extension to reinitialize the connection
- to AllTalk and check if AllTalk is started correctly.
If you have a voice that the model doesnt quite reproduce correctly,
- or indeed you just want to improve the reproduced voice, then finetuning is a way to train your "XTTSv2
- local" model (stored in /alltalk_tts/models/xxxxx/) on a specific voice.
- For this you will need:
-
-
An Nvidia graphics card To install a few portions of the Nvidia CUDA 11.8 Toolkit (this
- will not impact text-generation-webui's cuda setup.
-
18GB of disk space free (most of this is used temporarily)
-
At least 2 minutes of good quality speech from your chosen speaker in mp3, wav or flacc format, in one or
- more files (have tested as far as 20 minutes worth of audio).
-
-
-
How will this work/How complicated is it?
-
Everything has been done to make this as simple as possible. At its
- simplest, you can literally just download a large chunk of audio from an interview, and tell the finetuning to
- strip through it, find spoken parts and build your dataset. You can literally click 4 buttons, then copy a few
- files and you are done. At it's more complicated end you will clean up the audio a little beforehand, but
- its still only 4x buttons and copying a few files.
-
-
The audio you will use:
-
I would suggest that if its in an interview format, you cut out the
- interviewer speaking in audacity or your chosen audio editing package. You don't have to worry about being
- perfect with your cuts, the finetuning Step 1 will go and find spoken audio and cut it out for you. Is there is
- music over the spoken parts, for best quality you would cut out those parts, though its not 100% necessary. As
- always, try to avoid bad quality audio with noises in it (humming sounds, hiss etc). You can try something like
- Audioenhancer to try clean up noisier audio. There is no need to down-sample any of the audio, all of that is
- handled for you. Just give the finetuning some good quality audio to work with.
-
-
Important requirements CUDA 11.8:
-
As mentioned you must have a small portion of the Nvidia CUDA Toolkit 11.8 installed. Not higher or
- lower versions. Specifically 11.8. You do not have to uninstall any other versions, change
- any graphics drivers, reinstall torch or anything like that. There are instructions within the finetuning
- interface on doing this or you can also find them on this link here
-
-
Starting Finetuning:
-
Ensure you have followed the instructions on setting up the Nvidia CUDA
- Toolkit 11.8 here or
- the below procedure will fail.
-
The below instructions are also available online here
-
-
-
-
Close all other applications that are using your GPU/VRAM and copy your audio samples into:
-
- /alltalk_tts/finetune/put-voice-samples-in-here/
-
-
-
-
In a command prompt/terminal window you need to move into your Text generation webUI folder:
-
- cd text-generation-webui
-
-
-
-
Start the Text generation webUI Python environment for your OS:
-
- cmd_windows.bat, ./cmd_linux.sh, cmd_macos.sh or cmd_wsl.bat
-
-
-
-
You can double check your search path environment still works correctly with nvcc --version. It should report back 11.8:
-
- Cuda compilation tools, release 11.8.
-
-
-
-
Move into your extensions folder:
-
- cd extensions
-
-
-
-
Move into the alltalk_tts folder:
-
- cd alltalk_tts
-
-
-
-
Install the finetune requirements file: pip install -r
- requirements_finetune.txt
-
-
-
Type python finetune.py and it should start up.
-
-
-
Follow the on-screen instructions when the web interface starts up.
-
-
-
When you have finished finetuning, the final tab will tell you what to do with your files and how to move
- your newly trained model to the correct location on disk.
DeepSpeed provides a 2x-3x speed boost for Text-to-Speech and AI
- tasks. It's all about making AI and TTS happen faster and more efficiently.
-
-
Model Parallelism: Spreads work across multiple GPUs, making AI/TTS models handle tasks
- more efficiently.
-
Memory Magic: Optimizes how memory is used, reducing the memory needed for large AI/TTS
- models.
-
Handles More Load: Scales up to handle larger workloads with improved performance.
-
Smart Resource Use: Uses your computer's resources smartly, getting the most out of your
- hardware.
-
-
-
DeepSpeed only works with the XTTSv2 Local model and will deactivate when other models are selected, even
- if the checkbox still shows as being selected.
-
Note: DeepSpeed/AllTalk may
- warn if the Nvidia Cuda Toolkit and CUDA_HOME environment variable isnt set correctly. On Linux you need
- CUDA_HOME configured correctly; on Windows, if you use the pre-built wheel its ok without.
-
Note:You do not need to set
- Text-generation-webUi's --deepspeed setting for AllTalk to be able to use DeepSpeed.
Preferably use your built in package manager to install the CUDA toolkit. Alternatively download and install
- the Nvidia Cuda Toolkit for Linux Nvidia Cuda
- Toolkit 11.8 or 12.1
-
Open a terminal console.
-
Install libaio-dev (however your Linux version installs things) e.g.
- sudo apt install libaio-dev
-
-
Move into your Text generation webUI folder e.g. cd text-generation-webui
-
Start the Text generation webUI Python environment ./cmd_linux.sh
-
Text generation webUI overwrites the CUDA_HOME
- environment variable each time you ./cmd_linux.sh or ./start_linux.sh so you will need to either permanently change
- within the python environment OR set CUDA_HOME it each time you ./cmd_linux.sh. Details to change it each time are on the next step.
- Below is a link to Conda's manual and changing environment variables permanently though its possible
- changing it permanently could affect other extensions, you would have to test.
- Conda manual - Environment variables
-
You can temporarily set the CUDA_HOME environment with (Standard Ubuntu
- path below, but it could vary on other Linux flavours):
export
- CUDA_HOME=/etc/alternatives/cudaevery time you run ./cmd_linux.sh
If you try to start
- DeepSpeed with the CUDA_HOME path set incorrectly, expect an error similar to:
[Errno 2] No such file or directory:
- /home/yourname/text-generation-webui/installer_files/env/bin/nvcc
-
Now install deepspeed with pip install
- deepspeed
-
You can now start Text generation webUI python server.py ensuring to
- activate your extensions.
Just to reiterate, starting Text-generation-webUI with ./start_linux.sh will overwrite the CUDA_HOME variable unless you have
- permanently changed it, hence always starting it with ./cmd_linux.shthen setting the environment
- variable manually export CUDA_HOME=/etc/alternatives/cuda and
- thenpython server.py which is how you would need
- to run it each time, unless you permanently set the environment variable for CUDA_HOME within
- Text-generation-webUI's standard Python environment.
Removal - If it became necessary to uninstall DeepSpeed,
- you can do so with ./cmd_linux.sh and then pip uninstall deepspeed
-
The atsetup utility for Windows can install DeepSpeed for you (Stanalone users whom
- installed via the atsetup utility will already have DeepSpeed installed).
-
DeepSpeed v11.2 will work on the current default text-generation-webui Python 3.11
- environment! You have 2x options for how to setup DeepSpeed on Windows. A quick way Option 1 and a long way Option 2.
-
Thanks to @S95Sedan They managed to get
- DeepSpeed 11.2 working on Windows via making some edits to the original Microsoft DeepSpeed v11.2 installation.
- The original post is here.
Download the correct wheel version for your Python/Cuda from here and save
- the file it inside your text-generation-webui folder.
-
Open a command prompt window, move into your text-generation-webui
- folder you can now start the Python environment for text-generation-webui cmd_windows.bat
-
With the file that you saved in the text-generation-webui folder you
- now type the following, replacing your-version with the name of the
- file you have:
This should install through cleanly and you should now have DeepSpeed v11.2 installed within the Python
- 3.11/3.10 environment of text-generation-webui.
-
When you start up text-generation-webui, and AllTalk starts, you should see:
- [AllTalk Startup] DeepSpeed
- Detected
-
Within AllTalk, you will now have a checkbox for Activate DeepSpeed
- though remember you can only change 1x setting every 15 or so seconds, so don't try to activate DeepSpeed
- and
- LowVRAM simultaneously. When you are happy it works, you can set the default start-up settings in the
- settings page.
Removal - If it became necessary to uninstall
- DeepSpeed, you can do so with cmd_windows.bat and then pip uninstall deepspeed
OPTION 2 - Manual build of DeepSpeed v11.2
- (Python 3.11 and 3.10)
Due to the
- complexity of this, and the complicated formatting, instructions can be found on this link
It's worth noting that all models and methods can and do sound
- different from one another. Many people complained about the quality of audio produced by the 2.0.3 model, so
- this extension will download the 2.0.2 model to your models folder and give you the choice to use 2.0.2 (API Local and XTTSv2 Local) or use the
- most current model 2.0.3 (API TTS). As/When a new model is released by
- Coqui it will be downloaded by the TTS service on startup and stored wherever the TTS service keeps new models
- for your operating system.
-
-
API TTS: Uses the current TTS model available that's downloaded by the
- TTS API process ( version 2.0.3 at the time of writing). This model is not stored in your "models" folder,
- but elsewhere on your system and managed by the TTS software.
-
API Local: Utilizes the 2.0.2 local model stored at /alltalk_tts/models/xttsv2_2.0.2
-
XTTSv2 Local: Utilizes the 2.0.2 local model /alltalk_tts/models/xttsv2_2.0.2 and utilizes a distinct TTS generation
- method. Supports DeepSpeed acceleration.
It is recommended not to modify these settings unless you
- fully comprehend their effects. A general overview is provided below for reference.
-
Changes to these settings won't take effect until you
- restart AllTalk/Text generation webUI.
-
These settings only affect API Local and XTTSv2 Local
- methods.
-
-
Repetition Penalty:
-
In the context of text-to-speech (TTS), the Repetition
- Penaltyinfluences how the model handles the repetition of sounds, phonemes, or
- intonation patterns. Here's how it works:
-
-
Higher Repetition Penalty (e.g. 16.0): The model is less likely to repeat sounds or
- patterns. It promotes diversity in the generated speech. This can result in a more varied and expressive
- output, though introduce elements of unpredictability in the TTS output.
-
Lower Repetition Penalty (e.g. 2.0): The model is more tolerant of repeating sounds or
- patterns. This might lead to more repetition in the generated speech, potentially making it sound more
- structured or rhythmically consistent. Lower values can still introduce expressive variations, but to a
- lesser extent. This tendency means that the generated speech may remain closer to the original sample.
-
-
-
The factory setting for repetition penalty is 10.0
-
-
Temperature:
-
Temperature influences the randomness of the generated
- speech. Here's how it affects the output:
-
-
Higher Temperature (e.g. 0.95): Increases randomness in how the model selects and
- pronounces phonemes or intonation patterns. This can result in more creative, but potentially less
- controlled or "stable," speech that may deviate from the input sample. It adds an element of
- unpredictability and variety, contributing to expressiveness in the voice output created.
-
Lower Temperature (e.g. 0.20): Reduces randomness, making the model more likely to
- closely mimic the input sample's voice, intonation, and overall style. This tends to produce more
- "coherent" speech that aligns closely with the characteristics of the training data or input voice
- sample. It adds a level of predictability and consistency, potentially reducing expressive variations.
- So it could end up sounding too monotone.
-
-
The factory setting for temperature is 0.70
-
-
Temperature and Repetition Settings Examples:
-
-
Temp High (0.90) and Repetition High (16.0): Result: Speech may sound highly
- creative and diverse, with reduced repetition. It could be more expressive and unpredictable.
-
Temp Low (0.20) and Repetition High (16.0): Result: Output tends to be focused
- and deterministic, but with reduced repetition. It may sound structured and less expressive.
-
Temp High (0.90) and Repetition Low (2.0): Result: Speech may be more creative
- and diverse, with tolerance for repeating sounds. It could have expressive variations but with some
- structured patterns.
-
Temp Low (0.20) and Repetition Low (2.0): Result: Output is focused and
- deterministic, with tolerance for repeating sounds. It may sound more structured and less expressive.
-
-
-
Factory settings should be fine for most people, however if you choose to
- experiment, setting extremely high or low values, especially without a good understanding of their effects,
- may lead to flat-sounding output or very strange-sounding output. It's advisable to experiment with
- adjustments incrementally and observe the impact on the generated speech to find a balance that suits your
- desired outcome.
AllTalk performs a variety of checks on startup and will
- warn out messages at the console should you need to do something such as update your TTS version.
-
A basic environment check to ensure everything should work
- e.g. is the model already downloaded, are the configuration files set correctly etc.
-
AllTalk will download the Xtts model (version 2.0.2) into
- your models folder. Many people didn't like the quality of the 2.0.3 model, however the latest model will be
- accessible on the API TTS setting (2.0.3 at the time of writing) so you have the best of both worlds.
Its possible to set a custom model for the API Local and XTTSv2 Local methods,
- or indeed point it at the same model that API TTS uses (wherever it is
- stored on your OS of choice).
-
Many people did not like the sound quality of the Coqui
- 2.0.3 model, and as such the AllTalk downloads the 2.0.2 model separately from the 2.0.3 model that TTS service downloads and manages.
-
-
Typically the 2.0.2
- model is stored in your /alltalk_tts/models folder and it is downloaded
- on first start-up of the AllTalk_tts extension. However, you may either want to use a custom model
- version of your choosing, or re-point AllTalk to a different path on your system, or even point it so
- that API Local and XTTSv2
- Local both use the same model that API TTS is using.
-
If you do choose to change the location there are a couple
- of things to note.
-
-
The folder you place the model in, cannot be called "models". This name is reserved solely for the system to identify you
- are or are not using a custom model.
-
On each startup, the AllTalk tts extension will check the custom location and if it does not exist,
- it will create it and download the files it needs. It will also re download any missing files in that
- location that are needed for the model to function.
-
There will be extra output at the console to inform you that you are using a custom model and each time
- you load up AllTalk extension or switch between models.
-
-
To change the model path, there are at minimum 2x settings
- you need to alter in the modeldownload.json file, base_path and model_path.
-
You would edit the settings in the modeldownload.json file as follows (make a backup of your current file in
- case):
Windows path example: c:\\mystuff\\mydownloads\\myTTSmodel\\{files in here}
- base_path would be "c:\\mystuff\\mydownloads" model_path would be: "myTTSmodel"
-
Note: On Windows systems, you have to
- specify a double backslash
- \\ for each folder level in the path (as above)
-
Linux path example: /home/myaccount/myTTSmodel/{files in
- here} base_path
- would be "/home/myaccount"
- model_path would be: "TTSmodel"
Once you restart Alltalk, it will check this
- path for the files and output any details at the console.
-
When you are happy it's' working correctly, you are welcome to go delete the
- models folder stored at /alltalk_tts/models.
-
If you wish to change the files that the modeldownloader is pulling at startup,
- you can further edit the modeldownload.json and change the https
- addresses within this files files_to_download section e.g.
-
Sets the model files required to be downloaded into \base_path\model_path\ and where
- to download them from.
-
-
-
-
JSON calls & CURL Commands
-
-
Overview
-
The Text-to-Speech (TTS) Generation API allows you to
- generate speech from text input using various configuration options. This API supports both character and
- narrator voices, providing flexibility for creating dynamic and engaging audio content.
-
-
TTS Generation Endpoint
-
-
URL: http://127.0.0.1:7851/api/tts-generate
-
Method: POST
-
Content-Type: application/x-www-form-urlencoded
-
-
-
-
Example command line
-
Standard TTS speech Example (standard text) generating a time-stamped file
-
curl -X POST
- "http://127.0.0.1:7851/api/tts-generate" -d "text_input=All of this
- is text spoken by the character. This is text not inside quotes, though that doesnt matter in the
- slightest" -d "text_filtering=standard" -d
- "character_voice_gen=female_01.wav" -d "narrator_enabled=false" -d "narrator_voice_gen=male_01.wav" -d "text_not_inside=character" -d
- "language=en" -d "output_file_name=myoutputfile" -d "output_file_timestamp=true" -d "autoplay=true" -d
- "autoplay_volume=0.8"
-
Narrator Example (standard text) generating a time-stamped file
-
curl -X POST
- "http://127.0.0.1:7851/api/tts-generate" -d "text_input=*This is
- text spoken by the narrator* \"This is text spoken by the character\". This is text not inside
- quotes." -d "text_filtering=standard" -d
- "character_voice_gen=female_01.wav" -d "narrator_enabled=true" -d "narrator_voice_gen=male_01.wav" -d
- "text_not_inside=character" -d
- "language=en" -d "output_file_name=myoutputfile" -d "output_file_timestamp=true" -d "autoplay=true" -d
- "autoplay_volume=0.8"
-
Note
- that if your text that needs to be generated contains double quotes you will need to escape them
- with \" (Please see the narrator example).
-
-
Request Parameters
-
-
-
text_input: The text you want the TTS engine to
- produce. Use escaped double quotes for character speech and asterisks for narrator speech if using
- the narrator function. Example:
-
-d "text_input=*This is text spoken by the narrator* \"This is
- text spoken by the character\". This is text not inside quotes."
-
-
-
text_filtering: Filter for text. Options:
-
-
none No filtering.
- Whatever is sent will go over to the TTS engine as raw text, which may result in some odd sounds
- with some special characters.
-
standard Human-readable text and a basic level of
- filtering, just to clean up some special characters.
-
html HTML content. Where you are using HTML entity's
- like "
Standard Example: *This is text spoken by the narrator* "This is
- text spoken by the character" This is text not inside quotes.
-
HTML Example: *This is text spoken by the
- narrator* "This is text spoken by the character" This is text
- not inside quotes.
-
None will just pass whatever characters/text you send at it.
-
-
-
-
character_voice_gen: The WAV file name for the character's voice.
-
-d "character_voice_gen=female_01.wav"
-
-
-
narrator_enabled: Enable or disable the narrator function. If true, minimum text
- filtering is set to standard. Anything between double quotes is considered the character's speech,
- and anything between asterisks is considered the narrator's speech.
output_file_name: The name of the output file (excluding the .wav extension).
-
-d "output_file_name=myoutputfile"
-
-
-
output_file_timestamp: Add a timestamp to the output file name. If true, each file
- will have a unique timestamp; otherwise, the same file name will be overwritten each time you
- generate TTS.
curl -X POST
- "http://127.0.0.1:7851/api/reload?tts_method=API%20Local"
- curl -X POST "http://127.0.0.1:7851/api/reload?tts_method=API%20TTS" curl -X POST
- "http://127.0.0.1:7851/api/reload?tts_method=XTTSv2%20Local"
-
Switch between the 3 models respectively.
-
JSON return {"status": "model-success"}
-
-
Switch DeepSpeed
-
curl -X POST
- "http://127.0.0.1:7851/api/deepspeed?new_deepspeed_value=True"
-
Replace True with False to disable DeepSpeed mode.
-
JSON return {"status": "deepspeed-success"}
-
-
-
Switching Low VRAM
-
curl -X POST
- "http://127.0.0.1:7851/api/lowvramsetting?new_low_vram_value=True"
-
Replace True with False to disable Low VRAM mode.
-
JSON return {"status": "lowvram-success"}
-
-
-
Ready Endpoint
-
Check if the Text-to-Speech (TTS) service is ready to accept requests.
-
-
URL: http://127.0.0.1:7851/api/ready
-
Method: GET
-
Response: Ready
-
-
curl -X GET
- "http://127.0.0.1:7851/api/ready"
-
-
Voices List Endpoint
-
Retrieve a list of available voices for generating speech.
Command line outputs are more verbose to assist in
- understanding backend processes and debugging.
-
Its possible during startup you can get a warning message
- such as:
[AllTalk
- Startup]WarningTTS Subprocess
- has NOT started up yet, Will keep trying for 80 seconds maximum
This is normal
- behaviour if the subprocess is taking a while to start, however, if there is an issue starting the
- subprocess, you may see multiples of this message and an it will time out after 80 seconds, resulting in the
- TTS engine not starting. Its likely that you are not in the correct python environment or one that has a TTS
- engine inside, if this happens, though the system will output a warning about that ahead of this message
-
Typically, the command line console will output any warning
- or error messages. If you need to reset your default configuration, the settings are all listed above in the
- configuration details.
