Merge pull request #655 from Paladinium/fix_disk_space

Make free disk space to also consider large packages and swap storage

.github/actions/docker-setup/action.yml

@@ -0,0 +1,73 @@
+name: 'Setting up docker'
+
+inputs:
+  DOCKERHUB_USERNAME:
+    description: "Docker hub username"
+    required: true
+  DOCKERHUB_REPO_NAME:
+    description: "Docker hub repo name"
+    required: true
+  DOCKERHUB_TOKEN:
+    description: "Docker hub token"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Free Disk Space (Ubuntu)
+      uses: jlumbroso/free-disk-space@v1.3.1
+      with:
+        tool-cache: true
+        swap-storage: true
+        large-packages: true
+    - name: Read variables
+      shell: bash
+      run: |
+        source ./docker/variables.sh
+        for key in "${!ALLTALK_VARS[@]}"
+        do
+          echo "${key}=${ALLTALK_VARS[${key}]}"
+          echo "${key}=${ALLTALK_VARS[${key}]}" >> $GITHUB_ENV
+        done
+
+    - name: Extract version from Git Tag
+      shell: bash {0}
+      run: |
+        # Use the git tag if existing, otherwise 'latest':
+        DOCKER_TAG=$( echo "${{ github.ref }}" | grep -q tags | cut -d'/' -f 3 )
+        if [ -z "${DOCKER_TAG}" ]; then
+          DOCKER_TAG="latest"
+        fi
+        echo "Using DOCKER_TAG=${DOCKER_TAG}"
+        echo "DOCKER_TAG=${DOCKER_TAG}" >> $GITHUB_ENV
+
+    - name: Extract Docker Hub User and Repo Name
+      shell: bash
+      env:
+        DOCKERHUB_USERNAME: ${{ inputs.DOCKERHUB_USERNAME }}
+        DOCKERHUB_REPO_NAME: ${{ inputs.DOCKERHUB_REPO_NAME }}
+      run: |
+        if [ -n "${DOCKERHUB_USERNAME}" ]; then
+          echo "Using Github repository variable 'DOCKERHUB_USERNAME' as Docker hub username with value: '${DOCKERHUB_USERNAME}'"
+          echo "DOCKERHUB_USERNAME=${DOCKERHUB_USERNAME}" >> $GITHUB_ENV
+        else
+          echo "Using Github repository name as Docker hub username"
+          echo "DOCKERHUB_USERNAME=$(echo ${{ github.repository }} | cut -d'/' -f 1)" >> $GITHUB_ENV
+        fi
+
+        if [ -n "${DOCKERHUB_REPO_NAME}" ]; then
+          echo "Using Github repository variable 'DOCKERHUB_REPO_NAME' as Docker hub repo name with value: '${DOCKERHUB_REPO_NAME}'"
+          echo "DOCKERHUB_REPO_NAME=${DOCKERHUB_REPO_NAME}" >> $GITHUB_ENV
+        else
+          echo "Using Github repository name as Docker hub repo name"
+          echo "DOCKERHUB_REPO_NAME=$(echo ${{ github.repository }} | cut -d'/' -f 2)" >> $GITHUB_ENV
+        fi
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ inputs.DOCKERHUB_USERNAME }}
+        password: ${{ inputs.DOCKERHUB_TOKEN }}

.github/workflows/publish-docker-v2.yml

@@ -0,0 +1,124 @@
+name: Publish Docker v2
+
+on:
+  workflow_dispatch:
+  release:
+    types: [ published ]
+  push:
+    branches:
+      - 'alltalkbeta'
+    tags:
+      - 'v*'
+
+# Limit the concurrency of entire workflow runs for a specific branch:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build_base_environment_docker_image:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup docker
+        uses: ./.github/actions/docker-setup
+        with:
+          DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }}
+          DOCKERHUB_REPO_NAME: ${{ vars.DOCKERHUB_REPO_NAME }}
+          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ${{ env.DOCKERHUB_USERNAME }}/${{ env.DOCKERHUB_REPO_NAME }}_environment
+          tags: |
+            # Use sematic versioning on push tag event:
+            type=semver,pattern={{version}}
+            # set latest tag for main and alltalkbeta branch:
+            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') || github.ref == format('refs/heads/{0}', 'alltalkbeta') }}
+
+      - uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          base: 'alltalkbeta'
+          filters: |
+            docker-base:
+              - 'docker/variables.sh'
+              - 'docker/base/Dockerfile'
+            docker-workflow:
+              - '.github/workflows/publish-docker*.yml'
+
+      - name: Build and push base Docker image
+        if: |
+          steps.changes.outputs.docker-base == 'true' ||
+          steps.changes.outputs.docker-workflow == 'true' ||
+          github.event_name == 'release' ||
+          (github.event_name == 'push' && contains(github.ref, 'refs/tags/'))
+        uses: docker/build-push-action@v6
+        with:
+          context: ./docker/base
+          file: ./docker/base/Dockerfile
+          platforms: "linux/amd64"
+          push: true
+          build-args: |
+            CUDA_VERSION=${{ env.CUDA_VERSION }}
+            PYTHON_VERSION=${{ env.PYTHON_VERSION }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=registry,ref=${{ env.DOCKERHUB_USERNAME }}/${{ env.DOCKERHUB_REPO_NAME }}_environment:buildcache
+          cache-to: type=registry,ref=${{ env.DOCKERHUB_USERNAME }}/${{ env.DOCKERHUB_REPO_NAME }}_environment:buildcache,mode=max
+
+  build_main_docker_image:
+    needs: build_base_environment_docker_image
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tts: [ xtts, vits, piper ]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup docker
+        uses: ./.github/actions/docker-setup
+        with:
+          DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }}
+          DOCKERHUB_REPO_NAME: ${{ vars.DOCKERHUB_REPO_NAME }}
+          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ${{ env.DOCKERHUB_USERNAME }}/${{ env.DOCKERHUB_REPO_NAME }}
+          flavor: |
+            suffix=-${{ matrix.tts }},onlatest=true
+          tags: |
+            # Use sematic versioning on push tag event:
+            type=semver,pattern={{version}}
+            # set latest tag for main and alltalkbeta branch:
+            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') || github.ref == format('refs/heads/{0}', 'alltalkbeta') }}
+
+      - name: Build and push base Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: "linux/amd64"
+          push: true
+          build-args: |
+            TTS_MODEL=${{ matrix.tts }}
+            ALLTALK_DIR=${{ env.ALLTALK_DIR }}
+            DEEPSPEED_VERSION=${{ env.DEEPSPEED_VERSION }}
+            DOCKER_TAG=${{ env.DOCKER_TAG }}
+            DOCKER_REPOSITORY=${{ env.DOCKERHUB_USERNAME }}/
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=registry,ref=${{ env.DOCKERHUB_USERNAME }}/${{ env.DOCKERHUB_REPO_NAME }}:buildcache-${{ matrix.tts }}
+          cache-to: type=registry,ref=${{ env.DOCKERHUB_USERNAME }}/${{ env.DOCKERHUB_REPO_NAME }}:buildcache-${{ matrix.tts }},mode=max

DOCKER_README.md

@@ -1,21 +1,19 @@
 # Docker
-The Docker image currently works on Windows and Linux, optionally supporting NVIDIA GPUs.
+The Docker image currently works on Windows and Linux supporting NVIDIA GPUs.
 
-## General Remarks
-- The resulting Docker image is 21 GB in size. Building might require even more disk space temporarily.
-    - Another 15 GB is required for building DeepSpeed
-- Build time depends on your hardware and internet connection. Expect at least 20-30min to be normal for a full build.
-  - This includes building the conda environment as well as DeepSpeed, which is the basis for the alltalk Docker image.
-- The Docker build for alltalk:
-  - Downloads XTTS as default TTS engine
-  - Enables RVC by default
-  - Downloads all supported RVC models
-  - Enables deepspeed by default
-- Starting the Docker image should only a few seconds due to all the steps that were already executed during build.
+## System requirements
+- AMD compatible Windows or Linux system
+- CUDA 12.8
+- Nvidia Docker Container Toolkit
+- At least 25GB of free disk space
 
-## Docker for Linux
+## Quickstart
+Use `docker-start.sh` to pull the latest docker image using the XTTS model from Docker Hub and 
+visit `http://localhost:7851/`. See instructions below for passing more arguments to the start script.
 
-### Ubuntu Specific Setup for GPUs
+## Installing Prerequisites
+### Docker for Linux
+#### Ubuntu Specific Setup for GPUs
 1. Make sure the latest nvidia drivers are installed: `sudo ubuntu-drivers install`
 1. Install Docker your preferred way. One way to do it is to follow the official documentation [here](https://docs.docker.com/engine/install/ubuntu/#uninstall-old-versions).
     - Start by uninstalling the old versions
@@ -27,8 +25,8 @@ The Docker image currently works on Windows and Linux, optionally supporting NVI
       ```sudo nvidia-ctk runtime configure --runtime=docker```
     - Restart docker
 
-## Docker for Windows (WSL2)
-### Windows Specific Setup for GPUs
+### Docker for Windows (WSL2)
+#### Windows Specific Setup for GPUs
 > Make sure your Nvidia drivers are up to date: https://www.nvidia.com/download/index.aspx
 1. Install WSL2 in PowerShell with `wsl --install` and restart
 2. Open PowerShell, type and enter ```ubuntu```.  It should now load you into wsl2
@@ -36,31 +34,16 @@ The Docker image currently works on Windows and Linux, optionally supporting NVI
 4. Download CUDA toolkit keyring: `wget https://developer.download.nvidia.com/compute/cuda/repos/wsl-ubuntu/x86_64/cuda-keyring_1.1-1_all.deb`
 5. Install keyring: `sudo dpkg -i cuda-keyring_1.1-1_all.deb`
 6. Update package list: `sudo apt-get update`
-7. Install CUDA toolkit: `sudo apt-get -y install cuda-toolkit-12-6`
+7. Install CUDA toolkit: `sudo apt-get -y install cuda-toolkit-12-8`
 8. Install Docker Desktop using WSL2 as the backend
 9. Restart
 10. If you wish to monitor the terminal remotely via SSH, follow [this guide](https://www.hanselman.com/blog/how-to-ssh-into-wsl2-on-windows-10-from-an-external-machine).
-11. Open PowerShell, type ```ubuntu```, [then follow below](#building-and-running-in-docker)
+11. Open PowerShell, type ```ubuntu```, [then follow below](#quickstart)
 
-## Building and Running in Docker
+## Arguments for starting the Docker container
+To make it as simple as possible to use Docker, the script `docker-start.sh` can be used. It provides the 
+following optional arguments:
 
-1. Open a terminal (or Ubuntu WSL) and go where you cloned the repo
-3. Build the image with `./docker-build.sh`
-4. Start the container with `./docker-start.sh`
-5. Visit `http://localhost:7851/` or remotely with `http://<ip>:7851`
-
-## Arguments for building and starting docker
-There are various arguments to customize the build and start of the docker image.
-
-### Arguments for `docker-build.sh`
-- `--tts_model` allows to choose the TTS model that is used by default. Valid values are `piper`, `vits`, `xtts`. Defaults to `xtts`.
-  - Example: `docker-build.sh --tts_model piper`
-- `--tag` allows to choose the docker tag. Defaults to `latest`.
-  - Example: `docker-build.sh --tag mytag`
-- `--clean` allows remove existing dependency build like conda environment or DeepSpeed.
-    - Example: `docker-build.sh --clean`
-
-### Arguments for `docker-start.sh`
 - `--config` lets you choose a config JSON file which can subset of `confignew.json`. This allows you to change only
   few values and leave the rest as defined in the default `confignew.json` file.
     - Example: `docker-start.sh --config /my/config/file.json` with content `{"branding": "My Brand "}` will just change
@@ -68,11 +51,65 @@ There are various arguments to customize the build and start of the docker image
 - `--voices` lets you add voices for the TTS engine in WAV format. You have to specify the folder containing all
   voice files.
     - Example: `docker-start.sh --voices /my/voices/dir`
-- `--rvc_voices` similar to voices, this option lets you pick the folder containing the RVC models.
+- `--rvc-voices` similar to voices, this option lets you pick the folder containing the [RVC models](https://github.com/erew123/alltalk_tts/wiki/RVC-(Retrieval%E2%80%90based-Voice-Conversion)).
     - Example: `docker-start.sh --rvc_voices /my/rvc/voices/dir`
-- `--no_ui` allows you to not expose port 7852 for the gradio interface. Note that you still have to set `launch_gradio`
-  to `false` via JSON file passed to `--config`.
-- `--tag` allows to choose the docker tag of the image to run. Defaults to `latest`.
+- `--no-ui` allows you to not expose port 7852 for the gradio interface and sets `launch_gradio` and `gradio_interface` to `false`.
+- `--auto-cleanup` periodically deletes all generated WAV files that are older than 1min.
+- `--with-multi-engine-manager` enables the use of the [multi engine manager (MEM)](https://github.com/erew123/alltalk_tts/wiki/Multi-Engine-Manager)
+    which allows for more parallel requests. By default, one TTS engine is started. Optionally, you can pass the
+    file path of a JSON file which can be a subset of `mem_config.json` with more fine-grained configuration options.
+  - Example: `docker-start.sh --with-multi_engine_manager` to use MEM with default settings or 
+    `docker-start.sh --with-multi_engine_manager /my/config/file.json` to pass a JSON file containing more settings.
+- `--tag` allows to choose the docker tag of the image to run. Defaults to `latest-xtts`.
     - Example: `docker-start.sh --tag mytag`
-- Since the above commands only address the most important options, you might pass additional arbitrary docker commands
+- `--docker-repository` allows to choose another Docker repository for pulling the image from. Use an empty 
+  string for the local repo.
+- Since the above commands only address the most important options, you might pass additional arbitrary docker arguments
   to the `docker-start.sh`.
+
+Of course, like for any other Docker image, you can also directly use `docker run` directly and use 
+[docker-start.sh](https://github.com/erew123/alltalk_tts/blob/alltalkbeta/docker-start.sh) as an inspiration.
+
+### Using environment variables for setting config values
+Instead of using config JSON files, there are config settings that can be directly set using environment (ENV) variables 
+when starting the application. This is also what is used internally in `docker-start.sh`.
+
+Since `docker-start.sh` allows to pass arbitrary params, it can also be used to set env variables, e.g:
+
+```
+./docker-start.sh -e ALLTALK_BRANDING="My Brand " -e ALLTALK_API_PORT_NUMBER=9876
+```
+
+or when using Docker directly:
+```
+docker run -e ALLTALK_BRANDING="My Brand " -e ALLTALK_API_PORT_NUMBER=9876 ...
+```
+
+Consult [docker_default_confignew.json](https://github.com/erew123/alltalk_tts/blob/alltalkbeta/docker_default_confignew.json) 
+and [docker_default_mem_config.json](https://github.com/erew123/alltalk_tts/blob/alltalkbeta/docker_default_mem_config.json) 
+for a list of all available env variables.
+
+
+## Building your own images (advanced)
+> Be aware that building the images requires good hardware and bandwidth. Expect this process to require >
+> 40 GB of disk space and > 30min execution time depending on your hardware.
+
+Under normal circumstances, there should be no need to build the Docker images locally. However, if needed for some
+reason, you may want to use `docker-build.sh` with the following arguments:
+
+- `--tts_model` allows to choose the TTS model that is used by default. Valid values are `piper`, `vits`, `xtts`. Defaults to `xtts`.
+    - Example: `docker-build.sh --tts_model piper`
+- `--tag` allows to choose the docker tag. Defaults to `latest-xtts`.
+    - Example: `docker-build.sh --tag mytag`
+- `--docker-repository` allows to choose another Docker repository for tagging the image from. Use an empty
+  string for the local repo.
+- `--local-deepspeed-build` should only be used in rare cases where DeepSpeed needs to be rebuilt. Defaults to `false`.
+- `--clean` allows remove existing dependency build like conda environment or DeepSpeed. Defaults to `false`.
+    - Example: `docker-build.sh --clean`
+
+
+# Older images
+- CUDA 12.6:
+  - XTTS  Dockerhub hash: sha256:fd4741714af0c82e4e7a66bf9ae47c98f3e51ebefdb3af5795b47d41ca5fb159
+  - VITS Dockerhub hash: sha256:9f411e8fdb997486bc217fcd316bf6b163c39736655ea3400028ad4fcc5c60f9
+  - Piper:  Dockerhub hash: sha256:6ae2520051bb00a0356dc4ed80a33bf6784cb567b59759af46046ddcf356b765

Dockerfile

@@ -1,32 +1,81 @@
-ARG GITHUB_REPOSITORY
-FROM ${GITHUB_REPOSITORY}alltalk_environment:latest
+ARG DOCKER_REPOSITORY
+ARG DOCKER_TAG=latest
+FROM ${DOCKER_REPOSITORY}alltalk_tts_environment:${DOCKER_TAG}
 
 # Argument to choose the model: piper, vits, xtts
 ARG TTS_MODEL="xtts"
 ENV TTS_MODEL=$TTS_MODEL
 
-ARG DEEPSPEED_VERSION=0.16.2
+ARG DEEPSPEED_VERSION=0.17.2
 ENV DEEPSPEED_VERSION=$DEEPSPEED_VERSION
 
-ARG ALLTALK_DIR=/opt/alltalk
-
 ENV GRADIO_SERVER_NAME="0.0.0.0"
 
+# Switch for using the multi engine manager:
+ENV ALLTALK_ENABLE_MULTI_ENGINE_MANAGER=false
+
+# Default settings used in docker_default_confignew.json:
+ENV ALLTALK_DELETE_OUTPUT_WAVS="1"
+ENV ALLTALK_GRADIO_INTERFACE=true
+ENV ALLTALK_RVC_ENABLED=true
+ENV ALLTALK_RVC_F0METHOD="rmvpe"
+
+# Default settings used in docker_default_mem_config.json:
+ENV ALLTALK_MEM_AUTO_START_ENGINES=1
+ENV ALLTALK_MEM_MAX_INSTANCES=1
+
 WORKDIR ${ALLTALK_DIR}
 
+##############################################################################
+# Download TTS models:
+##############################################################################
+COPY system/tts_engines/piper/available_models.json system/tts_engines/piper/
+COPY system/tts_engines/vits/available_models.json system/tts_engines/vits/
+COPY system/tts_engines/xtts/available_models.json system/tts_engines/xtts/
+RUN <<EOR
+    available_models="system/tts_engines/${TTS_MODEL}/available_models.json"
+    first_start_model=$(cat ${available_models} | jq -r '.first_start_model')
+    model=$(cat ${available_models} | jq ".models[] | select(.model_name==\"${first_start_model}\")")
+    folder_path=$( echo "${model}" | jq -r '.folder_path // empty' )
+    files_to_download=$( echo "${model}" | jq '.files_to_download[]?' )
+    github_rls_url=$( echo "${model}" | jq '.github_rls_url // empty' )
+
+    # Special case for vits: firstrun.py expects the ZIP download directly in the model folder.
+    # When firstrun.py unzips the file, the correct folder will be created automatically.
+    if [[ "$TTS_MODEL" == "vits" ]]; then
+      folder_path=""
+    fi
+
+    # Merging both fields:
+    files_to_download="${files_to_download} ${github_rls_url}"
+    target_dir="models/${TTS_MODEL}/${folder_path}"
+
+    echo "$files_to_download" | xargs -n 1 curl --create-dirs --output-dir "${target_dir}" -LO
+EOR
+
+##############################################################################
+# Download all RVC models:
+##############################################################################
+COPY system/tts_engines/rvc_files.json system/tts_engines/
+RUN jq -r '.[]' system/tts_engines/rvc_files.json | xargs -n 1 curl --create-dirs --output-dir models/rvc_base -LO
+
 ##############################################################################
 # Install python dependencies (cannot use --no-deps because requirements are not complete)
 ##############################################################################
-COPY system/config system/config
+COPY system/config/*.whl system/config/
 COPY system/requirements/requirements_standalone.txt system/requirements/requirements_standalone.txt
 COPY system/requirements/requirements_parler.txt system/requirements/requirements_parler.txt
 ENV PIP_CACHE_DIR=${ALLTALK_DIR}/pip_cache
 RUN <<EOR
     conda activate alltalk
 
-    mkdir ${ALLTALK_DIR}/pip_cache
+    mkdir -p ${ALLTALK_DIR}/pip_cache
     pip install --no-cache-dir --cache-dir=${ALLTALK_DIR}/pip_cache -r system/requirements/requirements_standalone.txt
     pip install --no-cache-dir --cache-dir=${ALLTALK_DIR}/pip_cache --upgrade gradio==4.32.2
+
+    # By default, version 1.9.10 is used causing this warning on startup: 'FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated'
+    pip install --no-cache-dir --cache-dir=${ALLTALK_DIR}/pip_cache local-attention==1.11.1
+
     # Parler:
     pip install --no-cache-dir --cache-dir=${ALLTALK_DIR}/pip_cache -r system/requirements/requirements_parler.txt
 
@@ -36,12 +85,23 @@ EOR
 ##############################################################################
 # Install DeepSpeed
 ##############################################################################
-RUN mkdir -p /tmp/deepseped
-COPY docker/deepspeed/build/*.whl /tmp/deepspeed/
+RUN mkdir -p /tmp/deepspeed
+COPY docker/deepspeed/build*/*.whl /tmp/deepspeed/
 RUN <<EOR
-    DEEPSPEED_WHEEL=$(realpath /tmp/deepspeed/*.whl)
+    DEEPSPEED_WHEEL=$(realpath -q /tmp/deepspeed/*.whl)
     conda activate alltalk
 
+    # Download DeepSpeed wheel if it was not built locally:
+    if [ -z "${DEEPSPEED_WHEEL}" ] || [ ! -f $DEEPSPEED_WHEEL ] ; then
+      echo "Downloading pre-built DeepSpeed wheel"
+      CURL_ERROR=$( { curl --output-dir /tmp/deepspeed -fLO "https://github.com/erew123/alltalk_tts/releases/download/DeepSpeed-for-docker/deepspeed-0.17.2+15f054d9-cp311-cp311-linux_x86_64.whl" ; } 2>&1 )
+      if [ $? -ne 0 ] ; then
+        echo "Failed to download DeepSpeed: $CURL_ERROR"
+        exit 1
+      fi
+      DEEPSPEED_WHEEL=$(realpath -q /tmp/deepspeed/*.whl)
+    fi
+
     echo "Using precompiled DeepSpeed wheel at ${DEEPSPEED_WHEEL}"
     CFLAGS="-I$CONDA_PREFIX/include/" LDFLAGS="-L$CONDA_PREFIX/lib/" \
       pip install --no-cache-dir ${DEEPSPEED_WHEEL}
@@ -63,24 +123,55 @@ RUN <<EOR
 #!/usr/bin/env bash
 source ~/.bashrc
 
-# Merging config from docker_confignew.json into confignew.json:
-jq -s '.[0] * .[1] * .[2]' confignew.json docker_default_config.json docker_confignew.json  > confignew.json.tmp
-mv confignew.json.tmp confignew.json
+replace_env_vars() {
+  echo "{}" > /tmp/empty.json
+  jq "\$( cat \$1 ) | del(.. | select(. == null))" /tmp/empty.json > \$1.tmp
+  mv \$1.tmp \$1
+  rm -f /tmp/empty.json
+}
 
-conda activate alltalk
+merge_json_files() {
+  # Merging JSON config with docker default values followed by values set on startup:
+  jq -s '.[0] * .[1] * .[2]' \$1 docker_default_\$1 docker_\$1  > \$1.tmp
+  mv \$1.tmp \$1
+}
+
+replace_env_vars docker_default_confignew.json
+merge_json_files confignew.json
+
+# Script for deleting WAV files that are older than 1 minute:
+cat << EOF2 > ${ALLTALK_DIR}/cleanup-wavs.sh
+#!/usr/bin/env bash
+[ \"\$ALLTALK_AUTO_CLEANUP\" = \"true\" ] && find ${ALLTALK_DIR}/outputs -type f -name "*.wav" -mmin +1 -exec rm {} \;
+EOF2
+chmod u+x ${ALLTALK_DIR}/cleanup-wavs.sh
+
+# Starting the cron job:
+cron
+
+source ${ALLTALK_DIR}/conda_env.sh
+
+if [ "\$ALLTALK_ENABLE_MULTI_ENGINE_MANAGER" = "true" ] ; then
+  echo "Starting alltalk using multi engine manager"
+  replace_env_vars docker_default_mem_config.json
+  merge_json_files mem_config.json
+  python tts_mem.py
+else
+  echo "Starting alltalk"
   python script.py
+fi
 EOF
     cat << EOF > start_finetune.sh
 #!/usr/bin/env bash
 source ~/.bashrc
 export TRAINER_TELEMETRY=0
-conda activate alltalk
+source ${ALLTALK_DIR}/conda_env.sh
 python finetune.py
 EOF
     cat << EOF > start_diagnostics.sh
 #!/usr/bin/env bash
 source ~/.bashrc
-conda activate alltalk
+source ${ALLTALK_DIR}/conda_env.sh
 python diagnostics.py
 EOF
     chmod +x start_alltalk.sh
@@ -88,21 +179,9 @@ EOF
     chmod +x start_diagnostics.sh
 EOR
 
-COPY . .
+COPY --chown=alltalk:alltalk . .
 
-##############################################################################
-# Create script to execute firstrun.py and run it:
-##############################################################################
-RUN echo $'#!/usr/bin/env bash \n\
-source ~/.bashrc \n\
-conda activate alltalk \n\
-python ./system/config/firstrun.py $@' > ./start_firstrun.sh
-
-RUN chmod +x start_firstrun.sh
-RUN ./start_firstrun.sh --tts_model $TTS_MODEL
-
-RUN mkdir -p ${ALLTALK_DIR}/outputs
-RUN mkdir -p /root/.triton/autotune
+RUN mkdir -p ${ALLTALK_DIR}/outputs ${ALLTALK_DIR}/.triton/autotune
 
 ##############################################################################
 # Enable deepspeed for all models:
@@ -110,15 +189,29 @@ RUN mkdir -p /root/.triton/autotune
 RUN find . -name model_settings.json -exec sed -i -e 's/"deepspeed_enabled": false/"deepspeed_enabled": true/g' {} \;
 
 ##############################################################################
-# Download all RVC models:
+# Cronjob running every minute to delete output WAV files:
 ##############################################################################
+USER root
 RUN <<EOR
-  jq -r '.[]' system/tts_engines/rvc_files.json > /tmp/rvc_files.txt
-  xargs -n 1 curl --create-dirs --output-dir models/rvc_base -LO < /tmp/rvc_files.txt
-  rm -f /tmp/rvc_files.txt
+    echo "* * * * * ${ALLTALK_DIR}/cleanup-wavs.sh" > /etc/cron.d/cleanup-wavs
+    crontab -u alltalk /etc/cron.d/cleanup-wavs
+    chmod u+s /usr/sbin/cron
 EOR
 
+USER alltalk
+
+##############################################################################
+# Create script to execute firstrun.py and run it:
+##############################################################################
+RUN echo $'#!/usr/bin/env bash \n\
+source ~/.bashrc \n\
+source ${ALLTALK_DIR}/conda_env.sh \n\
+python ./system/config/firstrun.py $@' > ./start_firstrun.sh
+
+RUN chmod +x start_firstrun.sh
+RUN ./start_firstrun.sh --tts_model $TTS_MODEL
+
 ##############################################################################
 # Start alltalk:
 ##############################################################################
-ENTRYPOINT ["sh", "-c", "./start_alltalk.sh"]
+ENTRYPOINT ["bash", "-c", "./start_alltalk.sh"]

atsetup.sh

@@ -553,6 +553,10 @@ reapply_standalone() {
     echo
 }
 
-
+# Check for "-silent" install command-line argument
+if [[ $1 == "-silent" ]]; then
+	install_custom_standalone
+else 
 	# Start the main menu
 	main_menu
+fi

config.py

@@ -38,6 +38,7 @@ class AlltalkConfigRvcSettings(BaseModel):
     f0method: str = "fcpe"
     embedder_model: str = "hubert"
     training_data_size: int = 45000
+    model_cache_size: int = 1
 
 class AlltalkConfigTgwUi(BaseModel):
     tgwui_activate_tts: bool = True
@@ -134,7 +135,7 @@ class AbstractJsonConfig(ABC):
 
     def _reload_on_change(self):
         # Check if config file has been modified and reload if needed
-        if time.time() - self.__last_read_time >= self.__file_check_interval:
+        if self.__file_check_interval != 0 and time.time() - self.__last_read_time >= self.__file_check_interval:
             try:
                 most_recent_modification = self.get_config_path().stat().st_mtime
                 if most_recent_modification > self.__last_read_time:
@@ -369,3 +370,45 @@ class AlltalkConfig(AbstractJsonConfig, AlltalkConfigFields):
         model = AlltalkConfigModel.model_validate_json(json_string)
         model.get_output_directory().mkdir(parents=True, exist_ok=True)
         return model
+
+class AlltalkMultiEngineManagerConfigFields:
+    base_port: int = 7001
+    api_server_port: int = 7851
+    auto_start_engines: int = 0
+    max_instances: int = 8
+    gradio_interface_port: int = 7500
+    max_retries: int = 12
+    initial_wait: float = 2
+    backoff_factor: float = 1.2
+    debug_mode: bool = False
+    max_queue_time: int = 60,  # Maximum time a request can wait in the queue (in seconds)
+    queue_check_interval: float = 0.1,  # Time between checks for available instances (in seconds)
+    tts_request_timeout: int = 30,  # Timeout for individual TTS requests (in seconds)
+    text_length_factor: float = 0.2,  # Increase timeout by 20% per 100 characters
+    concurrent_request_factor: float = 0.5,  # Increase timeout by 50% per concurrent request
+    diminishing_factor: float = 0.5,  # Reduce additional time for long-running requests by 50%
+    queue_position_factor: float = 1.0  # Add 100% of base timeout for each queue position
+
+class AlltalkMultiEngineManagerConfigModel(BaseModel, AlltalkMultiEngineManagerConfigFields):
+    pass
+
+class AlltalkMultiEngineManagerConfig(AbstractJsonConfig, AlltalkMultiEngineManagerConfigFields):
+    __instance = None
+    __this_dir = Path(__file__).parent.resolve()
+
+    def __init__(self, config_path: Path | str = os.path.join(__this_dir, "mem_config.json")):
+        super().__init__(config_path, 0)
+        self._load_config()
+
+    @staticmethod
+    def get_instance():
+        if AlltalkMultiEngineManagerConfig.__instance is None:
+            AlltalkMultiEngineManagerConfig.__instance = AlltalkMultiEngineManagerConfig()
+        AlltalkMultiEngineManagerConfig.__instance._reload_on_change()
+        return AlltalkMultiEngineManagerConfig.__instance
+
+    def _handle_loaded_config(self, json_string: str):
+        return AlltalkMultiEngineManagerConfigModel.model_validate_json(json_string)
+
+    def save(self, path: Path | str | None = None):
+        self._save_file(path)

confignew.json

@@ -26,7 +26,8 @@
         "hop_length": 130,
         "f0method": "fcpe",
         "embedder_model": "hubert",
-        "training_data_size": 45000
+        "training_data_size": 45000,
+        "model_cache_size": 1
     },
     "tgwui": {
         "tgwui_activate_tts": true,

docker-build.sh

@@ -6,8 +6,10 @@ cd $SCRIPT_DIR
 . ${SCRIPT_DIR=}/docker/variables.sh
 
 TTS_MODEL=xtts
-DOCKER_TAG=latest
+DOCKER_TAG=latest-${TTS_MODEL}
 CLEAN=false
+LOCAL_DEEPSPEED_BUILD=false
+DOCKER_REPOSITORY=erew123
 
 # Create required build directories if they don't exist
 mkdir -p ${SCRIPT_DIR=}/docker/deepspeed/build
@@ -36,12 +38,17 @@ while [ "$#" -gt 0 ]; do
       DOCKER_TAG="$2"
       shift
       ;;
-    --github-repository)
-      if [ -n "${GITHUB_REPOSITORY}" ] && ! [[ $GITHUB_REPOSITORY =~ ^--.* ]]; then
-        GITHUB_REPOSITORY="$2"
+    --docker-repository)
+      if [ -n "$2" ] && ! [[ $2 =~ ^--.* ]]; then
+        DOCKER_REPOSITORY="$2"
         shift
+      else
+        DOCKER_REPOSITORY=""
       fi
       ;;
+    --local-deepspeed-build)
+      LOCAL_DEEPSPEED_BUILD=true
+      ;;
     --clean)
       CLEAN=true
       ;;
@@ -53,6 +60,9 @@ while [ "$#" -gt 0 ]; do
   shift
 done
 
+# Append slash if missing:
+DOCKER_REPOSITORY=$(echo "$DOCKER_REPOSITORY" | sed 's![^/]$!&/!')
+
 if [ "$CLEAN" = true ]; then
   rm -rf ${SCRIPT_DIR=}/docker/deepspeed/build
   # Recreate directories after clean
@@ -64,7 +74,7 @@ echo "Building base environment image"
 $SCRIPT_DIR/docker/base/build-base-env.sh \
   --cuda-version ${CUDA_VERSION} \
   --python-version ${PYTHON_VERSION} \
-  --github-repository ${GITHUB_REPOSITORY} \
+  --docker-repository ${DOCKER_REPOSITORY} \
   --tag ${DOCKER_TAG}
 
 if [ $? -ne 0 ]; then
@@ -72,16 +82,18 @@ if [ $? -ne 0 ]; then
   exit 1
 fi
 
+if [ "$LOCAL_DEEPSPEED_BUILD" = true ]; then
   echo "Building DeepSpeed"
   $SCRIPT_DIR/docker/deepspeed/build-deepspeed.sh \
     --python-version ${PYTHON_VERSION} \
-  --github-repository ${GITHUB_REPOSITORY} \
+    --docker-repository ${DOCKER_REPOSITORY} \
     --tag ${DOCKER_TAG}
 
   if [ $? -ne 0 ]; then
     echo "Failed to build DeepSpeed"
     exit 1
   fi
+fi
 
 echo "Starting docker build process using TTS model '${TTS_MODEL}' and docker tag '${DOCKER_TAG}'"
 echo "Building for CUDA $CUDA_VERSION using python ${PYTHON_VERSION}"
@@ -90,9 +102,10 @@ docker buildx \
   build \
   --progress=plain \
   --build-arg TTS_MODEL=$TTS_MODEL \
-  --build-arg ALLTALK_DIR=$ALLTALK_DIR \
   --build-arg DEEPSPEED_VERSION=$DEEPSPEED_VERSION \
-  -t ${GITHUB_REPOSITORY}alltalk_beta:${DOCKER_TAG} \
+  --build-arg DOCKER_TAG=$DOCKER_TAG \
+  --build-arg DOCKER_REPOSITORY=$DOCKER_REPOSITORY \
+  -t ${DOCKER_REPOSITORY}alltalk_tts:${DOCKER_TAG} \
   .
 
 echo "Docker build process finished. Use docker-start.sh to start the container."

docker-start.sh

@@ -3,8 +3,13 @@
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 . ${SCRIPT_DIR=}/docker/variables.sh
 
-WITH_UI=true
-DOCKER_TAG=latest
+ALLTALK_GRADIO_INTERFACE=true
+ALLTALK_ENABLE_MULTI_ENGINE_MANAGER=false
+MULTI_ENGINE_MANAGER_CONFIG=
+DOCKER_TAG=latest-xtts
+DOCKER_REPOSITORY=erew123
+ALLTALK_DIR=/home/alltalk
+ALLTALK_AUTO_CLEANUP=false
 declare -a ADDITIONAL_ARGS=()
 
 # Parse arguments
@@ -18,21 +23,33 @@ while [ "$#" -gt 0 ]; do
       VOICES="$2"
       shift
       ;;
-    --rvc_voices)
+    --rvc-voices)
       RVC_VOICES="$2"
       shift
       ;;
-    --no_ui)
-      WITH_UI=false
+    --no-ui)
+      ALLTALK_GRADIO_INTERFACE=false
+      ;;
+    --auto-cleanup)
+      ALLTALK_AUTO_CLEANUP=true
+      ;;
+    --with-multi-engine-manager)
+      ALLTALK_ENABLE_MULTI_ENGINE_MANAGER=true
+      if [ -n "$2" ] && ! [[ $2 =~ ^--.* ]]; then
+        MULTI_ENGINE_MANAGER_CONFIG=$2
+        shift
+      fi
       ;;
     --tag)
       DOCKER_TAG="$2"
       shift
       ;;
-    --github-repository)
-      if [ -n "${GITHUB_REPOSITORY}" ] && ! [[ $GITHUB_REPOSITORY =~ ^--.* ]]; then
-        GITHUB_REPOSITORY="$2"
+    --docker-repository)
+      if [ -n "$2" ] && ! [[ $2 =~ ^--.* ]]; then
+        DOCKER_REPOSITORY="$2"
         shift
+      else
+        DOCKER_REPOSITORY=""
       fi
       ;;
     *)
@@ -43,6 +60,9 @@ while [ "$#" -gt 0 ]; do
   shift
 done
 
+# Append slash if missing:
+DOCKER_REPOSITORY=$(echo "$DOCKER_REPOSITORY" | sed 's![^/]$!&/!')
+
 # Compose docker arguments based on user input to the script:
 declare -a DOCKER_ARGS=()
 
@@ -59,9 +79,24 @@ if [[ -n $RVC_VOICES ]]; then
   DOCKER_ARGS+=( -v ${RVC_VOICES}:${ALLTALK_DIR}/models/rvc_voices )
 fi
 
-if [ "$WITH_UI" = true ] ; then
+if [ "$ALLTALK_GRADIO_INTERFACE" = true ] ; then
+    if [ "$ALLTALK_ENABLE_MULTI_ENGINE_MANAGER" = true ] ; then
+      DOCKER_ARGS+=( -p 7500:7500 )
+    else
       DOCKER_ARGS+=( -p 7852:7852 )
     fi
+fi
+
+if [[ -n $MULTI_ENGINE_MANAGER_CONFIG ]]; then
+  DOCKER_ARGS+=( -v ${MULTI_ENGINE_MANAGER_CONFIG}:${ALLTALK_DIR}/docker_mem_config.json )
+fi
+
+# Pass env variables:
+DOCKER_ARGS+=( -e ALLTALK_ENABLE_MULTI_ENGINE_MANAGER=${ALLTALK_ENABLE_MULTI_ENGINE_MANAGER} )
+DOCKER_ARGS+=( -e ALLTALK_GRADIO_INTERFACE=${ALLTALK_GRADIO_INTERFACE} )
+DOCKER_ARGS+=( -e ALLTALK_LAUNCH_GRADIO=${ALLTALK_GRADIO_INTERFACE} )
+DOCKER_ARGS+=( -e ALLTALK_AUTO_CLEANUP=${ALLTALK_AUTO_CLEANUP} )
+
 
 docker run \
   --rm \
@@ -71,4 +106,4 @@ docker run \
   --name alltalk \
  "${DOCKER_ARGS[@]}" \
  "${ADDITIONAL_ARGS[@]}" \
-  ${GITHUB_REPOSITORY}alltalk_beta:${DOCKER_TAG} &> /dev/stdout
+  ${DOCKER_REPOSITORY}alltalk_tts:${DOCKER_TAG} &> /dev/stdout

docker/base/Dockerfile

@@ -1,6 +1,6 @@
 FROM continuumio/miniconda3:24.7.1-0 AS build
 
-ARG CUDA_VERSION=12.6.0
+ARG CUDA_VERSION=12.8.1
 ENV CUDA_VERSION=$CUDA_VERSION
 
 ARG PYTHON_VERSION=3.11.11
@@ -14,7 +14,7 @@ ENV CUDA_DOCKER_ARCH=all
 ENV NVIDIA_VISIBLE_DEVICES=all
 ENV CONDA_AUTO_UPDATE_CONDA="false"
 
-ENV ALLTALK_DIR=/opt/alltalk
+ENV ALLTALK_DIR=/home/alltalk
 
 ##############################################################################
 # Installation/Basic Utilities
@@ -24,6 +24,7 @@ RUN <<EOR
     apt-get upgrade -y
     apt-get install --no-install-recommends -y \
       espeak-ng \
+      cron \
       curl \
       wget \
       jq \
@@ -32,6 +33,10 @@ RUN <<EOR
     apt-get clean && rm -rf /var/lib/apt/lists/*
 EOR
 
+# Create a non-root user and switch to it for security
+RUN useradd -ms /bin/bash alltalk
+USER alltalk
+
 ##############################################################################
 # Create a conda environment and install dependencies:
 ##############################################################################
@@ -48,7 +53,8 @@ RUN <<EOR
       cuda-command-line-tools=${CUDA_VERSION} cuda-compiler=${CUDA_VERSION} cuda-runtime=${CUDA_VERSION} \
       cuda-libraries=${CUDA_VERSION} cuda-libraries-dev=${CUDA_VERSION} cuda-libraries-static=${CUDA_VERSION} \
       cuda-cudart=${CUDA_SHORT_VERSION} cuda-cudart_linux-64=${CUDA_SHORT_VERSION} \
-      cudnn=9.3 \
+      conda-forge::nccl=2.27.5.1 \
+      cudnn=9.10 \
       py-cpuinfo=9 \
       conda-forge::ffmpeg=7.1.0 \
       conda-forge::portaudio=19.7.0 \
@@ -71,10 +77,23 @@ RUN <<EOR
   conda activate alltalk
   mkdir -p ${ALLTALK_DIR}/pip_cache
   pip install --no-cache-dir --cache-dir=${ALLTALK_DIR}/pip_cache \
-      torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu${CUDA_SHORT_VERSION_NO_DOT}
+      torch==2.7.1 torchvision==0.22.1 torchaudio==2.7.1 --index-url https://download.pytorch.org/whl/cu${CUDA_SHORT_VERSION_NO_DOT}
 
   if [ $? -ne 0 ] ; then
     echo "Failed to install pip dependencies: $RESULT"
     exit 1
   fi
 EOR
+
+RUN <<EOR
+# See https://gist.github.com/xkortex/5ae49d7e6e969405bd2c3152a949c1f1
+cat << EOF > ${ALLTALK_DIR}/conda_env.sh
+#!/usr/bin/env bash
+source ~/.bashrc
+__conda_setup="\$('/opt/conda/bin/conda' 'shell.bash' 'hook' 2> /dev/null)"
+eval "\$__conda_setup"
+unset __conda_setup
+
+conda activate alltalk
+EOF
+EOR

docker/base/build-base-env.sh

@@ -6,6 +6,7 @@ cd $SCRIPT_DIR
 . ${SCRIPT_DIR=}/../variables.sh
 
 DOCKER_TAG=latest
+DOCKER_REPOSITORY=erew123
 
 # Parse arguments
 while [ "$#" -gt 0 ]; do
@@ -22,10 +23,12 @@ while [ "$#" -gt 0 ]; do
       DOCKER_TAG="$2"
       shift
       ;;
-    --github-repository)
-      if [ -n "${GITHUB_REPOSITORY}" ] && ! [[ $GITHUB_REPOSITORY =~ ^--.* ]]; then
-        GITHUB_REPOSITORY="$2"
+    --docker-repository)
+      if [ -n "$2" ] && ! [[ $2 =~ ^--.* ]]; then
+        DOCKER_REPOSITORY="$2"
         shift
+      else
+        DOCKER_REPOSITORY=""
       fi
       ;;
     *)
@@ -36,6 +39,9 @@ while [ "$#" -gt 0 ]; do
   shift
 done
 
+# Append slash if missing:
+DOCKER_REPOSITORY=$(echo "$DOCKER_REPOSITORY" | sed 's![^/]$!&/!')
+
 echo "Building conda environment using python ${PYTHON_VERSION} with CUDA ${CUDA_VERSION}"
 
 docker buildx \
@@ -43,5 +49,5 @@ docker buildx \
   --progress=plain \
   --build-arg CUDA_VERSION=$CUDA_VERSION \
   --build-arg PYTHON_VERSION=$PYTHON_VERSION \
-  -t ${GITHUB_REPOSITORY}alltalk_environment:${DOCKER_TAG} \
+  -t ${DOCKER_REPOSITORY}alltalk_tts_environment:${DOCKER_TAG} \
   .

docker/deepspeed/Dockerfile

@@ -1,14 +1,11 @@
-ARG GITHUB_REPOSITORY
-FROM ${GITHUB_REPOSITORY}alltalk_environment:latest
+ARG DOCKER_REPOSITORY
+ARG DOCKER_TAG=latest
+FROM ${DOCKER_REPOSITORY}alltalk_tts_environment:${DOCKER_TAG}
 
-ARG DEEPSPEED_VERSION=0.16.3
+ARG DEEPSPEED_VERSION=0.17.2
 ENV DEEPSPEED_VERSION=$DEEPSPEED_VERSION
 
-##############################################################################
-# Directories:
-##############################################################################
-ENV STAGE_DIR=/tmp
-RUN mkdir -p ${STAGE_DIR}
+WORKDIR ${ALLTALK_DIR}
 
 ##############################################################################
 # Install python dependencies
@@ -26,40 +23,30 @@ EOR
 # Clone DeepSpeed sources
 ##############################################################################
 RUN <<EOR
-  git clone https://github.com/microsoft/DeepSpeed.git ${STAGE_DIR}/DeepSpeed
-  cd ${STAGE_DIR}/DeepSpeed
+  git clone https://github.com/microsoft/DeepSpeed.git ${ALLTALK_DIR}/DeepSpeed
+  cd ${ALLTALK_DIR}/DeepSpeed
   git checkout .
   git checkout "tags/v${DEEPSPEED_VERSION}" -b "v${DEEPSPEED_VERSION}"
-
-  # Patching some files due to bug https://github.com/microsoft/DeepSpeed/issues/6709
-  START_LINE=`awk '/#ifndef BF16_AVAILABLE/{ print NR; exit }' csrc/transformer/inference/csrc/gelu.cu`
-  TO_LINE=$((START_LINE + 2))
-  sed -i "${START_LINE},${TO_LINE}d" ./csrc/transformer/inference/csrc/gelu.cu
-
-  START_LINE=`awk '/#ifndef BF16_AVAILABLE/{ print NR; exit }' csrc/transformer/inference/csrc/transform.cu`
-  TO_LINE=$((START_LINE + 2))
-  sed -i "${START_LINE},${TO_LINE}d" ./csrc/transformer/inference/csrc/transform.cu
 EOR
 
 ##############################################################################
 # Create DeepSpeed build file
 ##############################################################################
 RUN <<EOR
-  cat << EOF > build_deepspeed.sh
+  cat << EOF > ${ALLTALK_DIR}/build_deepspeed.sh
 #!/usr/bin/env bash
-source ~/.bashrc
-mkdir -p /deepspeed
+source ${ALLTALK_DIR}/conda_env.sh
+mkdir -p ${ALLTALK_DIR}/build
 
-conda activate alltalk
-cd ${STAGE_DIR}/DeepSpeed
+cd ${ALLTALK_DIR}/DeepSpeed
 DS_BUILD_OPS=1 python setup.py build_ext -j8 bdist_wheel
-mv ${STAGE_DIR}/DeepSpeed/dist/*.whl /deepspeed/
+mv ${ALLTALK_DIR}/DeepSpeed/dist/*.whl ${ALLTALK_DIR}/build
 EOF
 EOR
 
-RUN chmod +x /build_deepspeed.sh
+RUN chmod +x ${ALLTALK_DIR}/build_deepspeed.sh
 
 ##############################################################################
 # Build DeepSpeed:
 ##############################################################################
-ENTRYPOINT ["/build_deepspeed.sh"]
+ENTRYPOINT ["bash", "-c", "-l", "$ALLTALK_DIR/build_deepspeed.sh"]

docker/deepspeed/build-deepspeed.sh

@@ -6,6 +6,8 @@ cd $SCRIPT_DIR
 . ${SCRIPT_DIR=}/../variables.sh
 
 DOCKER_TAG=latest
+DOCKER_REPOSITORY=erew123
+ALLTALK_DIR=/home/alltalk
 
 # Parse arguments
 while [ "$#" -gt 0 ]; do
@@ -22,10 +24,12 @@ while [ "$#" -gt 0 ]; do
       DOCKER_TAG="$2"
       shift
       ;;
-    --github-repository)
-      if [ -n "${GITHUB_REPOSITORY}" ] && ! [[ $GITHUB_REPOSITORY =~ ^--.* ]]; then
-        GITHUB_REPOSITORY="$2"
+    --docker-repository)
+      if [ -n "$2" ] && ! [[ $2 =~ ^--.* ]]; then
+        DOCKER_REPOSITORY="$2"
         shift
+      else
+        DOCKER_REPOSITORY=""
       fi
       ;;
     *)
@@ -36,6 +40,9 @@ while [ "$#" -gt 0 ]; do
   shift
 done
 
+# Append slash if missing:
+DOCKER_REPOSITORY=$(echo "$DOCKER_REPOSITORY" | sed 's![^/]$!&/!')
+
 PYTHON_SHORT_VERSION=${PYTHON_VERSION%.*}
 PYTHON_VERSION_NO_DOT=${PYTHON_SHORT_VERSION//./}
 if [[ -n $(find build -name "deepspeed-${DEEPSPEED_VERSION}*-cp${PYTHON_VERSION_NO_DOT}-cp${PYTHON_VERSION_NO_DOT}-*.whl") ]]
@@ -51,9 +58,9 @@ mkdir -p build
 
 docker buildx \
   build \
-  --build-arg GITHUB_REPOSITORY=$GITHUB_REPOSITORY \
+  --build-arg DOCKER_REPOSITORY=$DOCKER_REPOSITORY \
   --build-arg DEEPSPEED_VERSION=$DEEPSPEED_VERSION \
-  -t ${GITHUB_REPOSITORY}alltalk_deepspeed:${DOCKER_TAG} \
+  -t ${DOCKER_REPOSITORY}alltalk_deepspeed:${DOCKER_TAG} \
   .
 
 docker run \
@@ -61,5 +68,5 @@ docker run \
   -it \
   --gpus=all \
   --name deepspeed \
-  -v $SCRIPT_DIR/build:/deepspeed \
-  ${GITHUB_REPOSITORY}alltalk_deepspeed:${DOCKER_TAG}
+  -v $SCRIPT_DIR/build:${ALLTALK_DIR}/build \
+  ${DOCKER_REPOSITORY}alltalk_deepspeed:${DOCKER_TAG}

docker/variables.sh

@@ -1,7 +1,15 @@
 #!/usr/bin/env bash
 
-export CUDA_VERSION=12.6.0
-export PYTHON_VERSION=3.11.11
-export DEEPSPEED_VERSION=0.16.2
-export ALLTALK_DIR=/opt/alltalk
-export GITHUB_REPOSITORY=
+declare -A ALLTALK_VARS
+ALLTALK_VARS["CUDA_VERSION"]=12.8.1
+ALLTALK_VARS["PYTHON_VERSION"]=3.11.11
+ALLTALK_VARS["DEEPSPEED_VERSION"]=0.17.2
+
+# Export single variables (needed by Docker locally)
+for key in "${!ALLTALK_VARS[@]}"
+do
+  export "${key}=${ALLTALK_VARS[${key}]}"
+done
+
+# Export the entire associative array (needed by Github action)
+export ALLTALK_VARS

docker_default_config.json

@@ -1,7 +0,0 @@
-{
-  "delete_output_wavs": "1",
-  "rvc_settings": {
-    "rvc_enabled": true,
-    "f0method": "rmvpe"
-  }
-}

docker_default_confignew.json

@@ -0,0 +1,41 @@
+{
+  "branding": $ENV.ALLTALK_BRANDING,
+  "delete_output_wavs": $ENV.ALLTALK_DELETE_OUTPUT_WAVS,
+  "gradio_interface": $ENV.ALLTALK_GRADIO_INTERFACE,
+  "output_folder": $ENV.ALLTALK_OUTPUT_FOLDER,
+  "gradio_port_number": $ENV.ALLTALK_GRADIO_PORT_NUMBER,
+  "launch_gradio": $ENV.ALLTALK_LAUNCH_GRADIO,
+  "transcode_audio_format": $ENV.ALLTALK_TRANSCODE_AUDIO_FORMAT,
+  "rvc_settings": {
+    "rvc_enabled": $ENV.ALLTALK_RVC_ENABLED,
+    "rvc_char_model_file": $ENV.ALLTALK_RVC_CHAR_MODEL_FILE,
+    "rvc_narr_model_file": $ENV.ALLTALK_RVC_NARR_MODEL_FILE,
+    "split_audio": $ENV.ALLTALK_RVC_SPLIT_AUDIO,
+    "autotune": $ENV.ALLTALK_RVC_AUTOTUNE,
+    "pitch": $ENV.ALLTALK_RVC_PITCH,
+    "filter_radius": $ENV.ALLTALK_RVC_FILTER_RADIUS,
+    "index_rate": $ENV.ALLTALK_RVC_INDEX_RATE,
+    "rms_mix_rate": $ENV.ALLTALK_RVC_RMS_MIX_RATE,
+    "protect": $ENV.ALLTALK_RVC_PROTECT,
+    "hop_length": $ENV.ALLTALK_RVC_HOP_LENGTH,
+    "f0method": $ENV.ALLTALK_RVC_F0METHOD,
+    "embedder_model": $ENV.ALLTALK_RVC_EMBEDDER_MODEL,
+    "training_data_size": $ENV.ALLTALK_RVC_TRAINING_DATA_SIZE,
+    "model_cache_size": $ENV.ALLTALK_RVC_MODEL_CACHE_SIZE
+  },
+  "api_def": {
+    "api_port_number": $ENV.ALLTALK_API_PORT_NUMBER,
+    "api_allowed_filter": $ENV.ALLTALK_API_ALLOWED_FILTER,
+    "api_length_stripping": $ENV.ALLTALK_API_LENGTH_STRIPPING,
+    "api_max_characters": $ENV.ALLTALK_API_MAX_CHARACTERS,
+    "api_use_legacy_api": $ENV.ALLTALK_API_USE_LEGACY_API,
+    "api_text_filtering": $ENV.ALLTALK_API_TEXT_FILTERING,
+    "api_narrator_enabled": $ENV.ALLTALK_API_NARRATOR_ENABLED,
+    "api_text_not_inside": $ENV.ALLTALK_API_TEXT_NOT_INSIDE,
+    "api_language": $ENV.ALLTALK_API_LANGUAGE,
+    "api_output_file_name": $ENV.ALLTALK_API_OUTPUT_FILE_NAME,
+    "api_output_file_timestamp": $ENV.ALLTALK_API_OUTPUT_FILE_TIMESTAMP,
+    "api_autoplay": $ENV.ALLTALK_API_AUTOPLAY,
+    "api_autoplay_volume": $ENV.ALLTALK_API_AUTOPLAY_VOLUME
+  }
+}

docker_default_mem_config.json

@@ -0,0 +1,18 @@
+{
+  "base_port": $ENV.ALLTALK_MEM_BASE_PORT,
+  "api_server_port": $ENV.ALLTALK_MEM_API_SERVER_PORT,
+  "auto_start_engines": $ENV.ALLTALK_MEM_AUTO_START_ENGINES,
+  "max_instances": $ENV.ALLTALK_MEM_MAX_INSTANCES,
+  "gradio_interface_port": $ENV.ALLTALK_MEM_GRADIO_INTERFACE_PORT,
+  "max_retries": $ENV.ALLTALK_MEM_MAX_RETRIES,
+  "initial_wait": $ENV.ALLTALK_MEM_INITIAL_WAIT,
+  "backoff_factor": $ENV.ALLTALK_MEM_BACKOFF_FACTOR,
+  "debug_mode": $ENV.ALLTALK_MEM_DEBUG_MODE,
+  "max_queue_time": $ENV.ALLTALK_MEM_MAX_QUEUE_TIME,
+  "queue_check_interval": $ENV.ALLTALK_MEM_QUEUE_CHECK_INTERVAL,
+  "tts_request_timeout": $ENV.ALLTALK_MEM_TTS_REQUEST_TIMEOUT,
+  "text_length_factor": $ENV.ALLTALK_MEM_TEXT_LENGTH_FACTOR,
+  "concurrent_request_factor": $ENV.ALLTALK_MEM_CONCURRENT_REQUEST_FACTOR,
+  "diminishing_factor": $ENV.ALLTALK_MEM_DIMINISHING_FACTOR,
+  "queue_position_factor": $ENV.ALLTALK_MEM_QUEUE_POSITION_FACTOR
+}

docker_mem_config.json

@@ -0,0 +1 @@
+{}

finetune.py

@@ -4805,5 +4805,5 @@ if __name__ == "__main__":
         share=False,
         debug=False,
         server_port=7052,
-        server_name="127.0.0.1",
+        server_name="0.0.0.0",
     )

mem_config.json

@@ -0,0 +1,18 @@
+{
+  "base_port": 7001,
+  "api_server_port": 7851,
+  "auto_start_engines": 0,
+  "max_instances": 8,
+  "gradio_interface_port": 7500,
+  "max_retries": 12,
+  "initial_wait": 2.0,
+  "backoff_factor": 1.2,
+  "debug_mode": false,
+  "max_queue_time": 60,
+  "queue_check_interval": 0.1,
+  "tts_request_timeout": 30,
+  "text_length_factor": 0.2,
+  "concurrent_request_factor": 0.5,
+  "diminishing_factor": 0.5,
+  "queue_position_factor": 1.0
+}

script.py

@@ -1227,6 +1227,7 @@ else:
         + str(config.api_def.api_port_number)
         + "\033[00m"
     )
+    if config.launch_gradio:
         print_message(
             "\033[94mGradio Light:\033[00m \033[92mhttp://127.0.0.1:"
             + str(config.gradio_port_number)
@@ -2038,7 +2039,7 @@ def process_audio_files(
                     "processing_date": datetime.now().isoformat(),
                     "validation_messages": validation_messages,
                 },
-                f,
+                file,
                 indent=4,
             )
 

system/config/firstrun.py

@@ -41,9 +41,8 @@ Functions:
 - **is_running_in_colab**: Detects if the script is running in Google Colab.
 - **is_running_in_docker**: Detects if the script is running in a Docker container.
 - **download_file**: Downloads a file with a progress bar and retry logic.
-- **setup_piper**: Sets up the Piper TTS engine and downloads associated files.
-- **setup_vits**: Sets up the VITS TTS engine and downloads associated files.
-- **setup_xtts**: Sets up the XTTS TTS engine and downloads associated files.
+- **setup_tts**: Sets up the TTS engine passed as argument.
+- **download_tts_model**: Downloads the TTS engine's associated files.
 - **update_tts_engines**: Updates the configuration to reflect the selected TTS engine.
 - **set_firstrun_model_false**: Marks the initial setup as complete in the configuration.
 - **warning_message**: Displays post-setup warnings or tips for the user.
@@ -65,6 +64,7 @@ import zipfile
 import requests
 from tqdm import tqdm
 from inputimeout import inputimeout, TimeoutOccurred
+from urllib.parse import urlsplit
 
 # Setting the module search path:
 sys.path.insert(0, str(Path(__file__).parent.parent.parent.resolve()))
@@ -103,6 +103,11 @@ def download_file(url: str, dest_path: str, timeout: int = 30, retries: int = 3)
     Raises:
         ValueError: If the downloaded file size does not match the expected size.
     """
+    # Don't re-download if file already exists (which is the case when using Docker):
+    if Path(dest_path).exists():
+        print(f"File already exists at: {dest_path}. Skipping download.")
+        return True
+
     progress_bar = None
     for attempt in range(1, retries + 1):
         try:
@@ -166,41 +171,44 @@ def download_file(url: str, dest_path: str, timeout: int = 30, retries: int = 3)
     # Return False if all attempts fail
     return False
 
-def setup_piper():
-    """Download base piper files"""
-    os.makedirs(this_dir / 'models/piper', exist_ok=True)
-    os.makedirs(this_dir / 'models/xtts', exist_ok=True)
-    download_file("https://huggingface.co/rhasspy/piper-voices/resolve/v1.0.0/en/en_US/ljspeech/high/en_US-ljspeech-high.onnx?download=true",
-                  this_dir / "models/piper/en_US-ljspeech-high.onnx")
-    download_file("https://huggingface.co/rhasspy/piper-voices/resolve/v1.0.0/en/en_US/ljspeech/high/en_US-ljspeech-high.onnx.json?download=true.json",
-                  this_dir / "models/piper/en_US-ljspeech-high.onnx.json")
+def unzip_compressed_files(downloaded_files: []):
+    for file in downloaded_files:
+        _, file_extension = os.path.splitext(file)
+        if file_extension == '.zip':
+            with zipfile.ZipFile(file, 'r') as zip_ref:
+                zip_ref.extractall(file.parent)
+            os.remove(file)
 
-def setup_vits():
-    """Download base vits files"""
-    os.makedirs(this_dir / 'models/vits', exist_ok=True)
-    os.makedirs(this_dir / 'models/xtts', exist_ok=True)
-    zip_path = this_dir / 'models/vits/vits.zip'
-    download_file("https://coqui.gateway.scarf.sh/v0.6.1_models/tts_models--en--vctk--vits.zip", zip_path)
-    with zipfile.ZipFile(zip_path, 'r') as zip_ref:
-        zip_ref.extractall(this_dir / 'models/vits')
-    os.remove(zip_path)
+def setup_tts(tts_model: str, on_download_completed = unzip_compressed_files):
+    download_tts_model(tts_model, on_download_completed)
+    update_tts_engines(tts_model)
+    set_firstrun_model_false()
 
-def setup_xtts():
-    """Download base xtts files"""
-    os.makedirs(this_dir / 'models/xtts/xttsv2_2.0.3', exist_ok=True)
-    base_url = "https://huggingface.co/coqui/XTTS-v2/resolve/v2.0.3/"
-    files = [
-        "LICENSE.txt",
-        "README.md",
-        "config.json",
-        "model.pth",
-        "dvae.pth",
-        "mel_stats.pth",
-        "speakers_xtts.pth",
-        "vocab.json"
-    ]
-    for each_file in files:
-        download_file(base_url + each_file + "?download=true", this_dir / f"models/xtts/xttsv2_2.0.3/{each_file}")
+def download_tts_model(tts_model: str, on_download_completed = None):
+    with open(os.path.join(this_dir, "system", "tts_engines", tts_model, "available_models.json")) as f:
+        available_models = json.load(f)
+        first_start_model = available_models["first_start_model"]
+        model = next(m for m in available_models["models"] if m["model_name"] == first_start_model)
+
+        # folder_path: Special case for vits: firstrun.py expects the ZIP download directly in the model folder.
+        # When firstrun.py unzips the file, the correct folder will be created automatically.
+        folder_path = "" if tts_model == "vits" else model.get("folder_path", "")
+        files_to_download = model.get("files_to_download", []) or [model.get("github_rls_url", None)]
+        target_dir = this_dir / f"models/{tts_model}/{folder_path}"
+
+        if type(files_to_download) == dict:
+            files_to_download = list(files_to_download.values())
+
+        downloaded_files = []
+        for file_download in files_to_download:
+            urlpath = urlsplit(file_download).path
+            file_name = os.path.basename(urlpath)
+            target_file = target_dir / file_name
+            download_file(file_download, target_file)
+            downloaded_files.append(target_file)
+
+        if on_download_completed:
+            on_download_completed(downloaded_files)
 
 def update_tts_engines(engine):
     """Set engine to users choice that matches the download"""
@@ -282,18 +290,9 @@ args = parser.parse_args()
 if config.firstrun_model:
     # Handle command-line argument for tts_model
     if args.tts_model:
-        if args.tts_model == 'piper':
-            setup_piper()
-            update_tts_engines('piper')
-            set_firstrun_model_false()
-        elif args.tts_model == 'vits':
-            setup_vits()
-            update_tts_engines('vits')
-            set_firstrun_model_false()
-        elif args.tts_model == 'xtts':
-            setup_xtts()
-            update_tts_engines('xtts')
-            set_firstrun_model_false()
+        if args.tts_model in {'piper', 'vits', 'xtts'}:
+            print(f"[{branding}TTS] Setting up model '{args.tts_model}'.")
+            setup_tts(args.tts_model)
         elif args.tts_model == 'none':
             print(f"[{branding}TTS] No TTS model setup requested.")
         print(f"[{branding}TTS] Setup completed for {args.tts_model}. Exiting.")
@@ -302,9 +301,7 @@ if config.firstrun_model:
     # Check for Colab or Docker environment first
     if is_running_in_colab() or is_running_in_docker():
         print(f"[{branding}TTS] Detected Colab/Docker environment - automatically selecting Piper")
-        setup_piper()
-        update_tts_engines('piper')
-        set_firstrun_model_false()
+        setup_tts('piper')
         warning_message()
         sys.exit()
 
@@ -363,19 +360,9 @@ if config.firstrun_model:
         warning_message()
 
     if user_choice is None or user_choice.lower() != 'own':
-        if selected_model['name'] == 'piper':
-            setup_piper()
-            update_tts_engines('piper')
-        elif selected_model['name'] == 'vits':
-            setup_vits()
-            update_tts_engines('vits')
-        elif selected_model['name'] == 'xtts':
-            setup_xtts()
-            update_tts_engines('xtts')
-
+        if selected_model['name'] in {'piper', 'vits', 'xtts'}:
+            setup_tts(selected_model['name'])
             print(f"[{branding}TTS] {selected_model['name']} model downloaded and configuration updated successfully.")
-        # Set firstrun_model to false
-        set_firstrun_model_false()
             warning_message()
             sys.exit()
 else:

system/requirements/requirements_colab.txt

@@ -10,7 +10,7 @@ Jinja2>=3.1.0
 librosa>=0.10.2.post1
 local_attention==1.9.10
 onnxruntime-gpu>=1.18.0
-pydantic>=2.7.0
+pydantic==2.10.6
 python-Levenshtein>=0.25.1
 praat-parselmouth>=0.4.0
 pyworld>=0.3.0

system/requirements/requirements_standalone.txt

@@ -1,5 +1,7 @@
 coqui-tts==0.24.3
 filelock>=3.13.4
+flask[async]==3.1.0
+flask-cors==5.0.1
 fuzzywuzzy>=0.18.0
 ffmpeg-python>=0.2.0
 gradio==4.44.1
@@ -10,7 +12,7 @@ librosa>=0.10.2.post1
 local_attention==1.9.10
 openai-whisper
 onnxruntime-gpu>=1.18.1
-pydantic>=2.8.2
+pydantic==2.10.6
 python-Levenshtein>=0.25.1
 praat-parselmouth>=0.4.4
 pyworld>=0.3.4

system/requirements/requirements_textgen.txt

@@ -9,7 +9,7 @@ librosa>=0.10.2.post1
 local_attention>=1.9.0
 onnxruntime-gpu>=1.18.0
 openai-whisper
-pydantic>=2.7.0
+pydantic==2.10.6
 python-Levenshtein>=0.25.1
 praat-parselmouth>=0.4.0
 pyworld>=0.3.0

system/requirements/requirements_unit_test.txt

@@ -1,4 +1,4 @@
 filelock>=3.16.1
-pydantic>=2.7.0
+pydantic==2.10.6
 pytest>=8.3.3
 pytest-cov>=6.0.0

system/tts_engines/f5tts/available_models.json

@@ -1,19 +1,19 @@
 {
-  "first_start_model": "f5tts_v1",
+  "first_start_model": "f5tts_v1a",
   "models": [
       {
-          "model_name": "f5tts_v1",
-          "folder_path": "f5tts_v1",
+          "model_name": "f5tts_v1a",
+          "folder_path": "f5tts_v1a",
           "files_to_download": {
-              "model_1200000.safetensors": "https://huggingface.co/SWivid/F5-TTS/resolve/main/F5TTS_Base/model_1200000.safetensors",
+              "model_1250000.safetensors": "https://huggingface.co/SWivid/F5-TTS/resolve/main/F5TTS_v1_Base/model_1250000.safetensors",
               "vocab.txt": "https://raw.githubusercontent.com/SWivid/F5-TTS/refs/heads/main/src/f5_tts/infer/examples/vocab.txt",
               "vocos/config.yaml": "https://huggingface.co/charactr/vocos-mel-24khz/resolve/main/config.yaml",
               "vocos/pytorch_model.bin": "https://huggingface.co/charactr/vocos-mel-24khz/resolve/main/pytorch_model.bin"
           }
       },
       {
-        "model_name": "e2tts_v1",
-        "folder_path": "e2tts_v1",
+        "model_name": "e2tts_v1a",
+        "folder_path": "e2tts_v1a",
         "files_to_download": {
             "model_1200000.safetensors": "https://huggingface.co/SWivid/E2-TTS/resolve/main/E2TTS_Base/model_1200000.safetensors",
             "vocab.txt": "https://raw.githubusercontent.com/SWivid/F5-TTS/refs/heads/main/src/f5_tts/infer/examples/vocab.txt",

system/tts_engines/f5tts/model_engine.py

@@ -722,34 +722,36 @@ class tts_class:
             if clip_short:
                 # 1. try to find long silence for clipping
                 non_silent_segs = silence.split_on_silence(
-                    aseg, min_silence_len=1000, silence_thresh=-50, keep_silence=1000
+                    aseg, min_silence_len=1000, silence_thresh=-50, keep_silence=1000, seek_step=10
                 )
                 non_silent_wave = AudioSegment.silent(duration=0)
                 for non_silent_seg in non_silent_segs:
-                    if len(non_silent_wave) > 6000 and len(non_silent_wave + non_silent_seg) > 15000:
-                        print(f"[{self.branding}ENG] Audio is over 15s, clipping short. (1)") if self.debug_tts else None
+                    if len(non_silent_wave) > 6000 and len(non_silent_wave + non_silent_seg) > 12000:
+                        print(f"[{self.branding}ENG] Audio is over 12s, clipping short. (1)") if self.debug_tts else None
                         break
                     non_silent_wave += non_silent_seg
 
                 # 2. try to find short silence for clipping if 1. failed
-                if len(non_silent_wave) > 15000:
+                if len(non_silent_wave) > 12000:
                     non_silent_segs = silence.split_on_silence(
-                        aseg, min_silence_len=100, silence_thresh=-40, keep_silence=1000
+                        aseg, min_silence_len=100, silence_thresh=-40, keep_silence=1000, seek_step=10
                     )
                     non_silent_wave = AudioSegment.silent(duration=0)
                     for non_silent_seg in non_silent_segs:
-                        if len(non_silent_wave) > 6000 and len(non_silent_wave + non_silent_seg) > 15000:
-                            print(f"[{self.branding}ENG] Audio is over 15s, clipping short. (2)") if self.debug_tts else None
+                        if len(non_silent_wave) > 6000 and len(non_silent_wave + non_silent_seg) > 12000:
+                            print(f"[{self.branding}ENG] Audio is over 12s, clipping short. (2)") if self.debug_tts else None
                             break
                         non_silent_wave += non_silent_seg
 
                 aseg = non_silent_wave
 
                 # 3. if no proper silence found for clipping
-                if len(aseg) > 15000:
-                    aseg = aseg[:15000]
-                    print(f"[{self.branding}ENG] Audio is over 15s, clipping short. (3)") if self.debug_tts else None
+                if len(aseg) > 12000:
+                    aseg = aseg[:12000]
+                    print(f"[{self.branding}ENG] Audio is over 12s, clipping short. (3)") if self.debug_tts else None
 
+            # Add the silence removal function
+            aseg = self.remove_silence_edges(aseg) + AudioSegment.silent(duration=50)
             aseg.export(f.name, format="wav")
             ref_audio = f.name
 
@@ -762,6 +764,36 @@ class tts_class:
 
         return ref_audio, ref_text
 
+    def remove_silence_edges(self, audio, silence_threshold=-42):
+        try:
+            # Remove silence from the start
+            non_silent_start_idx = silence.detect_leading_silence(audio, silence_threshold=silence_threshold)
+            audio = audio[non_silent_start_idx:]
+
+            # Remove silence from the end
+            non_silent_end_duration = audio.duration_seconds
+            for ms in reversed(audio):
+                if ms.dBFS > silence_threshold:
+                    break
+                non_silent_end_duration -= 0.001
+            trimmed_audio = audio[: int(non_silent_end_duration * 1000)]
+
+            return trimmed_audio
+        except Exception as e:
+            print(f"[{self.branding}ENG] Error in remove_silence_edges: {str(e)}") if self.debug_tts else None
+            return audio  # Return original audio if there's an error
+    
+    def remove_silence_for_generated_wav(self, filename):
+        aseg = AudioSegment.from_file(filename)
+        non_silent_segs = silence.split_on_silence(
+            aseg, min_silence_len=1000, silence_thresh=-50, keep_silence=500, seek_step=10
+        )
+        non_silent_wave = AudioSegment.silent(duration=0)
+        for non_silent_seg in non_silent_segs:
+            non_silent_wave += non_silent_seg
+        aseg = non_silent_wave
+        aseg.export(filename, format="wav")
+
     async def infer_process(
         self,
         ref_audio,
@@ -779,9 +811,10 @@ class tts_class:
         device=None
     ):
         """Process text and prepare for batch inference"""
+        try:
             # Split the input text into batches
             audio, sr = torchaudio.load(str(ref_audio))
-        max_chars = int(len(ref_text.encode("utf-8")) / (audio.shape[-1] / sr) * (25 - audio.shape[-1] / sr))
+            max_chars = int(len(ref_text.encode("utf-8")) / (audio.shape[-1] / sr) * (22 - audio.shape[-1] / sr))
             gen_text_batches = self.chunk_text(gen_text, max_chars=max_chars)
             
             for i, gen_text_batch in enumerate(gen_text_batches):
@@ -789,7 +822,7 @@ class tts_class:
 
             print(f"[{self.branding}ENG] Generating audio in {len(gen_text_batches)} batches...") if self.debug_tts else None
             
-        return await self.infer_batch_process(
+            result = await self.infer_batch_process(
                 (audio, sr),
                 ref_text,
                 gen_text_batches,
@@ -805,6 +838,18 @@ class tts_class:
                 device=device
             )
             
+            if result is None:
+                print(f"[{self.branding}ENG] Error: No audio was generated") if self.debug_tts else None
+                # Return some default values to avoid unpacking errors
+                return np.zeros(1000), self.target_sample_rate, None
+                
+            return result
+            
+        except Exception as e:
+            print(f"[{self.branding}ENG] Error in infer_process: {str(e)}") if self.debug_tts else None
+            # Return some default values to avoid unpacking errors
+            return np.zeros(1000), self.target_sample_rate, None
+
     async def infer_batch_process(
         self,
         ref_audio,
@@ -853,6 +898,14 @@ class tts_class:
             ref_text = ref_text + " "
             
         for gen_text in gen_text_batches:
+            local_speed = speed
+            if len(gen_text.encode("utf-8")) < 10:
+                local_speed = 0.3
+                print(f"[{self.branding}Debug] Short text detected, using speed={local_speed}") if self.debug_tts else None
+            else:
+                print(f"[{self.branding}Debug] Using speed={local_speed}") if self.debug_tts else None
+                    
+            
             # Prepare the text
             text_list = [ref_text + gen_text]
             final_text_list = convert_char_to_pinyin(text_list)
@@ -861,10 +914,14 @@ class tts_class:
             if fix_duration is not None:
                 duration = int(fix_duration * self.target_sample_rate / self.hop_length)
             else:
-                # Calculate duration
                 ref_text_len = len(ref_text.encode("utf-8"))
                 gen_text_len = len(gen_text.encode("utf-8"))
-                duration = ref_audio_len + int(ref_audio_len / ref_text_len * gen_text_len / speed)
+                print(f"[{self.branding}Debug] ref_text_len={ref_text_len}, gen_text_len={gen_text_len}") if self.debug_tts else None
+                print(f"[{self.branding}Debug] ref_audio_len={ref_audio_len}") if self.debug_tts else None
+                estimated_duration = ref_audio_len + int(ref_audio_len / ref_text_len * gen_text_len / local_speed)
+                print(f"[{self.branding}Debug] Estimated duration={estimated_duration} frames") if self.debug_tts else None
+                duration = ref_audio_len + int(ref_audio_len / ref_text_len * gen_text_len / local_speed)
+
 
             # inference
             with torch.inference_mode():
@@ -878,19 +935,13 @@ class tts_class:
                 )
 
                 # Handle precision for post-processing
-                if self.device == 'cuda':
-                    generated = generated.half()
-                else:
-                    generated = generated.float()
-
+                generated = generated.to(torch.float32)
                 generated = generated[:, ref_audio_len:, :]
                 generated_mel_spec = generated.permute(0, 2, 1)
                 
                 # Convert to the correct precision for vocoder
-                if self.device == 'cuda':
+                if device == 'cuda':
                     generated_mel_spec = generated_mel_spec.half()
-                else:
-                    generated_mel_spec = generated_mel_spec.float()
                     
                 generated_wave = vocoder.decode(generated_mel_spec)
                 if rms < target_rms:
@@ -903,6 +954,11 @@ class tts_class:
                 spectrograms.append(generated_mel_spec[0].cpu().numpy())
 
         # Combine all generated waves with cross-fading
+        if not generated_waves:
+            # Handle the case where no waves were generated
+            print(f"[{self.branding}ENG] Warning: No audio was generated") if self.debug_tts else None
+            return None, self.target_sample_rate, None
+
         if cross_fade_duration <= 0:
             final_wave = np.concatenate(generated_waves)
         else:
@@ -938,7 +994,7 @@ class tts_class:
                 final_wave = new_wave
 
         # Create a combined spectrogram
-        combined_spectrogram = np.concatenate(spectrograms, axis=1)
+        combined_spectrogram = np.concatenate(spectrograms, axis=1) if spectrograms else None
 
         return final_wave, self.target_sample_rate, combined_spectrogram
 
@@ -1083,6 +1139,8 @@ class tts_class:
             )
             
             text = self.process_text_for_tts(text)
+            speed_value = float(speed)
+            print(f"[{self.branding}Debug] Using speed setting: {speed_value}") if self.debug_tts else None
             
             # Generate the audio using infer_process
             final_wave, final_sample_rate, _ = await self.infer_process(
@@ -1096,7 +1154,7 @@ class tts_class:
                 nfe_step=self.nfe_step,
                 cfg_strength=self.cfg_strength,
                 sway_sampling_coef=self.sway_sampling_coef,
-                speed=float(speed),
+                speed=speed_value,
                 fix_duration=None,
                 device=self.device
             )

system/tts_engines/f5tts/model_settings.json

@@ -25,7 +25,7 @@
         "def_narrator_voice": "female_01.wav",
         "deepspeed_enabled": false,
         "engine_installed": true,
-        "generationspeed_set": 0.9,
+        "generationspeed_set": 1,
         "lowvram_enabled": true,
         "pitch_set": 0,
         "repetitionpenalty_set": 10,

system/tts_engines/piper/available_models.json

@@ -1,5 +1,5 @@
 {
-  "first_start_model": "en_GB-jenny_dioco-medium",
+  "first_start_model": "en_US-ljspeech-high",
   "models": [
     {
       "model_name": "ar_JO-kareem-low",

system/tts_engines/rvc/infer/infer.py

@@ -9,6 +9,8 @@ import soundfile as sf
 import librosa
 from functools import lru_cache
 
+from config import AlltalkConfig
+
 now_dir = os.getcwd()
 sys.path.append(now_dir)
 
@@ -171,7 +173,7 @@ def voice_conversion(
         return None, None
 
 
-@lru_cache
+@lru_cache(maxsize=AlltalkConfig.get_instance().rvc_settings.model_cache_size)
 def get_vc(weight_root, sid, file_index=None, training_data_size=10000, debug_rvc=False):
     net_g = None
     branding="AllTalk "

system/tts_engines/rvc/lib/utils.py

@@ -1,20 +1,20 @@
 import numpy as np
 import re
 import unicodedata
-from fairseq import checkpoint_utils
-from fairseq.data import Dictionary
-import torch
 
+# Next 2 lines are a fix for fairseq to be compatible with pytorch > 2.6
+import sys
+sys.path = [str(p) for p in sys.path]
+
+from fairseq import checkpoint_utils
+import torch
 import logging
 
 logging.getLogger("fairseq").setLevel(logging.WARNING)
 import sys
-import os
 import numpy as np
-import asyncio
 import ffmpeg
 from pathlib import Path
-import subprocess
 
 # Get the parent directory of the current file
 this_dir = Path(__file__).resolve().parent.parent.parent.parent.parent
@@ -56,7 +56,6 @@ def format_title(title):
 
 
 def load_embedding(embedder_model):
-    #print("EMBEDDER MODEL IS", embedder_model)
     embedding_list = {
         "contentvec": "contentvec_base.pt",
         "hubert": "hubert_base.pt",
@@ -65,16 +64,26 @@ def load_embedding(embedder_model):
     try:
         this_dir = Path(__file__).resolve().parent.parent.parent.parent.parent
         model_path = this_dir / "models" / "rvc_base" / embedding_list[embedder_model]
-        #print("MODEL PATH IS", model_path)
         
-        # Load model ensemble and task
+        # Import Dictionary class from fairseq
+        from fairseq.data.dictionary import Dictionary
+        
+        # For PyTorch 2.2+, try using add_safe_globals directly but with proper import 
+        # of the Dictionary class from fairseq
+        try:
             torch.serialization.add_safe_globals([Dictionary])
+        except (AttributeError, ImportError) as e:
+            # If add_safe_globals doesn't exist, we'll need to load with weights_only=False
+            # But that's a security risk with untrusted models
+            logging.warning("Could not use add_safe_globals, loading model with reduced security")
+            pass
+        
+        # Load model
         models = checkpoint_utils.load_model_ensemble_and_task(
             [f"{model_path}"],
             suffix="",
         )
         
-        #print(f"Embedding model {embedder_model} loaded successfully.")
         return models
     except KeyError as e:
         logging.error(f"Invalid embedder model name: {embedder_model}")

system/tts_engines/vits/model_engine.py

@@ -267,6 +267,7 @@ class tts_class:
         model_files = ["model_file.pth", "model.pth"]
         
         found_any_models = False
+        model_name = None
         for subfolder in models_folder.iterdir():
             if subfolder.is_dir():
                 model_name = subfolder.name

test/test_config.py

@@ -2,7 +2,7 @@ import os.path
 import tempfile
 from unittest import TestCase
 from pathlib import Path
-from config import AlltalkConfig, AlltalkTTSEnginesConfig, AlltalkNewEnginesConfig
+from config import AlltalkConfig, AlltalkTTSEnginesConfig, AlltalkNewEnginesConfig, AlltalkMultiEngineManagerConfig
 
 
 class TestAlltalkConfig(TestCase):
@@ -242,3 +242,27 @@ class TestAlltalkNewEnginesConfig(TestCase):
     def test_no_private_fields(self):
         for attr in self.new_engines_config.to_dict().keys():
             self.assertTrue(not attr.startswith("_"))
+
+class TestAlltalkMultiEngineManagerConfig(TestCase):
+
+    def setUp(self):
+        self.mem_config = AlltalkMultiEngineManagerConfig.get_instance()
+        self.mem_config.reload()
+
+    def test_defaults(self):
+        self.assertEqual(self.mem_config.base_port, 7001)
+        self.assertEqual(self.mem_config.api_server_port, 7851)
+        self.assertEqual(self.mem_config.auto_start_engines, 0)
+        self.assertEqual(self.mem_config.max_instances, 8)
+        self.assertEqual(self.mem_config.gradio_interface_port, 7500)
+        self.assertEqual(self.mem_config.max_retries, 12)
+        self.assertEqual(self.mem_config.initial_wait, 2.0)
+        self.assertEqual(self.mem_config.backoff_factor, 1.2)
+        self.assertFalse(self.mem_config.debug_mode)
+        self.assertEqual(self.mem_config.max_queue_time, 60)
+        self.assertEqual(self.mem_config.queue_check_interval, 0.1)
+        self.assertEqual(self.mem_config.tts_request_timeout, 30)
+        self.assertEqual(self.mem_config.text_length_factor, 0.2)
+        self.assertEqual(self.mem_config.concurrent_request_factor, 0.5)
+        self.assertEqual(self.mem_config.diminishing_factor, 0.5)
+        self.assertEqual(self.mem_config.queue_position_factor, 1.0)

tts_mem.py

@@ -40,15 +40,12 @@ import os
 import io
 import sys
 import time
-import json
 import queue
 import socket
-import random
 import signal
 import asyncio
 import aiohttp
 import requests
-import importlib
 import threading
 import subprocess
 import gradio as gr
@@ -58,43 +55,7 @@ from collections import deque
 from datetime import datetime, timedelta
 from requests.exceptions import RequestException
 from werkzeug.serving import make_server, WSGIRequestHandler
-from concurrent.futures import ThreadPoolExecutor
-
-def ensure_flask_dependencies():
-    # Check for Flask async support
-    try:
-        import flask.json
-        if not hasattr(flask.json, 'loads'):
-            raise ImportError("Flask async support not found")
-    except ImportError:
-        print("Flask async support not found. Installing...")
-        try:
-            subprocess.check_call([sys.executable, "-m", "pip", "install", "flask[async]"])
-            print("Flask async support installed successfully.")
-            import flask
-            importlib.reload(flask)  # Reload the module to ensure fresh state after install
-        except subprocess.CalledProcessError as e:
-            print(f"Failed to install Flask async support. Error: {e}")
-            print("Please run 'pip install flask[async]' manually.")
-            sys.exit(1)
-
-    # Check for Flask-CORS
-    try:
-        import flask_cors
-    except ImportError:
-        print("Flask-CORS not found. Installing...")
-        try:
-            subprocess.check_call([sys.executable, "-m", "pip", "install", "flask-cors"])
-            print("Flask-CORS installed successfully.")
-        except subprocess.CalledProcessError as e:
-            print(f"Failed to install Flask-CORS. Error: {e}")
-            print("Please run 'pip install flask-cors' manually.")
-            sys.exit(1)
-
-# Check Flask Dependencies
-ensure_flask_dependencies()
-
-# Flask app setup
+from config import AlltalkMultiEngineManagerConfig, AlltalkConfig
 from flask_cors import CORS, cross_origin
 from flask import Flask, request, jsonify, send_from_directory, send_file
 
@@ -120,47 +81,9 @@ flask_app = Flask(__name__)
 CORS(flask_app)
 flask_app.config['OUTPUT_FOLDER'] = str(Path(os.getcwd()) / 'outputs')
 
-# Default configuration
-default_config = {
-    "base_port": 7001,
-    "api_server_port": 7851,
-    "auto_start_engines": 0,
-    "max_instances": 8,
-    "gradio_interface_port": 7500,
-    "max_retries": 12,
-    "initial_wait": 2,
-    "backoff_factor": 1.2,
-    "debug_mode": False,
-    "max_queue_time": 60,  # Maximum time a request can wait in the queue (in seconds)
-    "queue_check_interval": 0.1,  # Time between checks for available instances (in seconds)
-    "tts_request_timeout": 30,  # Timeout for individual TTS requests (in seconds)
-    "text_length_factor": 0.2,  # Increase timeout by 20% per 100 characters
-    "concurrent_request_factor": 0.5,  # Increase timeout by 50% per concurrent request
-    "diminishing_factor": 0.5,  # Reduce additional time for long-running requests by 50%
-    "queue_position_factor": 1.0  # Add 100% of base timeout for each queue position    
-}
-
-def load_config():
-    if os.path.exists(CONFIG_FILE):
-        with open(CONFIG_FILE, 'r') as f:
-            loaded_config = json.load(f)
-            # Update loaded_config with any missing keys from default_config
-            for key, value in default_config.items():
-                if key not in loaded_config:
-                    loaded_config[key] = value
-            # Ensure debug_mode is a boolean
-            loaded_config['debug_mode'] = bool(loaded_config['debug_mode'])
-            return loaded_config
-    return default_config.copy()
-
-def save_config(config):
-    with open(CONFIG_FILE, 'w') as f:
-        json.dump(config, f, indent=2)
-
 # Load configuration at the start of the script
-config = load_config()
-current_base_port = config['base_port']
-max_instances = config['max_instances']
+config = AlltalkMultiEngineManagerConfig.get_instance()
+USE_GRADIO = AlltalkConfig.get_instance().gradio_interface
 
 def signal_handler(signum, frame):
     print("[AllTalk MEM] Interrupt received, shutting down...")
@@ -201,9 +124,10 @@ print(f"[AllTalk MEM]")
 print(f"[AllTalk MEM] \033[93m     MEM is not intended for production use and\033[00m")
 print(f"[AllTalk MEM] \033[93m      there is NO support being offered on MEM\033[00m")
 print(f"[AllTalk MEM]")
-print(f"[AllTalk MEM] \033[94mAPI/Queue   :\033[00m \033[92mhttp://127.0.0.1:{config['api_server_port']}/api/tts-generate\033[00m")
-print(f"[AllTalk MEM] \033[94mGradio Light:\033[00m \033[92mhttp://127.0.0.1:{config['gradio_interface_port']}\033[00m")
-print(f"[AllTalk MEM] \033[94mGradio Dark :\033[00m \033[92mhttp://127.0.0.1:{config['gradio_interface_port']}?__theme=dark\033[00m")
+print(f"[AllTalk MEM] \033[94mAPI/Queue   :\033[00m \033[92mhttp://127.0.0.1:{config.api_server_port}/api/tts-generate\033[00m")
+if USE_GRADIO:
+    print(f"[AllTalk MEM] \033[94mGradio Light:\033[00m \033[92mhttp://127.0.0.1:{config.gradio_interface_port}\033[00m")
+    print(f"[AllTalk MEM] \033[94mGradio Dark :\033[00m \033[92mhttp://127.0.0.1:{config.gradio_interface_port}?__theme=dark\033[00m")
 print(f"[AllTalk MEM]")
 
 def is_port_in_use(port):
@@ -267,21 +191,21 @@ def check_subprocess_status(instance_id, port):
 # to allow for longer startup times of the TTS engine. 
 def retry_with_backoff(func):
     retries = 0
-    wait_time = config['initial_wait']
-    while retries < config['max_retries']:
+    wait_time = config.initial_wait
+    while retries < config.max_retries:
         try:
             result = func()
-            print(f"[AllTalk MEM] Operation successful on attempt {retries + 1}") if config['debug_mode'] else None
+            print(f"[AllTalk MEM] Operation successful on attempt {retries + 1}") if config.debug_mode else None
             return result
         except RequestException as e:
             retries += 1
-            print(f"[AllTalk MEM] Attempt {retries}") if config['debug_mode'] else None
-            if retries == config['max_retries']:
-                print(f"[AllTalk MEM] All {max_retries} attempts failed waiting for this instance of the Engine to load.") if config['debug_mode'] else None
+            print(f"[AllTalk MEM] Attempt {retries}") if config.debug_mode else None
+            if retries == config.max_retries:
+                print(f"[AllTalk MEM] All {config.max_retries} attempts failed waiting for this instance of the Engine to load.") if config.debug_mode else None
                 return False
-            print(f"[AllTalk MEM] Retrying in {wait_time} seconds...") if config['debug_mode'] else None
+            print(f"[AllTalk MEM] Retrying in {wait_time} seconds...") if config.debug_mode else None
             time.sleep(wait_time)
-            wait_time = round(wait_time * config['backoff_factor'], 1)
+            wait_time = round(wait_time * config.backoff_factor, 1)
     return False
 
 def is_server_ready(port, timeout=3):
@@ -294,14 +218,14 @@ def is_server_ready(port, timeout=3):
 
 def stop_all_instances():
     instance_ids = list(processes.keys())
-    results = ["❌ Not running"] * max_instances
+    results = ["❌ Not running"] * config.max_instances
     for instance_id in instance_ids:
         stop_subprocess(instance_id)
         results[instance_id-1] = "❌ Not running"
     return results
 
-def update_all_statuses(base_port):
-    return [check_subprocess_status(i, base_port + i - 1) for i in range(1, max_instances + 1)]            
+def update_all_statuses():
+    return [check_subprocess_status(i, config.base_port + i - 1) for i in range(1, config.max_instances + 1)]
 
 def count_running_instances():
     return sum(1 for p in processes.values() if p.poll() is None)
@@ -336,7 +260,7 @@ def test_tts(engine, voice, text):
         return None, "Please select both an engine and a voice" 
     try:
         engine_num = int(engine.split()[1])
-        port = current_base_port + engine_num - 1
+        port = config.base_port + engine_num - 1
         formatted_text = text.format(engine=engine_num, port=port, voice=voice)
         
         audio_url, debug_msg = generate_tts(port, engine_num, formatted_text, voice)
@@ -351,7 +275,7 @@ def test_tts(engine, voice, text):
 # Gradio Code and functions #
 #############################
 def create_gradio_interface():
-    global current_base_port, max_instances, config, monitor
+    global monitor
     monitor = MonitoringControl()
     with gr.Blocks(title="AllTalk Multi Engine Manager", theme=gr.themes.Base()) as interface:
         with gr.Row():
@@ -371,17 +295,17 @@ def create_gradio_interface():
             }""", show_api=False)
         with gr.Tab("Engine Management"):
             with gr.Row():
-                num_instances_input = gr.Slider(minimum=1, maximum=max_instances, step=1, value=1, label="Number of Instances to Start")
+                num_instances_input = gr.Slider(minimum=1, maximum=config.max_instances, step=1, value=1, label="Number of Instances to Start")
                 start_MEMple_button = gr.Button("Start Instances")
                 stop_all_button = gr.Button("Stop All Instances")
                 refresh_button = gr.Button("Refresh Status")
             error_box = gr.Textbox(label="Error Messages", visible=False)
             
             instance_statuses = []
-            for i in range(0, max_instances, 8):  # Create rows with 8 instances each
+            for i in range(0, config.max_instances, 8):  # Create rows with 8 instances each
                 with gr.Row():
                     for j in range(8):
-                        if i + j < max_instances:
+                        if i + j < config.max_instances:
                             instance_id = i + j + 1
                             
                             # Use a column for each engine instance to control layout
@@ -396,15 +320,15 @@ def create_gradio_interface():
                                     restart_btn = gr.Button("♻️ Re-St", size="sm", scale=1, min_width=10)
 
                                 # Define click behavior for each button
-                                start_btn.click(lambda id=instance_id: start_and_update(id, current_base_port + id - 1), outputs=status)
+                                start_btn.click(lambda id=instance_id: start_and_update(id, config.base_port + id - 1), outputs=status)
                                 stop_btn.click(lambda id=instance_id: stop_and_update(id), outputs=status)
-                                restart_btn.click(lambda id=instance_id: restart_and_update(id, current_base_port + id - 1), outputs=status)
+                                restart_btn.click(lambda id=instance_id: restart_and_update(id, config.base_port + id - 1), outputs=status)
 
                             instance_statuses.append(status)
             
             def refresh_and_update_slider():
                 running_count = count_running_instances()
-                statuses = update_all_statuses(current_base_port)
+                statuses = update_all_statuses()
                 return [gr.update(value=running_count)] + statuses
 
             refresh_button.click(
@@ -414,9 +338,9 @@ def create_gradio_interface():
                 
             def validate_and_start(num_instances):
                 if num_instances < 1:
-                    return [gr.update(value=1)] + ["❌ Not running"] * max_instances + ["Invalid number of instances"]
+                    return [gr.update(value=1)] + ["❌ Not running"] * config.max_instances + ["Invalid number of instances"]
                 
-                results = start_MEMple_instances(num_instances, current_base_port)
+                results = start_MEMple_instances(num_instances)
                 running_count = count_running_instances()
                 return [gr.update(value=running_count)] + results + [""]
 
@@ -463,7 +387,7 @@ def create_gradio_interface():
             async def update_voice_selector(engine):
                 if engine:
                     engine_num = int(engine.split()[1])
-                    port = current_base_port + engine_num - 1
+                    port = config.base_port + engine_num - 1
                     voices = await get_available_voices(port)
                     debug_str = f"Fetched voices for Engine {engine_num} (Port {port}): {voices}"
                     return gr.update(choices=voices, value=voices[0] if voices else None), debug_str
@@ -489,17 +413,17 @@ def create_gradio_interface():
                     gr.Markdown("### 🟦 Engine Instance Management")
                     with gr.Row():
                         base_port_input = gr.Number(
-                            value=config['base_port'],
-                            label=f"Starting Port Number (Current: {config['base_port']})",
+                            value=config.base_port,
+                            label=f"Starting Port Number (Current: {config.base_port})",
                             step=1
                         )
                         max_instances_input = gr.Number(
-                            value=config['max_instances'],
-                            label=f"Maximum Engine Instances (Current: {config['max_instances']})",
+                            value=config.max_instances,
+                            label=f"Maximum Engine Instances (Current: {config.max_instances})",
                             step=1
                         )
                         auto_start_engines = gr.Number(
-                            value=config['auto_start_engines'],
+                            value=config.auto_start_engines,
                             label="Auto-start Engine Instances (0 to disable)",
                             step=1
                         )
@@ -508,13 +432,13 @@ def create_gradio_interface():
                     gr.Markdown("### 🟨 Gradio Port & API Port")                                      
                     with gr.Row():
                         gradio_port_input = gr.Number(
-                            value=config['gradio_interface_port'],
-                            label=f"Gradio Interface Port (Current: {config['gradio_interface_port']})",
+                            value=config.gradio_interface_port,
+                            label=f"Gradio Interface Port (Current: {config.gradio_interface_port})",
                             step=1
                         )
                         api_server_port_input = gr.Number(
-                            value=config['api_server_port'],
-                            label=f"API Server Port (Current: {config['api_server_port']})",
+                            value=config.api_server_port,
+                            label=f"API Server Port (Current: {config.api_server_port})",
                             step=1
                         )                                     
                                         
@@ -523,18 +447,18 @@ def create_gradio_interface():
                     gr.Markdown("### 🟥 Engine Start-up Time Contol")
                     with gr.Row():
                         max_retries_input = gr.Number(
-                            value=config['max_retries'],
-                            label=f"Max Retries (Current: {config['max_retries']})",
+                            value=config.max_retries,
+                            label=f"Max Retries (Current: {config.max_retries})",
                             step=1
                         )
                         initial_wait_input = gr.Number(
-                            value=config['initial_wait'],
-                            label=f"Initial Wait (Current: {config['initial_wait']})",
+                            value=config.initial_wait,
+                            label=f"Initial Wait (Current: {config.initial_wait})",
                             step=0.1
                         )
                         backoff_factor_input = gr.Number(
-                            value=config['backoff_factor'],
-                            label=f"Backoff Factor (Current: {config['backoff_factor']})",
+                            value=config.backoff_factor,
+                            label=f"Backoff Factor (Current: {config.backoff_factor})",
                             step=0.1
                         )
                 with gr.Column(scale=1):
@@ -542,7 +466,7 @@ def create_gradio_interface():
                     with gr.Row():                        
                         debug_mode_input = gr.Dropdown(
                             choices=["Disabled", "Enabled"],
-                            value="Enabled" if config['debug_mode'] else "Disabled",
+                            value="Enabled" if config.debug_mode else "Disabled",
                             label="Debug Mode"
                         ) 
 
@@ -551,41 +475,41 @@ def create_gradio_interface():
                     gr.Markdown("### 🟩 Core Queue Management")                  
                     with gr.Row(): 
                         max_queue_time_input = gr.Number(
-                            value=config['max_queue_time'],
-                            label=f"Max Queue Time (seconds) (Current: {config['max_queue_time']})",
+                            value=config.max_queue_time,
+                            label=f"Max Queue Time (seconds) (Current: {config.max_queue_time})",
                             step=1
                         )
                         queue_check_interval_input = gr.Number(
-                            value=config['queue_check_interval'],
-                            label=f"Queue Check Interval (seconds) (Current: {config['queue_check_interval']})",
+                            value=config.queue_check_interval,
+                            label=f"Queue Check Interval (seconds) (Current: {config.queue_check_interval})",
                             step=0.1
                         )
                 with gr.Column(scale=3):
                     gr.Markdown("### 🟩 TTS Request Dynamic Timeout Factors")                  
                     with gr.Row():
                         tts_request_timeout_input = gr.Number(
-                            value=config['tts_request_timeout'],
-                            label=f"TTS Request Timeout (seconds) (Current: {config['tts_request_timeout']})",
+                            value=config.tts_request_timeout,
+                            label=f"TTS Request Timeout (seconds) (Current: {config.tts_request_timeout})",
                             step=1
                         )                                                         
                         text_length_factor_input = gr.Number(
-                            value=config['text_length_factor'],
-                            label=f"Text Length Factor Per 100 chrs (Current: {config['text_length_factor']})",
+                            value=config.text_length_factor,
+                            label=f"Text Length Factor Per 100 chrs (Current: {config.text_length_factor})",
                             step=0.1
                         )
                         concurrent_request_factor_input = gr.Number(
-                            value=config['concurrent_request_factor'],
-                            label=f"Concurrent Request Factor (Current: {config['concurrent_request_factor']})",
+                            value=config.concurrent_request_factor,
+                            label=f"Concurrent Request Factor (Current: {config.concurrent_request_factor})",
                             step=0.1
                         )
                         diminishing_factor_input = gr.Number(
-                            value=config['diminishing_factor'],
-                            label=f"Diminishing Factor Calc (Current: {config['diminishing_factor']})",
+                            value=config.diminishing_factor,
+                            label=f"Diminishing Factor Calc (Current: {config.diminishing_factor})",
                             step=0.1
                         )
                         queue_position_factor_input = gr.Number(
-                            value=config['queue_position_factor'],
-                            label=f"Queue Position Factor (Current: {config['queue_position_factor']})",
+                            value=config.queue_position_factor,
+                            label=f"Queue Position Factor (Current: {config.queue_position_factor})",
                             step=0.1
                         )                                                           
                                           
@@ -599,69 +523,63 @@ def create_gradio_interface():
                                 new_queue_check_interval, new_tts_request_timeout,
                                 new_text_length_factor, new_concurrent_request_factor,
                                 new_diminishing_factor, new_queue_position_factor):
-                global current_base_port, max_instances, config, server_port
-                
                 if new_port < 1024:
                     new_port = 7001
                 
                 new_auto_start = min(int(new_auto_start), int(new_max_instances))
                 
-                current_base_port = int(new_port)
-                max_instances = int(new_max_instances)
-                server_port = int(new_gradio_port)
+                config.base_port = int(new_port)
+                config.max_instances = int(new_max_instances)
+                config.auto_start_engines = new_auto_start
+                config.gradio_interface_port = int(new_gradio_port)
+                config.api_server_port = int(new_api_server_port)
+                config.max_retries = int(new_max_retries)
+                config.initial_wait = float(new_initial_wait)
+                config.backoff_factor = float(new_backoff_factor)
+                config.debug_mode = (new_debug_mode == "Enabled")
+                config.max_queue_time = int(new_max_queue_time)
+                config.queue_check_interval = float(new_queue_check_interval)
+                config.tts_request_timeout = int(new_tts_request_timeout)
+                config.text_length_factor = float(new_text_length_factor)
+                config.concurrent_request_factor = float(new_concurrent_request_factor)
+                config.diminishing_factor = float(new_diminishing_factor)
+                config.queue_position_factor = float(new_queue_position_factor)
                 
-                config['base_port'] = current_base_port
-                config['max_instances'] = max_instances
-                config['auto_start_engines'] = new_auto_start
-                config['gradio_interface_port'] = server_port
-                config['api_server_port'] = int(new_api_server_port)
-                config['max_retries'] = int(new_max_retries)
-                config['initial_wait'] = float(new_initial_wait)
-                config['backoff_factor'] = float(new_backoff_factor)
-                config['debug_mode'] = (new_debug_mode == "Enabled")
-                config['max_queue_time'] = int(new_max_queue_time)
-                config['queue_check_interval'] = float(new_queue_check_interval)
-                config['tts_request_timeout'] = int(new_tts_request_timeout)
-                config['text_length_factor'] = float(new_text_length_factor)
-                config['concurrent_request_factor'] = float(new_concurrent_request_factor)
-                config['diminishing_factor'] = float(new_diminishing_factor)
-                config['queue_position_factor'] = float(new_queue_position_factor)
-                
-                save_config(config)
+                config.save()
                 
                 status_message = (
-                    f"Settings updated: Start port: {current_base_port}, Max instances: {max_instances}, "
-                    f"Auto-start engines: {config['auto_start_engines']}, Gradio port: {server_port}, "
-                    f"API Server Port: {config['api_server_port']}, "
-                    f"Max retries: {config['max_retries']}, Initial wait: {config['initial_wait']}, "
-                    f"Backoff factor: {config['backoff_factor']}, Debug mode: {config['debug_mode']}, "
-                    f"Max queue time: {config['max_queue_time']}, "
-                    f"Queue check interval: {config['queue_check_interval']}, "
-                    f"TTS request timeout: {config['tts_request_timeout']}, "
-                    f"Text length factor: {config['text_length_factor']}, "
-                    f"Concurrent request factor: {config['concurrent_request_factor']}, "
-                    f"Diminishing factor: {config['diminishing_factor']}, "
-                    f"Queue position factor: {config['queue_position_factor']}"
+                    f"Settings updated: Start port: {config.base_port}, Max instances: {config.max_instances}, "
+                    f"Auto-start engines: {config.auto_start_engines}, Gradio port: {config.gradio_interface_port}, "
+                    f"API Server Port: {config.api_server_port}, "
+                    f"Max retries: {config.max_retries}, Initial wait: {config.initial_wait}, "
+                    f"Backoff factor: {config.backoff_factor}, Debug mode: {config.debug_mode}, "
+                    f"Max queue time: {config.max_queue_time}, "
+                    f"Queue check interval: {config.queue_check_interval}, "
+                    f"TTS request timeout: {config.tts_request_timeout}, "
+                    f"Text length factor: {config.text_length_factor}, "
+                    f"Concurrent request factor: {config.concurrent_request_factor}, "
+                    f"Diminishing factor: {config.diminishing_factor}, "
+                    f"Queue position factor: {config.queue_position_factor}"
                 )
                 
                 return (
                     status_message,
-                    gr.update(value=current_base_port, label=f"Starting Port Number (Current: {current_base_port})"),
-                    gr.update(value=max_instances, label=f"Maximum Instances (Current: {max_instances})"),
-                    gr.update(value=config['auto_start_engines']),
-                    gr.update(value=server_port, label=f"Gradio Interface Port (Current: {server_port})"),
-                    gr.update(value=config['api_server_port'], label=f"API Server Port (Current: {config['api_server_port']})"),
-                    gr.update(value=config['max_retries']),
-                    gr.update(value=config['initial_wait']),
-                    gr.update(value=config['backoff_factor']),
-                    gr.update(value="Enabled" if config['debug_mode'] else "Disabled"),
-                    gr.update(value=config['max_queue_time'], label=f"Max Queue Time (seconds) (Current: {config['max_queue_time']})"),
-                    gr.update(value=config['queue_check_interval'], label=f"Queue Check Interval (seconds) (Current: {config['queue_check_interval']})"),
-                    gr.update(value=config['tts_request_timeout'], label=f"TTS Request Timeout (seconds) (Current: {config['tts_request_timeout']})"),
-                    gr.update(value=config['text_length_factor'], label=f"Text Length Factor (Current: {config['text_length_factor']})"),
-                    gr.update(value=config['concurrent_request_factor'], label=f"Concurrent Request Factor (Current: {config['concurrent_request_factor']})"),
-                    gr.update(value=config['diminishing_factor'], label=f"Diminishing Factor (Current: {config['diminishing_factor']})"),
-                    gr.update(value=config['queue_position_factor'], label=f"Queue Position Factor (Current: {config['queue_position_factor']})")
+                    gr.update(value=config.base_port, label=f"Starting Port Number (Current: {config.base_port})"),
+                    gr.update(value=config.max_instances, label=f"Maximum Instances (Current: {config.max_instances})"),
+                    gr.update(value=config.auto_start_engines),
+                    gr.update(value=config.gradio_interface_port, label=f"Gradio Interface Port (Current: {config.gradio_interface_port})"),
+                    gr.update(value=config.api_server_port, label=f"API Server Port (Current: {config.api_server_port})"),
+                    gr.update(value=config.max_retries),
+                    gr.update(value=config.initial_wait),
+                    gr.update(value=config.backoff_factor),
+                    gr.update(value="Enabled" if config.debug_mode else "Disabled"),
+                    gr.update(value=config.max_queue_time, label=f"Max Queue Time (seconds) (Current: {config.max_queue_time})"),
+                    gr.update(value=config.queue_check_interval, label=f"Queue Check Interval (seconds) (Current: {config.queue_check_interval})"),
+                    gr.update(value=config.tts_request_timeout, label=f"TTS Request Timeout (seconds) (Current: {config.tts_request_timeout})"),
+                    gr.update(value=config.text_length_factor, label=f"Text Length Factor (Current: {config.text_length_factor})"),
+                    gr.update(value=config.concurrent_request_factor, label=f"Concurrent Request Factor (Current: {config.concurrent_request_factor})"),
+                    gr.update(value=config.diminishing_factor, label=f"Diminishing Factor (Current: {config.diminishing_factor})"),
+                    gr.update(value=config.queue_position_factor, label=f"Queue Position Factor (Current: {config.queue_position_factor})")
                 )
 
             set_settings_button.click(
@@ -866,7 +784,7 @@ def create_gradio_interface():
                     queue_data = []
                     for i, item in enumerate(list(queue_items)[:10]):
                         wait_time = datetime.now() - item.start_time
-                        timeout = timedelta(seconds=config['max_queue_time']) - wait_time
+                        timeout = timedelta(seconds=config.max_queue_time) - wait_time
                         queue_data.append([
                             i+1,
                             str(wait_time).split(".")[0],
@@ -1114,19 +1032,19 @@ def restart_and_update(instance_id, port):
     result = restart_subprocess(instance_id, port)
     return check_subprocess_status(instance_id, port)
 
-def start_MEMple_instances(num_instances, base_port):
+def start_MEMple_instances(num_instances):
     currently_running = count_running_instances()
     
     if currently_running >= num_instances:
-        return update_all_statuses(base_port)  # All requested instances are already running
+        return update_all_statuses()  # All requested instances are already running
     
     instances_to_start = num_instances - currently_running
-    start_port, available_ports = find_available_port(base_port + currently_running, instances_to_start)
+    start_port, available_ports = find_available_port(config.base_port + currently_running, instances_to_start)
     
     if len(available_ports) < instances_to_start:
-        return ["Not enough available ports"] * max_instances
+        return ["Not enough available ports"] * config.max_instances
 
-    results = update_all_statuses(base_port)
+    results = update_all_statuses()
     for i, port in enumerate(available_ports[:instances_to_start], start=currently_running+1):
         result = start_subprocess(i, port)
         results[i-1] = result
@@ -1152,8 +1070,8 @@ def initialize_instances():
     global tts_instances
     with lock:
         tts_instances.clear()
-        for i in range(1, max_instances + 1):
-            port = current_base_port + i - 1
+        for i in range(1, config.max_instances + 1):
+            port = config.base_port + i - 1
             tts_instances[i] = {"port": port, "locked": False}
 
 def get_available_instance():
@@ -1183,23 +1101,23 @@ def update_tts_instances():
         tts_instances.clear()
         for i, process in processes.items():
             if process.poll() is None:  # Check if the process is running
-                port = current_base_port + i - 1
+                port = config.base_port + i - 1
                 tts_instances[i] = {"port": port, "locked": False}
 
 def calculate_dynamic_timeout(text, concurrent_requests, queue_position, total_queue_length, request_start_time):
-    base_timeout = config['tts_request_timeout'] * (1 + (concurrent_requests * 0.5))
+    base_timeout = config.tts_request_timeout * (1 + (concurrent_requests * 0.5))
     
     text_length_factor = len(text) / 100
     adjusted_timeout = base_timeout * (1 + (text_length_factor * 0.2))
     
     time_in_process = time.time() - request_start_time
     diminishing_factor = max(0, 1 - (time_in_process / base_timeout))
-    timeout_with_diminishing = adjusted_timeout * (1 + (diminishing_factor * config['diminishing_factor']))
+    timeout_with_diminishing = adjusted_timeout * (1 + (diminishing_factor * config.diminishing_factor))
     
     queue_position_factor = queue_position / total_queue_length
-    final_timeout = timeout_with_diminishing + (config['tts_request_timeout'] * queue_position_factor)
+    final_timeout = timeout_with_diminishing + (config.tts_request_timeout * queue_position_factor)
     
-    return min(final_timeout, config['max_queue_time'])  # Ensure we don't exceed max queue time
+    return min(final_timeout, config.max_queue_time)  # Ensure we don't exceed max queue time
 
 ########################################
 ## WEBSERVER # Client Facing Endpoints #
@@ -1207,7 +1125,7 @@ def calculate_dynamic_timeout(text, concurrent_requests, queue_position, total_q
 @flask_app.route('/api/tts-generate', methods=['POST'])
 @cross_origin(origins='*', methods=['POST', 'OPTIONS'], allow_headers=['Content-Type'])
 def tts_generate():
-    print("[AllTalk MEM] Received TTS generate request") if config['debug_mode'] else None
+    print("[AllTalk MEM] Received TTS generate request") if config.debug_mode else None
     start_time = datetime.now()
     request_data = request.form.to_dict()
     
@@ -1215,11 +1133,11 @@ def tts_generate():
     queue_item = QueueItem(request_data['text_input'], start_time)
     queue_items.append(queue_item)
     
-    while (datetime.now() - start_time).total_seconds() < config['max_queue_time']:
+    while (datetime.now() - start_time).total_seconds() < config.max_queue_time:
         instance = get_available_instance()
         if instance:
             try:
-                print(f"[AllTalk MEM] Processing request with instance {instance}") if config['debug_mode'] else None
+                print(f"[AllTalk MEM] Processing request with instance {instance}") if config.debug_mode else None
                 
                 # Update engine status
                 engine_statuses[instance] = EngineStatus()
@@ -1236,7 +1154,7 @@ def tts_generate():
                 )
                 
                 result = process_tts_request(instance, request_data, dynamic_timeout)
-                print(f"[AllTalk MEM] Request processed, result: {result}") if config['debug_mode'] else None
+                print(f"[AllTalk MEM] Request processed, result: {result}") if config.debug_mode else None
                 
                 # Remove request from queue
                 queue_items.popleft()
@@ -1249,7 +1167,7 @@ def tts_generate():
                 print(f"[AllTalk MEM] Error processing request: {str(e)}")
                 release_instance(instance)
                 return jsonify({"status": "error", "message": str(e)})
-        time.sleep(config['queue_check_interval'])
+        time.sleep(config.queue_check_interval)
     
     # Remove request from queue if it times out
     queue_items.popleft()
@@ -1270,7 +1188,7 @@ def process_tts_request(instance, data, timeout):
 @flask_app.route('/v1/audio/speech', methods=['POST'])
 @cross_origin(origins='*', methods=['POST', 'OPTIONS'], allow_headers=['Content-Type'])
 def openai_speech():
-    print("[AllTalk MEM] Received OpenAI-style TTS generate request") if config['debug_mode'] else None
+    print("[AllTalk MEM] Received OpenAI-style TTS generate request") if config.debug_mode else None
     start_time = datetime.now()
     request_data = request.json
 
@@ -1278,11 +1196,11 @@ def openai_speech():
     queue_item = QueueItem(request_data['input'], start_time)
     queue_items.append(queue_item)
 
-    while (datetime.now() - start_time).total_seconds() < config['max_queue_time']:
+    while (datetime.now() - start_time).total_seconds() < config.max_queue_time:
         instance = get_available_instance()
         if instance:
             try:
-                print(f"[AllTalk MEM] Processing OpenAI-style request with instance {instance}") if config['debug_mode'] else None
+                print(f"[AllTalk MEM] Processing OpenAI-style request with instance {instance}") if config.debug_mode else None
 
                 # Update engine status
                 engine_statuses[instance] = EngineStatus()
@@ -1299,7 +1217,7 @@ def openai_speech():
                 )
 
                 result = process_openai_tts_request(instance, request_data, dynamic_timeout)
-                print(f"[AllTalk MEM] OpenAI-style request processed, result: {result}") if config['debug_mode'] else None
+                print(f"[AllTalk MEM] OpenAI-style request processed, result: {result}") if config.debug_mode else None
 
                 # Remove request from queue
                 queue_items.popleft()
@@ -1322,7 +1240,7 @@ def openai_speech():
                 release_instance(instance)
                 return jsonify({"error": str(e)}), 500
 
-        time.sleep(config['queue_check_interval'])
+        time.sleep(config.queue_check_interval)
 
     # Remove request from queue if it times out
     queue_items.popleft()
@@ -1427,7 +1345,7 @@ async def get_available_voices(port):
 async def update_voice_selector(engine):
     if engine:
         engine_num = int(engine.split()[1])
-        port = current_base_port + engine_num - 1
+        port = config.base_port + engine_num - 1
         voices = await get_available_voices(port)
         debug_str = f"Fetched voices for Engine {engine_num} (Port {port}): {voices}"
         return gr.update(choices=voices, value=voices[0] if voices else None), debug_str
@@ -1437,7 +1355,7 @@ async def update_voice_selector(engine):
 ## WEBSERVER # Load Testing #
 #############################
 async def run_load_test_gradio(num_requests, base_text, voice):
-    api_url = f"http://127.0.0.1:{config['api_server_port']}/api/tts-generate"
+    api_url = f"http://127.0.0.1:{config.api_server_port}/api/tts-generate"
     start_time = time.time()
 
     async with aiohttp.ClientSession() as session:
@@ -1455,7 +1373,7 @@ async def run_load_test_gradio(num_requests, base_text, voice):
     results = []
     for i, response in enumerate(responses):
         if response and response.get("status") == "generate-success":
-            audio_url = f"http://127.0.0.1:{config['api_server_port']}{response['output_file_url']}"
+            audio_url = f"http://127.0.0.1:{config.api_server_port}{response['output_file_url']}"
             # Round time to 1 decimal place
             rounded_time = round(response['time'], 1)
             results.append([f"Request {i+1}", rounded_time, "Success", audio_url])
@@ -1505,12 +1423,12 @@ class SilentWSGIRequestHandler(WSGIRequestHandler):
 class ServerThread(threading.Thread):
     def __init__(self, app):
         threading.Thread.__init__(self)
-        self.server = make_server('0.0.0.0', config['api_server_port'], app, threaded=True, request_handler=SilentWSGIRequestHandler)
+        self.server = make_server('0.0.0.0', config.api_server_port, app, threaded=True, request_handler=SilentWSGIRequestHandler)
         self.ctx = app.app_context()
         self.ctx.push()
 
     def run(self):
-        print(f"[AllTalk MEM] Starting API server on port {config['api_server_port']}") if config['debug_mode'] else None
+        print(f"[AllTalk MEM] Starting API server on port {config.api_server_port}") if config.debug_mode else None
         self.server.serve_forever()
 
     def shutdown(self):
@@ -1520,7 +1438,7 @@ def start_api_server():
     global server_thread
     server_thread = ServerThread(flask_app)  # Use flask_app here
     server_thread.start()
-    print(f"[AllTalk MEM] API server thread started on port {config['api_server_port']}") if config['debug_mode'] else None
+    print(f"[AllTalk MEM] API server thread started on port {config.api_server_port}") if config.debug_mode else None
 
 def stop_api_server():
     global server_thread
@@ -1563,10 +1481,10 @@ print(f"[AllTalk MEM] ")
 print(f"[AllTalk MEM] MEM Server Ready")
 
 def auto_start_engines():
-    num_engines = config['auto_start_engines']
+    num_engines = config.auto_start_engines
     if num_engines > 0:
         print(f"[AllTalk MEM] Auto-starting {num_engines} engines...")
-        results = start_MEMple_instances(num_engines, current_base_port)
+        results = start_MEMple_instances(num_engines)
         update_tts_instances()
         print(f"[AllTalk MEM] Auto-start complete.")
         print(f"[AllTalk MEM] {results}")   
@@ -1574,9 +1492,13 @@ def auto_start_engines():
 # Main execution
 if __name__ == "__main__":
     signal.signal(signal.SIGINT, signal_handler)
+    if USE_GRADIO:
+        print("[AllTalk MEM] Starting gradio...")
         interface = create_gradio_interface()
         # Start the Gradio interface
-    interface.launch(quiet=True, server_port=config['gradio_interface_port'], prevent_thread_lock=True)
+        interface.launch(quiet=True, server_port=config.gradio_interface_port, prevent_thread_lock=True)
+    else:
+        print("[AllTalk MEM] Skipping gradio start...")
     # Start the API server
     start_api_server()
     initialize_instances()