From 284c100860c7c01657e03d942f0db3c0473c5943 Mon Sep 17 00:00:00 2001 From: ekoc Date: Wed, 12 Feb 2025 21:59:35 +0100 Subject: [PATCH] Refactoring docker images, updating CUDA to 12.6 and pytorch 2.6 --- .gitignore | 1 - DOCKER_README.md | 2 +- Dockerfile | 61 ++++--------------- docker-build.sh | 38 +++++++++--- docker-start.sh | 8 ++- docker/{conda => base}/Dockerfile | 45 +++++++------- .../build-base-env.sh} | 22 ++++--- docker/deepspeed/Dockerfile | 56 ++++------------- docker/deepspeed/build-deepspeed.sh | 25 +++++--- docker/variables.sh | 5 +- system/tts_engines/rvc/lib/utils.py | 3 + 11 files changed, 123 insertions(+), 143 deletions(-) rename docker/{conda => base}/Dockerfile (59%) rename docker/{conda/build-conda-env.sh => base/build-base-env.sh} (66%) diff --git a/.gitignore b/.gitignore index 99a590e..40cdb9b 100644 --- a/.gitignore +++ b/.gitignore @@ -173,5 +173,4 @@ cython_debug/ .idea/* # Docker build -docker/conda/build docker/deepspeed/build \ No newline at end of file diff --git a/DOCKER_README.md b/DOCKER_README.md index 02f7a99..0021ab7 100644 --- a/DOCKER_README.md +++ b/DOCKER_README.md @@ -36,7 +36,7 @@ The Docker image currently works on Windows and Linux, optionally supporting NVI 4. Download CUDA toolkit keyring: `wget https://developer.download.nvidia.com/compute/cuda/repos/wsl-ubuntu/x86_64/cuda-keyring_1.1-1_all.deb` 5. Install keyring: `sudo dpkg -i cuda-keyring_1.1-1_all.deb` 6. Update package list: `sudo apt-get update` -7. Install CUDA toolkit: `sudo apt-get -y install cuda-toolkit-12-4` +7. Install CUDA toolkit: `sudo apt-get -y install cuda-toolkit-12-6` 8. Install Docker Desktop using WSL2 as the backend 9. Restart 10. If you wish to monitor the terminal remotely via SSH, follow [this guide](https://www.hanselman.com/blog/how-to-ssh-into-wsl2-on-windows-10-from-an-external-machine). diff --git a/Dockerfile b/Dockerfile index c7d44b8..ffbcf77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,54 +1,19 @@ -FROM continuumio/miniconda3:24.7.1-0 +ARG GITHUB_REPOSITORY +FROM ${GITHUB_REPOSITORY}alltalk_environment:latest # Argument to choose the model: piper, vits, xtts ARG TTS_MODEL="xtts" ENV TTS_MODEL=$TTS_MODEL -ARG ALLTALK_DIR=/opt/alltalk +ARG DEEPSPEED_VERSION=0.16.2 +ENV DEEPSPEED_VERSION=$DEEPSPEED_VERSION -SHELL ["/bin/bash", "-l", "-c"] -ENV SHELL=/bin/bash -ENV HOST=0.0.0.0 -ENV DEBIAN_FRONTEND=noninteractive -ENV CUDA_DOCKER_ARCH=all -ENV NVIDIA_VISIBLE_DEVICES=all -ENV CONDA_AUTO_UPDATE_CONDA="false" +ARG ALLTALK_DIR=/opt/alltalk ENV GRADIO_SERVER_NAME="0.0.0.0" -############################################################################## -# Installation/Basic Utilities -############################################################################## -RUN <&1 ) - - if [ $? -ne 0 ] ; then - echo "Failed to install conda dependencies: $RESULT" - exit 1 - fi - - conda clean -a && pip cache purge -EOR - ############################################################################## # Install python dependencies (cannot use --no-deps because requirements are not complete) ############################################################################## @@ -59,7 +24,7 @@ ENV PIP_CACHE_DIR=${ALLTALK_DIR}/pip_cache RUN <&1 ) + echo "Using precompiled DeepSpeed wheel at ${DEEPSPEED_WHEEL}" + CFLAGS="-I$CONDA_PREFIX/include/" LDFLAGS="-L$CONDA_PREFIX/lib/" \ + pip install --no-cache-dir ${DEEPSPEED_WHEEL} - if echo $RESULT | grep -izq error ; then + if [ $? -ne 0 ] ; then echo "Failed to install pip dependencies: $RESULT" exit 1 fi @@ -118,7 +84,6 @@ conda activate alltalk python diagnostics.py EOF chmod +x start_alltalk.sh - chmod +x start_environment.sh chmod +x start_finetune.sh chmod +x start_diagnostics.sh EOR @@ -129,9 +94,9 @@ COPY . . # Create script to execute firstrun.py and run it: ############################################################################## RUN echo $'#!/usr/bin/env bash \n\ - source ~/.bashrc \n\ - conda activate alltalk \n\ - python ./system/config/firstrun.py $@' > ./start_firstrun.sh +source ~/.bashrc \n\ +conda activate alltalk \n\ +python ./system/config/firstrun.py $@' > ./start_firstrun.sh RUN chmod +x start_firstrun.sh RUN ./start_firstrun.sh --tts_model $TTS_MODEL diff --git a/docker-build.sh b/docker-build.sh index c61f089..b91311f 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -10,7 +10,6 @@ DOCKER_TAG=latest CLEAN=false # Create required build directories if they don't exist -mkdir -p ${SCRIPT_DIR=}/docker/conda/build mkdir -p ${SCRIPT_DIR=}/docker/deepspeed/build echo "Created build directories" @@ -29,10 +28,20 @@ while [ "$#" -gt 0 ]; do TTS_MODEL="$2" shift ;; + --deepspeed-version) + DEEPSPEED_VERSION="$2" + shift + ;; --tag) DOCKER_TAG="$2" shift ;; + --github-repository) + if [ -n "${GITHUB_REPOSITORY}" ] && ! [[ $GITHUB_REPOSITORY =~ ^--.* ]]; then + GITHUB_REPOSITORY="$2" + shift + fi + ;; --clean) CLEAN=true ;; @@ -45,20 +54,34 @@ while [ "$#" -gt 0 ]; do done if [ "$CLEAN" = true ]; then - rm -rf ${SCRIPT_DIR=}/docker/conda/build rm -rf ${SCRIPT_DIR=}/docker/deepspeed/build # Recreate directories after clean - mkdir -p ${SCRIPT_DIR=}/docker/conda/build mkdir -p ${SCRIPT_DIR=}/docker/deepspeed/build echo "Cleaned and recreated build directories" fi -$SCRIPT_DIR/docker/conda/build-conda-env.sh \ +echo "Building base environment image" +$SCRIPT_DIR/docker/base/build-base-env.sh \ --cuda-version ${CUDA_VERSION} \ - --python-version ${PYTHON_VERSION} + --python-version ${PYTHON_VERSION} \ + --github-repository ${GITHUB_REPOSITORY} \ + --tag ${DOCKER_TAG} +if [ $? -ne 0 ]; then + echo "Failed to build base environment image" + exit 1 +fi + +echo "Building DeepSpeed" $SCRIPT_DIR/docker/deepspeed/build-deepspeed.sh \ - --python-version ${PYTHON_VERSION} + --python-version ${PYTHON_VERSION} \ + --github-repository ${GITHUB_REPOSITORY} \ + --tag ${DOCKER_TAG} + +if [ $? -ne 0 ]; then + echo "Failed to build DeepSpeed" + exit 1 +fi echo "Starting docker build process using TTS model '${TTS_MODEL}' and docker tag '${DOCKER_TAG}'" echo "Building for CUDA $CUDA_VERSION using python ${PYTHON_VERSION}" @@ -68,7 +91,8 @@ docker buildx \ --progress=plain \ --build-arg TTS_MODEL=$TTS_MODEL \ --build-arg ALLTALK_DIR=$ALLTALK_DIR \ - -t alltalk_beta:${DOCKER_TAG} \ + --build-arg DEEPSPEED_VERSION=$DEEPSPEED_VERSION \ + -t ${GITHUB_REPOSITORY}alltalk_beta:${DOCKER_TAG} \ . echo "Docker build process finished. Use docker-start.sh to start the container." diff --git a/docker-start.sh b/docker-start.sh index 0069d07..59e6bbc 100755 --- a/docker-start.sh +++ b/docker-start.sh @@ -29,6 +29,12 @@ while [ "$#" -gt 0 ]; do DOCKER_TAG="$2" shift ;; + --github-repository) + if [ -n "${GITHUB_REPOSITORY}" ] && ! [[ $GITHUB_REPOSITORY =~ ^--.* ]]; then + GITHUB_REPOSITORY="$2" + shift + fi + ;; *) # Allow to pass arbitrary arguments to docker as well to be flexible: ADDITIONAL_ARGS+=( $1 ) @@ -65,4 +71,4 @@ docker run \ --name alltalk \ "${DOCKER_ARGS[@]}" \ "${ADDITIONAL_ARGS[@]}" \ - alltalk_beta:${DOCKER_TAG} &> /dev/stdout \ No newline at end of file + ${GITHUB_REPOSITORY}alltalk_beta:${DOCKER_TAG} &> /dev/stdout \ No newline at end of file diff --git a/docker/conda/Dockerfile b/docker/base/Dockerfile similarity index 59% rename from docker/conda/Dockerfile rename to docker/base/Dockerfile index c179bbd..8df6590 100644 --- a/docker/conda/Dockerfile +++ b/docker/base/Dockerfile @@ -1,6 +1,6 @@ FROM continuumio/miniconda3:24.7.1-0 AS build -ARG CUDA_VERSION=12.4.1 +ARG CUDA_VERSION=12.6.0 ENV CUDA_VERSION=$CUDA_VERSION ARG PYTHON_VERSION=3.11.11 @@ -14,6 +14,8 @@ ENV CUDA_DOCKER_ARCH=all ENV NVIDIA_VISIBLE_DEVICES=all ENV CONDA_AUTO_UPDATE_CONDA="false" +ENV ALLTALK_DIR=/opt/alltalk + ############################################################################## # Installation/Basic Utilities ############################################################################## @@ -35,24 +37,24 @@ EOR ############################################################################## RUN <&1 ) + -c nvidia \ + -c anaconda if [ $? -ne 0 ] ; then echo "Failed to install conda dependencies: $RESULT" @@ -62,16 +64,17 @@ RUN < /environment-cu-${CUDA_VERSION}-cp-${PYTHON_VERSION}.yml -EOR + CUDA_SHORT_VERSION=${CUDA_VERSION%.*} + CUDA_SHORT_VERSION_NO_DOT=${CUDA_SHORT_VERSION//./} -############################################################################## -# Make environment file available to docker build output: -############################################################################## -FROM scratch -COPY --from=build /environment-*.yml / \ No newline at end of file + conda activate alltalk + mkdir -p ${ALLTALK_DIR}/pip_cache + pip install --no-cache-dir --cache-dir=${ALLTALK_DIR}/pip_cache \ + torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu${CUDA_SHORT_VERSION_NO_DOT} + + if [ $? -ne 0 ] ; then + echo "Failed to install pip dependencies: $RESULT" + exit 1 + fi +EOR \ No newline at end of file diff --git a/docker/conda/build-conda-env.sh b/docker/base/build-base-env.sh similarity index 66% rename from docker/conda/build-conda-env.sh rename to docker/base/build-base-env.sh index 6ff6df5..b96fee8 100755 --- a/docker/conda/build-conda-env.sh +++ b/docker/base/build-base-env.sh @@ -5,10 +5,7 @@ cd $SCRIPT_DIR . ${SCRIPT_DIR=}/../variables.sh -if [ -f $SCRIPT_DIR/build/environment-cu-${CUDA_VERSION}-cp-${PYTHON_VERSION}.yml ]; then - echo "Environment file already exists - skipping..." - exit 0 -fi +DOCKER_TAG=latest # Parse arguments while [ "$#" -gt 0 ]; do @@ -21,8 +18,17 @@ while [ "$#" -gt 0 ]; do PYTHON_VERSION="$2" shift ;; + --tag) + DOCKER_TAG="$2" + shift + ;; + --github-repository) + if [ -n "${GITHUB_REPOSITORY}" ] && ! [[ $GITHUB_REPOSITORY =~ ^--.* ]]; then + GITHUB_REPOSITORY="$2" + shift + fi + ;; *) - # Allow to pass arbitrary arguments to docker as well to be flexible: echo "Unknown argument '$1'" exit 1 ;; @@ -32,12 +38,10 @@ done echo "Building conda environment using python ${PYTHON_VERSION} with CUDA ${CUDA_VERSION}" -rm -rf build # make sure to properly clean up -mkdir -p build - docker buildx \ build \ + --progress=plain \ --build-arg CUDA_VERSION=$CUDA_VERSION \ --build-arg PYTHON_VERSION=$PYTHON_VERSION \ - --output=$SCRIPT_DIR/build \ + -t ${GITHUB_REPOSITORY}alltalk_environment:${DOCKER_TAG} \ . \ No newline at end of file diff --git a/docker/deepspeed/Dockerfile b/docker/deepspeed/Dockerfile index 869cfd5..6d69b5d 100644 --- a/docker/deepspeed/Dockerfile +++ b/docker/deepspeed/Dockerfile @@ -1,53 +1,15 @@ -FROM continuumio/miniconda3:24.7.1-0 +ARG GITHUB_REPOSITORY +FROM ${GITHUB_REPOSITORY}alltalk_environment:latest -ARG DEEPSPEED_VERSION=0.16.2 +ARG DEEPSPEED_VERSION=0.16.3 ENV DEEPSPEED_VERSION=$DEEPSPEED_VERSION -SHELL ["/bin/bash", "-l", "-c"] -ENV SHELL=/bin/bash -ENV HOST=0.0.0.0 -ENV DEBIAN_FRONTEND=noninteractive -ENV CUDA_DOCKER_ARCH=all -ENV NVIDIA_VISIBLE_DEVICES=all -ENV CONDA_AUTO_UPDATE_CONDA="false" - ############################################################################## # Directories: ############################################################################## ENV STAGE_DIR=/tmp RUN mkdir -p ${STAGE_DIR} -############################################################################## -# Installation/Basic Utilities -############################################################################## -RUN <&1 ) - - if [ $? -ne 0 ] ; then - echo "Failed to install conda dependencies: $RESULT" - exit 1 - fi - - conda clean -a && pip cache purge -EOR - ############################################################################## # Install python dependencies ############################################################################## @@ -56,8 +18,7 @@ RUN <