From c7f9b04826d49369f8590ea5a11473f6de381c2a Mon Sep 17 00:00:00 2001 From: ekoc Date: Fri, 20 Dec 2024 21:18:20 +0100 Subject: [PATCH] Fixing docker build for DeepSpeed, introducing build for conda environment --- .gitignore | 5 +- DOCKER_README.md | 20 ++-- Dockerfile | 88 ++++++++--------- docker-build.sh | 42 ++++---- docker-start.sh | 11 ++- docker/conda/Dockerfile | 77 +++++++++++++++ docker/conda/build-conda-env.sh | 43 +++++++++ {deepspeed => docker/deepspeed}/Dockerfile | 96 +++++++++---------- .../deepspeed}/build-deepspeed.sh | 37 ++++--- docker/variables.sh | 6 ++ 10 files changed, 272 insertions(+), 153 deletions(-) create mode 100644 docker/conda/Dockerfile create mode 100755 docker/conda/build-conda-env.sh rename {deepspeed => docker/deepspeed}/Dockerfile (56%) rename {deepspeed => docker/deepspeed}/build-deepspeed.sh (54%) create mode 100644 docker/variables.sh diff --git a/.gitignore b/.gitignore index 4428008..99a590e 100644 --- a/.gitignore +++ b/.gitignore @@ -172,5 +172,6 @@ cython_debug/ .idea/* -# Deepspeed build -deepspeed/build \ No newline at end of file +# Docker build +docker/conda/build +docker/deepspeed/build \ No newline at end of file diff --git a/DOCKER_README.md b/DOCKER_README.md index 62a32b6..e2343fa 100644 --- a/DOCKER_README.md +++ b/DOCKER_README.md @@ -2,9 +2,11 @@ The Docker image currently works on Windows and Linux, optionally supporting NVIDIA GPUs. ## General Remarks -- The resulting Docker image is 22 GB in size. Building might require even more disk space temporarily. -- Build time depends on your hardware and internet connection. Expect at least 10min to be normal. -- The Docker build: +- The resulting Docker image is 21 GB in size. Building might require even more disk space temporarily. + - Another 15 GB is required for building DeepSpeed +- Build time depends on your hardware and internet connection. Expect at least 20-30min to be normal for a full build. + - This includes building the conda environment as well as DeepSpeed, which is the basis for the alltalk Docker image. +- The Docker build for alltalk: - Downloads XTTS as default TTS engine - Enables RVC by default - Downloads all supported RVC models @@ -55,18 +57,22 @@ There are various arguments to customize the build and start of the docker image - Example: `docker-build.sh --tts_model piper` - `--tag` allows to choose the docker tag. Defaults to `latest`. - Example: `docker-build.sh --tag mytag` +- `--clean` allows remove existing dependency build like conda environment or DeepSpeed. + - Example: `docker-build.sh --clean` ### Arguments for `docker-start.sh` - `--config` lets you choose a config JSON file which can subset of `confignew.json`. This allows you to change only few values and leave the rest as defined in the default `confignew.json` file. - - Example: `docker-build.sh --config /my/config/file.json` with content `{"branding": "My Brand "}` will just change + - Example: `docker-start.sh --config /my/config/file.json` with content `{"branding": "My Brand "}` will just change the branding in `confignew.json`. - `--voices` lets you add voices for the TTS engine in WAV format. You have to specify the folder containing all voice files. - - Example: `docker-build.sh --voices /my/voices/dir` + - Example: `docker-start.sh --voices /my/voices/dir` - `--rvc_voices` similar to voices, this option lets you pick the folder containing the RVC models. - - Example: `docker-build.sh --rvc_vices /my/rvc/voices/dir` + - Example: `docker-start.sh --rvc_vices /my/rvc/voices/dir` - `--no_ui` allows you to not expose port 7852 for the gradio interface. Note that you still have to set `launch_gradio` - to `false` via JSON file passed to `--config`. + to `false` via JSON file passed to `--config`. +- `--tag` allows to choose the docker tag of the image to run. Defaults to `latest`. + - Example: `docker-start.sh --tag mytag` - Since the above commands only address the most important options, you might pass additional arbitrary docker commands to the `docker-start.sh`. diff --git a/Dockerfile b/Dockerfile index d865510..71b83d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,23 +4,21 @@ FROM continuumio/miniconda3:24.7.1-0 ARG TTS_MODEL="xtts" ENV TTS_MODEL=$TTS_MODEL -ARG CUDA_VERSION="12.1.1" -ENV CUDA_VERSION=$CUDA_VERSION - -ARG PYTHON_VERSION=3.11.9 -ENV PYTHON_VERSION=$PYTHON_VERSION - -ARG PYTORCH_VERSION=2.2.1 -ENV PYTORCH_VERSION=$PYTORCH_VERSION +ARG ALLTALK_DIR=/opt/alltalk SHELL ["/bin/bash", "-l", "-c"] ENV SHELL=/bin/bash ENV HOST=0.0.0.0 ENV DEBIAN_FRONTEND=noninteractive ENV CUDA_DOCKER_ARCH=all -ENV GRADIO_SERVER_NAME="0.0.0.0" ENV NVIDIA_VISIBLE_DEVICES=all +ENV CONDA_AUTO_UPDATE_CONDA="false" +ENV GRADIO_SERVER_NAME="0.0.0.0" + +############################################################################## +# Installation/Basic Utilities +############################################################################## RUN <&1 ) + RESULT=$( { conda env create -f environment.yml ; } 2>&1 ) if echo $RESULT | grep -izq error ; then - echo "Failed to install conda dependencies 2: $RESULT" + echo "Failed to install conda dependencies: $RESULT" exit 1 fi + conda clean -a && pip cache purge EOR +############################################################################## # Install python dependencies (cannot use --no-deps because requirements are not complete) +############################################################################## COPY system/config system/config COPY system/requirements/requirements_standalone.txt system/requirements/requirements_standalone.txt COPY system/requirements/requirements_parler.txt system/requirements/requirements_parler.txt -ENV PIP_CACHE_DIR=/alltalk/pip_cache +ENV PIP_CACHE_DIR=${ALLTALK_DIR}/pip_cache RUN < start_alltalk.sh #!/usr/bin/env bash @@ -141,7 +125,9 @@ EOR COPY . . -# Create script to execute firstrun.py: +############################################################################## +# Create script to execute firstrun.py and run it: +############################################################################## RUN echo $'#!/usr/bin/env bash \n\ source ~/.bashrc \n\ conda activate alltalk \n\ @@ -150,18 +136,24 @@ python ./system/config/firstrun.py $@' > ./start_firstrun.sh RUN chmod +x start_firstrun.sh RUN ./start_firstrun.sh --tts_model $TTS_MODEL -RUN mkdir -p /alltalk/outputs +RUN mkdir -p ${ALLTALK_DIR}/outputs RUN mkdir -p /root/.triton/autotune -# Enabling deepspeed for all models: +############################################################################## +# Enable deepspeed for all models: +############################################################################## RUN find . -name model_settings.json -exec sed -i -e 's/"deepspeed_enabled": false/"deepspeed_enabled": true/g' {} \; -# Downloading all RVC models: +############################################################################## +# Download all RVC models: +############################################################################## RUN < /tmp/rvc_files.txt xargs -n 1 curl --create-dirs --output-dir models/rvc_base -LO < /tmp/rvc_files.txt rm -f /tmp/rvc_files.txt EOR -## Start alltalk: +############################################################################## +# Start alltalk: +############################################################################## ENTRYPOINT ["sh", "-c", "./start_alltalk.sh"] diff --git a/docker-build.sh b/docker-build.sh index 3c0e1aa..0d52e0a 100755 --- a/docker-build.sh +++ b/docker-build.sh @@ -1,10 +1,13 @@ #!/usr/bin/env bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR + +. ${SCRIPT_DIR=}/docker/variables.sh + TTS_MODEL=xtts -CUDA_VERSION=12.1.1 -PYTHON_VERSION=3.11.9 -PYTORCH_VERSION=2.2.1 DOCKER_TAG=latest +CLEAN=false # Parse arguments while [ "$#" -gt 0 ]; do @@ -17,10 +20,6 @@ while [ "$#" -gt 0 ]; do PYTHON_VERSION="$2" shift ;; - --pytorch-version) - PYTORCH_VERSION="$2" - shift - ;; --tts_model) TTS_MODEL="$2" shift @@ -29,6 +28,9 @@ while [ "$#" -gt 0 ]; do DOCKER_TAG="$2" shift ;; + --clean) + CLEAN=true + ;; *) printf '%s\n' "Invalid argument ($1)" exit 1 @@ -37,29 +39,27 @@ while [ "$#" -gt 0 ]; do shift done -echo "$PYTHON_VERSION -> ${PYTHON_VERSION%.*}" +if [ "$CLEAN" = true ]; then + rm -rf ${SCRIPT_DIR=}/docker/conda/build + rm -rf ${SCRIPT_DIR=}/docker/deepspeed/build +fi -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -cd $SCRIPT_DIR - -PYTHON_MAJOR_MINOR=${PYTHON_VERSION%.*} -$SCRIPT_DIR/deepspeed/build-deepspeed.sh \ +$SCRIPT_DIR/docker/conda/build-conda-env.sh \ --cuda-version ${CUDA_VERSION} \ - --python-version ${PYTHON_MAJOR_MINOR} \ - --pytorch-version ${PYTORCH_VERSION} + --python-version ${PYTHON_VERSION} + +$SCRIPT_DIR/docker/deepspeed/build-deepspeed.sh \ + --python-version ${PYTHON_VERSION} echo "Starting docker build process using TTS model '${TTS_MODEL}' and docker tag '${DOCKER_TAG}'" -echo "Building for CUDA $CUDA_VERSION using python ${PYTHON_VERSION} with PyTorch ${PYTORCH_VERSION}" - +echo "Building for CUDA $CUDA_VERSION using python ${PYTHON_VERSION}" docker buildx \ build \ --progress=plain \ --build-arg TTS_MODEL=$TTS_MODEL \ - --build-arg CUDA_VERSION=$CUDA_VERSION \ - --build-arg PYTHON_VERSION=$PYTHON_VERSION \ - --build-arg PYTORCH_VERSION=$PYTORCH_VERSION \ + --build-arg ALLTALK_DIR=$ALLTALK_DIR \ -t alltalk_beta:${DOCKER_TAG} \ . -echo "Docker build process finished" \ No newline at end of file +echo "Docker build process finished. Use docker-start.sh to start the container." \ No newline at end of file diff --git a/docker-start.sh b/docker-start.sh index d76486a..0069d07 100755 --- a/docker-start.sh +++ b/docker-start.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash -ALLTALK_DIR="/alltalk" +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +. ${SCRIPT_DIR=}/docker/variables.sh + WITH_UI=true +DOCKER_TAG=latest declare -a ADDITIONAL_ARGS=() # Parse arguments @@ -22,6 +25,10 @@ while [ "$#" -gt 0 ]; do --no_ui) WITH_UI=false ;; + --tag) + DOCKER_TAG="$2" + shift + ;; *) # Allow to pass arbitrary arguments to docker as well to be flexible: ADDITIONAL_ARGS+=( $1 ) @@ -58,4 +65,4 @@ docker run \ --name alltalk \ "${DOCKER_ARGS[@]}" \ "${ADDITIONAL_ARGS[@]}" \ - alltalk_beta:latest &> /dev/stdout \ No newline at end of file + alltalk_beta:${DOCKER_TAG} &> /dev/stdout \ No newline at end of file diff --git a/docker/conda/Dockerfile b/docker/conda/Dockerfile new file mode 100644 index 0000000..57ff742 --- /dev/null +++ b/docker/conda/Dockerfile @@ -0,0 +1,77 @@ +FROM continuumio/miniconda3:24.7.1-0 AS build + +ARG CUDA_VERSION=12.4.1 +ENV CUDA_VERSION=$CUDA_VERSION + +ARG PYTHON_VERSION=3.11.11 +ENV PYTHON_VERSION=$PYTHON_VERSION + +SHELL ["/bin/bash", "-l", "-c"] +ENV SHELL=/bin/bash +ENV HOST=0.0.0.0 +ENV DEBIAN_FRONTEND=noninteractive +ENV CUDA_DOCKER_ARCH=all +ENV NVIDIA_VISIBLE_DEVICES=all +ENV CONDA_AUTO_UPDATE_CONDA="false" + +############################################################################## +# Installation/Basic Utilities +############################################################################## +RUN <&1 ) + + if echo $RESULT | grep -izq error ; then + echo "Failed to install conda dependencies: $RESULT" + exit 1 + fi + + conda clean -a && pip cache purge +EOR + +############################################################################## +# Export conda environment: +############################################################################## +RUN < /environment-cu-${CUDA_VERSION}-cp-${PYTHON_VERSION}.yml +EOR + +############################################################################## +# Make environment file available to docker build output: +############################################################################## +FROM scratch +COPY --from=build /environment-*.yml / \ No newline at end of file diff --git a/docker/conda/build-conda-env.sh b/docker/conda/build-conda-env.sh new file mode 100755 index 0000000..6ff6df5 --- /dev/null +++ b/docker/conda/build-conda-env.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR + +. ${SCRIPT_DIR=}/../variables.sh + +if [ -f $SCRIPT_DIR/build/environment-cu-${CUDA_VERSION}-cp-${PYTHON_VERSION}.yml ]; then + echo "Environment file already exists - skipping..." + exit 0 +fi + +# Parse arguments +while [ "$#" -gt 0 ]; do + case "$1" in + --cuda-version) + CUDA_VERSION="$2" + shift + ;; + --python-version) + PYTHON_VERSION="$2" + shift + ;; + *) + # Allow to pass arbitrary arguments to docker as well to be flexible: + echo "Unknown argument '$1'" + exit 1 + ;; + esac + shift +done + +echo "Building conda environment using python ${PYTHON_VERSION} with CUDA ${CUDA_VERSION}" + +rm -rf build # make sure to properly clean up +mkdir -p build + +docker buildx \ + build \ + --build-arg CUDA_VERSION=$CUDA_VERSION \ + --build-arg PYTHON_VERSION=$PYTHON_VERSION \ + --output=$SCRIPT_DIR/build \ + . \ No newline at end of file diff --git a/deepspeed/Dockerfile b/docker/deepspeed/Dockerfile similarity index 56% rename from deepspeed/Dockerfile rename to docker/deepspeed/Dockerfile index 26df530..05c256c 100644 --- a/deepspeed/Dockerfile +++ b/docker/deepspeed/Dockerfile @@ -1,16 +1,15 @@ -ARG CUDA_VERSION=12.1.1 -FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04 +FROM continuumio/miniconda3:24.7.1-0 -ARG PYTHON_VERSION=3.11 -ENV PYTHON_VERSION=$PYTHON_VERSION - -ARG PYTORCH_VERSION=2.2.1 -ENV PYTORCH_VERSION=$PYTORCH_VERSION - -ARG DEEPSPEED_VERSION=0.16.1 +ARG DEEPSPEED_VERSION=0.16.2 ENV DEEPSPEED_VERSION=$DEEPSPEED_VERSION +SHELL ["/bin/bash", "-l", "-c"] +ENV SHELL=/bin/bash +ENV HOST=0.0.0.0 ENV DEBIAN_FRONTEND=noninteractive +ENV CUDA_DOCKER_ARCH=all +ENV NVIDIA_VISIBLE_DEVICES=all +ENV CONDA_AUTO_UPDATE_CONDA="false" ############################################################################## # Directories: @@ -21,62 +20,49 @@ RUN mkdir -p ${STAGE_DIR} ############################################################################## # Installation/Basic Utilities ############################################################################## -SHELL ["/bin/bash", "-l", "-c"] -ENV SHELL=/bin/bash - RUN <&1 ) - curl -O https://bootstrap.pypa.io/pip/3.7/get-pip.py - python get-pip.py - rm get-pip.py - pip install --upgrade pip + if echo $RESULT | grep -izq error ; then + echo "Failed to install conda dependencies: $RESULT" + exit 1 + fi + + conda clean -a && pip cache purge +EOR + +############################################################################## +# Install python dependencies +############################################################################## +ENV PIP_CACHE_DIR=/tmp/pip_cache +RUN < build_deepspeed.sh #!/usr/bin/env bash +source ~/.bashrc mkdir -p /deepspeed + +conda activate alltalk cd ${STAGE_DIR}/DeepSpeed DS_BUILD_OPS=1 python setup.py build_ext -j8 bdist_wheel mv ${STAGE_DIR}/DeepSpeed/dist/*.whl /deepspeed/ @@ -100,4 +89,7 @@ EOR RUN chmod +x /build_deepspeed.sh +############################################################################## +# Build DeepSpeed: +############################################################################## ENTRYPOINT ["/build_deepspeed.sh"] \ No newline at end of file diff --git a/deepspeed/build-deepspeed.sh b/docker/deepspeed/build-deepspeed.sh similarity index 54% rename from deepspeed/build-deepspeed.sh rename to docker/deepspeed/build-deepspeed.sh index 35977bb..8def559 100755 --- a/deepspeed/build-deepspeed.sh +++ b/docker/deepspeed/build-deepspeed.sh @@ -1,28 +1,17 @@ #!/usr/bin/env bash -CUDA_VERSION=12.1.1 -PYTHON_VERSION=3.11 -PYTORCH_VERSION=2.2.1 -DEEPSPEED_VERSION=0.16.1 - SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd $SCRIPT_DIR +. ${SCRIPT_DIR=}/../variables.sh + # Parse arguments while [ "$#" -gt 0 ]; do case "$1" in - --cuda-version) - CUDA_VERSION="$2" - shift - ;; --python-version) PYTHON_VERSION="$2" shift ;; - --pytorch-version) - PYTORCH_VERSION="$2" - shift - ;; --deepspeed-version) DEEPSPEED_VERSION="$2" shift @@ -36,24 +25,30 @@ while [ "$#" -gt 0 ]; do shift done -PYTHON_VERSION_NO_DOT=${PYTHON_VERSION//./} +PYTHON_SHORT_VERSION=${PYTHON_VERSION%.*} +PYTHON_VERSION_NO_DOT=${PYTHON_SHORT_VERSION//./} if [[ -n $(find build -name "deepspeed-${DEEPSPEED_VERSION}*-cp${PYTHON_VERSION_NO_DOT}-cp${PYTHON_VERSION_NO_DOT}-*.whl") ]] then echo "DeepSpeed was already built - skipping..." exit 0 fi -echo "Building DeepSpeed $DEEPSPEED_VERSION for CUDA $CUDA_VERSION using python ${PYTHON_VERSION} with PyTorch ${PYTORCH_VERSION}" +CONDA_ENV=${SCRIPT_DIR=}/../conda/build/environment-cu-${CUDA_VERSION}-cp-${PYTHON_VERSION}.yml +if [ ! -f ${CONDA_ENV} ]; then + echo "No conda environment found. Please run 'build-conda-env.sh' first." + exit 1 +fi -rm -rf build # make sure to properly clean up - we only want 1 wheel at the time +echo "Building DeepSpeed $DEEPSPEED_VERSION using python ${PYTHON_VERSION}" + +rm -rf build # make sure to properly clean up mkdir -p build +cp ${CONDA_ENV} ${SCRIPT_DIR=}/build + docker buildx \ build \ - --build-arg CUDA_VERSION=$CUDA_VERSION \ - --build-arg PYTHON_VERSION=$PYTHON_VERSION \ - --build-arg PYTORCH_VERSION=$PYTORCH_VERSION \ --build-arg DEEPSPEED_VERSION=$DEEPSPEED_VERSION \ - -t deepspeed:cu-$CUDA_VERSION-ds-$DEEPSPEED_VERSION \ + -t deepspeed:$DEEPSPEED_VERSION \ . docker run \ @@ -62,4 +57,4 @@ docker run \ --gpus=all \ --name deepspeed \ -v $SCRIPT_DIR/build:/deepspeed \ - deepspeed:cu-$CUDA_VERSION-ds-$DEEPSPEED_VERSION \ No newline at end of file + deepspeed:$DEEPSPEED_VERSION \ No newline at end of file diff --git a/docker/variables.sh b/docker/variables.sh new file mode 100644 index 0000000..9cc0d0f --- /dev/null +++ b/docker/variables.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +export CUDA_VERSION=12.4.1 +export PYTHON_VERSION=3.11.11 +export DEEPSPEED_VERSION=0.16.2 +export ALLTALK_DIR=/opt/alltalk \ No newline at end of file