Refactoring docker images, updating CUDA to 12.6 and pytorch 2.6
This commit is contained in:
ekoc committed
1 parent
73d9d9450b
commit
284c100860
11 files changed
+123
-143
No files matched your search
@@ -1,6 +1,6 @@
|
||||
FROM continuumio/miniconda3:24.7.1-0 AS build
|
||||
|
||||
ARG CUDA_VERSION=12.4.1
|
||||
ARG CUDA_VERSION=12.6.0
|
||||
ENV CUDA_VERSION=$CUDA_VERSION
|
||||
|
||||
ARG PYTHON_VERSION=3.11.11
|
||||
@@ -14,6 +14,8 @@ ENV CUDA_DOCKER_ARCH=all
|
||||
ENV NVIDIA_VISIBLE_DEVICES=all
|
||||
ENV CONDA_AUTO_UPDATE_CONDA="false"
|
||||
|
||||
ENV ALLTALK_DIR=/opt/alltalk
|
||||
|
||||
##############################################################################
|
||||
# Installation/Basic Utilities
|
||||
##############################################################################
|
||||
@@ -35,24 +37,24 @@ EOR
|
||||
##############################################################################
|
||||
RUN <<EOR
|
||||
CUDA_SHORT_VERSION=${CUDA_VERSION%.*}
|
||||
|
||||
conda create -y -n "alltalk" -c conda-forge python=${PYTHON_VERSION}
|
||||
conda activate alltalk
|
||||
RESULT=$( { conda install -y \
|
||||
conda install -y \
|
||||
gcc_linux-64 \
|
||||
gxx_linux-64 \
|
||||
pytorch \
|
||||
pytorch-cuda=${CUDA_SHORT_VERSION} \
|
||||
torchvision \
|
||||
torchaudio \
|
||||
libaio \
|
||||
nvidia/label/cuda-${CUDA_SHORT_VERSION}.0::cuda-toolkit \
|
||||
faiss-gpu=1.9.0 \
|
||||
cuda=${CUDA_VERSION} cuda-tools=${CUDA_VERSION} cuda-toolkit=${CUDA_VERSION} cuda-version=${CUDA_SHORT_VERSION} \
|
||||
cuda-command-line-tools=${CUDA_VERSION} cuda-compiler=${CUDA_VERSION} cuda-runtime=${CUDA_VERSION} \
|
||||
cuda-libraries=${CUDA_VERSION} cuda-libraries-dev=${CUDA_VERSION} cuda-libraries-static=${CUDA_VERSION} \
|
||||
cuda-cudart=${CUDA_SHORT_VERSION} cuda-cudart_linux-64=${CUDA_SHORT_VERSION} \
|
||||
cudnn=9.3 \
|
||||
py-cpuinfo=9 \
|
||||
conda-forge::ffmpeg=7.1.0 \
|
||||
conda-forge::portaudio=19.7.0 \
|
||||
-c pytorch \
|
||||
-c anaconda \
|
||||
-c nvidia ; } 2>&1 )
|
||||
-c nvidia \
|
||||
-c anaconda
|
||||
|
||||
if [ $? -ne 0 ] ; then
|
||||
echo "Failed to install conda dependencies: $RESULT"
|
||||
@@ -62,16 +64,17 @@ RUN <<EOR
|
||||
conda clean -a && pip cache purge
|
||||
EOR
|
||||
|
||||
##############################################################################
|
||||
# Export conda environment:
|
||||
##############################################################################
|
||||
RUN <<EOR
|
||||
conda activate alltalk
|
||||
conda env export > /environment-cu-${CUDA_VERSION}-cp-${PYTHON_VERSION}.yml
|
||||
EOR
|
||||
CUDA_SHORT_VERSION=${CUDA_VERSION%.*}
|
||||
CUDA_SHORT_VERSION_NO_DOT=${CUDA_SHORT_VERSION//./}
|
||||
|
||||
##############################################################################
|
||||
# Make environment file available to docker build output:
|
||||
##############################################################################
|
||||
FROM scratch
|
||||
COPY --from=build /environment-*.yml /
|
||||
conda activate alltalk
|
||||
mkdir -p ${ALLTALK_DIR}/pip_cache
|
||||
pip install --no-cache-dir --cache-dir=${ALLTALK_DIR}/pip_cache \
|
||||
torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu${CUDA_SHORT_VERSION_NO_DOT}
|
||||
|
||||
if [ $? -ne 0 ] ; then
|
||||
echo "Failed to install pip dependencies: $RESULT"
|
||||
exit 1
|
||||
fi
|
||||
EOR
|
||||
@@ -5,10 +5,7 @@ cd $SCRIPT_DIR
|
||||
|
||||
. ${SCRIPT_DIR=}/../variables.sh
|
||||
|
||||
if [ -f $SCRIPT_DIR/build/environment-cu-${CUDA_VERSION}-cp-${PYTHON_VERSION}.yml ]; then
|
||||
echo "Environment file already exists - skipping..."
|
||||
exit 0
|
||||
fi
|
||||
DOCKER_TAG=latest
|
||||
|
||||
# Parse arguments
|
||||
while [ "$#" -gt 0 ]; do
|
||||
@@ -21,8 +18,17 @@ while [ "$#" -gt 0 ]; do
|
||||
PYTHON_VERSION="$2"
|
||||
shift
|
||||
;;
|
||||
--tag)
|
||||
DOCKER_TAG="$2"
|
||||
shift
|
||||
;;
|
||||
--github-repository)
|
||||
if [ -n "${GITHUB_REPOSITORY}" ] && ! [[ $GITHUB_REPOSITORY =~ ^--.* ]]; then
|
||||
GITHUB_REPOSITORY="$2"
|
||||
shift
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
# Allow to pass arbitrary arguments to docker as well to be flexible:
|
||||
echo "Unknown argument '$1'"
|
||||
exit 1
|
||||
;;
|
||||
@@ -32,12 +38,10 @@ done
|
||||
|
||||
echo "Building conda environment using python ${PYTHON_VERSION} with CUDA ${CUDA_VERSION}"
|
||||
|
||||
rm -rf build # make sure to properly clean up
|
||||
mkdir -p build
|
||||
|
||||
docker buildx \
|
||||
build \
|
||||
--progress=plain \
|
||||
--build-arg CUDA_VERSION=$CUDA_VERSION \
|
||||
--build-arg PYTHON_VERSION=$PYTHON_VERSION \
|
||||
--output=$SCRIPT_DIR/build \
|
||||
-t ${GITHUB_REPOSITORY}alltalk_environment:${DOCKER_TAG} \
|
||||
.
|
||||
+13
-43
@@ -1,53 +1,15 @@
|
||||
FROM continuumio/miniconda3:24.7.1-0
|
||||
ARG GITHUB_REPOSITORY
|
||||
FROM ${GITHUB_REPOSITORY}alltalk_environment:latest
|
||||
|
||||
ARG DEEPSPEED_VERSION=0.16.2
|
||||
ARG DEEPSPEED_VERSION=0.16.3
|
||||
ENV DEEPSPEED_VERSION=$DEEPSPEED_VERSION
|
||||
|
||||
SHELL ["/bin/bash", "-l", "-c"]
|
||||
ENV SHELL=/bin/bash
|
||||
ENV HOST=0.0.0.0
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ENV CUDA_DOCKER_ARCH=all
|
||||
ENV NVIDIA_VISIBLE_DEVICES=all
|
||||
ENV CONDA_AUTO_UPDATE_CONDA="false"
|
||||
|
||||
##############################################################################
|
||||
# Directories:
|
||||
##############################################################################
|
||||
ENV STAGE_DIR=/tmp
|
||||
RUN mkdir -p ${STAGE_DIR}
|
||||
|
||||
##############################################################################
|
||||
# Installation/Basic Utilities
|
||||
##############################################################################
|
||||
RUN <<EOR
|
||||
apt-get update
|
||||
apt-get upgrade -y
|
||||
apt-get install --no-install-recommends -y \
|
||||
espeak-ng \
|
||||
curl \
|
||||
wget \
|
||||
jq \
|
||||
vim
|
||||
|
||||
apt-get clean && rm -rf /var/lib/apt/lists/*
|
||||
EOR
|
||||
|
||||
##############################################################################
|
||||
# Create a conda environment and install dependencies:
|
||||
##############################################################################
|
||||
COPY build/environment-*.yml environment.yml
|
||||
RUN <<EOR
|
||||
RESULT=$( { conda env create -f environment.yml ; } 2>&1 )
|
||||
|
||||
if [ $? -ne 0 ] ; then
|
||||
echo "Failed to install conda dependencies: $RESULT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
conda clean -a && pip cache purge
|
||||
EOR
|
||||
|
||||
##############################################################################
|
||||
# Install python dependencies
|
||||
##############################################################################
|
||||
@@ -56,8 +18,7 @@ RUN <<EOR
|
||||
conda activate alltalk
|
||||
|
||||
pip install \
|
||||
deepspeed-kernels \
|
||||
scikit-learn
|
||||
deepspeed-kernels
|
||||
EOR
|
||||
|
||||
|
||||
@@ -69,6 +30,15 @@ RUN <<EOR
|
||||
cd ${STAGE_DIR}/DeepSpeed
|
||||
git checkout .
|
||||
git checkout "tags/v${DEEPSPEED_VERSION}" -b "v${DEEPSPEED_VERSION}"
|
||||
|
||||
# Patching some files due to bug https://github.com/microsoft/DeepSpeed/issues/6709
|
||||
START_LINE=`awk '/#ifndef BF16_AVAILABLE/{ print NR; exit }' csrc/transformer/inference/csrc/gelu.cu`
|
||||
TO_LINE=$((START_LINE + 2))
|
||||
sed -i "${START_LINE},${TO_LINE}d" ./csrc/transformer/inference/csrc/gelu.cu
|
||||
|
||||
START_LINE=`awk '/#ifndef BF16_AVAILABLE/{ print NR; exit }' csrc/transformer/inference/csrc/transform.cu`
|
||||
TO_LINE=$((START_LINE + 2))
|
||||
sed -i "${START_LINE},${TO_LINE}d" ./csrc/transformer/inference/csrc/transform.cu
|
||||
EOR
|
||||
|
||||
##############################################################################
|
||||
|
||||
@@ -5,6 +5,8 @@ cd $SCRIPT_DIR
|
||||
|
||||
. ${SCRIPT_DIR=}/../variables.sh
|
||||
|
||||
DOCKER_TAG=latest
|
||||
|
||||
# Parse arguments
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
@@ -16,8 +18,17 @@ while [ "$#" -gt 0 ]; do
|
||||
DEEPSPEED_VERSION="$2"
|
||||
shift
|
||||
;;
|
||||
--tag)
|
||||
DOCKER_TAG="$2"
|
||||
shift
|
||||
;;
|
||||
--github-repository)
|
||||
if [ -n "${GITHUB_REPOSITORY}" ] && ! [[ $GITHUB_REPOSITORY =~ ^--.* ]]; then
|
||||
GITHUB_REPOSITORY="$2"
|
||||
shift
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
# Allow to pass arbitrary arguments to docker as well to be flexible:
|
||||
echo "Unknown argument '$1'"
|
||||
exit 1
|
||||
;;
|
||||
@@ -33,22 +44,16 @@ then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
CONDA_ENV=${SCRIPT_DIR=}/../conda/build/environment-cu-${CUDA_VERSION}-cp-${PYTHON_VERSION}.yml
|
||||
if [ ! -f ${CONDA_ENV} ]; then
|
||||
echo "No conda environment found. Please run 'build-conda-env.sh' first."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Building DeepSpeed $DEEPSPEED_VERSION using python ${PYTHON_VERSION}"
|
||||
|
||||
rm -rf build # make sure to properly clean up
|
||||
mkdir -p build
|
||||
cp ${CONDA_ENV} ${SCRIPT_DIR=}/build
|
||||
|
||||
docker buildx \
|
||||
build \
|
||||
--build-arg GITHUB_REPOSITORY=$GITHUB_REPOSITORY \
|
||||
--build-arg DEEPSPEED_VERSION=$DEEPSPEED_VERSION \
|
||||
-t deepspeed:$DEEPSPEED_VERSION \
|
||||
-t ${GITHUB_REPOSITORY}alltalk_deepspeed:${DOCKER_TAG} \
|
||||
.
|
||||
|
||||
docker run \
|
||||
@@ -57,4 +62,4 @@ docker run \
|
||||
--gpus=all \
|
||||
--name deepspeed \
|
||||
-v $SCRIPT_DIR/build:/deepspeed \
|
||||
deepspeed:$DEEPSPEED_VERSION
|
||||
${GITHUB_REPOSITORY}alltalk_deepspeed:${DOCKER_TAG}
|
||||
+3
-2
@@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
export CUDA_VERSION=12.4.1
|
||||
export CUDA_VERSION=12.6.0
|
||||
export PYTHON_VERSION=3.11.11
|
||||
export DEEPSPEED_VERSION=0.16.2
|
||||
export ALLTALK_DIR=/opt/alltalk
|
||||
export ALLTALK_DIR=/opt/alltalk
|
||||
export GITHUB_REPOSITORY=
|
||||
Reference in new issue
Block a user