From fbd09a24b6b9d653add4aa5b89d4983b9f91de25 Mon Sep 17 00:00:00 2001 From: ekoc Date: Sat, 22 Feb 2025 16:40:25 +0100 Subject: [PATCH 1/3] Adding Github actions for building the docker images --- .github/actions/docker-setup/action.yml | 67 ++++++++++++++ .github/workflows/publish-docker-v2.yml | 117 ++++++++++++++++++++++++ 2 files changed, 184 insertions(+) create mode 100644 .github/actions/docker-setup/action.yml create mode 100644 .github/workflows/publish-docker-v2.yml diff --git a/.github/actions/docker-setup/action.yml b/.github/actions/docker-setup/action.yml new file mode 100644 index 0000000..820f794 --- /dev/null +++ b/.github/actions/docker-setup/action.yml @@ -0,0 +1,67 @@ +name: 'Setting up docker' + +inputs: + DOCKERHUB_USERNAME: + description: "Docker hub username" + required: true + DOCKERHUB_REPO_NAME: + description: "Docker hub repo name" + required: true + DOCKERHUB_TOKEN: + description: "Docker hub token" + required: true + +runs: + using: "composite" + steps: + - name: Read variables + shell: bash + run: | + source ./docker/variables.sh + for key in "${!ALLTALK_VARS[@]}" + do + echo "${key}=${ALLTALK_VARS[${key}]}" + echo "${key}=${ALLTALK_VARS[${key}]}" >> $GITHUB_ENV + done + + - name: Extract version from Git Tag + shell: bash {0} + run: | + # Use the git tag if existing, otherwise 'latest': + DOCKER_TAG=$( echo "${{ github.ref }}" | grep -q tags | cut -d'/' -f 3 ) + if [ -z "${DOCKER_TAG}" ]; then + DOCKER_TAG="latest" + fi + echo "Using DOCKER_TAG=${DOCKER_TAG}" + echo "DOCKER_TAG=${DOCKER_TAG}" >> $GITHUB_ENV + + - name: Extract Docker Hub User and Repo Name + shell: bash + env: + DOCKERHUB_USERNAME: ${{ inputs.DOCKERHUB_USERNAME }} + DOCKERHUB_REPO_NAME: ${{ inputs.DOCKERHUB_REPO_NAME }} + run: | + if [ -n "${DOCKERHUB_USERNAME}" ]; then + echo "Using Github repository variable 'DOCKERHUB_USERNAME' as Docker hub username with value: '${DOCKERHUB_USERNAME}'" + echo "DOCKERHUB_USERNAME=${DOCKERHUB_USERNAME}" >> $GITHUB_ENV + else + echo "Using Github repository name as Docker hub username" + echo "DOCKERHUB_USERNAME=$(echo ${{ github.repository }} | cut -d'/' -f 1)" >> $GITHUB_ENV + fi + + if [ -n "${DOCKERHUB_REPO_NAME}" ]; then + echo "Using Github repository variable 'DOCKERHUB_REPO_NAME' as Docker hub repo name with value: '${DOCKERHUB_REPO_NAME}'" + echo "DOCKERHUB_REPO_NAME=${DOCKERHUB_REPO_NAME}" >> $GITHUB_ENV + else + echo "Using Github repository name as Docker hub repo name" + echo "DOCKERHUB_REPO_NAME=$(echo ${{ github.repository }} | cut -d'/' -f 2)" >> $GITHUB_ENV + fi + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ inputs.DOCKERHUB_USERNAME }} + password: ${{ inputs.DOCKERHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/publish-docker-v2.yml b/.github/workflows/publish-docker-v2.yml new file mode 100644 index 0000000..f047b23 --- /dev/null +++ b/.github/workflows/publish-docker-v2.yml @@ -0,0 +1,117 @@ +name: Publish Docker v2 + +on: + workflow_dispatch: + release: + types: [ published ] + push: + branches: + - 'alltalkbeta' + tags: + - 'v*' + +jobs: + build_base_environment_docker_image: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup docker + uses: ./.github/actions/docker-setup + with: + DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }} + DOCKERHUB_REPO_NAME: ${{ vars.DOCKERHUB_REPO_NAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.DOCKERHUB_USERNAME }}/${{ env.DOCKERHUB_REPO_NAME }}_environment + tags: | + # Use sematic versioning on push tag event: + type=semver,pattern={{version}} + # set latest tag for main and alltalkbeta branch: + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') || github.ref == format('refs/heads/{0}', 'alltalkbeta') }} + + - uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + docker-base: + - 'docker/variables.sh' + - 'docker/base/Dockerfile' + docker-workflow: + - '.github/workflows/publish-docker*.yml' + + - name: Build and push base Docker image + if: | + steps.changes.outputs.docker-base == 'true' || + steps.changes.outputs.docker-workflow == 'true' || + github.event_name == 'release' || + (github.event_name == 'push' && contains(github.ref, 'refs/tags/')) + uses: docker/build-push-action@v6 + with: + context: ./docker/base + file: ./docker/base/Dockerfile + platforms: "linux/amd64" + push: true + build-args: | + CUDA_VERSION=${{ env.CUDA_VERSION }} + PYTHON_VERSION=${{ env.PYTHON_VERSION }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=${{ env.DOCKERHUB_USERNAME }}/${{ env.DOCKERHUB_REPO_NAME }}_environment:buildcache + cache-to: type=registry,ref=${{ env.DOCKERHUB_USERNAME }}/${{ env.DOCKERHUB_REPO_NAME }}_environment:buildcache,mode=max + + build_main_docker_image: + needs: build_base_environment_docker_image + runs-on: ubuntu-latest + strategy: + matrix: + tts: [ xtts, vits, piper ] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup docker + uses: ./.github/actions/docker-setup + with: + DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }} + DOCKERHUB_REPO_NAME: ${{ vars.DOCKERHUB_REPO_NAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.DOCKERHUB_USERNAME }}/${{ env.DOCKERHUB_REPO_NAME }} + flavor: | + suffix=-${{ matrix.tts }},onlatest=true + tags: | + # Use sematic versioning on push tag event: + type=semver,pattern={{version}} + # set latest tag for main and alltalkbeta branch: + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') || github.ref == format('refs/heads/{0}', 'alltalkbeta') }} + + - name: Build and push base Docker image + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + platforms: "linux/amd64" + push: true + build-args: | + TTS_MODEL=${{ matrix.tts }} + ALLTALK_DIR=${{ env.ALLTALK_DIR }} + DEEPSPEED_VERSION=${{ env.DEEPSPEED_VERSION }} + DOCKER_TAG=${{ env.DOCKER_TAG }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=${{ env.DOCKERHUB_USERNAME }}/${{ env.DOCKERHUB_REPO_NAME }}:buildcache + cache-to: type=registry,ref=${{ env.DOCKERHUB_USERNAME }}/${{ env.DOCKERHUB_REPO_NAME }}:buildcache,mode=max \ No newline at end of file From 3aacf3f27f20f006e7807840ce5d6fae4ef9ffc0 Mon Sep 17 00:00:00 2001 From: ekoc Date: Wed, 26 Feb 2025 20:21:46 +0100 Subject: [PATCH 2/3] Adding Github actions for building the docker images --- .github/actions/docker-setup/action.yml | 6 ++++++ .github/workflows/publish-docker-v2.yml | 10 ++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/actions/docker-setup/action.yml b/.github/actions/docker-setup/action.yml index 820f794..774e016 100644 --- a/.github/actions/docker-setup/action.yml +++ b/.github/actions/docker-setup/action.yml @@ -14,6 +14,12 @@ inputs: runs: using: "composite" steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: true + swap-storage: false + large-packages: false - name: Read variables shell: bash run: | diff --git a/.github/workflows/publish-docker-v2.yml b/.github/workflows/publish-docker-v2.yml index f047b23..1f30fba 100644 --- a/.github/workflows/publish-docker-v2.yml +++ b/.github/workflows/publish-docker-v2.yml @@ -10,6 +10,11 @@ on: tags: - 'v*' +# Limit the concurrency of entire workflow runs for a specific branch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build_base_environment_docker_image: runs-on: ubuntu-latest @@ -111,7 +116,8 @@ jobs: ALLTALK_DIR=${{ env.ALLTALK_DIR }} DEEPSPEED_VERSION=${{ env.DEEPSPEED_VERSION }} DOCKER_TAG=${{ env.DOCKER_TAG }} + GITHUB_REPOSITORY=${{ env.DOCKERHUB_USERNAME }}/ tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=${{ env.DOCKERHUB_USERNAME }}/${{ env.DOCKERHUB_REPO_NAME }}:buildcache - cache-to: type=registry,ref=${{ env.DOCKERHUB_USERNAME }}/${{ env.DOCKERHUB_REPO_NAME }}:buildcache,mode=max \ No newline at end of file + cache-from: type=registry,ref=${{ env.DOCKERHUB_USERNAME }}/${{ env.DOCKERHUB_REPO_NAME }}:buildcache-${{ matrix.tts }} + cache-to: type=registry,ref=${{ env.DOCKERHUB_USERNAME }}/${{ env.DOCKERHUB_REPO_NAME }}:buildcache-${{ matrix.tts }},mode=max \ No newline at end of file From 871a100a8a023ea35dd782ef3024130aa01a433c Mon Sep 17 00:00:00 2001 From: ekoc Date: Wed, 26 Feb 2025 21:28:48 +0100 Subject: [PATCH 3/3] Adding Github actions for building the docker images --- .github/workflows/publish-docker-v2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-docker-v2.yml b/.github/workflows/publish-docker-v2.yml index 1f30fba..6e8a380 100644 --- a/.github/workflows/publish-docker-v2.yml +++ b/.github/workflows/publish-docker-v2.yml @@ -116,7 +116,7 @@ jobs: ALLTALK_DIR=${{ env.ALLTALK_DIR }} DEEPSPEED_VERSION=${{ env.DEEPSPEED_VERSION }} DOCKER_TAG=${{ env.DOCKER_TAG }} - GITHUB_REPOSITORY=${{ env.DOCKERHUB_USERNAME }}/ + DOCKER_REPOSITORY=${{ env.DOCKERHUB_USERNAME }}/ tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=registry,ref=${{ env.DOCKERHUB_USERNAME }}/${{ env.DOCKERHUB_REPO_NAME }}:buildcache-${{ matrix.tts }}