Update README.md

remind people to update the requirements when updating.
2023-12-17 06:41:26 +00:00 · 2023-12-17 06:40:49 +00:00 · 2023-12-17 05:45:51 +00:00 · 2023-12-16 22:04:42 +00:00 · 2023-12-16 22:02:46 +00:00 · 2023-12-16 21:37:01 +00:00
23 changed files with 5584 additions and 316 deletions
@@ -1,160 +1,5 @@
-# Byte-compiled / optimized / DLL files
+voices/*.*
-__pycache__/
+models/*.*
-*.py[cod]
+outputs/*.*
-*$py.class
+config.json
-
+models.json
 # C extensions
 *.so
 # Distribution / packaging
 .Python
 build/
 develop-eggs/
 dist/
 downloads/
 eggs/
 .eggs/
 lib/
 lib64/
 parts/
 sdist/
 var/
 wheels/
 share/python-wheels/
 *.egg-info/
 .installed.cfg
 *.egg
 MANIFEST
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
 *.spec
 # Installer logs
 pip-log.txt
 pip-delete-this-directory.txt
 # Unit test / coverage reports
 htmlcov/
 .tox/
 .nox/
 .coverage
 .coverage.*
 .cache
 nosetests.xml
 coverage.xml
 *.cover
 *.py,cover
 .hypothesis/
 .pytest_cache/
 cover/
 # Translations
 *.mo
 *.pot
 # Django stuff:
 *.log
 local_settings.py
 db.sqlite3
 db.sqlite3-journal
 # Flask stuff:
 instance/
 .webassets-cache
 # Scrapy stuff:
 .scrapy
 # Sphinx documentation
 docs/_build/
 # PyBuilder
 .pybuilder/
 target/
 # Jupyter Notebook
 .ipynb_checkpoints
 # IPython
 profile_default/
 ipython_config.py
 # pyenv
 #   For a library or package, you might want to ignore these files since the code is
 #   intended to run in multiple environments; otherwise, check them in:
 # .python-version
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
 #   However, in case of collaboration, if having platform-specific dependencies or dependencies
 #   having no cross-platform support, pipenv may install dependencies that don't work, or not
 #   install all needed dependencies.
 #Pipfile.lock
 # poetry
 #   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
 #   This is especially recommended for binary packages to ensure reproducibility, and is more
 #   commonly ignored for libraries.
 #   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
 #poetry.lock
 # pdm
 #   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
 #pdm.lock
 #   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
 #   in version control.
 #   https://pdm.fming.dev/#use-with-ide
 .pdm.toml
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
 __pypackages__/
 # Celery stuff
 celerybeat-schedule
 celerybeat.pid
 # SageMath parsed files
 *.sage.py
 # Environments
 .env
 .venv
 env/
 venv/
 ENV/
 env.bak/
 venv.bak/
 # Spyder project settings
 .spyderproject
 .spyproject
 # Rope project settings
 .ropeproject
 # mkdocs documentation
 /site
 # mypy
 .mypy_cache/
 .dmypy.json
 dmypy.json
 # Pyre type checker
 .pyre/
 # pytype static type analyzer
 .pytype/
 # Cython debug symbols
 cython_debug/
 # PyCharm
 #  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
@@ -0,0 +1,285 @@
 // Copyright (c) Microsoft Corporation.
 // SPDX-License-Identifier: Apache-2.0
 // DeepSpeed Team
 #include <ATen/cuda/CUDAContext.h>
 #include <torch/extension.h>
 #include <cassert>
 #include <vector>
 #include "quantization.h"
 template <typename T>
 at::Tensor ds_quantize(at::Tensor& vals, int groups, int bits)
 {
    auto t_size = vals.sizes();
    int size = 1;
    for (auto dim : t_size) size *= dim;
    if ((((size / groups) - 1) / 4096 + 1) <= 256) {
        launch_fake_quantize_kernel(
            (T*)vals.data_ptr(), size, groups, bits, at::cuda::getCurrentCUDAStream());
    }
    return vals;
 }
 template <typename T>
 at::Tensor ds_sr_quantize(at::Tensor& vals, int groups, int bits)
 {
    auto t_size = vals.sizes();
    int size = 1;
    for (auto dim : t_size) size *= dim;
    if (((size / groups) / 4 / 1024) <= 256) {
        launch_sr_fake_quantize_kernel(
            (T*)vals.data_ptr(), size, groups, bits, at::cuda::getCurrentCUDAStream());
    }
    return vals;
 }
 template <typename T>
 at::Tensor ds_quantize_asym(at::Tensor& vals, int groups, int bits)
 {
    auto t_size = vals.sizes();
    int size = 1;
    for (auto dim : t_size) size *= dim;
    if ((((size / groups) - 1) / 4096 + 1) <= 256) {
        launch_fake_quantize_kernel_asym(
            (T*)vals.data_ptr(), size, groups, bits, at::cuda::getCurrentCUDAStream());
    }
    return vals;
 }
 template <typename T>
 at::Tensor ds_sr_quantize_asym(at::Tensor& vals, int groups, int bits)
 {
    auto t_size = vals.sizes();
    int size = 1;
    for (auto dim : t_size) size *= dim;
    if (((size / groups) / 4 / 1024) <= 256) {
        launch_sr_fake_quantize_kernel_asym(
            (T*)vals.data_ptr(), size, groups, bits, at::cuda::getCurrentCUDAStream());
    }
    return vals;
 }
 std::vector<at::Tensor> quantize_kernel(at::Tensor& input_vals,
                                        int groups,
                                        int numBits,
                                        quantize::Type quantType)
 {
    auto dtype = at::kFloat;
    auto params_options = at::TensorOptions()
                              .dtype(dtype)
                              .layout(at::kStrided)
                              .device(at::kCUDA)
                              .requires_grad(false);
    const int param_elems = (quantize::requires_offset(quantType)) ? 2 : 1;
    auto params = torch::empty({groups, param_elems}, params_options);
    auto output_options = at::TensorOptions()
                              .dtype(at::kChar)
                              .layout(at::kStrided)
                              .device(at::kCUDA)
                              .requires_grad(false);
    auto output_sizes = input_vals.sizes().vec();
    output_sizes[output_sizes.size() - 1] /= numBits == 8 ? 1 : 2;
    auto output = torch::empty(output_sizes, output_options);
    const int elems_per_group = at::numel(input_vals) / groups;
    launch_quant((int8_t*)output.data_ptr(),
                 (float*)params.data_ptr(),
                 (__half*)input_vals.data_ptr(),
                 groups,
                 elems_per_group,
                 numBits,
                 quantType,
                 at::cuda::getCurrentCUDAStream());
    return {output, params};
 }
 template <typename T>
 at::Tensor dequantize(at::Tensor& quantized_data,
                      at::Tensor& params,
                      int groups,
                      int num_bits,
                      quantize::Type quant_type)
 {
    auto dtype = (std::is_same<T, float>::value) ? torch::kFloat32 : torch::kFloat16;
    auto output_options = at::TensorOptions()
                              .dtype(dtype)
                              .layout(at::kStrided)
                              .device(at::kCUDA)
                              .requires_grad(false);
    auto output_sizes = quantized_data.sizes().vec();
    output_sizes[output_sizes.size() - 1] *= num_bits == 8 ? 1 : 2;
    auto output = torch::empty(output_sizes, output_options);
    const int total_elems = at::numel(output);
    const int elems_per_group = total_elems / groups;
    launch_dequantize_kernel((T*)output.data_ptr(),
                             (const int8_t*)quantized_data.data_ptr(),
                             (const float*)params.data_ptr(),
                             quant_type,
                             num_bits,
                             elems_per_group,
                             total_elems,
                             at::cuda::getCurrentCUDAStream());
    return output;
 }
 at::Tensor dequantize_int4_to_half_experimental(at::Tensor& data_in,
                                                at::Tensor& scale_buffer,
                                                at::Tensor& min_val_buffer,
                                                int num_group,
                                                int group_size)
 {
    auto output_options = at::TensorOptions().dtype(at::kHalf).device(at::kCUDA);
    auto output = torch::empty({num_group, group_size}, output_options);
    launch_dequantize_int4_to_half_experimental((uint8_t*)data_in.data_ptr(),
                                                (half*)output.data_ptr(),
                                                (half*)scale_buffer.data_ptr(),
                                                (half*)min_val_buffer.data_ptr(),
                                                num_group,
                                                group_size,
                                                at::cuda::getCurrentCUDAStream());
    return output;
 }
 std::vector<at::Tensor> ds_swizzle_quant(at::Tensor& input_vals,
                                         int groups,
                                         int num_bits,
                                         quantize::Type quant_type,
                                         int pipeline_size,
                                         int nodes,
                                         int devices_per_node)
 {
    auto scales_options = at::TensorOptions()
                              .dtype(at::kFloat)
                              .layout(at::kStrided)
                              .device(at::kCUDA)
                              .requires_grad(false);
    const int scales_elems = (quantize::requires_offset(quant_type)) ? 2 : 1;
    auto scales = torch::empty({groups, scales_elems}, scales_options);
    auto output_options = at::TensorOptions()
                              .dtype(at::kChar)
                              .layout(at::kStrided)
                              .device(at::kCUDA)
                              .requires_grad(false);
    const int quantization_scalar = 8 / num_bits;
    const int compressed_vals = at::numel(input_vals) / quantization_scalar;
    auto output = torch::empty({compressed_vals}, output_options);
    const int elems_per_group = at::numel(input_vals) / groups;
    launch_swizzled_quant((int8_t*)output.data_ptr(),
                          (float*)scales.data_ptr(),
                          (__half*)input_vals.data_ptr(),
                          num_bits,
                          quant_type,
                          groups,
                          elems_per_group,
                          pipeline_size,
                          nodes,
                          devices_per_node,
                          at::cuda::getCurrentCUDAStream());
    return {output, scales};
 }
 std::vector<at::Tensor> quantized_reduction(at::Tensor& input_vals,
                                            at::Tensor& input_scales,
                                            int in_groups,
                                            int out_groups,
                                            int num_bits,
                                            quantize::Type quant_type,
                                            int devices_per_node)
 {
    auto scales_options = at::TensorOptions()
                              .dtype(at::kFloat)
                              .layout(at::kStrided)
                              .device(at::kCUDA)
                              .requires_grad(false);
    const int scales_elems = (quantize::requires_offset(quant_type)) ? 2 : 1;
    auto scales = torch::empty({out_groups, scales_elems}, scales_options);
    auto output_options = at::TensorOptions()
                              .dtype(at::kChar)
                              .layout(at::kStrided)
                              .device(at::kCUDA)
                              .requires_grad(false);
     std::vector<int64_t> sz_vector(input_vals.sizes().begin(), input_vals.sizes().end());
     sz_vector[sz_vector.size() - 1] = sz_vector.back() / devices_per_node;  // num of GPU per nodes
     at::IntArrayRef sz(sz_vector);
     auto output = torch::empty(sz, output_options);
     const int elems_per_in_tensor = at::numel(input_vals) / devices_per_node;
     const int elems_per_in_group = elems_per_in_tensor / (in_groups / devices_per_node);
     const int elems_per_out_group = elems_per_in_tensor / out_groups;
 // old code:
 //    std::vector<long int> sz(input_vals.sizes().begin(), input_vals.sizes().end());
 //    sz[sz.size() - 1] = sz.back() / devices_per_node;  // num of GPU per nodes
 //    const int elems_per_in_tensor = at::numel(input_vals) / devices_per_node;
 //    auto output = torch::empty(sz, output_options);
 //    const int elems_per_in_group = elems_per_in_tensor / (in_groups / devices_per_node);
 //    const int elems_per_out_group = elems_per_in_tensor / out_groups;
    launch_dequant_reduce((int8_t*)output.data_ptr(),
                          (float*)scales.data_ptr(),
                          (const int8_t*)input_vals.data_ptr(),
                          (const float*)input_scales.data_ptr(),
                          devices_per_node,
                          num_bits,
                          quant_type,
                          out_groups,
                          elems_per_out_group,
                          elems_per_in_tensor,
                          in_groups / devices_per_node,
                          elems_per_in_group,
                          at::cuda::getCurrentCUDAStream());
    return {output, scales};
 }
 PYBIND11_MODULE(TORCH_EXTENSION_NAME, m)
 {
    m.def("ds_quantize_fp32", &ds_quantize<float>, "DeepSpeed Quantize with fp32 (CUDA)");
    m.def("ds_quantize_fp16", &ds_quantize<__half>, "DeepSpeed Quantize with fp16 (CUDA)");
    m.def("ds_sr_quantize_fp32", &ds_sr_quantize<float>, "DeepSpeed Quantize with fp32 (CUDA)");
    m.def("ds_sr_quantize_fp16", &ds_sr_quantize<__half>, "DeepSpeed Quantize with fp16 (CUDA)");
    m.def("ds_quantize_asym_fp32", &ds_quantize_asym<float>, "DeepSpeed Quantize with fp32 (CUDA)");
    m.def(
        "ds_quantize_asym_fp16", &ds_quantize_asym<__half>, "DeepSpeed Quantize with fp16 (CUDA)");
    m.def("ds_sr_quantize_asym_fp32",
          &ds_sr_quantize_asym<float>,
          "DeepSpeed Quantize with fp32 (CUDA)");
    m.def("ds_sr_quantize_asym_fp16",
          &ds_sr_quantize_asym<__half>,
          "DeepSpeed Quantize with fp16 (CUDA)");
    pybind11::enum_<quantize::Type>(m, "QuantizationType")
        .value("Symmetric", quantize::Type::Symmetric)
        .value("Asymmetric", quantize::Type::Asymmetric)
        .export_values();
    m.def("quantize", &quantize_kernel);
    m.def("dequantize", &dequantize<__half>);
    m.def("dequantize_fp32", &dequantize<float>);
    m.def("dequantize_int4_to_half_experimental",
          &dequantize_int4_to_half_experimental,
          "Dequantize int4 to half (experimental)");
    m.def("swizzle_quant", &ds_swizzle_quant);
    m.def("quantized_reduction", &quantized_reduction);
 }
@@ -0,0 +1,160 @@
 # Copyright (c) Microsoft Corporation.
 # SPDX-License-Identifier: Apache-2.0
 # DeepSpeed Team
 import os
 import torch
 import deepspeed
 import subprocess
 import argparse
 import psutil
 from .ops.op_builder.all_ops import ALL_OPS
 from .git_version_info import installed_ops, torch_info
 from deepspeed.accelerator import get_accelerator
 GREEN = '\033[92m'
 RED = '\033[91m'
 YELLOW = '\033[93m'
 END = '\033[0m'
 SUCCESS = f"{GREEN} [SUCCESS] {END}"
 OKAY = f"{GREEN}[OKAY]{END}"
 WARNING = f"{YELLOW}[WARNING]{END}"
 FAIL = f'{RED}[FAIL]{END}'
 INFO = '[INFO]'
 color_len = len(GREEN) + len(END)
 okay = f"{GREEN}[OKAY]{END}"
 warning = f"{YELLOW}[WARNING]{END}"
 def op_report(verbose=True):
    max_dots = 23
    max_dots2 = 11
    h = ["op name", "installed", "compatible"]
    print("-" * (max_dots + max_dots2 + len(h[0]) + len(h[1])))
    print("DeepSpeed C++/CUDA extension op report")
    print("-" * (max_dots + max_dots2 + len(h[0]) + len(h[1])))
    print("NOTE: Ops not installed will be just-in-time (JIT) compiled at\n"
          "      runtime if needed. Op compatibility means that your system\n"
          "      meet the required dependencies to JIT install the op.")
    print("-" * (max_dots + max_dots2 + len(h[0]) + len(h[1])))
    print("JIT compiled ops requires ninja")
    ninja_status = OKAY if ninja_installed() else FAIL
    print('ninja', "." * (max_dots - 5), ninja_status)
    print("-" * (max_dots + max_dots2 + len(h[0]) + len(h[1])))
    print(h[0], "." * (max_dots - len(h[0])), h[1], "." * (max_dots2 - len(h[1])), h[2])
    print("-" * (max_dots + max_dots2 + len(h[0]) + len(h[1])))
    installed = f"{GREEN}[YES]{END}"
    no = f"{YELLOW}[NO]{END}"
    for op_name, builder in ALL_OPS.items():
        dots = "." * (max_dots - len(op_name))
        is_compatible = OKAY if builder.is_compatible(verbose) else no
        is_installed = installed if installed_ops.get(op_name, False) else no
        dots2 = '.' * ((len(h[1]) + (max_dots2 - len(h[1]))) - (len(is_installed) - color_len))
        print(op_name, dots, is_installed, dots2, is_compatible)
    print("-" * (max_dots + max_dots2 + len(h[0]) + len(h[1])))
 def ninja_installed():
    try:
        import ninja  # noqa: F401 # type: ignore
    except ImportError:
        return False
    return True
 def nvcc_version():
    import torch.utils.cpp_extension
    cuda_home = torch.utils.cpp_extension.CUDA_HOME
    if cuda_home is None:
        return f"{RED} [FAIL] cannot find CUDA_HOME via torch.utils.cpp_extension.CUDA_HOME={torch.utils.cpp_extension.CUDA_HOME} {END}"
    try:
        output = subprocess.check_output([cuda_home + "/bin/nvcc", "-V"], universal_newlines=True)
    except FileNotFoundError:
        return f"{RED} [FAIL] nvcc missing {END}"
    output_split = output.split()
    release_idx = output_split.index("release")
    release = output_split[release_idx + 1].replace(',', '').split(".")
    return ".".join(release)
 def get_shm_size():
    try:
        temp_dir = os.getenv('TEMP') or os.getenv('TMP') or os.path.join(os.path.expanduser('~'), 'tmp')
        shm_stats = psutil.disk_usage(temp_dir)
        shm_size = shm_stats.total
        shm_hbytes = human_readable_size(shm_size)
        warn = []
        if shm_size < 512 * 1024**2:
            warn.append(
                f" {YELLOW} [WARNING] Shared memory size might be too small, consider increasing it. {END}"
            )
            # Add additional warnings specific to your use case if needed.
        return shm_hbytes, warn
    except Exception as e:
        return "UNKNOWN", [f"Error getting shared memory size: {e}"]
 def human_readable_size(size):
    units = ['B', 'KB', 'MB', 'GB', 'TB']
    i = 0
    while size >= 1024 and i < len(units) - 1:
        size /= 1024
        i += 1
    return f'{size:.2f} {units[i]}'
 def debug_report():
    max_dots = 33
    report = [("torch install path", torch.__path__), ("torch version", torch.__version__),
              ("deepspeed install path", deepspeed.__path__),
              ("deepspeed info", f"{deepspeed.__version__}, {deepspeed.__git_hash__}, {deepspeed.__git_branch__}")]
    if get_accelerator().device_name() == 'cuda':
        hip_version = getattr(torch.version, "hip", None)
        report.extend([("torch cuda version", torch.version.cuda), ("torch hip version", hip_version),
                       ("nvcc version", (None if hip_version else nvcc_version())),
                       ("deepspeed wheel compiled w.", f"torch {torch_info['version']}, " +
                        (f"hip {torch_info['hip_version']}" if hip_version else f"cuda {torch_info['cuda_version']}"))
                       ])
    else:
        report.extend([("deepspeed wheel compiled w.", f"torch {torch_info['version']} ")])
    report.append(("shared memory (/dev/shm) size", get_shm_size()))
    print("DeepSpeed general environment info:")
    for name, value in report:
        warns = []
        if isinstance(value, tuple):
            value, warns = value
        print(name, "." * (max_dots - len(name)), value)
        if warns:
            for warn in warns:
                print(warn)
 def parse_arguments():
    parser = argparse.ArgumentParser()
    parser.add_argument('--hide_operator_status',
                        action='store_true',
                        help='Suppress display of installation and compatibility statuses of DeepSpeed operators. ')
    parser.add_argument('--hide_errors_and_warnings', action='store_true', help='Suppress warning and error messages.')
    args = parser.parse_args()
    return args
 def main(hide_operator_status=False, hide_errors_and_warnings=False):
    if not hide_operator_status:
        op_report(verbose=not hide_errors_and_warnings)
    debug_report()
 def cli_main():
    args = parse_arguments()
    main(hide_operator_status=args.hide_operator_status, hide_errors_and_warnings=args.hide_errors_and_warnings)
 if __name__ == "__main__":
    main()
@@ -0,0 +1,299 @@
 // Copyright (c) Microsoft Corporation.
 // SPDX-License-Identifier: Apache-2.0
 // DeepSpeed Team
 #include <ATen/cuda/CUDAContext.h>
 #include <torch/extension.h>
 #include <cassert>
 #include <vector>
 #include "quantization.h"
 template <typename T>
 at::Tensor ds_quantize(at::Tensor& vals, int groups, int bits)
 {
    auto t_size = vals.sizes();
    int size = 1;
    for (auto dim : t_size) size *= dim;
    if ((((size / groups) - 1) / 4096 + 1) <= 256) {
        launch_fake_quantize_kernel(
            (T*)vals.data_ptr(), size, groups, bits, at::cuda::getCurrentCUDAStream());
    }
    return vals;
 }
 template <typename T>
 at::Tensor ds_sr_quantize(at::Tensor& vals, int groups, int bits)
 {
    auto t_size = vals.sizes();
    int size = 1;
    for (auto dim : t_size) size *= dim;
    if (((size / groups) / 4 / 1024) <= 256) {
        launch_sr_fake_quantize_kernel(
            (T*)vals.data_ptr(), size, groups, bits, at::cuda::getCurrentCUDAStream());
    }
    return vals;
 }
 template <typename T>
 at::Tensor ds_quantize_asym(at::Tensor& vals, int groups, int bits)
 {
    auto t_size = vals.sizes();
    int size = 1;
    for (auto dim : t_size) size *= dim;
    if ((((size / groups) - 1) / 4096 + 1) <= 256) {
        launch_fake_quantize_kernel_asym(
            (T*)vals.data_ptr(), size, groups, bits, at::cuda::getCurrentCUDAStream());
    }
    return vals;
 }
 template <typename T>
 at::Tensor ds_sr_quantize_asym(at::Tensor& vals, int groups, int bits)
 {
    auto t_size = vals.sizes();
    int size = 1;
    for (auto dim : t_size) size *= dim;
    if (((size / groups) / 4 / 1024) <= 256) {
        launch_sr_fake_quantize_kernel_asym(
            (T*)vals.data_ptr(), size, groups, bits, at::cuda::getCurrentCUDAStream());
    }
    return vals;
 }
 std::vector<at::Tensor> quantize_kernel(at::Tensor& input_vals,
                                        int groups,
                                        int numBits,
                                        quantize::Type quantType)
 {
    auto dtype = at::kFloat;
    auto params_options = at::TensorOptions()
                              .dtype(dtype)
                              .layout(at::kStrided)
                              .device(at::kCUDA)
                              .requires_grad(false);
    const int param_elems = (quantize::requires_offset(quantType)) ? 2 : 1;
    auto params = torch::empty({groups, param_elems}, params_options);
    auto output_options = at::TensorOptions()
                              .dtype(at::kChar)
                              .layout(at::kStrided)
                              .device(at::kCUDA)
                              .requires_grad(false);
    auto output_sizes = input_vals.sizes().vec();
    output_sizes[output_sizes.size() - 1] /= numBits == 8 ? 1 : 2;
    auto output = torch::empty(output_sizes, output_options);
    const int elems_per_group = at::numel(input_vals) / groups;
    launch_quant((int8_t*)output.data_ptr(),
                 (float*)params.data_ptr(),
                 (__half*)input_vals.data_ptr(),
                 groups,
                 elems_per_group,
                 numBits,
                 quantType,
                 at::cuda::getCurrentCUDAStream());
    return {output, params};
 }
 template <typename T>
 at::Tensor dequantize(at::Tensor& quantized_data,
                      at::Tensor& params,
                      int groups,
                      int num_bits,
                      quantize::Type quant_type)
 {
    auto dtype = (std::is_same<T, float>::value) ? torch::kFloat32 : torch::kFloat16;
    auto output_options = at::TensorOptions()
                              .dtype(dtype)
                              .layout(at::kStrided)
                              .device(at::kCUDA)
                              .requires_grad(false);
    auto output_sizes = quantized_data.sizes().vec();
    output_sizes[output_sizes.size() - 1] *= num_bits == 8 ? 1 : 2;
    auto output = torch::empty(output_sizes, output_options);
    const int total_elems = at::numel(output);
    const int elems_per_group = total_elems / groups;
    launch_dequantize_kernel((T*)output.data_ptr(),
                             (const int8_t*)quantized_data.data_ptr(),
                             (const float*)params.data_ptr(),
                             quant_type,
                             num_bits,
                             elems_per_group,
                             total_elems,
                             at::cuda::getCurrentCUDAStream());
    return output;
 }
 at::Tensor dequantize_int4_to_half_experimental(at::Tensor& data_in,
                                                at::Tensor& scale_buffer,
                                                at::Tensor& min_val_buffer,
                                                int num_group,
                                                int group_size)
 {
    auto output_options = at::TensorOptions().dtype(at::kHalf).device(at::kCUDA);
    auto output = torch::empty({num_group, group_size}, output_options);
    launch_dequantize_int4_to_half_experimental((uint8_t*)data_in.data_ptr(),
                                                (half*)output.data_ptr(),
                                                (half*)scale_buffer.data_ptr(),
                                                (half*)min_val_buffer.data_ptr(),
                                                num_group,
                                                group_size,
                                                at::cuda::getCurrentCUDAStream());
    return output;
 }
 at::Tensor dequantize_int8_to_half_experimental(at::Tensor& data_in,
                                                at::Tensor& scale_buffer,
                                                at::Tensor& min_val_buffer,
                                                int num_group,
                                                int group_size)
 {
    auto output_options = at::TensorOptions().dtype(at::kHalf).device(at::kCUDA);
    auto output = torch::empty({num_group, group_size}, output_options);
    launch_dequantize_int8_to_half_experimental((uint8_t*)data_in.data_ptr(),
                                                (half*)output.data_ptr(),
                                                (half*)scale_buffer.data_ptr(),
                                                (half*)min_val_buffer.data_ptr(),
                                                num_group,
                                                group_size,
                                                at::cuda::getCurrentCUDAStream());
    return output;
 }
 std::vector<at::Tensor> ds_swizzle_quant(at::Tensor& input_vals,
                                         int groups,
                                         int num_bits,
                                         quantize::Type quant_type,
                                         int pipeline_size,
                                         int nodes,
                                         int devices_per_node)
 {
    auto scales_options = at::TensorOptions()
                              .dtype(at::kFloat)
                              .layout(at::kStrided)
                              .device(at::kCUDA)
                              .requires_grad(false);
    const int scales_elems = (quantize::requires_offset(quant_type)) ? 2 : 1;
    auto scales = torch::empty({groups, scales_elems}, scales_options);
    auto output_options = at::TensorOptions()
                              .dtype(at::kChar)
                              .layout(at::kStrided)
                              .device(at::kCUDA)
                              .requires_grad(false);
    const int quantization_scalar = 8 / num_bits;
    const int compressed_vals = at::numel(input_vals) / quantization_scalar;
    auto output = torch::empty({compressed_vals}, output_options);
    const int elems_per_group = at::numel(input_vals) / groups;
    launch_swizzled_quant((int8_t*)output.data_ptr(),
                          (float*)scales.data_ptr(),
                          (__half*)input_vals.data_ptr(),
                          num_bits,
                          quant_type,
                          groups,
                          elems_per_group,
                          pipeline_size,
                          nodes,
                          devices_per_node,
                          at::cuda::getCurrentCUDAStream());
    return {output, scales};
 }
 std::vector<at::Tensor> quantized_reduction(at::Tensor& input_vals,
                                            at::Tensor& input_scales,
                                            int in_groups,
                                            int out_groups,
                                            int num_bits,
                                            quantize::Type quant_type,
                                            int devices_per_node)
 {
    auto scales_options = at::TensorOptions()
                              .dtype(at::kFloat)
                              .layout(at::kStrided)
                              .device(at::kCUDA)
                              .requires_grad(false);
    const int scales_elems = (quantize::requires_offset(quant_type)) ? 2 : 1;
    auto scales = torch::empty({out_groups, scales_elems}, scales_options);
    auto output_options = at::TensorOptions()
                              .dtype(at::kChar)
                              .layout(at::kStrided)
                              .device(at::kCUDA)
                              .requires_grad(false);
    std::vector<int64_t> sz_vector(input_vals.sizes().begin(), input_vals.sizes().end());
    sz_vector[sz_vector.size() - 1] = sz_vector.back() / devices_per_node;  // num of GPU per nodes
    at::IntArrayRef sz(sz_vector);
    auto output = torch::empty(sz, output_options);
    const int elems_per_in_tensor = at::numel(input_vals) / devices_per_node;
    const int elems_per_in_group = elems_per_in_tensor / (in_groups / devices_per_node);
    const int elems_per_out_group = elems_per_in_tensor / out_groups;
    launch_dequant_reduce((int8_t*)output.data_ptr(),
                          (float*)scales.data_ptr(),
                          (const int8_t*)input_vals.data_ptr(),
                          (const float*)input_scales.data_ptr(),
                          devices_per_node,
                          num_bits,
                          quant_type,
                          out_groups,
                          elems_per_out_group,
                          elems_per_in_tensor,
                          in_groups / devices_per_node,
                          elems_per_in_group,
                          at::cuda::getCurrentCUDAStream());
    return {output, scales};
 }
 PYBIND11_MODULE(TORCH_EXTENSION_NAME, m)
 {
    m.def("ds_quantize_fp32", &ds_quantize<float>, "DeepSpeed Quantize with fp32 (CUDA)");
    m.def("ds_quantize_fp16", &ds_quantize<__half>, "DeepSpeed Quantize with fp16 (CUDA)");
    m.def("ds_sr_quantize_fp32", &ds_sr_quantize<float>, "DeepSpeed Quantize with fp32 (CUDA)");
    m.def("ds_sr_quantize_fp16", &ds_sr_quantize<__half>, "DeepSpeed Quantize with fp16 (CUDA)");
    m.def("ds_quantize_asym_fp32", &ds_quantize_asym<float>, "DeepSpeed Quantize with fp32 (CUDA)");
    m.def(
        "ds_quantize_asym_fp16", &ds_quantize_asym<__half>, "DeepSpeed Quantize with fp16 (CUDA)");
    m.def("ds_sr_quantize_asym_fp32",
          &ds_sr_quantize_asym<float>,
          "DeepSpeed Quantize with fp32 (CUDA)");
    m.def("ds_sr_quantize_asym_fp16",
          &ds_sr_quantize_asym<__half>,
          "DeepSpeed Quantize with fp16 (CUDA)");
    pybind11::enum_<quantize::Type>(m, "QuantizationType")
        .value("Symmetric", quantize::Type::Symmetric)
        .value("Asymmetric", quantize::Type::Asymmetric)
        .export_values();
    m.def("quantize", &quantize_kernel);
    m.def("dequantize", &dequantize<__half>);
    m.def("dequantize_fp32", &dequantize<float>);
    m.def("dequantize_int4_to_half_experimental",
          &dequantize_int4_to_half_experimental,
          "Dequantize int4 to half (experimental)");
    m.def("dequantize_int8_to_half_experimental",
          &dequantize_int8_to_half_experimental,
          "Dequantize int8 to half (experimental)");
    m.def("swizzle_quant", &ds_swizzle_quant);
    m.def("quantized_reduction", &quantized_reduction);
 }
@@ -0,0 +1,161 @@
 # Copyright (c) Microsoft Corporation.
 # SPDX-License-Identifier: Apache-2.0
 # DeepSpeed Team
 import os
 import torch
 import deepspeed
 import subprocess
 import argparse
 import psutil
 from .ops.op_builder.all_ops import ALL_OPS
 from .git_version_info import installed_ops, torch_info
 from deepspeed.accelerator import get_accelerator
 GREEN = '\033[92m'
 RED = '\033[91m'
 YELLOW = '\033[93m'
 END = '\033[0m'
 SUCCESS = f"{GREEN} [SUCCESS] {END}"
 OKAY = f"{GREEN}[OKAY]{END}"
 WARNING = f"{YELLOW}[WARNING]{END}"
 FAIL = f'{RED}[FAIL]{END}'
 INFO = '[INFO]'
 color_len = len(GREEN) + len(END)
 okay = f"{GREEN}[OKAY]{END}"
 warning = f"{YELLOW}[WARNING]{END}"
 def op_report(verbose=True):
    max_dots = 23
    max_dots2 = 11
    h = ["op name", "installed", "compatible"]
    print("-" * (max_dots + max_dots2 + len(h[0]) + len(h[1])))
    print("DeepSpeed C++/CUDA extension op report")
    print("-" * (max_dots + max_dots2 + len(h[0]) + len(h[1])))
    print("NOTE: Ops not installed will be just-in-time (JIT) compiled at\n"
          "      runtime if needed. Op compatibility means that your system\n"
          "      meet the required dependencies to JIT install the op.")
    print("-" * (max_dots + max_dots2 + len(h[0]) + len(h[1])))
    print("JIT compiled ops requires ninja")
    ninja_status = OKAY if ninja_installed() else FAIL
    print('ninja', "." * (max_dots - 5), ninja_status)
    print("-" * (max_dots + max_dots2 + len(h[0]) + len(h[1])))
    print(h[0], "." * (max_dots - len(h[0])), h[1], "." * (max_dots2 - len(h[1])), h[2])
    print("-" * (max_dots + max_dots2 + len(h[0]) + len(h[1])))
    installed = f"{GREEN}[YES]{END}"
    no = f"{YELLOW}[NO]{END}"
    for op_name, builder in ALL_OPS.items():
        dots = "." * (max_dots - len(op_name))
        is_compatible = OKAY if builder.is_compatible(verbose) else no
        is_installed = installed if installed_ops.get(op_name, False) else no
        dots2 = '.' * ((len(h[1]) + (max_dots2 - len(h[1]))) - (len(is_installed) - color_len))
        print(op_name, dots, is_installed, dots2, is_compatible)
    print("-" * (max_dots + max_dots2 + len(h[0]) + len(h[1])))
 def ninja_installed():
    try:
        import ninja  # noqa: F401 # type: ignore
    except ImportError:
        return False
    return True
 def nvcc_version():
    import torch.utils.cpp_extension
    cuda_home = torch.utils.cpp_extension.CUDA_HOME
    if cuda_home is None:
        return f"{RED} [FAIL] cannot find CUDA_HOME via torch.utils.cpp_extension.CUDA_HOME={torch.utils.cpp_extension.CUDA_HOME} {END}"
    try:
        output = subprocess.check_output([cuda_home + "/bin/nvcc", "-V"], universal_newlines=True)
    except FileNotFoundError:
        return f"{RED} [FAIL] nvcc missing {END}"
    output_split = output.split()
    release_idx = output_split.index("release")
    release = output_split[release_idx + 1].replace(',', '').split(".")
    return ".".join(release)
 def get_shm_size():
    try:
        temp_dir = os.getenv('TEMP') or os.getenv('TMP') or os.path.join(os.path.expanduser('~'), 'tmp')
        shm_stats = psutil.disk_usage(temp_dir)
        shm_size = shm_stats.total
        shm_hbytes = human_readable_size(shm_size)
        warn = []
        if shm_size < 512 * 1024**2:
            warn.append(
                f" {YELLOW} [WARNING] Shared memory size might be too small, consider increasing it. {END}"
            )
            # Add additional warnings specific to your use case if needed.
        return shm_hbytes, warn
    except Exception as e:
        return "UNKNOWN", [f"Error getting shared memory size: {e}"]
 def human_readable_size(size):
    units = ['B', 'KB', 'MB', 'GB', 'TB']
    i = 0
    while size >= 1024 and i < len(units) - 1:
        size /= 1024
        i += 1
    return f'{size:.2f} {units[i]}'
 def debug_report():
    max_dots = 33
    report = [("torch install path", torch.__path__), ("torch version", torch.__version__),
              ("deepspeed install path", deepspeed.__path__),
              ("deepspeed info", f"{deepspeed.__version__}, {deepspeed.__git_hash__}, {deepspeed.__git_branch__}")]
    if get_accelerator().device_name() == 'cuda':
        hip_version = getattr(torch.version, "hip", None)
        report.extend([("torch cuda version", torch.version.cuda), ("torch hip version", hip_version),
                       ("nvcc version", (None if hip_version else nvcc_version())),
                       ("deepspeed wheel compiled w.", f"torch {torch_info['version']}, " +
                        (f"hip {torch_info['hip_version']}" if hip_version else f"cuda {torch_info['cuda_version']}"))
                       ])
    else:
        report.extend([("deepspeed wheel compiled w.", f"torch {torch_info['version']} ")])
    report.append(("shared memory (/dev/shm) size", get_shm_size()))
    print("DeepSpeed general environment info:")
    for name, value in report:
        warns = []
        if isinstance(value, tuple):
            value, warns = value
        print(name, "." * (max_dots - len(name)), value)
        if warns:
            for warn in warns:
                print(warn)
 def parse_arguments():
    parser = argparse.ArgumentParser()
    parser.add_argument('--hide_operator_status',
                        action='store_true',
                        help='Suppress display of installation and compatibility statuses of DeepSpeed operators. ')
    parser.add_argument('--hide_errors_and_warnings', action='store_true', help='Suppress warning and error messages.')
    args = parser.parse_args()
    return args
 def main(hide_operator_status=False, hide_errors_and_warnings=False):
    if not hide_operator_status:
        op_report(verbose=not hide_errors_and_warnings)
    debug_report()
 def cli_main():
    args = parse_arguments()
    main(hide_operator_status=args.hide_operator_status, hide_errors_and_warnings=args.hide_errors_and_warnings)
 if __name__ == "__main__":
    main()
@@ -0,0 +1,18 @@
 These are the modified files you would need for a MANUAL build of DeepSpeed 11.1 or 11.2.
 They were created by a mix of instructions from erew123 and S95Sedan.
 - You still have to download the DeepSpeed v11.1 or v11.2 installation files from Microsoft to use these files.
 https://github.com/microsoft/DeepSpeed/releases/tag/v0.11.2
 - These files were provided by S95Sedan here: 
 https://github.com/S95Sedan/Deepspeed-Windows
 - They are related to the manual build of DeepSpeed for AllTalk, which is linked here: 
 https://github.com/erew123/alltalk_tts/tree/main?tab=readme-ov-file#-option-2---a-bit-more-complicated
@@ -1,18 +1,41 @@
 # AllTalk TTS
 AllTalk is an updated version of the Coqui_tts extension for Text Generation web UI. Features include:
- **Custom Startup Settings:** Adjust your standard startup settings. [Screenshot](https://github.com/erew123/alltalk_tts#screenshots)
+- **Custom Start-up Settings:** Adjust your default start-up settings. [Screenshot](https://github.com/erew123/alltalk_tts#screenshots)
- **Nararator:** Use different voices for main character and narration. [Screenshot](https://github.com/erew123/alltalk_tts#screenshots)
+- **Narrarator:** Use different voices for main character and narration. [Example Narration](https://vocaroo.com/18fYWVxiQpk1)
 - **Low VRAM mode:** Improve generation performance if your VRAM is filled by your LLM. [Screenshot](https://github.com/erew123/alltalk_tts#screenshots)
- **DeepSpeed:** When DeepSpeed is installed you can get a 3-4x performance boost generating TTS.
+- **DeepSpeed:** A 3-4x performance boost generating TTS. [DeepSpeed Windows/Linux Instructions](https://github.com/erew123/alltalk_tts?tab=readme-ov-file#deepspeed-installation-options) [Screenshot](https://github.com/erew123/alltalk_tts/assets/35898566/548619c8-5f1b-47d0-a73d-54d2fee3f3db)
 - **Local/Custom models:** Use any of the XTTSv2 models (API Local and XTTSv2 Local).
 - **Optional wav file maintenance:** Configurable deletion of old output wav files. [Screenshot](https://github.com/erew123/alltalk_tts#screenshots)
 - **Documentation:** Fully documented with a built in webpage. [Screenshot](https://github.com/erew123/alltalk_tts#screenshots)
 - **Console output** Clear command line output for any warnings or issues.
 - **Standalone/3rd Party support via JSON calls** Can be used with 3rd party applications via JSON calls.
-## Installation on Text generation web UI
+## Index
-This has been tested on the current Dec 2023 release of Text generation webUI. If you have not updated it for a while, you may wish to update Text generation webUI, instructions [here](https://github.com/oobabooga/text-generation-webui#getting-updates)
+
 - 🟩 [Installation](https://github.com/erew123/alltalk_tts?#-installation-on-text-generation-web-ui)
 - 🟪 [Updating & problems with updating](https://github.com/erew123/alltalk_tts?#-updating)
 - 🟫 [Screenshots](https://github.com/erew123/alltalk_tts#-screenshots)
 - 🟨 [Help with problems](https://github.com/erew123/alltalk_tts?#-help-with-problems)
 - 🔵🟢🟡 [DeepSpeed Installation (Windows & Linux)](https://github.com/erew123/alltalk_tts?#-deepspeed-installation-options)
 - 🔴 [Future to-do list & Upcoming updates](https://github.com/erew123/alltalk_tts?#-future-to-do-list)
 #### Updates
 The latest build (13 Dec 2023) has had the entire text filtering engine and narration engine rebuilt from scratch. It's highly complicated how its actually working, but the end result it a much clearer TTS output and much better control over the narrator option and how to handle text that isnt within quotes or asterisks. Its a highly recommened update, for the improved quality it gives to the TTS output, if nothing else.
 Should you want the older version of the narrator engine+text filtering, I will leave this older copy [here](https://github.com/erew123/alltalk_tts/releases/tag/v1-old-narrator)
 DeepSpeed **v11.2** can be installed within the **default text-generation-webui Python 3.11 environment**. Instructions [here](https://github.com/erew123/alltalk_tts#deepspeed-112-for-windows--python-311) (or scroll down). Please note, this is **not** an official Microsoft method (currently) so they will not support you with this style installation. Officially, only DeepSpeed v8.3 is installing on Python 3.9.x.
 #### The one thing I cant easily work around
 With a RP chat with your AI, **on your character card** `parameters menu` > `character tab` > `greeting` make sure that anything in there that is the **narrator is in asterisks** and anything **spoken is in double quotes**, then hit the `save` (disk) button. Greeting paragraphs/sentences are handled differently from how the AI sends text and so its difficut to account for them both.
 I could force a delimeter in at this stage, but I know it would/may affect things further down the line in the chat and I need a good think about that before just making a change. This issue **only** affects the greeting card/start of conversation and the "example" card that comes with text-generation-webui suffers this issue (if you want to try it for yourself). So you would put double quotes around like this (from the example card):
 `"`Hey! I'm so excited to finally meet you. I've heard so many great things about you and I'm eager to pick your brain about computers. I'm sure you have a wealth of knowledge that I can learn from.`"`
 ## 🟩 Installation on Text generation web UI
 This has been tested on the current Dec 2023 release of Text generation webUI. If you have not updated it for a while, you may wish to update Text generation webUI, [instructions here](https://github.com/oobabooga/text-generation-webui?tab=readme-ov-file#how-to-install)
 1) In a command prompt/terminal window you need to move into your Text generation webUI folder:
@@ -40,7 +63,13 @@ This has been tested on the current Dec 2023 release of Text generation webUI. I
 *Other machines (mac, amd etc)* - `pip install -r requirements_other.txt`
-7) You can now start move back to the main Text generation webUI folder `cd ..` (a few times), start Text generation webUI (`start_windows.bat`,`./start_linux.sh`, `start_macos.sh` or `start_wsl.bat`)  and load the AllTalk extension in the Text generation webUI **session** tab.
+7) **(Optional DeepSpeed)** If you have an Nvidia Graphics card on a system running Linux or Windows and wish to use **DeepSpeed** please follow these instructions [here](https://github.com/erew123/alltalk_tts?#-deepspeed-installation-options). Though you may wish to check things are generally working with the below steps before installing DeepSpeed.
 8) You can now start move back to the main Text generation webUI folder `cd ..` (a few times), start Text generation webUI (`start_windows.bat`,`./start_linux.sh`, `start_macos.sh` or `start_wsl.bat`)  and load the AllTalk extension in the Text generation webUI **session** tab.
 9) Please read the note below about start-up times and also the note above about ensuring your character cards are set up [correctly](https://github.com/erew123/alltalk_tts#the-one-thing-i-cant-easily-work-around)
 10) Some extra voices downloadable [here](https://drive.google.com/file/d/1bYdZdr3L69kmzUN3vSiqZmLRD7-A3M47/view?usp=drive_link)
 **Note: It can take a while to start up.** Check the command prompt/terminal window if you want to know what its doing. After it says "Model Loaded" the Text generation webUI is usually available on its IP address a few seconds later, for you to connect to in your browser.
@@ -48,13 +77,236 @@ This has been tested on the current Dec 2023 release of Text generation webUI. I
 **Where to find voices** https://aiartes.com/voiceai or https://commons.wikimedia.org/ or interviews on youtube etc. Instructions on how to cut down and prepare a voice sample are within the built in documentation.
-### Other installation notes
+### 🟩 Other installation notes
 On first startup, AllTalk will download the Coqui XTTSv2 2.0.2 model to its **models** folder (1.8GB space required). You can customse your model or use the TTS latest model within the interface (details in documentation).
 Once the extension is loaded, please find all documentation and settings on the link provided in the interface (as shown in the screenshot below).
 To start AllTalk every Text generation webUI loads, edit the Text generation webUI `CMD_FLAGS.txt` file in the main `text-generation-webui` folder and add `--extensions alltalk_tts`.
-## Screenshots
+### 🟪 Updating
-|![image](https://github.com/erew123/alltalk_tts/assets/35898566/dfb972dd-817b-45f8-8878-3d3f82ac1ecc) | ![image](https://github.com/erew123/alltalk_tts/assets/35898566/3497d656-9729-4cb7-8d0d-6367078794ee) |
+This is pretty much a repeat of the installation process. 
 1) In a command prompt/terminal window you need to move into your Text generation webUI folder:
 `cd text-generation-webui`
 2) Move into your extensions and alltalk_tts folder:
 `cd extensions` then `cd alltalk_tts`
 3) At the command prompt/terminal, type:
 `git pull`
 4) Install the requirements:
 *Nvidia graphics card machines* - `pip install -r requirements_nvidia.txt`
 *Other machines (mac, amd etc)* - `pip install -r requirements_other.txt`
 #### 🟪 **git pull error** 
 I did leave a mistake in the `/extensions/alltalk_tts/.gitignore` file at one point. If your `git pull` doesnt work, you can either follow the Problems Updating section below, or edit the `.gitignore` file and **replace its entire contents** with the below, save the file, then re-try the `git pull`
 ```
 voices/*.*
 models/*.*
 outputs/*.*
 config.json
 models.json
 ```
 ### 🟪 Problems Updating
 If you do experience any problems, the simplest method to resolve this will be:
 1) re-name the existing `alltalk_tts` folder to something like `alltalk_tts.old`
 2) Start a console/terminal then:
 `cd text-generation-webui` and start your python environment `cmd_windows.bat`, `./cmd_linux.sh`, `cmd_macos.sh` or `cmd_wsl.bat`
 3) Move into the extensions folder, same as if you were doing a fresh installation:
 `cd extensions` then 
 `git clone https://github.com/erew123/alltalk_tts`
 This will download a fresh installation. 
 3) Move into the **alltalk_tts** folder:
 `cd alltalk_tts`
 4) Install the requirements:
 *Nvidia graphics card machines* - `pip install -r requirements_nvidia.txt`
 *Other machines (mac, amd etc)* - `pip install -r requirements_other.txt`
 5) Before starting it up, copy/merge the `models`, `voices` and `outputs` folders over from the `alltalk_tts.old` folder to the newly created `alltalk_tts` folder. This will keep your voices history and also stop it re-downloading the model again.
 You can now start text-generation-webui or AllTalk (standalone) and it should start up fine. You will need to re-set any saved configuration changes on the configuration page. 
 Assuming its all working fine and you are happy, you can delete the old alltalk_tts.old folder.
 ## 🟫 Screenshots
 |![image](https://github.com/erew123/alltalk_tts/assets/35898566/4ca9b4c7-60ce-4ac6-82e5-fd1989b84644) | ![image](https://github.com/erew123/alltalk_tts/assets/35898566/b0e13dba-c6b1-4ab7-845d-244ac1158330) |
 |:---:|:---:|
 |![image](https://github.com/erew123/alltalk_tts/assets/35898566/548619c8-5f1b-47d0-a73d-54d2fee3f3db) | ![image](https://github.com/erew123/alltalk_tts/assets/35898566/e35e987c-543a-486b-b4fb-ee6ebe6f59c6) |
 ## 🟨 Help with problems
 #### 🟨 I activated DeepSpeed in the settings page, but I didnt install DeepSpeed yet and now I have issues starting up
 You can either follow the [Problems Updating](https://github.com/erew123/alltalk_tts#-problems-updating) and fresh install your config. Or you can edit the `config.json` file within the `alltalk_tts` folder. You would look for '"deepspeed_activate": true,' and change the word true to false `"deepspeed_activate": false,' ,then save the file and try starting again.
 If you want to use DeepSpeed, you need an Nvidia Graphics card and to install DeepSpeed on your system. Instructions are [here](https://github.com/erew123/alltalk_tts?tab=readme-ov-file#deepspeed-installation-options).
 #### 🟨 I am having problems getting AllTalk to start after changing settings or making a custom setup/model setup.
 I would suggest following [Problems Updating](https://github.com/erew123/alltalk_tts#-problems-updating) and if you still have issues after that, you can raise an issue [here](https://github.com/erew123/alltalk_tts/issues)
 #### 🟨 I see some red "asyncio" messages
 As far as I am aware, these are to do with the chrome browser the gradio text-generation-webui in some way. I raised an issue about this on the text-generation-webui [here](https://github.com/oobabooga/text-generation-webui/issues/4788) where you can see that AllTalk is not loaded and the messages persist. Either way, this is more a warning than an actual issue, so shouldnt affect any functionality of either AllTalk or text-generation-webui, they are more just an annoyance.
 #### 🟨 I am having problems updating/some other issue where it wont start up/Im sure this is a bug
 Please see [Problems Updating](https://github.com/erew123/alltalk_tts#-problems-updating). If that doesnt help you can raise an issue [here](https://github.com/erew123/alltalk_tts/issues). It would be handy to have any log files from the console where your error is being shown. I can only losely support custom built Python environments and give general pointers.
 Also, is your text-generation-webui up to date? [instructions here](https://github.com/oobabooga/text-generation-webui?tab=readme-ov-file#how-to-install)
 ## 🔵🟢🟡 DeepSpeed Installation Options
 ### 🔵 For Linux
 ➡️DeepSpeed requires an Nvidia Graphics card!⬅️
 Covered in the online/buit-in documentation, but a nice easy install.
 ### 🟢🟡 For Windows & Python 3.11
 DeepSpeed v11.1 and v11.2 will work on the current text-generation-webui Python 3.11 environment! You have 2x options for how to setup DeepSpeed on Windows. A quick way (🟢Option 1) and a long way (🟡Option 2).
 Thanks to [@S95Sedan](https://github.com/S95Sedan) - They managed to get DeepSpeed 11.2 working on Windows via making some edits to the original Microsoft DeepSpeed v11.2 installation. The original post is [here](https://github.com/oobabooga/text-generation-webui/issues/4734#issuecomment-1843984142).
 #### 🟢 OPTION 1 - Quick and easy!
 #### Pre-Compiled Wheel Deepspeed v11.1 (for Windows and Python 3.11) 
 ➡️DeepSpeed requires an Nvidia Graphics card!⬅️
 [@S95Sedan](https://github.com/S95Sedan) has kindly provided a pre-compiled DeepSpeed v11.1 wheel file, which you can download and use. To use this, you will need to:
 **Note:** In my tests, with this method you will **not** need to install the Nvidia CUDA toolkit to make this work, but AllTalk may warn you when starting DeepSpeed that it doesnt see the CUDA Toolkit, however, it works fine for TTS purposes.
 1) Download the file [deepspeed-0.11.1+e9503fe-cp311-cp311-win_amd64.whl](https://drive.google.com/file/d/1PFsf6uSPY5Cb4o9VxiZ7DLv-j35L7Y41/view?usp=sharing) by going to the link and clicking the **download** icon at the top right of the screen and save the file it inside your **text-generation-webui** folder.
 2) Open a command prompt window, move into your **text-generation-webui folder**, you can now start the Python environment for text-generation-webui:
 `cmd_windows.bat`
 3) With the file that you saved in the **text-generation-webui folder** you now type the following:
 `pip install "deepspeed-0.11.1+e9503fe-cp311-cp311-win_amd64.whl"`
 4) This should install through cleanly and you should now have DeepSpeed v11.1 installed within the Python 3.11 environment of text-generation-webui.
 5) When you start up text-generation-webui, and AllTalk starts, you should see **[AllTalk Startup] DeepSpeed Detected**
 6) Within AllTalk, you will now have a checkbox for **Activate DeepSpeed** though remember you can only change **1x setting every 15 or so seconds**, so dont try to activate DeepSpeed **and** LowVRAM/Change your model simultantiously. Do one of those, wait 15-20 seconds until the change is confirmed in the terminal/command prompt, then you can change the other. When you are happy it works, you can set the default start-up settings in the settings page.
 #### 🟡 OPTION 2 - A bit more complicated!
 #### Manual Build DeepSpeed v11.2 (for Windows and Python 3.11) 
 ➡️DeepSpeed requires an Nvidia Graphics card!⬅️
 DeepSpeed Version 11.2 with CUDA 12.1 - Installation Instructions:
 1. Download the 11.2 release of [DeepSpeed](https://github.com/microsoft/DeepSpeed/releases/tag/v0.11.2) extract it to a folder. 
 2. Install Visual C++ build tools, such as [VS2019 C++ x64/x86](https://learn.microsoft.com/en-us/visualstudio/releases/2019/redistribution#vs2019-download) build tools.
 3. Download and install the [Nvidia Cuda Toolkit 12.1](https://developer.nvidia.com/cuda-12-1-0-download-archive)
 4. Edit your Windows environment variables to ensure that CUDA_HOME and CUDA_PATH are set to your Nvidia Cuda Toolkit path. (The folder above the bin folder that nvcc.exe is installed in). Examples are:<br>
 ```set CUDA_HOME=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1```<br>
 ```set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1```<br>
 5. OPTIONAL If you do not have an python environment already created, you can install [Miniconda](https://docs.conda.io/projects/miniconda/en/latest/miniconda-install.html), then at a command prompt, create and activate your environment with:<br>
 ```conda create -n pythonenv python=3.11```<br>
 ```activate pythonenv```<br>
 6. Launch the Command Prompt cmd with Administrator privilege as it requires admin to allow creating symlink folders.
 7. Install PyTorch, 2.1.0 with CUDA 12.1 into your Python 3.11 environment e.g:<br>
 ```activate pythonenv``` (activate your python environment)<br>
 ```conda install pytorch==2.1.0 torchvision==0.16.0 torchaudio==2.1.0 pytorch-cuda=12.1 -c pytorch -c nvidia```
 8. In your python environment check that your CUDA_HOME and CUDA_PATH are still pointing to the correct location.<br>
 ```set``` (to list and check the windows environment variables. Refer to step 4 if not)
 9. Navigate to your deepspeed folder in the Command Prompt:<br>
 ```cd c:\deepspeed``` (wherever you extracted it to)
 10. Modify the following files:<br>
 **(These modified files are included in the git-pull of AllTalk, but if you want to modify them yourself, please follow the below)**
 deepspeed-0.11.2/build_win.bat** - at the top of the file, add:<br>
 ```set DS_BUILD_EVOFORMER_ATTN=0```
 deepspeed-0.11.2/csrc/quantization/pt_binding.cpp - lines 244-250 - change to:
 ```
    std::vector<int64_t> sz_vector(input_vals.sizes().begin(), input_vals.sizes().end());
    sz_vector[sz_vector.size() - 1] = sz_vector.back() / devices_per_node;  // num of GPU per nodes
    at::IntArrayRef sz(sz_vector);
    auto output = torch::empty(sz, output_options);
    const int elems_per_in_tensor = at::numel(input_vals) / devices_per_node;
    const int elems_per_in_group = elems_per_in_tensor / (in_groups / devices_per_node);
    const int elems_per_out_group = elems_per_in_tensor / out_groups;
 ```
 deepspeed-0.11.2/csrc/transformer/inference/csrc/pt_binding.cpp
 lines 541-542 - change to:
 ```
 									 {static_cast<unsigned>(hidden_dim * InferenceContext::Instance().GetMaxTokenLength()),
 									  static_cast<unsigned>(k * InferenceContext::Instance().GetMaxTokenLength()),
 ```
 lines 550-551 - change to:
 ```
 						 {static_cast<unsigned>(hidden_dim * InferenceContext::Instance().GetMaxTokenLength()),
 						  static_cast<unsigned>(k * InferenceContext::Instance().GetMaxTokenLength()),
 ```
 line 1581 - change to:
 ```
 		at::from_blob(intermediate_ptr, {input.size(0), input.size(1), static_cast<int64_t>(mlp_1_out_neurons)}, options);
 ```
 deepspeed-0.11.2/deepspeed/env_report.py
 line 10 - add:
 ```
 import psutil
 ```
 line 83 - 100 - change to:
 ```
 def get_shm_size():
    try:
        temp_dir = os.getenv('TEMP') or os.getenv('TMP') or os.path.join(os.path.expanduser('~'), 'tmp')
        shm_stats = psutil.disk_usage(temp_dir)
        shm_size = shm_stats.total
        shm_hbytes = human_readable_size(shm_size)
        warn = []
        if shm_size < 512 * 1024**2:
            warn.append(
                f" {YELLOW} [WARNING] Shared memory size might be too small, consider increasing it. {END}"
            )
            # Add additional warnings specific to your use case if needed.
        return shm_hbytes, warn
    except Exception as e:
        return "UNKNOWN", [f"Error getting shared memory size: {e}"]
 ```
 11. While still in your command line with python environment enabled run:<br>
 ```build_win.bat```
 12. Now cd dist to go into your dist folder and you can now pip install deepspeed-YOURFILENAME.whl (or whatever your WHL file is called).
 ### 🔴 Future to-do list
 - Complete & document the new/full standalone mode API.
 - Voice output within the command prompt/terminal (TBD).
 - Correct an issue on incorrect output folder path when running as a standalone app.
 - Correct a few spelling mistakes in the documnentation.
 - Possibly add some additional TTS engines (TBD).
 - Have a break!
@@ -1 +1 @@
-{"activate": true, "autoplay": true, "branding": "AllTalk ", "narrator_enabled": false, "deepspeed_activate": false, "delete_output_wavs": "Disabled", "ip_address": "127.0.0.1", "language": "English", "low_vram": false, "local_temperature": "0.7", "local_repetition_penalty": "12.0", "model_loaded": true, "model_name": "tts_models/multilingual/multi-dataset/xtts_v2", "narrator_voice": "female_01.wav", "output_folder_wav": "extensions/alltalk_tts/outputs/", "port_number": "7851", "remove_trailing_dots": false, "show_text": true, "tts_method_api_local": false, "tts_method_api_tts": false, "tts_method_xtts_local": true, "voice": "female_01.wav"}
+{"activate": true, "autoplay": true, "branding": "AllTalk ", "narrator_enabled": false, "deepspeed_activate": false, "delete_output_wavs": "Disabled", "ip_address": "127.0.0.1", "language": "English", "low_vram": false, "local_temperature": "0.7", "local_repetition_penalty": "10.0", "model_loaded": true, "model_name": "tts_models/multilingual/multi-dataset/xtts_v2", "narrator_voice": "female_01.wav", "output_folder_wav": "extensions/alltalk_tts/outputs/", "port_number": "7851", "remove_trailing_dots": false, "show_text": true, "tts_method_api_local": false, "tts_method_api_tts": false, "tts_method_xtts_local": true, "voice": "female_01.wav"}
@@ -48,7 +48,6 @@ except ModuleNotFoundError:
 # IMPORT - Attempt Importing DeepSpeed (required for displaying Deepspeed checkbox in gradio)
 try:
    import deepspeed
    deepspeed_installed = True
 except ImportError:
    deepspeed_installed = False
@@ -155,8 +154,8 @@ elif params["tts_method_api_local"] == True:
 elif params["tts_method_xtts_local"] == True:
    gr_modelchoice = "XTTSv2 Local"
-gr_narrator_enabled = str(params["narrator_enabled"]).lower()
+# Set the default for Narrated text without asterisk or quotes to be Narrator
-
+non_quoted_text_is = True
 ######################
 #### GRADIO STUFF ####
@@ -272,7 +271,6 @@ else:
    # Cleanly kill off this script, but allow text-generation-webui to keep running, albeit without this alltalk_tts
    sys.exit(1)
 #####################################
 #### MODEL LOADING AND UNLOADING ####
 #####################################
@@ -289,7 +287,25 @@ def send_reload_request(tts_method):
        if json_response.get("status") == "model-success":
            # Update model_loaded to True if the reload was successful
            params["model_loaded"] = True
-        return json_response
+            # Update local script parameters based on the tts_method
            if tts_method == "API TTS":
                params["tts_method_api_local"] = False
                params["tts_method_xtts_local"] = False
                params["tts_method_api_tts"] = True
                params["deepspeed_activate"] = False
                audio_path = this_dir / "templates" / "apitts.wav"
            elif tts_method == "API Local":
                params["tts_method_api_tts"] = False
                params["tts_method_xtts_local"] = False
                params["tts_method_api_local"] = True
                params["deepspeed_activate"] = False
                audio_path = this_dir / "templates" / "apilocal.wav"
            elif tts_method == "XTTSv2 Local":
                params["tts_method_api_tts"] = False
                params["tts_method_api_local"] = False
                params["tts_method_xtts_local"] = True
                audio_path = this_dir / "templates" / "xttslocal.wav"
        return f'<audio src="file/{audio_path}" controls autoplay></audio>'
    except requests.exceptions.RequestException as e:
        # Handle the HTTP request error
        print(
@@ -297,7 +313,6 @@ def send_reload_request(tts_method):
        )
        return {"status": "error", "message": str(e)}
 ##################
 #### LOW VRAM ####
 ##################
@@ -305,6 +320,10 @@ def send_reload_request(tts_method):
 def send_lowvram_request(low_vram):
    try:
        params["model_loaded"] = False
        if low_vram:
            audio_path = this_dir / "templates" / "lowvramenabled.wav"
        else:
            audio_path = this_dir / "templates" / "lowvramdisabled.wav"
        url = f"{base_url}/api/lowvramsetting?new_low_vram_value={low_vram}"
        headers = {"Content-Type": "application/json"}
        response = requests.post(url, headers=headers)
@@ -314,7 +333,7 @@ def send_lowvram_request(low_vram):
        if json_response.get("status") == "lowvram-success":
            # Update any relevant variables or perform other actions on success
            params["model_loaded"] = True
-        return json_response
+        return f'<audio src="file/{audio_path}" controls autoplay></audio>'
    except requests.exceptions.RequestException as e:
        # Handle the HTTP request error
        print(
@@ -330,6 +349,11 @@ def send_lowvram_request(low_vram):
 def send_deepspeed_request(deepspeed_param):
    try:
        params["model_loaded"] = False
        if deepspeed_param:
            audio_path = this_dir / "templates" / "deepspeedenabled.wav"
        else:
            audio_path = this_dir / "templates" / "deepspeeddisabled.wav"
        url = f"{base_url}/api/deepspeed?new_deepspeed_value={deepspeed_param}"
        headers = {"Content-Type": "application/json"}
        response = requests.post(url, headers=headers)
@@ -339,7 +363,7 @@ def send_deepspeed_request(deepspeed_param):
        if json_response.get("status") == "deepspeed-success":
            # Update any relevant variables or perform other actions on success
            params["model_loaded"] = True
-        return json_response
+        return f'<audio src="file/{audio_path}" controls autoplay></audio>'
    except requests.exceptions.RequestException as e:
        # Handle the HTTP request error
        print(
@@ -402,11 +426,11 @@ def combine(audio_files, output_folder, state):
            audio = np.concatenate((audio, audio_data))
    # Save the combined audio to a file with a specified sample rate
-    output_file_path = os.path.join(
+    if "character_menu" in state:
-        output_folder, f'{state["character_menu"]}_{int(time.time())}_combined.wav'
+        output_file_path = os.path.join(output_folder, f'{state["character_menu"]}_{int(time.time())}_combined.wav')
-    )
+    else:
        output_file_path = os.path.join(output_folder, f'TTSOUT_{int(time.time())}_combined.wav')
    sf.write(output_file_path, audio, samplerate=sample_rate)
    print(f"[{params['branding']}TTSGen] Narrated audio generated")
    # Clean up unnecessary files
    for audio_file in audio_files:
        os.remove(audio_file)
@@ -463,7 +487,6 @@ def voice_preview(string):
 #################################
 # STANDARD VOICE - Generate TTS Function
 def output_modifier(string, state):
    # DEBUG print("THE ORIGINAL STRING IS:", string,"\n")
    if not params["activate"]:
        return string
    original_string = string
@@ -477,65 +500,94 @@ def output_modifier(string, state):
    if process_lock.acquire(blocking=False):
        try:
            if params["narrator_enabled"]:
-                processed_string = original_string
+                # print(original_string)
-                processed_string = processed_string.replace("***", "*")
+                # Do Some basic stripping and remove any double CR's
-                processed_string = processed_string.replace("**", "*")
+                processed_string = (
-                processed_string = processed_string.replace(".*", "*")
+                    original_string
-                processed_string = processed_string.replace(".'", "'")
+                    .replace("***", "*")
-                processed_string = processed_string.replace(".&#x27;", "'")
+                    .replace("**", "*")
-                # Remove new lines
+                    .replace("\n\n", "\n")
-                processed_string = processed_string.replace("\n", " ")
+                )
-                #DEBUG print("PROCESSED STRING IS NOW:", processed_string)
+                # Clean up two asterisks (narrators) being made between a newline, making it one sentence.
-                # Split the processed string into lines
+                processed_string = re.sub(r'\.\*\n\*', '. ', processed_string)
-                lines = processed_string.split("\n")
+                # Clean up a few other bits.
                processed_string = (
                    processed_string
                    .replace("&#x27;", "'")
                    .replace("\n", " ")
                    # Add special characters to the quote so that we can use it to identify things later after its been split
                    .replace('&quot;', '&quot;<')
                    # Capture new conversations which wont have things like &quote in them
                    .replace('"', '&quot;<')
                )
                #capturing another outlier in inital character paragraph
                # print("processed string 1 is:", processed_string)
                processed_string = processed_string.replace('&quot;<. *', '&quot;< *"')
                processed_string = processed_string.replace('< *"', '< *')
                processed_string = processed_string.replace('. *', '< *')
                # print("processed string 2 is:", processed_string)
                # Set up a tracking of the individual wav files.
                audio_files_all_paragraphs = []
-                is_narrator = False
+                # Split the line using &quot; and ".* " (so end of sentences, leaving special characters added to the start of all OTHER sentences, bar possibly the first one if its starting with a *
-                for line in lines:
+                parts = re.split(r'&quot;|\.\*', processed_string)
-                    # Decode HTML entities
+                audio_files_paragraph = []
-                    decoded_text = html.unescape(line)
+                for i, part in enumerate(parts):
-                    # Initialize variables to track narrator and voice parts
+                    # Skip parts that are too short
-                    is_narrator = "*" in line or is_narrator
+                    if len(part.strip()) <= 1:
-                    # Split the line using double quotes
+                        continue
-                    parts = re.split(r'"', decoded_text)
+                    # Figure out which type of line it is, then replace characters as necessary to avoid TTS trying to pronunce them, htmlunescape after. 
-                    audio_files_paragraph = []
+                    # Character will always be a < with a letter immediately after it
-                    for i, part in enumerate(parts):
+                    if '<' in part and '< *' not in part and '<*' not in part and '<  *' not in part and '< ' not in part and '<  ' not in part:
-                        # Skip parts that are too short
+                        cleaned_part = html.unescape(part.replace('<', ''))
-                        if len(part.strip()) <= 1:
+                        voice_to_use = params["voice"]
-                            continue
+                    #Narrator will always be an * or < with an * a position or two after it.
-                        # Determine if it's a narrator or voice part within double quotes
+                    elif '<*' in part or '< *' in part or '<  *' in part or '*' in part:
-                        is_narrator_part = is_narrator if i % 2 == 0 else False
+                        cleaned_part = html.unescape(part.replace('<*', '').replace('< *', '').replace('<  *', '').replace('*', ''))
-                        voice_to_use = (
+                        voice_to_use = params["narrator_voice"]
-                            params["narrator_voice"] if is_narrator_part else params["voice"]
+                    #If the other two dont capture it, aka, the AI gave no * or &quot; on the line, use non_quoted_text_is aka user interface, user can choose Char or Narrator
-                        )
+                    elif non_quoted_text_is:
-                        # DEBUG print(f"THE STRING TO BE USED: {voice_to_use} {part.strip()}")
+                        cleaned_part = html.unescape(part.replace('< ', '').replace('<  ', '').replace('<  ', ''))
-                        # Process the part
+                        voice_to_use = params["voice"]
-                        output_file = Path(
+                    else:
-                            f'{params["output_folder_wav"]}/{state["character_menu"]}_{int(time.time())}_{i}.wav'
+                        cleaned_part = html.unescape(part.replace('< ', '').replace('<  ', '').replace('<  ', ''))
-                        )
+                        voice_to_use = params["narrator_voice"]
-                        output_file_str = output_file.as_posix()
+                    # Check if character name exists and if not, just call it TTSOUT_
-                        generate_response = send_generate_request(
+                    if "character_menu" in state:
-                            part, voice_to_use, language_code, output_file_str
+                        output_file = Path(f'{params["output_folder_wav"]}/{state["character_menu"]}_{int(time.time())}_{i}.wav')
-                        )
+                    else:
-                        audio_path = generate_response.get("data", {}).get("audio_path")
+                        output_file = Path(f'{params["output_folder_wav"]}/TTSOUT_{int(time.time())}_{i}.wav')
-                        audio_files_paragraph.append(audio_path)
+                    # Generate that TTS and output to a file
-                    is_narrator_part = is_narrator if i % 2 == 0 else False
+                    output_file_str = output_file.as_posix()
-                    voice_to_use = (
+                    generate_response = send_generate_request(
-                        params["narrator_voice"] if is_narrator_part else params["voice"]
+                        cleaned_part, voice_to_use, language_code, output_file_str
                    )
-                    # Accumulate audio files within the paragraph
+                    audio_path = generate_response.get("data", {}).get("audio_path")
-                    audio_files_all_paragraphs.extend(audio_files_paragraph)
+                    audio_files_paragraph.append(audio_path)
                # Accumulate audio files within the paragraph
                audio_files_all_paragraphs.extend(audio_files_paragraph)
                # Combine audio files across paragraphs
                final_output_file = combine(
                    audio_files_all_paragraphs, params["output_folder_wav"], state
                )
            else:
-                output_file = Path(
+                processed_string = (
-                    f'{params["output_folder_wav"]}/{state["character_menu"]}_{int(time.time())}.wav'
+                    original_string
                    .replace("***", "")
                    .replace("**", "")
                    .replace("*", "")
                    .replace("\n\n", "\n")
                    .replace("&#x27;", "'")
                )
                cleaned_part = html.unescape(processed_string)                
                # Process the part and give it a non-character name if being used vai API or standalone.
                if "character_menu" in state:
                    output_file = Path(f'{params["output_folder_wav"]}/{state["character_menu"]}_{int(time.time())}.wav')
                else:
                    output_file = Path(f'{params["output_folder_wav"]}/TTSOUT_{int(time.time())}.wav')
                output_file_str = output_file.as_posix()
                output_file = get_output_filename(state)
                generate_response = send_generate_request(
-                    string, params["voice"], language_code, output_file_str
+                    cleaned_part, params["voice"], language_code, output_file_str
                )
                audio_path = generate_response.get("data", {}).get("audio_path")
                final_output_file = audio_path
@@ -613,13 +665,18 @@ def state_modifier(state):
    state["stream"] = False
    return state
 def update_narrator_enabled(value):
    if value == "Enabled":
        params["narrator_enabled"] = True
    elif value == "Disabled":
        params["narrator_enabled"] = False
 def update_non_quoted_text_is(value):
    global non_quoted_text_is
    if value == "Narrator":
        non_quoted_text_is = False
    elif value == "Char":
        non_quoted_text_is = True
 def input_modifier(string, state):
    if not params["activate"]:
@@ -628,7 +685,6 @@ def input_modifier(string, state):
    shared.processing_message = "*Is recording a voice message...*"
    return string
 def custom_css():
    path_to_css = Path(f"{this_dir}/style.css")
    return open(path_to_css, "r").read()
@@ -702,21 +758,28 @@ def ui():
                    },
                    "refresh-button",
                )
-            narrator_enabled_gr = gr.Radio(
+                narrator_enabled_gr = gr.Radio(
-                choices={"Enabled": "true", "Disabled": "false"},
+                    choices={"Enabled": "true", "Disabled": "false"},
-                label="Narrator Activation",
+                    label="Narrator Activation",
-                value="Enabled" if gr_narrator_enabled == "true" else "Disabled",
+                    value="Enabled" if params.get("narrator_enabled") else "Disabled",
-            )
+                )
                non_quoted_text_is_gr = gr.Radio(
                    choices={"Char": "true", "Narrator": "false"},
                    label='Text NOT inside of * or " is',
                    value="Char" if non_quoted_text_is else "Narrator",
                )
        with gr.Row():
            low_vram = gr.Checkbox(
                value=params["low_vram"], label="Low VRAM mode (Read NOTE)"
            )
            low_vram_play = gr.HTML(visible=False)
            deepspeed_checkbox = gr.Checkbox(
                value=params["deepspeed_activate"],
                label="Activate DeepSpeed (Read NOTE)",
                visible=deepspeed_installed,
            )
            deepspeed_checkbox_play = gr.HTML(visible=False)
        with gr.Row():
            tts_radio_buttons = gr.Radio(
@@ -724,6 +787,7 @@ def ui():
                label="Select TTS Generation Method (Read NOTE)",
                value=gr_modelchoice,  # Set the default value
            )
            tts_radio_buttons_play = gr.HTML(visible=False)
            explanation_text = gr.HTML(
                f"<p>NOTE: Switching Model Type, Low VRAM & DeepSpeed takes 15 seconds. Each TTS generation method has a slightly different sound. DeepSpeed checkbox is only visible if DeepSpeed is present. Readme & Settings: <a href='http://{params['ip_address']}:{params['port_number']}'>http://{params['ip_address']}:{params['port_number']}</a>"
            )
@@ -796,9 +860,9 @@ def ui():
    activate.change(lambda x: params.update({"activate": x}), activate, None)
    autoplay.change(lambda x: params.update({"autoplay": x}), autoplay, None)
    low_vram.change(lambda x: params.update({"low_vram": x}), low_vram, None)
-    low_vram.change(lambda x: send_lowvram_request(x), low_vram, None)
+    low_vram.change(lambda x: send_lowvram_request(x), low_vram, low_vram_play, None)
-    tts_radio_buttons.change(send_reload_request, tts_radio_buttons, None)
+    tts_radio_buttons.change(send_reload_request, tts_radio_buttons, tts_radio_buttons_play, None)
-    deepspeed_checkbox.change(send_deepspeed_request, deepspeed_checkbox, None)
+    deepspeed_checkbox.change(send_deepspeed_request, deepspeed_checkbox, deepspeed_checkbox_play, None)
    remove_trailing_dots.change(
        lambda x: params.update({"remove_trailing_dots": x}), remove_trailing_dots, None
    )
@@ -807,9 +871,8 @@ def ui():
    # Narrator selection actions
    narrator_enabled_gr.change(update_narrator_enabled, narrator_enabled_gr, None)
-    narrator_voice_gr.change(
+    non_quoted_text_is_gr.change(update_non_quoted_text_is, non_quoted_text_is_gr, None)
-        lambda x: params.update({"narrator_voice": x}), narrator_voice_gr, None
+    narrator_voice_gr.change(lambda x: params.update({"narrator_voice": x}), narrator_voice_gr, None)
    )
    # Play preview
    preview_text.submit(voice_preview, preview_text, preview_audio)
@@ -1,5 +1,4 @@
 import json
 import random
 import time
 import os
 from pathlib import Path
@@ -15,13 +14,13 @@ from fastapi import (
    FastAPI,
    Form,
    Request,
    HTTPException,
    BackgroundTasks,
    Response,
    Depends,
 )
-from fastapi.responses import JSONResponse, HTMLResponse, RedirectResponse
+from fastapi.responses import JSONResponse, HTMLResponse, RedirectResponse, FileResponse
 from fastapi.templating import Jinja2Templates
 from fastapi.staticfiles import StaticFiles
 from jinja2 import Template
 from contextlib import asynccontextmanager
 ###########################
@@ -188,9 +187,7 @@ async def api_manual_load_model():
        )
        model = TTS(
            model_path=modeldownload_base_path / modeldownload_model_path,
-            config_path=modeldownload_base_path
+            config_path=modeldownload_base_path / modeldownload_model_path / "config.json",
            / modeldownload_model_path
            / "config.json",
        ).to(device)
    return model
@@ -202,6 +199,7 @@ async def xtts_manual_load_model():
    # check to see if a custom path has been set in modeldownload.json and use that path to load the model if so
    if str(modeldownload_base_path) == "models":
        config_path = this_dir / "models" / modeldownload_model_path / "config.json"
        vocab_path_dir = this_dir / "models" / modeldownload_model_path / "vocab.json"
        checkpoint_dir = this_dir / "models" / modeldownload_model_path
    else:
        print(
@@ -209,12 +207,14 @@ async def xtts_manual_load_model():
            modeldownload_base_path / modeldownload_model_path,
        )
        config_path = modeldownload_base_path / modeldownload_model_path / "config.json"
        vocab_path_dir = modeldownload_base_path / modeldownload_model_path / "vocab.json"
        checkpoint_dir = modeldownload_base_path / modeldownload_model_path
    config.load_json(str(config_path))
    model = Xtts.init_from_config(config)
    model.load_checkpoint(
        config,
        checkpoint_dir=str(checkpoint_dir),
        vocab_path=str(vocab_path_dir),
        use_deepspeed=params["deepspeed_activate"],
    )
    model.to(device)
@@ -511,7 +511,6 @@ def list_files(directory):
    ]
    return files
 #############################
 #### JSON CONFIG UPDATER ####
 #############################
@@ -519,7 +518,6 @@ def list_files(directory):
 # Create an instance of Jinja2Templates for rendering HTML templates
 templates = Jinja2Templates(directory=this_dir / "templates")
 # Create a dependency to get the current JSON data
 def get_json_data():
    with open(this_dir / "config.json", "r") as json_file:
@@ -542,6 +540,8 @@ async def get_settings(request: Request):
        },
    )
 # Define an endpoint to serve static files
 app.mount("/static", StaticFiles(directory=str(this_dir / "templates")), name="static")
@app.post("/update-settings")
 async def update_settings(
@@ -598,6 +598,25 @@ async def update_settings(
    return RedirectResponse(url="/settings", status_code=303)
 ##################################
 #### SETTINGS PAGE DEMO VOICE ####
 ##################################
 # Submit demo post here
@app.post("/tts-demo-request", response_class=JSONResponse)
 async def tts_demo_request(request: Request, text: str = Form(...), voice: str = Form(...), language: str = Form(...), output_file: str = Form(...)):
    try:
        output_file_path = this_dir / "outputs" / output_file
        await generate_audio(text, voice, language, output_file_path)
        return JSONResponse(content={"output_file_path": str(output_file)}, status_code=200)
    except Exception as e:
        return JSONResponse(content={"error": "An error occurred"}, status_code=500)
 # Gives web access to the output files
@app.get("/audio/{filename}")
 async def get_audio(filename: str):
    audio_path = this_dir / "outputs" / filename
    return FileResponse(audio_path)
 #############################################################
 #### DOCUMENTATION - README ETC - PRESENTED AS A WEBPAGE ####
 #############################################################
@@ -699,6 +718,64 @@ simple_webpage = """
        .option-b {
            color: red;
            font-weight: bold;
        }
                /* New styles for TTS Request Page */
        #container {
            max-width: 600px;
            margin: 50px auto;
            padding: 20px;
            background-color: #fff;
            border-radius: 8px;
            box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
        }
        form {
            display: flex;
            flex-direction: column;
            gap: 4px;
        }
        label {
            font-weight: bold;
            font-size: 14px;
            padding: 2px;
        }
        textarea, input, select, button {
            padding: 4px;
            font-size: 14px;
            border: 1px solid #ccc;
            border-radius: 4px;
            padding: 4px;
        }
        button {
            background-color: #4caf50;
            color: #fff;
            cursor: pointer;
            transition: background-color 0.3s;
        }
        button:hover {
            background-color: #45a049;
        }
        p {
            margin-top: 20px;
        }
        #outputFilePath {
            font-weight: bold;
        }
        #audioSource {
        display: block;
        margin: auto;
        }
        #outputFilePath {
        display: none;
        }
    </style>
@@ -706,16 +783,17 @@ simple_webpage = """
 <body>
    <h1 id="toc">AllTalk TTS for Text generation webUI</h1>
-    <p>If you are looking for your <b>Text generation webUI</b> webpage, try visiting <a href="http://127.0.0.1:7860" target="_blank">http://127.0.0.1:7860</a>.</p>
+    <p><b>Text generation webUI</b> webpage <a href="http://{{ params["ip_address"] }}:7860" target="_blank">here</a> and documentation <a href="https://github.com/oobabooga/text-generation-webui/wiki" target="_blank">here</a></p>
-    <p><b>Text generation webUI</b> documentation and wiki <a href="https://github.com/oobabooga/text-generation-webui/wiki" target="_blank">https://github.com/oobabooga/text-generation-webui/wiki</a>.</p>
+    <p><b>AllTalk Github</b> <a href="https://github.com/erew123/alltalk_tts" target="_blank">here</a> 
-    <iframe src="http://127.0.0.1:7851/settings" width="100%" height="500" frameborder="0" style="margin: 0; padding: 0;"></iframe>
+    <iframe src="http://{{ params["ip_address"] }}:{{ params["port_number"] }}/settings" width="100%" height="500" frameborder="0" style="margin: 0; padding: 0;"></iframe>
    <h3>Table of Contents</h3>
    <ul>
        <li><a href="#getting-started">Getting Started with AllTalk TTS</a></li>
        <li><a href="#server-information">Server Information</a></li>
        <li><a href="#using-voice-samples">Using Voice Samples</a></li>
        <li><a href="#text-not-inside">Text Not inside function</a></li>
        <li><a href="#local-model-temperature-and-repetition-settings">Local Model temperature and repetition settings</a></li>
        <li><a href="#where-are-the-outputs-stored">Automatic output wav file deletion</a></li>
        <li><a href="#low-vram-option-overview">Low VRAM Overview</a></li>
@@ -724,6 +802,7 @@ simple_webpage = """
        <li><a href="#other-features">Other Features of AllTalk_TTS Extension for Text generation webUI</a></li>
        <li><a href="#TTSmodels">TTS Models/Methods</a></li>
        <li><a href="#customTTSmodels">Custom TTS Models and Model path</a></li>
        <li><a href="#demotesttts">Demo/Test TTS output</a></li>
        <li><a href="#curl-commands">CURL Commands</a></li>
        <li><a href="#configuration-details">Configuration Details</a></li>
        <li><a href="#debugging-and-tts-generation-information">Debugging and TTS Generation Information</a></li>
@@ -733,11 +812,13 @@ simple_webpage = """
    <h2 id="getting-started">Getting Started</h2>
    <p>AllTalk is a web interface, based around the Coqui TTS speech generation system. To generate TTS, you can use the provided interface or interact with the server using CURL commands. Below are some details and examples:</p>
    <p><b>Note:</b> When loading up a new character in Text generation webUI it may look like nothing is happening for 20-30 seconds. Its actually processing the introduction section of the text and once that is completed, it will appear. You can see the activity occuring in the console window. Refreshing the page multiple times will try force the TTS engine to keep re-generating the text, so please just wait and check the console if needed. </p>
    <p><b>IMPORTANT Note:</b> Ensure that your RP character card has asterisks around anything for the narration and double quotes around anything spoken. There is a complication ONLY with the greeting card so, ensuring it has the correct use of quotes and asterisks will help make sure that the greeting card sounds correct. I will aim to address this issue in a future update.</p>
    <p>Details on how to edit your greeting card, and where I'm at with solving this, can be found here <a href="https://github.com/erew123/alltalk_tts/tree/main#the-one-thing-i-cant-easily-work-around" target="_blank">erew123's Github</a> and you can look here at an example of how text should be formatted with quotes and asterisks <a href="#text-not-inside">Text Not inside function</a></p>
    <h3 id="server-information">Server Information</h3>
    <ul>
-        <li>Base URL: <code>http://127.0.0.1:7851</code></li>
+        <li>Base URL: <code>http://{{ params["ip_address"] }}:{{ params["port_number"] }}</code></li>
-        <li>Server Status: <code><a href="http://127.0.0.1:7851/ready">http://127.0.0.1:7851/ready</a></code></li>
+        <li>Server Status: <code><a href="http://{{ params["ip_address"] }}:{{ params["port_number"] }}/ready">http://{{ params["ip_address"] }}:{{ params["port_number"] }}/ready</a></code></li>
    </ul>
    <p><a href="#toc">Back to top of page</a></p>
@@ -751,7 +832,7 @@ simple_webpage = """
    <p>This extension will download the 2.0.2 model to <b>/extensions/alltalk_tts/models/</b></p>
    <p>This TTS engine will also download the latest available model and store it wherever it normally stores it for your OS (Windows/Linux/Mac).</p>
    <h4>How do I create a new voice sample?</h4>
-    <p>To create a new voice sample you need to make a <b>wav</b> file that is <b>22050Hz</b>, <b>Mono</b>, <b>16 bit</b> and between <b>6 to 12 seconds long</b>, though 7 to 9 seconds is usually good.</p>
+    <p>To create a new voice sample you need to make a <b>wav</b> file that is <b>22050Hz</b>, <b>Mono</b>, <b>16 bit</b> and between <b>6 to 30 seconds long</b>, though 8 to 10 seconds is usually good enough. The model <b>can handle up to a 30 second sample</b>, however Ive not noticed any improvemnt in voice output from a much shorter clip.</p>
    <p>You want to find a nice clear selection of audio, so lets say you wanted to clone your favourite celebrity. You may go looking for an interview where they are talking. Pay close attention to the audio you are listening to and trying to sample, are there noises in the backgroud, hiss on the soundtrack, a low humm, some quiet music playing or something? The better quality the audio the better the result. Dont forget, the AI that processes the sounds can hear everything, all those liitle noises, and it will use them in the voice its trying to recreate. </p>
    <p>Try make your clip one of nice flowing speech, like the included example files. No big pauses, gaps or other sounds. Preferably one that the person you are trying to copy will show a little vocal range and emotion in their voice. Also, try to avoid a clip starting or ending with breathy sounds (breathing in/out etc).</p>
    <h4>Generating the sample!</h4>
@@ -768,6 +849,15 @@ simple_webpage = """
    </ul>
    <p><a href="#toc">Back to top of page</a></p>
    <h3 id="text-not-inside"><strong>Text Not inside function</strong></h3>
    <p>When using the Narrator function, most AI models should be using asterisks or double quotes to differentiate between the Narrator or the Character, however, many models sometimes switch between using asterisks and double quotes or nothing at all. This leaves a bit of a mess because sometimes that text is narration and sometimes its the character talking and there's no clear way to know where to split sentences. There is no 100&percnt; solution at the moment.</p>
    <p>Most models usually lean more way than the other as to it is narration or the character talking when it does this. So you can use the "<b>Text NOT inside of * or &quot; is</b>" function to decide what the TTS engine should do in situations like this. It's an either/or situation. It will either use the Character voice or the narrator voice when text is not inside asterisks or double quotes.</p>
    <div style="text-align: center;">
        <img src="/static/textnotinside.jpg" alt="When the AI doesnt use an asterisk or a quote">
    </div>
    <p><a href="#toc">Back to top of page</a></p>
    <h3 id="local-model-temperature-and-repetition-settings"><strong>Local Model Temperature and Repetition Settings</strong></h3>
    <p><strong>Caution:</strong> It is recommended not to modify these settings unless you fully comprehend their effects. A general overview is provided below for reference.</p>
@@ -830,13 +920,20 @@ simple_webpage = """
    <h3 id="low-vram-option-overview"><strong>Low VRAM Overview:</strong></h3>
    <p>The Low VRAM option is a crucial feature designed to enhance performance under constrained Video Random Access Memory (VRAM) conditions, as the TTS models require 2GB-3GB of VRAM to run effectively. This feature strategically manages the relocation of the Text-to-Speech (TTS) model between your system's Random Access Memory (RAM) and VRAM, moving it between the two on the fly.</p>
    <p>When you dont have enough VRAM free after loading your LLM model into your VRAM (Normal Mode example below), you can see that with so little working space, your GPU will have to swap in and out bits of the TTS model, which causes horrible slowdown.</p>
    <p><b>Note: </b>An Nvidia Graphics card is required for LowVRAM, as you will be using system memory for the models otherwise.</p>
    <h4>How It Works:</h4>
-    <p>The Low VRAM option intelligently orchestrates the relocation of the entire TTS model. When the TTS engine requires VRAM for processing, the entire model seamlessly moves into VRAM, causing your LLM to unload/displace some layers, ensuring optimal performance of the TTS engine.</p>
+    <p>The Low VRAM mode intelligently orchestrates the relocation of the entire TTS model. When the TTS engine requires VRAM for processing, the entire model seamlessly moves into VRAM, causing your LLM to unload/displace some layers, ensuring optimal performance of the TTS engine.</p>
    <p>The TTS model is fully loaded into VRAM, facilitating uninterrupted and efficient TTS generation, creating contiguous space for the TTS model and significantly accelerating TTS processing, especially for long paragraphs. Post-TTS processing, the model promptly moves back to RAM, freeing up VRAM space for your Language Model (LLM) to load back in the missing layers. This adds about 1-2 seconds to both text generation by the LLM and the TTS engine.</p>
    <p>By transferring the entire model between RAM and VRAM, the Low VRAM option avoids fragmentation, ensuring the TTS model remains cohesive and accessible.</p>
    <p>This creates a TTS generation performance Boost for Low VRAM Users and is particularly beneficial for users with less than 2GB of free VRAM after loading their LLM, delivering a substantial 5-10x improvement in TTS generation speed.</p>
    <div style="text-align: center;">
        <img src="/static/lowvrammode.png" alt="How Low VRAM Works">
    </div>
    <p><a href="#toc">Back to top of page</a></p>
    <h3 id="deepspeed-simplified"><strong>DeepSpeed Simplified:</strong></h3>
@@ -859,6 +956,10 @@ simple_webpage = """
        <li><strong>Smart Resource Use:</strong> Uses your computer's resources smartly, getting the most out of your hardware.</li>
    </ul>
    <div style="text-align: center;">
        <img src="/static/deepspeedexample.jpg" alt="DeepSpeed on vs off">
    </div>
    <p><strong>Note:</strong> DeepSpeed only works with the XTTSv2 Local model.</p>
    <p><strong>Note:</strong> Requires Nvidia Cuda Toolkit installation and correct CUDA_HOME path configuration.</p>
@@ -867,7 +968,7 @@ simple_webpage = """
    <p><a href="#toc">Back to top of page</a></p>
    <h3 id="setup_deepspeed">Setup DeepSpeed</h3>
-      <p><strong>Note:</strong> DeepSpeed WILL throw up a load of errors if you have not installed it fully, correctly, or installed the Nvidia Cuda Toolkit and correctly set the CUDA_HOME environment variable or if load Text generation webUI via its start-up scripts.</p>
+      <p><strong>Note:</strong> DeepSpeed/AllTalk may warn if the Nvidia Cuda Toolkit and CUDA_HOME environment variables arent set. On Linux I believe you need these installed, on Windows, if you use the pre-built wheel it seems ok without.</p>
    <h4>Linux - DeepSpeed - Installation Instructions</h4>
    <ol>
@@ -884,64 +985,34 @@ simple_webpage = """
    </ol>
    <p><a href="#toc">Back to top of page</a></p>
-    <h4>Windows - DeepSpeed Version 8.3 & CUDA 11.8 or CUDA 12.1 - Installation Instructions</h4>
+    <h4>Windows - DeepSpeed v11.1 or v11.2 Installation Instructions</h4>
-    <p>Currently DeepSpeed 8.3 is the latest build that works on Windows and it has to run inside a Python 3.9.18 environment. As such, either you need to have your own, already set up Python 3.9.18 environment and have installed the correct Text generation webUI requirements file into it. Or you can create a new Python environment. DeepSpeed version 9.x, 10.x, 11.x, and 12.x will all fail to compile <strong>(on Windows)</strong> for a multitude of reasons. As such, DeepSpeed v8.3 is the most recent build for Windows.</p>
+    <p>Currently/Officially only DeepSpeed v8.3 is installing on Windows, due to the broken installation routine by Microsoft, however, between myself and <a href="https://github.com/S95Sedan" target="_blank">S95Sedan</a> its now possible to install DeepSpeed v11.1 or v11.2 on Windows.</p>
    <h4>DeepSpeed on Windows - <span class="option-a">Option A</span></h4>
-    <p><span class="option-a">Option A</span> is to fully go through the process of compiling DeepSpeed yourself. This takes a longer time to perform, but you can be sure it's set up for your system correctly.</p>
+    <p>This is to use the pre built DeepSpeed v11.1 wheel file. This is quite a quick process and should work for 99&percnt; of people.</p>
-    <h4>DeepSpeed on Windows - <span class="option-b">Option B</span></h4>
+    <p><strong>Note:</strong> In my tests, with this method you will <strong>not</strong> need to install the Nvidia CUDA toolkit to make this work, but AllTalk may warn you when starting DeepSpeed that it doesn't see the CUDA Toolkit; however, it works fine for TTS purposes.</p>
    <p><span class="option-b">Option B</span> is to download someone else's precompiled wheels. If you can find someone's precompiled wheel file, you only need to follow <span class="option-b">Option B</span>.</p>
    <ol>
-    <li>(<span class="option-a">OPTION A</span>) Download the 8.3 release of <a href="https://github.com/microsoft/DeepSpeed/releases/tag/v0.8.3">DeepSpeed</a> and extract it to a folder.</li>
+    <li>Download the file <a href="https://drive.google.com/file/d/1PFsf6uSPY5Cb4o9VxiZ7DLv-j35L7Y41/view?usp=sharing" target="_blank">deepspeed-0.11.1+e9503fe-cp311-cp311-win_amd64.whl</a> by clicking the <strong>download</strong> icon at the top right of the screen and save the file it inside your <strong>text-generation-webui</strong> folder.</li>
    <li>(<span class="option-a">OPTION A</span>) Install Visual C++ build tools, such as <a href="https://learn.microsoft.com/en-us/visualstudio/releases/2019/redistribution#vs2019-download">VS2019 C++ x64/x86</a> build tools.</li>
    <li>(<span class="option-a">OPTION A</span> <span class="option-b">and B</span>) Download and install the <a href="https://developer.nvidia.com/cuda-toolkit-archive">Nvidia Cuda Toolkit 11.8 OR 12.1</a> <b>(11.8 or 12.1)</b></li>
    <li>(<span class="option-a">OPTION A</span> <span class="option-b">and B</span>) Edit your Windows environment variables to ensure that CUDA_HOME and CUDA_PATH are set to your Nvidia Cuda Toolkit path. (The folder <strong>above</strong> the bin folder that <strong>nvcc.exe</strong> is installed in). Examples are:<br>
        <br>CUDA 11.8<br><br>
        <code>CUDA_HOME=C:&#92;Program Files&#92;NVIDIA GPU Computing Toolkit&#92;CUDA&#92;v11.8</code><br>
        <code>CUDA_PATH=C:&#92;Program Files&#92;NVIDIA GPU Computing Toolkit&#92;CUDA&#92;v11.8</code><br>
        <br>
        CUDA 12.1<br><br>
        <code>CUDA_HOME=C:&#92;Program Files&#92;NVIDIA GPU Computing Toolkit&#92;CUDA&#92;v12.1</code><br>
        <code>CUDA_PATH=C:&#92;Program Files&#92;NVIDIA GPU Computing Toolkit&#92;CUDA&#92;v12.1</code><br><br>
-        It is possible to permanently set these or use the <code>set</code> command at the command prompt when inside your python environment e.g. <b>set CUDA_HOME=C:&#92;Program Files&#92;NVIDIA GPU Computing Toolkit&#92;CUDA&#92;v12.1'</b>. You can then just type <code>set</code> again, and it will show you the list of all currently set environment variables so that you can confirm you have them set correctly.
+    <li>At a command prompt window, move into your <strong>text-generation-webui folder</strong>, you can now start the Python environment for text-generation-webui:
-    </li>
+        <br><code>cmd_windows.bat</code></li>
-    <li>(<span class="option-a">OPTION A</span> <span class="option-b">and B</span>) <strong>Currently Python versions 3.9.x are supported</strong>. If you do not have a Python environment already created, you can install <a href="https://docs.conda.io/projects/miniconda/en/latest/miniconda-install.html">Miniconda</a>, then at a command prompt, create and activate your environment with:
+
-        <br><br><code>conda create -n pythonenv python=3.9.18</code>
+    <li>With the file that you saved in the <strong>text-generation-webui folder</strong>, you now type the following:
-        <br><br><code>activate pythonenv</code>
+        <br><code>pip install "deepspeed-0.11.1+e9503fe-cp311-cp311-win_amd64.whl"</code></li>
-    </li>
+
-    <li>(<span class="option-a">OPTION A</span> <span class="option-b">and B</span>) Launch the Command Prompt <strong>cmd</strong> with <strong>Administrator privilege</strong> as it requires admin to allow creating symlink folders.</li>
+    <li>This should install through cleanly and you should now have DeepSpeed v11.1 installed within the Python 3.11 environment of text-generation-webui.</li>
-    <li>(<span class="option-a">OPTION A</span> <span class="option-b">and B</span>) Install <a href="https://pytorch.org/get-started/previous-versions/#v210">PyTorch</a>, 2.1.0 with CUDA 11.8 or CUDA 12.1 into your Python 3.9.x environment e.g.<br><br>
+
-        CUDA 11.8<br>
+    <li>When you start up text-generation-webui, and AllTalk starts, you should see <strong>[AllTalk Startup] DeepSpeed Detected</strong></li>
-        <br><code>activate pythonenv</code> (activate your python environment)<br>
+
-        <br><code>conda install pytorch==2.1.0 torchvision==0.16.0 torchaudio==2.1.0 pytorch-cuda=11.8 -c pytorch -c nvidia</code><br>
+    <li>Within AllTalk, you will now have a checkbox for <strong>Activate DeepSpeed</strong> though remember you can only change <strong>1x setting every 15 or so seconds</strong>, so don't try to activate DeepSpeed <strong>and</strong> LowVRAM/Change your model simultaneously. Do one of those, wait 15-20 seconds until the change is confirmed in the terminal/command prompt, then you can change the other. When you are happy it works, you can set the default start-up settings in the settings page.</li>
        <br>CUDA 12.1<br>
        <br><code>activate pythonenv</code> (activate your python environment)<br>
        <br><code>conda install pytorch==2.1.0 torchvision==0.16.0 torchaudio==2.1.0 pytorch-cuda=12.1 -c pytorch -c nvidia</code>
    </li>
    <li>(<span class="option-a">OPTION A</span> <span class="option-b">and B</span>) In your python environment double-check that your <b>CUDA_HOME</b> and <b>CUDA_PATH</b> are still pointing to the correct location.
        <br>Type <code>set</code> to list and check the Windows environment variables. Refer to step 4 if not.
    </li>
    <li>(<span class="option-a">OPTION A</span> <span class="option-b">and B</span>) Navigate to your deepspeed folder in the Command Prompt:<code>cd c:\deepspeed</code> (wherever you extracted it to)
    </li>
    <li>(<span class="option-a">OPTIONAL A</span>) You can try using the <strong>build_win.bat</strong> however you may need to set options to not install certain features e.g. <strong>set DS_BUILD_AIO=0</strong> <a href="https://www.deepspeed.ai/tutorials/advanced-install/#pre-install-deepspeed-ops">https://www.deepspeed.ai/tutorials/advanced-install/#pre-install-deepspeed-ops</a>. Type the following into the command prompt:
        <br><br><code>set DS_BUILD_AIO=0</code>
        <br><br><code>set DS_BUILD_SPARSE_ATTN=0</code>
        <br><br><code>set DS_BUILD_EVOFORMER_ATTN=0</code>
        <br><br><code>build_win.bat</code> (This will start building your wheel file and may take a while)
    </li>
    <li>(<span class="option-a">OPTION A</span> <span class="option-b">and B</span>) Now <strong>cd dist</strong> to go into your <strong>dist</strong> folder or wherever you have put your downloaded wheel file and you can now <strong>pip install deepspeed-_YOURFILENAME_.whl</strong> (or whatever your WHL file is called).
        <br><strong><br>Note:</strong> You can run <code>ds_report</code> when you have installed DeepSpeed on your system to see if it is working correctly
    </li>
    <li>(<span class="option-a">OPTION A</span> <span class="option-b">and B</span>) Move into your Text generation webUI folder e.g. <code>cd Text generation webUI</code></li>
    <li>(<span class="option-a">OPTION A</span> <span class="option-b">and B</span>) Don't forget to have <code>pip install -r requirements.txt</code> or whichever requirements file you want to use for your system, within the Text generation webUI folder.</li>
    <li>(<span class="option-a">OPTION A</span> <span class="option-b">and B</span>) Start the text-generation=web-ui with Python e.g. <code>python server.py</code> ensuring to activate your extensions.
        <br><strong><br>Note:</strong> you can't use <strong>start_windows.bat</strong> or <strong>cmd_windows.bat</strong> because it will use a different environment, incorrect version of Python and it will overwrite the CUDA_HOME path.
    </li>
    </ol>
    <h4>DeepSpeed on Windows - <span class="option-b">Option B</span></h4>
    <p>Due to the complexity of this, Im not keeping the instructions within this document as it would be too complex to format.</p>
    <p>As such, the instuctions can be found on this <a href="https://github.com/erew123/alltalk_tts?tab=readme-ov-file#-option-2---a-bit-more-complicated" target="_blank">link</a>
    <p><a href="#toc">Back to top of page</a></p>
    <h2 id="other-features"><strong>Other Features of AllTalk TTS Extension for Text generation webUI</strong></h2>
@@ -961,9 +1032,78 @@ simple_webpage = """
    </ul>
    <p><a href="#toc">Back to top of page</a></p>
    <h3 id="demotesttts">Demo/Test TTS</h3>
    <div id="container">
        <form method="post" action="/tts-demo-request" id="ttsForm">
            <label for="text">Text:</label>
            <textarea id="text" name="text" rows="4" required></textarea>
            <label for="voice">Voice:</label>
            <input type="text" id="voice" name="voice" value="female_01.wav" required>
            <label for="language">Language:</label>
            <select id="language" name="language" value="English" required>
                <option value="en" selected>English</option>
                <option value="ar">Arabic</option>
                <option value="zh-cn">Chinese</option>
                <option value="cs">Czech</option>
                <option value="nl">Dutch</option>
                <option value="fr">French</option>
                <option value="de">German</option>
                <option value="hu">Hungarian</option>
                <option value="it">Italian</option>
                <option value="ja">Japanese</option>
                <option value="ko">Korean</option>
                <option value="pl">Polish</option>
                <option value="pt">Portuguese</option>
                <option value="ru">Russian</option>
                <option value="es">Spanish</option>
                <option value="tr">Turkish</option>
            </select>
            <label for="outputFile">Output File:</label>
            <input type="text" id="outputFile" name="output_file" value="demo_output.wav" required>
            <!-- Audio player with autoplay -->
            <audio controls autoplay id="audioSource">
                <source type="audio/wav" />
                Your browser does not support the audio element.
            </audio>
            <span id="outputFilePath" style="height: 0px;">{{ output_file_path }}</span>
            <button type="submit">Generate TTS</button>
        </form>
    </div>
    <script>
        const form = document.getElementById('ttsForm');
        form.addEventListener('submit', async (event) => {
            event.preventDefault();
            const formData = new FormData(form);
            const response = await fetch('/tts-demo-request', {
                method: 'POST',
                body: formData
            });
            const result = await response.json();
            // Update the generated audio file path
            const outputFilePath = document.getElementById('outputFilePath');
            outputFilePath.textContent = result.output_file_path;
            // Update the audio player source
            const audioPlayer = document.getElementById('audioSource');
            audioPlayer.src = `/audio/${result.output_file_path}`;
            audioPlayer.load(); // Reload the audio player
            audioPlayer.play(); // Play the audio;
        });
    </script>
    <p><a href="#toc">Back to top of page</a></p>
    <h3 id="customTTSmodels">Custom TTS Models and Model path</h3>
-    <p>Its possible to set a custom model for the <strong>API Local</strong> and<strong> XTTSv2</strong> Local methods, or ideed point it at the same model that <strong>API TTS</strong> uses (wherever it is stored on your OS of choice). Many people did not like the sound quality of the Coqui <strong>2.0.<span style="color:#e74c3c">3</span></strong> model, and as such the AllTalk tts extension downloads the <strong>2.0.<span style="color:#2980b9">2</span></strong> model seperately to the <strong>2.0.<span style="color:#e74c3c">3</span></strong> model that TTS service downloads and manages. Typically the <strong>2.0.<span style="color:#2980b9">2</span></strong> model is stored in your <strong>/extensions/alltalk_tts/models</strong> folder and it is always downloaded on first start-up of the&nbsp;AllTalk_tts extension. However, you may either want to use a custom model version of your choosing, or point it to a different path on your system, or even point it so that&nbsp;API Local and XTTSv2 <strong>both</strong> use the same model that API TTS is using.</p>
+    <p>Its possible to set a custom model for the <strong>API Local</strong> and<strong> XTTSv2</strong> Local methods, or indeed point it at the same model that <strong>API TTS</strong> uses (wherever it is stored on your OS of choice). Many people did not like the sound quality of the Coqui <strong>2.0.<span style="color:#e74c3c">3</span></strong> model, and as such the AllTalk tts extension downloads the <strong>2.0.<span style="color:#2980b9">2</span></strong> model seperately to the <strong>2.0.<span style="color:#e74c3c">3</span></strong> model that TTS service downloads and manages. Typically the <strong>2.0.<span style="color:#2980b9">2</span></strong> model is stored in your <strong>/extensions/alltalk_tts/models</strong> folder and it is always downloaded on first start-up of the&nbsp;AllTalk_tts extension. However, you may either want to use a custom model version of your choosing, or point it to a different path on your system, or even point it so that&nbsp;API Local and XTTSv2 <strong>both</strong> use the same model that API TTS is using.</p>
    <p>If you do choose to change the location there are a couple of things to note.&nbsp;</p>
    <p>- The folder you place the model in, <strong><span style="color:#e74c3c">cannot</span></strong> be called &quot;<strong>models</strong>&quot;. This name is reserved solely for the system to identify you are or are not using a custom model.</p>
    <p>- On each startup, the&nbsp;AllTalk tts extension will check the custom location and if it does not exist, it will create it and download the files it needs. It will also re download any missing files in that location that are needed for the model to function.</p>
@@ -1089,28 +1229,33 @@ simple_webpage = """
    <h3>Thanks to</h3>
    <ul>
        <li><a href="https://github.com/daswer123" target="_blank">daswer123 GitHub Profile</a> (Assistance with cuda to cpu moving)</li>
        <li><a href="https://github.com/S95Sedan" target="_blank">S95Sedan GitHub Profile</a> (Editing the Microsoft DeepSpeed v11.x installation files so they work)</li>
        <li><a href="https://github.com/kanttouchthis" target="_blank">kanttouchthis GitHub Profile</a> (Portions of orginal Coquii_TTS extension)</li>
        <li><a href="https://github.com/kanttouchthis" target="_blank">Wuzzooy GitHub Profile</a> (Trying out the code while in development)</li>
    </ul>
    <p><a href="#toc">Back to top of page</a></p>
 </body>
 </html>
 """
 ###################################################
 #### Webserver Startup & Initial model Loading ####
 ###################################################
 # Create a Jinja2 template object
 template = Template(simple_webpage)
 # Render the template with the dynamic values
 rendered_html = template.render(params=params)
@app.get("/ready")
 async def ready():
    return Response("Ready endpoint")
@app.get("/")
 async def read_root():
-    return HTMLResponse(content=simple_webpage, status_code=200)
+    return HTMLResponse(content=rendered_html, status_code=200)
 # Start Uvicorn Webserver
 host_parameter = {params["ip_address"]}
Author	SHA1	Message	Date
erew123	34621ebabc	Update README.md	2023-12-17 06:41:26 +00:00
erew123	98dae5cff8	Update README.md remind people to update the requirements when updating.	2023-12-17 06:40:49 +00:00
erew123	5b174bab35	pull params correctly into HTML Only affects custom IP/ports. No changes otherwise.	2023-12-17 05:45:51 +00:00
erew123	dbb51bdb41	Update README.md	2023-12-16 22:04:42 +00:00
erew123	91330111fa	Update README.md	2023-12-16 22:02:46 +00:00
erew123	242ba3dc9f	Update README.md	2023-12-16 21:37:01 +00:00
erew123	aae9568b72	Update README.md	2023-12-16 21:30:45 +00:00
erew123	9d002de65a	Update README.md	2023-12-16 21:21:39 +00:00
erew123	48ce2421a4	Update .gitignore	2023-12-16 21:18:03 +00:00
erew123	2d095063a3	Update README.md	2023-12-16 20:30:13 +00:00
erew123	dbacc3d352	Update README.md	2023-12-16 20:28:13 +00:00
erew123	49489bda89	Update README.md	2023-12-16 20:25:35 +00:00
erew123	5260df4e2d	Update README.md	2023-12-16 20:11:25 +00:00
erew123	a2efb1abd8	Update README.md	2023-12-16 20:01:10 +00:00
erew123	006b5a7f58	Update README.md	2023-12-16 19:58:32 +00:00
erew123	5720b56dea	Update README.md	2023-12-16 19:52:46 +00:00
erew123	e0f10db484	Update README.md	2023-12-16 19:45:27 +00:00
erew123	25b14c9c21	Update README.md	2023-12-16 11:01:12 +00:00
erew123	e069c33633	Update README.md Added updating instructions	2023-12-16 11:00:13 +00:00
erew123	7dafb0f32a	Update config.json pulling the default repetition penalty back to 10 now that the vocab if working correctly	2023-12-16 05:24:36 +00:00
erew123	4bcafc16cf	update to v1.5 1 more mistake	2023-12-15 22:39:36 +00:00
erew123	7ba991235b	Update to 1.5 forgot to remove some debugging bits.	2023-12-15 22:35:25 +00:00
erew123	926574e107	Update README.md	2023-12-15 22:28:24 +00:00
erew123	900e52ca27	Update README.md	2023-12-15 22:26:54 +00:00
erew123	bcb5832c4c	Upgrade to v1.5 Added: - new documentation - demo in the documentation - extra API endpoints (yet to be documented) - voice notification on change of model or setting - DeepSpeed update to v11.x and re-documented - improved vocab bringing the XTTSv2 model closer to sounding like the sample, with improved pronunciation and intonation.	2023-12-15 22:07:06 +00:00
erew123	c4c70d6c9c	Update README.md	2023-12-15 07:02:04 +00:00
erew123	c75b0e5cf0	Update README.md	2023-12-15 06:36:02 +00:00
erew123	7583d15c9d	Update README.md	2023-12-14 14:26:05 +00:00
erew123	422b5e7884	Update README.md	2023-12-14 08:42:52 +00:00
erew123	aff4f4ae7d	Update README.md	2023-12-14 08:15:15 +00:00
erew123	a704bb2be3	Update README.md	2023-12-14 08:10:43 +00:00
erew123	9488df662b	Update README.md	2023-12-14 07:59:11 +00:00
erew123	70e6f81663	Update README.md	2023-12-14 07:27:47 +00:00
erew123	4ab2a0c53a	Update README.md	2023-12-14 07:19:56 +00:00
erew123	bebd34b1d8	Update README.md	2023-12-14 07:16:33 +00:00
erew123	7a5b557bb5	Update README.md	2023-12-14 06:56:24 +00:00
erew123	2ec1cbcd43	Update README.md	2023-12-14 06:54:23 +00:00
erew123	281706688f	Update README.md	2023-12-14 06:53:14 +00:00
erew123	7d53dd0d9c	Update README.md	2023-12-14 06:51:36 +00:00
erew123	e1eab6a644	Update README.md	2023-12-14 06:45:39 +00:00
erew123	1c29a8391d	Update README.md DeepSpeed 11.2 instructions	2023-12-14 06:42:03 +00:00
erew123	d9cf355b79	Update README.md	2023-12-13 12:28:04 +00:00
erew123	3762ccd396	Update README.md	2023-12-13 12:19:44 +00:00
erew123	7c8e57d6b3	Update README.md	2023-12-13 12:15:03 +00:00
erew123	e8dde29d20	Documentation update Updated a note in the documentation about correct formatting of RP character greeting cards	2023-12-13 12:11:28 +00:00
erew123	8d7b7018ae	Update README.md	2023-12-13 11:51:12 +00:00
erew123	2262046fd0	Update README.md	2023-12-13 11:50:28 +00:00
erew123	bdc74d3333	Update README.md	2023-12-13 11:42:46 +00:00
erew123	8c8e65fb1d	Fixed outlier on character paragraph. On new chat, character paragraphs, found an outlier possibility where character/narrator wouldn't swap, so performed replace('"<. ', '"< "').replace('< "', '< ') which should capture it.	2023-12-13 10:00:33 +00:00
erew123	fedea7cc09	Update README.md	2023-12-13 06:52:27 +00:00
erew123	992e9625ef	Update README.md	2023-12-13 05:48:58 +00:00
erew123	36f44fce11	Update README.md	2023-12-13 05:39:30 +00:00
erew123	e12cf5f0a2	Update README.md	2023-12-13 05:28:46 +00:00
erew123	b7de2988b0	Updated note about clip length Turns out the models default config file is setting "max_ref_len": 30, allowing for up to 30 second long voice clips to be used.	2023-12-13 05:21:38 +00:00
erew123	bbfbeba246	Update README.md	2023-12-13 04:58:01 +00:00
erew123	40b775459e	New Narrator Version The narrator and filtering has been re-written to vastly improve filtering of text, resulting in less unwanted sounds and better control over narrator/character.	2023-12-13 04:51:41 +00:00
		`@@ -1 +1 @@`
			{"activate": true, "autoplay": true, "branding": "AllTalk ", "narrator_enabled": false, "deepspeed_activate": false, "delete_output_wavs": "Disabled", "ip_address": "127.0.0.1", "language": "English", "low_vram": false, "local_temperature": "0.7", "local_repetition_penalty": "12.0", "model_loaded": true, "model_name": "tts_models/multilingual/multi-dataset/xtts_v2", "narrator_voice": "female_01.wav", "output_folder_wav": "extensions/alltalk_tts/outputs/", "port_number": "7851", "remove_trailing_dots": false, "show_text": true, "tts_method_api_local": false, "tts_method_api_tts": false, "tts_method_xtts_local": true, "voice": "female_01.wav"}				{"activate": true, "autoplay": true, "branding": "AllTalk ", "narrator_enabled": false, "deepspeed_activate": false, "delete_output_wavs": "Disabled", "ip_address": "127.0.0.1", "language": "English", "low_vram": false, "local_temperature": "0.7", "local_repetition_penalty": "10.0", "model_loaded": true, "model_name": "tts_models/multilingual/multi-dataset/xtts_v2", "narrator_voice": "female_01.wav", "output_folder_wav": "extensions/alltalk_tts/outputs/", "port_number": "7851", "remove_trailing_dots": false, "show_text": true, "tts_method_api_local": false, "tts_method_api_tts": false, "tts_method_xtts_local": true, "voice": "female_01.wav"}