From caa28e395cb40b41598097372e268ccd2c557d4d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 6 Oct 2025 23:16:36 +0200 Subject: [PATCH 01/10] feat(neutts): add backend Signed-off-by: Ettore Di Giacinto --- Makefile | 9 + backend/python/neutts/Makefile | 23 +++ backend/python/neutts/backend.py | 162 ++++++++++++++++++ backend/python/neutts/install.sh | 33 ++++ backend/python/neutts/requirements-after.txt | 2 + backend/python/neutts/requirements-cpu.txt | 11 ++ .../python/neutts/requirements-cublas12.txt | 10 ++ .../python/neutts/requirements-hipblas.txt | 10 ++ backend/python/neutts/requirements-l4t.txt | 10 ++ backend/python/neutts/requirements.txt | 6 + backend/python/neutts/run.sh | 9 + backend/python/neutts/test.py | 82 +++++++++ backend/python/neutts/test.sh | 11 ++ 13 files changed, 378 insertions(+) create mode 100644 backend/python/neutts/Makefile create mode 100644 backend/python/neutts/backend.py create mode 100755 backend/python/neutts/install.sh create mode 100644 backend/python/neutts/requirements-after.txt create mode 100644 backend/python/neutts/requirements-cpu.txt create mode 100644 backend/python/neutts/requirements-cublas12.txt create mode 100644 backend/python/neutts/requirements-hipblas.txt create mode 100644 backend/python/neutts/requirements-l4t.txt create mode 100644 backend/python/neutts/requirements.txt create mode 100755 backend/python/neutts/run.sh create mode 100644 backend/python/neutts/test.py create mode 100755 backend/python/neutts/test.sh diff --git a/Makefile b/Makefile index 5d32926aed18..03e4efccbaaf 100644 --- a/Makefile +++ b/Makefile @@ -376,6 +376,9 @@ backends/llama-cpp-darwin: build bash ./scripts/build/llama-cpp-darwin.sh ./local-ai backends install "ocifile://$(abspath ./backend-images/llama-cpp.tar)" +backends/neutts: docker-build-neutts docker-save-neutts build + ./local-ai backends install "ocifile://$(abspath ./backend-images/neutts.tar)" + build-darwin-python-backend: build bash ./scripts/build/python-darwin.sh @@ -432,6 +435,12 @@ docker-save-kitten-tts: backend-images docker-save-chatterbox: backend-images docker save local-ai-backend:chatterbox -o backend-images/chatterbox.tar +docker-build-neutts: + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:neutts -f backend/Dockerfile.python --build-arg BACKEND=neutts ./backend + +docker-save-neutts: backend-images + docker save local-ai-backend:neutts -o backend-images/neutts.tar + docker-build-kokoro: docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:kokoro -f backend/Dockerfile.python --build-arg BACKEND=kokoro ./backend diff --git a/backend/python/neutts/Makefile b/backend/python/neutts/Makefile new file mode 100644 index 000000000000..7d50ed07be29 --- /dev/null +++ b/backend/python/neutts/Makefile @@ -0,0 +1,23 @@ +.PHONY: neutts +neutts: + bash install.sh + +.PHONY: run +run: neutts + @echo "Running neutts..." + bash run.sh + @echo "neutts run." + +.PHONY: test +test: neutts + @echo "Testing neutts..." + bash test.sh + @echo "neutts tested." + +.PHONY: protogen-clean +protogen-clean: + $(RM) backend_pb2_grpc.py backend_pb2.py + +.PHONY: clean +clean: protogen-clean + rm -rf venv __pycache__ \ No newline at end of file diff --git a/backend/python/neutts/backend.py b/backend/python/neutts/backend.py new file mode 100644 index 000000000000..e765436d104f --- /dev/null +++ b/backend/python/neutts/backend.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +""" +This is an extra gRPC server of LocalAI for NeuTTSAir +""" +from concurrent import futures +import time +import argparse +import signal +import sys +import os +import backend_pb2 +import backend_pb2_grpc +import torch +from neuttsair.neutts import NeuTTSAir +import soundfile as sf + +import grpc + +def is_float(s): + """Check if a string can be converted to float.""" + try: + float(s) + return True + except ValueError: + return False +def is_int(s): + """Check if a string can be converted to int.""" + try: + int(s) + return True + except ValueError: + return False + +_ONE_DAY_IN_SECONDS = 60 * 60 * 24 + +# If MAX_WORKERS are specified in the environment use it, otherwise default to 1 +MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1')) + +# Implement the BackendServicer class with the service methods +class BackendServicer(backend_pb2_grpc.BackendServicer): + """ + BackendServicer is the class that implements the gRPC service + """ + def Health(self, request, context): + return backend_pb2.Reply(message=bytes("OK", 'utf-8')) + def LoadModel(self, request, context): + + # Get device + # device = "cuda" if request.CUDA else "cpu" + if torch.cuda.is_available(): + print("CUDA is available", file=sys.stderr) + device = "cuda" + else: + print("CUDA is not available", file=sys.stderr) + device = "cpu" + mps_available = hasattr(torch.backends, "mps") and torch.backends.mps.is_available() + if mps_available: + device = "mps" + if not torch.cuda.is_available() and request.CUDA: + return backend_pb2.Result(success=False, message="CUDA is not available") + + + options = request.Options + + # empty dict + self.options = {} + self.ref_text = None + + # The options are a list of strings in this form optname:optvalue + # We are storing all the options in a dict so we can use it later when + # generating the images + for opt in options: + if ":" not in opt: + continue + key, value = opt.split(":") + # if value is a number, convert it to the appropriate type + if is_float(value): + value = float(value) + elif is_int(value): + value = int(value) + elif value.lower() in ["true", "false"]: + value = value.lower() == "true" + self.options[key] = value + + codec_repo = "neuphonic/neucodec" + if "codec_repo" in self.options: + codec_repo = self.options["codec_repo"] + del self.options["codec_repo"] + if "ref_text" in self.options: + self.ref_text = self.options["ref_text"] + del self.options["ref_text"] + + self.AudioPath = None + + if os.path.isabs(request.AudioPath): + self.AudioPath = request.AudioPath + elif request.AudioPath and request.ModelFile != "" and not os.path.isabs(request.AudioPath): + # get base path of modelFile + modelFileBase = os.path.dirname(request.ModelFile) + # modify LoraAdapter to be relative to modelFileBase + self.AudioPath = os.path.join(modelFileBase, request.AudioPath) + try: + print("Preparing models, please wait", file=sys.stderr) + self.model = NeuTTSAir(backbone_repo=request.Model, backbone_device=device, codec_repo=codec_repo, codec_device=device) + except Exception as err: + return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}") + # Implement your logic here for the LoadModel service + # Replace this with your desired response + return backend_pb2.Result(message="Model loaded successfully", success=True) + + def TTS(self, request, context): + try: + kwargs = {} + + # add options to kwargs + kwargs.update(self.options) + + ref_codes = self.model.encode_reference(self.AudioPath) + + wav = self.model.infer(request.text, ref_codes, self.ref_text) + + sf.write(request.dst, wav, 24000) + except Exception as err: + return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}") + return backend_pb2.Result(success=True) + +def serve(address): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=MAX_WORKERS), + options=[ + ('grpc.max_message_length', 50 * 1024 * 1024), # 50MB + ('grpc.max_send_message_length', 50 * 1024 * 1024), # 50MB + ('grpc.max_receive_message_length', 50 * 1024 * 1024), # 50MB + ]) + backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server) + server.add_insecure_port(address) + server.start() + print("Server started. Listening on: " + address, file=sys.stderr) + + # Define the signal handler function + def signal_handler(sig, frame): + print("Received termination signal. Shutting down...") + server.stop(0) + sys.exit(0) + + # Set the signal handlers for SIGINT and SIGTERM + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + try: + while True: + time.sleep(_ONE_DAY_IN_SECONDS) + except KeyboardInterrupt: + server.stop(0) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run the gRPC server.") + parser.add_argument( + "--addr", default="localhost:50051", help="The address to bind the server to." + ) + args = parser.parse_args() + + serve(args.addr) diff --git a/backend/python/neutts/install.sh b/backend/python/neutts/install.sh new file mode 100755 index 000000000000..38143193afcb --- /dev/null +++ b/backend/python/neutts/install.sh @@ -0,0 +1,33 @@ +#!/bin/bash +set -e + +backend_dir=$(dirname $0) +if [ -d $backend_dir/common ]; then + source $backend_dir/common/libbackend.sh +else + source $backend_dir/../common/libbackend.sh +fi + +# This is here because the Intel pip index is broken and returns 200 status codes for every package name, it just doesn't return any package links. +# This makes uv think that the package exists in the Intel pip index, and by default it stops looking at other pip indexes once it finds a match. +# We need uv to continue falling through to the pypi default index to find optimum[openvino] in the pypi index +# the --upgrade actually allows us to *downgrade* torch to the version provided in the Intel pip index +if [ "x${BUILD_PROFILE}" == "xintel" ]; then + EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match" +fi + +if [ "x${BUILD_TYPE}" == "xcublas" ] || [ "x${BUILD_TYPE}" == "l4t" ]; then + export CMAKE_ARGS="-DGGML_CUDA=on" +fi + +if [ "x${BUILD_TYPE}" == "xhipblas" ]; then + export CMAKE_ARGS="-DGGML_HIPBLAS=on" +fi + +EXTRA_PIP_INSTALL_FLAGS+=" --no-build-isolation" + +git clone https://github.com/neuphonic/neutts-air neutts-air + +cp -rfv neutts-air/neuttsair ./ + +installRequirements diff --git a/backend/python/neutts/requirements-after.txt b/backend/python/neutts/requirements-after.txt new file mode 100644 index 000000000000..dfa969a39d32 --- /dev/null +++ b/backend/python/neutts/requirements-after.txt @@ -0,0 +1,2 @@ +datasets==4.1.1 +torchtune==0.6.1 \ No newline at end of file diff --git a/backend/python/neutts/requirements-cpu.txt b/backend/python/neutts/requirements-cpu.txt new file mode 100644 index 000000000000..688998d94a75 --- /dev/null +++ b/backend/python/neutts/requirements-cpu.txt @@ -0,0 +1,11 @@ +--extra-index-url https://download.pytorch.org/whl/cpu +--extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu +accelerate +torch==2.8.0 +transformers==4.56.1 +librosa==0.11.0 +neucodec>=0.0.4 +phonemizer==3.3.0 +soundfile==0.13.1 +resemble-perth==1.0.1 +llama-cpp-python \ No newline at end of file diff --git a/backend/python/neutts/requirements-cublas12.txt b/backend/python/neutts/requirements-cublas12.txt new file mode 100644 index 000000000000..a4fd0e575e9b --- /dev/null +++ b/backend/python/neutts/requirements-cublas12.txt @@ -0,0 +1,10 @@ +librosa==0.11.0 +neucodec>=0.0.4 +phonemizer==3.3.0 +soundfile==0.13.1 +torch==2.8.0 +transformers==4.56.1 +resemble-perth==1.0.1 +accelerate +llama-cpp-python +--extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu125 \ No newline at end of file diff --git a/backend/python/neutts/requirements-hipblas.txt b/backend/python/neutts/requirements-hipblas.txt new file mode 100644 index 000000000000..c6d8936ccacd --- /dev/null +++ b/backend/python/neutts/requirements-hipblas.txt @@ -0,0 +1,10 @@ +--extra-index-url https://download.pytorch.org/whl/rocm6.3 +torch==2.8.0+rocm6.3 +transformers==4.56.1 +accelerate +librosa==0.11.0 +neucodec>=0.0.4 +neuttsair>=0.0.1 +phonemizer==3.3.0 +soundfile==0.13.1 +resemble-perth==1.0.1 \ No newline at end of file diff --git a/backend/python/neutts/requirements-l4t.txt b/backend/python/neutts/requirements-l4t.txt new file mode 100644 index 000000000000..597a1bab9482 --- /dev/null +++ b/backend/python/neutts/requirements-l4t.txt @@ -0,0 +1,10 @@ +--extra-index-url https://pypi.jetson-ai-lab.io/jp6/cu126/ +torch +transformers +accelerate +librosa==0.11.0 +neucodec>=0.0.4 +phonemizer==3.3.0 +soundfile==0.13.1 +resemble-perth==1.0.1 +llama-cpp-python diff --git a/backend/python/neutts/requirements.txt b/backend/python/neutts/requirements.txt new file mode 100644 index 000000000000..76d46f1b29a4 --- /dev/null +++ b/backend/python/neutts/requirements.txt @@ -0,0 +1,6 @@ +grpcio==1.71.0 +protobuf +certifi +packaging +setuptools +numpy==2.2.6 \ No newline at end of file diff --git a/backend/python/neutts/run.sh b/backend/python/neutts/run.sh new file mode 100755 index 000000000000..82b7b09ecc7d --- /dev/null +++ b/backend/python/neutts/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash +backend_dir=$(dirname $0) +if [ -d $backend_dir/common ]; then + source $backend_dir/common/libbackend.sh +else + source $backend_dir/../common/libbackend.sh +fi + +startBackend $@ \ No newline at end of file diff --git a/backend/python/neutts/test.py b/backend/python/neutts/test.py new file mode 100644 index 000000000000..878345ab64b4 --- /dev/null +++ b/backend/python/neutts/test.py @@ -0,0 +1,82 @@ +""" +A test script to test the gRPC service +""" +import unittest +import subprocess +import time +import backend_pb2 +import backend_pb2_grpc + +import grpc + + +class TestBackendServicer(unittest.TestCase): + """ + TestBackendServicer is the class that tests the gRPC service + """ + def setUp(self): + """ + This method sets up the gRPC service by starting the server + """ + self.service = subprocess.Popen(["python3", "backend.py", "--addr", "localhost:50051"]) + time.sleep(30) + + def tearDown(self) -> None: + """ + This method tears down the gRPC service by terminating the server + """ + self.service.terminate() + self.service.wait() + + def test_server_startup(self): + """ + This method tests if the server starts up successfully + """ + try: + self.setUp() + with grpc.insecure_channel("localhost:50051") as channel: + stub = backend_pb2_grpc.BackendStub(channel) + response = stub.Health(backend_pb2.HealthMessage()) + self.assertEqual(response.message, b'OK') + except Exception as err: + print(err) + self.fail("Server failed to start") + finally: + self.tearDown() + + def test_load_model(self): + """ + This method tests if the model is loaded successfully + """ + try: + self.setUp() + with grpc.insecure_channel("localhost:50051") as channel: + stub = backend_pb2_grpc.BackendStub(channel) + response = stub.LoadModel(backend_pb2.ModelOptions()) + print(response) + self.assertTrue(response.success) + self.assertEqual(response.message, "Model loaded successfully") + except Exception as err: + print(err) + self.fail("LoadModel service failed") + finally: + self.tearDown() + + def test_tts(self): + """ + This method tests if the embeddings are generated successfully + """ + try: + self.setUp() + with grpc.insecure_channel("localhost:50051") as channel: + stub = backend_pb2_grpc.BackendStub(channel) + response = stub.LoadModel(backend_pb2.ModelOptions()) + self.assertTrue(response.success) + tts_request = backend_pb2.TTSRequest(text="80s TV news production music hit for tonight's biggest story") + tts_response = stub.TTS(tts_request) + self.assertIsNotNone(tts_response) + except Exception as err: + print(err) + self.fail("TTS service failed") + finally: + self.tearDown() \ No newline at end of file diff --git a/backend/python/neutts/test.sh b/backend/python/neutts/test.sh new file mode 100755 index 000000000000..eb59f2aaf3f3 --- /dev/null +++ b/backend/python/neutts/test.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +backend_dir=$(dirname $0) +if [ -d $backend_dir/common ]; then + source $backend_dir/common/libbackend.sh +else + source $backend_dir/../common/libbackend.sh +fi + +runUnittests From 2c845f481ad90d92214b2807bcdfd448c1f18610 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 7 Oct 2025 18:20:46 +0200 Subject: [PATCH 02/10] chore(ci): add images to CI Signed-off-by: Ettore Di Giacinto --- .github/workflows/backend.yml | 49 +++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 85b8737d6bbe..cb5a46b3d7ee 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -993,6 +993,55 @@ jobs: backend: "kitten-tts" dockerfile: "./backend/Dockerfile.python" context: "./backend" + # neutts + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64,linux/arm64' + tag-latest: 'auto' + tag-suffix: '-cpu-neutts' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:22.04" + skip-drivers: 'false' + backend: "neutts" + dockerfile: "./backend/Dockerfile.python" + context: "./backend" + - build-type: 'cublas' + cuda-major-version: "12" + cuda-minor-version: "0" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-12-neutts' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:22.04" + skip-drivers: 'false' + backend: "neutts" + dockerfile: "./backend/Dockerfile.python" + context: "./backend" + - build-type: 'hipblas' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-rocm-hipblas-neutts' + runs-on: 'ubuntu-latest' + base-image: "rocm/dev-ubuntu-22.04:6.4.3" + skip-drivers: 'false' + backend: "neutts" + dockerfile: "./backend/Dockerfile.python" + context: "./backend" + - build-type: 'l4t' + cuda-major-version: "12" + cuda-minor-version: "0" + platforms: 'linux/arm64' + skip-drivers: 'true' + tag-latest: 'auto' + tag-suffix: '-nvidia-l4t-arm64-neutts' + base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0" + runs-on: 'ubuntu-24.04-arm' + backend: "neutts" + dockerfile: "./backend/Dockerfile.python" + context: "./backend" backend-jobs-darwin: uses: ./.github/workflows/backend_build_darwin.yml strategy: From c6183d35f12debae537d3fca61fa9de0ea3886ef Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 7 Oct 2025 18:24:59 +0200 Subject: [PATCH 03/10] chore(gallery): add Neutts Signed-off-by: Ettore Di Giacinto --- backend/index.yaml | 62 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/backend/index.yaml b/backend/index.yaml index c55df26368ee..da142b0998a8 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -427,6 +427,68 @@ - text-to-speech - TTS license: apache-2.0 +- &neutts + name: "neutts" + urls: + - https://github.com/neuphonic/neutts-air + description: | + NeuTTS Air is the world’s first super-realistic, on-device, TTS speech language model with instant voice cloning. Built off a 0.5B LLM backbone, NeuTTS Air brings natural-sounding speech, real-time performance, built-in security and speaker cloning to your local device - unlocking a new category of embedded voice agents, assistants, toys, and compliance-safe apps. + tags: + - text-to-speech + - TTS + license: apache-2.0 + capabilities: + default: "cpu-neutts" + nvidia: "cuda12-neutts" + amd: "rocm-neutts" + nvidia-l4t: "nvidia-l4t-neutts" +- !!merge <<: *neutts + name: "neutts-development" + capabilities: + default: "cpu-neutts-development" + nvidia: "cuda12-neutts-development" + amd: "rocm-neutts-development" + nvidia-l4t: "nvidia-l4t-neutts-development" +- !!merge <<: *neutts + name: "cpu-neutts" + uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-neutts" + mirrors: + - localai/localai-backends:latest-cpu-neutts +- !!merge <<: *neutts + name: "cuda12-neutts" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-neutts" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-12-neutts +- !!merge <<: *neutts + name: "rocm-neutts" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-neutts" + mirrors: + - localai/localai-backends:latest-gpu-rocm-hipblas-neutts +- !!merge <<: *neutts + name: "nvidia-l4t-neutts" + uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-arm64-neutts" + mirrors: + - localai/localai-backends:latest-nvidia-l4t-arm64-neutts +- !!merge <<: *neutts + name: "cpu-neutts-development" + uri: "quay.io/go-skynet/local-ai-backends:master-cpu-neutts" + mirrors: + - localai/localai-backends:master-cpu-neutts +- !!merge <<: *neutts + name: "cuda12-neutts-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-neutts" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-12-neutts +- !!merge <<: *neutts + name: "rocm-neutts-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-neutts" + mirrors: + - localai/localai-backends:master-gpu-rocm-hipblas-neutts +- !!merge <<: *neutts + name: "nvidia-l4t-neutts-development" + uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-arm64-neutts" + mirrors: + - localai/localai-backends:master-nvidia-l4t-arm64-neutts - !!merge <<: *mlx name: "mlx-development" uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-mlx" From c107ddb452b8f97f2aff1037f98b9ad5624828e2 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 7 Oct 2025 18:49:16 +0200 Subject: [PATCH 04/10] Make it work with quantized versions Signed-off-by: Ettore Di Giacinto --- backend/Dockerfile.python | 2 +- backend/python/neutts/requirements-cpu.txt | 1 - backend/python/neutts/requirements-cublas12.txt | 3 +-- backend/python/neutts/requirements-hipblas.txt | 3 ++- backend/python/neutts/requirements-l4t.txt | 2 +- backend/python/neutts/requirements.txt | 3 ++- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/Dockerfile.python b/backend/Dockerfile.python index 9850e9808678..b069ce164eca 100644 --- a/backend/Dockerfile.python +++ b/backend/Dockerfile.python @@ -28,7 +28,7 @@ RUN apt-get update && \ curl python3-pip \ python-is-python3 \ python3-dev llvm \ - python3-venv make && \ + python3-venv make cmake && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ pip install --upgrade pip diff --git a/backend/python/neutts/requirements-cpu.txt b/backend/python/neutts/requirements-cpu.txt index 688998d94a75..d6f972df9d95 100644 --- a/backend/python/neutts/requirements-cpu.txt +++ b/backend/python/neutts/requirements-cpu.txt @@ -1,5 +1,4 @@ --extra-index-url https://download.pytorch.org/whl/cpu ---extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu accelerate torch==2.8.0 transformers==4.56.1 diff --git a/backend/python/neutts/requirements-cublas12.txt b/backend/python/neutts/requirements-cublas12.txt index a4fd0e575e9b..227608f98618 100644 --- a/backend/python/neutts/requirements-cublas12.txt +++ b/backend/python/neutts/requirements-cublas12.txt @@ -6,5 +6,4 @@ torch==2.8.0 transformers==4.56.1 resemble-perth==1.0.1 accelerate -llama-cpp-python ---extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu125 \ No newline at end of file +llama-cpp-python \ No newline at end of file diff --git a/backend/python/neutts/requirements-hipblas.txt b/backend/python/neutts/requirements-hipblas.txt index c6d8936ccacd..aa900b228431 100644 --- a/backend/python/neutts/requirements-hipblas.txt +++ b/backend/python/neutts/requirements-hipblas.txt @@ -7,4 +7,5 @@ neucodec>=0.0.4 neuttsair>=0.0.1 phonemizer==3.3.0 soundfile==0.13.1 -resemble-perth==1.0.1 \ No newline at end of file +resemble-perth==1.0.1 +llama-cpp-python \ No newline at end of file diff --git a/backend/python/neutts/requirements-l4t.txt b/backend/python/neutts/requirements-l4t.txt index 597a1bab9482..7932d192eb32 100644 --- a/backend/python/neutts/requirements-l4t.txt +++ b/backend/python/neutts/requirements-l4t.txt @@ -7,4 +7,4 @@ neucodec>=0.0.4 phonemizer==3.3.0 soundfile==0.13.1 resemble-perth==1.0.1 -llama-cpp-python +llama-cpp-python \ No newline at end of file diff --git a/backend/python/neutts/requirements.txt b/backend/python/neutts/requirements.txt index 76d46f1b29a4..9262a3934510 100644 --- a/backend/python/neutts/requirements.txt +++ b/backend/python/neutts/requirements.txt @@ -3,4 +3,5 @@ protobuf certifi packaging setuptools -numpy==2.2.6 \ No newline at end of file +numpy==2.2.6 +scikit_build_core \ No newline at end of file From b0336b16e16d1f3498cc547255dc53a7e0d37b03 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 7 Oct 2025 19:22:06 +0200 Subject: [PATCH 05/10] Fixups Signed-off-by: Ettore Di Giacinto --- .github/workflows/backend.yml | 2 +- backend/python/neutts/requirements-hipblas.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index cb5a46b3d7ee..f304fb0e53f6 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -1024,7 +1024,7 @@ jobs: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-neutts' - runs-on: 'ubuntu-latest' + runs-on: 'arc-runner-set' base-image: "rocm/dev-ubuntu-22.04:6.4.3" skip-drivers: 'false' backend: "neutts" diff --git a/backend/python/neutts/requirements-hipblas.txt b/backend/python/neutts/requirements-hipblas.txt index aa900b228431..012d3c8bf6f5 100644 --- a/backend/python/neutts/requirements-hipblas.txt +++ b/backend/python/neutts/requirements-hipblas.txt @@ -4,7 +4,6 @@ transformers==4.56.1 accelerate librosa==0.11.0 neucodec>=0.0.4 -neuttsair>=0.0.1 phonemizer==3.3.0 soundfile==0.13.1 resemble-perth==1.0.1 From c262e66e7d2f3ec8e89063d47d8c3845bb2ad916 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 8 Oct 2025 08:33:22 +0200 Subject: [PATCH 06/10] Docs Signed-off-by: Ettore Di Giacinto --- README.md | 3 ++- docs/content/docs/reference/compatibility-table.md | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d21b2886f0f5..4bf777bcc1db 100644 --- a/README.md +++ b/README.md @@ -269,6 +269,7 @@ LocalAI supports a comprehensive range of AI backends with multiple acceleration | **piper** | Fast neural TTS system | CPU | | **kitten-tts** | Kitten TTS models | CPU | | **silero-vad** | Voice Activity Detection | CPU | +| **neutts** | Text-to-speech with voice cloning | CUDA 12, ROCm, CPU | ### Image & Video Generation | Backend | Description | Acceleration Support | @@ -290,7 +291,7 @@ LocalAI supports a comprehensive range of AI backends with multiple acceleration |-------------------|-------------------|------------------| | **NVIDIA CUDA 11** | llama.cpp, whisper, stablediffusion, diffusers, rerankers, bark, chatterbox | Nvidia hardware | | **NVIDIA CUDA 12** | All CUDA-compatible backends | Nvidia hardware | -| **AMD ROCm** | llama.cpp, whisper, vllm, transformers, diffusers, rerankers, coqui, kokoro, bark | AMD Graphics | +| **AMD ROCm** | llama.cpp, whisper, vllm, transformers, diffusers, rerankers, coqui, kokoro, bark, neutts | AMD Graphics | | **Intel oneAPI** | llama.cpp, whisper, stablediffusion, vllm, transformers, diffusers, rfdetr, rerankers, exllama2, coqui, kokoro, bark | Intel Arc, Intel iGPUs | | **Apple Metal** | llama.cpp, whisper, diffusers, MLX, MLX-VLM, bark-cpp | Apple M1/M2/M3+ | | **Vulkan** | llama.cpp, whisper, stablediffusion | Cross-platform GPUs | diff --git a/docs/content/docs/reference/compatibility-table.md b/docs/content/docs/reference/compatibility-table.md index 74f7f8eeafe6..87c9dc5fe1bb 100644 --- a/docs/content/docs/reference/compatibility-table.md +++ b/docs/content/docs/reference/compatibility-table.md @@ -43,6 +43,7 @@ LocalAI will attempt to automatically load models which are not explicitly confi | [chatterbox](https://github.com/resemble-ai/chatterbox) | Chatterbox TTS | no | Text-to-speech | no | no | CUDA 11/12, CPU | | [kitten-tts](https://github.com/KittenML/KittenTTS) | Kitten TTS | no | Text-to-speech | no | no | CPU | | [silero-vad](https://github.com/snakers4/silero-vad) with [Golang bindings](https://github.com/streamer45/silero-vad-go) | Silero VAD | no | Voice Activity Detection | no | no | CPU | +| [neutts](https://github.com/neuphonic/neuttsair) | NeuTTSAir | no | Text-to-speech with voice cloning | no | no | CUDA 12, ROCm, CPU | | [mlx-audio](https://github.com/Blaizzy/mlx-audio) | MLX | no | Text-tospeech | no | no | Metal (Apple Silicon) | {{< /table >}} From ce15065a2dd92ae82eb5825adc1d514327d00e4b Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 8 Oct 2025 08:33:43 +0200 Subject: [PATCH 07/10] Fixups Signed-off-by: Ettore Di Giacinto --- backend/python/neutts/install.sh | 3 ++- backend/python/neutts/run.sh | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/python/neutts/install.sh b/backend/python/neutts/install.sh index 38143193afcb..299f3ab38e9f 100755 --- a/backend/python/neutts/install.sh +++ b/backend/python/neutts/install.sh @@ -16,8 +16,9 @@ if [ "x${BUILD_PROFILE}" == "xintel" ]; then EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match" fi -if [ "x${BUILD_TYPE}" == "xcublas" ] || [ "x${BUILD_TYPE}" == "l4t" ]; then +if [ "x${BUILD_TYPE}" == "xcublas" ] || [ "x${BUILD_TYPE}" == "xl4t" ]; then export CMAKE_ARGS="-DGGML_CUDA=on" + export LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/cuda/compat:${LD_LIBRARY_PATH:-}" fi if [ "x${BUILD_TYPE}" == "xhipblas" ]; then diff --git a/backend/python/neutts/run.sh b/backend/python/neutts/run.sh index 82b7b09ecc7d..cf1d45d36123 100755 --- a/backend/python/neutts/run.sh +++ b/backend/python/neutts/run.sh @@ -6,4 +6,6 @@ else source $backend_dir/../common/libbackend.sh fi +export LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/cuda/compat:${LD_LIBRARY_PATH:-}" + startBackend $@ \ No newline at end of file From efaffffc80fdb4341d3f85b862d205ab4d2750aa Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 9 Oct 2025 19:16:59 +0200 Subject: [PATCH 08/10] Apply suggestion from @mudler Signed-off-by: Ettore Di Giacinto --- backend/python/neutts/requirements-cublas12.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/python/neutts/requirements-cublas12.txt b/backend/python/neutts/requirements-cublas12.txt index 227608f98618..13afd3b86d64 100644 --- a/backend/python/neutts/requirements-cublas12.txt +++ b/backend/python/neutts/requirements-cublas12.txt @@ -5,5 +5,4 @@ soundfile==0.13.1 torch==2.8.0 transformers==4.56.1 resemble-perth==1.0.1 -accelerate -llama-cpp-python \ No newline at end of file +accelerate \ No newline at end of file From 6ffb62c261e430f8661c45193df88018cf843736 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 9 Oct 2025 19:17:15 +0200 Subject: [PATCH 09/10] Apply suggestion from @mudler Signed-off-by: Ettore Di Giacinto --- backend/python/neutts/install.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/python/neutts/install.sh b/backend/python/neutts/install.sh index 299f3ab38e9f..886f931ee50c 100755 --- a/backend/python/neutts/install.sh +++ b/backend/python/neutts/install.sh @@ -18,7 +18,6 @@ fi if [ "x${BUILD_TYPE}" == "xcublas" ] || [ "x${BUILD_TYPE}" == "xl4t" ]; then export CMAKE_ARGS="-DGGML_CUDA=on" - export LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/cuda/compat:${LD_LIBRARY_PATH:-}" fi if [ "x${BUILD_TYPE}" == "xhipblas" ]; then From cc0cfd0c095fec7edf2599da633bc78d2ccdb3c3 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 9 Oct 2025 19:17:27 +0200 Subject: [PATCH 10/10] Apply suggestion from @mudler Signed-off-by: Ettore Di Giacinto --- backend/python/neutts/run.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/python/neutts/run.sh b/backend/python/neutts/run.sh index cf1d45d36123..7d9d321eded7 100755 --- a/backend/python/neutts/run.sh +++ b/backend/python/neutts/run.sh @@ -6,6 +6,5 @@ else source $backend_dir/../common/libbackend.sh fi -export LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/cuda/compat:${LD_LIBRARY_PATH:-}" startBackend $@ \ No newline at end of file