diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 85b8737d6bbe..f304fb0e53f6 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -993,6 +993,55 @@ jobs: backend: "kitten-tts" dockerfile: "./backend/Dockerfile.python" context: "./backend" + # neutts + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64,linux/arm64' + tag-latest: 'auto' + tag-suffix: '-cpu-neutts' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:22.04" + skip-drivers: 'false' + backend: "neutts" + dockerfile: "./backend/Dockerfile.python" + context: "./backend" + - build-type: 'cublas' + cuda-major-version: "12" + cuda-minor-version: "0" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-12-neutts' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:22.04" + skip-drivers: 'false' + backend: "neutts" + dockerfile: "./backend/Dockerfile.python" + context: "./backend" + - build-type: 'hipblas' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-rocm-hipblas-neutts' + runs-on: 'arc-runner-set' + base-image: "rocm/dev-ubuntu-22.04:6.4.3" + skip-drivers: 'false' + backend: "neutts" + dockerfile: "./backend/Dockerfile.python" + context: "./backend" + - build-type: 'l4t' + cuda-major-version: "12" + cuda-minor-version: "0" + platforms: 'linux/arm64' + skip-drivers: 'true' + tag-latest: 'auto' + tag-suffix: '-nvidia-l4t-arm64-neutts' + base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0" + runs-on: 'ubuntu-24.04-arm' + backend: "neutts" + dockerfile: "./backend/Dockerfile.python" + context: "./backend" backend-jobs-darwin: uses: ./.github/workflows/backend_build_darwin.yml strategy: diff --git a/Makefile b/Makefile index 5d32926aed18..03e4efccbaaf 100644 --- a/Makefile +++ b/Makefile @@ -376,6 +376,9 @@ backends/llama-cpp-darwin: build bash ./scripts/build/llama-cpp-darwin.sh ./local-ai backends install "ocifile://$(abspath ./backend-images/llama-cpp.tar)" +backends/neutts: docker-build-neutts docker-save-neutts build + ./local-ai backends install "ocifile://$(abspath ./backend-images/neutts.tar)" + build-darwin-python-backend: build bash ./scripts/build/python-darwin.sh @@ -432,6 +435,12 @@ docker-save-kitten-tts: backend-images docker-save-chatterbox: backend-images docker save local-ai-backend:chatterbox -o backend-images/chatterbox.tar +docker-build-neutts: + docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:neutts -f backend/Dockerfile.python --build-arg BACKEND=neutts ./backend + +docker-save-neutts: backend-images + docker save local-ai-backend:neutts -o backend-images/neutts.tar + docker-build-kokoro: docker build --build-arg BUILD_TYPE=$(BUILD_TYPE) --build-arg BASE_IMAGE=$(BASE_IMAGE) -t local-ai-backend:kokoro -f backend/Dockerfile.python --build-arg BACKEND=kokoro ./backend diff --git a/README.md b/README.md index d21b2886f0f5..4bf777bcc1db 100644 --- a/README.md +++ b/README.md @@ -269,6 +269,7 @@ LocalAI supports a comprehensive range of AI backends with multiple acceleration | **piper** | Fast neural TTS system | CPU | | **kitten-tts** | Kitten TTS models | CPU | | **silero-vad** | Voice Activity Detection | CPU | +| **neutts** | Text-to-speech with voice cloning | CUDA 12, ROCm, CPU | ### Image & Video Generation | Backend | Description | Acceleration Support | @@ -290,7 +291,7 @@ LocalAI supports a comprehensive range of AI backends with multiple acceleration |-------------------|-------------------|------------------| | **NVIDIA CUDA 11** | llama.cpp, whisper, stablediffusion, diffusers, rerankers, bark, chatterbox | Nvidia hardware | | **NVIDIA CUDA 12** | All CUDA-compatible backends | Nvidia hardware | -| **AMD ROCm** | llama.cpp, whisper, vllm, transformers, diffusers, rerankers, coqui, kokoro, bark | AMD Graphics | +| **AMD ROCm** | llama.cpp, whisper, vllm, transformers, diffusers, rerankers, coqui, kokoro, bark, neutts | AMD Graphics | | **Intel oneAPI** | llama.cpp, whisper, stablediffusion, vllm, transformers, diffusers, rfdetr, rerankers, exllama2, coqui, kokoro, bark | Intel Arc, Intel iGPUs | | **Apple Metal** | llama.cpp, whisper, diffusers, MLX, MLX-VLM, bark-cpp | Apple M1/M2/M3+ | | **Vulkan** | llama.cpp, whisper, stablediffusion | Cross-platform GPUs | diff --git a/backend/Dockerfile.python b/backend/Dockerfile.python index 9850e9808678..b069ce164eca 100644 --- a/backend/Dockerfile.python +++ b/backend/Dockerfile.python @@ -28,7 +28,7 @@ RUN apt-get update && \ curl python3-pip \ python-is-python3 \ python3-dev llvm \ - python3-venv make && \ + python3-venv make cmake && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ pip install --upgrade pip diff --git a/backend/index.yaml b/backend/index.yaml index c55df26368ee..da142b0998a8 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -427,6 +427,68 @@ - text-to-speech - TTS license: apache-2.0 +- &neutts + name: "neutts" + urls: + - https://github.com/neuphonic/neutts-air + description: | + NeuTTS Air is the world’s first super-realistic, on-device, TTS speech language model with instant voice cloning. Built off a 0.5B LLM backbone, NeuTTS Air brings natural-sounding speech, real-time performance, built-in security and speaker cloning to your local device - unlocking a new category of embedded voice agents, assistants, toys, and compliance-safe apps. + tags: + - text-to-speech + - TTS + license: apache-2.0 + capabilities: + default: "cpu-neutts" + nvidia: "cuda12-neutts" + amd: "rocm-neutts" + nvidia-l4t: "nvidia-l4t-neutts" +- !!merge <<: *neutts + name: "neutts-development" + capabilities: + default: "cpu-neutts-development" + nvidia: "cuda12-neutts-development" + amd: "rocm-neutts-development" + nvidia-l4t: "nvidia-l4t-neutts-development" +- !!merge <<: *neutts + name: "cpu-neutts" + uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-neutts" + mirrors: + - localai/localai-backends:latest-cpu-neutts +- !!merge <<: *neutts + name: "cuda12-neutts" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-neutts" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-12-neutts +- !!merge <<: *neutts + name: "rocm-neutts" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-neutts" + mirrors: + - localai/localai-backends:latest-gpu-rocm-hipblas-neutts +- !!merge <<: *neutts + name: "nvidia-l4t-neutts" + uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-arm64-neutts" + mirrors: + - localai/localai-backends:latest-nvidia-l4t-arm64-neutts +- !!merge <<: *neutts + name: "cpu-neutts-development" + uri: "quay.io/go-skynet/local-ai-backends:master-cpu-neutts" + mirrors: + - localai/localai-backends:master-cpu-neutts +- !!merge <<: *neutts + name: "cuda12-neutts-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-neutts" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-12-neutts +- !!merge <<: *neutts + name: "rocm-neutts-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-neutts" + mirrors: + - localai/localai-backends:master-gpu-rocm-hipblas-neutts +- !!merge <<: *neutts + name: "nvidia-l4t-neutts-development" + uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-arm64-neutts" + mirrors: + - localai/localai-backends:master-nvidia-l4t-arm64-neutts - !!merge <<: *mlx name: "mlx-development" uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-mlx" diff --git a/backend/python/neutts/Makefile b/backend/python/neutts/Makefile new file mode 100644 index 000000000000..7d50ed07be29 --- /dev/null +++ b/backend/python/neutts/Makefile @@ -0,0 +1,23 @@ +.PHONY: neutts +neutts: + bash install.sh + +.PHONY: run +run: neutts + @echo "Running neutts..." + bash run.sh + @echo "neutts run." + +.PHONY: test +test: neutts + @echo "Testing neutts..." + bash test.sh + @echo "neutts tested." + +.PHONY: protogen-clean +protogen-clean: + $(RM) backend_pb2_grpc.py backend_pb2.py + +.PHONY: clean +clean: protogen-clean + rm -rf venv __pycache__ \ No newline at end of file diff --git a/backend/python/neutts/backend.py b/backend/python/neutts/backend.py new file mode 100644 index 000000000000..e765436d104f --- /dev/null +++ b/backend/python/neutts/backend.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +""" +This is an extra gRPC server of LocalAI for NeuTTSAir +""" +from concurrent import futures +import time +import argparse +import signal +import sys +import os +import backend_pb2 +import backend_pb2_grpc +import torch +from neuttsair.neutts import NeuTTSAir +import soundfile as sf + +import grpc + +def is_float(s): + """Check if a string can be converted to float.""" + try: + float(s) + return True + except ValueError: + return False +def is_int(s): + """Check if a string can be converted to int.""" + try: + int(s) + return True + except ValueError: + return False + +_ONE_DAY_IN_SECONDS = 60 * 60 * 24 + +# If MAX_WORKERS are specified in the environment use it, otherwise default to 1 +MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1')) + +# Implement the BackendServicer class with the service methods +class BackendServicer(backend_pb2_grpc.BackendServicer): + """ + BackendServicer is the class that implements the gRPC service + """ + def Health(self, request, context): + return backend_pb2.Reply(message=bytes("OK", 'utf-8')) + def LoadModel(self, request, context): + + # Get device + # device = "cuda" if request.CUDA else "cpu" + if torch.cuda.is_available(): + print("CUDA is available", file=sys.stderr) + device = "cuda" + else: + print("CUDA is not available", file=sys.stderr) + device = "cpu" + mps_available = hasattr(torch.backends, "mps") and torch.backends.mps.is_available() + if mps_available: + device = "mps" + if not torch.cuda.is_available() and request.CUDA: + return backend_pb2.Result(success=False, message="CUDA is not available") + + + options = request.Options + + # empty dict + self.options = {} + self.ref_text = None + + # The options are a list of strings in this form optname:optvalue + # We are storing all the options in a dict so we can use it later when + # generating the images + for opt in options: + if ":" not in opt: + continue + key, value = opt.split(":") + # if value is a number, convert it to the appropriate type + if is_float(value): + value = float(value) + elif is_int(value): + value = int(value) + elif value.lower() in ["true", "false"]: + value = value.lower() == "true" + self.options[key] = value + + codec_repo = "neuphonic/neucodec" + if "codec_repo" in self.options: + codec_repo = self.options["codec_repo"] + del self.options["codec_repo"] + if "ref_text" in self.options: + self.ref_text = self.options["ref_text"] + del self.options["ref_text"] + + self.AudioPath = None + + if os.path.isabs(request.AudioPath): + self.AudioPath = request.AudioPath + elif request.AudioPath and request.ModelFile != "" and not os.path.isabs(request.AudioPath): + # get base path of modelFile + modelFileBase = os.path.dirname(request.ModelFile) + # modify LoraAdapter to be relative to modelFileBase + self.AudioPath = os.path.join(modelFileBase, request.AudioPath) + try: + print("Preparing models, please wait", file=sys.stderr) + self.model = NeuTTSAir(backbone_repo=request.Model, backbone_device=device, codec_repo=codec_repo, codec_device=device) + except Exception as err: + return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}") + # Implement your logic here for the LoadModel service + # Replace this with your desired response + return backend_pb2.Result(message="Model loaded successfully", success=True) + + def TTS(self, request, context): + try: + kwargs = {} + + # add options to kwargs + kwargs.update(self.options) + + ref_codes = self.model.encode_reference(self.AudioPath) + + wav = self.model.infer(request.text, ref_codes, self.ref_text) + + sf.write(request.dst, wav, 24000) + except Exception as err: + return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}") + return backend_pb2.Result(success=True) + +def serve(address): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=MAX_WORKERS), + options=[ + ('grpc.max_message_length', 50 * 1024 * 1024), # 50MB + ('grpc.max_send_message_length', 50 * 1024 * 1024), # 50MB + ('grpc.max_receive_message_length', 50 * 1024 * 1024), # 50MB + ]) + backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server) + server.add_insecure_port(address) + server.start() + print("Server started. Listening on: " + address, file=sys.stderr) + + # Define the signal handler function + def signal_handler(sig, frame): + print("Received termination signal. Shutting down...") + server.stop(0) + sys.exit(0) + + # Set the signal handlers for SIGINT and SIGTERM + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + try: + while True: + time.sleep(_ONE_DAY_IN_SECONDS) + except KeyboardInterrupt: + server.stop(0) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run the gRPC server.") + parser.add_argument( + "--addr", default="localhost:50051", help="The address to bind the server to." + ) + args = parser.parse_args() + + serve(args.addr) diff --git a/backend/python/neutts/install.sh b/backend/python/neutts/install.sh new file mode 100755 index 000000000000..886f931ee50c --- /dev/null +++ b/backend/python/neutts/install.sh @@ -0,0 +1,33 @@ +#!/bin/bash +set -e + +backend_dir=$(dirname $0) +if [ -d $backend_dir/common ]; then + source $backend_dir/common/libbackend.sh +else + source $backend_dir/../common/libbackend.sh +fi + +# This is here because the Intel pip index is broken and returns 200 status codes for every package name, it just doesn't return any package links. +# This makes uv think that the package exists in the Intel pip index, and by default it stops looking at other pip indexes once it finds a match. +# We need uv to continue falling through to the pypi default index to find optimum[openvino] in the pypi index +# the --upgrade actually allows us to *downgrade* torch to the version provided in the Intel pip index +if [ "x${BUILD_PROFILE}" == "xintel" ]; then + EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match" +fi + +if [ "x${BUILD_TYPE}" == "xcublas" ] || [ "x${BUILD_TYPE}" == "xl4t" ]; then + export CMAKE_ARGS="-DGGML_CUDA=on" +fi + +if [ "x${BUILD_TYPE}" == "xhipblas" ]; then + export CMAKE_ARGS="-DGGML_HIPBLAS=on" +fi + +EXTRA_PIP_INSTALL_FLAGS+=" --no-build-isolation" + +git clone https://github.com/neuphonic/neutts-air neutts-air + +cp -rfv neutts-air/neuttsair ./ + +installRequirements diff --git a/backend/python/neutts/requirements-after.txt b/backend/python/neutts/requirements-after.txt new file mode 100644 index 000000000000..dfa969a39d32 --- /dev/null +++ b/backend/python/neutts/requirements-after.txt @@ -0,0 +1,2 @@ +datasets==4.1.1 +torchtune==0.6.1 \ No newline at end of file diff --git a/backend/python/neutts/requirements-cpu.txt b/backend/python/neutts/requirements-cpu.txt new file mode 100644 index 000000000000..d6f972df9d95 --- /dev/null +++ b/backend/python/neutts/requirements-cpu.txt @@ -0,0 +1,10 @@ +--extra-index-url https://download.pytorch.org/whl/cpu +accelerate +torch==2.8.0 +transformers==4.56.1 +librosa==0.11.0 +neucodec>=0.0.4 +phonemizer==3.3.0 +soundfile==0.13.1 +resemble-perth==1.0.1 +llama-cpp-python \ No newline at end of file diff --git a/backend/python/neutts/requirements-cublas12.txt b/backend/python/neutts/requirements-cublas12.txt new file mode 100644 index 000000000000..13afd3b86d64 --- /dev/null +++ b/backend/python/neutts/requirements-cublas12.txt @@ -0,0 +1,8 @@ +librosa==0.11.0 +neucodec>=0.0.4 +phonemizer==3.3.0 +soundfile==0.13.1 +torch==2.8.0 +transformers==4.56.1 +resemble-perth==1.0.1 +accelerate \ No newline at end of file diff --git a/backend/python/neutts/requirements-hipblas.txt b/backend/python/neutts/requirements-hipblas.txt new file mode 100644 index 000000000000..012d3c8bf6f5 --- /dev/null +++ b/backend/python/neutts/requirements-hipblas.txt @@ -0,0 +1,10 @@ +--extra-index-url https://download.pytorch.org/whl/rocm6.3 +torch==2.8.0+rocm6.3 +transformers==4.56.1 +accelerate +librosa==0.11.0 +neucodec>=0.0.4 +phonemizer==3.3.0 +soundfile==0.13.1 +resemble-perth==1.0.1 +llama-cpp-python \ No newline at end of file diff --git a/backend/python/neutts/requirements-l4t.txt b/backend/python/neutts/requirements-l4t.txt new file mode 100644 index 000000000000..7932d192eb32 --- /dev/null +++ b/backend/python/neutts/requirements-l4t.txt @@ -0,0 +1,10 @@ +--extra-index-url https://pypi.jetson-ai-lab.io/jp6/cu126/ +torch +transformers +accelerate +librosa==0.11.0 +neucodec>=0.0.4 +phonemizer==3.3.0 +soundfile==0.13.1 +resemble-perth==1.0.1 +llama-cpp-python \ No newline at end of file diff --git a/backend/python/neutts/requirements.txt b/backend/python/neutts/requirements.txt new file mode 100644 index 000000000000..9262a3934510 --- /dev/null +++ b/backend/python/neutts/requirements.txt @@ -0,0 +1,7 @@ +grpcio==1.71.0 +protobuf +certifi +packaging +setuptools +numpy==2.2.6 +scikit_build_core \ No newline at end of file diff --git a/backend/python/neutts/run.sh b/backend/python/neutts/run.sh new file mode 100755 index 000000000000..7d9d321eded7 --- /dev/null +++ b/backend/python/neutts/run.sh @@ -0,0 +1,10 @@ +#!/bin/bash +backend_dir=$(dirname $0) +if [ -d $backend_dir/common ]; then + source $backend_dir/common/libbackend.sh +else + source $backend_dir/../common/libbackend.sh +fi + + +startBackend $@ \ No newline at end of file diff --git a/backend/python/neutts/test.py b/backend/python/neutts/test.py new file mode 100644 index 000000000000..878345ab64b4 --- /dev/null +++ b/backend/python/neutts/test.py @@ -0,0 +1,82 @@ +""" +A test script to test the gRPC service +""" +import unittest +import subprocess +import time +import backend_pb2 +import backend_pb2_grpc + +import grpc + + +class TestBackendServicer(unittest.TestCase): + """ + TestBackendServicer is the class that tests the gRPC service + """ + def setUp(self): + """ + This method sets up the gRPC service by starting the server + """ + self.service = subprocess.Popen(["python3", "backend.py", "--addr", "localhost:50051"]) + time.sleep(30) + + def tearDown(self) -> None: + """ + This method tears down the gRPC service by terminating the server + """ + self.service.terminate() + self.service.wait() + + def test_server_startup(self): + """ + This method tests if the server starts up successfully + """ + try: + self.setUp() + with grpc.insecure_channel("localhost:50051") as channel: + stub = backend_pb2_grpc.BackendStub(channel) + response = stub.Health(backend_pb2.HealthMessage()) + self.assertEqual(response.message, b'OK') + except Exception as err: + print(err) + self.fail("Server failed to start") + finally: + self.tearDown() + + def test_load_model(self): + """ + This method tests if the model is loaded successfully + """ + try: + self.setUp() + with grpc.insecure_channel("localhost:50051") as channel: + stub = backend_pb2_grpc.BackendStub(channel) + response = stub.LoadModel(backend_pb2.ModelOptions()) + print(response) + self.assertTrue(response.success) + self.assertEqual(response.message, "Model loaded successfully") + except Exception as err: + print(err) + self.fail("LoadModel service failed") + finally: + self.tearDown() + + def test_tts(self): + """ + This method tests if the embeddings are generated successfully + """ + try: + self.setUp() + with grpc.insecure_channel("localhost:50051") as channel: + stub = backend_pb2_grpc.BackendStub(channel) + response = stub.LoadModel(backend_pb2.ModelOptions()) + self.assertTrue(response.success) + tts_request = backend_pb2.TTSRequest(text="80s TV news production music hit for tonight's biggest story") + tts_response = stub.TTS(tts_request) + self.assertIsNotNone(tts_response) + except Exception as err: + print(err) + self.fail("TTS service failed") + finally: + self.tearDown() \ No newline at end of file diff --git a/backend/python/neutts/test.sh b/backend/python/neutts/test.sh new file mode 100755 index 000000000000..eb59f2aaf3f3 --- /dev/null +++ b/backend/python/neutts/test.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +backend_dir=$(dirname $0) +if [ -d $backend_dir/common ]; then + source $backend_dir/common/libbackend.sh +else + source $backend_dir/../common/libbackend.sh +fi + +runUnittests diff --git a/docs/content/docs/reference/compatibility-table.md b/docs/content/docs/reference/compatibility-table.md index 74f7f8eeafe6..87c9dc5fe1bb 100644 --- a/docs/content/docs/reference/compatibility-table.md +++ b/docs/content/docs/reference/compatibility-table.md @@ -43,6 +43,7 @@ LocalAI will attempt to automatically load models which are not explicitly confi | [chatterbox](https://github.com/resemble-ai/chatterbox) | Chatterbox TTS | no | Text-to-speech | no | no | CUDA 11/12, CPU | | [kitten-tts](https://github.com/KittenML/KittenTTS) | Kitten TTS | no | Text-to-speech | no | no | CPU | | [silero-vad](https://github.com/snakers4/silero-vad) with [Golang bindings](https://github.com/streamer45/silero-vad-go) | Silero VAD | no | Voice Activity Detection | no | no | CPU | +| [neutts](https://github.com/neuphonic/neuttsair) | NeuTTSAir | no | Text-to-speech with voice cloning | no | no | CUDA 12, ROCm, CPU | | [mlx-audio](https://github.com/Blaizzy/mlx-audio) | MLX | no | Text-tospeech | no | no | Metal (Apple Silicon) | {{< /table >}}