diff --git a/Dockerfile b/Dockerfile
index 54de0e2285ac..f4e547a1366b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,7 +14,7 @@ ARG TARGETARCH
 ARG TARGETVARIANT
 
 ENV BUILD_TYPE=${BUILD_TYPE}
-ENV EXTERNAL_GRPC_BACKENDS="huggingface-embeddings:/build/extra/grpc/huggingface/run.sh,autogptq:/build/extra/grpc/autogptq/run.sh,bark:/build/extra/grpc/bark/run.sh,diffusers:/build/extra/grpc/diffusers/run.sh,exllama:/build/extra/grpc/exllama/exllama.py,vall-e-x:/build/extra/grpc/vall-e-x/run.sh,vllm:/build/extra/grpc/vllm/run.sh"
+ENV EXTERNAL_GRPC_BACKENDS="huggingface-embeddings:/build/extra/grpc/huggingface/run.sh,autogptq:/build/extra/grpc/autogptq/run.sh,bark:/build/extra/grpc/bark/run.sh,diffusers:/build/extra/grpc/diffusers/run.sh,exllama:/build/extra/grpc/exllama/run.sh,vall-e-x:/build/extra/grpc/vall-e-x/run.sh,vllm:/build/extra/grpc/vllm/run.sh"
 ENV GALLERIES='[{"name":"model-gallery", "url":"github:go-skynet/model-gallery/index.yaml"}, {"url": "github:go-skynet/model-gallery/huggingface.yaml","name":"huggingface"}]'
 ARG GO_TAGS="stablediffusion tts"
 
diff --git a/Makefile b/Makefile
index 5a5a5fa541aa..d883af0de9da 100644
--- a/Makefile
+++ b/Makefile
@@ -416,6 +416,7 @@ prepare-extra-conda-environments:
 	$(MAKE) -C extra/grpc/vllm
 	$(MAKE) -C extra/grpc/huggingface
 	$(MAKE) -C extra/grpc/vall-e-x
+	$(MAKE) -C extra/grpc/exllama
 
 
 backend-assets/grpc:
diff --git a/extra/grpc/exllama/Makefile b/extra/grpc/exllama/Makefile
new file mode 100644
index 000000000000..8410bc5eb40e
--- /dev/null
+++ b/extra/grpc/exllama/Makefile
@@ -0,0 +1,11 @@
+.PONY: exllama
+exllama:
+	@echo "Creating virtual environment..."
+	@conda env create --name exllama --file exllama.yml
+	@echo "Virtual environment created."
+
+.PONY: run
+run:
+	@echo "Running exllama..."
+	bash run.sh
+	@echo "exllama run."
\ No newline at end of file
diff --git a/extra/grpc/exllama/README.md b/extra/grpc/exllama/README.md
new file mode 100644
index 000000000000..f9ed5e9fbdb7
--- /dev/null
+++ b/extra/grpc/exllama/README.md
@@ -0,0 +1,5 @@
+# Creating a separate environment for the exllama project
+
+```
+make exllama
+```
\ No newline at end of file
diff --git a/extra/grpc/exllama/exllama.yml b/extra/grpc/exllama/exllama.yml
new file mode 100644
index 000000000000..20be0df1a23e
--- /dev/null
+++ b/extra/grpc/exllama/exllama.yml
@@ -0,0 +1,55 @@
+name: exllama
+channels:
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - bzip2=1.0.8=h7b6447c_0
+  - ca-certificates=2023.08.22=h06a4308_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - libffi=3.4.4=h6a678d5_0
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - libuuid=1.41.5=h5eee18b_0
+  - ncurses=6.4=h6a678d5_0
+  - openssl=3.0.11=h7f8727e_2
+  - pip=23.2.1=py311h06a4308_0
+  - python=3.11.5=h955ad1f_0
+  - readline=8.2=h5eee18b_0
+  - setuptools=68.0.0=py311h06a4308_0
+  - sqlite=3.41.2=h5eee18b_0
+  - tk=8.6.12=h1ccaba5_0
+  - tzdata=2023c=h04d1e81_0
+  - wheel=0.41.2=py311h06a4308_0
+  - xz=5.4.2=h5eee18b_0
+  - zlib=1.2.13=h5eee18b_0
+  - pip:
+      - filelock==3.12.4
+      - fsspec==2023.9.2
+      - grpcio==1.59.0
+      - jinja2==3.1.2
+      - markupsafe==2.1.3
+      - mpmath==1.3.0
+      - networkx==3.1
+      - ninja==1.11.1
+      - nvidia-cublas-cu12==12.1.3.1
+      - nvidia-cuda-cupti-cu12==12.1.105
+      - nvidia-cuda-nvrtc-cu12==12.1.105
+      - nvidia-cuda-runtime-cu12==12.1.105
+      - nvidia-cudnn-cu12==8.9.2.26
+      - nvidia-cufft-cu12==11.0.2.54
+      - nvidia-curand-cu12==10.3.2.106
+      - nvidia-cusolver-cu12==11.4.5.107
+      - nvidia-cusparse-cu12==12.1.0.106
+      - nvidia-nccl-cu12==2.18.1
+      - nvidia-nvjitlink-cu12==12.2.140
+      - nvidia-nvtx-cu12==12.1.105
+      - protobuf==4.24.4
+      - safetensors==0.3.2
+      - sentencepiece==0.1.99
+      - sympy==1.12
+      - torch==2.1.0
+      - triton==2.1.0
+      - typing-extensions==4.8.0
+prefix: /opt/conda/envs/exllama
diff --git a/extra/grpc/exllama/run.sh b/extra/grpc/exllama/run.sh
new file mode 100644
index 000000000000..e91c2a3a9626
--- /dev/null
+++ b/extra/grpc/exllama/run.sh
@@ -0,0 +1,10 @@
+##
+## A bash script wrapper that runs the exllama server with conda
+
+# Activate conda environment
+source activate exllama
+
+# get the directory where the bash script is located
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+python $DIR/exllama.py
\ No newline at end of file