NGINX Plus R33 support (#2760)

Adding support for NGINX Plus R33. The major change with this release is that NGINX Plus now requires a JWT in order to run. A user must create a Secret with this JWT and supply the secret name to NGF when installing. A user can also create client SSL and CA Secrets for NIM connections. All of these Secrets are mounted to the nginx container.

Because of the new usage reporting method, the old usage reporting method has been removed and CLI arguments have been altered. Since this release is a breaking change for N+ users, the choice was made to remove the unused usage reporting flags instead of deprecating them.

Updated documentation to describe this process, while also cleaning up the JWT docker registry process for N+.

Author: sjberman

.github/workflows/ci.yml

@@ -238,6 +238,7 @@ jobs:
     with:
       image: ${{ matrix.image }}
       k8s-version: ${{ matrix.k8s-version }}
+    secrets: inherit
     permissions:
       contents: read
 
@@ -259,6 +260,7 @@ jobs:
       image: ${{ matrix.image }}
       k8s-version: ${{ matrix.k8s-version }}
       enable-experimental: ${{ matrix.enable-experimental }}
+    secrets: inherit
     permissions:
       contents: write
 

.github/workflows/conformance.yml

@@ -135,6 +135,12 @@ jobs:
           kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }}
           kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}
 
+      - name: Setup license file for plus
+        if: ${{ inputs.image == 'plus' }}
+        env:
+          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
+        run: echo "${PLUS_LICENSE}" > license.jwt
+
       - name: Setup conformance tests
         run: |
           ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric

.github/workflows/functional.yml

@@ -100,6 +100,12 @@ jobs:
             NGINX_CONF_DIR=internal/mode/static/nginx/conf
             BUILD_AGENT=gha
 
+      - name: Setup license file for plus
+        if: ${{ inputs.image == 'plus' }}
+        env:
+          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
+        run: echo "${PLUS_LICENSE}" > license.jwt
+
       - name: Install cloud-provider-kind
         run: |
           CLOUD_PROVIDER_KIND_VERSION=v0.4.0 # renovate: datasource=github-tags depName=kubernetes-sigs/cloud-provider-kind

.github/workflows/helm.yml

@@ -98,6 +98,15 @@ jobs:
           kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }}
           kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}
           kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f -
+          kubectl create namespace nginx-gateway
+
+      - name: Create plus secret
+        if: ${{ inputs.image == 'plus' }}
+        env:
+          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
+        run: |
+          echo "${PLUS_LICENSE}" > license.jwt
+          kubectl create secret generic nplus-license --from-file license.jwt -n nginx-gateway
 
       - name: Set up Python
         uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
@@ -110,7 +119,7 @@ jobs:
 
       - name: Install Chart
         run: |
-          ct install --config .ct.yaml --helm-extra-set-args="--set=nginxGateway.image.tag=${{ steps.ngf-meta.outputs.version }} \
+          ct install --config .ct.yaml --namespace nginx-gateway --helm-extra-set-args="--set=nginxGateway.image.tag=${{ steps.ngf-meta.outputs.version }} \
           --set=nginx.image.repository=ghcr.io/nginxinc/nginx-gateway-fabric/nginx${{ inputs.image == 'plus' && '-plus' || ''}} \
           --set=nginx.plus=${{ inputs.image == 'plus' }} \
           --set=nginx.image.tag=${{ steps.nginx-meta.outputs.version }} \
@@ -143,10 +152,14 @@ jobs:
           kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f -
           kubectl create namespace nginx-gateway
 
-      - name: Create k8s secret
+      - name: Create plus secrets
         if: ${{ inputs.image == 'plus' }}
+        env:
+          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
         run: |
+          echo "${PLUS_LICENSE}" > license.jwt
           kubectl create secret docker-registry nginx-plus-registry-secret --docker-server=private-registry.nginx.com --docker-username=${{ secrets.JWT_PLUS_REGISTRY }} --docker-password=none -n nginx-gateway
+          kubectl create secret generic nplus-license --from-file license.jwt -n nginx-gateway
 
       - name: Set up Python
         uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0

.github/workflows/nfr.yml

@@ -111,6 +111,12 @@ jobs:
           echo "GKE_NUM_NODES=12" >> vars.env
           echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env
 
+      - name: Setup license file for plus
+        if: matrix.type == 'plus'
+        env:
+          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
+        run: echo "${PLUS_LICENSE}" > license.jwt
+
       - name: Create GKE cluster
         working-directory: ./tests
         run: make create-gke-cluster CI=true

.gitignore

@@ -46,6 +46,9 @@ internal/mode/static/nginx/modules/coverage
 *.crt
 *.key
 
+# JWT files
+*.jwt
+
 # Dotenv files
 **/*.env
 

.hugo_build.lock

@@ -3,6 +3,7 @@ ignore:
   - charts/nginx-gateway-fabric/templates
   - config/crd/bases/
   - deploy/crds.yaml
+  - deploy/*nginx-plus
   - site/static
 
 rules:

.yamllint.yaml

@@ -1,13 +1,12 @@
 # variables that should not be overridden by the user
 VERSION = edge
-SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
+SELF_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 CHART_DIR = $(SELF_DIR)charts/nginx-gateway-fabric
 NGINX_CONF_DIR = internal/mode/static/nginx/conf
 NJS_DIR = internal/mode/static/nginx/modules/src
 KIND_CONFIG_FILE = $(SELF_DIR)config/cluster/kind-cluster.yaml
-NGINX_DOCKER_BUILD_PLUS_ARGS = --secret id=nginx-repo.crt,src=nginx-repo.crt --secret id=nginx-repo.key,src=nginx-repo.key
-BUILD_AGENT=local
-PLUS_ENABLED ?= false
+NGINX_DOCKER_BUILD_PLUS_ARGS = --secret id=nginx-repo.crt,src=$(SELF_DIR)nginx-repo.crt --secret id=nginx-repo.key,src=$(SELF_DIR)nginx-repo.key
+BUILD_AGENT = local
 
 PROD_TELEMETRY_ENDPOINT = oss.edge.df.f5.com:443
 # the telemetry related variables below are also configured in goreleaser.yml
@@ -49,6 +48,8 @@ TARGET ?= local## The target of the build. Possible values: local and container
 OUT_DIR ?= build/out## The folder where the binary will be stored
 GOARCH ?= amd64## The architecture of the image and/or binary. For example: amd64 or arm64
 GOOS ?= linux## The OS of the image and/or binary. For example: linux or darwin
+PLUS_ENABLED ?= false
+PLUS_LICENSE_FILE ?= $(SELF_DIR)license.jwt
 override NGINX_DOCKER_BUILD_OPTIONS += --build-arg NJS_DIR=$(NJS_DIR) --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) --build-arg BUILD_AGENT=$(BUILD_AGENT)
 
 .DEFAULT_GOAL := help
@@ -227,7 +228,9 @@ helm-install-local: install-gateway-crds ## Helm install NGF on configured kind
 
 .PHONY: helm-install-local-with-plus
 helm-install-local-with-plus: install-gateway-crds ## Helm install NGF with NGINX Plus on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build-with-plus.
-	helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --create-namespace --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway --set nginx.plus=true $(HELM_PARAMETERS)
+	kubectl create namespace nginx-gateway || true
+	kubectl -n nginx-gateway create secret generic nplus-license --from-file $(PLUS_LICENSE_FILE) || true
+	helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway --set nginx.plus=true $(HELM_PARAMETERS)
 
 # Debug Targets
 .PHONY: debug-build

Makefile

@@ -6,7 +6,7 @@ ARG NGINX_CONF_DIR
 ARG BUILD_AGENT
 
 RUN apk add --no-cache libcap \
-    && mkdir -p /var/lib/nginx /usr/lib/nginx/modules \
+    && mkdir -p /usr/lib/nginx/modules \
 	&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
 	&& setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx \
     && setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug \
@@ -18,7 +18,7 @@ COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-pages.conf /etc/nginx/grpc-error-pages.conf
 
-RUN chown -R 101:1001 /etc/nginx /var/cache/nginx /var/lib/nginx
+RUN chown -R 101:1001 /etc/nginx /var/cache/nginx
 
 LABEL org.nginx.ngf.image.build.agent="${BUILD_AGENT}"