From 4c623b392d72c05951f05b145b30dd5dcf4421bb Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Tue, 24 Sep 2024 17:01:45 +0100 Subject: [PATCH 01/27] commit poc --- charts/nginx-ingress/values.yaml | 33 +++-- .../transport-server-sni/README.md | 95 +++++++++++++++ .../transport-server-sni/cafe-secret.yaml | 9 ++ .../cafe-transport-server.yaml | 17 +++ .../transport-server-sni/mongo-secret.yaml | 9 ++ .../mongo-transport-server.yaml | 18 +++ .../transport-server-sni/mongo.yaml | 38 ++++++ .../transport-server-sni/tcp-echo-server.yaml | 39 ++++++ internal/configs/transportserver.go | 1 + .../version2/nginx.transportserver.tmpl | 2 + internal/configs/version2/stream.go | 1 + internal/k8s/configuration.go | 114 +++++++++++------- .../validation/transportserver.go | 57 +++++++-- tmp/build-errors.log | 2 + 14 files changed, 370 insertions(+), 65 deletions(-) create mode 100644 examples/custom-resources/transport-server-sni/README.md create mode 100644 examples/custom-resources/transport-server-sni/cafe-secret.yaml create mode 100644 examples/custom-resources/transport-server-sni/cafe-transport-server.yaml create mode 100644 examples/custom-resources/transport-server-sni/mongo-secret.yaml create mode 100644 examples/custom-resources/transport-server-sni/mongo-transport-server.yaml create mode 100644 examples/custom-resources/transport-server-sni/mongo.yaml create mode 100644 examples/custom-resources/transport-server-sni/tcp-echo-server.yaml create mode 100644 tmp/build-errors.log diff --git a/charts/nginx-ingress/values.yaml b/charts/nginx-ingress/values.yaml index ebece1f3a6..feb1539757 100644 --- a/charts/nginx-ingress/values.yaml +++ b/charts/nginx-ingress/values.yaml @@ -127,7 +127,10 @@ controller: logLevel: 1 ## A list of custom ports to expose on the NGINX Ingress Controller pod. Follows the conventional Kubernetes yaml syntax for container ports. - customPorts: [] + customPorts: + - name: port + containerPort: 7000 + protocol: TCP image: ## The image repository of the Ingress Controller. @@ -357,17 +360,20 @@ controller: globalConfiguration: ## Creates the GlobalConfiguration custom resource. Requires controller.enableCustomResources. - create: false + create: true ## The spec of the GlobalConfiguration for defining the global configuration parameters of the Ingress Controller. - spec: {} ## Ensure both curly brackets are removed when adding listeners in YAML format. - # listeners: - # - name: dns-udp - # port: 5353 - # protocol: UDP - # - name: dns-tcp - # port: 5353 - # protocol: TCP + spec: ## Ensure both curly brackets are removed when adding listeners in YAML format. + listeners: + - name: tcp-listener + port: 7000 + protocol: TCP + - name: dns-udp + port: 5353 + protocol: UDP + # - name: dns-tcp + # port: 5353 + # protocol: TCP ## Enable custom NGINX configuration snippets in Ingress, VirtualServer, VirtualServerRoute and TransportServer resources. enableSnippets: false @@ -456,7 +462,12 @@ controller: targetPort: 443 ## A list of custom ports to expose through the Ingress Controller service. Follows the conventional Kubernetes yaml syntax for service ports. - customPorts: [] + customPorts: + - name: tcp-port + port: 7000 + protocol: TCP + targetPort: 7000 + serviceAccount: ## The annotations of the service account of the Ingress Controller pods. diff --git a/examples/custom-resources/transport-server-sni/README.md b/examples/custom-resources/transport-server-sni/README.md new file mode 100644 index 0000000000..b8e29d52ee --- /dev/null +++ b/examples/custom-resources/transport-server-sni/README.md @@ -0,0 +1,95 @@ +TransportServer SNI +In this example we create two different TransportServers that listen on the same interface, which are distinguished by their Host field. + +1. Create a GlobalConfiguration resource with the following listener. + +``` + listeners: + - name: tcp-listener + port: 7000 + protocol: TCP +``` + +2. Add a custom port to the NIC pod via the helm chart + +``` +controller.customPorts: + - name: port + containerPort: 7000 + protocol: TCP +``` + +3. Add a custom port to the NIC service + +``` +controller.service.customPorts: + - name: tcp-port + port: 7000 + protocol: TCP + targetPort: 7000 +``` + +4. Create the cafe-secret, and mongo-secret. These secrets are used for TLS in the TransportServers +`kubectl apply -f cafe-secret.yaml` +`kubectl apply -f mongo-secret.yaml` + +5. Create the mongo and tcp echo example applications +`kubectl apply -f mongo.yaml` +`kubectl apply -f tcp-echo-server.yaml` + +6. Wait until these are ready. +`kubectl get deploy -w` + +7. Create the TransportServers for each application +`kubectl apply -f cafe-transport-server.yaml` +`kubectl apply -f mongo-transport-server.yaml` + +8. Ensure they are in valid state +`kubectl get ts` + +``` +NAME STATE REASON AGE +cafe-ts Valid AddedOrUpdated 2m +mongo-ts Valid AddedOrUpdated 2m +``` + +9. Set up /etc/hosts or DNS +In this example I am running NIC locally, so I will set +cafe.example.com and mongo.example.com in my /etc/hosts to localhost +If running NIC as a live service, the server names would be set at the DNS layer +`cat /etc/hosts` + +``` +... +127.0.0.1 cafe.example.com +127.0.0.1 mongo.example.com +``` + +10. Expose port 7000 of the NIC Loadbalancer Service +`k port-forward svc/my-release-nginx-ingress-controller 7000:7000` + +11. Use ncat to ping cafe.example.com on port 7000 with ssl +`ncat --ssl cafe.example.com 7000` +When you write a message you should get sent back + +``` +hi +hi +``` + +And once the connection is closed (ctrl + c) you should see the request and response in the NIC logs are both 2 bytes + +``` +127.0.0.1 [24/Sep/2024:15:48:58 +0000] TCP 200 3 3 2.702 "- +``` + +12. Use mongosh to connect to the mongodb container, via the TransportServer, also on port 7000 +`mongosh --host mongo.example.com --port 7000 --tls --tlsAllowInvalidCertificates` + +``` +test> show dbs +admin 40.00 KiB +config 60.00 KiB +local 40.00 KiB +test> +``` diff --git a/examples/custom-resources/transport-server-sni/cafe-secret.yaml b/examples/custom-resources/transport-server-sni/cafe-secret.yaml new file mode 100644 index 0000000000..4129b88e76 --- /dev/null +++ b/examples/custom-resources/transport-server-sni/cafe-secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUY1ekNDQTgrZ0F3SUJBZ0lVR2dXT0JNTkR2TXNWOGY3OWc0MlpSZy9KaWc0d0RRWUpLb1pJaHZjTkFRRUwKQlFBd2dZSXhDekFKQmdOVkJBWVRBbGhZTVJJd0VBWURWUVFJREFsVGRHRjBaVTVoYldVeEVUQVBCZ05WQkFjTQpDRU5wZEhsT1lXMWxNUlF3RWdZRFZRUUtEQXREYjIxd1lXNTVUbUZ0WlRFYk1Ca0dBMVVFQ3d3U1EyOXRjR0Z1CmVWTmxZM1JwYjI1T1lXMWxNUmt3RndZRFZRUUREQkJqWVdabExtVjRZVzF3YkdVdVkyOXRNQjRYRFRJME1Ea3kKTXpFd01EUXdNVm9YRFRNME1Ea3lNVEV3TURRd01Wb3dnWUl4Q3pBSkJnTlZCQVlUQWxoWU1SSXdFQVlEVlFRSQpEQWxUZEdGMFpVNWhiV1V4RVRBUEJnTlZCQWNNQ0VOcGRIbE9ZVzFsTVJRd0VnWURWUVFLREF0RGIyMXdZVzU1ClRtRnRaVEViTUJrR0ExVUVDd3dTUTI5dGNHRnVlVk5sWTNScGIyNU9ZVzFsTVJrd0Z3WURWUVFEREJCallXWmwKTG1WNFlXMXdiR1V1WTI5dE1JSUNJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBZzhBTUlJQ0NnS0NBZ0VBbjkvZwpkMml5NEZyaGNtS3ZXQTVHZXFiS1c5WU1xeml6Vm4yRExkd215enkvS2FUdTdtUlkvUWRsTkxHaVhIM1NEa0FkCkhtZFJEK0Zob1ZHaWxoTjEzcTI0azdyRSt6dk1iMlRqdnBnMmVLYVd1dDBHaGQ3V3l3TnpIUU9WcWJCM2o2U2QKSE1sNlJKK25ERHo2Yi9adkRiNm9nQjBucUowZjBWN0tBMHFFalYxVld2dXI4YVZpVlVxK0tBOVMveEdJMHZSSwo2NE9BS2xIUUF4a2xGWWhheHVjcWRsS0owQlRXekE0Z0gzUTZlQi9CRjNOMTJDQjgrMVEzZG54bU5TVnpLbXp1CmJYdm1jK1RPUjg0V3ovemlaMTl1K3RuRHg2aklNajZZQXd2M0tzSVlYbmV1ZEgycUZKRG5FMmZwYjNnZzRxbm0KQnduY0ZlU1poY09nZWdnR1RmKzRTc3YwRUNuamNNdXhteTEwMTNwY0h1a2prTUNmVGJsUGZTU0NNS24rVnlGZQpUNzlSeUkxM3hmL0JXOSt0aTFad1IwYVpkWk5jV1R2Um53R1M1bWhnelJXclhRUmdJQlVVVjk2N3cxRVVONDJVCkE3ZjBVSzZlYks5bExBMnNZWmJJSWUrbllRY1d1OVRuNXo5YnVwK2w3aU5hUDYxWXdwWGdrUTFyNHlSamhMeUEKajVha1Z5VnROYzZmSWltaXlWV0Y1QzBaZnNIYzF3cUpLb2lXQzBtT1dyNEVRSVorRkY2WC9FRytSMmhodjZkQQpGcndpd3BHUE9kWkROQzNqNFBqVzVreWlYMlh2Vm1RWTUvbkxNdFpUaFF6VHpadG8zT3M4d1RsRjFyQlZkd05iCi9zNTh2dnlTQitsYk5Sd215TEErQVYvOGQ2L2VNcGZjS0ZyYU85OENBd0VBQWFOVE1GRXdIUVlEVlIwT0JCWUUKRktEejZlREZLaC93ZDVLTHpPMXpCMDR2UGVRSU1COEdBMVVkSXdRWU1CYUFGS0R6NmVERktoL3dkNUtMek8xegpCMDR2UGVRSU1BOEdBMVVkRXdFQi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dJQkFKQnd5WVlyCllXa3RvRGJPYlpHRUFXT0lEcjFTb3NWRDVIRXdkazV6ZTJuT05oQXFHVWszOHpUT3V1VVZRNk5lT1c1UWtML3oKVTFUa3dwUlNFNjFpbVZHOUdjYWY5ampkSjd5VGhsOGpmZXUyWTF4RHo1OUpmbWJ1WUo2SGE4ZmR4STAxQXZ3RgoxaERYY1ZEcTZoalhhb0pKMVZta1NiTEJPc1JXaGVzWGl1K3FiN3NSOUhlZmJmaTUxandQb2NFblE4YzNiWXF5CmpIRkwxS1JJekIyR2VYVGk2WngwN2lqcDZIeGJSQUFEWnBuRGN0blM1UVQvTGFoYklIZVNNTExMK21vdnUrSUgKTVgwQmtaYmJjRk94L3d2ZlN1SXdveEVXR1RjVGJIYnJGanVUUDRkalpPdFhybElSK1RJTmRiaFhaMm1XZVdHbgpRa0tkcyt4a3puNzNZbG9xZCtPZGZ2d1dJYXVFYktuaXdQUk01NHZqK0dnbndiWmZNWXpnY292cE5BYzZjMmlmCkNNdHlHWDBUREptSjRVWC9FaFFSYTArSURMbFZTMWc2Y3IxWmVQM1N1MWpkSnhBSFVwUjZwSVYrWGdsTENDdGYKQXdDUW9BUWtUeXkyb0Y1Z0hWOGVuc28zVE15cmI5R1NBWDF0UEdJL080L3VCRDBqRzQ4anZsZEI3dzZKbjdmSwoyS21DWnIwYlRDdU9vWnVuZHA3OHA2R1ozb3lVOXBxcTFTUmU0MTdjSlVuNzR3S05TTkd0U0xTNm9OZ3FQQ2h6Cm9ZTy9zK0NHL295c0JUcTBma2VBYXdMZ2oxVG9ybGNsaEt6M05uWUtLZmhDVFBLTDVVUFZLNjVXQ3V2djlSVWgKVUZvM2F2TnhhSmpWUEY4V0FVUnRZem82bXBadFRITm53ZkJTCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUpRd0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQ1Mwd2dna3BBZ0VBQW9JQ0FRQ2YzK0IzYUxMZ1d1RnkKWXE5WURrWjZwc3BiMWd5ck9MTldmWU10M0NiTFBMOHBwTzd1WkZqOUIyVTBzYUpjZmRJT1FCMGVaMUVQNFdHaApVYUtXRTNYZXJiaVR1c1Q3Tzh4dlpPTyttRFo0cHBhNjNRYUYzdGJMQTNNZEE1V3BzSGVQcEowY3lYcEVuNmNNClBQcHY5bThOdnFpQUhTZW9uUi9SWHNvRFNvU05YVlZhKzZ2eHBXSlZTcjRvRDFML0VZalM5RXJyZzRBcVVkQUQKR1NVVmlGckc1eXAyVW9uUUZOYk1EaUFmZERwNEg4RVhjM1hZSUh6N1ZEZDJmR1kxSlhNcWJPNXRlK1p6NU01SAp6aGJQL09KblgyNzYyY1BIcU1neVBwZ0RDL2Nxd2hoZWQ2NTBmYW9Va09jVForbHZlQ0RpcWVZSENkd1Y1Sm1GCnc2QjZDQVpOLzdoS3kvUVFLZU53eTdHYkxYVFhlbHdlNlNPUXdKOU51VTk5SklJd3FmNVhJVjVQdjFISWpYZkYKLzhGYjM2MkxWbkJIUnBsMWsxeFpPOUdmQVpMbWFHRE5GYXRkQkdBZ0ZSUlgzcnZEVVJRM2paUUR0L1JRcnA1cwpyMlVzRGF4aGxzZ2g3NmRoQnhhNzFPZm5QMXU2bjZYdUkxby9yVmpDbGVDUkRXdmpKR09FdklDUGxxUlhKVzAxCnpwOGlLYUxKVllYa0xSbCt3ZHpYQ29rcWlKWUxTWTVhdmdSQWhuNFVYcGY4UWI1SGFHRy9wMEFXdkNMQ2tZODUKMWtNMExlUGcrTmJtVEtKZlplOVdaQmpuK2NzeTFsT0ZETlBObTJqYzZ6ekJPVVhXc0ZWM0Exdit6bnkrL0pJSAo2VnMxSENiSXNENEJYL3gzcjk0eWw5d29XdG83M3dJREFRQUJBb0lDQUNkQmRZQmNlTytWNFIyUkZiVHRiR2paClkzN0JSRU1XblJKenB5NHZqR2NDOTMxbVBqVFM5dmJLUmhOMk9vT3pjVXlHZVovcGhvSDd1VmsvRGtrRFprSFQKTGlzNEJQNGJaTXRGWHBhQ0VYMzJpYlJBYVVXZHZlZ0RaTlNPK01TOXk5MjljY2FMd2pYdmJia1hqL2JGNytiVQpGZE8vVk9tV0N5WUJ2R0NxZjNtbW5UckY2U1pna1pDWDFiRkljZnluZFkwMjV0NkZYNGNFcDZyYkZidi95eXBqCndJMWxIdW0wOURrT2p0eXFVV0VGaXdnVEZiQ0g2YWhjdVhHaWdnWXl0K0NHOXRSelE5YlpLNzE5NFNRWTJBN0IKNUNJOExsSnNJeHdUT29naysvL0h3T3dSUHdqamdrdWllTnJPL1FhZDNKVkxXbXdJQTc1c2J6WGxIeFpYdWhReApFZTlTTVNaRE5JMGMvcGdDVXloL2h5Y3ZYWVJKb1dIMnZYTDYzL1EyRWJVVVZVcXpzcEpjV0xMbkhSSFdBQjZQCmtPb0FMeEs2M3lpQTU5KzREQ0ZZMGM0dXd4WHM5YnZlVWk5UUdxaUtoZkFaNmE3Ymh4ajNvRWIvRDZ0dWZKb1QKMnIxUXFKTlJ4N0RzMmJUc3J6SEgyYmVKR0R4bFJ4djdaNmdSTzdHd2lrU25veDlMcnY3YzMwUHp0WjZENXhwQwpoVHdmM0VpRWlrZ2pOM1FGd2RaUXVqV3RxWXQ1UWVGYUxpL2Q3VXZteFdZNWcrWTYyQnFkNnBpWFFJSlFVbDZUCmZmU0U2OEYvZWF6QnhPQm9INUMvWWcrd1kyVEpTb1BzK3NFUUtnYlNJTmRQNHU4N3FiQ056YWZ4WjFySTN1dmgKR3NZTWpSbm50c3RKamJXVEJkakJBb0lCQVFEZXpsY3dVNFJYbnhKVENSTjFQZUx1UHI2aXFnc09uY1lRRXNuTgpZemdwOXF6eC9SSTJodGZCYnN6TkM2RUZ3eDBDZy8wRnpRbVhqWEQ3VmhQS0UrOHBXMmQzNTg4YTBwU0xSbFBOCkprUnFxR2ovbzhIaEtmTGdCYy9BSGU3dkJEZG1UTXh1d0o1SmFmMWZ3OGtxNEJyRXROTi83SVF3MkNYcFNIK2MKSEhSZnBTM1JidDArbW9xK2RTRW1pU2h3UGgxL2FJbng4ZnBKbTFmaU1QdXhvYStzOXNUVmQ3Ky9GSjdIaWNZSApSQThQRHdQT2RMQ2loQlZFT3liSkduTlpaSFhYQTA4MThhNnpjdjU2aUM2VGZzSkhWZUg4a2Nma1IzblI4eTUvCkx2MnI1QzVaZWJlbW9ZaVhBTC9sVnZ0QUNGMXE3UW1zdlRkQ2R3R1lvaXBMbksvNUFvSUJBUUMzc1YvcFFOVlEKSjNsZExFem9OVUdBZWFwL0NNU2tuY0lUUXNnMUI3SDJHclIyR201VGNuVGE0SE5Rbmt3a2h3YUxyTUlmVDFXZAorMFVvR3M5ZXdyYkZERWtKR0MxQVd2RmFIREk3Q3NOYUFUM2FWYTE5R1ZRWFg2NnkrWEF2QXFTOXpOU25id0J0CnM5UDFFVURwMko1SGtyNlVLUHZjSnY3Q0MyR3J5emZaTy85MFpTTlFQUXdKMXFqZ2NNOUJpMlBYSkE0OENldnMKK3RvaGFqUWFIMStVY0VqNkQ0SHhoQ1RtcTBxM2YvdFNudk9iaTNOYklEWjNEWmR4ejVPelJSVmpLNWlSOGZDNQpzSVVKckxCSXRZeWpUeHAzTzROUmRFaHhzamdBZTZrSTV2dm95RVAvRFZwTFlQU0s3cFpoVjJ4YUpXUkZUUVZzCjhrKzZDbGduWGZDWEFvSUJBUUM5QjY4dFR3NHZFTVNKTW1BUnprbWovQlBkQ2d1TGdRd3pRdDEzcGNCV3lmUDgKOHNycS9BZzlFbllyV0x4cW1Sa1pzMFdPRUdFYzlXRnZ1NTNhaW9NVVFYcE5YcHgxazBkM3lsajY2b2FOUHdpbQpLeGNvbzJCdDlFQklMSjAwcUEwZ2UvUE4yeG53Q3o1dWF6dFhadjhPK0tPZ0d0Z2tZSjM1aUFyTU5jLzkvYlFiCnhjVnJnYzVJdkRNOThJd2dmbktrVDlzSkxGVSs4YzdrRnM3VDYrdVNBV01LQVNqclF1RmJSV1ovYjV5ZkdBd1EKc3l2UkZlSzlHcnBUVUYrZzdmeVVTVGlBK2VWUVZqWFZXNGk0bG9qWjRPRjBXWEtRR0p3Z0pnUEMzK2xVVnFtRQpQQ0kxKzBKWmFzZGtHaUhjTjd5YUpUVmFHc2F4V3lvOWh3Zi9VcFp4QW9JQkFDQ0hDem5ObmpoTVZTUlhsT0xGCmsyekJucHhTSENnZU8yQ1h3Y1lLTDh3cG5HMFJieG5kdWEyTWN6OENXTzlhN2FETUhhL1hwNHlMRXdydi9HcUcKUmtFTVZONkVabmJ2NDY4V01ScmRaQXhMRGYzY2tCVUg2Q2tmYTFzTDZuNllsRDE3eU9oQk1xMDZXNzBZcWdyKwpyY0IwenNTRG9WMnhsZ2tjWk5ZNzdRN05uZ1dwWnlCdFB2VjdDbnA3MzJkMjNGNGJaMTNnVCtPdDQvUm96d01WCkxTS200M1ZNUzdGTnVnOFNvKzlzZlQ5N0lCNGFDbnBIY1AyUjdaQmN0b1hYSk50anUrZVVGUkY4bllKQ0R4RkEKL0w5cVlZQmRqSHBmQWZrSUd2eVM2VExIWERJelREOGN5VEZ4NEx1OVZlbTB4bDRNSXY1V2pqQmxsQktZaEZXcwpQODhDZ2dFQkFKUjR5UUIrREtoRE1ZOGlHOXpYWUJpWmZ5MFI2c21oeEJycTREZWd4cElkajBydHUyMFNuWDdsCnN3YlZsN2NmdWRxUi9tVEhnZ1lYeGJSS3UxUXhmN3NHTW4zWVpRY1AxZDVJelFMOWZFQitidGJsdWY5VTB4NDIKUzM0U0l6Z25pd0hPQnN3M2ZLYm1Gbnc1dmtNd2RoemlBT1R0elRwcmU3ckdhaEpwZlk0eFRWWjZpK3pjQ1pCWgo0ZHhSTjlQWnBiQm4vNmJDTFY3S0I3ZmY0Uk04b3c3K1l4aHFFTFhxcnVQS2paMnRRSWs2M2hzZklMd0tiRWIrCkhEM2VLZEhNc2hhbXFXZzI0NnlORW5nQUZCQzhoVU5kQ0pmeUthUlhkSlV5eDhqcWlKM3ErdnpnNFBieUhqeW0KZEQzM2pVT2UwdHdoSWJxKytUaXZCa1ptSDBsSStnVT0KLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= +kind: Secret +metadata: + creationTimestamp: null + name: cafe-secret +type: kubernetes.io/tls diff --git a/examples/custom-resources/transport-server-sni/cafe-transport-server.yaml b/examples/custom-resources/transport-server-sni/cafe-transport-server.yaml new file mode 100644 index 0000000000..638d6496a8 --- /dev/null +++ b/examples/custom-resources/transport-server-sni/cafe-transport-server.yaml @@ -0,0 +1,17 @@ +apiVersion: k8s.nginx.org/v1 +kind: TransportServer +metadata: + name: cafe-ts +spec: + host: cafe.example.com + listener: + name: tcp-listener + protocol: TCP + tls: + secret: cafe-secret + upstreams: + - name: tcp-echo + service: tcp-echo-service + port: 7000 + action: + pass: tcp-echo diff --git a/examples/custom-resources/transport-server-sni/mongo-secret.yaml b/examples/custom-resources/transport-server-sni/mongo-secret.yaml new file mode 100644 index 0000000000..54c0c3d964 --- /dev/null +++ b/examples/custom-resources/transport-server-sni/mongo-secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUY2VENDQTlHZ0F3SUJBZ0lVTGRHUktVc3FrZktiV1JheWwvaTdrSkVvWnRjd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2dZTXhDekFKQmdOVkJBWVRBbGhZTVJJd0VBWURWUVFJREFsVGRHRjBaVTVoYldVeEVUQVBCZ05WQkFjTQpDRU5wZEhsT1lXMWxNUlF3RWdZRFZRUUtEQXREYjIxd1lXNTVUbUZ0WlRFYk1Ca0dBMVVFQ3d3U1EyOXRjR0Z1CmVWTmxZM1JwYjI1T1lXMWxNUm93R0FZRFZRUUREQkZ0YjI1bmJ5NWxlR0Z0Y0d4bExtTnZiVEFlRncweU5EQTUKTWpNeE1EQTFNVGxhRncwek5EQTVNakV4TURBMU1UbGFNSUdETVFzd0NRWURWUVFHRXdKWVdERVNNQkFHQTFVRQpDQXdKVTNSaGRHVk9ZVzFsTVJFd0R3WURWUVFIREFoRGFYUjVUbUZ0WlRFVU1CSUdBMVVFQ2d3TFEyOXRjR0Z1CmVVNWhiV1V4R3pBWkJnTlZCQXNNRWtOdmJYQmhibmxUWldOMGFXOXVUbUZ0WlRFYU1CZ0dBMVVFQXd3UmJXOXUKWjI4dVpYaGhiWEJzWlM1amIyMHdnZ0lpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElDRHdBd2dnSUtBb0lDQVFEUAppQngwMDU3MjB2S2JVMGJsZUNjcTg2N0RJREkzbnZ1VjdiMVhCaEJHOFJ5S1MveDREc2pPYjMwaVZtVm1MR29xCm5WVnk5bnN5VDBEMmhydnE2eUp4VEtBMmtTNFVOd3JJZDBhc3FqSmxITWFpbkNmbmtGLzB4RmwxZERvYUEydncKelZZYnpQd3NBRlQrZGk2cG1hcnFQVFlNditiL0NhQnFib3dBQWFtUjhLemdGRTZDSFNZc1EwSzFEV2xhbmRBOAoxRXlFT0p0dndjQXFjVHVXTXl6eGMzUUx6dmM4Z0hGcG1NdlNEUUtvL2xpbytUYlFaOW1ldXJKTTY0dUVtRThzCmloVDNxREo2OUE0VkpDQWs0cGVmZ0p6cjNnN1ExOTBvVmpwOGtXZ0NIMkl3OHdXUGVjYUVySXFHcmo2YkNzcHUKSHVQaGlaQnB4aDlmOUhmWCt1NHBpNytGU2VPaC8xU0xEdE9qVzAwV256TnFJSGJvSmhLd2MyRXU3Z2xkc2VNdgp5a0VQbWtKNDkyNXN1Rmsxc3U5RWdDeDBhM1JwWCtxNERsSjFHaEl6QUJja0hOVXplUXJWVGdYTHkwdk1wZGdCClVXU3hkMWdGdGM2ZGp6WEoxZUxwcEVPaEtQdmQrVWN2eEpPb2NLekx3a1BGNmNueVl2NElFUnFZWE9UUktlNFcKamFKR05xWisvNGRJQXh4M0UxamtyZUR0UUNzMVByaFBLaGpVdVlKaVI5UFFsdEFUa2w4MXQ1NzEyeDExYUQ0SgpNMWRsOXdoSUlNcjFoQzF6Wms5SkZXdkluRytCTEdBY3d4cjduY2pLTEpzcjBpS3R2aFEzbndWdG9GWnlzWERCClBBdU5QclFMM0JaT3JKanpvV3FJdU5lNVRqY3FiTG9WREx2NzMwMytXUUlEQVFBQm8xTXdVVEFkQmdOVkhRNEUKRmdRVVpjTjdpUitRNGkrb3U2YS9aR01JUXQrRWJRY3dId1lEVlIwakJCZ3dGb0FVWmNON2lSK1E0aStvdTZhLwpaR01JUXQrRWJRY3dEd1lEVlIwVEFRSC9CQVV3QXdFQi96QU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FnRUFKYVRXCnFENnEyRm51RmtJcmJIbkxPdFdFTDUxTC9QMncyUWZqMTcrSWlJeDRNd3hzS082ZEtHbUdlazY2Sy9sSjdBaEIKWngyRkFiOFdERmpPeVFtL3Axc1lQME82L1RyUzYrYjRlbndaR3p3YXVoVWdXR0M3djByRlp0VnNMNm5pdGJkMAp6VExGZ1V6QkJHR3cwZ3E2ekF4ZG5BNXo1VkFyYjhzcGZFNzZ1Q1FWbEdYVXhua0Fka2FYVXlOYXU1YXo0VW1WClMzRjIvaG53RG5XYUpZUmxqMTJ2SFYxWmxiamwxMzE3Uk0rRHZMSkRwV29JTkdzRzd0SG9pTGQ1dXJJeWYxZ0oKZXdJN1FXYTQ5UE1MWCszYnNSbjQ4dnJZUGxqWGcvdkhqYlZvUWlzMmFCak1sSDNCczdMVGVvVWZqVERsU2c3cApHd3ZzaW41dWJPZFAvTGRrWllaTFhiMWkvT1E3cjFpZTJ3OTQyd1RZT3NKRk5QVlB4N3JPV1BNeHd0Q0tUeE41Ck5URXFPM0F5dXZpUW9tRVhhL1RyL2xybnFSWS8rcXNpL2J2d01QWWlIS0RiTmJMM1NQaSt4bmU0Y3NxQ0d5L3QKMGVsaHVYVHpDTXUvcXdvM2ZNVXc4VFBBMXcvOEZTMWh4ZWkzY0JhL3VxQzFPS2s3aDRTLzJocXljaFd5ekhIRApMNUdob2RGZ0NkVTZaSUZJSnJzQS9uQ09maGZWUXJJMFRRZWlMSHR6OXd0T2tWTXhPeWRpcGpVMEgySjErRElwCi8vQzUrM2FpNWlxTkZnUTNZNEp5SGFlYmh3NW5lcjh1U3JmN3NnMit1a1dVU0QxamhnRVQxVWEzTTNUeG00bmYKNXhXOFV1aTFDYjVMdEM2clgvV1A4SG4veTJid2Iyc0swMGJYb1Y4PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUpRZ0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQ1N3d2dna29BZ0VBQW9JQ0FRRFBpQngwMDU3MjB2S2IKVTBibGVDY3E4NjdESURJM252dVY3YjFYQmhCRzhSeUtTL3g0RHNqT2IzMGlWbVZtTEdvcW5WVnk5bnN5VDBEMgpocnZxNnlKeFRLQTJrUzRVTndySWQwYXNxakpsSE1haW5DZm5rRi8weEZsMWREb2FBMnZ3elZZYnpQd3NBRlQrCmRpNnBtYXJxUFRZTXYrYi9DYUJxYm93QUFhbVI4S3pnRkU2Q0hTWXNRMEsxRFdsYW5kQTgxRXlFT0p0dndjQXEKY1R1V015enhjM1FMenZjOGdIRnBtTXZTRFFLby9saW8rVGJRWjltZXVySk02NHVFbUU4c2loVDNxREo2OUE0VgpKQ0FrNHBlZmdKenIzZzdRMTkwb1ZqcDhrV2dDSDJJdzh3V1BlY2FFcklxR3JqNmJDc3B1SHVQaGlaQnB4aDlmCjlIZlgrdTRwaTcrRlNlT2gvMVNMRHRPalcwMFduek5xSUhib0poS3djMkV1N2dsZHNlTXZ5a0VQbWtKNDkyNXMKdUZrMXN1OUVnQ3gwYTNScFgrcTREbEoxR2hJekFCY2tITlV6ZVFyVlRnWEx5MHZNcGRnQlVXU3hkMWdGdGM2ZApqelhKMWVMcHBFT2hLUHZkK1VjdnhKT29jS3pMd2tQRjZjbnlZdjRJRVJxWVhPVFJLZTRXamFKR05xWisvNGRJCkF4eDNFMWprcmVEdFFDczFQcmhQS2hqVXVZSmlSOVBRbHRBVGtsODF0NTcxMngxMWFENEpNMWRsOXdoSUlNcjEKaEMxelprOUpGV3ZJbkcrQkxHQWN3eHI3bmNqS0xKc3IwaUt0dmhRM253VnRvRlp5c1hEQlBBdU5QclFMM0JaTwpySmp6b1dxSXVOZTVUamNxYkxvVkRMdjczMDMrV1FJREFRQUJBb0lDQUU3ZDJYNldPMmR1WkE4ZUZ5ZTJRU0JFCkNlcVNUak13QWtrSVYzZCtVT284ejgxSXNqSEg0SXorOW0xNXFzQW82ZEczQjlXUUVPSmVGd0I0MUdvaW9HeXgKSTRPSktaczZEYW1BRm9ZZ2lkVStHY2lMRW1rZ1J5OEQvVUV6QWErSUZGbW5GdTJxdVR4WmhmTkw0MURGbXB1NAoxbFVEQ3B4cVFxR2YwQ2xpZUZnRFFCZEo4RW5uSE80ZVEzZjltRWQ5Q0xsTkxxVGl4RU0wdkx3RVd4SXA4WTd5CmdxdklJOUhFdUJUYW9iNTUva1JOb0ZEYW9IZVR0N0pvSGNFNGxFVTRBb0taR1AzQzJDZzhuaXR2bHAyZDFPUWoKSXI5S0hKUkdMSUFiUU0rOURHc2VGUmtvQ2Jsc0hFS29OVjZZVWlkbWN1WmxhOUYyajBCN0w4b3Q0K3RhcTIzRApYendTSWFrWXBlaFVrYlRrbDJDak9FTHkxaCsrOFYzcFZqak5LdW9pRHhKWTdSSzFMQk9obExlSG5YUzlGY2hGCmVGRVZlb0c2ZlY0QWtFdGJocDJjRExacGF0UWg5ZUNzbmhUT2d2a002WjZ2ZWlTcGpvTUtleGM3bnRVZEJyWFAKQ3lSemVkSEFaaEh5MnBvREFvYU80aW5yOEI2U2VwWlJiaTBPK1ZXQXJkNDJ3UWhXdWZRUFQ1VExVSEoyR3VjeQpkME5wZk9VRXJYT3RVdU11NFNRU2lQbHJ6Ykx3NThrOFdKUVU2TXRGTnpXQ29yTG9TNWVmNU9LclBFYXh3SmhNCnh5Q1FPaWhLQk1SWE1IUXNib2JQZjhOQlJqYVNtK1ZzL0ZpOXdiYnVLZTVpYUhQenB2YVlzMmpqYzcyYWIzMVcKL2V4cUtSQmtEUVQvcmRPSHdJWHZBb0lCQVFEKzdIKzJyM3EwVVMvY0VHalZlSlovdzJXQzJJRCsxNGY3eHdvKwo5V05iWUdyY3NoZTRVSC9JSVpEL3RHQ2MrZ01mSlhTckk5OVhXejZ5Y0p5YXhTdTFsQmxsWEduTk5JM1AwYXpPCmdRc2FjenpjR1JLdk5WTWIyZmJDVEdHR3JVMXZjQVNET1ltZDZha3laemxBaWN4aHI3bStuMWZrLzY1bWFBeEgKTFBleXQwYWMydEdVL293MWJwV1hvRi9OL05QMTRsZ2ZMU2pXQmpWVEQ0bnByenNBU1RsK1hCLzRuK01wUG9CZwo4K2poYW5mUHFDY2MyQW5VMXAvTGZKSDZ6MWJkcHpDNVVQaFF6K2t3a2gwa3Q1cDRQOW5kV3FIWkw1cDlzS1J6Cld3S3pVQld4VjB3VDBOQzRnRlBtYzJaRGpTR0hlSXJLeWx1YVdIS2w4eVcwNGRldkFvSUJBUURRYUdVSlBtT08KQlZzUFROanhTdStzVGg4czNUZFhLUVIwQU02Y0l4UmVrVHpaODg2L3ptUFhlQThsNmROV2c1L0hCTE4rbUllNApnZnV2YWxnMEh5TWI3TC8vbEtYcHpFQTRncVdxU29MaXd4c1JHcUU1NFNud1g3QXV6K0NsT1lTUlNSQXpmeC9BCnE2QS9rOHR6SzN5NVF4Rk5Sc0NFak1GdWY5ak1yWCs0aXU0c3JvZEdUMXkvRE9hZ2twRmphVWZpL2NiaTM4V2sKdVhvK0ZoYmV1Yys0N1VzWjRXWDJSOGg1Uy9sUVhBYkxKU0lwMjlSZy9EK2pZdUllZTZXbk9pdzZWVkRweHhlYgp6RDh3SGVJTHhuRHdZbXprTzVrSG5ZVU5ubUtmWFovU1NpdWpkN0ZvQ1pMZld0R1gzQlFSdVNtRHlQSnBmSTlKCmVtNTl3bkkxVEFSM0FvSUJBUUNBa1l5VCtZcThPSm9YdGhyNVZ2a29kTWJVcUJiZThKci9xOUlLRUw3TWppTTMKTFliekNYNTQxQjBLS2RIME9jK3JQTHZMdUtyaXB2MUhCNjZrREQ5UU0rSmZFYTIydGdPenhYOFBJMXdUT2YxKwowQkp4VlVhV0xHYmNkYU5XUmo5Z3JiRkk4WkxybHJZajJwV3diQTh0VVhBdnFMT3VwaGt5UXRXMmJBSjlHeHc4CjdjdDRCcTEySVZESENUWm9jRlFDbGVaMXl0UG1wWGp0YkUvVkVQQ0Q0MnBneFZ2R2kvVUlqeUkyUTYyM1NuZ2sKdmgwRDJoMlRQNitWOUR5M3J5eVRXOGdpSHFrdU1MM2VKa01XaXBjWWdMT0RoVHROaTBteWpJMVVOSmwzRURQdwpuaU9iZHR4ZHdUTVBiaklzYlpoMGQ2SWdSdERPVmo5MFhONHVqUnkzQW9JQkFFNTRRREt1endWV2R1Ylg1SWRWCjA5Qm95Y3cycnZPZWVoTERpd2UzSGFCTnh6KzVVUXRmUnJDR2dBMmljUFNPTXNiWXVremNXWjNiTTB6bEdiam0KVUczZlFwdVUrTE9ET0ZzT3RobmNYRlBOYW8rU0cwcVR3UnJFcksyemo3NG1YZ2ZtSHJlRkVndVZrNHpjdFNuMApJYzRQdHFBR0Y4N1F3TFErWnY4S0JLRVRqb0k4WktyUWp2ZFFnRFhOZWZpWVYzemNXTnBycnh0S3l3QTlpUGJyCnQ0N0ZxaFZnak9laU41V1VTWmM4VDBLR0JNc0YvbjFWL1JBajEwZnEvb0Jzb3VLRDVTZGcwejdTTktpRlYrdGYKR0g4cVVCM1BZdHMvTUMza2lQWEFac0RqTkhNa1NpUUdGc3NLZ3doTzBTK3JMRHAybXUraytyNkwzclp6VkZWRQovaGtDZ2dFQUxpdGI5U29mLy9HenBpcmpTMW16SXNRWUxBWW9ETkZGNE5wYzV1Q0pDWEJJTmVFQnl4YlVzZFRVCkRyTnZ2NG1oNTF5SU1WVHVuM3MzKzFUU25lVjZwdjlIS3JicTlEdU85YXBUOFZnV1E5ZGU5YkVaU0lLVG5iVE0KcUVoLzNIelpKcmNRWjMxWEdTZm5NN0NsVjdPQWM4bVNiT3M4NDlGLzg4UEg0Y3VrYVRWNmZPUnFOaW54RXhuZQpqbnk3ajUxT01aWTQ4TVIvaGJhOVk3QWM3TE1Ec0pQdFZ5cStBemU4ODRreU5ZV2dtZHJHanBaVnhLeDQzMjFJCmhXM1NUam9SZEZBMFVCdENyT2ZUalF0cThPQzZqL1QrYWdwa0g2VjFQVkw3Y29PbjREU2RIb1RqdnJFZ0QrYTgKVzY1ODFvdHdha2tMK0FGalU0MkVHaTkxWnJLSUx3PT0KLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= +kind: Secret +metadata: + creationTimestamp: null + name: mongo-secret +type: kubernetes.io/tls diff --git a/examples/custom-resources/transport-server-sni/mongo-transport-server.yaml b/examples/custom-resources/transport-server-sni/mongo-transport-server.yaml new file mode 100644 index 0000000000..4dd3608e58 --- /dev/null +++ b/examples/custom-resources/transport-server-sni/mongo-transport-server.yaml @@ -0,0 +1,18 @@ + +apiVersion: k8s.nginx.org/v1 +kind: TransportServer +metadata: + name: mongo-ts +spec: + host: mongo.example.com + tls: + secret: mongo-secret + listener: + name: tcp-listener + protocol: TCP + upstreams: + - name: mongo + service: mongodb + port: 27017 + action: + pass: mongo diff --git a/examples/custom-resources/transport-server-sni/mongo.yaml b/examples/custom-resources/transport-server-sni/mongo.yaml new file mode 100644 index 0000000000..ef3498704f --- /dev/null +++ b/examples/custom-resources/transport-server-sni/mongo.yaml @@ -0,0 +1,38 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: mongodb +spec: + selector: + app: mongodb + ports: + - protocol: TCP + port: 27017 + targetPort: 27017 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mongodb +spec: + replicas: 1 + selector: + matchLabels: + app: mongodb + template: + metadata: + labels: + app: mongodb + spec: + containers: + - name: mongodb + image: mongo:latest + ports: + - containerPort: 27017 + volumeMounts: + - name: storage + mountPath: /data/db + volumes: + - name: storage + emptyDir: {} diff --git a/examples/custom-resources/transport-server-sni/tcp-echo-server.yaml b/examples/custom-resources/transport-server-sni/tcp-echo-server.yaml new file mode 100644 index 0000000000..4be65f2f56 --- /dev/null +++ b/examples/custom-resources/transport-server-sni/tcp-echo-server.yaml @@ -0,0 +1,39 @@ + apiVersion: apps/v1 + kind: Deployment + metadata: + name: tcp-echo-server + spec: + replicas: 1 + selector: + matchLabels: + app: tcp-echo-server + template: + metadata: + labels: + app: tcp-echo-server + spec: + containers: + - name: tcp-echo-server + image: alpine + command: ["/bin/sh"] + args: + - -c + - | + apk add --no-cache socat + echo "TCP Echo Server starting..." + socat -v tcp-listen:7000,fork,reuseaddr,keepalive exec:'/bin/cat' + ports: + - containerPort: 7000 +--- +apiVersion: v1 +kind: Service +metadata: + name: tcp-echo-service +spec: + selector: + app: tcp-echo-server + ports: + - protocol: TCP + port: 7000 + targetPort: 7000 + type: ClusterIP diff --git a/internal/configs/transportserver.go b/internal/configs/transportserver.go index c78db4e217..b5fa31a552 100644 --- a/internal/configs/transportserver.go +++ b/internal/configs/transportserver.go @@ -101,6 +101,7 @@ func generateTransportServerConfig(p transportServerConfigParams) (*version2.Tra tsConfig := &version2.TransportServerConfig{ Server: version2.StreamServer{ + ServerName: p.transportServerEx.TransportServer.Spec.Host, TLSPassthrough: p.transportServerEx.TransportServer.Spec.Listener.Name == conf_v1.TLSPassthroughListenerName, UnixSocket: generateUnixSocket(p.transportServerEx), Port: p.listenerPort, diff --git a/internal/configs/version2/nginx.transportserver.tmpl b/internal/configs/version2/nginx.transportserver.tmpl index eff9df8665..d00b262d71 100644 --- a/internal/configs/version2/nginx.transportserver.tmpl +++ b/internal/configs/version2/nginx.transportserver.tmpl @@ -26,6 +26,8 @@ server { {{ makeTransportListener $s | printf }} {{- end }} + server_name {{ $s.ServerName }}; + {{- if $ssl.Enabled }} ssl_certificate {{ makeSecretPath $ssl.Certificate $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; ssl_certificate_key {{ makeSecretPath $ssl.CertificateKey $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; diff --git a/internal/configs/version2/stream.go b/internal/configs/version2/stream.go index ef2e67daba..b7d7ff9e8b 100644 --- a/internal/configs/version2/stream.go +++ b/internal/configs/version2/stream.go @@ -37,6 +37,7 @@ type StreamUpstreamBackupServer struct { // StreamServer defines a server in the stream module. type StreamServer struct { + ServerName string TLSPassthrough bool UnixSocket string Port int diff --git a/internal/k8s/configuration.go b/internal/k8s/configuration.go index b600213754..d71791451c 100644 --- a/internal/k8s/configuration.go +++ b/internal/k8s/configuration.go @@ -101,6 +101,16 @@ type IngressConfiguration struct { ChildWarnings map[string][]string } +type listenerHostKey struct { + ListenerName string + Host string +} + +// used for sorting +func (lhk listenerHostKey) String() string { + return fmt.Sprintf("%s|%s", lhk.ListenerName, lhk.Host) +} + // NewRegularIngressConfiguration creates an IngressConfiguration from an Ingress resource. func NewRegularIngressConfiguration(ing *networking.Ingress) *IngressConfiguration { return &IngressConfiguration{ @@ -333,7 +343,7 @@ type TransportServerMetrics struct { // of the objects in the Configuration. type Configuration struct { hosts map[string]Resource - listeners map[string]*TransportServerConfiguration + listeners map[listenerHostKey]*TransportServerConfiguration listenerMap map[string]conf_v1.Listener // only valid resources with the matching IngressClass are stored @@ -389,7 +399,7 @@ func NewConfiguration( ) *Configuration { return &Configuration{ hosts: make(map[string]Resource), - listeners: make(map[string]*TransportServerConfiguration), + listeners: make(map[listenerHostKey]*TransportServerConfiguration), ingresses: make(map[string]*networking.Ingress), virtualServers: make(map[string]*conf_v1.VirtualServer), virtualServerRoutes: make(map[string]*conf_v1.VirtualServerRoute), @@ -766,15 +776,11 @@ func (c *Configuration) rebuildListeners() ([]ResourceChange, []ConfigurationPro return changes, newOrUpdatedProblems } -func (c *Configuration) buildListenersAndTSConfigurations() (newListeners map[string]*TransportServerConfiguration, newTSConfigs map[string]*TransportServerConfiguration) { - newListeners = make(map[string]*TransportServerConfiguration) - newTSConfigs = make(map[string]*TransportServerConfiguration) +func (c *Configuration) buildListenersAndTSConfigurations() (map[listenerHostKey]*TransportServerConfiguration, map[string]*TransportServerConfiguration) { + newListeners := make(map[listenerHostKey]*TransportServerConfiguration) + newTSConfigs := make(map[string]*TransportServerConfiguration) for key, ts := range c.transportServers { - if ts.Spec.Listener.Protocol == conf_v1.TLSPassthroughListenerProtocol { - continue - } - tsc := NewTransportServerConfiguration(ts) newTSConfigs[key] = tsc @@ -782,8 +788,8 @@ func (c *Configuration) buildListenersAndTSConfigurations() (newListeners map[st continue } - found := false var listener conf_v1.Listener + found := false for _, l := range c.globalConfiguration.Spec.Listeners { if ts.Spec.Listener.Name == l.Name && ts.Spec.Listener.Protocol == l.Protocol { listener = l @@ -800,17 +806,21 @@ func (c *Configuration) buildListenersAndTSConfigurations() (newListeners map[st tsc.IPv4 = listener.IPv4 tsc.IPv6 = listener.IPv6 - holder, exists := newListeners[listener.Name] + host := ts.Spec.Host + listenerKey := listenerHostKey{ListenerName: listener.Name, Host: host} + + holder, exists := newListeners[listenerKey] if !exists { - newListeners[listener.Name] = tsc + newListeners[listenerKey] = tsc continue } - warning := fmt.Sprintf("listener %s is taken by another resource", listener.Name) + // another TransportServer exists with the same listener and host + warning := fmt.Sprintf("listener %s and host %s are taken by another resource", listener.Name, host) if !holder.Wins(tsc) { holder.AddWarning(warning) - newListeners[listener.Name] = tsc + newListeners[listenerKey] = tsc } else { tsc.AddWarning(warning) } @@ -973,7 +983,7 @@ func (c *Configuration) findResourcesForResourceReference(namespace string, name } } - for _, l := range getSortedTransportServerConfigurationKeys(c.listeners) { + for _, l := range getSortedListenerHostKeys(c.listeners) { tsConfig := c.listeners[l] if checker.IsReferencedByTransportServer(namespace, name, tsConfig.TransportServer) { @@ -1061,15 +1071,21 @@ func detectChangesInProblems(newProblems map[string]ConfigurationProblem, oldPro return result } -func (c *Configuration) addProblemsForTSConfigsWithoutActiveListener(tsConfigs map[string]*TransportServerConfiguration, problems map[string]ConfigurationProblem) { +func (c *Configuration) addProblemsForTSConfigsWithoutActiveListener( + tsConfigs map[string]*TransportServerConfiguration, + problems map[string]ConfigurationProblem, +) { for _, tsc := range tsConfigs { - holder, exists := c.listeners[tsc.TransportServer.Spec.Listener.Name] + listenerName := tsc.TransportServer.Spec.Listener.Name + host := tsc.TransportServer.Spec.Host + key := listenerHostKey{ListenerName: listenerName, Host: host} + holder, exists := c.listeners[key] if !exists { p := ConfigurationProblem{ Object: tsc.TransportServer, IsError: false, Reason: "Rejected", - Message: fmt.Sprintf("Listener %s doesn't exist", tsc.TransportServer.Spec.Listener.Name), + Message: fmt.Sprintf("Listener %s with host %s doesn't exist", listenerName, host), //TODO: rethink this error as the user doesnt define the listener with host struct } problems[tsc.GetKeyWithKind()] = p continue @@ -1080,7 +1096,7 @@ func (c *Configuration) addProblemsForTSConfigsWithoutActiveListener(tsConfigs m Object: tsc.TransportServer, IsError: false, Reason: "Rejected", - Message: fmt.Sprintf("Listener %s is taken by another resource", tsc.TransportServer.Spec.Listener.Name), + Message: fmt.Sprintf("Listener %s with host %s is taken by another resource", listenerName, host), } problems[tsc.GetKeyWithKind()] = p } @@ -1301,8 +1317,12 @@ func createResourceChangesForHosts(removedHosts []string, updatedHosts []string, return append(deleteChanges, changes...) } -func createResourceChangesForListeners(removedListeners []string, updatedListeners []string, addedListeners []string, oldListeners map[string]*TransportServerConfiguration, - newListeners map[string]*TransportServerConfiguration, +func createResourceChangesForListeners( + removedListeners []listenerHostKey, + updatedListeners []listenerHostKey, + addedListeners []listenerHostKey, + oldListeners map[listenerHostKey]*TransportServerConfiguration, + newListeners map[listenerHostKey]*TransportServerConfiguration, ) []ResourceChange { var changes []ResourceChange var deleteChanges []ResourceChange @@ -1758,6 +1778,20 @@ func getSortedTransportServerConfigurationKeys(m map[string]*TransportServerConf return keys } +func getSortedListenerHostKeys(m map[listenerHostKey]*TransportServerConfiguration) []listenerHostKey { + var keys []listenerHostKey + + for k := range m { + keys = append(keys, k) + } + + sort.Slice(keys, func(i, j int) bool { + return keys[i].String() < keys[j].String() + }) + + return keys +} + func detectChangesInHosts(oldHosts map[string]Resource, newHosts map[string]Resource) (removedHosts []string, updatedHosts []string, addedHosts []string) { for _, h := range getSortedResourceKeys(oldHosts) { _, exists := newHosts[h] @@ -1806,31 +1840,29 @@ func detectChangesInHosts(oldHosts map[string]Resource, newHosts map[string]Reso return removedHosts, updatedHosts, addedHosts } -func detectChangesInListeners(oldListeners map[string]*TransportServerConfiguration, newListeners map[string]*TransportServerConfiguration) (removedListeners []string, - updatedListeners []string, addedListeners []string, -) { - for _, l := range getSortedTransportServerConfigurationKeys(oldListeners) { - _, exists := newListeners[l] - if !exists { +func detectChangesInListeners( + oldListeners map[listenerHostKey]*TransportServerConfiguration, + newListeners map[listenerHostKey]*TransportServerConfiguration, +) (removedListeners []listenerHostKey, updatedListeners []listenerHostKey, addedListeners []listenerHostKey) { + oldKeys := getSortedListenerHostKeys(oldListeners) + newKeys := getSortedListenerHostKeys(newListeners) + + oldKeysSet := make(map[listenerHostKey]struct{}) + for _, l := range oldKeys { + oldKeysSet[l] = struct{}{} + if _, exists := newListeners[l]; !exists { removedListeners = append(removedListeners, l) } } - for _, l := range getSortedTransportServerConfigurationKeys(newListeners) { - _, exists := oldListeners[l] - if !exists { + for _, l := range newKeys { + if _, exists := oldListeners[l]; !exists { addedListeners = append(addedListeners, l) - } - } - - for _, l := range getSortedTransportServerConfigurationKeys(newListeners) { - oldR, exists := oldListeners[l] - if !exists { - continue - } - - if !oldR.IsEqual(newListeners[l]) { - updatedListeners = append(updatedListeners, l) + } else { + oldR := oldListeners[l] + if !oldR.IsEqual(newListeners[l]) { + updatedListeners = append(updatedListeners, l) + } } } diff --git a/pkg/apis/configuration/validation/transportserver.go b/pkg/apis/configuration/validation/transportserver.go index a4dac4f54c..29f9dec2d4 100644 --- a/pkg/apis/configuration/validation/transportserver.go +++ b/pkg/apis/configuration/validation/transportserver.go @@ -38,7 +38,7 @@ func (tsv *TransportServerValidator) validateTransportServerSpec(spec *conf_v1.T allErrs := tsv.validateTransportListener(&spec.Listener, fieldPath.Child("listener")) isTLSPassthroughListener := isPotentialTLSPassthroughListener(&spec.Listener) - allErrs = append(allErrs, validateTransportServerHost(spec.Host, fieldPath.Child("host"), isTLSPassthroughListener)...) + allErrs = append(allErrs, validateTransportServerHost(spec.Host, fieldPath.Child("host"), isTLSPassthroughListener, spec.Listener.Protocol, spec.TLS)...) upstreamErrs, upstreamNames := validateTransportServerUpstreams(spec.Upstreams, fieldPath.Child("upstreams"), tsv.isPlus) allErrs = append(allErrs, upstreamErrs...) @@ -57,22 +57,37 @@ func (tsv *TransportServerValidator) validateTransportServerSpec(spec *conf_v1.T allErrs = append(allErrs, validateSnippets(spec.StreamSnippets, fieldPath.Child("streamSnippets"), tsv.snippetsEnabled)...) - allErrs = append(allErrs, validateTLS(spec.TLS, isTLSPassthroughListener, fieldPath.Child("tls"))...) + hostSpecified := spec.Host != "" + allErrs = append(allErrs, validateTLS(spec.TLS, isTLSPassthroughListener, fieldPath.Child("tls"), hostSpecified)...) return allErrs } -func validateTLS(tls *conf_v1.TransportServerTLS, isTLSPassthrough bool, fieldPath *field.Path) field.ErrorList { - if tls == nil { +func validateTLS( + tls *conf_v1.TransportServerTLS, + isTLSPassthroughListener bool, + fieldPath *field.Path, + hostSpecified bool, +) field.ErrorList { + if isTLSPassthroughListener { + if tls != nil { + return field.ErrorList{field.Forbidden(fieldPath, "cannot specify tls for TLS Passthrough TransportServers")} + } return nil } - if isTLSPassthrough { - return field.ErrorList{field.Forbidden(fieldPath, "cannot specify secret for tls passthrough")} + + if hostSpecified { + if tls == nil || tls.Secret == "" { + return field.ErrorList{field.Required(fieldPath, "must specify spec.tls.secret when host is specified")} + } + return validateSecretName(tls.Secret, fieldPath.Child("secret")) } - if tls.Secret == "" { - return field.ErrorList{field.Required(fieldPath, "must specify secret for tls")} + + if tls != nil && tls.Secret != "" { + return validateSecretName(tls.Secret, fieldPath.Child("secret")) } - return validateSecretName(tls.Secret, fieldPath.Child("secret")) + + return nil } func validateSnippets(serverSnippet string, fieldPath *field.Path, snippetsEnabled bool) field.ErrorList { @@ -82,14 +97,30 @@ func validateSnippets(serverSnippet string, fieldPath *field.Path, snippetsEnabl return nil } -func validateTransportServerHost(host string, fieldPath *field.Path, isTLSPassthroughListener bool) field.ErrorList { - if !isTLSPassthroughListener { +func validateTransportServerHost(host string, fieldPath *field.Path, isTLSPassthroughListener bool, protocol string, tls *conf_v1.TransportServerTLS) field.ErrorList { + // TODO: Are UDP listeners allowed for TLS Passthrough, if so I should loosen this restriction below + if protocol == "UDP" { if host != "" { - return field.ErrorList{field.Forbidden(fieldPath, "host field is allowed only for TLS Passthrough TransportServers")} + return field.ErrorList{field.Forbidden(fieldPath, "host field is not allowed for UDP TransportServers")} } return nil } - return validateHost(host, fieldPath) + + if host != "" { + if !isTLSPassthroughListener { + if tls == nil || tls.Secret == "" { + return field.ErrorList{field.Required(fieldPath, "must specify spec.tls.secret when host is specified for non-TLS Passthrough TransportServers")} + } + return validateHost(host, fieldPath) + } + return validateHost(host, fieldPath) + } + + if isTLSPassthroughListener { + return field.ErrorList{field.Required(fieldPath, "must specify host for TLS Passthrough TransportServers")} + } + + return nil } func (tsv *TransportServerValidator) validateTransportListener(listener *conf_v1.TransportServerListener, fieldPath *field.Path) field.ErrorList { diff --git a/tmp/build-errors.log b/tmp/build-errors.log new file mode 100644 index 0000000000..b9456ae561 --- /dev/null +++ b/tmp/build-errors.log @@ -0,0 +1,2 @@ +exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1 +exit status 1 \ No newline at end of file From bd1a6ac185028ebdf939987b26f05376fae37301 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 24 Sep 2024 16:04:04 +0000 Subject: [PATCH 02/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tmp/build-errors.log | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tmp/build-errors.log b/tmp/build-errors.log index b9456ae561..e1d820ccaf 100644 --- a/tmp/build-errors.log +++ b/tmp/build-errors.log @@ -1,2 +1,2 @@ exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1 -exit status 1 \ No newline at end of file +exit status 1 From 9234db313c1b96919c5b327801af225cf1f4de8f Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Tue, 24 Sep 2024 17:05:30 +0100 Subject: [PATCH 03/27] remove build errors --- tmp/build-errors.log | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 tmp/build-errors.log diff --git a/tmp/build-errors.log b/tmp/build-errors.log deleted file mode 100644 index e1d820ccaf..0000000000 --- a/tmp/build-errors.log +++ /dev/null @@ -1,2 +0,0 @@ -exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1 -exit status 1 From 6f6dd49e633aff4d74e8f2e07a9b35eee9a6f8d1 Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Wed, 25 Sep 2024 12:06:28 +0100 Subject: [PATCH 04/27] fix go tests --- internal/configs/transportserver_test.go | 5 ++ internal/k8s/configuration.go | 11 +++- internal/k8s/configuration_test.go | 10 +-- .../validation/transportserver_test.go | 61 ++++++++++++++++--- 4 files changed, 71 insertions(+), 16 deletions(-) diff --git a/internal/configs/transportserver_test.go b/internal/configs/transportserver_test.go index cf8edfbc1e..08a71b9470 100644 --- a/internal/configs/transportserver_test.go +++ b/internal/configs/transportserver_test.go @@ -524,6 +524,7 @@ func TestGenerateTransportServerConfigForTLSPassthrough(t *testing.T) { Port: 2020, UDP: false, StatusZone: "example.com", + ServerName: "example.com", ProxyPass: "ts_default_tcp-server_tcp-app", Name: "tcp-server", Namespace: "default", @@ -642,6 +643,7 @@ func TestGenerateTransportServerConfigForBackupServiceNGINXPlus(t *testing.T) { Port: 2020, UDP: false, StatusZone: "example.com", + ServerName: "example.com", ProxyPass: "ts_default_tcp-server_tcp-app", Name: "tcp-server", Namespace: "default", @@ -715,6 +717,7 @@ func TestGenerateTransportServerConfig_DoesNotGenerateBackupOnMissingBackupName( Port: 2020, UDP: false, StatusZone: "example.com", + ServerName: "example.com", ProxyPass: "ts_default_tcp-server_tcp-app", Name: "tcp-server", Namespace: "default", @@ -789,6 +792,7 @@ func TestGenerateTransportServerConfig_DoesNotGenerateBackupOnMissingBackupPort( Port: 2020, UDP: false, StatusZone: "example.com", + ServerName: "example.com", ProxyPass: "ts_default_tcp-server_tcp-app", Name: "tcp-server", Namespace: "default", @@ -863,6 +867,7 @@ func TestGenerateTransportServerConfig_DoesNotGenerateBackupOnMissingBackupPortA Port: 2020, UDP: false, StatusZone: "example.com", + ServerName: "example.com", ProxyPass: "ts_default_tcp-server_tcp-app", Name: "tcp-server", Namespace: "default", diff --git a/internal/k8s/configuration.go b/internal/k8s/configuration.go index d71791451c..e947155674 100644 --- a/internal/k8s/configuration.go +++ b/internal/k8s/configuration.go @@ -781,6 +781,9 @@ func (c *Configuration) buildListenersAndTSConfigurations() (map[listenerHostKey newTSConfigs := make(map[string]*TransportServerConfiguration) for key, ts := range c.transportServers { + if ts.Spec.Listener.Protocol == conf_v1.TLSPassthroughListenerProtocol { + continue + } tsc := NewTransportServerConfiguration(ts) newTSConfigs[key] = tsc @@ -1078,6 +1081,10 @@ func (c *Configuration) addProblemsForTSConfigsWithoutActiveListener( for _, tsc := range tsConfigs { listenerName := tsc.TransportServer.Spec.Listener.Name host := tsc.TransportServer.Spec.Host + hostDescription := "empty host" + if host != "" { + hostDescription = host + } key := listenerHostKey{ListenerName: listenerName, Host: host} holder, exists := c.listeners[key] if !exists { @@ -1085,7 +1092,7 @@ func (c *Configuration) addProblemsForTSConfigsWithoutActiveListener( Object: tsc.TransportServer, IsError: false, Reason: "Rejected", - Message: fmt.Sprintf("Listener %s with host %s doesn't exist", listenerName, host), //TODO: rethink this error as the user doesnt define the listener with host struct + Message: fmt.Sprintf("Listener %s doesn't exist", listenerName), } problems[tsc.GetKeyWithKind()] = p continue @@ -1096,7 +1103,7 @@ func (c *Configuration) addProblemsForTSConfigsWithoutActiveListener( Object: tsc.TransportServer, IsError: false, Reason: "Rejected", - Message: fmt.Sprintf("Listener %s with host %s is taken by another resource", listenerName, host), + Message: fmt.Sprintf("Listener %s with host %s is taken by another resource", listenerName, hostDescription), } problems[tsc.GetKeyWithKind()] = p } diff --git a/internal/k8s/configuration_test.go b/internal/k8s/configuration_test.go index e7674a2ccc..5720e1f85e 100644 --- a/internal/k8s/configuration_test.go +++ b/internal/k8s/configuration_test.go @@ -3280,7 +3280,7 @@ func TestPortCollisions(t *testing.T) { Object: ts2, IsError: false, Reason: "Rejected", - Message: "Listener tcp-7777 is taken by another resource", + Message: "Listener tcp-7777 with host empty host is taken by another resource", }, } @@ -3300,7 +3300,7 @@ func TestPortCollisions(t *testing.T) { Object: ts3, IsError: false, Reason: "Rejected", - Message: "Listener tcp-7777 is taken by another resource", + Message: "Listener tcp-7777 with host empty host is taken by another resource", }, } @@ -4065,7 +4065,7 @@ func TestFindResourcesForResourceReference(t *testing.T) { }, expected: []Resource{ configuration.hosts["ts.example.com"], - configuration.listeners["tcp-7777"], + configuration.listeners[listenerHostKey{ListenerName: "tcp-7777", Host: ""}], }, msg: "only TransportServers", }, @@ -4121,7 +4121,7 @@ func TestGetResources(t *testing.T) { expected := []Resource{ configuration.hosts["foo.example.com"], configuration.hosts["abc.example.com"], - configuration.listeners["tcp-7777"], + configuration.listeners[listenerHostKey{ListenerName: "tcp-7777", Host: ""}], configuration.hosts["qwe.example.com"], } @@ -4150,7 +4150,7 @@ func TestGetResources(t *testing.T) { expected = []Resource{ configuration.hosts["abc.example.com"], - configuration.listeners["tcp-7777"], + configuration.listeners[listenerHostKey{ListenerName: "tcp-7777", Host: ""}], } result = configuration.GetResourcesWithFilter(resourceFilter{TransportServers: true}) diff --git a/pkg/apis/configuration/validation/transportserver_test.go b/pkg/apis/configuration/validation/transportserver_test.go index a9613748b8..e600d38df5 100644 --- a/pkg/apis/configuration/validation/transportserver_test.go +++ b/pkg/apis/configuration/validation/transportserver_test.go @@ -282,19 +282,29 @@ func TestValidateTransportServerHost(t *testing.T) { tests := []struct { host string isTLSPassthroughListener bool + protocol string + tls *conf_v1.TransportServerTLS }{ { host: "", isTLSPassthroughListener: false, + protocol: "TCP", + tls: &conf_v1.TransportServerTLS{ + Secret: "secret-name", + }, }, { host: "nginx.org", isTLSPassthroughListener: true, + protocol: "TCP", + tls: &conf_v1.TransportServerTLS{ + Secret: "secret-name", + }, }, } for _, test := range tests { - allErrs := validateTransportServerHost(test.host, field.NewPath("host"), test.isTLSPassthroughListener) + allErrs := validateTransportServerHost(test.host, field.NewPath("host"), test.isTLSPassthroughListener, test.protocol, test.tls) if len(allErrs) > 0 { t.Errorf("validateTransportServerHost(%q, %v) returned errors %v for valid input", test.host, test.isTLSPassthroughListener, allErrs) } @@ -460,19 +470,35 @@ func TestValidateTransportServerHost_FailsOnInvalidInput(t *testing.T) { tests := []struct { host string isTLSPassthroughListener bool + protocol string + tls *conf_v1.TransportServerTLS }{ { host: "nginx.org", isTLSPassthroughListener: false, + protocol: "TCP", + tls: nil, // TLS and TLS.secret are required when not using tls passthrough and using a host + }, + { + host: "nginx.org", + isTLSPassthroughListener: false, + protocol: "TCP", + tls: &conf_v1.TransportServerTLS{ + Secret: "", // TLS and TLS.secret are required when not using tls passthrough and using a host + }, }, { host: "", isTLSPassthroughListener: true, + protocol: "TCP", + tls: &conf_v1.TransportServerTLS{ + Secret: "secret-name", + }, }, } for _, test := range tests { - allErrs := validateTransportServerHost(test.host, field.NewPath("host"), test.isTLSPassthroughListener) + allErrs := validateTransportServerHost(test.host, field.NewPath("host"), test.isTLSPassthroughListener, test.protocol, test.tls) if len(allErrs) == 0 { t.Errorf("validateTransportServerHost(%q, %v) returned no errors for invalid input", test.host, test.isTLSPassthroughListener) } @@ -1041,17 +1067,29 @@ func TestValidateMatchExpect_FailsOnInvalidInput(t *testing.T) { func TestValidateTsTLS(t *testing.T) { t.Parallel() - validTLSes := []*conf_v1.TransportServerTLS{ - nil, + + type testCase struct { + tls *conf_v1.TransportServerTLS + hostSpecified bool + } + + validTestCases := []testCase{ { - Secret: "my-secret", + tls: nil, + hostSpecified: false, + }, + { + tls: &conf_v1.TransportServerTLS{ + Secret: "my-secret", + }, + hostSpecified: true, }, } - for _, tls := range validTLSes { - allErrs := validateTLS(tls, false, field.NewPath("tls")) + for _, tc := range validTestCases { + allErrs := validateTLS(tc.tls, false, field.NewPath("tls"), tc.hostSpecified) if len(allErrs) > 0 { - t.Errorf("validateTLS() returned errors %v for valid input %v", allErrs, tls) + t.Errorf("validateTLS() returned errors %v for valid input %+v", allErrs, tc) } } } @@ -1061,35 +1099,40 @@ func TestValidateTsTLS_FailsOnInvalidInput(t *testing.T) { invalidTLSes := []struct { tls *conf_v1.TransportServerTLS isTLSPassthrough bool + hostSpecified bool }{ { tls: &conf_v1.TransportServerTLS{ Secret: "-", }, isTLSPassthrough: false, + hostSpecified: true, }, { tls: &conf_v1.TransportServerTLS{ Secret: "a/b", }, isTLSPassthrough: false, + hostSpecified: true, }, { tls: &conf_v1.TransportServerTLS{ Secret: "my-secret", }, isTLSPassthrough: true, + hostSpecified: true, }, { tls: &conf_v1.TransportServerTLS{ Secret: "", }, isTLSPassthrough: false, + hostSpecified: true, }, } for _, test := range invalidTLSes { - allErrs := validateTLS(test.tls, test.isTLSPassthrough, field.NewPath("tls")) + allErrs := validateTLS(test.tls, test.isTLSPassthrough, field.NewPath("tls"), test.hostSpecified) if len(allErrs) == 0 { t.Errorf("validateTLS() returned no errors for invalid input %v", test) } From 88f24b6d311f2d954f349688e9ba5e5c8ef7bd85 Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Wed, 25 Sep 2024 14:01:32 +0100 Subject: [PATCH 05/27] fix example readme linting --- README.md | 1 + .../security-monitoring/README.md | 2 +- .../tls-passthrough/README.md | 2 +- .../transport-server-sni/README.md | 41 ++++++++++--------- 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 9322b4aea9..3759031dfd 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ We value community input and would love to see you at the next community call. A | **2024-10-21** | | **2024-11-04** | | **2024-11-18** | + --- NGINX Ingress Controller works with both NGINX and NGINX Plus and supports the standard Ingress features - content-based diff --git a/examples/custom-resources/security-monitoring/README.md b/examples/custom-resources/security-monitoring/README.md index 5a970ae08d..d735d317ad 100644 --- a/examples/custom-resources/security-monitoring/README.md +++ b/examples/custom-resources/security-monitoring/README.md @@ -64,7 +64,7 @@ Note that the VirtualServer references the policy `waf-policy` created in Step 3 ## Step 5 - Test the Application To access the application, **curl`** the coffee and the tea services. Use the --resolve option to set the Host header -of a request with `webapp.example.com` +of a request with`webapp.example.com` 1. Send a request to the application: diff --git a/examples/custom-resources/tls-passthrough/README.md b/examples/custom-resources/tls-passthrough/README.md index 0e2c844d44..adb5a556ce 100644 --- a/examples/custom-resources/tls-passthrough/README.md +++ b/examples/custom-resources/tls-passthrough/README.md @@ -47,7 +47,7 @@ You can see how the Secure App is implemented in the `secure-app.yaml` file. 1. Save the HTTPS port of the Ingress Controller where TLS Passthrough is enabled into a shell variable: ```console - $ IC_HTTPS_PORT= + IC_HTTPS_PORT= 1. Save the HTTPS port of the Ingress Controller into a shell variable: diff --git a/examples/custom-resources/transport-server-sni/README.md b/examples/custom-resources/transport-server-sni/README.md index b8e29d52ee..c1dbaba88b 100644 --- a/examples/custom-resources/transport-server-sni/README.md +++ b/examples/custom-resources/transport-server-sni/README.md @@ -1,27 +1,28 @@ -TransportServer SNI +# TransportServer SNI + In this example we create two different TransportServers that listen on the same interface, which are distinguished by their Host field. 1. Create a GlobalConfiguration resource with the following listener. -``` +```yaml listeners: - name: tcp-listener port: 7000 protocol: TCP ``` -2. Add a custom port to the NIC pod via the helm chart +1. Add a custom port to the NIC pod via the helm chart -``` +```yaml controller.customPorts: - name: port containerPort: 7000 protocol: TCP ``` -3. Add a custom port to the NIC service +1. Add a custom port to the NIC service -``` +```yaml controller.service.customPorts: - name: tcp-port port: 7000 @@ -29,64 +30,64 @@ controller.service.customPorts: targetPort: 7000 ``` -4. Create the cafe-secret, and mongo-secret. These secrets are used for TLS in the TransportServers +1. Create the cafe-secret, and mongo-secret. These secrets are used for TLS in the TransportServers `kubectl apply -f cafe-secret.yaml` `kubectl apply -f mongo-secret.yaml` -5. Create the mongo and tcp echo example applications +1. Create the mongo and tcp echo example applications `kubectl apply -f mongo.yaml` `kubectl apply -f tcp-echo-server.yaml` -6. Wait until these are ready. +1. Wait until these are ready. `kubectl get deploy -w` -7. Create the TransportServers for each application +1. Create the TransportServers for each application `kubectl apply -f cafe-transport-server.yaml` `kubectl apply -f mongo-transport-server.yaml` -8. Ensure they are in valid state +1. Ensure they are in valid state `kubectl get ts` -``` +```bash NAME STATE REASON AGE cafe-ts Valid AddedOrUpdated 2m mongo-ts Valid AddedOrUpdated 2m ``` -9. Set up /etc/hosts or DNS +1. Set up /etc/hosts or DNS In this example I am running NIC locally, so I will set cafe.example.com and mongo.example.com in my /etc/hosts to localhost If running NIC as a live service, the server names would be set at the DNS layer `cat /etc/hosts` -``` +```bash ... 127.0.0.1 cafe.example.com 127.0.0.1 mongo.example.com ``` -10. Expose port 7000 of the NIC Loadbalancer Service +1. Expose port 7000 of the NIC Loadbalancer Service `k port-forward svc/my-release-nginx-ingress-controller 7000:7000` -11. Use ncat to ping cafe.example.com on port 7000 with ssl +1. Use ncat to ping cafe.example.com on port 7000 with ssl `ncat --ssl cafe.example.com 7000` When you write a message you should get sent back -``` +```bash hi hi ``` And once the connection is closed (ctrl + c) you should see the request and response in the NIC logs are both 2 bytes -``` +```bash 127.0.0.1 [24/Sep/2024:15:48:58 +0000] TCP 200 3 3 2.702 "- ``` -12. Use mongosh to connect to the mongodb container, via the TransportServer, also on port 7000 +1. Use mongosh to connect to the mongodb container, via the TransportServer, also on port 7000 `mongosh --host mongo.example.com --port 7000 --tls --tlsAllowInvalidCertificates` -``` +```bash test> show dbs admin 40.00 KiB config 60.00 KiB From 9af72cffb343ea3e7cf247c4282ba1f1e2fed02b Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Wed, 25 Sep 2024 15:32:18 +0100 Subject: [PATCH 06/27] fix tests --- charts/nginx-ingress/values.yaml | 33 +++++++------------ .../version2/nginx-plus.transportserver.tmpl | 1 + .../version2/nginx.transportserver.tmpl | 2 +- internal/configs/version2/template_helper.go | 8 +++++ 4 files changed, 21 insertions(+), 23 deletions(-) diff --git a/charts/nginx-ingress/values.yaml b/charts/nginx-ingress/values.yaml index feb1539757..ebece1f3a6 100644 --- a/charts/nginx-ingress/values.yaml +++ b/charts/nginx-ingress/values.yaml @@ -127,10 +127,7 @@ controller: logLevel: 1 ## A list of custom ports to expose on the NGINX Ingress Controller pod. Follows the conventional Kubernetes yaml syntax for container ports. - customPorts: - - name: port - containerPort: 7000 - protocol: TCP + customPorts: [] image: ## The image repository of the Ingress Controller. @@ -360,20 +357,17 @@ controller: globalConfiguration: ## Creates the GlobalConfiguration custom resource. Requires controller.enableCustomResources. - create: true + create: false ## The spec of the GlobalConfiguration for defining the global configuration parameters of the Ingress Controller. - spec: ## Ensure both curly brackets are removed when adding listeners in YAML format. - listeners: - - name: tcp-listener - port: 7000 - protocol: TCP - - name: dns-udp - port: 5353 - protocol: UDP - # - name: dns-tcp - # port: 5353 - # protocol: TCP + spec: {} ## Ensure both curly brackets are removed when adding listeners in YAML format. + # listeners: + # - name: dns-udp + # port: 5353 + # protocol: UDP + # - name: dns-tcp + # port: 5353 + # protocol: TCP ## Enable custom NGINX configuration snippets in Ingress, VirtualServer, VirtualServerRoute and TransportServer resources. enableSnippets: false @@ -462,12 +456,7 @@ controller: targetPort: 443 ## A list of custom ports to expose through the Ingress Controller service. Follows the conventional Kubernetes yaml syntax for service ports. - customPorts: - - name: tcp-port - port: 7000 - protocol: TCP - targetPort: 7000 - + customPorts: [] serviceAccount: ## The annotations of the service account of the Ingress Controller pods. diff --git a/internal/configs/version2/nginx-plus.transportserver.tmpl b/internal/configs/version2/nginx-plus.transportserver.tmpl index 8b959b548f..52e5237fa2 100644 --- a/internal/configs/version2/nginx-plus.transportserver.tmpl +++ b/internal/configs/version2/nginx-plus.transportserver.tmpl @@ -40,6 +40,7 @@ server { set_real_ip_from unix:; {{- else }} {{ makeTransportListener $s | printf }} + {{ makeServerName $s | printf }} {{- end }} {{- if $ssl.Enabled }} diff --git a/internal/configs/version2/nginx.transportserver.tmpl b/internal/configs/version2/nginx.transportserver.tmpl index d00b262d71..506faeb157 100644 --- a/internal/configs/version2/nginx.transportserver.tmpl +++ b/internal/configs/version2/nginx.transportserver.tmpl @@ -24,9 +24,9 @@ server { set_real_ip_from unix:; {{- else }} {{ makeTransportListener $s | printf }} + {{ makeServerName $s | printf }} {{- end }} - server_name {{ $s.ServerName }}; {{- if $ssl.Enabled }} ssl_certificate {{ makeSecretPath $ssl.Certificate $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; diff --git a/internal/configs/version2/template_helper.go b/internal/configs/version2/template_helper.go index 95e8f59d78..89291eafc5 100644 --- a/internal/configs/version2/template_helper.go +++ b/internal/configs/version2/template_helper.go @@ -224,6 +224,13 @@ func makeHeaderQueryValue(apiKey APIKey) string { return fmt.Sprintf("\"%s\"", strings.Join(parts, "")) } +func makeServerName(s StreamServer) string { + if s.TLSPassthrough || s.ServerName == "" { + return "" + } + return fmt.Sprintf("server_name %s", s.ServerName) +} + var helperFunctions = template.FuncMap{ "headerListToCIMap": headerListToCIMap, "hasCIKey": hasCIKey, @@ -238,4 +245,5 @@ var helperFunctions = template.FuncMap{ "makeSecretPath": commonhelpers.MakeSecretPath, "makeHeaderQueryValue": makeHeaderQueryValue, "makeTransportListener": makeTransportListener, + "makeServerName": makeServerName, } From 4c3c49c33524045dd80786165e61185ec16d1c37 Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Wed, 25 Sep 2024 15:37:14 +0100 Subject: [PATCH 07/27] remove unnessary changes --- README.md | 1 - examples/custom-resources/security-monitoring/README.md | 2 +- examples/custom-resources/tls-passthrough/README.md | 2 +- internal/configs/version2/nginx.transportserver.tmpl | 1 - internal/k8s/configuration.go | 2 +- 5 files changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3759031dfd..9322b4aea9 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,6 @@ We value community input and would love to see you at the next community call. A | **2024-10-21** | | **2024-11-04** | | **2024-11-18** | - --- NGINX Ingress Controller works with both NGINX and NGINX Plus and supports the standard Ingress features - content-based diff --git a/examples/custom-resources/security-monitoring/README.md b/examples/custom-resources/security-monitoring/README.md index d735d317ad..5a970ae08d 100644 --- a/examples/custom-resources/security-monitoring/README.md +++ b/examples/custom-resources/security-monitoring/README.md @@ -64,7 +64,7 @@ Note that the VirtualServer references the policy `waf-policy` created in Step 3 ## Step 5 - Test the Application To access the application, **curl`** the coffee and the tea services. Use the --resolve option to set the Host header -of a request with`webapp.example.com` +of a request with `webapp.example.com` 1. Send a request to the application: diff --git a/examples/custom-resources/tls-passthrough/README.md b/examples/custom-resources/tls-passthrough/README.md index adb5a556ce..deec45d25a 100644 --- a/examples/custom-resources/tls-passthrough/README.md +++ b/examples/custom-resources/tls-passthrough/README.md @@ -47,7 +47,7 @@ You can see how the Secure App is implemented in the `secure-app.yaml` file. 1. Save the HTTPS port of the Ingress Controller where TLS Passthrough is enabled into a shell variable: ```console - IC_HTTPS_PORT= + $ IC_HTTPS_PORT= 1. Save the HTTPS port of the Ingress Controller into a shell variable: diff --git a/internal/configs/version2/nginx.transportserver.tmpl b/internal/configs/version2/nginx.transportserver.tmpl index 506faeb157..a0cde13fab 100644 --- a/internal/configs/version2/nginx.transportserver.tmpl +++ b/internal/configs/version2/nginx.transportserver.tmpl @@ -27,7 +27,6 @@ server { {{ makeServerName $s | printf }} {{- end }} - {{- if $ssl.Enabled }} ssl_certificate {{ makeSecretPath $ssl.Certificate $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; ssl_certificate_key {{ makeSecretPath $ssl.CertificateKey $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; diff --git a/internal/k8s/configuration.go b/internal/k8s/configuration.go index e947155674..a6d1535cf5 100644 --- a/internal/k8s/configuration.go +++ b/internal/k8s/configuration.go @@ -791,8 +791,8 @@ func (c *Configuration) buildListenersAndTSConfigurations() (map[listenerHostKey continue } - var listener conf_v1.Listener found := false + var listener conf_v1.Listener for _, l := range c.globalConfiguration.Spec.Listeners { if ts.Spec.Listener.Name == l.Name && ts.Spec.Listener.Protocol == l.Protocol { listener = l From c812f7d00822c75552276400640bb87ee481a2e3 Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Wed, 25 Sep 2024 15:39:03 +0100 Subject: [PATCH 08/27] update snaps --- internal/configs/version2/__snapshots__/templates_test.snap | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/configs/version2/__snapshots__/templates_test.snap b/internal/configs/version2/__snapshots__/templates_test.snap index c2a797fd0d..dfca059383 100644 --- a/internal/configs/version2/__snapshots__/templates_test.snap +++ b/internal/configs/version2/__snapshots__/templates_test.snap @@ -101,6 +101,7 @@ server { listen 127.0.0.1:1234 ssl udp; listen [::1]:1234 ssl udp; + ssl_certificate cafe-secret.pem; ssl_certificate_key cafe-secret.pem; @@ -146,6 +147,7 @@ server { listen 127.0.0.1:1234 ssl; listen [::1]:1234 ssl; + ssl_certificate cafe-secret.pem; ssl_certificate_key cafe-secret.pem; @@ -7911,6 +7913,7 @@ server { listen 1234 ssl udp; listen [::]:1234 ssl udp; + ssl_certificate cafe-secret.pem; ssl_certificate_key cafe-secret.pem; From 4c746b6092186aedfbae80e62d74f9b8ced3f762 Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Wed, 25 Sep 2024 15:44:58 +0100 Subject: [PATCH 09/27] tls passthrough example lint --- examples/custom-resources/tls-passthrough/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/custom-resources/tls-passthrough/README.md b/examples/custom-resources/tls-passthrough/README.md index deec45d25a..e8b2d4df88 100644 --- a/examples/custom-resources/tls-passthrough/README.md +++ b/examples/custom-resources/tls-passthrough/README.md @@ -46,8 +46,9 @@ You can see how the Secure App is implemented in the `secure-app.yaml` file. 1. Save the HTTPS port of the Ingress Controller where TLS Passthrough is enabled into a shell variable: - ```console - $ IC_HTTPS_PORT= +```console +IC_HTTPS_PORT= +``` 1. Save the HTTPS port of the Ingress Controller into a shell variable: From 6a0644bae45386ba3644eff15da496a0f940d922 Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Wed, 25 Sep 2024 16:24:31 +0100 Subject: [PATCH 10/27] fix tests --- tests/suite/test_transport_server_tcp_load_balance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/suite/test_transport_server_tcp_load_balance.py b/tests/suite/test_transport_server_tcp_load_balance.py index bd812b8902..3a9d2eedff 100644 --- a/tests/suite/test_transport_server_tcp_load_balance.py +++ b/tests/suite/test_transport_server_tcp_load_balance.py @@ -186,7 +186,7 @@ def test_tcp_request_load_balanced_multiple(self, kube_apis, crd_ingress_control response["status"] and response["status"]["reason"] == "Rejected" and response["status"]["state"] == "Warning" - and response["status"]["message"] == "Listener tcp-server is taken by another resource" + and response["status"]["message"] == "Listener tcp-server with host empty host is taken by another resource" ) # Step 3, remove the default TransportServer with the same port From 7e8c45ce0255cdd766caaeaa739991c33144d14b Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Wed, 25 Sep 2024 16:40:25 +0100 Subject: [PATCH 11/27] fix test --- tests/suite/test_transport_server_udp_load_balance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/suite/test_transport_server_udp_load_balance.py b/tests/suite/test_transport_server_udp_load_balance.py index 841fc43bf9..86fa8596ec 100644 --- a/tests/suite/test_transport_server_udp_load_balance.py +++ b/tests/suite/test_transport_server_udp_load_balance.py @@ -204,7 +204,7 @@ def test_udp_request_load_balanced_multiple(self, kube_apis, crd_ingress_control response["status"] and response["status"]["reason"] == "Rejected" and response["status"]["state"] == "Warning" - and response["status"]["message"] == "Listener udp-server is taken by another resource" + and response["status"]["message"] == "Listener udp-server with host empty host is taken by another resource" ) # Step 3, remove the default TransportServer with the same port From 9db8c566043e2782bae4c2cf27e4ab63aaa80e12 Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Thu, 26 Sep 2024 11:22:48 +0100 Subject: [PATCH 12/27] add new snapshot test, fix makeServerName function --- .../__snapshots__/templates_test.snap | 33 +++++++++++++++ internal/configs/version2/template_helper.go | 2 +- internal/configs/version2/templates_test.go | 41 +++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) diff --git a/internal/configs/version2/__snapshots__/templates_test.snap b/internal/configs/version2/__snapshots__/templates_test.snap index dfca059383..4e5c46a309 100644 --- a/internal/configs/version2/__snapshots__/templates_test.snap +++ b/internal/configs/version2/__snapshots__/templates_test.snap @@ -71,6 +71,39 @@ server { health_check_timeout 5s; + proxy_timeout 10s; + proxy_connect_timeout 10s; + proxy_next_upstream on; + proxy_next_upstream_timeout 10s; + proxy_next_upstream_tries 5; +} + +--- + +[TestExecuteTemplateForTransportServerWithSNI - 1] + +upstream cafe-upstream { + zone cafe-upstream 512k; + server 10.0.0.20:5001 max_fails=0 fail_timeout= max_conns=0; +} + + +server { + listen 1234 ssl; + listen [::]:1234 ssl; + + server_name "cafe.example.com"; + ssl_certificate cafe-secret.pem; + ssl_certificate_key cafe-secret.pem; + + status_zone ; + proxy_requests 1; + proxy_responses 2; + + proxy_pass cafe-upstream; + + + proxy_timeout 10s; proxy_connect_timeout 10s; proxy_next_upstream on; diff --git a/internal/configs/version2/template_helper.go b/internal/configs/version2/template_helper.go index 89291eafc5..7337c4e3f6 100644 --- a/internal/configs/version2/template_helper.go +++ b/internal/configs/version2/template_helper.go @@ -228,7 +228,7 @@ func makeServerName(s StreamServer) string { if s.TLSPassthrough || s.ServerName == "" { return "" } - return fmt.Sprintf("server_name %s", s.ServerName) + return fmt.Sprintf("server_name \"%s\";", s.ServerName) } var helperFunctions = template.FuncMap{ diff --git a/internal/configs/version2/templates_test.go b/internal/configs/version2/templates_test.go index aa0f276c68..8d13e92ef3 100644 --- a/internal/configs/version2/templates_test.go +++ b/internal/configs/version2/templates_test.go @@ -486,6 +486,16 @@ func TestExecuteTemplateForTransportServerWithResolver(t *testing.T) { snaps.MatchSnapshot(t, string(got)) } +func TestExecuteTemplateForTransportServerWithSNI(t *testing.T) { + t.Parallel() + executor := newTmplExecutorNGINXPlus(t) + got, err := executor.ExecuteTransportServerTemplate(&transportServerCfgWithSNI) + if err != nil { + t.Errorf("Failed to execute template: %v", err) + } + snaps.MatchSnapshot(t, string(got)) +} + func TestTransportServerForNginx(t *testing.T) { t.Parallel() executor := newTmplExecutorNGINX(t) @@ -5321,6 +5331,37 @@ var ( }, } + transportServerCfgWithSNI = TransportServerConfig{ + Upstreams: []StreamUpstream{ + { + Name: "cafe-upstream", + Servers: []StreamUpstreamServer{ + { + Address: "10.0.0.20:5001", + }, + }, + }, + }, + Server: StreamServer{ + Port: 1234, + ServerName: "cafe.example.com", + TLSPassthrough: false, + SSL: &StreamSSL{ + Enabled: true, + Certificate: "cafe-secret.pem", + CertificateKey: "cafe-secret.pem", + }, + ProxyRequests: createPointerFromInt(1), + ProxyResponses: createPointerFromInt(2), + ProxyPass: "cafe-upstream", + ProxyTimeout: "10s", + ProxyConnectTimeout: "10s", + ProxyNextUpstream: true, + ProxyNextUpstreamTimeout: "10s", + ProxyNextUpstreamTries: 5, + }, + } + transportServerCfgWithSSL = TransportServerConfig{ Upstreams: []StreamUpstream{ { From a751f7878debb6f6eb144d6c54312df20423c7b1 Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Thu, 26 Sep 2024 13:40:11 +0100 Subject: [PATCH 13/27] add test for makeServerName function --- .../mongo-transport-server.yaml | 1 - internal/configs/version2/template_helper.go | 2 +- .../configs/version2/template_helper_test.go | 37 +++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/examples/custom-resources/transport-server-sni/mongo-transport-server.yaml b/examples/custom-resources/transport-server-sni/mongo-transport-server.yaml index 4dd3608e58..529fb13128 100644 --- a/examples/custom-resources/transport-server-sni/mongo-transport-server.yaml +++ b/examples/custom-resources/transport-server-sni/mongo-transport-server.yaml @@ -1,4 +1,3 @@ - apiVersion: k8s.nginx.org/v1 kind: TransportServer metadata: diff --git a/internal/configs/version2/template_helper.go b/internal/configs/version2/template_helper.go index 7337c4e3f6..8dec8347ab 100644 --- a/internal/configs/version2/template_helper.go +++ b/internal/configs/version2/template_helper.go @@ -225,7 +225,7 @@ func makeHeaderQueryValue(apiKey APIKey) string { } func makeServerName(s StreamServer) string { - if s.TLSPassthrough || s.ServerName == "" { + if s.TLSPassthrough || s.ServerName == "" || s.SSL == nil { return "" } return fmt.Sprintf("server_name \"%s\";", s.ServerName) diff --git a/internal/configs/version2/template_helper_test.go b/internal/configs/version2/template_helper_test.go index 8c2ea4cb9c..58d166657d 100644 --- a/internal/configs/version2/template_helper_test.go +++ b/internal/configs/version2/template_helper_test.go @@ -613,6 +613,43 @@ func TestMakeTransportIPListener(t *testing.T) { } } +func TestMakeServerName(t *testing.T) { + t.Parallel() + + testCases := []struct { + server StreamServer + expected string + }{ + {server: StreamServer{ + TLSPassthrough: false, + ServerName: "cafe.example.com", + SSL: &StreamSSL{}, + }, expected: "server_name \"cafe.example.com\";"}, + {server: StreamServer{ + TLSPassthrough: true, + ServerName: "cafe.example.com", + SSL: &StreamSSL{}, + }, expected: ""}, + {server: StreamServer{ + TLSPassthrough: false, + ServerName: "", + SSL: &StreamSSL{}, + }, expected: ""}, + {server: StreamServer{ + TLSPassthrough: false, + ServerName: "cafe.example.com", + SSL: nil, + }, expected: ""}, + } + + for _, tc := range testCases { + got := makeServerName(tc.server) + if got != tc.expected { + t.Errorf("Function generated wrong config, got %q but expected %q.", got, tc.expected) + } + } +} + func newContainsTemplate(t *testing.T) *template.Template { t.Helper() tmpl, err := template.New("testTemplate").Funcs(helperFunctions).Parse(`{{contains .InputString .Substring}}`) From 3d35774cb9fea699eb72a3937e259f2360838532 Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Thu, 26 Sep 2024 14:41:16 +0100 Subject: [PATCH 14/27] add go tests --- internal/configs/transportserver.go | 14 +- internal/configs/transportserver_test.go | 127 +++++++++++++++++- .../__snapshots__/templates_test.snap | 89 +++++++----- internal/configs/version2/templates_test.go | 12 +- 4 files changed, 203 insertions(+), 39 deletions(-) diff --git a/internal/configs/transportserver.go b/internal/configs/transportserver.go index b5fa31a552..c04939021d 100644 --- a/internal/configs/transportserver.go +++ b/internal/configs/transportserver.go @@ -98,11 +98,14 @@ func generateTransportServerConfig(p transportServerConfigParams) (*version2.Tra if p.transportServerEx.TransportServer.Spec.Listener.Name == conf_v1.TLSPassthroughListenerName { statusZone = p.transportServerEx.TransportServer.Spec.Host } + host := p.transportServerEx.TransportServer.Spec.Host + isTLSPassthrough := p.transportServerEx.TransportServer.Spec.Listener.Name == conf_v1.TLSPassthroughListenerName + serverName := generateServerName(host, isTLSPassthrough) tsConfig := &version2.TransportServerConfig{ Server: version2.StreamServer{ - ServerName: p.transportServerEx.TransportServer.Spec.Host, - TLSPassthrough: p.transportServerEx.TransportServer.Spec.Listener.Name == conf_v1.TLSPassthroughListenerName, + ServerName: serverName, + TLSPassthrough: isTLSPassthrough, UnixSocket: generateUnixSocket(p.transportServerEx), Port: p.listenerPort, UDP: p.transportServerEx.TransportServer.Spec.Listener.Protocol == "UDP", @@ -352,3 +355,10 @@ func generateLoadBalancingMethod(method string) string { } return method } + +func generateServerName(host string, isTLSPassthrough bool) string { + if isTLSPassthrough { + return "" + } + return host +} diff --git a/internal/configs/transportserver_test.go b/internal/configs/transportserver_test.go index 08a71b9470..c0abb19b66 100644 --- a/internal/configs/transportserver_test.go +++ b/internal/configs/transportserver_test.go @@ -524,7 +524,7 @@ func TestGenerateTransportServerConfigForTLSPassthrough(t *testing.T) { Port: 2020, UDP: false, StatusZone: "example.com", - ServerName: "example.com", + ServerName: "", ProxyPass: "ts_default_tcp-server_tcp-app", Name: "tcp-server", Namespace: "default", @@ -643,7 +643,7 @@ func TestGenerateTransportServerConfigForBackupServiceNGINXPlus(t *testing.T) { Port: 2020, UDP: false, StatusZone: "example.com", - ServerName: "example.com", + ServerName: "", ProxyPass: "ts_default_tcp-server_tcp-app", Name: "tcp-server", Namespace: "default", @@ -717,7 +717,7 @@ func TestGenerateTransportServerConfig_DoesNotGenerateBackupOnMissingBackupName( Port: 2020, UDP: false, StatusZone: "example.com", - ServerName: "example.com", + ServerName: "", ProxyPass: "ts_default_tcp-server_tcp-app", Name: "tcp-server", Namespace: "default", @@ -792,7 +792,7 @@ func TestGenerateTransportServerConfig_DoesNotGenerateBackupOnMissingBackupPort( Port: 2020, UDP: false, StatusZone: "example.com", - ServerName: "example.com", + ServerName: "", ProxyPass: "ts_default_tcp-server_tcp-app", Name: "tcp-server", Namespace: "default", @@ -867,7 +867,7 @@ func TestGenerateTransportServerConfig_DoesNotGenerateBackupOnMissingBackupPortA Port: 2020, UDP: false, StatusZone: "example.com", - ServerName: "example.com", + ServerName: "", ProxyPass: "ts_default_tcp-server_tcp-app", Name: "tcp-server", Namespace: "default", @@ -1290,6 +1290,123 @@ func TestGenerateTransportServerConfig_UsesNotExistignSocketOnNotPlusAndNoEndpoi } } +func TestGenerateTransportServerConfigForTCPWithTLSWithHost(t *testing.T) { + t.Parallel() + transportServerEx := TransportServerEx{ + TransportServer: &conf_v1.TransportServer{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "tcp-server", + Namespace: "default", + }, + Spec: conf_v1.TransportServerSpec{ + Host: "cafe.example.com", + Listener: conf_v1.TransportServerListener{ + Name: "tcp-listener", + Protocol: "TCP", + }, + TLS: &conf_v1.TransportServerTLS{ + Secret: "my-secret", + }, + Upstreams: []conf_v1.TransportServerUpstream{ + { + Name: "tcp-app", + Service: "tcp-app-svc", + Port: 5001, + MaxFails: intPointer(3), + FailTimeout: "40s", + }, + }, + UpstreamParameters: &conf_v1.UpstreamParameters{ + ConnectTimeout: "30s", + NextUpstream: false, + }, + SessionParameters: &conf_v1.SessionParameters{ + Timeout: "50s", + }, + Action: &conf_v1.TransportServerAction{ + Pass: "tcp-app", + }, + }, + }, + Endpoints: map[string][]string{ + "default/tcp-app-svc:5001": { + "10.0.0.20:5001", + }, + }, + DisableIPV6: false, + SecretRefs: map[string]*secrets.SecretReference{ + "default/my-secret": { + Secret: &api_v1.Secret{ + Type: api_v1.SecretTypeTLS, + }, + Path: "/etc/nginx/secrets/default-my-secret", + }, + }, + } + + listenerPort := 2020 + + expected := &version2.TransportServerConfig{ + Upstreams: []version2.StreamUpstream{ + { + Name: "ts_default_tcp-server_tcp-app", + Servers: []version2.StreamUpstreamServer{ + { + Address: "10.0.0.20:5001", + MaxFails: 3, + FailTimeout: "40s", + }, + }, + UpstreamLabels: version2.UpstreamLabels{ + ResourceName: "tcp-server", + ResourceType: "transportserver", + ResourceNamespace: "default", + Service: "tcp-app-svc", + }, + LoadBalancingMethod: "random two least_conn", + }, + }, + Server: version2.StreamServer{ + Port: 2020, + UDP: false, + StatusZone: "tcp-listener", + ServerName: "cafe.example.com", + ProxyPass: "ts_default_tcp-server_tcp-app", + Name: "tcp-server", + Namespace: "default", + ProxyConnectTimeout: "30s", + ProxyNextUpstream: false, + ProxyNextUpstreamTries: 0, + ProxyNextUpstreamTimeout: "0s", + ProxyTimeout: "50s", + HealthCheck: nil, + ServerSnippets: []string{}, + SSL: &version2.StreamSSL{ + Enabled: true, + Certificate: "/etc/nginx/secrets/default-my-secret", + CertificateKey: "/etc/nginx/secrets/default-my-secret", + }, + }, + StreamSnippets: []string{}, + StaticSSLPath: "/etc/nginx/secret", + } + + result, warnings := generateTransportServerConfig(transportServerConfigParams{ + transportServerEx: &transportServerEx, + listenerPort: listenerPort, + isPlus: true, + isResolverConfigured: false, + isDynamicReloadEnabled: false, + staticSSLPath: "/etc/nginx/secret", + }) + if len(warnings) != 0 { + t.Errorf("want no warnings, got %v", warnings) + } + if !cmp.Equal(expected, result) { + t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", cmp.Diff(expected, result)) + } +} + func TestGenerateTransportServerConfigForTCPWithTLS(t *testing.T) { t.Parallel() transportServerEx := TransportServerEx{ diff --git a/internal/configs/version2/__snapshots__/templates_test.snap b/internal/configs/version2/__snapshots__/templates_test.snap index 4e5c46a309..327d35224a 100644 --- a/internal/configs/version2/__snapshots__/templates_test.snap +++ b/internal/configs/version2/__snapshots__/templates_test.snap @@ -1,35 +1,54 @@ -[TestExecuteTemplateForTransportServerWithBackupServerForNGINXPlus - 1] +[TestExecuteTemplateForNGINXOSSTransportServerWithSNI - 1] -upstream udp-upstream { - zone udp-upstream 512k; +upstream cafe-upstream { + zone cafe-upstream 512k; server 10.0.0.20:5001 max_fails=0 fail_timeout= max_conns=0; - server clustertwo.corp.local:8080 resolve backup; } +server { + listen 1234 ssl; + listen [::]:1234 ssl; + server_name "cafe.example.com"; + ssl_certificate cafe-secret.pem; + ssl_certificate_key cafe-secret.pem; + proxy_requests 1; + proxy_responses 2; -match match_udp-upstream { - - send "GET / HTTP/1.0\r\nHost: localhost\r\n\r\n"; - + proxy_pass cafe-upstream; - - expect ~* "200 OK"; - + proxy_timeout 10s; + proxy_connect_timeout 10s; + proxy_next_upstream on; + proxy_next_upstream_timeout 10s; + proxy_next_upstream_tries 5; } + +--- + +[TestExecuteTemplateForNGINXPlusTransportServerWithSNI - 1] + +upstream cafe-upstream { + zone cafe-upstream 512k; + server 10.0.0.20:5001 max_fails=0 fail_timeout= max_conns=0; +} + + server { + listen 1234 ssl; + listen [::]:1234 ssl; - status_zone udp-app; + server_name "cafe.example.com"; + ssl_certificate cafe-secret.pem; + ssl_certificate_key cafe-secret.pem; + + status_zone ; proxy_requests 1; proxy_responses 2; - proxy_pass udp-upstream; + proxy_pass cafe-upstream; - health_check interval=5s port=8080 - passes=1 jitter=0 fails=1 udp match=match_udp-upstream; - health_check_timeout 5s; - proxy_timeout 10s; proxy_connect_timeout 10s; @@ -40,11 +59,12 @@ server { --- -[TestExecuteTemplateForTransportServerWithResolver - 1] +[TestExecuteTemplateForTransportServerWithBackupServerForNGINXPlus - 1] upstream udp-upstream { zone udp-upstream 512k; - server 10.0.0.20:5001 max_fails=0 fail_timeout= max_conns=0 resolve; + server 10.0.0.20:5001 max_fails=0 fail_timeout= max_conns=0; + server clustertwo.corp.local:8080 resolve backup; } @@ -80,29 +100,36 @@ server { --- -[TestExecuteTemplateForTransportServerWithSNI - 1] +[TestExecuteTemplateForTransportServerWithResolver - 1] -upstream cafe-upstream { - zone cafe-upstream 512k; - server 10.0.0.20:5001 max_fails=0 fail_timeout= max_conns=0; +upstream udp-upstream { + zone udp-upstream 512k; + server 10.0.0.20:5001 max_fails=0 fail_timeout= max_conns=0 resolve; } -server { - listen 1234 ssl; - listen [::]:1234 ssl; +match match_udp-upstream { + + send "GET / HTTP/1.0\r\nHost: localhost\r\n\r\n"; + - server_name "cafe.example.com"; - ssl_certificate cafe-secret.pem; - ssl_certificate_key cafe-secret.pem; + + expect ~* "200 OK"; + +} +server { - status_zone ; + status_zone udp-app; proxy_requests 1; proxy_responses 2; - proxy_pass cafe-upstream; + proxy_pass udp-upstream; + health_check interval=5s port=8080 + passes=1 jitter=0 fails=1 udp match=match_udp-upstream; + health_check_timeout 5s; + proxy_timeout 10s; proxy_connect_timeout 10s; diff --git a/internal/configs/version2/templates_test.go b/internal/configs/version2/templates_test.go index 8d13e92ef3..f5a1548bec 100644 --- a/internal/configs/version2/templates_test.go +++ b/internal/configs/version2/templates_test.go @@ -486,7 +486,17 @@ func TestExecuteTemplateForTransportServerWithResolver(t *testing.T) { snaps.MatchSnapshot(t, string(got)) } -func TestExecuteTemplateForTransportServerWithSNI(t *testing.T) { +func TestExecuteTemplateForNGINXOSSTransportServerWithSNI(t *testing.T) { + t.Parallel() + executor := newTmplExecutorNGINX(t) + got, err := executor.ExecuteTransportServerTemplate(&transportServerCfgWithSNI) + if err != nil { + t.Errorf("Failed to execute template: %v", err) + } + snaps.MatchSnapshot(t, string(got)) +} + +func TestExecuteTemplateForNGINXPlusTransportServerWithSNI(t *testing.T) { t.Parallel() executor := newTmplExecutorNGINXPlus(t) got, err := executor.ExecuteTransportServerTemplate(&transportServerCfgWithSNI) From 2748b2f550e1cad44d64bf2c9ea5e0eb1299afd1 Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Mon, 7 Oct 2024 11:28:33 +0100 Subject: [PATCH 15/27] change c.listeners to c.listenerHosts --- internal/k8s/configuration.go | 86 +++++++++++++++--------------- internal/k8s/configuration_test.go | 6 +-- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/internal/k8s/configuration.go b/internal/k8s/configuration.go index a6d1535cf5..c966bf5f97 100644 --- a/internal/k8s/configuration.go +++ b/internal/k8s/configuration.go @@ -342,9 +342,9 @@ type TransportServerMetrics struct { // The IC needs to ensure that at any point in time the NGINX config on the filesystem reflects the state // of the objects in the Configuration. type Configuration struct { - hosts map[string]Resource - listeners map[listenerHostKey]*TransportServerConfiguration - listenerMap map[string]conf_v1.Listener + hosts map[string]Resource + listenerHosts map[listenerHostKey]*TransportServerConfiguration + listenerMap map[string]conf_v1.Listener // only valid resources with the matching IngressClass are stored ingresses map[string]*networking.Ingress @@ -399,7 +399,7 @@ func NewConfiguration( ) *Configuration { return &Configuration{ hosts: make(map[string]Resource), - listeners: make(map[listenerHostKey]*TransportServerConfiguration), + listenerHosts: make(map[listenerHostKey]*TransportServerConfiguration), ingresses: make(map[string]*networking.Ingress), virtualServers: make(map[string]*conf_v1.VirtualServer), virtualServerRoutes: make(map[string]*conf_v1.VirtualServerRoute), @@ -621,7 +621,7 @@ func (c *Configuration) AddOrUpdateGlobalConfiguration(gc *conf_v1.GlobalConfigu c.globalConfiguration = gc c.setGlobalConfigListenerMap() - listenerChanges, listenerProblems := c.rebuildListeners() + listenerChanges, listenerProblems := c.rebuildListenerHosts() changes = append(changes, listenerChanges...) problems = append(problems, listenerProblems...) @@ -643,7 +643,7 @@ func (c *Configuration) DeleteGlobalConfiguration() ([]ResourceChange, []Configu c.globalConfiguration = nil c.setGlobalConfigListenerMap() - listenerChanges, listenerProblems := c.rebuildListeners() + listenerChanges, listenerProblems := c.rebuildListenerHosts() changes = append(changes, listenerChanges...) problems = append(problems, listenerProblems...) @@ -681,7 +681,7 @@ func (c *Configuration) AddOrUpdateTransportServer(ts *conf_v1.TransportServer) } } - changes, problems := c.rebuildListeners() + changes, problems := c.rebuildListenerHosts() if c.isTLSPassthroughEnabled { hostChanges, hostProblems := c.rebuildHosts() @@ -691,7 +691,7 @@ func (c *Configuration) AddOrUpdateTransportServer(ts *conf_v1.TransportServer) } if validationErr != nil { - // If the invalid resource has an active host/listener, rebuildHosts/rebuildListeners will create a change + // If the invalid resource has an active host/listener, rebuildHosts/rebuildListenerHosts will create a change // to remove the resource. // Here we add the validationErr to that change. kind := getResourceKeyWithKind(transportServerKind, &ts.ObjectMeta) @@ -706,7 +706,7 @@ func (c *Configuration) AddOrUpdateTransportServer(ts *conf_v1.TransportServer) // On the other hand, the invalid resource might not have any active host/listener. // Or the resource was invalid before and is still invalid (in some different way). - // In those cases, rebuildHosts/rebuildListeners will create no change for that resource. + // In those cases, rebuildHosts/rebuildListenerHosts will create no change for that resource. // To make sure the validationErr is reported to the user, we create a problem. p := ConfigurationProblem{ Object: ts, @@ -732,7 +732,7 @@ func (c *Configuration) DeleteTransportServer(key string) ([]ResourceChange, []C delete(c.transportServers, key) - changes, problems := c.rebuildListeners() + changes, problems := c.rebuildListenerHosts() if c.isTLSPassthroughEnabled { hostChanges, hostProblems := c.rebuildHosts() @@ -744,13 +744,13 @@ func (c *Configuration) DeleteTransportServer(key string) ([]ResourceChange, []C return changes, problems } -func (c *Configuration) rebuildListeners() ([]ResourceChange, []ConfigurationProblem) { - newListeners, newTSConfigs := c.buildListenersAndTSConfigurations() +func (c *Configuration) rebuildListenerHosts() ([]ResourceChange, []ConfigurationProblem) { + newListenerHosts, newTSConfigs := c.buildListenerHostsAndTSConfigurations() - removedListeners, updatedListeners, addedListeners := detectChangesInListeners(c.listeners, newListeners) - changes := createResourceChangesForListeners(removedListeners, updatedListeners, addedListeners, c.listeners, newListeners) + removedListenerHosts, updatedListenerHosts, addedListenerHosts := detectChangesInListenerHosts(c.listenerHosts, newListenerHosts) + changes := createResourceChangesForListeners(removedListenerHosts, updatedListenerHosts, addedListenerHosts, c.listenerHosts, newListenerHosts) - c.listeners = newListeners + c.listenerHosts = newListenerHosts changes = squashResourceChanges(changes) @@ -776,8 +776,8 @@ func (c *Configuration) rebuildListeners() ([]ResourceChange, []ConfigurationPro return changes, newOrUpdatedProblems } -func (c *Configuration) buildListenersAndTSConfigurations() (map[listenerHostKey]*TransportServerConfiguration, map[string]*TransportServerConfiguration) { - newListeners := make(map[listenerHostKey]*TransportServerConfiguration) +func (c *Configuration) buildListenerHostsAndTSConfigurations() (map[listenerHostKey]*TransportServerConfiguration, map[string]*TransportServerConfiguration) { + newListenerHosts := make(map[listenerHostKey]*TransportServerConfiguration) newTSConfigs := make(map[string]*TransportServerConfiguration) for key, ts := range c.transportServers { @@ -812,9 +812,9 @@ func (c *Configuration) buildListenersAndTSConfigurations() (map[listenerHostKey host := ts.Spec.Host listenerKey := listenerHostKey{ListenerName: listener.Name, Host: host} - holder, exists := newListeners[listenerKey] + holder, exists := newListenerHosts[listenerKey] if !exists { - newListeners[listenerKey] = tsc + newListenerHosts[listenerKey] = tsc continue } @@ -823,13 +823,13 @@ func (c *Configuration) buildListenersAndTSConfigurations() (map[listenerHostKey if !holder.Wins(tsc) { holder.AddWarning(warning) - newListeners[listenerKey] = tsc + newListenerHosts[listenerKey] = tsc } else { tsc.AddWarning(warning) } } - return newListeners, newTSConfigs + return newListenerHosts, newTSConfigs } func (c *Configuration) buildListenersForVSConfiguration(vsc *VirtualServerConfiguration) { @@ -890,7 +890,7 @@ func (c *Configuration) GetResourcesWithFilter(filter resourceFilter) []Resource } if filter.TransportServers { - for _, r := range c.listeners { + for _, r := range c.listenerHosts { resources[r.GetKeyWithKind()] = r } } @@ -986,8 +986,8 @@ func (c *Configuration) findResourcesForResourceReference(namespace string, name } } - for _, l := range getSortedListenerHostKeys(c.listeners) { - tsConfig := c.listeners[l] + for _, lh := range getSortedListenerHostKeys(c.listenerHosts) { + tsConfig := c.listenerHosts[lh] if checker.IsReferencedByTransportServer(namespace, name, tsConfig.TransportServer) { result = append(result, tsConfig) @@ -1086,7 +1086,7 @@ func (c *Configuration) addProblemsForTSConfigsWithoutActiveListener( hostDescription = host } key := listenerHostKey{ListenerName: listenerName, Host: host} - holder, exists := c.listeners[key] + holder, exists := c.listenerHosts[key] if !exists { p := ConfigurationProblem{ Object: tsc.TransportServer, @@ -1680,7 +1680,7 @@ func (c *Configuration) GetTransportServerMetrics() *TransportServerMetrics { } } - for _, tsConfig := range c.listeners { + for _, tsConfig := range c.listenerHosts { if tsConfig.TransportServer.Spec.Listener.Protocol == "TCP" { metrics.TotalTCP++ } else { @@ -1847,31 +1847,31 @@ func detectChangesInHosts(oldHosts map[string]Resource, newHosts map[string]Reso return removedHosts, updatedHosts, addedHosts } -func detectChangesInListeners( - oldListeners map[listenerHostKey]*TransportServerConfiguration, - newListeners map[listenerHostKey]*TransportServerConfiguration, -) (removedListeners []listenerHostKey, updatedListeners []listenerHostKey, addedListeners []listenerHostKey) { - oldKeys := getSortedListenerHostKeys(oldListeners) - newKeys := getSortedListenerHostKeys(newListeners) +func detectChangesInListenerHosts( + oldListenerHosts map[listenerHostKey]*TransportServerConfiguration, + newListenerHosts map[listenerHostKey]*TransportServerConfiguration, +) (removedListenerHosts []listenerHostKey, updatedListenerHosts []listenerHostKey, addedListenerHosts []listenerHostKey) { + oldKeys := getSortedListenerHostKeys(oldListenerHosts) + newKeys := getSortedListenerHostKeys(newListenerHosts) oldKeysSet := make(map[listenerHostKey]struct{}) - for _, l := range oldKeys { - oldKeysSet[l] = struct{}{} - if _, exists := newListeners[l]; !exists { - removedListeners = append(removedListeners, l) + for _, key := range oldKeys { + oldKeysSet[key] = struct{}{} + if _, exists := newListenerHosts[key]; !exists { + removedListenerHosts = append(removedListenerHosts, key) } } - for _, l := range newKeys { - if _, exists := oldListeners[l]; !exists { - addedListeners = append(addedListeners, l) + for _, key := range newKeys { + if _, exists := oldListenerHosts[key]; !exists { + addedListenerHosts = append(addedListenerHosts, key) } else { - oldR := oldListeners[l] - if !oldR.IsEqual(newListeners[l]) { - updatedListeners = append(updatedListeners, l) + oldConfig := oldListenerHosts[key] + if !oldConfig.IsEqual(newListenerHosts[key]) { + updatedListenerHosts = append(updatedListenerHosts, key) } } } - return removedListeners, updatedListeners, addedListeners + return removedListenerHosts, updatedListenerHosts, addedListenerHosts } diff --git a/internal/k8s/configuration_test.go b/internal/k8s/configuration_test.go index 5720e1f85e..6e34ab4bb5 100644 --- a/internal/k8s/configuration_test.go +++ b/internal/k8s/configuration_test.go @@ -4065,7 +4065,7 @@ func TestFindResourcesForResourceReference(t *testing.T) { }, expected: []Resource{ configuration.hosts["ts.example.com"], - configuration.listeners[listenerHostKey{ListenerName: "tcp-7777", Host: ""}], + configuration.listenerHosts[listenerHostKey{ListenerName: "tcp-7777", Host: ""}], }, msg: "only TransportServers", }, @@ -4121,7 +4121,7 @@ func TestGetResources(t *testing.T) { expected := []Resource{ configuration.hosts["foo.example.com"], configuration.hosts["abc.example.com"], - configuration.listeners[listenerHostKey{ListenerName: "tcp-7777", Host: ""}], + configuration.listenerHosts[listenerHostKey{ListenerName: "tcp-7777", Host: ""}], configuration.hosts["qwe.example.com"], } @@ -4150,7 +4150,7 @@ func TestGetResources(t *testing.T) { expected = []Resource{ configuration.hosts["abc.example.com"], - configuration.listeners[listenerHostKey{ListenerName: "tcp-7777", Host: ""}], + configuration.listenerHosts[listenerHostKey{ListenerName: "tcp-7777", Host: ""}], } result = configuration.GetResourcesWithFilter(resourceFilter{TransportServers: true}) From 5f4ebc0df55036255e4637790ab13f68b2f1c1b0 Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Mon, 7 Oct 2024 17:45:30 +0100 Subject: [PATCH 16/27] fix non tls passthrough hosts being added to tls passthrough template --- internal/configs/configurator.go | 3 ++- internal/configs/transportserver.go | 3 ++- pkg/apis/configuration/validation/transportserver.go | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 7aac432c39..5a1dcae9b7 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -771,7 +771,8 @@ func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *Transport // update TLS Passthrough Hosts config in case we have a TLS Passthrough TransportServer // only TLS Passthrough TransportServers have non-empty hosts - if transportServerEx.TransportServer.Spec.Host != "" { + isTLSPassthrough := transportServerEx.TransportServer.Spec.Listener.Name == conf_v1.TLSPassthroughListenerName + if transportServerEx.TransportServer.Spec.Host != "" && isTLSPassthrough { key := generateNamespaceNameKey(&transportServerEx.TransportServer.ObjectMeta) cnf.tlsPassthroughPairs[key] = tlsPassthroughPair{ Host: transportServerEx.TransportServer.Spec.Host, diff --git a/internal/configs/transportserver.go b/internal/configs/transportserver.go index c04939021d..7dabd46dfa 100644 --- a/internal/configs/transportserver.go +++ b/internal/configs/transportserver.go @@ -101,6 +101,7 @@ func generateTransportServerConfig(p transportServerConfigParams) (*version2.Tra host := p.transportServerEx.TransportServer.Spec.Host isTLSPassthrough := p.transportServerEx.TransportServer.Spec.Listener.Name == conf_v1.TLSPassthroughListenerName serverName := generateServerName(host, isTLSPassthrough) + isUDP := p.transportServerEx.TransportServer.Spec.Listener.Protocol == "UDP" tsConfig := &version2.TransportServerConfig{ Server: version2.StreamServer{ @@ -108,7 +109,7 @@ func generateTransportServerConfig(p transportServerConfigParams) (*version2.Tra TLSPassthrough: isTLSPassthrough, UnixSocket: generateUnixSocket(p.transportServerEx), Port: p.listenerPort, - UDP: p.transportServerEx.TransportServer.Spec.Listener.Protocol == "UDP", + UDP: isUDP, StatusZone: statusZone, ProxyRequests: proxyRequests, ProxyResponses: proxyResponses, diff --git a/pkg/apis/configuration/validation/transportserver.go b/pkg/apis/configuration/validation/transportserver.go index 29f9dec2d4..058cba0130 100644 --- a/pkg/apis/configuration/validation/transportserver.go +++ b/pkg/apis/configuration/validation/transportserver.go @@ -78,7 +78,7 @@ func validateTLS( if hostSpecified { if tls == nil || tls.Secret == "" { - return field.ErrorList{field.Required(fieldPath, "must specify spec.tls.secret when host is specified")} + return field.ErrorList{field.Required(fieldPath, "must specify spec.tls.secret when host is specified, and the TransportServer is not using the TLS Passthrough listener")} } return validateSecretName(tls.Secret, fieldPath.Child("secret")) } From 87a671f5f3e694de89f04120ff5e53f56956752e Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Thu, 10 Oct 2024 17:35:42 +0100 Subject: [PATCH 17/27] more go tests --- internal/configs/configurator.go | 2 +- internal/configs/configurator_test.go | 56 +++++++ internal/k8s/configuration_test.go | 138 ++++++++++++++++++ .../validation/transportserver.go | 1 - .../validation/transportserver_test.go | 60 +++++++- 5 files changed, 253 insertions(+), 4 deletions(-) diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 5a1dcae9b7..3f5ce7a0cb 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -770,7 +770,7 @@ func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *Transport cnf.transportServers[name] = transportServerEx // update TLS Passthrough Hosts config in case we have a TLS Passthrough TransportServer - // only TLS Passthrough TransportServers have non-empty hosts + // A non empty Host, may be a TLS Passthrough TransportServer but we have to check for the existence of the TLS Passthrough listener also, as TransportServers that terminate at the NGINX level can have non empty Hosts now too isTLSPassthrough := transportServerEx.TransportServer.Spec.Listener.Name == conf_v1.TLSPassthroughListenerName if transportServerEx.TransportServer.Spec.Host != "" && isTLSPassthrough { key := generateNamespaceNameKey(&transportServerEx.TransportServer.ObjectMeta) diff --git a/internal/configs/configurator_test.go b/internal/configs/configurator_test.go index 63be30821f..c5eef453af 100644 --- a/internal/configs/configurator_test.go +++ b/internal/configs/configurator_test.go @@ -15,9 +15,11 @@ import ( "github.com/nginxinc/kubernetes-ingress/internal/configs/version1" "github.com/nginxinc/kubernetes-ingress/internal/configs/version2" + "github.com/nginxinc/kubernetes-ingress/internal/k8s/secrets" "github.com/nginxinc/kubernetes-ingress/internal/nginx" conf_v1 "github.com/nginxinc/kubernetes-ingress/pkg/apis/configuration/v1" "github.com/nginxinc/kubernetes-ingress/pkg/apis/dos/v1beta1" + api_v1 "k8s.io/api/core/v1" ) func createTestStaticConfigParams() *StaticConfigParams { @@ -1606,6 +1608,21 @@ func TestGetVitualServerCountsNotExistingVS(t *testing.T) { } } +func TestAddOrUpdateTransportServer(t *testing.T) { + t.Parallel() + cnf := createTestConfigurator(t) + + ts := createTransportServerExWithHostNoTLSPassthrough() + + warnings, err := cnf.AddOrUpdateTransportServer(&ts) + if err != nil { + t.Errorf("AddOrUpdateTransportServer returned: \n%v, but expected: \n%v", err, nil) + } + if len(warnings) != 0 { + t.Errorf("AddOrUpdateTransportServer returned warnings: %v", warnings) + } +} + var ( invalidVirtualServerEx = &VirtualServerEx{ VirtualServer: &conf_v1.VirtualServer{}, @@ -1715,3 +1732,42 @@ func TestGenerateApDosAllowListFileContent(t *testing.T) { }) } } + +func createTransportServerExWithHostNoTLSPassthrough() TransportServerEx { + return TransportServerEx{ + SecretRefs: map[string]*secrets.SecretReference{ + "default/echo-secret": { + Secret: &api_v1.Secret{ + Type: api_v1.SecretTypeTLS, + }, + Path: "secret.pem", + }, + }, + TransportServer: &conf_v1.TransportServer{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "echo-app", + Namespace: "default", + }, + Spec: conf_v1.TransportServerSpec{ + Listener: conf_v1.TransportServerListener{ + Name: "tcp-listener", + Protocol: "TCP", + }, + Host: "example.com", + TLS: &conf_v1.TransportServerTLS{ + Secret: "echo-secret", + }, + Upstreams: []conf_v1.TransportServerUpstream{ + { + Name: "echo-app", + Service: "echo-app", + Port: 7000, + }, + }, + Action: &conf_v1.TransportServerAction{ + Pass: "echo-app", + }, + }, + }, + } +} diff --git a/internal/k8s/configuration_test.go b/internal/k8s/configuration_test.go index 479c7e12d9..9375a60b9f 100644 --- a/internal/k8s/configuration_test.go +++ b/internal/k8s/configuration_test.go @@ -2006,6 +2006,136 @@ func TestAddTransportServer(t *testing.T) { } } +func TestAddTransportServerWithHost(t *testing.T) { + configuration := createTestConfiguration() + + listeners := []conf_v1.Listener{ + { + Name: "tcp-7777", + Port: 7777, + Protocol: "TCP", + }, + } + + addOrUpdateGlobalConfiguration(t, configuration, listeners, noChanges, noProblems) + + secretName := "echo-secret" + + ts := createTestTransportServerWithHost("transportserver", "echo.example.com", "tcp-7777", "TCP", secretName) + + // no problems are expected for all cases + var expectedProblems []ConfigurationProblem + var expectedChanges []ResourceChange + + // Add TransportServer + + expectedChanges = []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &TransportServerConfiguration{ + ListenerPort: 7777, + TransportServer: ts, + }, + }, + } + + changes, problems := configuration.AddOrUpdateTransportServer(ts) + if diff := cmp.Diff(expectedChanges, changes); diff != "" { + t.Errorf("AddOrUpdateTransportServer() returned unexpected result (-want +got):\n%s", diff) + } + if diff := cmp.Diff(expectedProblems, problems); diff != "" { + t.Errorf("AddOrUpdateTransportServer() returned unexpected result (-want +got):\n%s", diff) + } + + // Update TransportServer + + updatedTS := ts.DeepCopy() + updatedTS.Generation++ + + expectedChanges = []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &TransportServerConfiguration{ + ListenerPort: 7777, + TransportServer: updatedTS, + }, + }, + } + + changes, problems = configuration.AddOrUpdateTransportServer(updatedTS) + if diff := cmp.Diff(expectedChanges, changes); diff != "" { + t.Errorf("AddOrUpdateTransportServer() returned unexpected result (-want +got):\n%s", diff) + } + if diff := cmp.Diff(expectedProblems, problems); diff != "" { + t.Errorf("AddOrUpdateTransportServer() returned unexpected result (-want +got):\n%s", diff) + } + + // Make TransportServer invalid + + invalidTS := updatedTS.DeepCopy() + invalidTS.Generation++ + invalidTS.Spec.Upstreams = nil + + expectedChanges = []ResourceChange{ + { + Op: Delete, + Resource: &TransportServerConfiguration{ + ListenerPort: 7777, + TransportServer: updatedTS, + }, + Error: `spec.action.pass: Not found: "myapp"`, + }, + } + + changes, problems = configuration.AddOrUpdateTransportServer(invalidTS) + if diff := cmp.Diff(expectedChanges, changes); diff != "" { + t.Errorf("AddOrUpdateTransportServer() returned unexpected result (-want +got):\n%s", diff) + } + if diff := cmp.Diff(expectedProblems, problems); diff != "" { + t.Errorf("AddOrUpdateTransportServer() returned unexpected result (-want +got):\n%s", diff) + } + + // Restore TransportServer + + expectedChanges = []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &TransportServerConfiguration{ + ListenerPort: 7777, + TransportServer: updatedTS, + }, + }, + } + + changes, problems = configuration.AddOrUpdateTransportServer(updatedTS) + if diff := cmp.Diff(expectedChanges, changes); diff != "" { + t.Errorf("AddOrUpdateTransportServer() returned unexpected result (-want +got):\n%s", diff) + } + if diff := cmp.Diff(expectedProblems, problems); diff != "" { + t.Errorf("AddOrUpdateTransportServer() returned unexpected result (-want +got):\n%s", diff) + } + + // Delete TransportServer + + expectedChanges = []ResourceChange{ + { + Op: Delete, + Resource: &TransportServerConfiguration{ + ListenerPort: 7777, + TransportServer: updatedTS, + }, + }, + } + + changes, problems = configuration.DeleteTransportServer("default/transportserver") + if diff := cmp.Diff(expectedChanges, changes); diff != "" { + t.Errorf("DeleteTransportServer() returned unexpected result (-want +got):\n%s", diff) + } + if diff := cmp.Diff(expectedProblems, problems); diff != "" { + t.Errorf("DeleteTransportServer() returned unexpected result (-want +got):\n%s", diff) + } +} + func TestAddTransportServerForTLSPassthrough(t *testing.T) { configuration := createTestConfiguration() @@ -3710,6 +3840,14 @@ func createTestTransportServer(name string, listenerName string, listenerProtoco } } +func createTestTransportServerWithHost(name string, host string, listenerName string, listenerProtocol string, secretName string) *conf_v1.TransportServer { + ts := createTestTransportServer(name, listenerName, listenerProtocol) + ts.Spec.Host = host + ts.Spec.TLS = &conf_v1.TransportServerTLS{Secret: secretName} + + return ts +} + func createTestTLSPassthroughTransportServer(name string, host string) *conf_v1.TransportServer { ts := createTestTransportServer(name, conf_v1.TLSPassthroughListenerName, conf_v1.TLSPassthroughListenerProtocol) ts.Spec.Host = host diff --git a/pkg/apis/configuration/validation/transportserver.go b/pkg/apis/configuration/validation/transportserver.go index 058cba0130..50a36ffb76 100644 --- a/pkg/apis/configuration/validation/transportserver.go +++ b/pkg/apis/configuration/validation/transportserver.go @@ -98,7 +98,6 @@ func validateSnippets(serverSnippet string, fieldPath *field.Path, snippetsEnabl } func validateTransportServerHost(host string, fieldPath *field.Path, isTLSPassthroughListener bool, protocol string, tls *conf_v1.TransportServerTLS) field.ErrorList { - // TODO: Are UDP listeners allowed for TLS Passthrough, if so I should loosen this restriction below if protocol == "UDP" { if host != "" { return field.ErrorList{field.Forbidden(fieldPath, "host field is not allowed for UDP TransportServers")} diff --git a/pkg/apis/configuration/validation/transportserver_test.go b/pkg/apis/configuration/validation/transportserver_test.go index e600d38df5..c5b5c9e6f6 100644 --- a/pkg/apis/configuration/validation/transportserver_test.go +++ b/pkg/apis/configuration/validation/transportserver_test.go @@ -279,7 +279,7 @@ func TestValidateTransportServerUpstreams_FailsOnInvalidInput(t *testing.T) { func TestValidateTransportServerHost(t *testing.T) { t.Parallel() - tests := []struct { + validCases := []struct { host string isTLSPassthroughListener bool protocol string @@ -296,6 +296,14 @@ func TestValidateTransportServerHost(t *testing.T) { { host: "nginx.org", isTLSPassthroughListener: true, + protocol: "TLS_PASSTHROUGH", + tls: &conf_v1.TransportServerTLS{ + Secret: "secret-name", + }, + }, + { + host: "nginx.org", + isTLSPassthroughListener: false, protocol: "TCP", tls: &conf_v1.TransportServerTLS{ Secret: "secret-name", @@ -303,12 +311,60 @@ func TestValidateTransportServerHost(t *testing.T) { }, } - for _, test := range tests { + for _, test := range validCases { allErrs := validateTransportServerHost(test.host, field.NewPath("host"), test.isTLSPassthroughListener, test.protocol, test.tls) if len(allErrs) > 0 { t.Errorf("validateTransportServerHost(%q, %v) returned errors %v for valid input", test.host, test.isTLSPassthroughListener, allErrs) } } + + invalidCases := []struct { + host string + isTLSPassthroughListener bool + protocol string + tls *conf_v1.TransportServerTLS + }{ + // Invalid case: host is empty but isTLSPassthroughListener is true + { + host: "", + isTLSPassthroughListener: true, + protocol: "TLS_PASSTHROUGH", + tls: &conf_v1.TransportServerTLS{ + Secret: "secret-name", + }, + }, + { + host: "nginx.org", + isTLSPassthroughListener: false, + protocol: "UDP", + tls: &conf_v1.TransportServerTLS{ + Secret: "secret-name", + }, + }, + { + host: "invalid host name", + isTLSPassthroughListener: true, + protocol: "TCP", + tls: &conf_v1.TransportServerTLS{ + Secret: "secret-name", + }, + }, + { + host: "nginx.org", + isTLSPassthroughListener: true, + protocol: "UDP", + tls: &conf_v1.TransportServerTLS{ + Secret: "secret-name", + }, + }, + } + + for _, test := range invalidCases { + allErrs := validateTransportServerHost(test.host, field.NewPath("host"), test.isTLSPassthroughListener, test.protocol, test.tls) + if len(allErrs) == 0 { + t.Errorf("validateTransportServerHost(%q, %v) returned no errors for invalid input", test.host, test.isTLSPassthroughListener) + } + } } func TestValidateTransportServerLoadBalancingMethod(t *testing.T) { From 70debbeda3ee8e62c6b78c28b6e277d4619aa520 Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Fri, 11 Oct 2024 12:59:03 +0100 Subject: [PATCH 18/27] add python tests --- .../validation/transportserver_test.go | 22 +++-- .../cafe-secret.yaml | 9 +++ .../transport-server-with-host.yaml | 17 ++++ .../suite/test_transport_server_with_host.py | 80 +++++++++++++++++++ 4 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 tests/data/transport-server-with-host/cafe-secret.yaml create mode 100644 tests/data/transport-server-with-host/transport-server-with-host.yaml create mode 100644 tests/suite/test_transport_server_with_host.py diff --git a/pkg/apis/configuration/validation/transportserver_test.go b/pkg/apis/configuration/validation/transportserver_test.go index c5b5c9e6f6..a3354785c6 100644 --- a/pkg/apis/configuration/validation/transportserver_test.go +++ b/pkg/apis/configuration/validation/transportserver_test.go @@ -1125,25 +1125,35 @@ func TestValidateTsTLS(t *testing.T) { t.Parallel() type testCase struct { - tls *conf_v1.TransportServerTLS - hostSpecified bool + tls *conf_v1.TransportServerTLS + isTLSPassthrough bool + hostSpecified bool } validTestCases := []testCase{ { - tls: nil, - hostSpecified: false, + tls: nil, + isTLSPassthrough: false, + hostSpecified: false, + }, + { + tls: &conf_v1.TransportServerTLS{ + Secret: "my-secret", + }, + isTLSPassthrough: false, + hostSpecified: true, }, { tls: &conf_v1.TransportServerTLS{ Secret: "my-secret", }, - hostSpecified: true, + isTLSPassthrough: false, + hostSpecified: false, }, } for _, tc := range validTestCases { - allErrs := validateTLS(tc.tls, false, field.NewPath("tls"), tc.hostSpecified) + allErrs := validateTLS(tc.tls, tc.isTLSPassthrough, field.NewPath("tls"), tc.hostSpecified) if len(allErrs) > 0 { t.Errorf("validateTLS() returned errors %v for valid input %+v", allErrs, tc) } diff --git a/tests/data/transport-server-with-host/cafe-secret.yaml b/tests/data/transport-server-with-host/cafe-secret.yaml new file mode 100644 index 0000000000..4129b88e76 --- /dev/null +++ b/tests/data/transport-server-with-host/cafe-secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUY1ekNDQTgrZ0F3SUJBZ0lVR2dXT0JNTkR2TXNWOGY3OWc0MlpSZy9KaWc0d0RRWUpLb1pJaHZjTkFRRUwKQlFBd2dZSXhDekFKQmdOVkJBWVRBbGhZTVJJd0VBWURWUVFJREFsVGRHRjBaVTVoYldVeEVUQVBCZ05WQkFjTQpDRU5wZEhsT1lXMWxNUlF3RWdZRFZRUUtEQXREYjIxd1lXNTVUbUZ0WlRFYk1Ca0dBMVVFQ3d3U1EyOXRjR0Z1CmVWTmxZM1JwYjI1T1lXMWxNUmt3RndZRFZRUUREQkJqWVdabExtVjRZVzF3YkdVdVkyOXRNQjRYRFRJME1Ea3kKTXpFd01EUXdNVm9YRFRNME1Ea3lNVEV3TURRd01Wb3dnWUl4Q3pBSkJnTlZCQVlUQWxoWU1SSXdFQVlEVlFRSQpEQWxUZEdGMFpVNWhiV1V4RVRBUEJnTlZCQWNNQ0VOcGRIbE9ZVzFsTVJRd0VnWURWUVFLREF0RGIyMXdZVzU1ClRtRnRaVEViTUJrR0ExVUVDd3dTUTI5dGNHRnVlVk5sWTNScGIyNU9ZVzFsTVJrd0Z3WURWUVFEREJCallXWmwKTG1WNFlXMXdiR1V1WTI5dE1JSUNJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBZzhBTUlJQ0NnS0NBZ0VBbjkvZwpkMml5NEZyaGNtS3ZXQTVHZXFiS1c5WU1xeml6Vm4yRExkd215enkvS2FUdTdtUlkvUWRsTkxHaVhIM1NEa0FkCkhtZFJEK0Zob1ZHaWxoTjEzcTI0azdyRSt6dk1iMlRqdnBnMmVLYVd1dDBHaGQ3V3l3TnpIUU9WcWJCM2o2U2QKSE1sNlJKK25ERHo2Yi9adkRiNm9nQjBucUowZjBWN0tBMHFFalYxVld2dXI4YVZpVlVxK0tBOVMveEdJMHZSSwo2NE9BS2xIUUF4a2xGWWhheHVjcWRsS0owQlRXekE0Z0gzUTZlQi9CRjNOMTJDQjgrMVEzZG54bU5TVnpLbXp1CmJYdm1jK1RPUjg0V3ovemlaMTl1K3RuRHg2aklNajZZQXd2M0tzSVlYbmV1ZEgycUZKRG5FMmZwYjNnZzRxbm0KQnduY0ZlU1poY09nZWdnR1RmKzRTc3YwRUNuamNNdXhteTEwMTNwY0h1a2prTUNmVGJsUGZTU0NNS24rVnlGZQpUNzlSeUkxM3hmL0JXOSt0aTFad1IwYVpkWk5jV1R2Um53R1M1bWhnelJXclhRUmdJQlVVVjk2N3cxRVVONDJVCkE3ZjBVSzZlYks5bExBMnNZWmJJSWUrbllRY1d1OVRuNXo5YnVwK2w3aU5hUDYxWXdwWGdrUTFyNHlSamhMeUEKajVha1Z5VnROYzZmSWltaXlWV0Y1QzBaZnNIYzF3cUpLb2lXQzBtT1dyNEVRSVorRkY2WC9FRytSMmhodjZkQQpGcndpd3BHUE9kWkROQzNqNFBqVzVreWlYMlh2Vm1RWTUvbkxNdFpUaFF6VHpadG8zT3M4d1RsRjFyQlZkd05iCi9zNTh2dnlTQitsYk5Sd215TEErQVYvOGQ2L2VNcGZjS0ZyYU85OENBd0VBQWFOVE1GRXdIUVlEVlIwT0JCWUUKRktEejZlREZLaC93ZDVLTHpPMXpCMDR2UGVRSU1COEdBMVVkSXdRWU1CYUFGS0R6NmVERktoL3dkNUtMek8xegpCMDR2UGVRSU1BOEdBMVVkRXdFQi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dJQkFKQnd5WVlyCllXa3RvRGJPYlpHRUFXT0lEcjFTb3NWRDVIRXdkazV6ZTJuT05oQXFHVWszOHpUT3V1VVZRNk5lT1c1UWtML3oKVTFUa3dwUlNFNjFpbVZHOUdjYWY5ampkSjd5VGhsOGpmZXUyWTF4RHo1OUpmbWJ1WUo2SGE4ZmR4STAxQXZ3RgoxaERYY1ZEcTZoalhhb0pKMVZta1NiTEJPc1JXaGVzWGl1K3FiN3NSOUhlZmJmaTUxandQb2NFblE4YzNiWXF5CmpIRkwxS1JJekIyR2VYVGk2WngwN2lqcDZIeGJSQUFEWnBuRGN0blM1UVQvTGFoYklIZVNNTExMK21vdnUrSUgKTVgwQmtaYmJjRk94L3d2ZlN1SXdveEVXR1RjVGJIYnJGanVUUDRkalpPdFhybElSK1RJTmRiaFhaMm1XZVdHbgpRa0tkcyt4a3puNzNZbG9xZCtPZGZ2d1dJYXVFYktuaXdQUk01NHZqK0dnbndiWmZNWXpnY292cE5BYzZjMmlmCkNNdHlHWDBUREptSjRVWC9FaFFSYTArSURMbFZTMWc2Y3IxWmVQM1N1MWpkSnhBSFVwUjZwSVYrWGdsTENDdGYKQXdDUW9BUWtUeXkyb0Y1Z0hWOGVuc28zVE15cmI5R1NBWDF0UEdJL080L3VCRDBqRzQ4anZsZEI3dzZKbjdmSwoyS21DWnIwYlRDdU9vWnVuZHA3OHA2R1ozb3lVOXBxcTFTUmU0MTdjSlVuNzR3S05TTkd0U0xTNm9OZ3FQQ2h6Cm9ZTy9zK0NHL295c0JUcTBma2VBYXdMZ2oxVG9ybGNsaEt6M05uWUtLZmhDVFBLTDVVUFZLNjVXQ3V2djlSVWgKVUZvM2F2TnhhSmpWUEY4V0FVUnRZem82bXBadFRITm53ZkJTCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUpRd0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQ1Mwd2dna3BBZ0VBQW9JQ0FRQ2YzK0IzYUxMZ1d1RnkKWXE5WURrWjZwc3BiMWd5ck9MTldmWU10M0NiTFBMOHBwTzd1WkZqOUIyVTBzYUpjZmRJT1FCMGVaMUVQNFdHaApVYUtXRTNYZXJiaVR1c1Q3Tzh4dlpPTyttRFo0cHBhNjNRYUYzdGJMQTNNZEE1V3BzSGVQcEowY3lYcEVuNmNNClBQcHY5bThOdnFpQUhTZW9uUi9SWHNvRFNvU05YVlZhKzZ2eHBXSlZTcjRvRDFML0VZalM5RXJyZzRBcVVkQUQKR1NVVmlGckc1eXAyVW9uUUZOYk1EaUFmZERwNEg4RVhjM1hZSUh6N1ZEZDJmR1kxSlhNcWJPNXRlK1p6NU01SAp6aGJQL09KblgyNzYyY1BIcU1neVBwZ0RDL2Nxd2hoZWQ2NTBmYW9Va09jVForbHZlQ0RpcWVZSENkd1Y1Sm1GCnc2QjZDQVpOLzdoS3kvUVFLZU53eTdHYkxYVFhlbHdlNlNPUXdKOU51VTk5SklJd3FmNVhJVjVQdjFISWpYZkYKLzhGYjM2MkxWbkJIUnBsMWsxeFpPOUdmQVpMbWFHRE5GYXRkQkdBZ0ZSUlgzcnZEVVJRM2paUUR0L1JRcnA1cwpyMlVzRGF4aGxzZ2g3NmRoQnhhNzFPZm5QMXU2bjZYdUkxby9yVmpDbGVDUkRXdmpKR09FdklDUGxxUlhKVzAxCnpwOGlLYUxKVllYa0xSbCt3ZHpYQ29rcWlKWUxTWTVhdmdSQWhuNFVYcGY4UWI1SGFHRy9wMEFXdkNMQ2tZODUKMWtNMExlUGcrTmJtVEtKZlplOVdaQmpuK2NzeTFsT0ZETlBObTJqYzZ6ekJPVVhXc0ZWM0Exdit6bnkrL0pJSAo2VnMxSENiSXNENEJYL3gzcjk0eWw5d29XdG83M3dJREFRQUJBb0lDQUNkQmRZQmNlTytWNFIyUkZiVHRiR2paClkzN0JSRU1XblJKenB5NHZqR2NDOTMxbVBqVFM5dmJLUmhOMk9vT3pjVXlHZVovcGhvSDd1VmsvRGtrRFprSFQKTGlzNEJQNGJaTXRGWHBhQ0VYMzJpYlJBYVVXZHZlZ0RaTlNPK01TOXk5MjljY2FMd2pYdmJia1hqL2JGNytiVQpGZE8vVk9tV0N5WUJ2R0NxZjNtbW5UckY2U1pna1pDWDFiRkljZnluZFkwMjV0NkZYNGNFcDZyYkZidi95eXBqCndJMWxIdW0wOURrT2p0eXFVV0VGaXdnVEZiQ0g2YWhjdVhHaWdnWXl0K0NHOXRSelE5YlpLNzE5NFNRWTJBN0IKNUNJOExsSnNJeHdUT29naysvL0h3T3dSUHdqamdrdWllTnJPL1FhZDNKVkxXbXdJQTc1c2J6WGxIeFpYdWhReApFZTlTTVNaRE5JMGMvcGdDVXloL2h5Y3ZYWVJKb1dIMnZYTDYzL1EyRWJVVVZVcXpzcEpjV0xMbkhSSFdBQjZQCmtPb0FMeEs2M3lpQTU5KzREQ0ZZMGM0dXd4WHM5YnZlVWk5UUdxaUtoZkFaNmE3Ymh4ajNvRWIvRDZ0dWZKb1QKMnIxUXFKTlJ4N0RzMmJUc3J6SEgyYmVKR0R4bFJ4djdaNmdSTzdHd2lrU25veDlMcnY3YzMwUHp0WjZENXhwQwpoVHdmM0VpRWlrZ2pOM1FGd2RaUXVqV3RxWXQ1UWVGYUxpL2Q3VXZteFdZNWcrWTYyQnFkNnBpWFFJSlFVbDZUCmZmU0U2OEYvZWF6QnhPQm9INUMvWWcrd1kyVEpTb1BzK3NFUUtnYlNJTmRQNHU4N3FiQ056YWZ4WjFySTN1dmgKR3NZTWpSbm50c3RKamJXVEJkakJBb0lCQVFEZXpsY3dVNFJYbnhKVENSTjFQZUx1UHI2aXFnc09uY1lRRXNuTgpZemdwOXF6eC9SSTJodGZCYnN6TkM2RUZ3eDBDZy8wRnpRbVhqWEQ3VmhQS0UrOHBXMmQzNTg4YTBwU0xSbFBOCkprUnFxR2ovbzhIaEtmTGdCYy9BSGU3dkJEZG1UTXh1d0o1SmFmMWZ3OGtxNEJyRXROTi83SVF3MkNYcFNIK2MKSEhSZnBTM1JidDArbW9xK2RTRW1pU2h3UGgxL2FJbng4ZnBKbTFmaU1QdXhvYStzOXNUVmQ3Ky9GSjdIaWNZSApSQThQRHdQT2RMQ2loQlZFT3liSkduTlpaSFhYQTA4MThhNnpjdjU2aUM2VGZzSkhWZUg4a2Nma1IzblI4eTUvCkx2MnI1QzVaZWJlbW9ZaVhBTC9sVnZ0QUNGMXE3UW1zdlRkQ2R3R1lvaXBMbksvNUFvSUJBUUMzc1YvcFFOVlEKSjNsZExFem9OVUdBZWFwL0NNU2tuY0lUUXNnMUI3SDJHclIyR201VGNuVGE0SE5Rbmt3a2h3YUxyTUlmVDFXZAorMFVvR3M5ZXdyYkZERWtKR0MxQVd2RmFIREk3Q3NOYUFUM2FWYTE5R1ZRWFg2NnkrWEF2QXFTOXpOU25id0J0CnM5UDFFVURwMko1SGtyNlVLUHZjSnY3Q0MyR3J5emZaTy85MFpTTlFQUXdKMXFqZ2NNOUJpMlBYSkE0OENldnMKK3RvaGFqUWFIMStVY0VqNkQ0SHhoQ1RtcTBxM2YvdFNudk9iaTNOYklEWjNEWmR4ejVPelJSVmpLNWlSOGZDNQpzSVVKckxCSXRZeWpUeHAzTzROUmRFaHhzamdBZTZrSTV2dm95RVAvRFZwTFlQU0s3cFpoVjJ4YUpXUkZUUVZzCjhrKzZDbGduWGZDWEFvSUJBUUM5QjY4dFR3NHZFTVNKTW1BUnprbWovQlBkQ2d1TGdRd3pRdDEzcGNCV3lmUDgKOHNycS9BZzlFbllyV0x4cW1Sa1pzMFdPRUdFYzlXRnZ1NTNhaW9NVVFYcE5YcHgxazBkM3lsajY2b2FOUHdpbQpLeGNvbzJCdDlFQklMSjAwcUEwZ2UvUE4yeG53Q3o1dWF6dFhadjhPK0tPZ0d0Z2tZSjM1aUFyTU5jLzkvYlFiCnhjVnJnYzVJdkRNOThJd2dmbktrVDlzSkxGVSs4YzdrRnM3VDYrdVNBV01LQVNqclF1RmJSV1ovYjV5ZkdBd1EKc3l2UkZlSzlHcnBUVUYrZzdmeVVTVGlBK2VWUVZqWFZXNGk0bG9qWjRPRjBXWEtRR0p3Z0pnUEMzK2xVVnFtRQpQQ0kxKzBKWmFzZGtHaUhjTjd5YUpUVmFHc2F4V3lvOWh3Zi9VcFp4QW9JQkFDQ0hDem5ObmpoTVZTUlhsT0xGCmsyekJucHhTSENnZU8yQ1h3Y1lLTDh3cG5HMFJieG5kdWEyTWN6OENXTzlhN2FETUhhL1hwNHlMRXdydi9HcUcKUmtFTVZONkVabmJ2NDY4V01ScmRaQXhMRGYzY2tCVUg2Q2tmYTFzTDZuNllsRDE3eU9oQk1xMDZXNzBZcWdyKwpyY0IwenNTRG9WMnhsZ2tjWk5ZNzdRN05uZ1dwWnlCdFB2VjdDbnA3MzJkMjNGNGJaMTNnVCtPdDQvUm96d01WCkxTS200M1ZNUzdGTnVnOFNvKzlzZlQ5N0lCNGFDbnBIY1AyUjdaQmN0b1hYSk50anUrZVVGUkY4bllKQ0R4RkEKL0w5cVlZQmRqSHBmQWZrSUd2eVM2VExIWERJelREOGN5VEZ4NEx1OVZlbTB4bDRNSXY1V2pqQmxsQktZaEZXcwpQODhDZ2dFQkFKUjR5UUIrREtoRE1ZOGlHOXpYWUJpWmZ5MFI2c21oeEJycTREZWd4cElkajBydHUyMFNuWDdsCnN3YlZsN2NmdWRxUi9tVEhnZ1lYeGJSS3UxUXhmN3NHTW4zWVpRY1AxZDVJelFMOWZFQitidGJsdWY5VTB4NDIKUzM0U0l6Z25pd0hPQnN3M2ZLYm1Gbnc1dmtNd2RoemlBT1R0elRwcmU3ckdhaEpwZlk0eFRWWjZpK3pjQ1pCWgo0ZHhSTjlQWnBiQm4vNmJDTFY3S0I3ZmY0Uk04b3c3K1l4aHFFTFhxcnVQS2paMnRRSWs2M2hzZklMd0tiRWIrCkhEM2VLZEhNc2hhbXFXZzI0NnlORW5nQUZCQzhoVU5kQ0pmeUthUlhkSlV5eDhqcWlKM3ErdnpnNFBieUhqeW0KZEQzM2pVT2UwdHdoSWJxKytUaXZCa1ptSDBsSStnVT0KLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= +kind: Secret +metadata: + creationTimestamp: null + name: cafe-secret +type: kubernetes.io/tls diff --git a/tests/data/transport-server-with-host/transport-server-with-host.yaml b/tests/data/transport-server-with-host/transport-server-with-host.yaml new file mode 100644 index 0000000000..2729dce546 --- /dev/null +++ b/tests/data/transport-server-with-host/transport-server-with-host.yaml @@ -0,0 +1,17 @@ +apiVersion: k8s.nginx.org/v1 +kind: TransportServer +metadata: + name: transport-server +spec: + host: cafe.example.com + listener: + name: dns-tcp + protocol: TCP + tls: + secret: cafe-secret + upstreams: + - name: dns-app + service: coredns + port: 5353 + action: + pass: dns-app diff --git a/tests/suite/test_transport_server_with_host.py b/tests/suite/test_transport_server_with_host.py new file mode 100644 index 0000000000..ca65df5556 --- /dev/null +++ b/tests/suite/test_transport_server_with_host.py @@ -0,0 +1,80 @@ +import pytest +from settings import TEST_DATA +from suite.utils.custom_resources_utils import patch_ts_from_yaml, read_custom_resource +from suite.utils.resources_utils import ( + create_secret_from_yaml, + get_events_for_object, + get_ts_nginx_template_conf, + wait_before_test, +) + + +@pytest.mark.ts +@pytest.mark.parametrize( + "crd_ingress_controller, transport_server_setup", + [ + ( + { + "type": "complete", + "extra_args": [ + "-global-configuration=nginx-ingress/nginx-configuration", + "-enable-leader-election=false", + "-enable-snippets", + ], + }, + {"example": "transport-server-status"}, + ) + ], + indirect=True, +) +class TestTransportServerWithHost: + def test_ts_with_host( + self, kube_apis, crd_ingress_controller, transport_server_setup, ingress_controller_prerequisites + ): + """ + Test TransportServer with Host field without TLS Passthrough + """ + + # TS with Host needs a secret + secret_src = f"{TEST_DATA}/transport-server-with-host/cafe-secret.yaml" + create_secret_from_yaml(kube_apis.v1, transport_server_setup.namespace, secret_src) + + # Update the status TS from the example with one which uses a Host + patch_src = f"{TEST_DATA}/transport-server-with-host/transport-server-with-host.yaml" + patch_ts_from_yaml( + kube_apis.custom_objects, + transport_server_setup.name, + patch_src, + transport_server_setup.namespace, + ) + wait_before_test() + + conf = get_ts_nginx_template_conf( + kube_apis.v1, + transport_server_setup.namespace, + transport_server_setup.name, + transport_server_setup.ingress_pod_name, + ingress_controller_prerequisites.namespace, + ) + print(conf) + + std_src = f"{TEST_DATA}/transport-server-status/standard/transport-server.yaml" + patch_ts_from_yaml( + kube_apis.custom_objects, + transport_server_setup.name, + std_src, + transport_server_setup.namespace, + ) + + conf_lines = [line.strip() for line in conf.split("\n")] + assert 'server_name "cafe.example.com";' in conf_lines + + ts_events = get_events_for_object(kube_apis.v1, transport_server_setup.namespace, transport_server_setup.name) + ts_latest_event = ts_events[-1] + print(ts_latest_event) + assert ts_latest_event.reason == "AddedOrUpdated" and ts_latest_event.type == "Normal" + + ts_info = read_custom_resource( + kube_apis.custom_objects, transport_server_setup.namespace, "transportservers", transport_server_setup.name + ) + assert ts_info["status"] and ts_info["status"]["state"] == "Valid" From 1b7621e08d27ff96f38cde257c8db1c205c18a03 Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Fri, 11 Oct 2024 13:36:52 +0100 Subject: [PATCH 19/27] add listenerhost tests --- internal/k8s/configuration_test.go | 144 ++++++++++++++++++++++++++++- 1 file changed, 141 insertions(+), 3 deletions(-) diff --git a/internal/k8s/configuration_test.go b/internal/k8s/configuration_test.go index 9375a60b9f..b8d0289f09 100644 --- a/internal/k8s/configuration_test.go +++ b/internal/k8s/configuration_test.go @@ -2021,7 +2021,7 @@ func TestAddTransportServerWithHost(t *testing.T) { secretName := "echo-secret" - ts := createTestTransportServerWithHost("transportserver", "echo.example.com", "tcp-7777", "TCP", secretName) + ts := createTestTransportServerWithHost("transportserver", "echo.example.com", "tcp-7777", secretName) // no problems are expected for all cases var expectedProblems []ConfigurationProblem @@ -3840,8 +3840,8 @@ func createTestTransportServer(name string, listenerName string, listenerProtoco } } -func createTestTransportServerWithHost(name string, host string, listenerName string, listenerProtocol string, secretName string) *conf_v1.TransportServer { - ts := createTestTransportServer(name, listenerName, listenerProtocol) +func createTestTransportServerWithHost(name string, host string, listenerName string, secretName string) *conf_v1.TransportServer { + ts := createTestTransportServer(name, listenerName, "TCP") ts.Spec.Host = host ts.Spec.TLS = &conf_v1.TransportServerTLS{Secret: secretName} @@ -4754,3 +4754,141 @@ var ( }, } ) + +func TestTransportServerListenerHostCollisions(t *testing.T) { + configuration := createTestConfiguration() + + listeners := []conf_v1.Listener{ + { + Name: "tcp-7777", + Port: 7777, + Protocol: "TCP", + }, + { + Name: "tcp-8888", + Port: 8888, + Protocol: "TCP", + }, + } + + addOrUpdateGlobalConfiguration(t, configuration, listeners, noChanges, noProblems) + + // Create TransportServers with the same listener and host + ts1 := createTestTransportServerWithHost("ts1", "example.com", "tcp-7777", "secret1") + ts2 := createTestTransportServerWithHost("ts2", "example.com", "tcp-7777", "secret2") // same listener and host + ts3 := createTestTransportServerWithHost("ts3", "example.org", "tcp-7777", "secret3") // different host + ts4 := createTestTransportServer("ts4", "tcp-7777", "TCP") // No host same listener + ts5 := createTestTransportServer("ts5", "tcp-7777", "TCP") // same as ts4 to induce error with empty host twice + ts6 := createTestTransportServerWithHost("ts6", "example.com", "tcp-8888", "secret4") // different listener + + // Add ts1 to the configuration + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &TransportServerConfiguration{ + ListenerPort: 7777, + TransportServer: ts1, + }, + }, + } + changes, problems := configuration.AddOrUpdateTransportServer(ts1) + if diff := cmp.Diff(expectedChanges, changes); diff != "" { + t.Errorf("AddOrUpdateTransportServer(ts1) returned unexpected result (-want +got):\n%s", diff) + } + if len(problems) != 0 { + t.Errorf("AddOrUpdateTransportServer(ts1) returned problems %v", problems) + } + + // Try to add ts2, should be rejected due to conflict + changes, problems = configuration.AddOrUpdateTransportServer(ts2) + expectedChanges = nil // No changes expected + expectedProblems := []ConfigurationProblem{ + { + Object: ts2, + IsError: false, + Reason: "Rejected", + Message: "Listener tcp-7777 with host example.com is taken by another resource", + }, + } + + if diff := cmp.Diff(expectedChanges, changes); diff != "" { + t.Errorf("AddOrUpdateTransportServer(ts2) returned unexpected changes (-want +got):\n%s", diff) + } + if diff := cmp.Diff(expectedProblems, problems); diff != "" { + t.Errorf("AddOrUpdateTransportServer(ts2) returned unexpected problems (-want +got):\n%s", diff) + } + + // Add ts3 with a different host, should be accepted + expectedChanges = []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &TransportServerConfiguration{ + ListenerPort: 7777, + TransportServer: ts3, + }, + }, + } + changes, problems = configuration.AddOrUpdateTransportServer(ts3) + if diff := cmp.Diff(expectedChanges, changes); diff != "" { + t.Errorf("AddOrUpdateTransportServer(ts3) returned unexpected result (-want +got):\n%s", diff) + } + if len(problems) != 0 { + t.Errorf("AddOrUpdateTransportServer(ts3) returned problems %v", problems) + } + + // Add ts4 with no host, should be accepted + expectedChanges = []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &TransportServerConfiguration{ + ListenerPort: 7777, + TransportServer: ts4, + }, + }, + } + changes, problems = configuration.AddOrUpdateTransportServer(ts4) + if diff := cmp.Diff(expectedChanges, changes); diff != "" { + t.Errorf("AddOrUpdateTransportServer(ts4) returned unexpected result (-want +got):\n%s", diff) + } + if len(problems) != 0 { + t.Errorf("AddOrUpdateTransportServer(ts4) returned problems %v", problems) + } + + // Try to add ts5 with no host, should be rejected due to conflict + changes, problems = configuration.AddOrUpdateTransportServer(ts5) + expectedChanges = nil + expectedProblems = []ConfigurationProblem{ + { + Object: ts5, + IsError: false, + Reason: "Rejected", + Message: "Listener tcp-7777 with host empty host is taken by another resource", + }, + } + + if diff := cmp.Diff(expectedChanges, changes); diff != "" { + t.Errorf("AddOrUpdateTransportServer(ts5) returned unexpected changes (-want +got):\n%s", diff) + } + if diff := cmp.Diff(expectedProblems, problems); diff != "" { + t.Errorf("AddOrUpdateTransportServer(ts5) returned unexpected problems (-want +got):\n%s", diff) + } + + // Try to add ts6 with different listener, but same domain as initial ts, should be fine as different listener + changes, problems = configuration.AddOrUpdateTransportServer(ts6) + expectedChanges = []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &TransportServerConfiguration{ + ListenerPort: 8888, + TransportServer: ts6, + }, + }, + } + if diff := cmp.Diff(expectedChanges, changes); diff != "" { + t.Errorf("AddOrUpdateTransportServer(ts6) returned unexpected changes (-want +got):\n%s", diff) + } + + if len(problems) != 0 { + t.Errorf("AddOrUpdateTransportServer(ts6) returned problems %v", problems) + } +} From c85c9a8c475e605c73551a637f7f5363e0150b0a Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Fri, 11 Oct 2024 15:10:16 +0100 Subject: [PATCH 20/27] add docs --- .../configuration/host-and-listener-collisions.md | 10 +++++++--- site/content/configuration/transportserver-resource.md | 7 +++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/site/content/configuration/host-and-listener-collisions.md b/site/content/configuration/host-and-listener-collisions.md index 6ff29cbeb3..5b0a03ede5 100644 --- a/site/content/configuration/host-and-listener-collisions.md +++ b/site/content/configuration/host-and-listener-collisions.md @@ -108,11 +108,13 @@ It is *not* possible to merge configuration for multiple TransportServer resourc --- -## Listener collisions +## Listener/Host collisions -Listener collisions occur when multiple TransportServer resources (Configured for TCP/UDP load balancing) target the same `listener`. +Listener/Host collisions occur when multiple TransportServer resources (configured for TCP/UDP load balancing) specify the same combination of `spec.listener.name` and `spec.host`. -NGINX Ingress Controller will choose the winner, which will own the listener. +The combination of `spec.listener.name` and `spec.host` must be unique among all TransportServer resources. If two TransportServer resources specify the same spec.listener.name and spec.host, one of them will be rejected to prevent conflicts. In the case where spec.host is not specified, it is considered an empty string. + +NGINX Ingress Controller will choose the winner, which will own that listener and host combination. --- @@ -128,6 +130,7 @@ Consider the following two resources: metadata: name: tcp-1 spec: + host: dns.example.com listener: name: dns-tcp protocol: TCP @@ -142,6 +145,7 @@ Consider the following two resources: metadata: name: tcp-2 spec: + host: dns.example.com listener: name: dns-tcp protocol: TCP diff --git a/site/content/configuration/transportserver-resource.md b/site/content/configuration/transportserver-resource.md index d49588006c..b6cf8cfbee 100644 --- a/site/content/configuration/transportserver-resource.md +++ b/site/content/configuration/transportserver-resource.md @@ -30,6 +30,7 @@ The TransportServer resource defines load balancing configuration for TCP, UDP, metadata: name: dns-tcp spec: + host: dns.example.com listener: name: dns-tcp protocol: TCP @@ -89,7 +90,7 @@ The TransportServer resource defines load balancing configuration for TCP, UDP, |Field | Description | Type | Required | | ---| ---| ---| --- | |``listener`` | The listener on NGINX that will accept incoming connections/datagrams. | [listener](#listener) | Yes | -|``host`` | The host (domain name) of the server. Must be a valid subdomain as defined in RFC 1123, such as ``my-app`` or ``hello.example.com``. Wildcard domains like ``*.example.com`` are not allowed. Required for TLS Passthrough load balancing. | ``string`` | No | +|``host`` | The host (domain name) of the server. Must be a valid subdomain as defined in RFC 1123, such as ``my-app`` or ``hello.example.com``. Wildcard domains like ``*.example.com`` are not allowed. When specified, NGINX will use this host for SNI-based routing. For TLS Passthrough, this field is required. For TCP with TLS termination, specifying the host enables SNI routing and requires specifying a TLS secret.| ``string`` | No | |``tls`` | The TLS termination configuration. Not supported for TLS Passthrough load balancing. | [tls](#tls) | No | |``upstreams`` | A list of upstreams. | [[]upstream](#upstream) | Yes | |``upstreamParameters`` | The upstream parameters. | [upstreamParameters](#upstreamparameters) | No | @@ -105,6 +106,8 @@ The TransportServer resource defines load balancing configuration for TCP, UDP, The listener field references a listener that NGINX will use to accept incoming traffic for the TransportServer. For TCP and UDP, the listener must be defined in the [GlobalConfiguration resource]({{< relref "configuration/global-configuration/globalconfiguration-resource.md" >}}). When referencing a listener, both the name and the protocol must match. For TLS Passthrough, use the built-in listener with the name `tls-passthrough` and the protocol `TLS_PASSTHROUGH`. +The combination of ``spec.listener.name`` and ``spec.host`` must be unique among all TransportServers. If two TransportServers specify the same combination of ``spec.listener.name`` and ``spec.host``, one of them will be rejected to prevent conflicts. In the case where no host is specified, it is considered an empty string. + An example: ```yaml @@ -122,7 +125,7 @@ listener: ### TLS -The tls field defines TLS configuration for a TransportServer. Please note the current implementation supports TLS termination on multiple ports, where each application owns a dedicated port - NGINX Ingress Controller terminates TLS connections on each port, where each application uses its own cert/key, and routes connections to appropriate application (service) based on that incoming port (any TLS connection regardless of the SNI on a port will be routed to the application that corresponds to that port). An example configuration is shown below: +The tls field defines TLS configuration for a TransportServer. When using TLS termination (not TLS Passthrough), you can specify the host field to enable SNI-based routing, allowing multiple applications to share the same listener port and be distinguished by the TLS SNI hostname. Each application can use its own TLS certificate and key specified via the secret field. ```yaml secret: cafe-secret From 66bedf5c114d6cae1f9d18207b114eff9b66b751 Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Fri, 11 Oct 2024 16:04:37 +0100 Subject: [PATCH 21/27] fix validateTSHost logic --- .../configuration/validation/transportserver.go | 3 +++ .../validation/transportserver_test.go | 17 +++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pkg/apis/configuration/validation/transportserver.go b/pkg/apis/configuration/validation/transportserver.go index 50a36ffb76..9812e1a5d9 100644 --- a/pkg/apis/configuration/validation/transportserver.go +++ b/pkg/apis/configuration/validation/transportserver.go @@ -112,6 +112,9 @@ func validateTransportServerHost(host string, fieldPath *field.Path, isTLSPassth } return validateHost(host, fieldPath) } + if tls != nil && tls.Secret != "" { + return field.ErrorList{field.Required(fieldPath, "must not specify spec.tls.secret when host using TLS Passthrough")} + } return validateHost(host, fieldPath) } diff --git a/pkg/apis/configuration/validation/transportserver_test.go b/pkg/apis/configuration/validation/transportserver_test.go index a3354785c6..9e7b163c36 100644 --- a/pkg/apis/configuration/validation/transportserver_test.go +++ b/pkg/apis/configuration/validation/transportserver_test.go @@ -293,14 +293,7 @@ func TestValidateTransportServerHost(t *testing.T) { Secret: "secret-name", }, }, - { - host: "nginx.org", - isTLSPassthroughListener: true, - protocol: "TLS_PASSTHROUGH", - tls: &conf_v1.TransportServerTLS{ - Secret: "secret-name", - }, - }, + { host: "nginx.org", isTLSPassthroughListener: false, @@ -357,6 +350,14 @@ func TestValidateTransportServerHost(t *testing.T) { Secret: "secret-name", }, }, + { + host: "nginx.org", + isTLSPassthroughListener: true, + protocol: "TLS_PASSTHROUGH", + tls: &conf_v1.TransportServerTLS{ + Secret: "secret-name", + }, + }, } for _, test := range invalidCases { From 7be8dc2eb6fdfef5181bf5881912417c849c1f4a Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Fri, 11 Oct 2024 16:22:56 +0100 Subject: [PATCH 22/27] remove unused function --- internal/k8s/configuration.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/internal/k8s/configuration.go b/internal/k8s/configuration.go index c966bf5f97..139bf3f7a4 100644 --- a/internal/k8s/configuration.go +++ b/internal/k8s/configuration.go @@ -1773,18 +1773,6 @@ func getSortedTransportServerKeys(m map[string]*conf_v1.TransportServer) []strin return keys } -func getSortedTransportServerConfigurationKeys(m map[string]*TransportServerConfiguration) []string { - var keys []string - - for k := range m { - keys = append(keys, k) - } - - sort.Strings(keys) - - return keys -} - func getSortedListenerHostKeys(m map[listenerHostKey]*TransportServerConfiguration) []listenerHostKey { var keys []listenerHostKey From 17b8e2f6f1ffafce1251dae525711918100b14d7 Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Fri, 11 Oct 2024 16:38:36 +0100 Subject: [PATCH 23/27] Apply suggestions from code review Co-authored-by: Alan Dooley Signed-off-by: Jim Ryan --- .../transport-server-sni/README.md | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/examples/custom-resources/transport-server-sni/README.md b/examples/custom-resources/transport-server-sni/README.md index c1dbaba88b..297354772d 100644 --- a/examples/custom-resources/transport-server-sni/README.md +++ b/examples/custom-resources/transport-server-sni/README.md @@ -11,7 +11,7 @@ In this example we create two different TransportServers that listen on the same protocol: TCP ``` -1. Add a custom port to the NIC pod via the helm chart +2. Add a custom port to the NGINX Ingress Controller pod with the Helm chart. ```yaml controller.customPorts: @@ -20,7 +20,7 @@ controller.customPorts: protocol: TCP ``` -1. Add a custom port to the NIC service +3. Add a custom port to the NGINX Ingress Controller service. ```yaml controller.service.customPorts: @@ -30,22 +30,22 @@ controller.service.customPorts: targetPort: 7000 ``` -1. Create the cafe-secret, and mongo-secret. These secrets are used for TLS in the TransportServers +4. Use `kubectl` to create the cafe-secret, and mongo-secret. These secrets are used for TLS in the TransportServers. `kubectl apply -f cafe-secret.yaml` `kubectl apply -f mongo-secret.yaml` -1. Create the mongo and tcp echo example applications +5. Create the mongo and tcp echo example applications. `kubectl apply -f mongo.yaml` `kubectl apply -f tcp-echo-server.yaml` -1. Wait until these are ready. +6. Wait until these are ready. `kubectl get deploy -w` -1. Create the TransportServers for each application +7. Create the TransportServers for each application. `kubectl apply -f cafe-transport-server.yaml` `kubectl apply -f mongo-transport-server.yaml` -1. Ensure they are in valid state +8. Ensure they are in valid state. `kubectl get ts` ```bash @@ -54,10 +54,10 @@ cafe-ts Valid AddedOrUpdated 2m mongo-ts Valid AddedOrUpdated 2m ``` -1. Set up /etc/hosts or DNS -In this example I am running NIC locally, so I will set -cafe.example.com and mongo.example.com in my /etc/hosts to localhost -If running NIC as a live service, the server names would be set at the DNS layer +9. Set up /etc/hosts or DNS +This example uses a local NGINX Ingress Controller instance, so the /etc/hosts file +is being used to set cafe.example.com and mongo.example.com to localhost. +In a production instance, the server names would be set at the DNS layer. `cat /etc/hosts` ```bash @@ -66,25 +66,27 @@ If running NIC as a live service, the server names would be set at the DNS layer 127.0.0.1 mongo.example.com ``` -1. Expose port 7000 of the NIC Loadbalancer Service +10. Expose port 7000 of the LoadBalancer service. `k port-forward svc/my-release-nginx-ingress-controller 7000:7000` -1. Use ncat to ping cafe.example.com on port 7000 with ssl +11. Use `ncat` to ping cafe.example.com on port 7000 with SSL. `ncat --ssl cafe.example.com 7000` -When you write a message you should get sent back +When you write a message you should receive the following response: ```bash hi hi ``` -And once the connection is closed (ctrl + c) you should see the request and response in the NIC logs are both 2 bytes +Close the connection (CTRL+ c), then view the NGINX Ingress Controller logs. + +The request and response should both be 2 bytes. ```bash 127.0.0.1 [24/Sep/2024:15:48:58 +0000] TCP 200 3 3 2.702 "- ``` -1. Use mongosh to connect to the mongodb container, via the TransportServer, also on port 7000 +12. Use mongosh to connect to the mongodb container through the TransportServer on port 7000. `mongosh --host mongo.example.com --port 7000 --tls --tlsAllowInvalidCertificates` ```bash From 20d71c363d24636561edb07ab5cb667367d062b3 Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Fri, 11 Oct 2024 16:43:36 +0100 Subject: [PATCH 24/27] test undo enumeration --- .../transport-server-sni/README.md | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/custom-resources/transport-server-sni/README.md b/examples/custom-resources/transport-server-sni/README.md index 297354772d..71046182b8 100644 --- a/examples/custom-resources/transport-server-sni/README.md +++ b/examples/custom-resources/transport-server-sni/README.md @@ -11,7 +11,7 @@ In this example we create two different TransportServers that listen on the same protocol: TCP ``` -2. Add a custom port to the NGINX Ingress Controller pod with the Helm chart. +1. Add a custom port to the NGINX Ingress Controller pod with the Helm chart. ```yaml controller.customPorts: @@ -20,7 +20,7 @@ controller.customPorts: protocol: TCP ``` -3. Add a custom port to the NGINX Ingress Controller service. +1. Add a custom port to the NGINX Ingress Controller service. ```yaml controller.service.customPorts: @@ -30,22 +30,22 @@ controller.service.customPorts: targetPort: 7000 ``` -4. Use `kubectl` to create the cafe-secret, and mongo-secret. These secrets are used for TLS in the TransportServers. +1. Use `kubectl` to create the cafe-secret, and mongo-secret. These secrets are used for TLS in the TransportServers. `kubectl apply -f cafe-secret.yaml` `kubectl apply -f mongo-secret.yaml` -5. Create the mongo and tcp echo example applications. +1. Create the mongo and tcp echo example applications. `kubectl apply -f mongo.yaml` `kubectl apply -f tcp-echo-server.yaml` -6. Wait until these are ready. +1. Wait until these are ready. `kubectl get deploy -w` -7. Create the TransportServers for each application. +1. Create the TransportServers for each application. `kubectl apply -f cafe-transport-server.yaml` `kubectl apply -f mongo-transport-server.yaml` -8. Ensure they are in valid state. +1. Ensure they are in valid state. `kubectl get ts` ```bash @@ -54,7 +54,7 @@ cafe-ts Valid AddedOrUpdated 2m mongo-ts Valid AddedOrUpdated 2m ``` -9. Set up /etc/hosts or DNS +1. Set up /etc/hosts or DNS This example uses a local NGINX Ingress Controller instance, so the /etc/hosts file is being used to set cafe.example.com and mongo.example.com to localhost. In a production instance, the server names would be set at the DNS layer. @@ -66,10 +66,10 @@ In a production instance, the server names would be set at the DNS layer. 127.0.0.1 mongo.example.com ``` -10. Expose port 7000 of the LoadBalancer service. +1. Expose port 7000 of the LoadBalancer service. `k port-forward svc/my-release-nginx-ingress-controller 7000:7000` -11. Use `ncat` to ping cafe.example.com on port 7000 with SSL. +1. Use `ncat` to ping cafe.example.com on port 7000 with SSL. `ncat --ssl cafe.example.com 7000` When you write a message you should receive the following response: @@ -86,7 +86,7 @@ The request and response should both be 2 bytes. 127.0.0.1 [24/Sep/2024:15:48:58 +0000] TCP 200 3 3 2.702 "- ``` -12. Use mongosh to connect to the mongodb container through the TransportServer on port 7000. +1. Use mongosh to connect to the mongodb container through the TransportServer on port 7000. `mongosh --host mongo.example.com --port 7000 --tls --tlsAllowInvalidCertificates` ```bash From 775a9b42d0d5a61009b1bf9b23dafd71861eb5bc Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Fri, 11 Oct 2024 16:52:06 +0100 Subject: [PATCH 25/27] change numbers to headings --- .../transport-server-sni/README.md | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/examples/custom-resources/transport-server-sni/README.md b/examples/custom-resources/transport-server-sni/README.md index 71046182b8..8b999c7b53 100644 --- a/examples/custom-resources/transport-server-sni/README.md +++ b/examples/custom-resources/transport-server-sni/README.md @@ -2,7 +2,7 @@ In this example we create two different TransportServers that listen on the same interface, which are distinguished by their Host field. -1. Create a GlobalConfiguration resource with the following listener. +## Create a GlobalConfiguration resource with the following listener ```yaml listeners: @@ -11,7 +11,7 @@ In this example we create two different TransportServers that listen on the same protocol: TCP ``` -1. Add a custom port to the NGINX Ingress Controller pod with the Helm chart. +## Add a custom port to the NGINX Ingress Controller pod with the Helm chart ```yaml controller.customPorts: @@ -20,7 +20,7 @@ controller.customPorts: protocol: TCP ``` -1. Add a custom port to the NGINX Ingress Controller service. +## Add a custom port to the NGINX Ingress Controller service ```yaml controller.service.customPorts: @@ -30,22 +30,27 @@ controller.service.customPorts: targetPort: 7000 ``` -1. Use `kubectl` to create the cafe-secret, and mongo-secret. These secrets are used for TLS in the TransportServers. +## Use `kubectl` to create the cafe-secret, and mongo-secret. These secrets are used for TLS in the TransportServers + `kubectl apply -f cafe-secret.yaml` `kubectl apply -f mongo-secret.yaml` -1. Create the mongo and tcp echo example applications. +## Create the mongo and tcp echo example applications + `kubectl apply -f mongo.yaml` `kubectl apply -f tcp-echo-server.yaml` -1. Wait until these are ready. +## Wait until these are ready + `kubectl get deploy -w` -1. Create the TransportServers for each application. +## Create the TransportServers for each application + `kubectl apply -f cafe-transport-server.yaml` `kubectl apply -f mongo-transport-server.yaml` -1. Ensure they are in valid state. +## Ensure they are in valid state + `kubectl get ts` ```bash @@ -54,7 +59,8 @@ cafe-ts Valid AddedOrUpdated 2m mongo-ts Valid AddedOrUpdated 2m ``` -1. Set up /etc/hosts or DNS +## Set up /etc/hosts or DNS + This example uses a local NGINX Ingress Controller instance, so the /etc/hosts file is being used to set cafe.example.com and mongo.example.com to localhost. In a production instance, the server names would be set at the DNS layer. @@ -66,10 +72,12 @@ In a production instance, the server names would be set at the DNS layer. 127.0.0.1 mongo.example.com ``` -1. Expose port 7000 of the LoadBalancer service. -`k port-forward svc/my-release-nginx-ingress-controller 7000:7000` +## Expose port 7000 of the LoadBalancer service + +`kubectl port-forward svc/my-release-nginx-ingress-controller 7000:7000` + +## Use `ncat` to ping cafe.example.com on port 7000 with SSL -1. Use `ncat` to ping cafe.example.com on port 7000 with SSL. `ncat --ssl cafe.example.com 7000` When you write a message you should receive the following response: @@ -86,7 +94,8 @@ The request and response should both be 2 bytes. 127.0.0.1 [24/Sep/2024:15:48:58 +0000] TCP 200 3 3 2.702 "- ``` -1. Use mongosh to connect to the mongodb container through the TransportServer on port 7000. +## Use mongosh to connect to the mongodb container through the TransportServer on port 7000 + `mongosh --host mongo.example.com --port 7000 --tls --tlsAllowInvalidCertificates` ```bash From e66a52c8dbb825497183a39d60bb26dcfe4aaa7f Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Mon, 14 Oct 2024 16:57:21 +0100 Subject: [PATCH 26/27] make changes from code review --- .../transport-server-sni/README.md | 19 +++++++++++-------- .../transport-server-sni/tcp-echo-server.yaml | 5 +---- .../__snapshots__/templates_test.snap | 5 ----- .../version2/nginx-plus.transportserver.tmpl | 2 +- .../version2/nginx.transportserver.tmpl | 2 +- 5 files changed, 14 insertions(+), 19 deletions(-) diff --git a/examples/custom-resources/transport-server-sni/README.md b/examples/custom-resources/transport-server-sni/README.md index 8b999c7b53..dbbbec83c6 100644 --- a/examples/custom-resources/transport-server-sni/README.md +++ b/examples/custom-resources/transport-server-sni/README.md @@ -1,14 +1,17 @@ # TransportServer SNI In this example we create two different TransportServers that listen on the same interface, which are distinguished by their Host field. +The applications (a TCP echo server, and MongoDB) will be accessed via `ncat` and `mongosh`. +The `ncat` binary is available via `nmap`. On mac/linux this can be installed via homebrew/linuxbrew with `brew install nmap` +`mongosh` installation instructions are [available here](https://www.mongodb.com/docs/mongodb-shell/install/). ## Create a GlobalConfiguration resource with the following listener ```yaml - listeners: - - name: tcp-listener - port: 7000 - protocol: TCP +listeners: + - name: tcp-listener + port: 7000 + protocol: TCP ``` ## Add a custom port to the NGINX Ingress Controller pod with the Helm chart @@ -24,10 +27,10 @@ controller.customPorts: ```yaml controller.service.customPorts: - - name: tcp-port - port: 7000 - protocol: TCP - targetPort: 7000 + - name: tcp-port + port: 7000 + protocol: TCP + targetPort: 7000 ``` ## Use `kubectl` to create the cafe-secret, and mongo-secret. These secrets are used for TLS in the TransportServers diff --git a/examples/custom-resources/transport-server-sni/tcp-echo-server.yaml b/examples/custom-resources/transport-server-sni/tcp-echo-server.yaml index 4be65f2f56..e505296f4a 100644 --- a/examples/custom-resources/transport-server-sni/tcp-echo-server.yaml +++ b/examples/custom-resources/transport-server-sni/tcp-echo-server.yaml @@ -18,10 +18,7 @@ command: ["/bin/sh"] args: - -c - - | - apk add --no-cache socat - echo "TCP Echo Server starting..." - socat -v tcp-listen:7000,fork,reuseaddr,keepalive exec:'/bin/cat' + - nc -lk -p 7000 -e /bin/cat ports: - containerPort: 7000 --- diff --git a/internal/configs/version2/__snapshots__/templates_test.snap b/internal/configs/version2/__snapshots__/templates_test.snap index 327d35224a..1b90fc3072 100644 --- a/internal/configs/version2/__snapshots__/templates_test.snap +++ b/internal/configs/version2/__snapshots__/templates_test.snap @@ -8,7 +8,6 @@ upstream cafe-upstream { server { listen 1234 ssl; listen [::]:1234 ssl; - server_name "cafe.example.com"; ssl_certificate cafe-secret.pem; ssl_certificate_key cafe-secret.pem; @@ -37,7 +36,6 @@ upstream cafe-upstream { server { listen 1234 ssl; listen [::]:1234 ssl; - server_name "cafe.example.com"; ssl_certificate cafe-secret.pem; ssl_certificate_key cafe-secret.pem; @@ -161,7 +159,6 @@ server { listen 127.0.0.1:1234 ssl udp; listen [::1]:1234 ssl udp; - ssl_certificate cafe-secret.pem; ssl_certificate_key cafe-secret.pem; @@ -207,7 +204,6 @@ server { listen 127.0.0.1:1234 ssl; listen [::1]:1234 ssl; - ssl_certificate cafe-secret.pem; ssl_certificate_key cafe-secret.pem; @@ -7973,7 +7969,6 @@ server { listen 1234 ssl udp; listen [::]:1234 ssl udp; - ssl_certificate cafe-secret.pem; ssl_certificate_key cafe-secret.pem; diff --git a/internal/configs/version2/nginx-plus.transportserver.tmpl b/internal/configs/version2/nginx-plus.transportserver.tmpl index 52e5237fa2..3d37d3dfa9 100644 --- a/internal/configs/version2/nginx-plus.transportserver.tmpl +++ b/internal/configs/version2/nginx-plus.transportserver.tmpl @@ -40,7 +40,7 @@ server { set_real_ip_from unix:; {{- else }} {{ makeTransportListener $s | printf }} - {{ makeServerName $s | printf }} + {{- with makeServerName $s }}{{ printf "\t%s" . }}{{- end }} {{- end }} {{- if $ssl.Enabled }} diff --git a/internal/configs/version2/nginx.transportserver.tmpl b/internal/configs/version2/nginx.transportserver.tmpl index a0cde13fab..7715fff38a 100644 --- a/internal/configs/version2/nginx.transportserver.tmpl +++ b/internal/configs/version2/nginx.transportserver.tmpl @@ -24,7 +24,7 @@ server { set_real_ip_from unix:; {{- else }} {{ makeTransportListener $s | printf }} - {{ makeServerName $s | printf }} + {{- with makeServerName $s }}{{ printf "\t%s" . }}{{- end }} {{- end }} {{- if $ssl.Enabled }} From c0bde77d3152014f6ed782a4ed53e420a6e9b5d2 Mon Sep 17 00:00:00 2001 From: Jim Ryan Date: Mon, 21 Oct 2024 11:25:10 +0100 Subject: [PATCH 27/27] bash to shell --- .../custom-resources/transport-server-sni/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/custom-resources/transport-server-sni/README.md b/examples/custom-resources/transport-server-sni/README.md index dbbbec83c6..9c287e541e 100644 --- a/examples/custom-resources/transport-server-sni/README.md +++ b/examples/custom-resources/transport-server-sni/README.md @@ -56,7 +56,7 @@ controller.service.customPorts: `kubectl get ts` -```bash +```shell NAME STATE REASON AGE cafe-ts Valid AddedOrUpdated 2m mongo-ts Valid AddedOrUpdated 2m @@ -69,7 +69,7 @@ is being used to set cafe.example.com and mongo.example.com to localhost. In a production instance, the server names would be set at the DNS layer. `cat /etc/hosts` -```bash +```shell ... 127.0.0.1 cafe.example.com 127.0.0.1 mongo.example.com @@ -84,7 +84,7 @@ In a production instance, the server names would be set at the DNS layer. `ncat --ssl cafe.example.com 7000` When you write a message you should receive the following response: -```bash +```shell hi hi ``` @@ -93,7 +93,7 @@ Close the connection (CTRL+ c), then view the NGINX Ingress Controller logs. The request and response should both be 2 bytes. -```bash +```shell 127.0.0.1 [24/Sep/2024:15:48:58 +0000] TCP 200 3 3 2.702 "- ``` @@ -101,7 +101,7 @@ The request and response should both be 2 bytes. `mongosh --host mongo.example.com --port 7000 --tls --tlsAllowInvalidCertificates` -```bash +```shell test> show dbs admin 40.00 KiB config 60.00 KiB