Skip to content

Allow injection of env vars in any deployment #120

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Sep 12, 2021
Merged

Allow injection of env vars in any deployment #120

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c994ed6
Allow injection of env vars in any deployment
valentintorikian Mar 10, 2020
72ce516
Merge branch 'master' into env-injection
cognifloyd Sep 9, 2021
8388b0b
DRY customEnv definition using a _helper
cognifloyd Sep 9, 2021
7f5ec95
add customEnv for jobs
cognifloyd Sep 9, 2021
3ced2e4
changelog entry
cognifloyd Sep 9, 2021
291d37d
Merge branch 'master' into env-injection
cognifloyd Sep 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 92 additions & 0 deletions templates/deployments.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,13 @@ spec:
# TODO: Add liveness/readiness probes (#3)
#livenessProbe:
#readinessProbe:
{{- if .Values.st2auth.env }}
env:
{{- range $env, $value := .Values.st2auth.env }}
- name: {{ $env | quote }}
value: {{ $value | quote }}
{{- end }}
{{- end }}
Copy link
Member

@cognifloyd cognifloyd Jun 18, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this could be extracted into a helper to avoid some repetition.
So, here it would have:

{{ include "custom-env" .Values.st2auth | indent 8 }}

And then the template would be:

{{- define "custom-env" -}}
  {{- if .env }}
env:
    {{- range $env, $value := .env }}
- name: {{ $env | quote }}
  value: {{ $value | quote }}
    {{- end }}
  {{- end }}
{{- end -}}

envFrom:
- configMapRef:
name: {{ .Release.Name }}-st2-urls
Expand Down Expand Up @@ -191,6 +198,13 @@ spec:
# TODO: Add liveness/readiness probes (#3)
#livenessProbe:
#readinessProbe:
{{- if .Values.st2api.env }}
env:
{{- range $env, $value := .Values.st2api.env }}
- name: {{ $env | quote }}
value: {{ $value | quote }}
{{- end }}
{{- end }}
envFrom:
- configMapRef:
name: {{ .Release.Name }}-st2-urls
Expand Down Expand Up @@ -282,6 +296,13 @@ spec:
# TODO: Add liveness/readiness probes (#3)
#livenessProbe:
#readinessProbe:
{{- if .Values.st2stream.env }}
env:
{{- range $env, $value := .Values.st2stream.env }}
- name: {{ $env | quote }}
value: {{ $value | quote }}
{{- end }}
{{- end }}
envFrom:
- configMapRef:
name: {{ .Release.Name }}-st2-urls
Expand Down Expand Up @@ -370,6 +391,13 @@ spec:
# path: /api/
# port: 443
# initialDelaySeconds: 3
{{- if .Values.st2web.env }}
env:
{{- range $env, $value := .Values.st2web.env }}
- name: {{ $env | quote }}
value: {{ $value | quote }}
{{- end }}
{{- end }}
envFrom:
- configMapRef:
name: {{ .Release.Name }}-st2-urls
Expand Down Expand Up @@ -437,6 +465,13 @@ spec:
# TODO: Add liveness/readiness probes (#3)
#livenessProbe:
#readinessProbe:
{{- if .Values.st2rulesengine.env }}
env:
{{- range $env, $value := .Values.st2rulesengine.env }}
- name: {{ $env | quote }}
value: {{ $value | quote }}
{{- end }}
{{- end }}
envFrom:
- configMapRef:
name: {{ .Release.Name }}-st2-urls
Expand Down Expand Up @@ -513,6 +548,13 @@ spec:
# TODO: Add liveness/readiness probes (#3)
#livenessProbe:
#readinessProbe:
{{- if .Values.st2timersengine.env }}
env:
{{- range $env, $value := .Values.st2timersengine.env }}
- name: {{ $env | quote }}
value: {{ $value | quote }}
{{- end }}
{{- end }}
envFrom:
- configMapRef:
name: {{ .Release.Name }}-st2-urls
Expand Down Expand Up @@ -588,6 +630,13 @@ spec:
# TODO: Add liveness/readiness probes (#3)
#livenessProbe:
#readinessProbe:
{{- if .Values.st2workflowengine.env }}
env:
{{- range $env, $value := .Values.st2workflowengine.env }}
- name: {{ $env | quote }}
value: {{ $value | quote }}
{{- end }}
{{- end }}
envFrom:
- configMapRef:
name: {{ .Release.Name }}-st2-urls
Expand Down Expand Up @@ -662,6 +711,13 @@ spec:
# TODO: Add liveness/readiness probes (#3)
#livenessProbe:
#readinessProbe:
{{- if .Values.st2scheduler.env }}
env:
{{- range $env, $value := .Values.st2scheduler.env }}
- name: {{ $env | quote }}
value: {{ $value | quote }}
{{- end }}
{{- end }}
envFrom:
- configMapRef:
name: {{ .Release.Name }}-st2-urls
Expand Down Expand Up @@ -737,6 +793,13 @@ spec:
# TODO: Add liveness/readiness probes (#3)
#livenessProbe:
#readinessProbe:
{{- if .Values.st2notifier.env }}
env:
{{- range $env, $value := .Values.st2notifier.env }}
- name: {{ $env | quote }}
value: {{ $value | quote }}
{{- end }}
{{- end }}
envFrom:
- configMapRef:
name: {{ .Release.Name }}-st2-urls
Expand Down Expand Up @@ -866,6 +929,13 @@ spec:
- --single-sensor-mode
- --sensor-ref={{ .ref }}
{{- end }}
{{- if .env }}
env:
{{- range $env, $value := .env }}
- name: {{ $env | quote }}
value: {{ $value | quote }}
{{- end }}
{{- end }}
Copy link
Member

@cognifloyd cognifloyd Jun 18, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And the template here would just be:

{{ include "custom-env" . | indent 8 }}

envFrom:
- configMapRef:
name: {{ $.Release.Name }}-st2-urls
Expand Down Expand Up @@ -997,6 +1067,13 @@ spec:
# TODO: Add liveness/readiness probes (#3)
#livenessProbe:
#readinessProbe:
{{- if .Values.st2actionrunner.env }}
env:
{{- range $env, $value := .Values.st2actionrunner.env }}
- name: {{ $env | quote }}
value: {{ $value | quote }}
{{- end }}
{{- end }}
envFrom:
- configMapRef:
name: {{ .Release.Name }}-st2-urls
Expand Down Expand Up @@ -1097,6 +1174,13 @@ spec:
# TODO: Add liveness/readiness probes (#3)
#livenessProbe:
#readinessProbe:
{{- if .Values.st2garbagecollector.env }}
env:
{{- range $env, $value := .Values.st2garbagecollector.env }}
- name: {{ $env | quote }}
value: {{ $value | quote }}
{{- end }}
{{- end }}
envFrom:
- configMapRef:
name: {{ .Release.Name }}-st2-urls
Expand Down Expand Up @@ -1242,6 +1326,10 @@ spec:
env:
- name: ST2CLIENT
value: "1"
{{- range $env, $value := .Values.st2client.env }}
- name: {{ $env | quote }}
value: {{ $value | quote }}
{{- end }}
Copy link
Member

@cognifloyd cognifloyd Jun 18, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A couple of jobs also have an env: block already. So, we could split the template into two to accommodate this usage.

{{- define "custom-env-entries" -}}
  {{- range $env, $value := .env }}
- name: {{ $env | quote }}
  value: {{ $value | quote }}
  {{- end }}
{{- end -}}
{{- define "custom-env" -}}
  {{- if .env }}
env:
{{ include "custom-env-entries" . }}
  {{- end }}
{{- end -}}

Which would make this block be:

{{ include "custom-env-entries" .Values.st2client }}

envFrom:
- configMapRef:
name: {{ .Release.Name }}-st2-urls
Expand Down Expand Up @@ -1369,6 +1457,10 @@ spec:
secretKeyRef:
name: {{ .Release.Name }}-st2-auth
key: password
{{- range $env, $value := .Values.st2chatops.env }}
- name: {{ $env | quote }}
value: {{ $value | quote }}
{{- end }}
Copy link
Member

@cognifloyd cognifloyd Jun 18, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I would drop this, and leave the st2chatops env as is (in envFrom[1].secretRef below).

Suggested change
{{- range $env, $value := .Values.st2chatops.env }}
- name: {{ $env | quote }}
value: {{ $value | quote }}
{{- end }}

envFrom:
- configMapRef:
name: {{ .Release.Name }}-st2-urls
Expand Down
2 changes: 1 addition & 1 deletion templates/secrets_st2chatops.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ metadata:
heritage: {{ .Release.Service }}
type: Opaque
data:
{{- range $env, $value := .Values.st2chatops.env }}
{{- range $env, $value := .Values.secrets.st2.st2chatops.env }}
Copy link
Member

@arm4b arm4b Jun 26, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think there is at least an agreement about the 💯 need of raw ENV for every Pod and that's super helpful.
Can we keep raw ENV functionality in this PR to be able to merge it asap, but split the st2chatops secrets ENV enhancement into another PR?

{{ $env }}: {{ $value | b64enc | quote }}
{{- end }}
{{- end }}
32 changes: 31 additions & 1 deletion values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,8 @@ st2:
affinity: {}
nodeSelector: {}
tolerations: []
env: {}
# HTTP_PROXY: http://proxy:1234
Comment on lines +177 to +178
Copy link
Member

@cognifloyd cognifloyd Jun 18, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice. I like the commented example. 👍

# Import data into StackStorm's Key/Value datastore (https://docs.stackstorm.com/datastore.html)
keyvalue:
#- name: st2_version
Expand Down Expand Up @@ -158,6 +160,9 @@ ingress:
# TODO: Alternatively as part of reorganizing Helm values, consider moving values to existing `st2` and `st2web` sections ? (#14)
secrets:
st2:
st2chatops:
Copy link

@mickmcgrath13 mickmcgrath13 Mar 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any reason this is secrets.st2.st2chatops instead of secrets.st2chatops (i.e. why the nesting under st2)?

Copy link

@mickmcgrath13 mickmcgrath13 Mar 10, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, any reason not to put the secrets specific to st2chatops under st2chatops (and the same for all other services) instead of aggregating all secrets to a central place?

i.e. instead of

secrets:
  st2:
    st2chatops:
      foo: bar

do

st2chatops:
  secrets:
    foo: bar

?

Copy link
Contributor Author

@valentintorikian valentintorikian Mar 10, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any reason this is secrets.st2.st2chatops instead of secrets.st2chatops (i.e. why the nesting under st2)?

For this part, no particular reason, it's true that it's a bit too verbose.

Also, any reason not to put the secrets specific to st2chatops under st2chatops (and the same for all other services) instead of aggregating all secrets to a central place?

See issue #14 :)

Copy link

@mickmcgrath13 mickmcgrath13 Mar 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Makes sense. I think it'd be preferable to spread the secrets out into their respective services, but we can tackle that when we tackle #14

Thanks!

Copy link
Member

@arm4b arm4b Mar 10, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree that secrets.st2.st2chatops looks too verbose and even counter-intuitive.

This also raises some inconsistencies in the context of #119 discussion. I'm thinking about exploring the alternatives for the Helm values definition.

env:
HUBOT_SLACK_TOKEN: xoxb-CHANGE-ME-PLEASE
# Username, used to login to StackStorm system
username: st2admin
# Password, used to login to StackStorm system
Expand Down Expand Up @@ -223,6 +228,8 @@ st2web:
nodeSelector: {}
tolerations: []
affinity: {}
env: {}
# HTTP_PROXY: http://proxy:1234
# https://docs.stackstorm.com/reference/ha.html#st2auth
# Multiple st2auth processes can be behind a load balancer in an active-active configuration.
st2auth:
Expand All @@ -233,6 +240,8 @@ st2auth:
nodeSelector: {}
tolerations: []
affinity: {}
env: {}
# HTTP_PROXY: http://proxy:1234
# https://docs.stackstorm.com/reference/ha.html#st2api
# Multiple st2api process can be behind a load balancer in an active-active configuration.
st2api:
Expand All @@ -243,6 +252,8 @@ st2api:
nodeSelector: {}
tolerations: []
affinity: {}
env: {}
# HTTP_PROXY: http://proxy:1234
# https://docs.stackstorm.com/reference/ha.html#st2stream
# Multiple st2stream process can be behind a load balancer in an active-active configuration.
st2stream:
Expand All @@ -253,6 +264,8 @@ st2stream:
nodeSelector: {}
tolerations: []
affinity: {}
env: {}
# HTTP_PROXY: http://proxy:1234
# https://docs.stackstorm.com/reference/ha.html#st2rulesengine
# Multiple st2rulesengine processes can run in active-active with only connections to MongoDB and RabbitMQ. All these will share the TriggerInstance load and naturally pick up more work if one or more of the processes becomes unavailable.
st2rulesengine:
Expand All @@ -263,6 +276,8 @@ st2rulesengine:
nodeSelector: {}
tolerations: []
affinity: {}
env: {}
# HTTP_PROXY: http://proxy:1234
# https://docs.stackstorm.com/reference/ha.html#st2timersengine
# Only single replica is created via K8s Deployment as timersengine can't work in active-active mode at the moment and it relies on K8s failover/reschedule capabilities to address cases of process failure.
st2timersengine:
Expand All @@ -272,6 +287,8 @@ st2timersengine:
nodeSelector: {}
tolerations: []
affinity: {}
env: {}
# HTTP_PROXY: http://proxy:1234
# https://docs.stackstorm.com/reference/ha.html#st2workflowengine
# Multiple st2workflowengine processes can run in active-active mode and will share the load and pick up more work if one or more of the processes become available.
st2workflowengine:
Expand All @@ -282,6 +299,8 @@ st2workflowengine:
nodeSelector: {}
tolerations: []
affinity: {}
env: {}
# HTTP_PROXY: http://proxy:1234
# https://docs.stackstorm.com/reference/ha.html#st2scheduler
# TODO: Description TBD
st2scheduler:
Expand All @@ -292,6 +311,8 @@ st2scheduler:
nodeSelector: {}
tolerations: []
affinity: {}
env: {}
# HTTP_PROXY: http://proxy:1234
# https://docs.stackstorm.com/reference/ha.html#st2notifier
# st2notifier runs in active-active mode and requires for that coordination backend like Redis or Zookeeper
st2notifier:
Expand All @@ -302,6 +323,8 @@ st2notifier:
nodeSelector: {}
tolerations: []
affinity: {}
env: {}
# HTTP_PROXY: http://proxy:1234
# https://docs.stackstorm.com/reference/ha.html#st2actionrunner
# Multiple st2actionrunner processes can run in active-active with only connections to MongoDB and RabbitMQ. Work gets naturally
# distributed across runners via RabbitMQ. Adding more st2actionrunner processes increases the ability of StackStorm to execute actions.
Expand All @@ -322,6 +345,8 @@ st2actionrunner:
# - hostnames:
# - bar
# ip: 8.8.8.8
env: {}
# HTTP_PROXY: http://proxy:1234

# https://docs.stackstorm.com/reference/ha.html#st2garbagecollector
# Optional service that cleans up old executions and other operations data based on setup configurations.
Expand All @@ -335,6 +360,12 @@ st2garbagecollector:
nodeSelector: {}
tolerations: []
affinity: {}
env: {}
# HTTP_PROXY: http://proxy:1234

st2client:
env: {}
# HTTP_PROXY: http://proxy:1234

##
## StackStorm ChatOps (https://docs.stackstorm.com/chatops/index.html)
Expand All @@ -348,7 +379,6 @@ st2chatops:
# for the full list of supported adapters and example ENV variables.
env:
HUBOT_ADAPTER: slack
HUBOT_SLACK_TOKEN: xoxb-CHANGE-ME-PLEASE
# Set custom generated st2chatops Docker image source
# Otherwise default https://hub.docker.com/r/stackstorm/st2chatops is used
image: {}
Expand Down