Skip to content

Commit 6e91f50

Browse files
simonswinesimonswine
committed
Add observability for Logs, Trace
- Using Loki for logs - Using Tempo for distributed tracing - Grafana Agent for collection
1 parent 15dc890 commit 6e91f50

File tree

6 files changed

+383
-16
lines changed

6 files changed

+383
-16
lines changed

‎Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -307,11 +307,11 @@ helm/check: $(BIN)/kubeval $(BIN)/helm
307307

308308
.PHONY: deploy
309309
deploy: $(BIN)/kind $(BIN)/helm docker-image/phlare/build
310-
$(call deploy,phlare-dev,)
310+
$(call deploy,phlare-dev,--set=phlare.extraEnvVars.JAEGER_AGENT_HOST=jaeger.monitoring.svc.cluster.local.)
311311

312312
.PHONY: deploy-micro-services
313313
deploy-micro-services: $(BIN)/kind $(BIN)/helm docker-image/phlare/build
314-
$(call deploy,phlare-micro-services,--values=operations/phlare/helm/phlare/values-micro-services.yaml)
314+
$(call deploy,phlare-micro-services,--values=operations/phlare/helm/phlare/values-micro-services.yaml --set=phlare.extraEnvVars.JAEGER_AGENT_HOST=jaeger.monitoring.svc.cluster.local.)
315315

316316
.PHONY: deploy-monitoring
317317
deploy-monitoring: $(BIN)/tk $(BIN)/kind tools/monitoring/environments/default/spec.json

‎tools/monitoring/environments/default/loki.libsonnet

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
local agent = import 'github.com/grafana/agent/production/tanka/grafana-agent/v2/main.libsonnet';
2+
local loki = import 'github.com/grafana/loki/production/ksonnet/loki-simple-scalable/loki.libsonnet';
3+
local k = import 'ksonnet-util/kausal.libsonnet';
4+
local prom_k_grafana = import 'prometheus-ksonnet/grafana/grafana.libsonnet';
5+
6+
7+
{
8+
local namespace = if std.objectHas($._config, 'namespace') then $._config.namespace else 'mynamespace',
9+
10+
local podsLogPath = '/var/log/pods',
11+
12+
_config+: {},
13+
14+
agent+: {
15+
_config+: { agent_config+: {
16+
logs+: {
17+
configs: [{
18+
name: 'kubernetes-logs',
19+
clients: [{
20+
url: 'http://%s:3100/loki/api/v1/push' % $.loki.fqdn,
21+
}],
22+
positions: {
23+
filename: podsLogPath + '/grafana-agent-positions.yaml',
24+
},
25+
scrape_configs: agent.newKubernetesLogs({}),
26+
}],
27+
},
28+
} },
29+
30+
controller+:
31+
local sts = k.apps.v1.statefulSet;
32+
sts.spec.template.spec.withVolumesMixin(
33+
local volume = k.core.v1.volume;
34+
[
35+
volume.fromHostPath('logs', podsLogPath),
36+
]
37+
),
38+
39+
container+:
40+
{
41+
volumeMounts+:
42+
local volumeMount = k.core.v1.volumeMount;
43+
[
44+
volumeMount.new('logs', podsLogPath),
45+
],
46+
47+
},
48+
},
49+
50+
loki:
51+
local upstream = loki {
52+
_config+:: {
53+
headless_service_name: 'loki',
54+
http_listen_port: 3100,
55+
read_replicas: 1,
56+
write_replicas: 1,
57+
loki: {
58+
auth_enabled: false,
59+
},
60+
},
61+
62+
63+
write_args: {
64+
'config.file': '/etc/loki/local-config.yaml',
65+
},
66+
67+
write_statefulset+:
68+
{
69+
spec+: {
70+
volumeClaimTemplates:: [],
71+
template+: {
72+
spec+: {
73+
volumes:
74+
local volume = k.core.v1.volume;
75+
[
76+
volume.fromEmptyDir('data'),
77+
],
78+
containers: [
79+
c {
80+
name: 'loki',
81+
volumeMounts: [
82+
x {
83+
name: 'data',
84+
}
85+
for x in super.volumeMounts
86+
if x.name == 'write-data'
87+
],
88+
}
89+
for c in super.containers
90+
],
91+
},
92+
},
93+
},
94+
},
95+
};
96+
97+
{
98+
fqdn:: 'loki.%s.svc.cluster.local.' % namespace,
99+
100+
stateful_set:
101+
local sts = k.apps.v1.statefulSet;
102+
103+
upstream.write_statefulset +
104+
sts.metadata.withName('loki') +
105+
sts.metadata.withLabelsMixin({ name: 'loki' }) +
106+
sts.spec.template.metadata.withLabelsMixin({ name: 'loki' }) +
107+
sts.spec.selector.withMatchLabelsMixin({ name: 'loki' }) +
108+
{ spec+: { template+: { spec+: { affinity:: {} } } } },
109+
110+
service:
111+
local svc = k.core.v1.service;
112+
svc.new('loki', { name: 'loki' }, [{ port: 3100 }]),
113+
114+
},
115+
116+
grafana_datasource_config_map+: k.core.v1.configMap.withDataMixin({
117+
'loki-datasource.yml': k.util.manifestYaml({
118+
apiVersion: 1,
119+
datasources: [
120+
prom_k_grafana.grafana_datasource(
121+
'Loki',
122+
'http://%s:3100' % $.loki.fqdn,
123+
type='loki'
124+
),
125+
],
126+
}),
127+
}),
128+
}

‎tools/monitoring/environments/default/monitoring.libsonnet

Lines changed: 99 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,17 @@ local k = import 'ksonnet-util/kausal.libsonnet';
44
local prom_k_grafana = import 'prometheus-ksonnet/grafana/grafana.libsonnet';
55
local prometheus = import 'prometheus-ksonnet/prometheus-ksonnet.libsonnet';
66

7+
local namespace = 'monitoring';
8+
local fqdn = {
9+
phlare: 'phlare-micro-services-query-frontend.default.svc.cluster.local.',
10+
};
11+
712

813
prometheus {
914
local cluster_name = 'dev',
1015
_config+:: {
1116
cluster_name: cluster_name,
12-
namespace: 'monitoring',
17+
namespace: namespace,
1318
grafana_ini+: {
1419
sections+: {
1520
feature_toggles+: {
@@ -42,18 +47,105 @@ prometheus {
4247
mixins+:: {
4348
phlare: phlare {},
4449
},
50+
51+
agent:
52+
local pvc = k.core.v1.persistentVolumeClaim;
53+
local volumeMount = k.core.v1.volumeMount;
54+
local containerPort = k.core.v1.containerPort;
55+
local extraPorts = [
56+
{
57+
name: 'thrift-compact',
58+
port: 6831,
59+
protocol: 'UDP',
60+
},
61+
{
62+
name: 'sampling',
63+
port: 5778,
64+
protocol: 'TCP',
65+
},
66+
];
67+
local agent = import 'github.com/grafana/agent/production/tanka/grafana-agent/v2/main.libsonnet';
68+
agent.new(name='grafana-agent', namespace=$._config.namespace) +
69+
agent.withStatefulSetController(
70+
replicas=1,
71+
) +
72+
// problems with remote sampling in anything later than v0.28.1
73+
// see
74+
// https://github.com/grafana/agent/issues/2911
75+
agent.withImagesMixin({ agent: 'grafana/agent:v0.28.1' }) +
76+
agent.withArgsMixin({
77+
'enable-features': 'integrations-next',
78+
},) +
79+
// add dummy config or else will fail
80+
agent.withAgentConfig({
81+
server: { log_level: 'debug' },
82+
}) +
83+
agent.withVolumeMountsMixin([volumeMount.new('agent-wal', '/var/lib/agent')]) +
84+
// headless svc needed by statefulset
85+
agent.withService() +
86+
{
87+
controller_service+: {
88+
spec+: {
89+
clusterIP: 'None',
90+
},
91+
},
92+
controller+: {
93+
spec+: {
94+
template+: { spec+: {
95+
volumes+:
96+
local volume = k.core.v1.volume;
97+
[
98+
volume.fromEmptyDir('agent-wal'),
99+
],
100+
} },
101+
},
102+
},
103+
104+
105+
container+: {
106+
ports+: [
107+
std.prune(p {
108+
containerPort: p.port,
109+
port::: null,
110+
})
111+
for p in extraPorts
112+
],
113+
},
114+
115+
configMap+: {
116+
data+: {
117+
'strategy.json': std.manifestJsonMinified({
118+
default_strategy: { param: 1.0, type: 'probabilistic' },
119+
}),
120+
},
121+
},
122+
123+
jaeger_service:
124+
local svc = k.core.v1.service;
125+
126+
super.controller_service +
127+
svc.metadata.withName('jaeger') +
128+
svc.metadata.withLabelsMixin({ name: 'jaeger' }) +
129+
svc.spec.withPorts([
130+
p {
131+
targetPort: p.port,
132+
}
133+
for p in extraPorts
134+
]),
135+
},
136+
45137
grafana_datasource_config_map+: k.core.v1.configMap.withDataMixin({
46138
'phlare-datasource.yml': k.util.manifestYaml({
47139
apiVersion: 1,
48140
datasources: [
49141
prom_k_grafana.grafana_datasource(
50-
'phlare',
51-
'http://phlare-micro-services-query-frontend.default.svc.cluster.local.:4100',
142+
'Phlare',
143+
'http://%s:4100' % fqdn.phlare,
52144
type='phlare'
53-
) + grafana.datasource.withJsonData({
54-
path: 'http://phlare-micro-services-query-frontend.default.svc.cluster.local.:4100/',
55-
},),
145+
),
56146
],
57147
}),
58148
}),
59-
}
149+
} +
150+
(import 'tempo.libsonnet') +
151+
(import 'loki.libsonnet')

‎tools/monitoring/environments/default/tempo.libsonnet

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
local agent = import 'github.com/grafana/agent/production/tanka/grafana-agent/v2/main.libsonnet';
2+
local tempo = import 'github.com/grafana/tempo/operations/jsonnet/single-binary/tempo.libsonnet';
3+
local k = import 'ksonnet-util/kausal.libsonnet';
4+
local prom_k_grafana = import 'prometheus-ksonnet/grafana/grafana.libsonnet';
5+
6+
{
7+
local namespace = if std.objectHas($._config, 'namespace') then $._config.namespace else 'mynamespace',
8+
9+
_config+: {},
10+
11+
agent+: {
12+
_config+: { agent_config+: {
13+
traces+: {
14+
configs: [{
15+
name: 'kubernetes-traces',
16+
receivers: {
17+
jaeger: {
18+
protocols: {
19+
grpc: null,
20+
thrift_binary: null,
21+
thrift_compact: null,
22+
thrift_http: null,
23+
},
24+
remote_sampling: {
25+
strategy_file: '/etc/agent/strategy.json',
26+
tls: {
27+
insecure: true,
28+
},
29+
},
30+
},
31+
},
32+
remote_write: [{
33+
endpoint: '%s:4317' % $.tempo.fqdn,
34+
insecure: true,
35+
retry_on_failure: {
36+
enabled: true,
37+
},
38+
}],
39+
scrape_configs: agent.newKubernetesTraces({}),
40+
}],
41+
},
42+
} },
43+
},
44+
45+
46+
tempo:
47+
tempo {
48+
local cluster_name = 'dev',
49+
_config+:: {
50+
cluster_name: cluster_name,
51+
namespace: 'monitoring',
52+
receivers: {
53+
jaeger: {
54+
protocols: {
55+
grpc: null,
56+
thrift_http: null,
57+
thrift_binary: null,
58+
thrift_compact: null,
59+
},
60+
},
61+
zipkin: null,
62+
otlp: {
63+
protocols: {
64+
http: null,
65+
grpc: null,
66+
},
67+
},
68+
opencensus: null,
69+
},
70+
71+
// need to set something, but will use empty dir anyhow
72+
pvc_size: null,
73+
pvc_storage_class: null,
74+
},
75+
76+
tempo_container+:
77+
k.util.resourcesRequests('25m', '128Mi') +
78+
k.util.resourcesLimits(null, null),
79+
80+
tempo_statefulset+: {
81+
spec+: {
82+
volumeClaimTemplates: [
83+
],
84+
template+: { spec+: {
85+
volumes+:
86+
local volume = k.core.v1.volume;
87+
[
88+
volume.fromEmptyDir('tempo-data'),
89+
],
90+
} },
91+
},
92+
},
93+
} {
94+
fqdn:: 'tempo.%s.svc.cluster.local.' % namespace,
95+
},
96+
97+
grafana_datasource_config_map+: k.core.v1.configMap.withDataMixin({
98+
'tempo-datasource.yml': k.util.manifestYaml({
99+
apiVersion: 1,
100+
datasources: [
101+
prom_k_grafana.grafana_datasource(
102+
'Tempo',
103+
'http://%s:3200' % $.tempo.fqdn,
104+
type='tempo'
105+
),
106+
],
107+
}),
108+
}),
109+
}

0 commit comments

Comments
 (0)