refact: Improving builder and moving two binaries to cmd/ dir

.gitignore

@@ -22,3 +22,4 @@ _testmain.go
 *.exe
 *.test
 *.prof
+bin/

Makefile

@@ -1,15 +1,46 @@
-all: namenode_exporter resourcemanager_exporter
+# ###############
+# Go Project
+# ###############
+CPWD ?= $(PWD)
+BIN_PATH ?= ./cmd
+BIN_NAMES ?= namenode-exporter resourcemanager-exporter
+GO_FLAGS ?= GOOS=linux GOARCH=amd64
+export GO111MODULE=on
+
+# #######
+# General
+clean:
+	rm -rfv bin/*
+
+all: build
 .PHONY: all
 
 deps:
-	go get github.com/prometheus/client_golang/prometheus
-	go get github.com/prometheus/log
+	@mkdir ./bin || true
 
-namenode_exporter: deps namenode_exporter.go
-	go build namenode_exporter.go
+# #######
+# Builder
 
-resourcemanager_exporter: deps resourcemanager_exporter.go
-	go build resourcemanager_exporter.go
+build: deps
+	@for CMD in $(BIN_NAMES); do \
+		cd $(CPWD)/$(BIN_PATH)/$${CMD}; \
+		echo -e "** Building cmd: $${CMD} **"; \
+		GOOS=linux GOARCH=amd64 \
+		$(GO_FLAGS) go build -o $(CPWD)/bin/$${CMD}; \
+	done
+	@cd $(CPWD)
 
-clean:
-	rm -rf namenode_exporter resourcemanager_exporter
+# #######
+# Release
+tag:
+	$(call deps_tag,$@)
+	git tag -a $(shell cat VERSION) -m "$(message)"
+	git push origin $(shell cat VERSION)
+
+tag-attach:
+	@for CMD in $(BIN_NAMES); do \
+		echo -e "** Publish bin to github: $${CMD} **"; \
+		strip $(CPWD)/bin/$${CMD}; \
+		./hack/github-tag-attachment.sh $(shell cat VERSION) $(CPWD)/bin/$${CMD}; \
+	done
+	@cd $(CPWD)

VERSION

@@ -0,0 +1 @@
+v0.2.0

namenode_exporter.go → cmd/namenode-exporter/namenode_exporter.go

@@ -11,7 +11,8 @@ import (
 )
 
 const (
-	namespace = "namenode"
+	namespace          = "namenode"
+	maxIdleConnections = 10
 )
 
 var (
@@ -21,26 +22,39 @@ var (
 )
 
 type Exporter struct {
-	url                      string
-	MissingBlocks            prometheus.Gauge
-	CapacityTotal            prometheus.Gauge
-	CapacityUsed             prometheus.Gauge
-	CapacityRemaining        prometheus.Gauge
-	CapacityUsedNonDFS       prometheus.Gauge
-	BlocksTotal              prometheus.Gauge
-	FilesTotal               prometheus.Gauge
-	CorruptBlocks            prometheus.Gauge
-	ExcessBlocks             prometheus.Gauge
-	StaleDataNodes           prometheus.Gauge
-	pnGcCount                prometheus.Gauge
-	pnGcTime                 prometheus.Gauge
-	cmsGcCount               prometheus.Gauge
-	cmsGcTime                prometheus.Gauge
-	heapMemoryUsageCommitted prometheus.Gauge
-	heapMemoryUsageInit      prometheus.Gauge
-	heapMemoryUsageMax       prometheus.Gauge
-	heapMemoryUsageUsed      prometheus.Gauge
-	isActive                 prometheus.Gauge
+	url                             string
+	MissingBlocks                   prometheus.Gauge
+	CapacityTotal                   prometheus.Gauge
+	CapacityUsed                    prometheus.Gauge
+	CapacityRemaining               prometheus.Gauge
+	CapacityUsedNonDFS              prometheus.Gauge
+	BlocksTotal                     prometheus.Gauge
+	FilesTotal                      prometheus.Gauge
+	CorruptBlocks                   prometheus.Gauge
+	ExcessBlocks                    prometheus.Gauge
+	StaleDataNodes                  prometheus.Gauge
+	pnGcCount                       prometheus.Gauge
+	pnGcTime                        prometheus.Gauge
+	cmsGcCount                      prometheus.Gauge
+	cmsGcTime                       prometheus.Gauge
+	heapMemoryUsageCommitted        prometheus.Gauge
+	heapMemoryUsageInit             prometheus.Gauge
+	heapMemoryUsageMax              prometheus.Gauge
+	heapMemoryUsageUsed             prometheus.Gauge
+	isActive                        prometheus.Gauge
+	BlockCapacity                   prometheus.Gauge
+	TotalLoad                       prometheus.Gauge
+	UnderReplicatedBlocks           prometheus.Gauge
+	VolumeFailuresTotal             prometheus.Gauge
+	NumLiveDataNodes                prometheus.Gauge
+	NumDeadDataNodes                prometheus.Gauge
+	GcCountConcurrentMarkSweep      prometheus.Gauge
+	GcTimeMillisConcurrentMarkSweep prometheus.Gauge
+	MemNonHeapUsedM                 prometheus.Gauge
+	MemNonHeapCommittedM            prometheus.Gauge
+	MemHeapUsedM                    prometheus.Gauge
+	MemHeapCommittedM               prometheus.Gauge
+	MemHeapMaxM                     prometheus.Gauge
 }
 
 func NewExporter(url string) *Exporter {
@@ -141,6 +155,71 @@ func NewExporter(url string) *Exporter {
 			Name:      "isActive",
 			Help:      "isActive",
 		}),
+		BlockCapacity: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "BlockCapacity",
+			Help:      "BlockCapacity",
+		}),
+		TotalLoad: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "TotalLoad",
+			Help:      "TotalLoad",
+		}),
+		UnderReplicatedBlocks: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "UnderReplicatedBlocks",
+			Help:      "UnderReplicatedBlocks",
+		}),
+		VolumeFailuresTotal: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "VolumeFailuresTotal",
+			Help:      "VolumeFailuresTotal",
+		}),
+		NumLiveDataNodes: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "NumLiveDataNodes",
+			Help:      "NumLiveDataNodes",
+		}),
+		NumDeadDataNodes: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "NumDeadDataNodes",
+			Help:      "NumDeadDataNodes",
+		}),
+		GcCountConcurrentMarkSweep: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "GcCountConcurrentMarkSweep",
+			Help:      "GcCountConcurrentMarkSweep",
+		}),
+		GcTimeMillisConcurrentMarkSweep: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "GcTimeMillisConcurrentMarkSweep",
+			Help:      "GcTimeMillisConcurrentMarkSweep",
+		}),
+		MemNonHeapUsedM: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "MemNonHeapUsedM",
+			Help:      "MemNonHeapUsedM",
+		}),
+		MemNonHeapCommittedM: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "MemNonHeapCommittedM",
+			Help:      "MemNonHeapCommittedM",
+		}),
+		MemHeapUsedM: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "MemHeapUsedM",
+			Help:      "MemHeapUsedM",
+		}),
+		MemHeapCommittedM: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "MemHeapCommittedM",
+			Help:      "MemHeapCommittedM",
+		}),
+		MemHeapMaxM: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "MemHeapMaxM",
+			Help:      "MemHeapMaxM",
+		}),
 	}
 }
 
@@ -165,11 +244,26 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
 	e.heapMemoryUsageMax.Describe(ch)
 	e.heapMemoryUsageUsed.Describe(ch)
 	e.isActive.Describe(ch)
+	e.BlockCapacity.Describe(ch)
+	e.TotalLoad.Describe(ch)
+	e.UnderReplicatedBlocks.Describe(ch)
+	e.VolumeFailuresTotal.Describe(ch)
+	e.NumLiveDataNodes.Describe(ch)
+	e.NumDeadDataNodes.Describe(ch)
+	e.GcCountConcurrentMarkSweep.Describe(ch)
+	e.GcTimeMillisConcurrentMarkSweep.Describe(ch)
+	e.MemNonHeapUsedM.Describe(ch)
+	e.MemNonHeapCommittedM.Describe(ch)
+	e.MemHeapUsedM.Describe(ch)
+	e.MemHeapCommittedM.Describe(ch)
+	e.MemHeapMaxM.Describe(ch)
 }
 
 // Collect implements the prometheus.Collector interface.
 func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
-	resp, err := http.Get(e.url)
+	tr := &http.Transport{MaxIdleConns: maxIdleConnections}
+	client := &http.Client{Transport: tr}
+	resp, err := client.Get(e.url)
 	if err != nil {
 		log.Error(err)
 	}
@@ -245,6 +339,27 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
 			e.CorruptBlocks.Set(nameDataMap["CorruptBlocks"].(float64))
 			e.ExcessBlocks.Set(nameDataMap["ExcessBlocks"].(float64))
 			e.StaleDataNodes.Set(nameDataMap["StaleDataNodes"].(float64))
+			e.BlockCapacity.Set(nameDataMap["BlockCapacity"].(float64))
+			e.TotalLoad.Set(nameDataMap["TotalLoad"].(float64))
+			e.UnderReplicatedBlocks.Set(nameDataMap["UnderReplicatedBlocks"].(float64))
+		}
+		if nameDataMap["name"] == "Hadoop:service=NameNode,name=FSNamesystemState" {
+			e.VolumeFailuresTotal.Set(nameDataMap["VolumeFailuresTotal"].(float64))
+			e.NumLiveDataNodes.Set(nameDataMap["NumLiveDataNodes"].(float64))
+			e.NumDeadDataNodes.Set(nameDataMap["NumDeadDataNodes"].(float64))
+		}
+		if nameDataMap["name"] == "Hadoop:service=NameNode,name=JvmMetrics" {
+			if _, ok := nameDataMap["GcCountConcurrentMarkSweep"]; ok {
+				e.GcCountConcurrentMarkSweep.Set(nameDataMap["GcCountConcurrentMarkSweep"].(float64))
+			}
+			if _, ok := nameDataMap["GcTimeMillisConcurrentMarkSweep"]; ok {
+				e.GcTimeMillisConcurrentMarkSweep.Set(nameDataMap["GcTimeMillisConcurrentMarkSweep"].(float64))
+			}
+			e.MemNonHeapUsedM.Set(nameDataMap["MemNonHeapUsedM"].(float64))
+			e.MemNonHeapCommittedM.Set(nameDataMap["MemNonHeapCommittedM"].(float64))
+			e.MemHeapUsedM.Set(nameDataMap["MemHeapUsedM"].(float64))
+			e.MemHeapCommittedM.Set(nameDataMap["MemHeapCommittedM"].(float64))
+			e.MemHeapMaxM.Set(nameDataMap["MemHeapMaxM"].(float64))
 		}
 		if nameDataMap["name"] == "java.lang:type=GarbageCollector,name=ParNew" {
 			e.pnGcCount.Set(nameDataMap["CollectionCount"].(float64))
@@ -300,6 +415,19 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
 	e.heapMemoryUsageMax.Collect(ch)
 	e.heapMemoryUsageUsed.Collect(ch)
 	e.isActive.Collect(ch)
+	e.BlockCapacity.Collect(ch)
+	e.TotalLoad.Collect(ch)
+	e.UnderReplicatedBlocks.Collect(ch)
+	e.VolumeFailuresTotal.Collect(ch)
+	e.NumLiveDataNodes.Collect(ch)
+	e.NumDeadDataNodes.Collect(ch)
+	e.GcCountConcurrentMarkSweep.Collect(ch)
+	e.GcTimeMillisConcurrentMarkSweep.Collect(ch)
+	e.MemNonHeapUsedM.Collect(ch)
+	e.MemNonHeapCommittedM.Collect(ch)
+	e.MemHeapUsedM.Collect(ch)
+	e.MemHeapCommittedM.Collect(ch)
+	e.MemHeapMaxM.Collect(ch)
 }
 
 func main() {
@@ -315,6 +443,7 @@ func main() {
 		<head><title>NameNode Exporter</title></head>
 		<body>
 		<h1>NameNode Exporter</h1>
+		<p>Parsing JMX counters over HTTP/HTTPS.</p>
 		<p><a href="` + *metricsPath + `">Metrics</a></p>
 		</body>
 		</html>`))

go.sum

@@ -0,0 +1,22 @@
+github.com/Sirupsen/logrus v1.0.6 h1:HCAGQRk48dRVPA5Y+Yh0qdCSTzPOyU1tBJ7Q9YzotII=
+github.com/Sirupsen/logrus v1.0.6/go.mod h1:rmk17hk6i8ZSAJkSDa7nOxamrG+SP4P0mm+DAvExv4U=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/prometheus/client_golang v0.8.0 h1:1921Yw9Gc3iSc4VQh3PIoOqgPCZS7G/4xQNVUp8Mda8=
+github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e h1:n/3MEhJQjQxrOUCzh1Y3Re6aJUUWRp2M9+Oc3eVn/54=
+github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/log v0.0.0-20151026012452-9a3136781e1f h1:G4tJ8/52J/rRmxob3LtolevHcHhCwtxo/2VD0unNM/E=
+github.com/prometheus/log v0.0.0-20151026012452-9a3136781e1f/go.mod h1:1CWrwKZ/oqmOpg817WPlG88DKb9xKdpnq009SEKTgqQ=
+github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273 h1:agujYaXJSxSo18YNX3jzl+4G6Bstwt+kqv47GS12uL0=
+github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b h1:2b9XGzhjiYsYPnKXoEfL7klWZQIt8IfyRCz62gCqqlQ=
+golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

hack/github-tag-attachment.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Upload file as release attachment using ghr.
+# go get -u github.com/tcnksm/ghr
+# https://github.com/tcnksm/ghr
+
+RELEASE=$1
+UPLOAD_FILE=$2
+GH_FILE=$(basename $UPLOAD_FILE)
+
+export GIT_REPO_URL=$(git config -l |awk -F'remote.origin.url=' '{print$2}' |grep -v ^$)
+export GIT_USER=$(echo $GIT_REPO_URL |awk -F':' '{print$2}' |awk -F'/' '{print$1}')
+export GIT_REPO=$(echo $GIT_REPO_URL |awk -F':' '{print$2}' |awk -F'/' '{print$2}' |sed 's/.git//')
+
+ghr -u ${GIT_USER} -r ${GIT_REPO} \
+    --replace "${RELEASE}" ${UPLOAD_FILE}