Update beats

NOTICE.txt

@@ -264,7 +264,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 --------------------------------------------------------------------
 Dependency: github.com/elastic/beats
 Version: master
-Revision: 554ddcd4c23f099e4e5f97c1ddaccec7d5e38b5f
+Revision: 23d07fd9b4dd3bd46ad74714b10427cfaadfe9e8
 License type (autodetected): Apache-2.0
 ./vendor/github.com/elastic/beats/LICENSE.txt:
 --------------------------------------------------------------------
@@ -2365,7 +2365,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------
 Dependency: golang.org/x/tools
-Revision: 5d2fd3ccab986d52112bf301d47a819783339d0e
+Revision: release-branch.go1.10
 License type (autodetected): BSD-3-Clause
 ./vendor/golang.org/x/tools/LICENSE:
 --------------------------------------------------------------------

_beats/dev-tools/cmd/asset/asset.go

@@ -37,6 +37,7 @@ var (
 	pkg     string
 	input   string
 	output  string
+	name    string
 	license = "ASL2"
 )
 
@@ -45,6 +46,7 @@ func init() {
 	flag.StringVar(&input, "in", "-", "Source of input. \"-\" means reading from stdin")
 	flag.StringVar(&output, "out", "-", "Output path. \"-\" means writing to stdout")
 	flag.StringVar(&license, "license", "ASL2", "License header for generated file.")
+	flag.StringVar(&name, "name", "", "Asset name")
 }
 
 func main() {
@@ -94,9 +96,12 @@ func main() {
 		os.Exit(1)
 	}
 	var buf bytes.Buffer
+	if name == "" {
+		name = file
+	}
 	asset.Template.Execute(&buf, asset.Data{
 		Beat:    beatName,
-		Name:    file,
+		Name:    name,
 		Data:    encData,
 		License: licenseHeader,
 		Package: pkg,

_beats/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go

@@ -19,51 +19,73 @@ package main
 
 import (
 	"flag"
-	"fmt"
-	"os"
+	"log"
+	"path/filepath"
 
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/kibana"
 	"github.com/elastic/beats/libbeat/version"
 )
 
+var usageText = `
+Usage: kibana_index_pattern [flags]
+  kibana_index_pattern generates Kibana index patterns from the Beat's
+  fields.yml file. It will create a index pattern file that is usable with both
+  Kibana 5.x and 6.x.
+Options:
+`[1:]
+
+var (
+	beatName       string
+	beatVersion    string
+	indexPattern   string
+	fieldsYAMLFile string
+	outputDir      string
+)
+
+func init() {
+	flag.StringVar(&beatName, "beat", "", "Name of the beat. (Required)")
+	flag.StringVar(&beatVersion, "version", version.GetDefaultVersion(), "Beat version. (Required)")
+	flag.StringVar(&indexPattern, "index", "", "Kibana index pattern. (Required)")
+	flag.StringVar(&fieldsYAMLFile, "fields", "fields.yml", "fields.yml file containing all fields used by the Beat.")
+	flag.StringVar(&outputDir, "out", "build/kibana", "Output dir.")
+}
+
 func main() {
-	index := flag.String("index", "", "The name of the index pattern. (required)")
-	beatName := flag.String("beat-name", "", "The name of the beat. (required)")
-	beatDir := flag.String("beat-dir", "", "The local beat directory. (required)")
-	beatVersion := flag.String("version", version.GetDefaultVersion(), "The beat version.")
+	log.SetFlags(0)
 	flag.Parse()
 
-	if *index == "" {
-		fmt.Fprint(os.Stderr, "The name of the index pattern must be set.")
-		os.Exit(1)
+	if beatName == "" {
+		log.Fatal("Name of the Beat must be set (-beat).")
 	}
 
-	if *beatName == "" {
-		fmt.Fprint(os.Stderr, "The name of the beat must be set.")
-		os.Exit(1)
+	if beatVersion == "" {
+		log.Fatal("Beat version must be set (-version).")
 	}
 
-	if *beatDir == "" {
-		fmt.Fprint(os.Stderr, "The beat directory must be set.")
-		os.Exit(1)
+	if indexPattern == "" {
+		log.Fatal("Index pattern must be set (-index).")
 	}
 
 	version5, _ := common.NewVersion("5.0.0")
 	version6, _ := common.NewVersion("6.0.0")
-	versions := []*common.Version{version5, version6}
+	versions := []common.Version{*version5, *version6}
 	for _, version := range versions {
+		indexPattern, err := kibana.NewGenerator(indexPattern, beatName, fieldsYAMLFile, outputDir, beatVersion, version)
+		if err != nil {
+			log.Fatal(err)
+		}
 
-		indexPatternGenerator, err := kibana.NewGenerator(*index, *beatName, *beatDir, *beatVersion, *version)
+		file, err := indexPattern.Generate()
 		if err != nil {
-			fmt.Fprintf(os.Stderr, err.Error())
-			os.Exit(1)
+			log.Fatal(err)
 		}
-		pattern, err := indexPatternGenerator.Generate()
+
+		// Log output file location.
+		absFile, err := filepath.Abs(file)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, err.Error())
-			os.Exit(1)
+			absFile = file
 		}
-		fmt.Fprintf(os.Stdout, "-- The index pattern was created under %v\n", pattern)
+		log.Printf(">> The index pattern was created under %v", absFile)
 	}
 }

_beats/dev-tools/cmd/module_fields/main.go

@@ -0,0 +1,118 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"go/format"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+
+	"github.com/elastic/beats/libbeat/asset"
+	"github.com/elastic/beats/libbeat/generator/fields"
+	"github.com/elastic/beats/licenses"
+)
+
+var usageText = `
+Usage: module_fields [flags] [module-dir]
+  module_fields generates a fields.go file containing a copy of the module's
+  field.yml data in a format that can be embedded in Beat's binary. module-dir
+  should be the directory containing modules (e.g. filebeat/module).
+Options:
+`[1:]
+
+var (
+	beatName string
+	license  string
+)
+
+func init() {
+	flag.StringVar(&beatName, "beat", "", "Name of the beat. (Required)")
+	flag.StringVar(&license, "license", "ASL2", "License header for generated file.")
+	flag.Usage = usageFlag
+}
+
+func main() {
+	log.SetFlags(0)
+	flag.Parse()
+
+	if beatName == "" {
+		log.Fatal("You must use -beat to specify the beat name.")
+	}
+
+	license, err := licenses.Find(license)
+	if err != nil {
+		log.Fatalf("Invalid license specifier: %v", err)
+	}
+
+	args := flag.Args()
+	if len(args) != 1 {
+		log.Fatal("module-dir must be passed as an argument.")
+	}
+	dir := args[0]
+
+	modules, err := fields.GetModules(dir)
+	if err != nil {
+		log.Fatalf("Error fetching modules: %v", err)
+	}
+
+	for _, module := range modules {
+		files, err := fields.CollectFiles(module, dir)
+		if err != nil {
+			log.Fatalf("Error fetching files for module %v: %v", module, err)
+		}
+
+		data, err := fields.GenerateFieldsYml(files)
+		if err != nil {
+			log.Fatalf("Error fetching files for module %v: %v", module, err)
+		}
+
+		encData, err := asset.EncodeData(string(data))
+		if err != nil {
+			log.Fatalf("Error encoding the data: %v", err)
+		}
+
+		var buf bytes.Buffer
+		asset.Template.Execute(&buf, asset.Data{
+			License: license,
+			Beat:    beatName,
+			Name:    module,
+			Data:    encData,
+			Package: module,
+		})
+
+		bs, err := format.Source(buf.Bytes())
+		if err != nil {
+			log.Fatalf("Error creating golang file from template: %v", err)
+		}
+
+		err = ioutil.WriteFile(path.Join(dir, module, "fields.go"), bs, 0644)
+		if err != nil {
+			log.Fatalf("Error writing fields.go: %v", err)
+		}
+	}
+}
+
+func usageFlag() {
+	fmt.Fprintf(os.Stderr, usageText)
+	flag.PrintDefaults()
+}

_beats/dev-tools/generate_notice.py

@@ -7,6 +7,7 @@
 import json
 import csv
 import re
+import pdb
 import copy
 
 
@@ -87,6 +88,8 @@ def gather_dependencies(vendor_dirs, overrides=None):
 
 # Allow to skip files that could match the `LICENSE` pattern but does not have any license information.
 SKIP_FILES = [
+    # AWS lambda go defines that some part of the code is APLv2 and other on a MIT Modified license.
+    "./vendor/github.com/aws/aws-lambda-go/LICENSE-SUMMARY"
 ]
 
 

_beats/dev-tools/packaging/packages.yml

@@ -358,7 +358,7 @@ specs:
         <<: *binary_spec
         <<: *elastic_license_for_binaries
         files:
-          /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}:
+          '{{.BeatName}}{{.BinaryExt}}':
             source: ../x-pack/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
 
     - os: darwin

_beats/dev-tools/packaging/templates/windows/install-service.ps1.tmpl

@@ -12,3 +12,9 @@ $workdir = Split-Path $MyInvocation.MyCommand.Path
 New-Service -name {{.BeatName}} `
   -displayName {{.BeatName | title}} `
   -binaryPathName "`"$workdir\{{.BeatName}}.exe`" -c `"$workdir\{{.BeatName}}.yml`" -path.home `"$workdir`" -path.data `"C:\ProgramData\{{.BeatName}}`" -path.logs `"C:\ProgramData\{{.BeatName}}\logs`""
+
+# Attempt to set the service to delayed start using sc config.
+Try {
+  Start-Process -FilePath sc.exe -ArgumentList 'config {{.BeatName}} start=delayed-auto'
+}
+Catch { Write-Host "An error occured setting the service to delayed start." -ForegroundColor Red }

_beats/libbeat/scripts/Makefile

@@ -48,7 +48,7 @@ TESTIFY_TOOL_REPO?=github.com/elastic/beats/vendor/github.com/stretchr/testify/a
 NOW=$(shell date -u '+%Y-%m-%dT%H:%M:%SZ')
 GOBUILD_FLAGS?=-i -ldflags "-X github.com/elastic/beats/libbeat/version.buildTime=$(NOW) -X github.com/elastic/beats/libbeat/version.commit=$(COMMIT_ID)"
 GOIMPORTS=goimports
-GOIMPORTS_REPO?=golang.org/x/tools/cmd/goimports
+GOIMPORTS_REPO?=github.com/elastic/beats/vendor/golang.org/x/tools/cmd/goimports
 GOIMPORTS_LOCAL_PREFIX?=github.com/elastic
 GOLINT=golint
 GOLINT_REPO?=github.com/golang/lint/golint
@@ -82,6 +82,7 @@ VIRTUALENV_PARAMS?=
 INTEGRATION_TESTS?=
 FIND=. ${PYTHON_ENV}/bin/activate; find . -type f -not -path "*/vendor/*" -not -path "*/build/*" -not -path "*/.git/*"
 PERM_EXEC?=$(shell [ `uname -s` = "Darwin" ] && echo "+111" || echo "/a+x")
+XPACK_ONLY?=false
 
 ifeq ($(DOCKER_CACHE),0)
 	DOCKER_NOCACHE=--no-cache
@@ -106,9 +107,11 @@ ${BEAT_NAME}.test: $(GOFILES_ALL)
 .PHONY: crosscompile
 crosscompile: ## @build Cross-compile beat for the OS'es specified in GOX_OS variable. The binaries are placed in the build/bin directory.
 crosscompile: $(GOFILES)
+ifneq ($(shell [[ $(BEAT_NAME) == journalbeat ]] && echo true ),true)
 	go get github.com/mitchellh/gox
 	mkdir -p ${BUILD_DIR}/bin
 	gox -output="${BUILD_DIR}/bin/{{.Dir}}-{{.OS}}-{{.Arch}}" -os="$(strip $(GOX_OS))" -osarch="$(strip $(GOX_OSARCH))" ${GOX_FLAGS}
+endif
 
 .PHONY: check
 check: check-headers python-env prepare-tests ## @build Checks project and source code if everything is according to standard
@@ -208,6 +211,13 @@ system-tests: prepare-tests ${BEAT_NAME}.test python-env
 system-tests-environment:  ## @testing Runs the system tests inside a virtual environment. This can be run on any docker-machine (local, remote)
 system-tests-environment: prepare-tests build-image
 	${DOCKER_COMPOSE} run -e INTEGRATION_TESTS=1 -e TESTING_ENVIRONMENT=${TESTING_ENVIRONMENT} -e DOCKER_COMPOSE_PROJECT_NAME=${DOCKER_COMPOSE_PROJECT_NAME} beat make system-tests
+	#This is a hack to run x-pack/filebeat module tests
+	@XPACKBEAT="${ES_BEATS}/x-pack/${BEAT_NAME}" ; \
+	if [ -e "$$XPACKBEAT/tests/system" ] && [ $(XPACK_ONLY) = false ]; then \
+	    $(MAKE) -C ../x-pack/${BEAT_NAME} fields; \
+	    ${DOCKER_COMPOSE} run -e INTEGRATION_TESTS=1 -e MODULES_PATH="../../x-pack/${BEAT_NAME}/module" -e TESTING_ENVIRONMENT=${TESTING_ENVIRONMENT} -e DOCKER_COMPOSE_PROJECT_NAME=${DOCKER_COMPOSE_PROJECT_NAME} beat make -C "$$XPACKBEAT" ${BEAT_NAME}.test system-tests ; \
+	    $(MAKE) -C ../x-pack/${BEAT_NAME} fix-permissions; \
+	fi
 
 
 .PHONY: fast-system-tests
@@ -330,7 +340,7 @@ endif
 
 ifneq ($(shell [[ $(BEAT_NAME) == libbeat || $(BEAT_NAME) == metricbeat ]] && echo true ),true)
 	mkdir -p include
-	go run  ${ES_BEATS}/dev-tools/cmd/asset/asset.go -pkg include -in fields.yml -out include/fields.go $(BEAT_NAME)
+	go run  ${ES_BEATS}/dev-tools/cmd/asset/asset.go -license $(LICENSE) -pkg include -in fields.yml -out include/fields.go $(BEAT_NAME)
 endif
 
 ifneq ($(shell [[ $(BEAT_NAME) == libbeat ]] && echo true ),true)
@@ -348,7 +358,9 @@ endif
 	@python ${ES_BEATS}/libbeat/scripts/unpack_dashboards.py --glob="./_meta/kibana.generated/6/dashboard/*.json"
 	@mkdir -p $(PWD)/_meta/kibana.generated/5/index-pattern
 	@mkdir -p $(PWD)/_meta/kibana.generated/6/index-pattern
-	@go run ${ES_BEATS}/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go -index '${BEAT_INDEX_PREFIX}-*' -beat-name ${BEAT_NAME} -beat-dir $(PWD) -version ${BEAT_VERSION}
+	@go run ${ES_BEATS}/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go \
+	  -index '${BEAT_INDEX_PREFIX}-*' -beat ${BEAT_NAME} -fields $(PWD)/fields.yml \
+	  -version ${BEAT_VERSION} -out _meta/kibana.generated
 
 .PHONY: docs
 docs:  ## @build Builds the documents for the beat

_beats/libbeat/scripts/cmd/global_fields/main.go

@@ -65,6 +65,8 @@ func main() {
 		fmt.Fprintf(os.Stderr, "Error getting file info of target Beat: %+v\n", err)
 		os.Exit(1)
 	}
+	beat.Close()
+	esBeats.Close()
 
 	// If a community Beat does not have its own fields.yml file, it still requires
 	// the fields coming from libbeat to generate e.g assets. In case of Elastic Beats,
@@ -78,9 +80,7 @@ func main() {
 
 	var fieldsFiles []*fields.YmlFile
 	for _, fieldsFilePath := range beatFieldsPaths {
-		pathToModules := filepath.Join(beatPath, fieldsFilePath)
-
-		fieldsFile, err := fields.CollectModuleFiles(pathToModules)
+		fieldsFile, err := fields.CollectModuleFiles(fieldsFilePath)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Cannot collect fields.yml files: %+v\n", err)
 			os.Exit(2)
@@ -95,5 +95,9 @@ func main() {
 		os.Exit(3)
 	}
 
-	fmt.Fprintf(os.Stderr, "Generated fields.yml for %s to %s\n", name, filepath.Join(beatPath, output))
+	outputPath, _ := filepath.Abs(output)
+	if err != nil {
+		outputPath = output
+	}
+	fmt.Fprintf(os.Stderr, "Generated fields.yml for %s to %s\n", name, outputPath)
 }