[Beats Management] Move to Ingest UI arch and initial TS effort

package.json

@@ -323,7 +323,7 @@
     "tree-kill": "^1.1.0",
     "ts-jest": "^22.4.6",
     "ts-loader": "^3.5.0",
-    "ts-node": "^6.0.3",
+    "ts-node": "^6.1.1",
     "tslint": "^5.10.0",
     "tslint-config-prettier": "^1.12.0",
     "tslint-plugin-prettier": "^1.3.0",

tsconfig.json

@@ -1,6 +1,7 @@
 {
   "compilerOptions": {
     "baseUrl": ".",
+    "sourceMap": true,
     "paths": {
       "ui/*": ["src/ui/public/*"]
     },
@@ -41,7 +42,5 @@
     // Disallow inconsistently-cased references to the same file.
     "forceConsistentCasingInFileNames": true
   },
-  "include": [
-    "src/**/*"
-  ]
+  "include": ["src/**/*"]
 }

x-pack/package.json

@@ -27,7 +27,11 @@
     "@kbn/es": "link:../packages/kbn-es",
     "@kbn/plugin-helpers": "link:../packages/kbn-plugin-helpers",
     "@kbn/test": "link:../packages/kbn-test",
+    "@types/boom": "^4.3.8",
+    "@types/hapi": "15.0.1",
     "@types/jest": "^22.2.3",
+    "@types/joi": "^10.4.4",
+    "@types/lodash": "^4.14.107",
     "@types/pngjs": "^3.3.1",
     "abab": "^1.0.4",
     "ansicolors": "0.3.2",
@@ -85,6 +89,7 @@
     "@kbn/ui-framework": "link:../packages/kbn-ui-framework",
     "@samverschueren/stream-to-observable": "^0.3.0",
     "@slack/client": "^4.2.2",
+    "@types/uuid": "^3.4.3",
     "angular-paging": "2.2.1",
     "angular-resource": "1.4.9",
     "angular-sanitize": "1.4.9",

x-pack/plugins/beats/common/constants/configuration_blocks.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export enum ConfigurationBlockTypes {
+  FilebeatInputs = 'filebeat.inputs',
+  FilebeatModules = 'filebeat.modules',
+  MetricbeatModules = 'metricbeat.modules',
+  Output = 'output',
+  Processors = 'processors',
+}
+
+export const UNIQUENESS_ENFORCING_TYPES = [ConfigurationBlockTypes.Output];

x-pack/plugins/beats/common/constants/index.js → x-pack/plugins/beats/common/constants/index.ts

@@ -6,4 +6,7 @@
 
 export { PLUGIN } from './plugin';
 export { INDEX_NAMES } from './index_names';
-export { CONFIGURATION_BLOCKS } from './configuration_blocks';
+export {
+  UNIQUENESS_ENFORCING_TYPES,
+  ConfigurationBlockTypes,
+} from './configuration_blocks';

x-pack/plugins/beats/common/constants/index_names.js → x-pack/plugins/beats/common/constants/index_names.ts

@@ -5,5 +5,5 @@
  */
 
 export const INDEX_NAMES = {
-  BEATS: '.management-beats'
+  BEATS: '.management-beats',
 };

x-pack/plugins/beats/common/constants/plugin.js → x-pack/plugins/beats/common/constants/plugin.ts

@@ -5,5 +5,5 @@
  */
 
 export const PLUGIN = {
-  ID: 'beats'
+  ID: 'beats',
 };

x-pack/plugins/beats/index.js → x-pack/plugins/beats/index.ts

@@ -4,24 +4,26 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { installIndexTemplate } from './server/lib/index_template';
-import { registerApiRoutes } from './server/routes/api';
 import { PLUGIN } from './common/constants';
+import { initServerWithKibana } from './server/kibana.index';
 
 const DEFAULT_ENROLLMENT_TOKENS_TTL_S = 10 * 60; // 10 minutes
 
-export function beats(kibana)  {
+export function beats(kibana) {
   return new kibana.Plugin({
+    config: Joi =>
+      Joi.object({
+        enabled: Joi.boolean().default(true),
+        enrollmentTokensTtlInSeconds: Joi.number()
+          .integer()
+          .min(1)
+          .default(DEFAULT_ENROLLMENT_TOKENS_TTL_S),
+      }).default(),
+    configPrefix: 'xpack.beats',
     id: PLUGIN.ID,
     require: ['kibana', 'elasticsearch', 'xpack_main'],
-    configPrefix: 'xpack.beats',
-    config: Joi => Joi.object({
-      enabled: Joi.boolean().default(true),
-      enrollmentTokensTtlInSeconds: Joi.number().integer().min(1).default(DEFAULT_ENROLLMENT_TOKENS_TTL_S)
-    }).default(),
-    init: async function (server) {
-      await installIndexTemplate(server);
-      registerApiRoutes(server);
-    }
+    async init(server) {
+      initServerWithKibana(server);
+    },
   });
 }

x-pack/plugins/beats/server/kibana.index.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Server } from 'hapi';
+import { compose } from './lib/compose/kibana';
+import { initManagementServer } from './management_server';
+
+export const initServerWithKibana = (hapiServer: Server) => {
+  const libs = compose(hapiServer);
+  initManagementServer(libs);
+};

x-pack/plugins/beats/server/lib/adapters/beats/elasticsearch_beats_adapter.ts

@@ -0,0 +1,217 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { flatten, get, omit } from 'lodash';
+import moment from 'moment';
+import { INDEX_NAMES } from '../../../../common/constants';
+import {
+  BackendFrameworkAdapter,
+  CMBeat,
+  CMBeatsAdapter,
+  CMTagAssignment,
+  FrameworkRequest,
+} from '../../lib';
+
+export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
+  private framework: BackendFrameworkAdapter;
+
+  constructor(framework: BackendFrameworkAdapter) {
+    this.framework = framework;
+  }
+
+  public async get(id: string) {
+    const params = {
+      id: `beat:${id}`,
+      ignore: [404],
+      index: INDEX_NAMES.BEATS,
+      type: '_doc',
+    };
+
+    const response = await this.framework.callWithInternalUser('get', params);
+    if (!response.found) {
+      return null;
+    }
+
+    return get(response, '_source.beat');
+  }
+
+  public async insert(beat: CMBeat) {
+    const body = {
+      beat,
+      type: 'beat',
+    };
+
+    const params = {
+      body,
+      id: `beat:${beat.id}`,
+      index: INDEX_NAMES.BEATS,
+      refresh: 'wait_for',
+      type: '_doc',
+    };
+    await this.framework.callWithInternalUser('create', params);
+  }
+
+  public async update(beat: CMBeat) {
+    const body = {
+      beat,
+      type: 'beat',
+    };
+
+    const params = {
+      body,
+      id: `beat:${beat.id}`,
+      index: INDEX_NAMES.BEATS,
+      refresh: 'wait_for',
+      type: '_doc',
+    };
+    return await this.framework.callWithInternalUser('index', params);
+  }
+
+  public async getWithIds(req: FrameworkRequest, beatIds: string[]) {
+    const ids = beatIds.map(beatId => `beat:${beatId}`);
+
+    const params = {
+      _source: false,
+      body: {
+        ids,
+      },
+      index: INDEX_NAMES.BEATS,
+      type: '_doc',
+    };
+    const response = await this.framework.callWithRequest(req, 'mget', params);
+    return get(response, 'docs', []);
+  }
+
+  // TODO merge with getBeatsWithIds
+  public async getVerifiedWithIds(req: FrameworkRequest, beatIds: string[]) {
+    const ids = beatIds.map(beatId => `beat:${beatId}`);
+
+    const params = {
+      _sourceInclude: ['beat.id', 'beat.verified_on'],
+      body: {
+        ids,
+      },
+      index: INDEX_NAMES.BEATS,
+      type: '_doc',
+    };
+    const response = await this.framework.callWithRequest(req, 'mget', params);
+    return get(response, 'docs', []);
+  }
+
+  public async verifyBeats(req: FrameworkRequest, beatIds: string[]) {
+    if (!Array.isArray(beatIds) || beatIds.length === 0) {
+      return [];
+    }
+
+    const verifiedOn = moment().toJSON();
+    const body = flatten(
+      beatIds.map(beatId => [
+        { update: { _id: `beat:${beatId}` } },
+        { doc: { beat: { verified_on: verifiedOn } } },
+      ])
+    );
+
+    const params = {
+      body,
+      index: INDEX_NAMES.BEATS,
+      refresh: 'wait_for',
+      type: '_doc',
+    };
+
+    const response = await this.framework.callWithRequest(req, 'bulk', params);
+    return get(response, 'items', []);
+  }
+
+  public async getAll(req: FrameworkRequest) {
+    const params = {
+      index: INDEX_NAMES.BEATS,
+      q: 'type:beat',
+      type: '_doc',
+    };
+    const response = await this.framework.callWithRequest(
+      req,
+      'search',
+      params
+    );
+
+    const beats = get(response, 'hits.hits', []);
+    return beats.map(beat => omit(beat._source.beat, ['access_token']));
+  }
+
+  public async removeTagsFromBeats(
+    req: FrameworkRequest,
+    removals: CMTagAssignment[]
+  ): Promise<CMTagAssignment[]> {
+    const body = flatten(
+      removals.map(({ beatId, tag }) => {
+        const script =
+          '' +
+          'def beat = ctx._source.beat; ' +
+          'if (beat.tags != null) { ' +
+          '  beat.tags.removeAll([params.tag]); ' +
+          '}';
+
+        return [
+          { update: { _id: `beat:${beatId}` } },
+          { script: { source: script, params: { tag } } },
+        ];
+      })
+    );
+
+    const params = {
+      body,
+      index: INDEX_NAMES.BEATS,
+      refresh: 'wait_for',
+      type: '_doc',
+    };
+
+    const response = await this.framework.callWithRequest(req, 'bulk', params);
+    return get(response, 'items', []).map((item, resultIdx) => ({
+      idxInRequest: removals[resultIdx].idxInRequest,
+      result: item.update.result,
+      status: item.update.status,
+    }));
+  }
+
+  // formerly persistAssignments
+  public async assignTagsToBeats(
+    req: FrameworkRequest,
+    assignments: CMTagAssignment[]
+  ): Promise<CMTagAssignment[]> {
+    const body = flatten(
+      assignments.map(({ beatId, tag }) => {
+        const script =
+          '' +
+          'def beat = ctx._source.beat; ' +
+          'if (beat.tags == null) { ' +
+          '  beat.tags = []; ' +
+          '} ' +
+          'if (!beat.tags.contains(params.tag)) { ' +
+          '  beat.tags.add(params.tag); ' +
+          '}';
+
+        return [
+          { update: { _id: `beat:${beatId}` } },
+          { script: { source: script, params: { tag } } },
+        ];
+      })
+    );
+
+    const params = {
+      body,
+      index: INDEX_NAMES.BEATS,
+      refresh: 'wait_for',
+      type: '_doc',
+    };
+
+    const response = await this.framework.callWithRequest(req, 'bulk', params);
+    return get(response, 'items', []).map((item, resultIdx) => ({
+      idxInRequest: assignments[resultIdx].idxInRequest,
+      result: item.update.result,
+      status: item.update.status,
+    }));
+  }
+}