[Beats Management] add more tests, update types, break out ES into it's own adapter

x-pack/plugins/beats/common/domain_types.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { ConfigurationBlockTypes } from './constants';
+
+export interface ConfigurationBlock {
+  type: ConfigurationBlockTypes;
+  block_yml: string;
+}
+
+export interface CMBeat {
+  id: string;
+  access_token: string;
+  verified_on?: string;
+  type: string;
+  version?: string;
+  host_ip: string;
+  host_name: string;
+  ephemeral_id?: string;
+  local_configuration_yml?: string;
+  tags?: string[];
+  central_configuration_yml?: string;
+  metadata?: {};
+}
+
+export interface BeatTag {
+  id: string;
+  configuration_blocks: ConfigurationBlock[];
+}

x-pack/plugins/beats/index.ts

@@ -10,18 +10,20 @@ import { initServerWithKibana } from './server/kibana.index';
 
 const DEFAULT_ENROLLMENT_TOKENS_TTL_S = 10 * 60; // 10 minutes
 
+export const config = Joi.object({
+  enabled: Joi.boolean().default(true),
+  encryptionKey: Joi.string(),
+  enrollmentTokensTtlInSeconds: Joi.number()
+    .integer()
+    .min(1)
+    .default(DEFAULT_ENROLLMENT_TOKENS_TTL_S),
+}).default();
+export const configPrefix = 'xpack.beats';
+
 export function beats(kibana: any) {
   return new kibana.Plugin({
-    config: () =>
-      Joi.object({
-        enabled: Joi.boolean().default(true),
-        encryptionKey: Joi.string(),
-        enrollmentTokensTtlInSeconds: Joi.number()
-          .integer()
-          .min(1)
-          .default(DEFAULT_ENROLLMENT_TOKENS_TTL_S),
-      }).default(),
-    configPrefix: 'xpack.beats',
+    config: () => config,
+    configPrefix,
     id: PLUGIN.ID,
     require: ['kibana', 'elasticsearch', 'xpack_main'],
     init(server: any) {

x-pack/plugins/beats/readme.md

@@ -1,7 +1,15 @@
 # Documentation for Beats CM in x-pack kibana
 
-### Run tests
+### Run tests (from x-pack dir)
+
+Functional tests
 
 ```
 node scripts/jest.js plugins/beats --watch
 ```
+
+Functional API tests
+
+```
+node scripts/functional_tests  --config test/api_integration/config
+```

x-pack/plugins/beats/server/lib/adapters/beats/adapter_types.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { CMBeat } from '../../../../common/domain_types';
+import { FrameworkRequest } from '../famework/adapter_types';
+
+// FIXME: fix getBeatsWithIds return type
+export interface CMBeatsAdapter {
+  insert(beat: CMBeat): Promise<void>;
+  update(beat: CMBeat): Promise<void>;
+  get(id: string): any;
+  getAll(req: FrameworkRequest): any;
+  getWithIds(req: FrameworkRequest, beatIds: string[]): any;
+  verifyBeats(req: FrameworkRequest, beatIds: string[]): any;
+  removeTagsFromBeats(
+    req: FrameworkRequest,
+    removals: BeatsTagAssignment[]
+  ): Promise<BeatsTagAssignment[]>;
+  assignTagsToBeats(
+    req: FrameworkRequest,
+    assignments: BeatsTagAssignment[]
+  ): Promise<BeatsTagAssignment[]>;
+}
+
+export interface BeatsTagAssignment {
+  beatId: string;
+  tag: string;
+  idxInRequest?: number;
+}
+
+interface BeatsReturnedTagAssignment {
+  status: number | null;
+  result?: string;
+}
+
+export interface CMAssignmentReturn {
+  assignments: BeatsReturnedTagAssignment[];
+}
+
+export interface BeatsRemovalReturn {
+  removals: BeatsReturnedTagAssignment[];
+}

x-pack/plugins/beats/server/lib/adapters/beats/elasticsearch_beats_adapter.ts

@@ -7,19 +7,16 @@
 import { flatten, get as _get, omit } from 'lodash';
 import moment from 'moment';
 import { INDEX_NAMES } from '../../../../common/constants';
-import {
-  BackendFrameworkAdapter,
-  CMBeat,
-  CMBeatsAdapter,
-  CMTagAssignment,
-  FrameworkRequest,
-} from '../../lib';
+import { CMBeat } from '../../../../common/domain_types';
+import { DatabaseAdapter } from '../database/adapter_types';
+import { FrameworkRequest } from '../famework/adapter_types';
+import { BeatsTagAssignment, CMBeatsAdapter } from './adapter_types';
 
 export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
-  private framework: BackendFrameworkAdapter;
+  private database: DatabaseAdapter;
 
-  constructor(framework: BackendFrameworkAdapter) {
-    this.framework = framework;
+  constructor(database: DatabaseAdapter) {
+    this.database = database;
   }
 
   public async get(id: string) {
@@ -30,7 +27,7 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
       type: '_doc',
     };
 
-    const response = await this.framework.callWithInternalUser('get', params);
+    const response = await this.database.get(null, params);
     if (!response.found) {
       return null;
     }
@@ -44,14 +41,13 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
       type: 'beat',
     };
 
-    const params = {
+    await this.database.create(null, {
       body,
       id: `beat:${beat.id}`,
       index: INDEX_NAMES.BEATS,
       refresh: 'wait_for',
       type: '_doc',
-    };
-    await this.framework.callWithInternalUser('create', params);
+    });
   }
 
   public async update(beat: CMBeat) {
@@ -67,7 +63,7 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
       refresh: 'wait_for',
       type: '_doc',
     };
-    return await this.framework.callWithInternalUser('index', params);
+    await this.database.index(null, params);
   }
 
   public async getWithIds(req: FrameworkRequest, beatIds: string[]) {
@@ -81,9 +77,9 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
       index: INDEX_NAMES.BEATS,
       type: '_doc',
     };
-    const response = await this.framework.callWithRequest(req, 'mget', params);
+    const response = await this.database.mget(req, params);
 
-    return get(response, 'docs', [])
+    return _get(response, 'docs', [])
       .filter((b: any) => b.found)
       .map((b: any) => b._source.beat);
   }
@@ -101,15 +97,13 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
       ])
     );
 
-    const params = {
+    const response = await this.database.bulk(req, {
       _sourceInclude: ['beat.id', 'beat.verified_on'],
       body,
       index: INDEX_NAMES.BEATS,
       refresh: 'wait_for',
       type: '_doc',
-    };
-
-    const response = await this.framework.callWithRequest(req, 'bulk', params);
+    });
 
     return _get(response, 'items', []).map(b => ({
       ..._get(b, 'update.get._source.beat', {}),
@@ -123,20 +117,16 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
       q: 'type:beat',
       type: '_doc',
     };
-    const response = await this.framework.callWithRequest(
-      req,
-      'search',
-      params
-    );
+    const response = await this.database.search(req, params);
 
-    const beats = get<any>(response, 'hits.hits', []);
+    const beats = _get<any>(response, 'hits.hits', []);
     return beats.map((beat: any) => omit(beat._source.beat, ['access_token']));
   }
 
   public async removeTagsFromBeats(
     req: FrameworkRequest,
-    removals: CMTagAssignment[]
-  ): Promise<CMTagAssignment[]> {
+    removals: BeatsTagAssignment[]
+  ): Promise<BeatsTagAssignment[]> {
     const body = flatten(
       removals.map(({ beatId, tag }) => {
         const script =
@@ -153,15 +143,13 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
       })
     );
 
-    const params = {
+    const response = await this.database.bulk(req, {
       body,
       index: INDEX_NAMES.BEATS,
       refresh: 'wait_for',
       type: '_doc',
-    };
-
-    const response = await this.framework.callWithRequest(req, 'bulk', params);
-    return get<any>(response, 'items', []).map(
+    });
+    return _get<any>(response, 'items', []).map(
       (item: any, resultIdx: number) => ({
         idxInRequest: removals[resultIdx].idxInRequest,
         result: item.update.result,
@@ -172,8 +160,8 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
 
   public async assignTagsToBeats(
     req: FrameworkRequest,
-    assignments: CMTagAssignment[]
-  ): Promise<CMTagAssignment[]> {
+    assignments: BeatsTagAssignment[]
+  ): Promise<BeatsTagAssignment[]> {
     const body = flatten(
       assignments.map(({ beatId, tag }) => {
         const script =
@@ -193,18 +181,18 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
       })
     );
 
-    const params = {
+    const response = await this.database.bulk(req, {
       body,
       index: INDEX_NAMES.BEATS,
       refresh: 'wait_for',
       type: '_doc',
-    };
-
-    const response = await this.framework.callWithRequest(req, 'bulk', params);
-    return get<any>(response, 'items', []).map((item: any, resultIdx: any) => ({
-      idxInRequest: assignments[resultIdx].idxInRequest,
-      result: item.update.result,
-      status: item.update.status,
-    }));
+    });
+    return _get<any>(response, 'items', []).map(
+      (item: any, resultIdx: any) => ({
+        idxInRequest: assignments[resultIdx].idxInRequest,
+        result: item.update.result,
+        status: item.update.status,
+      })
+    );
   }
 }

x-pack/plugins/beats/server/lib/adapters/beats/memory_beats_adapter.ts

@@ -7,12 +7,9 @@
 import { omit } from 'lodash';
 import moment from 'moment';
 
-import {
-  CMBeat,
-  CMBeatsAdapter,
-  CMTagAssignment,
-  FrameworkRequest,
-} from '../../lib';
+import { CMBeat } from '../../../../common/domain_types';
+import { FrameworkRequest } from '../famework/adapter_types';
+import { BeatsTagAssignment, CMBeatsAdapter } from './adapter_types';
 
 export class MemoryBeatsAdapter implements CMBeatsAdapter {
   private beatsDB: CMBeat[];
@@ -64,8 +61,8 @@ export class MemoryBeatsAdapter implements CMBeatsAdapter {
 
   public async removeTagsFromBeats(
     req: FrameworkRequest,
-    removals: CMTagAssignment[]
-  ): Promise<CMTagAssignment[]> {
+    removals: BeatsTagAssignment[]
+  ): Promise<BeatsTagAssignment[]> {
     const beatIds = removals.map(r => r.beatId);
 
     const response = this.beatsDB
@@ -89,15 +86,15 @@ export class MemoryBeatsAdapter implements CMBeatsAdapter {
 
   public async assignTagsToBeats(
     req: FrameworkRequest,
-    assignments: CMTagAssignment[]
-  ): Promise<CMTagAssignment[]> {
+    assignments: BeatsTagAssignment[]
+  ): Promise<BeatsTagAssignment[]> {
     const beatIds = assignments.map(r => r.beatId);
 
     this.beatsDB.filter(beat => beatIds.includes(beat.id)).map(beat => {
       // get tags that need to be assigned to this beat
       const tags = assignments
         .filter(a => a.beatId === beat.id)
-        .map((t: CMTagAssignment) => t.tag);
+        .map((t: BeatsTagAssignment) => t.tag);
 
       if (tags.length > 0) {
         if (!beat.tags) {
@@ -114,10 +111,12 @@ export class MemoryBeatsAdapter implements CMBeatsAdapter {
       return beat;
     });
 
-    return assignments.map<any>((item: CMTagAssignment, resultIdx: number) => ({
-      idxInRequest: assignments[resultIdx].idxInRequest,
-      result: 'updated',
-      status: 200,
-    }));
+    return assignments.map<any>(
+      (item: BeatsTagAssignment, resultIdx: number) => ({
+        idxInRequest: assignments[resultIdx].idxInRequest,
+        result: 'updated',
+        status: 200,
+      })
+    );
   }
 }

x-pack/plugins/beats/server/lib/adapters/database/__tests__/kibana.test.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+// file.skip
+
+// @ts-ignore
+import { createEsTestCluster } from '@kbn/test';
+// @ts-ignore
+import * as kbnTestServer from '../../../../../../../../src/test_utils/kbn_server';
+import { DatabaseKbnESPlugin } from '../adapter_types';
+import { KibanaDatabaseAdapter } from '../kibana_database_adapter';
+import { contractTests } from './test_contract';
+
+const kbnServer = kbnTestServer.createServerWithCorePlugins();
+const es = createEsTestCluster({});
+
+contractTests('Kibana Database Adapter', {
+  before: async () => {
+    await es.start();
+    await kbnServer.ready();
+
+    return await kbnServer.server.plugins.elasticsearch.waitUntilReady();
+  },
+  after: async () => {
+    await kbnServer.close();
+    return await es.cleanup();
+  },
+  adapterSetup: () => {
+    return new KibanaDatabaseAdapter(kbnServer.server.plugins
+      .elasticsearch as DatabaseKbnESPlugin);
+  },
+});