chore: project structure refactoring

przemyslawwalczak · przemyslawwalczak · commit 699e60138d0c · 2023-10-05T19:06:17.000+02:00
diff --git a/docs/decorating-document-catalog.md b/docs/decorating-document-catalog.md
@@ -0,0 +1,10 @@
+```typescript
+import { IsString } from 'class-validator'
+import { Catalog } from 'nestjs/decorators'
+
+@Catalog('example-catalog-name')
+export class ExampleDocument {
+    @IsString()
+    readonly field: string
+}
+```
diff --git a/docs/registering-elasticsearch-module.md b/docs/registering-elasticsearch-module.md
@@ -1,5 +1,7 @@
+
+```typescript
 import { Module } from '@nestjs/common'
-import { ElasticsearchModule } from '..'
+import { ElasticsearchModule } from '../src/nestjs'
 import { ExampleDocument } from './document'
 
 @Module({
@@ -19,4 +21,5 @@ import { ExampleDocument } from './document'
         ])
     ]
 })
-export class AppModule {}
+export class AppModule {}
+```
diff --git a/src/lib/aggregations/get-avg.spec.ts b/src/lib/aggregations/get-avg.spec.ts
@@ -1,10 +1,10 @@
 import { describe } from 'node:test'
-import { ExampleDocument } from 'nestjs/examples'
-import { getAvgAggregation } from './get-avg.aggregation'
+import { ExampleCatalogDocument } from 'test/module'
+import { getAvgAggregation } from './get-avg'
 
 describe('getAvgAggregation', () => {
     it('accepts only schema fields', () => {
-        const query = getAvgAggregation<ExampleDocument>('field')
+        const query = getAvgAggregation<ExampleCatalogDocument>('field')
 
         expect(query).toEqual({
             avg: {
@@ -14,7 +14,7 @@ describe('getAvgAggregation', () => {
     })
 
     it('accepts only schema fields with keyword', () => {
-        const query = getAvgAggregation<ExampleDocument>('field.keyword')
+        const query = getAvgAggregation<ExampleCatalogDocument>('field.keyword')
 
         expect(query).toEqual({
             avg: {
diff --git a/src/lib/aggregations/get-avg.ts b/src/lib/aggregations/get-avg.ts
diff --git a/src/lib/aggregations/get-sum.spec.ts b/src/lib/aggregations/get-sum.spec.ts
@@ -1,10 +1,10 @@
 import { describe } from 'node:test'
-import { ExampleDocument } from 'nestjs/examples'
-import { getSumAggregation } from './get-sum.aggregation'
+import { ExampleCatalogDocument } from 'test/module'
+import { getSumAggregation } from './get-sum'
 
 describe('getSumAggregation', () => {
     it('accepts only schema fields', () => {
-        const query = getSumAggregation<ExampleDocument>('field')
+        const query = getSumAggregation<ExampleCatalogDocument>('field')
 
         expect(query).toEqual({
             sum: {
@@ -14,7 +14,7 @@ describe('getSumAggregation', () => {
     })
 
     it('accepts only schema fields with keyword', () => {
-        const query = getSumAggregation<ExampleDocument>('field.keyword')
+        const query = getSumAggregation<ExampleCatalogDocument>('field.keyword')
 
         expect(query).toEqual({
             sum: {
diff --git a/src/lib/aggregations/get-sum.ts b/src/lib/aggregations/get-sum.ts
diff --git a/src/lib/aggregations/index.ts b/src/lib/aggregations/index.ts
@@ -1,2 +1,2 @@
-export * from './get-sum.aggregation'
-export * from './get-avg.aggregation'
+export * from './get-sum'
+export * from './get-avg'
diff --git a/src/lib/builders/index.ts b/src/lib/builders/index.ts
@@ -0,0 +1 @@
+export * from './query-builder'
diff --git a/src/lib/builders/query-builder.ts b/src/lib/builders/query-builder.ts
@@ -0,0 +1,8 @@
+import { Document } from 'lib/types'
+import { Catalog } from 'nestjs/injectables'
+
+export class QueryBuilder<TDocument extends Document> {
+    constructor(
+        private readonly catalog: Catalog<TDocument>
+    ) {}
+}
diff --git a/src/lib/elasticsearch/index.ts b/src/lib/elasticsearch/index.ts
@@ -0,0 +1,2 @@
+export * from './request'
+export * from './response'
diff --git a/src/lib/elasticsearch/request.ts b/src/lib/elasticsearch/request.ts
@@ -0,0 +1 @@
+export const getSearchRequest = () => {}
diff --git a/src/lib/elasticsearch/response.ts b/src/lib/elasticsearch/response.ts
@@ -0,0 +1 @@
+export const getSearchResponse = () => {}
diff --git a/src/lib/queries/get-bool-must.ts b/src/lib/queries/get-bool-must.ts
@@ -0,0 +1,10 @@
+import { Document } from 'lib/types'
+import { getTermQuery } from './get-term'
+import { getTermsQuery } from './get-terms'
+
+export type BoolMustQuery<TDocument extends Document> = {
+    term?: ReturnType<typeof getTermQuery<TDocument>>
+    terms?: ReturnType<typeof getTermsQuery<TDocument>>
+}
+
+export const getBoolMustQuery = <TDocument extends Document>(query: BoolMustQuery<TDocument> | Array<BoolMustQuery<TDocument>>) => query
diff --git a/src/lib/queries/get-bool-should.ts b/src/lib/queries/get-bool-should.ts
@@ -0,0 +1,10 @@
+import { Document } from 'lib/types'
+import { getTermQuery } from './get-term'
+import { getTermsQuery } from './get-terms'
+
+export type BoolShouldQuery<TDocument extends Document> = {
+    term?: ReturnType<typeof getTermQuery<TDocument>>
+    terms?: ReturnType<typeof getTermsQuery<TDocument>>
+}
+
+export const getBoolShouldQuery = <TDocument extends Document>(query: BoolShouldQuery<TDocument> | Array<BoolShouldQuery<TDocument>>) => query
diff --git a/src/lib/queries/get-bool.ts b/src/lib/queries/get-bool.ts
@@ -0,0 +1,12 @@
+import { Document } from 'lib/types'
+import { getBoolMustQuery } from './get-bool-must'
+import { getBoolShouldQuery } from './get-bool-should'
+
+export type BoolQuery<TDocument extends Document> = {
+    must?: ReturnType<typeof getBoolMustQuery<TDocument>>
+    should?: ReturnType<typeof getBoolShouldQuery<TDocument>>
+}
+
+export const getBoolQuery = <TDocument extends Document>(bool: BoolQuery<TDocument>) => ({
+    bool
+})
diff --git a/src/lib/queries/get-term.spec.ts b/src/lib/queries/get-term.spec.ts
@@ -1,10 +1,10 @@
 import { describe } from 'node:test'
-import { ExampleDocument } from 'nestjs/examples'
-import { getTermQuery } from './get-term.query'
+import { getTermQuery } from './get-term'
+import { ExampleCatalogDocument } from 'test/module'
 
 describe('getTermQuery', () => {
     it('accepts only schema fields', () => {
-        const query = getTermQuery<ExampleDocument>('field', 'test')
+        const query = getTermQuery<ExampleCatalogDocument>('field', 'test')
 
         expect(query).toEqual({
             term: {
@@ -16,7 +16,7 @@ describe('getTermQuery', () => {
     })
 
     it('accepts only schema fields with keyword', () => {
-        const query = getTermQuery<ExampleDocument>('field.keyword', 'test')
+        const query = getTermQuery<ExampleCatalogDocument>('field.keyword', 'test')
 
         expect(query).toEqual({
             term: {
diff --git a/src/lib/queries/get-term.ts b/src/lib/queries/get-term.ts
@@ -1,5 +1,11 @@
 import { Document, FieldKeyword, FieldType } from 'lib/types'
 
+/**
+ *
+ * @param field
+ * @param value
+ * @returns
+ */
 export const getTermQuery = <TDocument extends Document>(field: FieldKeyword<TDocument>, value: FieldType<TDocument, typeof field>) => ({
     term: {
         [field]: { value }
diff --git a/src/lib/queries/get-terms.spec.ts b/src/lib/queries/get-terms.spec.ts
@@ -1,10 +1,10 @@
 import { describe } from 'node:test'
-import { ExampleDocument } from 'nestjs/examples'
-import { getTermsQuery } from './get-terms.query'
+import { getTermsQuery } from './get-terms'
+import { ExampleCatalogDocument } from 'test/module'
 
 describe('getTermsQuery', () => {
     it('accepts only schema fields', () => {
-        const query = getTermsQuery<ExampleDocument>('field', ['one', 'two', 'three'])
+        const query = getTermsQuery<ExampleCatalogDocument>('field', ['one', 'two', 'three'])
 
         expect(query).toEqual({
             terms: {
@@ -14,7 +14,7 @@ describe('getTermsQuery', () => {
     })
 
     it('accepts only schema field with keyword', () => {
-        const query = getTermsQuery<ExampleDocument>('field.keyword', ['one', 'two', 'three'])
+        const query = getTermsQuery<ExampleCatalogDocument>('field.keyword', ['one', 'two', 'three'])
 
         expect(query).toEqual({
             terms: {
diff --git a/src/lib/queries/get-terms.ts b/src/lib/queries/get-terms.ts
@@ -1,5 +1,11 @@
 import { Document, FieldKeyword, FieldType } from 'lib/types'
 
+/**
+ *
+ * @param field
+ * @param values
+ * @returns
+ */
 export const getTermsQuery = <TDocument extends Document>(field: FieldKeyword<TDocument>, values: Array<FieldType<TDocument, typeof field>>) => ({
     terms: {
         [field]: values
diff --git a/src/lib/queries/index.ts b/src/lib/queries/index.ts
@@ -1 +1 @@
-export * from './get-term.query'
+export * from './get-term'
diff --git a/src/lib/types/elasticsearch.ts b/src/lib/types/elasticsearch.ts
@@ -1,10 +1,19 @@
 export type ElasticsearchCatalog = {
+    /**
+     *
+     * @returns void
+     */
     search: () => void
-    msearch: () => void
 
     /**
      *
      * @returns void
      */
     createQueryBuilder: () => void
+
+    /**
+     *
+     * @returns void
+     */
+    checkDocumentIntegrity: () => void
 }
diff --git a/src/nestjs/elasticsearch.module.ts b/src/nestjs/elasticsearch.module.ts
@@ -1,9 +1,10 @@
 import { ClientOptions } from '@elastic/elasticsearch'
-import { ElasticsearchModule as BaseElasticsearchModule, ElasticsearchService } from '@nestjs/elasticsearch'
+import { ElasticsearchModule as BaseElasticsearchModule, ElasticsearchService as ElasticsearchBaseService } from '@nestjs/elasticsearch'
 import { Module, DynamicModule, Provider } from '@nestjs/common'
 import { ClassConstructor } from 'lib/types'
 import { ELASTICSEARCH_CATALOG_NAME, ELASTICSEARCH_CATALOG_PREFIX } from 'lib/constants'
 import { Catalog } from './injectables'
+import { ElasticsearchService } from './elasticsearch.service'
 
 @Module({})
 export class ElasticsearchModule {
@@ -12,6 +13,7 @@ export class ElasticsearchModule {
             global: true,
             module: ElasticsearchModule,
             imports: [BaseElasticsearchModule.register(options)],
+            providers: [ElasticsearchService],
             exports: [ElasticsearchService]
         }
     }
@@ -26,9 +28,9 @@ export class ElasticsearchModule {
             }
 
             return {
-                inject: [ElasticsearchService],
+                inject: [ElasticsearchBaseService],
                 provide: `${ELASTICSEARCH_CATALOG_PREFIX}:${name}`,
-                useFactory: (es: ElasticsearchService) => new Catalog(document, es)
+                useFactory: (base: ElasticsearchBaseService) => new Catalog(base, document)
             }
         })
 
diff --git a/src/nestjs/elasticsearch.service.ts b/src/nestjs/elasticsearch.service.ts
@@ -0,0 +1,27 @@
+import { Injectable } from '@nestjs/common'
+import { ElasticsearchService as ElasticsearchBaseService } from '@nestjs/elasticsearch'
+import { ClassConstructor, Document } from 'lib/types'
+import { Catalog } from './injectables'
+
+@Injectable()
+export class ElasticsearchService {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    private readonly catalogs: Array<Catalog<any>> = []
+
+    constructor(private readonly elasticsearchBaseService: ElasticsearchBaseService) {}
+
+    checkCatalogsIntegrity() {
+        // todo: get the catalog schema and compare it with elasticsearch catalog field types
+        // note: ideal for situations where we want to run regression tests and ensure data integrity of the service and datasource
+
+        return Promise.allSettled(this.catalogs.map(catalog => catalog.checkDocumentIntegrity()))
+    }
+
+    withCatalog<TDocument extends Document>(catalogDocument: ClassConstructor<TDocument>) {
+        return new Catalog(this.elasticsearchBaseService, catalogDocument)
+    }
+
+    search() {
+        // todo: search spec
+    }
+}
diff --git a/src/nestjs/examples/document.ts b/src/nestjs/examples/document.ts
diff --git a/src/nestjs/examples/index.ts b/src/nestjs/examples/index.ts
diff --git a/src/nestjs/index.ts b/src/nestjs/index.ts
@@ -1 +1,2 @@
+export * from './elasticsearch.service'
 export * from './elasticsearch.module'
diff --git a/src/nestjs/injectables/catalog.injectable.ts b/src/nestjs/injectables/catalog.injectable.ts
@@ -1,17 +1,32 @@
 import { Injectable } from '@nestjs/common'
-import { ElasticsearchService as BaseElasticsearchService } from '@nestjs/elasticsearch'
+import { ElasticsearchService as ElasticsearchBaseService } from '@nestjs/elasticsearch'
 import { ClassConstructor, Document, ElasticsearchCatalog } from 'lib/types'
+import { QueryBuilder } from 'lib/builders'
 
 @Injectable()
 export class Catalog<TDocument extends Document> implements ElasticsearchCatalog {
     constructor(
-        private readonly document: ClassConstructor<TDocument>,
-        private readonly es: BaseElasticsearchService
+        private readonly elasticsearchBaseService: ElasticsearchBaseService,
+        private readonly document: ClassConstructor<TDocument>
     ) {}
 
     search() {}
 
-    msearch() {}
+    createQueryBuilder() {
+        return new QueryBuilder(this)
+    }
 
-    createQueryBuilder() {}
+    checkDocumentIntegrity() {
+        // todo: pull schema and example values from the index
+        // note: this can be potentially expensive operation, but we can make it cheap:
+        // - check if fields contains null values and check if document is marked to be nullable
+        // - better yet find documents with annomalies, that would cause the schema to break
+        // for example:
+        // check and pull all kind of fields that are available on the elasticsearch catalog
+        // select document fields and each value
+        // warn or error when field is either:
+        // 1. `null and not marked as optional in the schema`
+        // 2. `number/string/date/geopoint and not marked in the schema`
+        // 3. `check if document contains all fields on the source, or source contains all the fields in the document`
+    }
 }
diff --git a/src/test/data/elasticsearch-catalog.json b/src/test/data/elasticsearch-catalog.json
diff --git a/src/test/module/example.catalog.ts b/src/test/module/example.catalog.ts
@@ -0,0 +1,8 @@
+import { IsString } from 'class-validator'
+import { Catalog } from 'nestjs/decorators'
+
+@Catalog('example-catalog-name')
+export class ExampleCatalogDocument {
+    @IsString()
+    readonly field: string
+}
diff --git a/src/test/module/index.ts b/src/test/module/index.ts
@@ -0,0 +1 @@
+export * from './example.catalog'
diff --git a/tsconfig.build.json b/tsconfig.build.json
@@ -2,7 +2,11 @@
     "extends": "./tsconfig.json",
     "compilerOptions": {
         "baseUrl": ".",
-        "outDir": "./dist"
+        "outDir": "./dist",
+        "paths": {
+            "lib/*": ["./src/lib/*"],
+            "nestjs/*": ["./src/nestjs/*"]
+        }
     },
     "include": ["src"],
     "exclude": ["node_modules", "examples", "*.spec.ts"]
diff --git a/tsconfig.json b/tsconfig.json
@@ -14,15 +14,12 @@
         "outDir": "./dist",
         "baseUrl": ".",
         "paths": {
-            "lib/*": [
-                "./src/lib/*"
-            ],
-            "nestjs/*": [
-                "./src/nestjs/*"
-            ]
+            "lib/*": ["./src/lib/*"],
+            "nestjs/*": ["./src/nestjs/*"],
+            "test/*": ["./src/test/*"]
         }
     },
     "watchOptions": {
         "watchFile": "fixedpollinginterval"
     }
-}
+}

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+export * from './request'`
	`2`	`+export * from './response'`