feat(NODE-3589): support dot-notation attributes in Filter (#2972)

Author: avaly

.evergreen/run-checks.sh

@@ -20,12 +20,16 @@ npm run check:lint
 
 npm run check:unit
 
-echo "Typescript $(npx tsc -v)"
+export TSC="./node_modules/typescript/bin/tsc"
+
+echo "Typescript $($TSC -v)"
 # check resolution uses the default latest types
-echo "import * as mdb from '.'" > file.ts && npx tsc --noEmit --traceResolution file.ts | grep 'mongodb.d.ts' && rm file.ts
+echo "import * as mdb from '.'" > file.ts && $TSC --noEmit --traceResolution file.ts | grep 'mongodb.d.ts' && rm file.ts
 
-npm i --no-save typescript@4.0.2 # there is no 4.0.0
-echo "Typescript $(npx tsc -v)"
-npx tsc --noEmit mongodb.ts34.d.ts
+npm i --no-save typescript@4.1.6
+echo "Typescript $($TSC -v)"
+$TSC --noEmit mongodb.ts34.d.ts
 # check that resolution uses the downleveled types
-echo "import * as mdb from '.'" > file.ts && npx tsc --noEmit --traceResolution file.ts | grep 'mongodb.ts34.d.ts' && rm file.ts
+echo "import * as mdb from '.'" > file.ts && $TSC --noEmit --traceResolution file.ts | grep 'mongodb.ts34.d.ts' && rm file.ts
+
+rm -f file.ts

README.md

@@ -46,7 +46,7 @@ For version compatibility matrices, please refer to the following links:
 
 #### Typescript Version
 
-We recommend using the latest version of typescript, however we do provide a [downleveled](https://github.com/sandersn/downlevel-dts#readme) version of the type definitions that we test compiling against `typescript@4.0.2`.
+We recommend using the latest version of typescript, however we do provide a [downleveled](https://github.com/sandersn/downlevel-dts#readme) version of the type definitions that we test compiling against `typescript@4.1.6`.
 Since typescript [does not restrict breaking changes to major versions](https://github.com/Microsoft/TypeScript/wiki/Breaking-Changes) we consider this support best effort.
 If you run into any unexpected compiler failures please let us know and we will do our best to correct it.
 

src/index.ts

@@ -283,9 +283,11 @@ export type {
   InferIdType,
   IntegerType,
   IsAny,
+  Join,
   KeysOfAType,
   KeysOfOtherType,
   MatchKeysAndValues,
+  NestedPaths,
   NonObjectIdLikeDocument,
   NotAcceptedFields,
   NumericType,
@@ -295,6 +297,7 @@ export type {
   OptionalUnlessRequiredId,
   Projection,
   ProjectionOperators,
+  PropertyType,
   PullAllOperator,
   PullOperator,
   PushOperator,

src/mongo_types.ts

@@ -66,7 +66,9 @@ export type WithoutId<TSchema> = Omit<TSchema, '_id'>;
 
 /** A MongoDB filter can be some portion of the schema or a set of operators @public */
 export type Filter<TSchema> = {
-  [P in keyof WithId<TSchema>]?: Condition<WithId<TSchema>[P]>;
+  [Property in Join<NestedPaths<WithId<TSchema>>, '.'>]?: Condition<
+    PropertyType<WithId<TSchema>, Property>
+  >;
 } & RootFilterOperators<WithId<TSchema>>;
 
 /** @public */
@@ -440,3 +442,61 @@ export class TypedEventEmitter<Events extends EventsDescription> extends EventEm
 
 /** @public */
 export class CancellationToken extends TypedEventEmitter<{ cancel(): void }> {}
+
+/**
+ * Helper types for dot-notation filter attributes
+ */
+
+/** @public */
+export type Join<T extends unknown[], D extends string> = T extends []
+  ? ''
+  : T extends [string | number]
+  ? `${T[0]}`
+  : T extends [string | number, ...infer R]
+  ? `${T[0]}${D}${Join<R, D>}`
+  : string;
+
+/** @public */
+export type PropertyType<Type, Property extends string> = string extends Property
+  ? unknown
+  : Property extends keyof Type
+  ? Type[Property]
+  : Property extends `${number}`
+  ? Type extends ReadonlyArray<infer ArrayType>
+    ? ArrayType
+    : unknown
+  : Property extends `${infer Key}.${infer Rest}`
+  ? Key extends `${number}`
+    ? Type extends ReadonlyArray<infer ArrayType>
+      ? PropertyType<ArrayType, Rest>
+      : unknown
+    : Key extends keyof Type
+    ? Type[Key] extends Map<string, infer MapType>
+      ? MapType
+      : PropertyType<Type[Key], Rest>
+    : unknown
+  : unknown;
+
+// We dont't support nested circular references
+/** @public */
+export type NestedPaths<Type> = Type extends
+  | string
+  | number
+  | boolean
+  | Date
+  | RegExp
+  | Buffer
+  | Uint8Array
+  | ((...args: any[]) => any)
+  | { _bsontype: string }
+  ? []
+  : Type extends ReadonlyArray<infer ArrayType>
+  ? [number, ...NestedPaths<ArrayType>]
+  : Type extends Map<string, any>
+  ? [string]
+  : // eslint-disable-next-line @typescript-eslint/ban-types
+  Type extends object
+  ? {
+      [Key in Extract<keyof Type, string>]: [Key, ...NestedPaths<Type[Key]>];
+    }[Extract<keyof Type, string>]
+  : [];

test/types/community/collection/filterQuery.test-d.ts

@@ -1,4 +1,15 @@
-import { BSONRegExp, Decimal128, ObjectId } from 'bson';
+import {
+  Binary,
+  BSONRegExp,
+  BSONSymbol,
+  Code,
+  DBRef,
+  Decimal128,
+  Long,
+  MaxKey,
+  MinKey,
+  ObjectId
+} from 'bson';
 import { expectAssignable, expectError, expectNotType, expectType } from 'tsd';
 
 import { Collection, Filter, MongoClient, WithId } from '../../../../src';
@@ -16,6 +27,11 @@ const db = client.db('test');
  * Test the generic Filter using collection.find<T>() method
  */
 
+interface HumanModel {
+  _id: ObjectId;
+  name: string;
+}
+
 // a collection model for all possible MongoDB BSON types and TypeScript types
 interface PetModel {
   _id: ObjectId; // ObjectId field
@@ -24,14 +40,42 @@ interface PetModel {
   age: number; // number field
   type: 'dog' | 'cat' | 'fish'; // union field
   isCute: boolean; // boolean field
-  bestFriend?: PetModel; // object field (Embedded/Nested Documents)
+  bestFriend?: HumanModel; // object field (Embedded/Nested Documents)
   createdAt: Date; // date field
+  numOfPats: Long; // long field
   treats: string[]; // array of string
   playTimePercent: Decimal128; // bson Decimal128 type
-  readonly friends?: ReadonlyArray<PetModel>; // readonly array of objects
-  playmates?: PetModel[]; // writable array of objects
+  readonly friends?: ReadonlyArray<HumanModel>; // readonly array of objects
+  playmates?: HumanModel[]; // writable array of objects
+  laps?: Map<string, number>; // map field
+  // Object with multiple nested levels
+  meta?: {
+    updatedAt?: Date;
+    deep?: {
+      nestedArray: number[];
+      nested?: {
+        level?: number;
+      };
+    };
+  };
+
+  binary: Binary;
+  code: Code;
+  minKey: MinKey;
+  maxKey: MaxKey;
+  dBRef: DBRef;
+  bSONSymbol: BSONSymbol;
+
+  regex: RegExp;
+
+  fn: (...args: any[]) => any;
 }
 
+const john = {
+  _id: new ObjectId('577fa2d90c4cc47e31cf4b6a'),
+  name: 'John'
+};
+
 const spot = {
   _id: new ObjectId('577fa2d90c4cc47e31cf4b6f'),
   name: 'Spot',
@@ -40,16 +84,30 @@ const spot = {
   type: 'dog' as const,
   isCute: true,
   createdAt: new Date(),
+  numOfPats: Long.fromBigInt(100000000n),
   treats: ['kibble', 'bone'],
-  playTimePercent: new Decimal128('0.999999')
+  playTimePercent: new Decimal128('0.999999'),
+
+  binary: new Binary('', 2),
+  code: new Code(() => true),
+  minKey: new MinKey(),
+  maxKey: new MaxKey(),
+  dBRef: new DBRef('collection', new ObjectId()),
+  bSONSymbol: new BSONSymbol('hi'),
+
+  regex: /a/,
+
+  fn() {
+    return 'hi';
+  }
 };
 
 expectAssignable<PetModel>(spot);
 
 const collectionT = db.collection<PetModel>('test.filterQuery');
 
 // Assert that collection.find uses the Filter helper like so:
-const filter: Filter<PetModel> = {};
+const filter: Filter<PetModel> = {} as Filter<PetModel>;
 collectionT.find(filter);
 collectionT.find(spot); // a whole model definition is also a valid filter
 // Now tests below can directly test the Filter helper, and are implicitly checking collection.find
@@ -73,6 +131,10 @@ expectNotType<Filter<PetModel>>({ name: 23 });
 expectNotType<Filter<PetModel>>({ name: { suffix: 'Jr' } });
 expectNotType<Filter<PetModel>>({ name: ['Spot'] });
 
+// it should not accept wrong types for function fields
+expectNotType<Filter<PetModel>>({ fn: 3 });
+expectAssignable<WithId<PetModel>[]>(await collectionT.find({ fn: () => true }).toArray());
+
 /// it should query __number__ fields
 await collectionT.find({ age: 12 }).toArray();
 /// it should not accept wrong types for number fields
@@ -83,14 +145,67 @@ expectNotType<Filter<PetModel>>({ age: [23, 43] });
 
 /// it should query __nested document__ fields only by exact match
 // TODO: we currently cannot enforce field order but field order is important for mongo
-await collectionT.find({ bestFriend: spot }).toArray();
+await collectionT.find({ bestFriend: john }).toArray();
 /// nested documents query should contain all required fields
-expectNotType<Filter<PetModel>>({ bestFriend: { family: 'Andersons' } });
+expectNotType<Filter<PetModel>>({ bestFriend: { name: 'Andersons' } });
 /// it should not accept wrong types for nested document fields
 expectNotType<Filter<PetModel>>({ bestFriend: 21 });
 expectNotType<Filter<PetModel>>({ bestFriend: 'Andersons' });
 expectNotType<Filter<PetModel>>({ bestFriend: [spot] });
-expectNotType<Filter<PetModel>>({ bestFriend: [{ family: 'Andersons' }] });
+expectNotType<Filter<PetModel>>({ bestFriend: [{ name: 'Andersons' }] });
+
+// it should permit all our BSON types as query values
+expectAssignable<Filter<PetModel>>({ binary: new Binary('', 2) });
+expectAssignable<Filter<PetModel>>({ code: new Code(() => true) });
+expectAssignable<Filter<PetModel>>({ minKey: new MinKey() });
+expectAssignable<Filter<PetModel>>({ maxKey: new MaxKey() });
+expectAssignable<Filter<PetModel>>({ dBRef: new DBRef('collection', new ObjectId()) });
+expectAssignable<Filter<PetModel>>({ bSONSymbol: new BSONSymbol('hi') });
+
+// None of the bson types should be broken up into their nested keys
+expectNotType<Filter<PetModel>>({ 'binary.sub_type': 2 });
+expectNotType<Filter<PetModel>>({ 'code.code': 'string' });
+expectNotType<Filter<PetModel>>({ 'minKey._bsontype': 'MinKey' });
+expectNotType<Filter<PetModel>>({ 'maxKey._bsontype': 'MaxKey' });
+expectNotType<Filter<PetModel>>({ 'dBRef.collection': 'collection' });
+expectNotType<Filter<PetModel>>({ 'bSONSymbol.value': 'hi' });
+expectNotType<Filter<PetModel>>({ 'numOfPats.__isLong__': true });
+expectNotType<Filter<PetModel>>({ 'playTimePercent.bytes.BYTES_PER_ELEMENT': 1 });
+expectNotType<Filter<PetModel>>({ 'binary.sub_type': 'blah' });
+expectNotType<Filter<PetModel>>({ 'regex.dotAll': true });
+
+/// it should query __nested document__ fields using dot-notation
+collectionT.find({ 'meta.updatedAt': new Date() });
+collectionT.find({ 'meta.deep.nested.level': 123 });
+collectionT.find({ meta: { deep: { nested: { level: 123 } } } }); // no impact on actual nesting
+collectionT.find({ 'friends.0.name': 'John' });
+collectionT.find({ 'playmates.0.name': 'John' });
+// supports arrays with primitive types
+collectionT.find({ 'treats.0': 'bone' });
+collectionT.find({ 'laps.foo': 123 });
+
+// Handle special BSON types
+collectionT.find({ numOfPats: Long.fromBigInt(2n) });
+collectionT.find({ playTimePercent: new Decimal128('123.2') });
+
+// works with some extreme indexes
+collectionT.find({ 'friends.4294967295.name': 'John' });
+collectionT.find({ 'friends.999999999999999999999999999999999999.name': 'John' });
+
+/// it should not accept wrong types for nested document fields
+expectNotType<Filter<PetModel>>({ 'meta.updatedAt': 123 });
+expectNotType<Filter<PetModel>>({ 'meta.updatedAt': true });
+expectNotType<Filter<PetModel>>({ 'meta.updatedAt': 'now' });
+expectNotType<Filter<PetModel>>({ 'meta.deep.nested.level': 'string' });
+expectNotType<Filter<PetModel>>({ 'meta.deep.nested.level': true });
+expectNotType<Filter<PetModel>>({ 'meta.deep.nested.level': new Date() });
+expectNotType<Filter<PetModel>>({ 'friends.0.name': 123 });
+expectNotType<Filter<PetModel>>({ 'playmates.0.name': 123 });
+expectNotType<Filter<PetModel>>({ 'laps.foo': 'string' });
+expectNotType<Filter<PetModel>>({ 'treats.0': 123 });
+
+// Nested arrays aren't checked
+expectNotType<Filter<PetModel>>({ 'meta.deep.nestedArray.0': 'not a number' });
 
 /// it should query __array__ fields by exact match
 await collectionT.find({ treats: ['kibble', 'bone'] }).toArray();
@@ -233,10 +348,6 @@ expectNotType<Filter<PetModel>>({ name: { $all: ['world', 'world'] } });
 expectNotType<Filter<PetModel>>({ age: { $elemMatch: [1, 2] } });
 expectNotType<Filter<PetModel>>({ type: { $size: 2 } });
 
-// dot key case that shows it is assignable even when the referenced key is the wrong type
-expectAssignable<Filter<PetModel>>({ 'bestFriend.name': 23 }); // using dot notation permits any type for the key
-expectNotType<Filter<PetModel>>({ bestFriend: { name: 23 } });
-
 // ObjectId are not allowed to be used as a query predicate (issue described here: NODE-3758)
 // this only applies to schemas where the _id is not of type ObjectId.
 declare const nonObjectIdCollection: Collection<{ _id: number; otherField: string }>;