fix(NODE-2944): Reintroduce bson-ext support (#2823)

- Update benchmark imports
- Add bson-ext test to CI

Author: nbbeeken

.evergreen/config.yml

@@ -516,6 +516,16 @@ functions:
           rm -f ./prepare_client_encryption.sh
 
           MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh
+  run bson-ext test:
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: src
+        timeout_secs: 60
+        script: |
+          ${PREPARE_SHELL}
+
+          MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-bson-ext-test.sh
   upload test results:
     - command: attach.xunit_results
       params:
@@ -1353,6 +1363,20 @@ tasks:
           VERSION: '4.4'
           TOPOLOGY: server
       - func: run custom csfle tests
+  - name: run-bson-ext-test
+    tags:
+      - run-bson-ext-test
+    commands:
+      - func: install dependencies
+        vars:
+          NODE_LTS_NAME: fermium
+      - func: bootstrap mongo-orchestration
+        vars:
+          VERSION: '4.4'
+          TOPOLOGY: server
+      - func: run bson-ext test
+        vars:
+          NODE_LTS_NAME: fermium
 buildvariants:
   - name: macos-1014-dubnium
     display_name: macOS 10.14 Node Dubnium
@@ -1685,6 +1709,11 @@ buildvariants:
     run_on: ubuntu1804-test
     tasks:
       - run-custom-csfle-tests
+  - name: ubuntu1804-run-bson-ext-test
+    display_name: BSON EXT Test
+    run_on: ubuntu1804-test
+    tasks:
+      - run-bson-ext-test
   - name: mongosh_integration_tests
     display_name: mongosh integration tests
     run_on: ubuntu1804-test

.evergreen/config.yml.in

@@ -561,6 +561,17 @@ functions:
 
           MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh
 
+  "run bson-ext test":
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: "src"
+        timeout_secs: 60
+        script: |
+          ${PREPARE_SHELL}
+
+          MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-bson-ext-test.sh
+
   "upload test results":
     # Upload the xunit-format test results.
     - command: attach.xunit_results

.evergreen/generate_evergreen_tasks.js

@@ -533,6 +533,11 @@ BUILD_VARIANTS.push({
   display_name: 'Custom FLE Version Test',
   run_on: 'ubuntu1804-test',
   tasks: ['run-custom-csfle-tests']
+},{
+  name: 'ubuntu1804-run-bson-ext-test',
+  display_name: 'BSON EXT Test',
+  run_on: 'ubuntu1804-test',
+  tasks: ['run-bson-ext-test']
 });
 
 // singleton build variant for mongosh integration tests
@@ -591,6 +596,32 @@ SINGLETON_TASKS.push({
   ]
 });
 
+// special case for custom BSON-ext test
+SINGLETON_TASKS.push({
+  name: 'run-bson-ext-test',
+  tags: ['run-bson-ext-test'],
+  commands: [
+    {
+      func: 'install dependencies',
+      vars: {
+        NODE_LTS_NAME: 'fermium',
+      },
+    },
+    {
+      func: 'bootstrap mongo-orchestration',
+      vars: {
+        VERSION: '4.4',
+        TOPOLOGY: 'server'
+      }
+    },
+    { func: 'run bson-ext test',
+      vars: {
+        NODE_LTS_NAME: 'fermium',
+      }
+    }
+  ]
+});
+
 const fileData = yaml.safeLoad(fs.readFileSync(`${__dirname}/config.yml.in`, 'utf8'));
 fileData.tasks = (fileData.tasks || []).concat(BASE_TASKS).concat(TASKS).concat(SINGLETON_TASKS);
 fileData.buildvariants = (fileData.buildvariants || []).concat(BUILD_VARIANTS);

.evergreen/run-bson-ext-test.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+[ -s "$PROJECT_DIRECTORY/node-artifacts/nvm/nvm.sh" ] && source "$PROJECT_DIRECTORY"/node-artifacts/nvm/nvm.sh
+
+set -o xtrace   # Write all commands first to stderr
+set -o errexit  # Exit the script with error if any of the commands fail
+
+# Supported/used environment variables:
+#       SSL                     Set to enable SSL. Defaults to "nossl"
+#       MONGODB_URI             Set the suggested connection MONGODB_URI (including credentials and topology info)
+#       TEST_NPM_SCRIPT         Script to npm run. Defaults to "check:test"
+
+MONGODB_URI=${MONGODB_URI:-}
+TEST_NPM_SCRIPT=${TEST_NPM_SCRIPT:-check:test}
+
+# ssl setup
+SSL=${SSL:-nossl}
+if [ "$SSL" != "nossl" ]; then
+   export SSL_KEY_FILE="$DRIVERS_TOOLS/.evergreen/x509gen/client.pem"
+   export SSL_CA_FILE="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem"
+fi
+
+# run tests
+echo "Running $AUTH tests over $SSL, connecting to $MONGODB_URI"
+
+npm install bson-ext
+
+export MONGODB_API_VERSION=${MONGODB_API_VERSION}
+export MONGODB_URI=${MONGODB_URI}
+
+npm run "${TEST_NPM_SCRIPT}"

src/bson.ts

@@ -1,10 +1,21 @@
-// import type * as _BSON from 'bson';
-// let BSON: typeof _BSON = require('bson');
-// try {
-//   BSON = require('bson-ext');
-// } catch {} // eslint-disable-line
+import type {
+  serialize as serializeFn,
+  deserialize as deserializeFn,
+  calculateObjectSize as calculateObjectSizeFn
+} from 'bson';
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+let BSON = require('bson');
+try {
+  BSON = require('bson-ext');
+} catch {} // eslint-disable-line
 
-// export = BSON;
+/** @internal */
+export const deserialize = BSON.deserialize as typeof deserializeFn;
+/** @internal */
+export const serialize = BSON.serialize as typeof serializeFn;
+/** @internal */
+export const calculateObjectSize = BSON.calculateObjectSize as typeof calculateObjectSizeFn;
 
 export {
   Long,
@@ -21,38 +32,27 @@ export {
   BSONRegExp,
   BSONSymbol,
   Map,
-  deserialize,
-  serialize,
-  calculateObjectSize
+  Document
 } from 'bson';
 
-/** @public */
-export interface Document {
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  [key: string]: any;
-}
+import type { DeserializeOptions, SerializeOptions } from 'bson';
 
-import type { SerializeOptions } from 'bson';
-
-// TODO: Remove me when types from BSON are updated
 /**
  * BSON Serialization options.
  * @public
  */
-export interface BSONSerializeOptions extends Omit<SerializeOptions, 'index'> {
-  /** Return document results as raw BSON buffers */
-  fieldsAsRaw?: { [key: string]: boolean };
-  /** Promotes BSON values to native types where possible, set to false to only receive wrapper types */
-  promoteValues?: boolean;
-  /** Promotes Binary BSON values to native Node Buffers */
-  promoteBuffers?: boolean;
-  /** Promotes long values to number if they fit inside the 53 bits resolution */
-  promoteLongs?: boolean;
-  /** Serialize functions on any object */
-  serializeFunctions?: boolean;
-  /** Specify if the BSON serializer should ignore undefined fields */
-  ignoreUndefined?: boolean;
-
+export interface BSONSerializeOptions
+  extends Omit<SerializeOptions, 'index'>,
+    Omit<
+      DeserializeOptions,
+      | 'evalFunctions'
+      | 'cacheFunctions'
+      | 'cacheFunctionsCrc32'
+      | 'bsonRegExp'
+      | 'allowObjectSmallerThanBufferSize'
+      | 'index'
+    > {
+  /** Return BSON filled buffers from operations */
   raw?: boolean;
 }
 

src/index.ts

@@ -358,3 +358,4 @@ export type {
   MetaProjectionOperators,
   MetaSortOperators
 } from './mongo_types';
+export type { serialize, deserialize } from './bson';

test/benchmarks/driverBench/common.js

@@ -1,9 +1,10 @@
+/* eslint-disable no-restricted-modules */
 'use strict';
 
 const fs = require('fs');
 const path = require('path');
-const { MongoClient } = require('../../../src/mongo_client');
-const { GridFsBucket } = require('../../../src/gridfs-stream');
+const { MongoClient } = require('../../..');
+const { GridFSBucket } = require('../../..');
 
 const DB_NAME = 'perftest';
 const COLLECTION_NAME = 'corpus';
@@ -54,7 +55,7 @@ function dropCollection() {
 }
 
 function initBucket() {
-  this.bucket = new GridFsBucket(this.db);
+  this.bucket = new GridFSBucket(this.db);
 }
 
 function dropBucket() {

test/benchmarks/driverBench/index.js

@@ -5,7 +5,13 @@ const MongoBench = require('../mongoBench');
 const Runner = MongoBench.Runner;
 const commonHelpers = require('./common');
 
-const BSON = require('bson');
+let BSON = require('bson');
+try {
+  BSON = require('bson-ext');
+} catch (_) {
+  // do not care
+}
+
 const { EJSON } = require('bson');
 
 const makeClient = commonHelpers.makeClient;
@@ -359,5 +365,8 @@ benchmarkRunner
       driverBench
     };
   })
-  .then(data => console.log(data))
+  .then(data => {
+    data.bsonType = BSON.serialize.toString().includes('native code') ? 'bson-ext' : 'js-bson';
+    console.log(data);
+  })
   .catch(err => console.error(err));

test/functional/insert.test.js

@@ -1577,7 +1577,7 @@ describe('Insert', function () {
         var collection = db.collection('bson_types_insert_1');
 
         var document = {
-          symbol: new BSONSymbol('abcdefghijkl'),
+          string: 'abcdefghijkl',
           objid: new ObjectId('abcdefghijkl'),
           double: new Double(1),
           binary: new Binary(Buffer.from('hello world')),
@@ -1590,9 +1590,9 @@ describe('Insert', function () {
           expect(err).to.not.exist;
           test.ok(result);
 
-          collection.findOne({ symbol: new BSONSymbol('abcdefghijkl') }, function (err, doc) {
+          collection.findOne({ string: 'abcdefghijkl' }, function (err, doc) {
             expect(err).to.not.exist;
-            test.equal('abcdefghijkl', doc.symbol.toString());
+            test.equal('abcdefghijkl', doc.string.toString());
 
             collection.findOne({ objid: new ObjectId('abcdefghijkl') }, function (err, doc) {
               expect(err).to.not.exist;

test/types/bson.test-d.ts

@@ -0,0 +1,26 @@
+import { expectType } from 'tsd';
+import type { BSONSerializeOptions, Document } from '../../src/bson';
+
+const options: BSONSerializeOptions = {};
+
+expectType<boolean | undefined>(options.checkKeys);
+expectType<boolean | undefined>(options.serializeFunctions);
+expectType<boolean | undefined>(options.ignoreUndefined);
+expectType<boolean | undefined>(options.promoteLongs);
+expectType<boolean | undefined>(options.promoteBuffers);
+expectType<boolean | undefined>(options.promoteValues);
+expectType<Document | undefined>(options.fieldsAsRaw);
+
+type PermittedBSONOptionKeys =
+  | 'checkKeys'
+  | 'serializeFunctions'
+  | 'ignoreUndefined'
+  | 'promoteLongs'
+  | 'promoteBuffers'
+  | 'promoteValues'
+  | 'fieldsAsRaw'
+  | 'raw';
+
+const keys = (null as unknown) as PermittedBSONOptionKeys;
+// creates an explicit allow list assertion
+expectType<keyof BSONSerializeOptions>(keys);