From ce8cfa46ad785ee32a181b6f194be9076f8be5ce Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Tue, 11 Jun 2024 12:07:25 -0400 Subject: [PATCH 1/9] wip --- test/benchmark/main.mjs | 56 +++++++++++++++ test/benchmark/mongocrypt_binary_t.bench.mjs | 76 ++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 test/benchmark/main.mjs create mode 100644 test/benchmark/mongocrypt_binary_t.bench.mjs diff --git a/test/benchmark/main.mjs b/test/benchmark/main.mjs new file mode 100644 index 0000000..e72f98c --- /dev/null +++ b/test/benchmark/main.mjs @@ -0,0 +1,56 @@ +/* eslint-disable no-console */ +import util from 'node:util'; +import process from 'node:process'; +import path from 'node:path'; +import child_process, { spawn } from 'node:child_process'; +import events from 'node:events'; +import url from 'node:url'; + +const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); + +/** Resolves to the root of this repository */ +function resolveRoot(...paths) { + return path.resolve(__dirname, '..', '..', ...paths); +} + +/** `xtrace` style command runner, uses spawn so that stdio is inherited */ +async function run(command, args = [], options = {}) { + const commandDetails = `+ ${command} ${args.join(' ')}${options.cwd ? ` (in: ${options.cwd})` : ''}`; + console.error(commandDetails); + const proc = child_process.spawn(command, args, { + shell: process.platform === 'win32', + stdio: 'inherit', + cwd: resolveRoot('.'), + ...options + }); + await events.once(proc, 'exit'); + + if (proc.exitCode !== 0) throw new Error(`CRASH(${proc.exitCode}): ${commandDetails}`); +} + +function parseArguments() { + const options = { + help: { short: 'h', type: 'boolean', default: false } + }; + + const args = util.parseArgs({ args: process.argv.slice(2), options, allowPositionals: false }); + + if (args.values.help) { + console.log( + `${path.basename(process.argv[1])} ${[...Object.keys(options)] + .filter(k => k !== 'help') + .map(k => `[--${k}=${options[k].type}]`) + .join(' ')}` + ); + process.exit(0); + } + + return {}; +} + +async function main() { + parseArguments(); + bench({ spawn: true }); +} + +await main(); diff --git a/test/benchmark/mongocrypt_binary_t.bench.mjs b/test/benchmark/mongocrypt_binary_t.bench.mjs new file mode 100644 index 0000000..00fa974 --- /dev/null +++ b/test/benchmark/mongocrypt_binary_t.bench.mjs @@ -0,0 +1,76 @@ +import process from 'node:process'; +import { MongoClient, ClientEncryption } from 'mongodb'; + +let { MONGODB_URI = '', CRYPT_SHARED_LIB_PATH = '' } = process.env; +MONGODB_URI.length === 0 ? 'mongodb://127.0.0.1:27017' : MONGODB_URI; +const extraOptions = + CRYPT_SHARED_LIB_PATH.length !== 0 ? { cryptSharedLibPath: CRYPT_SHARED_LIB_PATH } : undefined; + +const LOCAL_KEY = Buffer.from( + 'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', + 'base64' +); + +const $jsonSchema = { + properties: { + encrypted: { + encrypt: { + keyId: [{ $binary: { base64: 'LOCALAAAAAAAAAAAAAAAAA==', subType: '04' } }], + bsonType: 'string', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' + } + } + }, + bsonType: 'object' +}; + +let client; + +let ae = { + autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, + bypassAutoEncryption: false, + keyVaultClient: undefined, + extraOptions + } +}; + +async function setupCollection() { + const client = new MongoClient(MONGODB_URI); + const collectionName = 'test_fle_bench'; + const db = client.db('test_fle_bench'); + await db.dropCollection(collectionName).catch(() => {}); + + const clientEncryption = new ClientEncryption(client, { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } } + }); + + const encryptedFields = { + escCollection: 'esc', + eccCollection: 'ecc', + ecocCollection: 'ecoc', + fields: Array.from({ length: 5000 }, (_, i) => ({ + path: `key${i.toString().padStart(4, '0')}`, + bsonType: 'string' + })) + }; + + const { collection } = await clientEncryption.createEncryptedCollection(db, collectionName, { + provider: 'local', + createCollectionOptions: { encryptedFields } + }); + + return collection; +} + +async function main() { + const collection = await setupCollection(); + + await client + .db('test_fle_bench') + .createCollection('test_fle_bench', { validator: { $jsonSchema } }); +} + +export async function bench(options) {} From 52da92537ed9f341fab7015b8247b1a274dde885 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Wed, 12 Jun 2024 18:06:17 -0400 Subject: [PATCH 2/9] chore(NODE-5455): benchmark FLE --- test/benchmark/main.mjs | 56 --------------- test/benchmark/mongocrypt_binary_t.bench.mjs | 76 -------------------- 2 files changed, 132 deletions(-) delete mode 100644 test/benchmark/main.mjs delete mode 100644 test/benchmark/mongocrypt_binary_t.bench.mjs diff --git a/test/benchmark/main.mjs b/test/benchmark/main.mjs deleted file mode 100644 index e72f98c..0000000 --- a/test/benchmark/main.mjs +++ /dev/null @@ -1,56 +0,0 @@ -/* eslint-disable no-console */ -import util from 'node:util'; -import process from 'node:process'; -import path from 'node:path'; -import child_process, { spawn } from 'node:child_process'; -import events from 'node:events'; -import url from 'node:url'; - -const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); - -/** Resolves to the root of this repository */ -function resolveRoot(...paths) { - return path.resolve(__dirname, '..', '..', ...paths); -} - -/** `xtrace` style command runner, uses spawn so that stdio is inherited */ -async function run(command, args = [], options = {}) { - const commandDetails = `+ ${command} ${args.join(' ')}${options.cwd ? ` (in: ${options.cwd})` : ''}`; - console.error(commandDetails); - const proc = child_process.spawn(command, args, { - shell: process.platform === 'win32', - stdio: 'inherit', - cwd: resolveRoot('.'), - ...options - }); - await events.once(proc, 'exit'); - - if (proc.exitCode !== 0) throw new Error(`CRASH(${proc.exitCode}): ${commandDetails}`); -} - -function parseArguments() { - const options = { - help: { short: 'h', type: 'boolean', default: false } - }; - - const args = util.parseArgs({ args: process.argv.slice(2), options, allowPositionals: false }); - - if (args.values.help) { - console.log( - `${path.basename(process.argv[1])} ${[...Object.keys(options)] - .filter(k => k !== 'help') - .map(k => `[--${k}=${options[k].type}]`) - .join(' ')}` - ); - process.exit(0); - } - - return {}; -} - -async function main() { - parseArguments(); - bench({ spawn: true }); -} - -await main(); diff --git a/test/benchmark/mongocrypt_binary_t.bench.mjs b/test/benchmark/mongocrypt_binary_t.bench.mjs deleted file mode 100644 index 00fa974..0000000 --- a/test/benchmark/mongocrypt_binary_t.bench.mjs +++ /dev/null @@ -1,76 +0,0 @@ -import process from 'node:process'; -import { MongoClient, ClientEncryption } from 'mongodb'; - -let { MONGODB_URI = '', CRYPT_SHARED_LIB_PATH = '' } = process.env; -MONGODB_URI.length === 0 ? 'mongodb://127.0.0.1:27017' : MONGODB_URI; -const extraOptions = - CRYPT_SHARED_LIB_PATH.length !== 0 ? { cryptSharedLibPath: CRYPT_SHARED_LIB_PATH } : undefined; - -const LOCAL_KEY = Buffer.from( - 'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', - 'base64' -); - -const $jsonSchema = { - properties: { - encrypted: { - encrypt: { - keyId: [{ $binary: { base64: 'LOCALAAAAAAAAAAAAAAAAA==', subType: '04' } }], - bsonType: 'string', - algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic' - } - } - }, - bsonType: 'object' -}; - -let client; - -let ae = { - autoEncryption: { - keyVaultNamespace: 'keyvault.datakeys', - kmsProviders: { local: { key: LOCAL_KEY } }, - bypassAutoEncryption: false, - keyVaultClient: undefined, - extraOptions - } -}; - -async function setupCollection() { - const client = new MongoClient(MONGODB_URI); - const collectionName = 'test_fle_bench'; - const db = client.db('test_fle_bench'); - await db.dropCollection(collectionName).catch(() => {}); - - const clientEncryption = new ClientEncryption(client, { - keyVaultNamespace: 'keyvault.datakeys', - kmsProviders: { local: { key: LOCAL_KEY } } - }); - - const encryptedFields = { - escCollection: 'esc', - eccCollection: 'ecc', - ecocCollection: 'ecoc', - fields: Array.from({ length: 5000 }, (_, i) => ({ - path: `key${i.toString().padStart(4, '0')}`, - bsonType: 'string' - })) - }; - - const { collection } = await clientEncryption.createEncryptedCollection(db, collectionName, { - provider: 'local', - createCollectionOptions: { encryptedFields } - }); - - return collection; -} - -async function main() { - const collection = await setupCollection(); - - await client - .db('test_fle_bench') - .createCollection('test_fle_bench', { validator: { $jsonSchema } }); -} - -export async function bench(options) {} From 9840e821d711958ef72b0ee9efcf96b689d71ad1 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 13 Jun 2024 14:35:46 -0400 Subject: [PATCH 3/9] feat(NODE-5455): use builtin native crypto --- .github/scripts/libmongocrypt.mjs | 89 ++++++++++++++++++++++++------- 1 file changed, 71 insertions(+), 18 deletions(-) diff --git a/.github/scripts/libmongocrypt.mjs b/.github/scripts/libmongocrypt.mjs index 7299ac1..b04b80d 100644 --- a/.github/scripts/libmongocrypt.mjs +++ b/.github/scripts/libmongocrypt.mjs @@ -23,6 +23,8 @@ async function parseArguments() { libVersion: { short: 'l', type: 'string', default: pkg['mongodb:libmongocrypt'] }, clean: { short: 'c', type: 'boolean', default: false }, build: { short: 'b', type: 'boolean', default: false }, + crypto: { type: 'boolean', default: false }, // Use Node.js builtin crypto + fastDownload: { type: 'boolean', default: false }, // Potentially incorrect download, only for the brave and impatient help: { short: 'h', type: 'boolean', default: false } }; @@ -39,7 +41,12 @@ async function parseArguments() { } return { - libmongocrypt: { url: args.values.gitURL, ref: args.values.libVersion }, + libmongocrypt: { + url: args.values.gitURL, + ref: args.values.libVersion, + crypto: args.values.crypto + }, + fastDownload: args.values.fastDownload, clean: args.values.clean, build: args.values.build, pkg @@ -77,7 +84,7 @@ export async function cloneLibMongoCrypt(libmongocryptRoot, { url, ref }) { } } -export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot) { +export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot, options) { console.error('building libmongocrypt...'); const nodeBuildRoot = resolveRoot(nodeDepsRoot, 'tmp', 'libmongocrypt-build'); @@ -88,7 +95,6 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot) { const CMAKE_FLAGS = toFlags({ /** * We provide crypto hooks from Node.js binding to openssl (so disable system crypto) - * TODO: NODE-5455 * * One thing that is not obvious from the build instructions for libmongocrypt * and the Node.js bindings is that the Node.js driver uses libmongocrypt in @@ -101,7 +107,7 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot) { * have a copy of OpenSSL available directly, but for now it seems to make sense * to stick with what the Node.js addon does here. */ - DDISABLE_NATIVE_CRYPTO: '1', + DDISABLE_NATIVE_CRYPTO: options.crypto ? '0' : '1', /** A consistent name for the output "library" directory */ DCMAKE_INSTALL_LIBDIR: 'lib', /** No warnings allowed */ @@ -128,15 +134,26 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot) { await run( 'cmake', - [...CMAKE_FLAGS, ...WINDOWS_CMAKE_FLAGS, ...MACOS_CMAKE_FLAGS, libmongocryptRoot], + [ + '--parallel', + '4', + ...CMAKE_FLAGS, + ...WINDOWS_CMAKE_FLAGS, + ...MACOS_CMAKE_FLAGS, + libmongocryptRoot + ], { cwd: nodeBuildRoot } ); - await run('cmake', ['--build', '.', '--target', 'install', '--config', 'RelWithDebInfo'], { - cwd: nodeBuildRoot - }); + await run( + 'cmake', + ['--parallel', '4', '--build', '.', '--target', 'install', '--config', 'RelWithDebInfo'], + { + cwd: nodeBuildRoot + } + ); } -export async function downloadLibMongoCrypt(nodeDepsRoot, { ref }) { +export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, crypto }, fastDownload) { const downloadURL = ref === 'latest' ? 'https://mciuploads.s3.amazonaws.com/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz' @@ -164,28 +181,62 @@ export async function downloadLibMongoCrypt(nodeDepsRoot, { ref }) { console.error(`Platform: ${detectedPlatform} Prebuild: ${prebuild}`); - const unzipArgs = ['-xzv', '-C', `_libmongocrypt-${ref}`, `${prebuild}/nocrypto`]; + const downloadDestination = crypto ? `${prebuild}` : `${prebuild}/nocrypto`; + const unzipArgs = ['-xzv', '-C', `_libmongocrypt-${ref}`, downloadDestination]; console.error(`+ tar ${unzipArgs.join(' ')}`); const unzip = child_process.spawn('tar', unzipArgs, { - stdio: ['pipe', 'inherit'], + stdio: ['pipe', 'inherit', 'pipe'], cwd: resolveRoot('.') }); const [response] = await events.once(https.get(downloadURL), 'response'); const start = performance.now(); - await stream.pipeline(response, unzip.stdin); + + let signal; + if (fastDownload) { + /** + * Tar will print out each file it finds inside MEMBER (ex. macos/nocrypto) + * For each file it prints, we give it a deadline of 10seconds to print the next one. + * If nothing prints after 10 seconds we exit early. + * This depends on the tar file being in order and un-tar-able in under 10sec. + * + * download time went from 230s to 80s + */ + const controller = new AbortController(); + signal = controller.signal; + let isFirstStdoutDataEv = true; + let timeout; + unzip.stderr.on('data', chunk => { + process.stderr.write(chunk, () => { + if (isFirstStdoutDataEv) { + isFirstStdoutDataEv = false; + timeout = setTimeout(() => controller.abort(), 10_000); + } + timeout?.refresh(); + }); + }); + } + + try { + await stream.pipeline(response, unzip.stdin, { signal }); + } catch { + await fs.access(path.join(`_libmongocrypt-${ref}`, downloadDestination)); + } const end = performance.now(); console.error(`downloaded libmongocrypt in ${(end - start) / 1000} secs...`); await fs.rm(nodeDepsRoot, { recursive: true, force: true }); - await fs.cp(resolveRoot(destination, prebuild, 'nocrypto'), nodeDepsRoot, { recursive: true }); - const currentPath = path.join(nodeDepsRoot, 'lib64'); + const source = crypto + ? resolveRoot(destination, prebuild) + : resolveRoot(destination, prebuild, 'nocrypto'); + await fs.cp(source, nodeDepsRoot, { recursive: true }); + const potentialLib64Path = path.join(nodeDepsRoot, 'lib64'); try { - await fs.rename(currentPath, path.join(nodeDepsRoot, 'lib')); + await fs.rename(potentialLib64Path, path.join(nodeDepsRoot, 'lib')); } catch (error) { - console.error(`error renaming ${currentPath}: ${error.message}`); + await fs.access(path.join(nodeDepsRoot, 'lib')); // Ensure there is a "lib" directory } } @@ -214,11 +265,13 @@ async function main() { const isBuilt = libmongocryptBuiltVersion.trim() === libmongocrypt.ref; if (clean || !isBuilt) { - await buildLibMongoCrypt(libmongocryptCloneDir, nodeDepsDir); + await buildLibMongoCrypt(libmongocryptCloneDir, nodeDepsDir, { + crypto: libmongocrypt.crypto + }); } } else { // Download - await downloadLibMongoCrypt(nodeDepsDir, libmongocrypt); + await downloadLibMongoCrypt(nodeDepsDir, libmongocrypt, fastDownload); } await fs.rm(resolveRoot('build'), { force: true, recursive: true }); From 1a5d641cffb1f6c338ce732d50e1d3bdb53e4bee Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 13 Jun 2024 15:09:51 -0400 Subject: [PATCH 4/9] fix: build --- .github/scripts/libmongocrypt.mjs | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/.github/scripts/libmongocrypt.mjs b/.github/scripts/libmongocrypt.mjs index b04b80d..01fcd8b 100644 --- a/.github/scripts/libmongocrypt.mjs +++ b/.github/scripts/libmongocrypt.mjs @@ -134,23 +134,13 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot, option await run( 'cmake', - [ - '--parallel', - '4', - ...CMAKE_FLAGS, - ...WINDOWS_CMAKE_FLAGS, - ...MACOS_CMAKE_FLAGS, - libmongocryptRoot - ], + [...CMAKE_FLAGS, ...WINDOWS_CMAKE_FLAGS, ...MACOS_CMAKE_FLAGS, libmongocryptRoot], { cwd: nodeBuildRoot } ); - await run( - 'cmake', - ['--parallel', '4', '--build', '.', '--target', 'install', '--config', 'RelWithDebInfo'], - { - cwd: nodeBuildRoot - } - ); + + await run('cmake', ['--build', '.', '--target', 'install', '--config', 'RelWithDebInfo'], { + cwd: nodeBuildRoot + }); } export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, crypto }, fastDownload) { @@ -265,9 +255,7 @@ async function main() { const isBuilt = libmongocryptBuiltVersion.trim() === libmongocrypt.ref; if (clean || !isBuilt) { - await buildLibMongoCrypt(libmongocryptCloneDir, nodeDepsDir, { - crypto: libmongocrypt.crypto - }); + await buildLibMongoCrypt(libmongocryptCloneDir, nodeDepsDir, libmongocrypt); } } else { // Download From ba7fea792b1e8c661e928bba0c7d5fe320a778d6 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 13 Jun 2024 15:54:50 -0400 Subject: [PATCH 5/9] feat: make crypto default --- .github/scripts/libmongocrypt.mjs | 36 +++++++++++++++---------------- .github/workflows/build.yml | 2 +- .github/workflows/codeql.yml | 2 +- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/scripts/libmongocrypt.mjs b/.github/scripts/libmongocrypt.mjs index 01fcd8b..001b552 100644 --- a/.github/scripts/libmongocrypt.mjs +++ b/.github/scripts/libmongocrypt.mjs @@ -1,3 +1,4 @@ +// @ts-check import util from 'node:util'; import process from 'node:process'; import fs from 'node:fs/promises'; @@ -23,7 +24,7 @@ async function parseArguments() { libVersion: { short: 'l', type: 'string', default: pkg['mongodb:libmongocrypt'] }, clean: { short: 'c', type: 'boolean', default: false }, build: { short: 'b', type: 'boolean', default: false }, - crypto: { type: 'boolean', default: false }, // Use Node.js builtin crypto + 'no-crypto': { type: 'boolean', default: false }, // Use Node.js builtin crypto fastDownload: { type: 'boolean', default: false }, // Potentially incorrect download, only for the brave and impatient help: { short: 'h', type: 'boolean', default: false } }; @@ -41,11 +42,9 @@ async function parseArguments() { } return { - libmongocrypt: { - url: args.values.gitURL, - ref: args.values.libVersion, - crypto: args.values.crypto - }, + url: args.values.gitURL, + ref: args.values.libVersion, + crypto: !args.values['no-crypto'], fastDownload: args.values.fastDownload, clean: args.values.clean, build: args.values.build, @@ -143,7 +142,7 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot, option }); } -export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, crypto }, fastDownload) { +export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, crypto, fastDownload }) { const downloadURL = ref === 'latest' ? 'https://mciuploads.s3.amazonaws.com/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz' @@ -231,11 +230,12 @@ export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, crypto }, fastD } async function main() { - const { libmongocrypt, build, clean, pkg } = await parseArguments(); + const args = await parseArguments(); + console.log(args); const nodeDepsDir = resolveRoot('deps'); - if (build) { + if (args.build) { const libmongocryptCloneDir = resolveRoot('_libmongocrypt'); const currentLibMongoCryptBranch = await fs @@ -243,23 +243,23 @@ async function main() { .catch(() => ''); const isClonedAndCheckedOut = currentLibMongoCryptBranch .trim() - .endsWith(`r-${libmongocrypt.ref}`); + .endsWith(`r-${args.ref}`); - if (clean || !isClonedAndCheckedOut) { - await cloneLibMongoCrypt(libmongocryptCloneDir, libmongocrypt); + if (args.clean || !isClonedAndCheckedOut) { + await cloneLibMongoCrypt(libmongocryptCloneDir, args); } const libmongocryptBuiltVersion = await fs .readFile(path.join(libmongocryptCloneDir, 'VERSION_CURRENT'), 'utf8') .catch(() => ''); - const isBuilt = libmongocryptBuiltVersion.trim() === libmongocrypt.ref; + const isBuilt = libmongocryptBuiltVersion.trim() === args.ref; - if (clean || !isBuilt) { - await buildLibMongoCrypt(libmongocryptCloneDir, nodeDepsDir, libmongocrypt); + if (args.clean || !isBuilt) { + await buildLibMongoCrypt(libmongocryptCloneDir, nodeDepsDir, args); } } else { // Download - await downloadLibMongoCrypt(nodeDepsDir, libmongocrypt, fastDownload); + await downloadLibMongoCrypt(nodeDepsDir, args); } await fs.rm(resolveRoot('build'), { force: true, recursive: true }); @@ -274,8 +274,8 @@ async function main() { if (process.platform === 'darwin') { // The "arm64" build is actually a universal binary await fs.copyFile( - resolveRoot('prebuilds', `mongodb-client-encryption-v${pkg.version}-napi-v4-darwin-arm64.tar.gz`), - resolveRoot('prebuilds', `mongodb-client-encryption-v${pkg.version}-napi-v4-darwin-x64.tar.gz`) + resolveRoot('prebuilds', `mongodb-client-encryption-v${args.pkg.version}-napi-v4-darwin-arm64.tar.gz`), + resolveRoot('prebuilds', `mongodb-client-encryption-v${args.pkg.version}-napi-v4-darwin-x64.tar.gz`) ); } } diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a4c3421..d6e73b4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@v4 - name: Build ${{ matrix.os }} Prebuild - run: node .github/scripts/libmongocrypt.mjs ${{ runner.os == 'Windows' && '--build' || '' }} + run: npm run install:libmongocrypt ${{ runner.os == 'Windows' && '-- --build' || '' }} shell: bash - name: Test ${{ matrix.os }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e1f501a..c44d69d 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -46,7 +46,7 @@ jobs: - if: matrix.build-mode == 'manual' shell: bash - run: node .github/scripts/libmongocrypt.mjs + run: npm run install:libmongocrypt - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 From 8376c53461ab2cd0191f5167f679597b1a30ea0b Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 13 Jun 2024 15:55:25 -0400 Subject: [PATCH 6/9] chore: make bench run native crypto by default --- test/benchmarks/bench.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/benchmarks/bench.mjs b/test/benchmarks/bench.mjs index 7062a5d..10a9409 100644 --- a/test/benchmarks/bench.mjs +++ b/test/benchmarks/bench.mjs @@ -15,7 +15,7 @@ const ERROR = 0; const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); -const { CRYPT_SHARED_LIB_PATH: cryptSharedLibPath = '', BENCH_WITH_NATIVE_CRYPTO = '' } = +const { CRYPT_SHARED_LIB_PATH: cryptSharedLibPath = '', BENCH_WITHOUT_NATIVE_CRYPTO = '' } = process.env; const warmupSecs = 2; @@ -122,7 +122,7 @@ function main() { ); const mongoCryptOptions = { kmsProviders: BSON.serialize(kmsProviders) }; - if (!BENCH_WITH_NATIVE_CRYPTO) mongoCryptOptions.cryptoCallbacks = cryptoCallbacks; + if (BENCH_WITHOUT_NATIVE_CRYPTO) mongoCryptOptions.cryptoCallbacks = cryptoCallbacks; if (cryptSharedLibPath) mongoCryptOptions.cryptSharedLibPath = cryptSharedLibPath; const mongoCrypt = new MongoCrypt(mongoCryptOptions); From f41e5d1d24cb47d92eb828fa748977ba2fa9bb68 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 13 Jun 2024 16:32:05 -0400 Subject: [PATCH 7/9] fix(NODE-5875): check for mongocrypt_is_crypto_available --- addon/mongocrypt.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon/mongocrypt.cc b/addon/mongocrypt.cc index 70046ca..d903b2d 100644 --- a/addon/mongocrypt.cc +++ b/addon/mongocrypt.cc @@ -472,7 +472,7 @@ MongoCrypt::MongoCrypt(const CallbackInfo& info) } } - if (options.Has("cryptoCallbacks")) { + if (!mongocrypt_is_crypto_available() && options.Has("cryptoCallbacks")) { Object cryptoCallbacks = options.Get("cryptoCallbacks").ToObject(); SetCallback("aes256CbcEncryptHook", cryptoCallbacks["aes256CbcEncryptHook"]); From b57af9edd2d305bcf5ed0e8a6b1e3c7d230d150e Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 13 Jun 2024 16:37:26 -0400 Subject: [PATCH 8/9] chore: remove cryptoCallbacks flag --- test/benchmarks/bench.mjs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/benchmarks/bench.mjs b/test/benchmarks/bench.mjs index 10a9409..7e87ffb 100644 --- a/test/benchmarks/bench.mjs +++ b/test/benchmarks/bench.mjs @@ -15,8 +15,7 @@ const ERROR = 0; const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); -const { CRYPT_SHARED_LIB_PATH: cryptSharedLibPath = '', BENCH_WITHOUT_NATIVE_CRYPTO = '' } = - process.env; +const { CRYPT_SHARED_LIB_PATH: cryptSharedLibPath = '' } = process.env; const warmupSecs = 2; const testInSecs = 57; @@ -121,8 +120,7 @@ function main() { `testInSecs=${testInSecs}` ); - const mongoCryptOptions = { kmsProviders: BSON.serialize(kmsProviders) }; - if (BENCH_WITHOUT_NATIVE_CRYPTO) mongoCryptOptions.cryptoCallbacks = cryptoCallbacks; + const mongoCryptOptions = { kmsProviders: BSON.serialize(kmsProviders), cryptoCallbacks }; if (cryptSharedLibPath) mongoCryptOptions.cryptSharedLibPath = cryptSharedLibPath; const mongoCrypt = new MongoCrypt(mongoCryptOptions); From eb13cdbc66b555ea0ac30b69f50ced290b678fdd Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 13 Jun 2024 17:44:13 -0400 Subject: [PATCH 9/9] docs: update setup readme --- .github/scripts/libmongocrypt.mjs | 2 +- README.md | 26 ++++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/.github/scripts/libmongocrypt.mjs b/.github/scripts/libmongocrypt.mjs index 001b552..e402c46 100644 --- a/.github/scripts/libmongocrypt.mjs +++ b/.github/scripts/libmongocrypt.mjs @@ -200,7 +200,7 @@ export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, crypto, fastDow process.stderr.write(chunk, () => { if (isFirstStdoutDataEv) { isFirstStdoutDataEv = false; - timeout = setTimeout(() => controller.abort(), 10_000); + timeout = setTimeout(() => controller.abort(), 5_000); } timeout?.refresh(); }); diff --git a/README.md b/README.md index 50f51ca..2f87705 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,32 @@ npm install mongodb-client-encryption #### Setup -Run the following command to build libmongocrypt and setup the node bindings for development: +Run the following command to build libmongocrypt and you are setup to develop the node bindings: ```shell -bash ./etc/build-static.sh +npm run install:libmongocrypt +``` + +#### `libmongocrypt.mjs` + +``` +node libmongocrypt.mjs [--gitURL=string] [--libVersion=string] [--clean] [--build] [--no-crypto] [--fastDownload] + +By default attempts to download and compile the bindings with the crypto prebuilds of libmongocrypt. +Can be configured to clone and build without crypto. + +--gitURL=string A custom remote git repository to clone libmongocrypt from. You must also set --build to use this. +--libVersion=string A custom version reference to either download or checkout after cloning. + You may use "latest" to get current libmongocrypt `HEAD`. +--clean Combined with --build, the script will not skip cloning and rebuilding libmongocrypt. +--build Instead of downloading, clone and build libmongocrypt along with the bindings. +--no-crypto Use libmongocrypt prebuild or build libmongocrypt without OpenSSL symbols. + Requires the bindings to provide cryptoCallbacks. + +Only suitable for local development: + +--fastDownload If you are improving this script or otherwise repeatedly downloading libmongocrypt, + this flag will interrupt the un-tar operation as early as possible. It should work, most of the time. ``` #### Prebuild Platforms