From 88bb752966b68e4552105da9ec957ad2abe60a66 Mon Sep 17 00:00:00 2001 From: Thodoris Greasidis Date: Mon, 26 Feb 2024 18:12:13 +0200 Subject: [PATCH 1/2] push,build: Use streaming while packing the build context files Resolves: #1031 Change-type: patch --- src/utils/compose_ts.ts | 58 ++++++++++++++++++++++----------- src/utils/eol-conversion.ts | 65 ++++++++++++++++++------------------- 2 files changed, 70 insertions(+), 53 deletions(-) diff --git a/src/utils/compose_ts.ts b/src/utils/compose_ts.ts index 37653cf3ba..caf3f9c5d8 100644 --- a/src/utils/compose_ts.ts +++ b/src/utils/compose_ts.ts @@ -18,7 +18,7 @@ import { Flags } from '@oclif/core'; import type { BalenaSDK } from 'balena-sdk'; import type { TransposeOptions } from '@balena/compose/dist/emulate'; import type * as Dockerode from 'dockerode'; -import { promises as fs } from 'fs'; +import { promises as fs, createReadStream } from 'fs'; import * as yaml from 'js-yaml'; import * as _ from 'lodash'; import * as path from 'path'; @@ -30,6 +30,7 @@ import type { import type * as MultiBuild from '@balena/compose/dist/multibuild'; import * as semver from 'semver'; import type { Duplex, Readable } from 'stream'; +import { pipeline } from 'node:stream/promises'; import type { Pack } from 'tar-stream'; import { ExpectedError } from '../errors'; import type { @@ -758,34 +759,53 @@ export async function tarDirectory( const { toPosixPath } = (await import('@balena/compose/dist/multibuild')) .PathUtils; - let readFile: (file: string) => Promise; - if (process.platform === 'win32') { - const { readFileWithEolConversion } = require('./eol-conversion'); - readFile = (file) => readFileWithEolConversion(file, convertEol); - } else { - readFile = fs.readFile; - } + const getFileEolConverter = + process.platform === 'win32' + ? (await import('./eol-conversion')).getFileEolConverter + : undefined; + const tar = await import('tar-stream'); const pack = tar.pack(); const serviceDirs = await getServiceDirsFromComposition(dir, composition); const { filteredFileList, dockerignoreFiles } = await filterFilesWithDockerignore(dir, multiDockerignore, serviceDirs); printDockerignoreWarn(dockerignoreFiles, serviceDirs, multiDockerignore); - for (const fileStats of filteredFileList) { - pack.entry( - { + void (async () => { + for (const fileStats of filteredFileList) { + const entryHeader = { name: toPosixPath(fileStats.relPath), mtime: fileStats.stats.mtime, mode: fileStats.stats.mode, size: fileStats.stats.size, - }, - await readFile(fileStats.filePath), - ); - } - if (preFinalizeCallback) { - await preFinalizeCallback(pack); - } - pack.finalize(); + }; + + const eolConverter = getFileEolConverter?.(fileStats, convertEol); + if (eolConverter != null) { + pack.entry( + { + ...entryHeader, + // When we need to convertEol we can't use streaming since pack.entry() + // only supports streaming when we do know the file size upfront, and + // we can't find the final size after the conversion upfront. + size: undefined, + }, + // TODO: Consider using a tmp file/stream in a follow-up PR + // to allow streaming in this case as well. Since though we only + // convert eol for files up to LARGE_FILE_THRESHOLD (currently 10MB) + // the impact of such change is limited. + eolConverter(await fs.readFile(fileStats.filePath)), + ); + } else { + const fileReadStream = createReadStream(fileStats.filePath); + const entry = pack.entry(entryHeader); + await pipeline(fileReadStream, entry); + } + } + if (preFinalizeCallback) { + await preFinalizeCallback(pack); + } + pack.finalize(); + })(); return pack; } diff --git a/src/utils/eol-conversion.ts b/src/utils/eol-conversion.ts index 114a9e3224..b938783646 100644 --- a/src/utils/eol-conversion.ts +++ b/src/utils/eol-conversion.ts @@ -15,8 +15,8 @@ * limitations under the License. */ -import { promises as fs } from 'fs'; import Logger = require('./logger'); +import type { FileStats } from './ignore'; const globalLogger = Logger.getLogger(); @@ -64,58 +64,55 @@ export function convertEolInPlace(buf: Buffer): Buffer { } /** - * Drop-in replacement for promisified fs.readFile() - * Attempts to convert EOLs from CRLF to LF for supported encodings, - * or otherwise logs warnings. + * Returns an EOL converter from CRLF to LF for supported encodings, + * or otherwise logs warnings. The result is either undefined or an + * async iterator, so that it can be used directly in pipeline(). * @param filepath * @param convertEol When true, performs conversions, otherwise just warns. */ -export async function readFileWithEolConversion( - filepath: string, - convertEol: boolean, -): Promise { - const fileBuffer = await fs.readFile(filepath); - +export function getFileEolConverter(fileStats: FileStats, convertEol: boolean) { // Skip processing of very large files - const fileStats = await fs.stat(filepath); - if (fileStats.size > LARGE_FILE_THRESHOLD) { - globalLogger.logWarn(`CRLF detection skipped for large file: ${filepath}`); - return fileBuffer; + if (fileStats.stats.size > LARGE_FILE_THRESHOLD) { + globalLogger.logWarn( + `CRLF detection skipped for large file: ${fileStats.filePath}`, + ); + return; } - // Analyse encoding - const encoding = detectEncoding(fileBuffer); + return function (fileBuffer: Buffer) { + // Analyse encoding + const encoding = detectEncoding(fileBuffer); - // Skip further processing of non-convertible encodings - if (!CONVERTIBLE_ENCODINGS.includes(encoding)) { - return fileBuffer; - } + // Skip further processing of non-convertible encodings + if (!CONVERTIBLE_ENCODINGS.includes(encoding)) { + return fileBuffer; + } - // Skip further processing of files that don't contain CRLF - if (!fileBuffer.includes('\r\n')) { - return fileBuffer; - } + // Skip further processing of files that don't contain CRLF + if (!fileBuffer.includes('\r\n')) { + return fileBuffer; + } - if (convertEol) { - // Convert CRLF->LF - globalLogger.logInfo( - `Converting line endings CRLF -> LF for file: ${filepath}`, - ); + if (convertEol) { + // Convert CRLF->LF + globalLogger.logInfo( + `Converting line endings CRLF -> LF for file: ${fileStats.filePath}`, + ); + + return convertEolInPlace(fileBuffer); + } - return convertEolInPlace(fileBuffer); - } else { // Immediate warning globalLogger.logWarn( - `CRLF (Windows) line endings detected in file: ${filepath}`, + `CRLF (Windows) line endings detected in file: ${fileStats.filePath}`, ); // And summary warning later globalLogger.deferredLog( 'Windows-format line endings were detected in some files, but were not converted due to `--noconvert-eol` option.', Logger.Level.WARN, ); - return fileBuffer; - } + }; } /** From eb1ebddf84c09e7ec0209415c30bb7d7cc592877 Mon Sep 17 00:00:00 2001 From: Thodoris Greasidis Date: Mon, 13 Oct 2025 12:52:04 +0300 Subject: [PATCH 2/2] Deduplicate dependencies --- npm-shrinkwrap.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 082a79593c..fccd510a54 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -7052,9 +7052,9 @@ } }, "node_modules/@inquirer/core/node_modules/@types/node": { - "version": "22.18.9", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.18.9.tgz", - "integrity": "sha512-5yBtK0k/q8PjkMXbTfeIEP/XVYnz1R9qZJ3yUicdEW7ppdDJfe+MqXEhpqDL3mtn4Wvs1u0KLEG0RXzCgNpsSg==", + "version": "22.18.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.18.10.tgz", + "integrity": "sha512-anNG/V/Efn/YZY4pRzbACnKxNKoBng2VTFydVu8RRs5hQjikP8CQfaeAV59VFSCzKNp90mXiVXW2QzV56rwMrg==", "dev": true, "license": "MIT", "dependencies": { @@ -10914,9 +10914,9 @@ } }, "node_modules/@types/node": { - "version": "20.19.20", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.20.tgz", - "integrity": "sha512-2Q7WS25j4pS1cS8yw3d6buNCVJukOTeQ39bAnwR6sOJbaxvyCGebzTMypDFN82CxBLnl+lSWVdCCWbRY6y9yZQ==", + "version": "20.19.21", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.21.tgz", + "integrity": "sha512-CsGG2P3I5y48RPMfprQGfy4JPRZ6csfC3ltBZSRItG3ngggmNY/qs2uZKp4p9VbrpqNNSMzUZNFZKzgOGnd/VA==", "license": "MIT", "dependencies": { "undici-types": "~6.21.0"