From 01de260d83f58b2dfe697539c7d7cc6f741d1cb1 Mon Sep 17 00:00:00 2001 From: TamsilAmani Date: Mon, 20 May 2024 13:15:35 +0530 Subject: [PATCH 1/5] Limit VCS info to 64kb --- bin/helpers/constants.js | 12 +++++++- bin/helpers/helper.js | 66 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/bin/helpers/constants.js b/bin/helpers/constants.js index a23a05c7..1de5d8a1 100644 --- a/bin/helpers/constants.js +++ b/bin/helpers/constants.js @@ -441,6 +441,14 @@ const CYPRESS_CONFIG_FILE_NAMES = Object.keys(CYPRESS_CONFIG_FILE_MAPPING); const CYPRESS_V10_AND_ABOVE_CONFIG_FILE_EXTENSIONS = ['js', 'ts', 'cjs', 'mjs'] +// Maximum size of VCS info which is allowed +const MAX_GIT_META_DATA_SIZE_IN_KB = 64; + +/* The value to be appended at the end if git metadata is larger than +MAX_GIT_META_DATA_SIZE_IN_KB +*/ +const GIT_META_DATA_TRUNCATED = '...[TRUNCATED]'; + const turboScaleObj = {}; module.exports = Object.freeze({ @@ -475,5 +483,7 @@ module.exports = Object.freeze({ CYPRESS_V10_AND_ABOVE_TYPE, CYPRESS_CONFIG_FILE_MAPPING, CYPRESS_CONFIG_FILE_NAMES, - CYPRESS_V10_AND_ABOVE_CONFIG_FILE_EXTENSIONS + CYPRESS_V10_AND_ABOVE_CONFIG_FILE_EXTENSIONS, + MAX_GIT_META_DATA_SIZE_IN_KB, + GIT_META_DATA_TRUNCATED }); diff --git a/bin/helpers/helper.js b/bin/helpers/helper.js index de5e7825..dceb9ccf 100644 --- a/bin/helpers/helper.js +++ b/bin/helpers/helper.js @@ -18,6 +18,7 @@ const glob = require('glob'); const pGitconfig = promisify(gitconfig); const { readCypressConfigFile } = require('./readCypressConfigUtil'); const CrashReporter = require('../testObservability/crashReporter'); +const { MAX_GIT_META_DATA_SIZE_IN_KB, GIT_META_DATA_TRUNCATED } = require('./constants') exports.debug = (text, shouldReport = false, throwable = null) => { if (process.env.BROWSERSTACK_OBSERVABILITY_DEBUG === "true" || process.env.BROWSERSTACK_OBSERVABILITY_DEBUG === "1") { @@ -119,7 +120,7 @@ exports.getGitMetaData = () => { const { remote } = await pGitconfig(info.commonGitDir); const remotes = Object.keys(remote).map(remoteName => ({name: remoteName, url: remote[remoteName]['url']})); - resolve({ + let gitMetaData = { "name": "git", "sha": info["sha"], "short_sha": info["abbreviatedSha"], @@ -136,7 +137,11 @@ exports.getGitMetaData = () => { "last_tag": info["lastTag"], "commits_since_last_tag": info["commitsSinceLastTag"], "remotes": remotes - }); + }; + + gitMetaData = this.checkAndTruncateVCSInfo(gitMetaData); + + resolve(gitMetaData); } catch(e) { exports.debug(`Exception in populating Git Metadata with error : ${e}`, true, e); logger.debug(`Exception in populating Git Metadata with error : ${e}`, true, e); @@ -146,7 +151,7 @@ exports.getGitMetaData = () => { } else { const { remote } = await pGitconfig(info.commonGitDir); const remotes = Object.keys(remote).map(remoteName => ({name: remoteName, url: remote[remoteName]['url']})); - resolve({ + let gitMetaData = { "name": "git", "sha": info["sha"], "short_sha": info["abbreviatedSha"], @@ -163,7 +168,11 @@ exports.getGitMetaData = () => { "last_tag": info["lastTag"], "commits_since_last_tag": info["commitsSinceLastTag"], "remotes": remotes - }); + }; + + gitMetaData = this.checkAndTruncateVCSInfo(gitMetaData); + + resolve(gitMetaData); } } catch(err) { exports.debug(`Exception in populating Git metadata with error : ${err}`, true, err); @@ -387,3 +396,52 @@ exports.getSupportFiles = (bsConfig, isA11y) => { cleanupParams: Object.keys(cleanupParams).length ? cleanupParams : null }; } + +exports.checkAndTruncateVCSInfo = (gitMetaData) => { + const gitMetaDataSizeInKb = this.getSizeOfJsonObjectInKb(gitMetaData); + + if (gitMetaDataSizeInKb && gitMetaDataSizeInKb > 0 && gitMetaDataSizeInKb > MAX_GIT_META_DATA_SIZE_IN_KB) { + const truncateSize = gitMetaDataSizeInKb - MAX_GIT_META_DATA_SIZE_IN_KB; + const truncatedCommitMessage = this.truncateString(gitMetaData.commit_message, truncateSize); + gitMetaData.commit_message = truncatedCommitMessage; + exports.debug('The commit has been truncated'); + logger.debug('The commit has been truncated'); + } + + return gitMetaData; +}; + +exports.getSizeOfJsonObjectInKb = (jsonData) => { + try { + if (jsonData) { + const buffer = Buffer.from(JSON.stringify(jsonData)); + + return Math.floor(buffer.length/1024); + } + } catch (error) { + exports.debug(`Something went wrong while calculating size of JSON object: ${error}`, true, error); + logger.debug(`Something went wrong while calculating size of JSON object: ${error}`, true, error); + } + + return -1; +}; + +exports.truncateString = (field, truncateSizeInKb) => { + try { + const bufferSizeInBytes = Buffer.from(GIT_META_DATA_TRUNCATED).length; + + const fieldBufferObj = Buffer.from(field); + const lenOfFieldBufferObj = fieldBufferObj.length; + const finalLen = Math.round(lenOfFieldBufferObj - (truncateSizeInKb * 1024) - (bufferSizeInBytes)); + if (finalLen > 0) { + const truncatedString = fieldBufferObj.subarray(0, finalLen).toString() + GIT_META_DATA_TRUNCATED; + + return truncatedString; + } + } catch (error) { + exports.debug(`Error while truncating field, nothing was truncated here: ${error}`, true, error); + logger.debug(`Error while truncating field, nothing was truncated here: ${error}`, true, error); + } + + return field; +}; \ No newline at end of file From 076bf3b3fe230c22f9661a4f6167890ab7362702 Mon Sep 17 00:00:00 2001 From: TamsilAmani Date: Mon, 20 May 2024 15:48:27 +0530 Subject: [PATCH 2/5] Add Object check and log info for new truncated size --- bin/helpers/helper.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bin/helpers/helper.js b/bin/helpers/helper.js index dceb9ccf..b66de299 100644 --- a/bin/helpers/helper.js +++ b/bin/helpers/helper.js @@ -404,8 +404,7 @@ exports.checkAndTruncateVCSInfo = (gitMetaData) => { const truncateSize = gitMetaDataSizeInKb - MAX_GIT_META_DATA_SIZE_IN_KB; const truncatedCommitMessage = this.truncateString(gitMetaData.commit_message, truncateSize); gitMetaData.commit_message = truncatedCommitMessage; - exports.debug('The commit has been truncated'); - logger.debug('The commit has been truncated'); + logger.info(`The commit has been truncated. Size of commit after truncation is ${ this.getSizeOfJsonObjectInKb(gitMetaData) }`); } return gitMetaData; @@ -413,7 +412,7 @@ exports.checkAndTruncateVCSInfo = (gitMetaData) => { exports.getSizeOfJsonObjectInKb = (jsonData) => { try { - if (jsonData) { + if (jsonData && jsonData instanceof Object) { const buffer = Buffer.from(JSON.stringify(jsonData)); return Math.floor(buffer.length/1024); From 34cf476247fe184c4edcaafa1d5db7884f3a7263 Mon Sep 17 00:00:00 2001 From: TamsilAmani Date: Thu, 23 May 2024 09:25:50 +0530 Subject: [PATCH 3/5] Fix: use math.ceil, calculations in bytes --- bin/helpers/constants.js | 6 +++--- bin/helpers/helper.js | 32 +++++++++++++++----------------- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/bin/helpers/constants.js b/bin/helpers/constants.js index 1de5d8a1..01027404 100644 --- a/bin/helpers/constants.js +++ b/bin/helpers/constants.js @@ -442,10 +442,10 @@ const CYPRESS_CONFIG_FILE_NAMES = Object.keys(CYPRESS_CONFIG_FILE_MAPPING); const CYPRESS_V10_AND_ABOVE_CONFIG_FILE_EXTENSIONS = ['js', 'ts', 'cjs', 'mjs'] // Maximum size of VCS info which is allowed -const MAX_GIT_META_DATA_SIZE_IN_KB = 64; +const MAX_GIT_META_DATA_SIZE_IN_BYTES = 65536; /* The value to be appended at the end if git metadata is larger than -MAX_GIT_META_DATA_SIZE_IN_KB +MAX_GIT_META_DATA_SIZE_IN_BYTES */ const GIT_META_DATA_TRUNCATED = '...[TRUNCATED]'; @@ -484,6 +484,6 @@ module.exports = Object.freeze({ CYPRESS_CONFIG_FILE_MAPPING, CYPRESS_CONFIG_FILE_NAMES, CYPRESS_V10_AND_ABOVE_CONFIG_FILE_EXTENSIONS, - MAX_GIT_META_DATA_SIZE_IN_KB, + MAX_GIT_META_DATA_SIZE_IN_BYTES, GIT_META_DATA_TRUNCATED }); diff --git a/bin/helpers/helper.js b/bin/helpers/helper.js index b66de299..ae3b8401 100644 --- a/bin/helpers/helper.js +++ b/bin/helpers/helper.js @@ -18,7 +18,7 @@ const glob = require('glob'); const pGitconfig = promisify(gitconfig); const { readCypressConfigFile } = require('./readCypressConfigUtil'); const CrashReporter = require('../testObservability/crashReporter'); -const { MAX_GIT_META_DATA_SIZE_IN_KB, GIT_META_DATA_TRUNCATED } = require('./constants') +const { MAX_GIT_META_DATA_SIZE_IN_BYTES, GIT_META_DATA_TRUNCATED } = require('./constants') exports.debug = (text, shouldReport = false, throwable = null) => { if (process.env.BROWSERSTACK_OBSERVABILITY_DEBUG === "true" || process.env.BROWSERSTACK_OBSERVABILITY_DEBUG === "1") { @@ -139,7 +139,7 @@ exports.getGitMetaData = () => { "remotes": remotes }; - gitMetaData = this.checkAndTruncateVCSInfo(gitMetaData); + gitMetaData = exports.checkAndTruncateVCSInfo(gitMetaData); resolve(gitMetaData); } catch(e) { @@ -170,7 +170,7 @@ exports.getGitMetaData = () => { "remotes": remotes }; - gitMetaData = this.checkAndTruncateVCSInfo(gitMetaData); + gitMetaData = exports.checkAndTruncateVCSInfo(gitMetaData); resolve(gitMetaData); } @@ -398,49 +398,47 @@ exports.getSupportFiles = (bsConfig, isA11y) => { } exports.checkAndTruncateVCSInfo = (gitMetaData) => { - const gitMetaDataSizeInKb = this.getSizeOfJsonObjectInKb(gitMetaData); + const gitMetaDataSizeInBytes = exports.getSizeOfJsonObjectInBytes(gitMetaData); - if (gitMetaDataSizeInKb && gitMetaDataSizeInKb > 0 && gitMetaDataSizeInKb > MAX_GIT_META_DATA_SIZE_IN_KB) { - const truncateSize = gitMetaDataSizeInKb - MAX_GIT_META_DATA_SIZE_IN_KB; - const truncatedCommitMessage = this.truncateString(gitMetaData.commit_message, truncateSize); + if (gitMetaDataSizeInBytes && gitMetaDataSizeInBytes > MAX_GIT_META_DATA_SIZE_IN_BYTES) { + const truncateSize = gitMetaDataSizeInBytes - MAX_GIT_META_DATA_SIZE_IN_BYTES; + const truncatedCommitMessage = exports.truncateString(gitMetaData.commit_message, truncateSize); gitMetaData.commit_message = truncatedCommitMessage; - logger.info(`The commit has been truncated. Size of commit after truncation is ${ this.getSizeOfJsonObjectInKb(gitMetaData) }`); + logger.info(`The commit has been truncated. Size of commit after truncation is ${ exports.getSizeOfJsonObjectInBytes(gitMetaData) / 1024} KB`); } return gitMetaData; }; -exports.getSizeOfJsonObjectInKb = (jsonData) => { +exports.getSizeOfJsonObjectInBytes = (jsonData) => { try { if (jsonData && jsonData instanceof Object) { const buffer = Buffer.from(JSON.stringify(jsonData)); - return Math.floor(buffer.length/1024); + return buffer.length; } } catch (error) { - exports.debug(`Something went wrong while calculating size of JSON object: ${error}`, true, error); - logger.debug(`Something went wrong while calculating size of JSON object: ${error}`, true, error); + logger.debug(`Something went wrong while calculating size of JSON object: ${error}`); } return -1; }; -exports.truncateString = (field, truncateSizeInKb) => { +exports.truncateString = (field, truncateSizeInBytes) => { try { const bufferSizeInBytes = Buffer.from(GIT_META_DATA_TRUNCATED).length; const fieldBufferObj = Buffer.from(field); const lenOfFieldBufferObj = fieldBufferObj.length; - const finalLen = Math.round(lenOfFieldBufferObj - (truncateSizeInKb * 1024) - (bufferSizeInBytes)); + const finalLen = Math.ceil(lenOfFieldBufferObj - truncateSizeInBytes - bufferSizeInBytes); if (finalLen > 0) { const truncatedString = fieldBufferObj.subarray(0, finalLen).toString() + GIT_META_DATA_TRUNCATED; return truncatedString; } } catch (error) { - exports.debug(`Error while truncating field, nothing was truncated here: ${error}`, true, error); - logger.debug(`Error while truncating field, nothing was truncated here: ${error}`, true, error); + logger.debug(`Error while truncating field, nothing was truncated here: ${error}`); } return field; -}; \ No newline at end of file +}; From dc926475c8a3e4d176b0c6f4d12194e91dfd2967 Mon Sep 17 00:00:00 2001 From: TamsilAmani Date: Thu, 23 May 2024 14:28:34 +0530 Subject: [PATCH 4/5] modify constant value --- bin/helpers/constants.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/helpers/constants.js b/bin/helpers/constants.js index 01027404..deba24db 100644 --- a/bin/helpers/constants.js +++ b/bin/helpers/constants.js @@ -442,7 +442,7 @@ const CYPRESS_CONFIG_FILE_NAMES = Object.keys(CYPRESS_CONFIG_FILE_MAPPING); const CYPRESS_V10_AND_ABOVE_CONFIG_FILE_EXTENSIONS = ['js', 'ts', 'cjs', 'mjs'] // Maximum size of VCS info which is allowed -const MAX_GIT_META_DATA_SIZE_IN_BYTES = 65536; +const MAX_GIT_META_DATA_SIZE_IN_BYTES = 64 * 1024; /* The value to be appended at the end if git metadata is larger than MAX_GIT_META_DATA_SIZE_IN_BYTES From 798bf97c72b556e5c226b651785b683f79df9aaf Mon Sep 17 00:00:00 2001 From: TamsilAmani Date: Thu, 23 May 2024 18:16:28 +0530 Subject: [PATCH 5/5] Remove variable use --- bin/helpers/helper.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/helpers/helper.js b/bin/helpers/helper.js index ae3b8401..8a985829 100644 --- a/bin/helpers/helper.js +++ b/bin/helpers/helper.js @@ -402,8 +402,7 @@ exports.checkAndTruncateVCSInfo = (gitMetaData) => { if (gitMetaDataSizeInBytes && gitMetaDataSizeInBytes > MAX_GIT_META_DATA_SIZE_IN_BYTES) { const truncateSize = gitMetaDataSizeInBytes - MAX_GIT_META_DATA_SIZE_IN_BYTES; - const truncatedCommitMessage = exports.truncateString(gitMetaData.commit_message, truncateSize); - gitMetaData.commit_message = truncatedCommitMessage; + gitMetaData.commit_message = exports.truncateString(gitMetaData.commit_message, truncateSize); logger.info(`The commit has been truncated. Size of commit after truncation is ${ exports.getSizeOfJsonObjectInBytes(gitMetaData) / 1024} KB`); }