From 29f70884bd300e95d0adc3fa8c57537d65995276 Mon Sep 17 00:00:00 2001 From: Sebastian Silbermann Date: Wed, 4 Mar 2020 11:30:57 +0100 Subject: [PATCH 1/5] [docs] Track bundle size of pages --- azure-pipelines.yml | 15 +++++---- scripts/sizeSnapshot/create.js | 61 ++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ea9f463816041c..c7a64a698f5cee 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,10 +1,10 @@ trigger: branches: include: - - '*' + - '*' exclude: - - l10n - - dependabot/* + - l10n + - dependabot/* pool: vmImage: 'ubuntu-latest' @@ -37,7 +37,7 @@ steps: globExpressions: '*.tgz' targetFolder: 'artifacts/$(Build.SourceBranchName)/$(Build.SourceVersion)' filesAcl: 'public-read' - displayName: "Upload distributables to S3" + displayName: 'Upload distributables to S3' condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) env: AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) @@ -49,8 +49,11 @@ steps: targetPath: 'material-ui-core.tgz' - script: | - yarn docs:build - displayName: 'build docs' + set -o pipefail + mkdir -p scripts/sizeSnapshot/build + yarn docs:build | tee scripts/sizeSnapshot/build/docs.next + set +o pipefail + displayName: 'build docs for size snapshot' - script: | yarn size:snapshot diff --git a/scripts/sizeSnapshot/create.js b/scripts/sizeSnapshot/create.js index 8ccc35c863bb50..9d52ae2306ba77 100644 --- a/scripts/sizeSnapshot/create.js +++ b/scripts/sizeSnapshot/create.js @@ -49,12 +49,73 @@ async function getWebpackSizes() { }); } +// waiting for String.prototype.matchAll in node 10 +function* matchAll(string, regex) { + let match = null; + do { + match = regex.exec(string); + if (match !== null) { + yield match; + } + } while (match !== null); +} + +/** + * Inverse to `pretty-bytes` + * + * @param {string} n + * @param {'B', 'kB' | 'MB' | 'GB' | 'TB' | 'PB'} unit + * @returns {number} + */ + +function prettyBytesInverse(n, unit) { + const metrixPrefix = unit.length < 2 ? '' : unit[0]; + const metricPrefixes = ['', 'k', 'M', 'G', 'T', 'P']; + const metrixPrefixIndex = metricPrefixes.indexOf(metrixPrefix); + if (metrixPrefixIndex === -1) { + throw new TypeError( + `unrecognized metric prefix '${metrixPrefix}' in unit '${unit}'. only '${metricPrefixes.join( + "', '", + )}' are allowed`, + ); + } + + const power = metrixPrefixIndex * 3; + return n * 10 ** power; +} + +/** + * parses output from next build to size snapshot format + * @returns {[string, { gzip: number, files: number, packages: number }][]} + */ + +async function getNextPagesSize() { + const consoleOutput = await fse.readFile(path.join(__dirname, 'build/docs.next'), { + encoding: 'utf8', + }); + const pageRegex = /^(?┌|├|└)\s+(?σ|⚡|)\s+(?[^\s]+)\s+(?[0-9.]+)\s+(?\w+)/gm; + + return Array.from(matchAll(consoleOutput, pageRegex), match => { + const { pageUrl, sizeFormatted, sizeUnit } = match.groups; + + const snapshotId = `docs:${pageUrl}`; + return [ + snapshotId, + { + parsed: prettyBytesInverse(sizeFormatted, sizeUnit), + gzip: -1, + }, + ]; + }); +} + async function run() { const rollupBundles = [path.join(workspaceRoot, 'packages/material-ui/size-snapshot.json')]; const bundleSizes = lodash.fromPairs([ ...(await getWebpackSizes()), ...lodash.flatten(await Promise.all(rollupBundles.map(getRollupSize))), + ...(await getNextPagesSize()), ]); await fse.writeJSON(snapshotDestPath, bundleSizes, { spaces: 2 }); From 464917a7b4e4c9209748b07556fe3499a2acfcf7 Mon Sep 17 00:00:00 2001 From: Sebastian Silbermann Date: Wed, 4 Mar 2020 14:33:02 +0100 Subject: [PATCH 2/5] Use next for docs.landing and docs.main --- scripts/sizeSnapshot/create.js | 9 +++++++-- scripts/sizeSnapshot/webpack.config.js | 23 ----------------------- 2 files changed, 7 insertions(+), 25 deletions(-) diff --git a/scripts/sizeSnapshot/create.js b/scripts/sizeSnapshot/create.js index 9d52ae2306ba77..66f4863efcc07a 100644 --- a/scripts/sizeSnapshot/create.js +++ b/scripts/sizeSnapshot/create.js @@ -98,7 +98,13 @@ async function getNextPagesSize() { return Array.from(matchAll(consoleOutput, pageRegex), match => { const { pageUrl, sizeFormatted, sizeUnit } = match.groups; - const snapshotId = `docs:${pageUrl}`; + let snapshotId = `docs:${pageUrl}`; + // used to be tracked with custom logic hence the different ids + if (pageUrl === '/') { + snapshotId = 'docs.main'; + } else if (pageUrl === '/_app') { + snapshotId = 'docs.main'; + } return [ snapshotId, { @@ -111,7 +117,6 @@ async function getNextPagesSize() { async function run() { const rollupBundles = [path.join(workspaceRoot, 'packages/material-ui/size-snapshot.json')]; - const bundleSizes = lodash.fromPairs([ ...(await getWebpackSizes()), ...lodash.flatten(await Promise.all(rollupBundles.map(getRollupSize))), diff --git a/scripts/sizeSnapshot/webpack.config.js b/scripts/sizeSnapshot/webpack.config.js index 6026b4b3c21e00..6a0f823fe2930d 100644 --- a/scripts/sizeSnapshot/webpack.config.js +++ b/scripts/sizeSnapshot/webpack.config.js @@ -1,4 +1,3 @@ -const fse = require('fs-extra'); const globCallback = require('glob'); const path = require('path'); const CompressionPlugin = require('compression-webpack-plugin'); @@ -9,18 +8,6 @@ const glob = promisify(globCallback); const workspaceRoot = path.join(__dirname, '..', '..'); async function getSizeLimitBundles() { - const nextDir = path.join(workspaceRoot, 'docs/.next'); - const buildId = await fse.readFile(path.join(nextDir, 'BUILD_ID'), 'utf8'); - - const dirname = path.join(nextDir, 'static/chunks'); - const [main] = (await fse.readdir(dirname)).reduce((result, filename) => { - if (filename.length === 31) { - return [...result, { path: `${dirname}/${filename}` }]; - } - - return result; - }, []); - const corePackagePath = path.join(workspaceRoot, 'packages/material-ui/build/esm'); const coreComponents = (await glob(path.join(corePackagePath, '[A-Z]*'))).map(componentPath => { const componentName = path.basename(componentPath); @@ -95,16 +82,6 @@ async function getSizeLimitBundles() { webpack: true, path: 'packages/material-ui/build/esm/useMediaQuery/index.js', }, - { - name: 'docs.main', - webpack: false, - path: path.relative(workspaceRoot, main.path), - }, - { - name: 'docs.landing', - webpack: false, - path: path.relative(workspaceRoot, path.join(nextDir, `static/${buildId}/pages/index.js`)), - }, ]; } From 7577928c0cda458765a5c52319d5b9111c5287fa Mon Sep 17 00:00:00 2001 From: Sebastian Silbermann Date: Wed, 4 Mar 2020 14:56:52 +0100 Subject: [PATCH 3/5] [ci] nest page table in pr comment --- dangerfile.js | 121 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 81 insertions(+), 40 deletions(-) diff --git a/dangerfile.js b/dangerfile.js index cfe09b018cfe4d..b886bbafaf98a6 100644 --- a/dangerfile.js +++ b/dangerfile.js @@ -97,6 +97,12 @@ function computeBundleLabel(bundleId) { if (bundleId === '@material-ui/core/Textarea') { return 'TextareaAutosize'; } + if (bundleId === 'docs.main') { + return 'docs:/_app'; + } + if (bundleId === 'docs.landing') { + return 'docs:/'; + } return bundleId.replace(/^@material-ui\/core\//, '').replace(/\.esm$/, ''); } @@ -129,6 +135,66 @@ function generateEmphasizedChange([bundle, { parsed, gzip }]) { return `**${bundle}**: parsed: ${changeParsed}, gzip: ${changeGzip}`; } +function createComparisonTable(entries) { + return generateMDTable( + [ + { label: 'bundle' }, + { label: 'Size Change', align: 'right' }, + { label: 'Size', align: 'right' }, + { label: 'Gzip Change', align: 'right' }, + { label: 'Gzip', align: 'right' }, + ], + entries + .map(([bundleId, size]) => [computeBundleLabel(bundleId), size]) + // orderBy(|parsedDiff| DESC, |gzipDiff| DESC, name ASC) + .sort(([labelA, statsA], [labelB, statsB]) => { + const compareParsedDiff = + Math.abs(statsB.parsed.absoluteDiff) - Math.abs(statsA.parsed.absoluteDiff); + const compareGzipDiff = + Math.abs(statsB.gzip.absoluteDiff) - Math.abs(statsA.gzip.absoluteDiff); + const compareName = labelA.localeCompare(labelB); + + if (compareParsedDiff === 0 && compareGzipDiff === 0) { + return compareName; + } + if (compareParsedDiff === 0) { + return compareGzipDiff; + } + return compareParsedDiff; + }) + .map(([label, { parsed, gzip }]) => { + return [ + label, + formatDiff(parsed.absoluteDiff, parsed.relativeDiff), + prettyBytes(parsed.current), + formatDiff(gzip.absoluteDiff, gzip.relativeDiff), + prettyBytes(gzip.current), + ]; + }), + ); +} + +/** + * Puts results in different buckets wh + * @param {*} results + */ +function sieveResults(results) { + const main = []; + const pages = []; + + results.forEach(entry => { + const [bundleId] = entry; + + if (bundleId.startsWith('docs:')) { + pages.push(entry); + } else { + main.push(entry); + } + }); + + return { all: results, main, pages }; +} + async function run() { // Use git locally to grab the commit which represents the place // where the branches differ @@ -145,11 +211,14 @@ async function run() { const commitRange = `${mergeBaseCommit}...${danger.github.pr.head.sha}`; const comparison = await loadComparison(mergeBaseCommit, upstreamRef); - const results = Object.entries(comparison.bundles); - const anyResultsChanges = results.filter(createComparisonFilter(1, 1)); + + const { all: allResults, main: mainResults, pages: pageResults } = sieveResults( + Object.entries(comparison.bundles), + ); + const anyResultsChanges = allResults.filter(createComparisonFilter(1, 1)); if (anyResultsChanges.length > 0) { - const importantChanges = results + const importantChanges = mainResults .filter(createComparisonFilter(parsedSizeChangeThreshold, gzipSizeChangeThreshold)) .filter(isPackageComparison) .map(generateEmphasizedChange); @@ -159,50 +228,22 @@ async function run() { markdown(importantChanges.join('\n')); } - const detailsTable = generateMDTable( - [ - { label: 'bundle' }, - { label: 'Size Change', align: 'right' }, - { label: 'Size', align: 'right' }, - { label: 'Gzip Change', align: 'right' }, - { label: 'Gzip', align: 'right' }, - ], - results - .map(([bundleId, size]) => [computeBundleLabel(bundleId), size]) - // orderBy(|parsedDiff| DESC, |gzipDiff| DESC, name ASC) - .sort(([labelA, statsA], [labelB, statsB]) => { - const compareParsedDiff = - Math.abs(statsB.parsed.absoluteDiff) - Math.abs(statsA.parsed.absoluteDiff); - const compareGzipDiff = - Math.abs(statsB.gzip.absoluteDiff) - Math.abs(statsA.gzip.absoluteDiff); - const compareName = labelA.localeCompare(labelB); - - if (compareParsedDiff === 0 && compareGzipDiff === 0) { - return compareName; - } - if (compareParsedDiff === 0) { - return compareGzipDiff; - } - return compareParsedDiff; - }) - .map(([label, { parsed, gzip }]) => { - return [ - label, - formatDiff(parsed.absoluteDiff, parsed.relativeDiff), - prettyBytes(parsed.current), - formatDiff(gzip.absoluteDiff, gzip.relativeDiff), - prettyBytes(gzip.current), - ]; - }), - ); + const mainDetailsTable = createComparisonTable(mainResults); + const pageDetailsTable = createComparisonTable(pageResults); const details = `
Details of bundle changes. +
+ Details of page changes + + ${pageDetailsTable} +
+

Comparing: ${commitRange}

- ${detailsTable} + ${mainDetailsTable}
`; From ae509446578a52522fc14db979cbaec01e6baa4c Mon Sep 17 00:00:00 2001 From: Sebastian Silbermann Date: Wed, 4 Mar 2020 15:22:36 +0100 Subject: [PATCH 4/5] use page link for bundle label in pages table --- dangerfile.js | 52 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/dangerfile.js b/dangerfile.js index b886bbafaf98a6..e8c619591a27c3 100644 --- a/dangerfile.js +++ b/dangerfile.js @@ -90,22 +90,6 @@ function formatDiff(absoluteChange, relativeChange) { )})`; } -function computeBundleLabel(bundleId) { - if (bundleId === 'packages/material-ui/build/umd/material-ui.production.min.js') { - return '@material-ui/core[umd]'; - } - if (bundleId === '@material-ui/core/Textarea') { - return 'TextareaAutosize'; - } - if (bundleId === 'docs.main') { - return 'docs:/_app'; - } - if (bundleId === 'docs.landing') { - return 'docs:/'; - } - return bundleId.replace(/^@material-ui\/core\//, '').replace(/\.esm$/, ''); -} - /** * Generates a Markdown table * @param {{ label: string, align: 'left' | 'center' | 'right'}[]} headers @@ -135,7 +119,15 @@ function generateEmphasizedChange([bundle, { parsed, gzip }]) { return `**${bundle}**: parsed: ${changeParsed}, gzip: ${changeGzip}`; } -function createComparisonTable(entries) { +/** + * + * @param {[string, object][]} entries + * @param {object} options + * @param {function (string): string} options.computeBundleLabel + */ +function createComparisonTable(entries, options) { + const { computeBundleLabel } = options; + return generateMDTable( [ { label: 'bundle' }, @@ -228,8 +220,30 @@ async function run() { markdown(importantChanges.join('\n')); } - const mainDetailsTable = createComparisonTable(mainResults); - const pageDetailsTable = createComparisonTable(pageResults); + const mainDetailsTable = createComparisonTable(mainResults, { + computeBundleLabel: bundleId => { + if (bundleId === 'packages/material-ui/build/umd/material-ui.production.min.js') { + return '@material-ui/core[umd]'; + } + if (bundleId === '@material-ui/core/Textarea') { + return 'TextareaAutosize'; + } + if (bundleId === 'docs.main') { + return 'docs:/_app'; + } + if (bundleId === 'docs.landing') { + return 'docs:/'; + } + return bundleId.replace(/^@material-ui\/core\//, '').replace(/\.esm$/, ''); + }, + }); + const pageDetailsTable = createComparisonTable(pageResults, { + computeBundleLabel: bundleId => { + const host = `https://deploy-preview-${danger.github.pr.number}--material-ui.netlify.com`; + const page = bundleId.replace(/^docs:/, ''); + return `[${page}](${host}${page})`; + }, + }); const details = `
From 5531b59cdcbc65ada37727da8b95492c210fbe78 Mon Sep 17 00:00:00 2001 From: Sebastian Silbermann Date: Wed, 4 Mar 2020 15:41:23 +0100 Subject: [PATCH 5/5] Move commit range to top --- dangerfile.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dangerfile.js b/dangerfile.js index e8c619591a27c3..cb4d2edd125215 100644 --- a/dangerfile.js +++ b/dangerfile.js @@ -249,14 +249,14 @@ async function run() {
Details of bundle changes. +

Comparing: ${commitRange}

+
Details of page changes ${pageDetailsTable}
-

Comparing: ${commitRange}

- ${mainDetailsTable}
`;