ember-fastboot · SparshithNR · Feb 19, 2020 · Feb 24, 2020 · Feb 25, 2020
diff --git a/README.md b/README.md
@@ -436,6 +436,20 @@ use the rehydration builder and your application will be using rehydration.
 
 Rehydration is only compatible with fastboot > 1.1.4-beta.1, and Ember.js > 3.2.
 
+### Update manifest from HTML
+
+This is an opt-in feature that allows updating app files information in fastboot manifest from `index.html`. Enabling this feature is required when using Embroider for building the application. We can opt-in into this feature by setting the following environmental variable:
+
+```
+FASTBOOT_HTML_MANIFEST=true
+```
+
+If we decide certain scirpt tags to be ignored in fastboot and shouldn't be loaded into fastboot sandbox, we need to use `data-fastboot-ignore` in script tag.
+Example:
+```
+<script src="fastboot-ignore.js" data-fastboot-ignore></script>
+```
+
 ## Build Hooks for FastBoot
 
 ### Disabling incompatible dependencies

diff --git a/index.js b/index.js
@@ -20,6 +20,7 @@ const Funnel = require('broccoli-funnel');
 const p = require('ember-cli-preprocess-registry/preprocessors');
 const fastbootTransform = require('fastboot-transform');
 const existsSync = fs.existsSync;
+const updateManifestFromHtml = require('./lib/embroider/update-manifest-from-html');
 
 let checker;
 function getVersionChecker(context) {
@@ -346,6 +347,13 @@ module.exports = {
   },
 
   postBuild(result) {
+    // Need to update manifest from html file.
+    // Set environment variable `FASTBOOT_HTML_MANIFEST` to `true`
+    // Usage `FASTBOOT_HTML_MANIFEST=true ember s`
+    if(process.env.FASTBOOT_HTML_MANIFEST === 'true') {
+      updateManifestFromHtml(result.directory);
+    }
+
     if (this.fastboot) {
       // should we reload fastboot if there are only css changes? Seems it maynot be needed.
       // TODO(future): we can do a smarter reload here by running fs-tree-diff on files loaded

diff --git a/lib/embroider/update-manifest-from-html.js b/lib/embroider/update-manifest-from-html.js
@@ -0,0 +1,61 @@
+'use strict';
+
+const fs = require('fs');
+const findScriptSrcs = require('find-scripts-srcs-in-document');
+const getScriptFileName = require('../utilities/get-script-file-name');
+
+module.exports = function updateManifestFromHtml(root) {
+  const pkg = JSON.parse(fs.readFileSync(root + '/package.json', 'UTF8'));
+  const htmlFileName = pkg.fastboot.manifest.htmlFile || 'index.html';
+
+  // otherwise, we must parse the index.html file to figure out what's next
+  const indexHtml = fs.readFileSync(root + '/' + htmlFileName, 'UTF8')
+  // TODO: fix simple-html-tokenizer: https://github.com/tildeio/simple-html-tokenizer/pull/71
+    .replace('<!DOCTYPE html>','');
+
+  const assetsDir = `assets/`;
+  const vendorFiles = pkg.fastboot.manifest.vendorFiles || [];
+
+  // Find script src which are not data-fastboot-ignore
+  // data-fastboot-ignore in the script tag is added by the developer
+  // These are those script files which shouldn't be loaded into fastboot sanbox.
+  const indexScriptFilePaths = findScriptSrcs(indexHtml, findScriptSrcs.ignoreWithAttribute('data-fastboot-ignore'))
+  .filter( src => {
+    // skipping files if they are part of vendorFiles list
+    // this would cover any vendor files (vendor or vendor-static)
+
+    // The App Files list from html in shouldn't include
+    // vendor files as they are already included in manifest.
+    // Checking if the script file from index.html
+    // is in the the list of vendor files in package.json and skip them.
+    const fileName = getScriptFileName(src);
+    return !vendorFiles.find(filePath => fileName === getScriptFileName(filePath));
+  });
+
+  /**
+   * Reading app files by parsing index.html so that FastBoot can load assets that
+   * are same as what is generated as part of the build flow and updated in
+   * index.html.
+   *
+   * Example: Embroider build flow upon WebPack as stage 3 processing in the build
+   * updates app.js to multiple chunks and updates index.html. Parsing from index.html
+   * allows loading same set of app files in FastBoot sandbox.
+   *
+   */
+  const appFiles = new Set();
+  indexScriptFilePaths.forEach(src => {
+    // Extract the file name from the src file paths and
+    // checks the file exists in the assets location on the disk.
+    const fileName = getScriptFileName(src);
+    const assetPath = `${assetsDir}${fileName}`;
+    const filePath = `${root}/${assetPath}`;
+    if(fileName && fs.existsSync(filePath)) {
+      appFiles.add(assetPath);
+    }
+  });
+
+  const fastbootFile = pkg.fastboot.manifest.appFiles.find(x => x.endsWith('-fastboot.js'));
+  appFiles.add(fastbootFile);
+  pkg.fastboot.manifest.appFiles = Array.from(appFiles);
+  fs.writeFileSync(`${root}/package.json`, JSON.stringify(pkg, null, 2));
+};
diff --git a/lib/utilities/fastboot-app-boot.js b/lib/utilities/fastboot-app-boot.js
@@ -7,7 +7,8 @@ module.exports = function fastbootAppBoot(prefix, configAppAsString, isModuleUni
   return [
     "",
     "if (typeof FastBoot === 'undefined') {",
-    "  if (!runningTests) {",
+    // In Embroider, global variable runnintTests is not set. As result we have to check if it is defined or not.
+    "  if (typeof runningTests === 'undefined' || !runningTests) {",
     "    require('{{MODULE_PREFIX}}/" + appSuffix + "')['default'].create({{CONFIG_APP}});",
     "  }",
     "}",

diff --git a/lib/utilities/get-script-file-name.js b/lib/utilities/get-script-file-name.js
@@ -0,0 +1,14 @@
+'use strict';
+// Helper to extract the script file name from a given path.
+// Used for getting the script filename from the html file, where
+// src is set to have custom context or wrapped with a placeholder.
+// Examples file paths:
+//  "%PLACEHOLDER[c.js]%"
+//  "/some_context_path/e.js"
+//  "chunk-12.js"
+// Expected output for file names: c.js, e.js, chunk-12.js
+module.exports = function getScriptFileName(filePath) {
+  const scriptFileNameRegEx = /([a-zA-Z0-9_\.\-\(\):])+(\.js)/ig;
+  const match = filePath.match(scriptFileNameRegEx);
+  return match ? match[0] : '';
+}
diff --git a/package.json b/package.json
@@ -34,6 +34,7 @@
     "fastboot": "^2.0.0",
     "fastboot-express-middleware": "^2.0.0",
     "fastboot-transform": "^0.1.3",
+    "find-scripts-srcs-in-document": "^1.0.0",
     "fs-extra": "^7.0.0",
     "json-stable-stringify": "^1.0.1",
     "md5-hex": "^2.0.0",
@@ -69,6 +70,8 @@
     "ember-source": "~3.8.0",
     "eslint-plugin-ember": "^6.0.0",
     "eslint-plugin-node": "^7.0.1",
+    "fixturify-project": "^1.9.1",
+    "fixturify": "^0.3.2",
     "glob": "^7.1.3",
     "loader.js": "^4.7.0",
     "mocha": "^5.2.0",

diff --git a/test/get-script-filename-test.js b/test/get-script-filename-test.js
@@ -0,0 +1,25 @@
+'use strict';
+const { expect } = require('chai');
+const getScriptFileName = require('../lib/utilities/get-script-file-name');
+
+describe('utilities/get-script-file-name', function() {
+  let filePaths = [
+    'a.js',
+    '"%PLACEHOLDER[c.js]%"',
+    '/some_context_path/e.js',
+    'chunk-12.js"',
+    'assets/vendor.js',
+    '/some_context_path/vendor-static.js'
+  ]
+  it('can parse different script filePaths and return script filenames', () => {
+    let fileNames = filePaths.map((filePath) => getScriptFileName(filePath));
+    expect(fileNames).to.deep.equal([
+      'a.js',
+      'c.js',
+      'e.js',
+      'chunk-12.js',
+      'vendor.js',
+      'vendor-static.js'
+    ]);
+  });
+});
diff --git a/test/update-manifest-from-html-test.js b/test/update-manifest-from-html-test.js
@@ -0,0 +1,134 @@
+'use strict';
+const { expect } = require('chai');
+const Project = require('fixturify-project');
+const fixturify = require('fixturify');
+const fs = require('fs');
+const updateManifestFromHtml = require('../lib/embroider/update-manifest-from-html')
+
+describe('embroider/update-manifest-from-html', function() {
+  const PROJECT = new Project('my-app');
+
+  PROJECT.files['index.html'] = `
+  <script src="a.js"></script>
+  <script src="b.js"></script>
+  <script src="%PLACEHOLDER[c.js]%"></script>
+  <script src="/some_context_path/e.js"></script>
+  <script src="chunk-12.js"></script>
+  <script src="chunk-12.test.js"></script>
+  <script src="fastboot-ignore.js" data-fastboot-ignore></script>
+  <script src="ember-cli-live-reload.js"></script>
+  <script src="testem.js"></script>
+  <script src="assets/vendor.js"></script>
+  <script src="assets/vendor-static.js"></script>
+  `;
+
+  PROJECT.pkg.fastboot = {
+    manifest: {
+      htmlFile: 'index.html',
+      appFiles: ['assets/app.js', 'assets/my-app-fastboot.js'],
+      vendorFiles:['assets/vendor.js', 'assets/vendor-static.js']
+    }
+  };
+  const project = PROJECT.clone();
+  project.writeSync();
+  const files = {};
+  files['a.js'] = '';
+  files['b.js'] = '';
+  files['c.js'] = '';
+  files['d.js'] = '';
+  files['e.js'] = '';
+  files['chunk-12.js'] = '';
+  files['chunk-12.test.js'] = '';
+
+  it(`works correctly as files are added/removed`,  function() {
+    fixturify.writeSync(project.root + '/my-app/assets', files);
+    updateManifestFromHtml(project.root + '/my-app');
+    project.readSync();
+
+    expect(JSON.parse(project.toJSON('package.json')).fastboot.manifest.appFiles).to.eql([
+      'assets/a.js',
+      'assets/b.js',
+      'assets/c.js',
+      'assets/e.js',
+      'assets/chunk-12.js',
+      'assets/chunk-12.test.js',
+      'assets/my-app-fastboot.js'
+    ]);
+  });
+
+  it(`Rebuild doesn't change anything if files didn't change`,  function() {
+    // let's try a rebuild
+    fixturify.writeSync(project.root + '/my-app/assets', files);
+    updateManifestFromHtml(project.root + '/my-app')
+    project.readSync();
+
+    expect(JSON.parse(project.toJSON('package.json')).fastboot.manifest.appFiles).to.eql([
+      'assets/a.js',
+      'assets/b.js',
+      'assets/c.js',
+      'assets/e.js',
+      'assets/chunk-12.js',
+      'assets/chunk-12.test.js',
+      'assets/my-app-fastboot.js'
+    ]);
+  });
+
+  it(`updates when remove a file, but leave it in the HTML`, function() {
+    // now lets remove a file, but leave it in the HTML
+    delete files['a.js'];
+    fs.unlinkSync(project.root + '/my-app/assets/a.js');
+    fixturify.writeSync(project.root + '/my-app/assets', files);
+
+    updateManifestFromHtml(project.root + '/my-app')
+    project.readSync();
+
+    expect(JSON.parse(project.toJSON('package.json')).fastboot.manifest.appFiles).to.eql([
+      'assets/b.js',
+      'assets/c.js',
+      'assets/e.js',
+      'assets/chunk-12.js',
+      'assets/chunk-12.test.js',
+      'assets/my-app-fastboot.js'
+    ]);
+  });
+
+  it(`updates when a file is added file deleted previously`, function() {
+    // lets add a file back
+    files['a.js'] = '';
+    fixturify.writeSync(project.root + '/my-app/assets', files);
+
+    updateManifestFromHtml(project.root + '/my-app')
+    project.readSync();
+
+    expect(JSON.parse(project.toJSON('package.json')).fastboot.manifest.appFiles).to.eql([
+      'assets/a.js',
+      'assets/b.js',
+      'assets/c.js',
+      'assets/e.js',
+      'assets/chunk-12.js',
+      'assets/chunk-12.test.js',
+      'assets/my-app-fastboot.js'
+    ]);
+  });
+
+  it(`updates when a new src added to html which is already on disk`, function() {
+    // lets add a new src to html which is already on disk
+
+    project.files['index.html'] += '<script src="d.js"></script>';
+
+    project.writeSync();
+    updateManifestFromHtml(project.root + '/my-app')
+    project.readSync();
+
+    expect(JSON.parse(project.toJSON('package.json')).fastboot.manifest.appFiles).to.eql([
+      'assets/a.js',
+      'assets/b.js',
+      'assets/c.js',
+      'assets/e.js',
+      'assets/chunk-12.js',
+      'assets/chunk-12.test.js',
+      'assets/d.js',
+      'assets/my-app-fastboot.js'
+    ]);
+  });
+});