Resolves #379: add metalsmith.watch option setter and watcher

Author: webketje

lib/index.js

@@ -3,6 +3,8 @@
 const assert = require('assert')
 const Mode = require('stat-mode')
 const path = require('path')
+const watcher = require('./watcher')
+
 const {
   readdir,
   batchAsync,
@@ -26,7 +28,9 @@ const { Debugger, fileLogHandler } = require('./debug')
 
 const symbol = {
   env: Symbol('env'),
-  log: Symbol('log')
+  log: Symbol('log'),
+  watch: Symbol('watch'),
+  closeWatcher: Symbol('closeWatcher')
 }
 
 /**
@@ -109,6 +113,9 @@ module.exports = Metalsmith
  * @return {Metalsmith}
  */
 
+/**
+ * @constructor
+ */
 function Metalsmith(directory) {
   if (!(this instanceof Metalsmith)) return new Metalsmith(directory)
   assert(directory, 'You must pass a working directory path.')
@@ -131,6 +138,16 @@ function Metalsmith(directory) {
     enumerable: false,
     writable: true
   })
+  Object.defineProperty(this, symbol.watch, {
+    value: false,
+    enumerable: false,
+    writable: true
+  })
+  Object.defineProperty(this, symbol.closeWatcher, {
+    value: null,
+    enumerable: false,
+    writable: true
+  })
 }
 
 /**
@@ -412,20 +429,22 @@ Metalsmith.prototype.build = function (callback) {
           if (this[symbol.log].pending) {
             this[symbol.log].on('ready', () => resolve())
           } else {
-            /* istanbul ignore next */
             resolve()
           }
         })
       }
     })
-    .then(this.process.bind(this))
-    .then((files) => {
-      return this.write(files).then(() => {
-        if (this[symbol.log]) this[symbol.log].end()
-        return files
+    .then(
+      this.process.bind(this, (err, files) => {
+        if (err) throw err
+        return this.write(files)
+          .then(() => {
+            if (this[symbol.log]) this[symbol.log].end()
+            if (isFunction(callback)) callback(null, files)
+          })
+          .catch(callback)
       })
-    })
-
+    )
   /* block required for Metalsmith 2.x callback-flow compat */
   if (isFunction(callback)) {
     result.then((files) => callback(null, files), callback)
@@ -434,6 +453,65 @@ Metalsmith.prototype.build = function (callback) {
   }
 }
 
+/**
+ * **EXPERIMENTAL — Caution**
+ * * not to be used with @metalsmith/metadata <= 0.2.0: a bug may trigger an infinite loop
+ * * not to be used with existing watch plugins
+ * * metalsmith.process/build are **not awaitable** when watching is enabled.
+ *   Instead of running once at the build's end, callbacks passed to these methods will run on every rebuild.
+ *
+ * Set the list of paths to watch and trigger rebuilds on. The watch method will skip files ignored with `metalsmith.ignore()`
+ * and will do partial (true) or full (false) rebuilds depending on the `metalsmith.clean()` setting.
+ * It can be used both for rebuilding in-memory with `metalsmith.process` or writing to file system with `metalsmith.build`,
+ * @method Metalsmith#watch
+ * @param {boolean|string|string[]} [paths]
+ * @return {Metalsmith|Promise<void>|boolean|import('chokidar').WatchOptions}
+ * @example
+ *
+ * metalsmith
+ *   .ignore(['wont-be-watched'])  // ignored
+ *   .clean(false)                 // do partial rebuilds
+ *   .watch(true)                  // watch all files in metalsmith.source()
+ *   .watch(['lib','src'])         // or watch files in directories 'lib' and 'src'
+ *
+ * if (process.argv[2] === '--dry-run') {
+ *   metalsmith.process(onRebuild) // reprocess in memory without writing to disk
+ * } else {
+ *   metalsmith.build(onRebuild)   // rewrite to disk
+ * }
+ *
+ * function onRebuild(err, files) {
+ *   if (err) {
+ *     metalsmith.watch(false)            // stop watching
+ *      .finally(() => console.log(err))  // and log build error
+ *   }
+ *   console.log('reprocessed files', Object.keys(files).join(', ')))
+ * }
+ */
+Metalsmith.prototype.watch = function (options) {
+  if (isUndefined(options)) return this[symbol.watch]
+  if (!options) {
+    // if watch has previously been enabled and is now passed false, close the watcher
+    this[symbol.watch] = false
+    if (options === false && typeof this[symbol.closeWatcher] === 'function') {
+      return this[symbol.closeWatcher]()
+    }
+  } else {
+    if (isString(options) || Array.isArray(options)) options = { paths: options }
+    else if (options === true) options = { paths: this.source() }
+
+    this[symbol.watch] = {
+      paths: options.paths,
+      alwaysStat: false,
+      cwd: this.directory(),
+      ignored: this.ignore(),
+      ignoreInitial: true,
+      awaitWriteFinish: true
+    }
+  }
+  return this
+}
+
 /**
  * Process files through plugins without writing out files.
  *
@@ -450,15 +528,24 @@ Metalsmith.prototype.build = function (callback) {
  */
 
 Metalsmith.prototype.process = function (callback) {
-  const result = this.read(this.source()).then((files) => {
-    return this.run(files, this.plugins)
-  })
+  const result = this.read(this.source())
 
-  /* block required for Metalsmith 2.x callback-flow compat */
-  if (callback) {
-    result.then((files) => callback(null, files), callback)
+  if (this.watch()) {
+    return result.then((files) => {
+      const msWatcher = watcher(files, this)
+      msWatcher(this[symbol.watch], callback).then((close) => {
+        this[symbol.closeWatcher] = close
+      })
+    })
   } else {
-    return result
+    result.then((files) => this.run(files, this.plugins))
+
+    /* block required for Metalsmith 2.x callback-flow compat */
+    if (callback) {
+      result.then((files) => callback(null, files), callback)
+    } else {
+      return result
+    }
   }
 }
 

lib/watcher.js

@@ -0,0 +1,151 @@
+const chokidar = require('chokidar')
+// to be replaced in distant future by native structuredClone when dropping Node <17 support
+const cloneDeep = require('lodash.clonedeep')
+const crypto = require('crypto')
+const { relative } = require('path')
+const { rm } = require('./helpers')
+
+function sourceRelPath(p, ms) {
+  return relative(ms.source(), ms.path(p))
+}
+function isInSource(p) {
+  return !p.startsWith('..')
+}
+
+function computeHashMap(files) {
+  return Object.entries(files).reduce((hashes, [path, file]) => {
+    hashes[path] = crypto.createHash('md5').update(file.contents).digest('hex')
+    return hashes
+  }, {})
+}
+
+/**
+ * @type {Object<string, string>} HashMap
+ */
+
+/**
+ * Return the keys of `map1` that are different from `map2`
+ * @param {HashMap} map1
+ * @param {HashMap} map2
+ * @returns {Array}
+ */
+function diffHashMap(map1, map2) {
+  return Object.keys(map1).filter((path) => map1[path] !== map2[path])
+}
+
+module.exports = function watchable(files, metalsmith) {
+  const clean = metalsmith.clean()
+  const meta = metalsmith.metadata()
+  const fileCache = files
+  let lastHashmap
+
+  function rerun() {
+    return metalsmith.metadata(meta).run(cloneDeep(fileCache), metalsmith.plugins)
+  }
+
+  function transformFilesObj(evt, p, metalsmith) {
+    // we only care about in-source files & dirs to update the fileCache
+    // other files are eventually added or processed by plugins
+    let filesTransform = Promise.resolve()
+    const relPath = sourceRelPath(p, metalsmith)
+
+    if (isInSource(relPath)) {
+      switch (evt) {
+        case 'unlinkDir':
+          metalsmith.match(relPath, Object.keys(fileCache)).forEach((r) => delete fileCache[r])
+          break
+        case 'unlink':
+          delete fileCache[relPath]
+          break
+        case 'add':
+        case 'change':
+          filesTransform = metalsmith.readFile(metalsmith.path(p)).then((file) => {
+            fileCache[relPath] = file
+          })
+          break
+      }
+    }
+
+    return filesTransform
+  }
+
+  return function watcher({ paths, ...options }, onRebuild) {
+    const watcher = chokidar.watch(paths || metalsmith.source(), options)
+
+    const eventqueue = []
+    // eslint-disable-next-line no-unused-vars
+    let inTheMiddleOfABuild = false
+    let run
+
+    watcher.on('all', (evt, p) => {
+      // eslint-disable-next-line no-console
+      console.log(evt, p)
+
+      // the metalsmith Files object does not output empty dirs,
+      // wait for the file add/change events instead
+      if (evt === 'addDir') return
+
+      eventqueue.push([evt, p])
+
+      clearTimeout(run)
+      run = setTimeout(() => {
+        inTheMiddleOfABuild = true
+        const fileTransforms = Promise.all(eventqueue.map(([evt, p]) => transformFilesObj(evt, p, metalsmith)))
+
+        fileTransforms.then(() => {
+          eventqueue.splice(0, eventqueue.length)
+          const latestRun = rerun()
+
+          if (clean) {
+            latestRun
+              .then(
+                (files) => onRebuild(null, files),
+                (err) => onRebuild(err)
+              )
+              .finally(() => {
+                inTheMiddleOfABuild = false
+              })
+            return
+          }
+
+          latestRun.then(
+            (files) => {
+              const newHashMap = computeHashMap(files)
+              const changedOrRemoved = diffHashMap(lastHashmap, newHashMap)
+              const addedFiles = diffHashMap(newHashMap, lastHashmap).filter((p) => !changedOrRemoved.includes(p))
+              const removedFiles = changedOrRemoved.filter((f) => !files[f])
+              const changedFiles = changedOrRemoved.filter((f) => !!files[f])
+              const output = [...addedFiles, ...changedFiles].reduce((all, current) => {
+                all[current] = files[current]
+                return all
+              }, {})
+              lastHashmap = newHashMap
+              // eslint-disable-next-line no-console
+              console.log({ addedFiles, removedFiles, changedFiles })
+
+              Promise.all(removedFiles.map((f) => rm(f)))
+                .then(() => onRebuild(null, output), onRebuild)
+                .finally(() => {
+                  inTheMiddleOfABuild = false
+                })
+            },
+            (err) => onRebuild(err)
+          )
+        })
+      }, 1000)
+    })
+    return new Promise((resolve, reject) => {
+      rerun()
+        .then((files) => {
+          if (!clean) lastHashmap = computeHashMap(files)
+          watcher.on('ready', () => {
+            onRebuild(null, files)
+            resolve(function closeWatcher() {
+              return watcher.unwatch(paths).close()
+            })
+          })
+        })
+        .catch(reject)
+    })
+  }
+}