Speeds up project reference build and doesnt store the result in memory (#1202)

* Output files are on disk so dont store their text in memory

* Get rid of version from output

* remove time

* Since the files are on disk, dont read them unless needed.

* add to CHANGELOG

Co-authored-by: John Reilly <[email protected]>

Author: sheetalkamat

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## v8.0.7
+* [Speeds up project reference build and doesnt store the result in memory](https://github.com/TypeStrong/ts-loader/pull/1202) - thanks @sheetalkamat
+
 ## v8.0.6
 * [Fixed further deprecation warning on webpack@5](https://github.com/TypeStrong/ts-loader/issues/1196) - thanks @appzuka
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ts-loader",
-  "version": "8.0.6",
+  "version": "8.0.7",
   "description": "TypeScript loader for webpack",
   "main": "index.js",
   "types": "dist",

src/index.ts

@@ -329,27 +329,18 @@ function updateFileInCache(
         instance.changedFilesList = true;
       }
     } else {
-      if (
-        instance.watchHost !== undefined ||
-        instance.solutionBuilderHost !== undefined
-      ) {
+      if (instance.watchHost !== undefined) {
         fileWatcherEventKind = instance.compiler.FileWatcherEventKind.Created;
       }
       file = { fileName: filePath, version: 0 };
       if (!isReferencedFile(instance, filePath)) {
         instance.files.set(key, file);
         instance.changedFilesList = true;
-      } else {
-        instance.otherFiles.set(key, file);
       }
     }
   }
 
-  if (
-    (instance.watchHost !== undefined ||
-      instance.solutionBuilderHost !== undefined) &&
-    contents === undefined
-  ) {
+  if (instance.watchHost !== undefined && contents === undefined) {
     fileWatcherEventKind = instance.compiler.FileWatcherEventKind.Deleted;
   }
 
@@ -376,8 +367,7 @@ function updateFileInCache(
     file.modifiedTime = new Date();
     instance.version++;
     if (
-      (instance.watchHost !== undefined ||
-        instance.solutionBuilderHost !== undefined) &&
+      instance.watchHost !== undefined &&
       fileWatcherEventKind === undefined
     ) {
       fileWatcherEventKind = instance.compiler.FileWatcherEventKind.Changed;
@@ -395,16 +385,6 @@ function updateFileInCache(
       instance.hasUnaccountedModifiedFiles;
   }
 
-  if (
-    instance.solutionBuilderHost !== undefined &&
-    fileWatcherEventKind !== undefined
-  ) {
-    instance.solutionBuilderHost.invokeFileWatcher(
-      filePath,
-      fileWatcherEventKind
-    );
-  }
-
   // push this file to modified files hash.
   if (!instance.modifiedFiles) {
     instance.modifiedFiles = new Map();
@@ -438,10 +418,9 @@ function getEmit(
         defFilePath.match(constants.dtsDtsxOrDtsDtsxMapRegex) &&
         // Remove the project reference d.ts as we are adding dependency for .ts later
         // This removed extra build pass (resulting in new stats object in initial build)
-        (!instance.solutionBuilderHost ||
-          !instance.solutionBuilderHost.getOutputFileKeyFromReferencedProject(
-            defFilePath
-          ))
+        !instance.solutionBuilderHost?.getOutputFileKeyFromReferencedProject(
+          defFilePath
+        )
       ) {
         addDependency(defFilePath);
       }
@@ -469,12 +448,18 @@ function getEmit(
     defFilePath =>
       path.relative(loaderContext.rootContext, defFilePath) +
       '@' +
-      (
-        instance.files.get(instance.filePathKeyMapper(defFilePath)) ||
-        instance.otherFiles.get(instance.filePathKeyMapper(defFilePath)) || {
-          version: '?',
-        }
-      ).version
+      (isReferencedFile(instance, defFilePath)
+        ? instance
+            .solutionBuilderHost!.getInputFileStamp(defFilePath)
+            .toString()
+        : (
+            instance.files.get(instance.filePathKeyMapper(defFilePath)) ||
+            instance.otherFiles.get(
+              instance.filePathKeyMapper(defFilePath)
+            ) || {
+              version: '?',
+            }
+          ).version)
   );
 
   return getOutputAndSourceMapFromOutputFiles(outputFiles);

src/instances.ts

@@ -507,7 +507,7 @@ export function buildSolutionReferences(
       { verbose: true }
     );
     solutionBuilder.build();
-    ensureAllReferences(instance);
+    instance.solutionBuilderHost.ensureAllReferenceTimestamps();
     instancesBySolutionBuilderConfigs.set(
       instance.filePathKeyMapper(instance.configFilePath!),
       instance
@@ -517,28 +517,6 @@ export function buildSolutionReferences(
   }
 }
 
-function ensureAllReferences(instance: TSInstance) {
-  // Return result from the json without errors so that the extra errors from config are digested here
-  for (const configInfo of instance.solutionBuilderHost!.configFileInfo.values()) {
-    if (!configInfo.config) {
-      continue;
-    }
-    // Load all the input files
-    configInfo.config.fileNames.forEach(file => {
-      const resolvedFileName = instance.filePathKeyMapper(file);
-      const existing = instance.otherFiles.get(resolvedFileName);
-      if (!existing) {
-        instance.otherFiles.set(resolvedFileName, {
-          fileName: path.resolve(file),
-          version: 1,
-          text: instance.compiler.sys.readFile(file),
-          modifiedTime: instance.compiler.sys.getModifiedTime!(file),
-        });
-      }
-    });
-  }
-}
-
 export function forEachResolvedProjectReference<T>(
   resolvedProjectReferences:
     | readonly (typescript.ResolvedProjectReference | undefined)[]

src/interfaces.ts

@@ -42,10 +42,11 @@ export type ResolveSync = (
   moduleName: string
 ) => string;
 
-export type Action = () => void;
-
 export interface HostMayBeCacheable {
-  clearCache?: Action;
+  clearCache?(): void;
+  fileExistsCache?: Map<string, boolean>;
+  directoryExistsCache?: Map<string, boolean>;
+  realpathCache?: Map<string, string>;
 }
 
 export interface CacheableHost extends HostMayBeCacheable {
@@ -130,39 +131,40 @@ export interface SolutionBuilderWithWatchHost
     >,
     WatchFactory {
   diagnostics: SolutionDiagnostics;
-  writtenFiles: OutputFile[];
+  writtenFiles: typescript.OutputFile[];
   configFileInfo: Map<FilePathKey, ConfigFileInfo>;
   outputAffectingInstanceVersion: Map<FilePathKey, true>;
+  getInputFileStamp(fileName: string): Date;
+  updateSolutionBuilderInputFile(fileName: string): void;
   getOutputFileKeyFromReferencedProject(
     outputFileName: string
   ): FilePathKey | undefined;
-  getOutputFileFromReferencedProject(
-    outputFileName: string
-  ): OutputFile | false | undefined;
   getOutputFileAndKeyFromReferencedProject(
     oututFileName: string
-  ): { key: FilePathKey; outputFile: OutputFile | false } | undefined;
+  ): { key: FilePathKey; outputFile: string | false } | undefined;
+  getOutputFileTextAndKeyFromReferencedProject(
+    oututFileName: string
+  ): { key: FilePathKey; text: string | undefined } | undefined;
   getInputFileNameFromOutput(outputFileName: string): string | undefined;
-  getOutputFilesFromReferencedProjectInput(inputFileName: string): OutputFile[];
+  getOutputFilesFromReferencedProjectInput(
+    inputFileName: string
+  ): typescript.OutputFile[];
   buildReferences(): void;
+  ensureAllReferenceTimestamps(): void;
   clearCache(): void;
+  close(): void;
 }
 
 export interface ConfigFileInfo {
   config: typescript.ParsedCommandLine | undefined;
   outputFileNames?: Map<
     FilePathKey,
-    { inputFileName: string; outputNames: FilePathKey[] }
+    { inputFileName: string; outputNames: string[] }
   >;
   tsbuildInfoFile?: string;
   dtsFiles?: string[];
 }
 
-export interface OutputFile extends typescript.OutputFile {
-  time: Date;
-  version: number;
-}
-
 export interface TSInstance {
   compiler: typeof typescript;
   compilerOptions: typescript.CompilerOptions;