From 90088bd5f817e6d50a76c428671589f01875b405 Mon Sep 17 00:00:00 2001
From: Eric Allam <eallam@icloud.com>
Date: Wed, 23 Jul 2025 14:02:38 +0100
Subject: [PATCH 1/2] Gracefully shutdown task run processes

---
 .changeset/early-points-jam.md                |  5 +++++
 .../cli-v3/src/executions/taskRunProcess.ts   | 19 ++++++++++++++++---
 .../src/indexing/indexWorkerManifest.ts       |  8 ++++----
 3 files changed, 25 insertions(+), 7 deletions(-)
 create mode 100644 .changeset/early-points-jam.md

diff --git a/.changeset/early-points-jam.md b/.changeset/early-points-jam.md
new file mode 100644
index 0000000000..645a50002f
--- /dev/null
+++ b/.changeset/early-points-jam.md
@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Gracefully shutdown task run processes using SIGTERM followed by SIGKILL after a 1s timeout. This also prevents cancelled or completed runs from leaving orphaned Ttask run processes behind
diff --git a/packages/cli-v3/src/executions/taskRunProcess.ts b/packages/cli-v3/src/executions/taskRunProcess.ts
index 74aecededf..af0872bfe6 100644
--- a/packages/cli-v3/src/executions/taskRunProcess.ts
+++ b/packages/cli-v3/src/executions/taskRunProcess.ts
@@ -51,6 +51,7 @@ export type TaskRunProcessOptions = {
   machineResources: MachinePresetResources;
   isWarmStart?: boolean;
   cwd?: string;
+  gracefulTerminationTimeoutInMs?: number;
 };
 
 export type TaskRunProcessExecuteParams = {
@@ -114,7 +115,7 @@ export class TaskRunProcess {
       console.error("Error cancelling task run process", { err });
     }
 
-    await this.kill();
+    await this.#gracefullyTerminate(this.options.gracefulTerminationTimeoutInMs);
   }
 
   async cleanup(kill = true) {
@@ -131,7 +132,7 @@ export class TaskRunProcess {
     }
 
     if (kill) {
-      await this.kill("SIGKILL");
+      await this.#gracefullyTerminate(this.options.gracefulTerminationTimeoutInMs);
     }
   }
 
@@ -395,6 +396,18 @@ export class TaskRunProcess {
     this._stderr.push(errorLine);
   }
 
+  async #gracefullyTerminate(timeoutInMs: number = 1_000) {
+    logger.debug("gracefully terminating task run process", { pid: this.pid, timeoutInMs });
+
+    await this.kill("SIGTERM", timeoutInMs);
+
+    if (this._child?.connected) {
+      logger.debug("child process is still connected, sending SIGKILL", { pid: this.pid });
+
+      await this.kill("SIGKILL");
+    }
+  }
+
   /** This will never throw. */
   async kill(signal?: number | NodeJS.Signals, timeoutInMs?: number) {
     logger.debug(`killing task run process`, {
@@ -420,7 +433,7 @@ export class TaskRunProcess {
     const [error] = await tryCatch(killTimeout);
 
     if (error) {
-      logger.debug("kill: failed to wait for child process to exit", { error });
+      logger.debug("kill: failed to wait for child process to exit", { killTimeout });
     }
   }
 
diff --git a/packages/cli-v3/src/indexing/indexWorkerManifest.ts b/packages/cli-v3/src/indexing/indexWorkerManifest.ts
index ff8de685ce..e4ae72283f 100644
--- a/packages/cli-v3/src/indexing/indexWorkerManifest.ts
+++ b/packages/cli-v3/src/indexing/indexWorkerManifest.ts
@@ -61,7 +61,7 @@ export async function indexWorkerManifest({
       }
 
       resolved = true;
-      child.kill();
+      child.kill("SIGKILL");
       reject(new Error("Worker timed out"));
     }, 20_000);
 
@@ -79,21 +79,21 @@ export async function indexWorkerManifest({
           } else {
             resolve(message.payload.manifest);
           }
-          child.kill();
+          child.kill("SIGKILL");
           break;
         }
         case "TASKS_FAILED_TO_PARSE": {
           clearTimeout(timeout);
           resolved = true;
           reject(new TaskMetadataParseError(message.payload.zodIssues, message.payload.tasks));
-          child.kill();
+          child.kill("SIGKILL");
           break;
         }
         case "UNCAUGHT_EXCEPTION": {
           clearTimeout(timeout);
           resolved = true;
           reject(new UncaughtExceptionError(message.payload.error, message.payload.origin));
-          child.kill();
+          child.kill("SIGKILL");
           break;
         }
       }

From 7e0f9a1200fa89ab785dfff0a544b1bb76bd892f Mon Sep 17 00:00:00 2001
From: Eric Allam <eallam@icloud.com>
Date: Wed, 23 Jul 2025 14:41:13 +0100
Subject: [PATCH 2/2] better log, thanks coderabbit

---
 packages/cli-v3/src/executions/taskRunProcess.ts | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/cli-v3/src/executions/taskRunProcess.ts b/packages/cli-v3/src/executions/taskRunProcess.ts
index af0872bfe6..7b2adb6eb6 100644
--- a/packages/cli-v3/src/executions/taskRunProcess.ts
+++ b/packages/cli-v3/src/executions/taskRunProcess.ts
@@ -433,7 +433,11 @@ export class TaskRunProcess {
     const [error] = await tryCatch(killTimeout);
 
     if (error) {
-      logger.debug("kill: failed to wait for child process to exit", { killTimeout });
+      logger.debug("kill: failed to wait for child process to exit", {
+        timeoutInMs,
+        signal,
+        pid: this.pid,
+      });
     }
   }