From f02f2c0db60e28c28b4ef036acf59319939c4c70 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 16 Sep 2024 14:00:49 +0100 Subject: [PATCH 001/114] Some notes on the new run engine --- packages/run-engine/README.md | 44 +++++++++++++++++++++++++++++++ packages/run-engine/package.json | 17 ++++++++++++ packages/run-engine/src/index.ts | 25 ++++++++++++++++++ packages/run-engine/tsconfig.json | 19 +++++++++++++ 4 files changed, 105 insertions(+) create mode 100644 packages/run-engine/README.md create mode 100644 packages/run-engine/package.json create mode 100644 packages/run-engine/src/index.ts create mode 100644 packages/run-engine/tsconfig.json diff --git a/packages/run-engine/README.md b/packages/run-engine/README.md new file mode 100644 index 0000000000..a88737c2b5 --- /dev/null +++ b/packages/run-engine/README.md @@ -0,0 +1,44 @@ +# Trigger.dev Run Engine + +The Run Engine process runs from triggering, to executing, and completing them. + +It is responsible for: + +- Creating and updating runs as they progress. +- Operating the run queue, including handling concurrency. + +## Components + +### Run Engine + +This is used to actually process a run and store the state at each step. It coordinates with the other components. + +#### Atomicity + +Operations on the run are "atomic" in the sense that only a single operation can mutate them at a time. We use RedLock to ensure this. + +#### Valid state transitions + +The run engine ensures that the run can only transition to valid states. + +#### State history + +When a run is mutated in any way, we store the state. This data is used for the next step for the run, and also for debugging. + +### Run Queue + +This is used to queue, dequeue, and manage concurrency. It also provides visibility into the concurrency for the env, org, etc. + +Run IDs are enqueued. They're pulled from the queue in a fair way with advanced options for debouncing and visibility. + +### Heartbeats + +Heartbeats are used to determine if a run has stopped responding. If a heartbeat isn't received within a defined period then the run is judged to have become stuck and the attempt is failed. + +### Checkpoints + +Checkpoints allow pausing an executing run and then resuming it later. + +## How does it work? + +It's very important that a run can only be acted on by one process at a time. We lock runs using RedLock while they're being mutated. This prevents some network-related race conditions like the timing of checkpoints and heartbeats permanently hanging runs. diff --git a/packages/run-engine/package.json b/packages/run-engine/package.json new file mode 100644 index 0000000000..e82b7fb13e --- /dev/null +++ b/packages/run-engine/package.json @@ -0,0 +1,17 @@ +{ + "name": "@trigger.dev/run-engine", + "private": true, + "version": "0.0.1", + "main": "./src/index.ts", + "types": "./src/index.ts", + "dependencies": { + "@trigger.dev/core": "workspace:*", + "@trigger.dev/database": "workspace:*", + "ioredis": "^5.3.2", + "typescript": "^4.8.4" + }, + "devDependencies": {}, + "scripts": { + "typecheck": "tsc --noEmit" + } +} diff --git a/packages/run-engine/src/index.ts b/packages/run-engine/src/index.ts new file mode 100644 index 0000000000..983035481b --- /dev/null +++ b/packages/run-engine/src/index.ts @@ -0,0 +1,25 @@ +import { PrismaClient, Prisma } from "@trigger.dev/database"; +import { type RedisOptions } from "ioredis"; + +type Options = { + prisma: PrismaClient; + redis: RedisOptions; + //todo + // queue: RunQueue; +}; + +export class RunEngine { + private prisma: PrismaClient; + + constructor(private readonly options: Options) { + this.prisma = options.prisma; + } + + /** Creates a new run with the options, returns metadata */ + async trigger() { + // const result = await this.options.prisma.taskRun.create({}); + // return result; + } + + async execute(runId: string) {} +} diff --git a/packages/run-engine/tsconfig.json b/packages/run-engine/tsconfig.json new file mode 100644 index 0000000000..83c7a33ef1 --- /dev/null +++ b/packages/run-engine/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "moduleResolution": "node", + "preserveWatchOutput": true, + "skipLibCheck": true, + "noEmit": true, + "strict": true, + "paths": { + "@trigger.dev/core": ["../../packages/core/src/index"], + "@trigger.dev/core/*": ["../../packages/core/src/*"], + "@trigger.dev/database": ["../../packages/database/src/index"], + "@trigger.dev/database/*": ["../../packages/database/src/*"] + } + }, + "exclude": ["node_modules"] +} From 46a202fa45517e1447eae66f3e68fc12b86f9af2 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 16 Sep 2024 14:01:01 +0100 Subject: [PATCH 002/114] lockfile with setup for the run engine --- pnpm-lock.yaml | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ca0e051fa7..1fc3701b9e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1234,6 +1234,21 @@ importers: specifier: ^5.5.0 version: 5.5.4 + packages/run-engine: + dependencies: + '@trigger.dev/core': + specifier: workspace:* + version: link:../core + '@trigger.dev/database': + specifier: workspace:* + version: link:../database + ioredis: + specifier: ^5.3.2 + version: 5.3.2 + typescript: + specifier: ^4.8.4 + version: 4.9.5 + packages/trigger-sdk: dependencies: '@opentelemetry/api': @@ -20310,7 +20325,7 @@ packages: '@types/node': 20.14.14 '@types/semver': 7.5.1 chalk: 4.1.2 - debug: 4.3.4 + debug: 4.3.6 interpret: 3.1.1 semver: 7.5.4 tslib: 2.6.2 @@ -20834,7 +20849,7 @@ packages: dependencies: '@ioredis/commands': 1.2.0 cluster-key-slot: 1.1.2 - debug: 4.3.4 + debug: 4.3.6 denque: 2.1.0 lodash.defaults: 4.2.0 lodash.isarguments: 3.1.0 @@ -24715,7 +24730,7 @@ packages: engines: {node: '>= 14'} dependencies: agent-base: 7.1.0 - debug: 4.3.4 + debug: 4.3.6 http-proxy-agent: 7.0.0 https-proxy-agent: 7.0.1 lru-cache: 7.18.3 @@ -25892,7 +25907,7 @@ packages: resolution: {integrity: sha512-OScOjQjrrjhAdFpQmnkE/qbIBGCRFhQB/YaJhcC3CPOlmhe7llnW46Ac1J5+EjcNXOTnDdpF96Erw/yedsGksQ==} engines: {node: '>=8.6.0'} dependencies: - debug: 4.3.4 + debug: 4.3.6 module-details-from-path: 1.0.3 resolve: 1.22.8 transitivePeerDependencies: @@ -28702,7 +28717,7 @@ packages: hasBin: true dependencies: cac: 6.7.14 - debug: 4.3.4 + debug: 4.3.6 mlly: 1.7.1 pathe: 1.1.2 picocolors: 1.0.0 @@ -28726,7 +28741,7 @@ packages: hasBin: true dependencies: cac: 6.7.14 - debug: 4.3.4 + debug: 4.3.6 mlly: 1.7.1 pathe: 1.1.2 picocolors: 1.0.0 @@ -28750,7 +28765,7 @@ packages: hasBin: true dependencies: cac: 6.7.14 - debug: 4.3.4 + debug: 4.3.6 pathe: 1.1.2 picocolors: 1.0.0 vite: 5.2.7(@types/node@18.11.18) From d11f422d175529dcd1122d7096de4e1a042eb038 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 19 Sep 2024 19:08:08 +0100 Subject: [PATCH 003/114] Documenting where TaskRun is currently mutated, to try figure out the shape of the new system --- packages/run-engine/README.md | 93 +++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/packages/run-engine/README.md b/packages/run-engine/README.md index a88737c2b5..4d3f25d1d0 100644 --- a/packages/run-engine/README.md +++ b/packages/run-engine/README.md @@ -42,3 +42,96 @@ Checkpoints allow pausing an executing run and then resuming it later. ## How does it work? It's very important that a run can only be acted on by one process at a time. We lock runs using RedLock while they're being mutated. This prevents some network-related race conditions like the timing of checkpoints and heartbeats permanently hanging runs. + +# Legacy system + +These are all the TaskRun mutations happening right now: + +## 1. Trigger + +## 2. Batch trigger + +## 3. DevQueueConsumer executing a run + +### a. Lock run and set status to `EXECUTING` + +[DevQueueConsumer.#doWorkInternal()](/apps/webapp/app/v3/marqs/devQueueConsumer.server.ts#371) + +### b. If an error is thrown, unlock the run and set status to `PENDING` + +[DevQueueConsumer.#doWorkInternal()](/apps/webapp/app/v3/marqs/devQueueConsumer.server.ts#477) + +## 4. SharedQueueConsumer executing a run + +### a. `EXECUTE`, lock the run + +We lock the run and update some basic metadata (but not status). +[SharedQueueConsumer.#doWorkInternal()](/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts#394) + +### b. `EXECUTE`, if an error is thrown, unlock the run + +We unlock the run, but don't change the status. +[SharedQueueConsumer.#doWorkInternal()](/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts#552) + +### c. `EXECUTE`, if the run has no deployment set the status to `WAITING_FOR_DEPLOY` + +[SharedQueueConsumer.#doWorkInternal()](/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts#876) + +## 5. CompleteAttemptService retrying a run + +### a. When an attempt has failed, we set the status to `RETRYING_AFTER_FAILURE` + +[CompleteAttemptService.#completeAttemptFailed()](/apps/webapp/app/v3/services/completeAttempt.server.ts#239) + +## 6. CreateTaskRunAttemptService creating a new attempt, setting the run to `EXECUTING` + +We call this when: + +- [Executing a DEV run from the CLI.](/packages/cli-v3//src/dev/workerRuntime.ts#305) +- [Deprecated: directly from the SharedQueueCOnsumer when we don't support lazy attempts](/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts#501) +- [When we receive a `CREATE_TASK_RUN_ATTEMPT` message from the coordinator](/apps/webapp//app/v3//handleSocketIo.server.ts#187) + +This is the actual very simple TaskRun update: +[CreateTaskRunAttemptService.call()](/apps/webapp/app/v3/services/createTaskRunAttempt.server.ts#134) + +## 7. EnqueueDelayedRunService set a run to `PENDING` when the `delay` has elapsed + +When the run attempt gets created it will be marked as `EXECUTING`. + +[EnqueueDelayedRunService.#call()](/apps/webapp/app/v3/services/enqueueDelayedRun.server.ts#41) + +## 8. FinalizeTaskRunService finalizing a run + +This service is called from many places, when a run is in a "final" state. This means the run can't be acted on anymore. + +We set the status, expiredAt and completedAt fields. + +[FinalizeTaskRunService.#call()](/apps/webapp/app/v3/services/finalizeTaskRun.server.ts#63) + +This function is called from: + +- [`FailedTaskRunService` when a run has SYSTEM_FAILURE](/apps/webapp/app/v3/failedTaskRun.server.ts#41) +- [`CancelAttemptService` when an attempt is canceled](/apps/webapp/app/v3/services/cancelAttempt.server.ts#66) +- [`CancelTaskRunService` when a run is canceled](/apps/webapp/app/v3/services/cancelTaskRun.server.ts#51) +- `CompleteAttemptService` when a SYSTEM_FAILURE happens + - [No attempt](/apps/webapp/app/v3/services/completeAttempt.server.ts#74) + - [`completeAttemptFailed` and there's no checkpoint](/apps/webapp/app/v3/services/completeAttempt.server.ts#280) + - [`completeAttemptFailed` and the error is internal and a graceful exit timeout](/apps/webapp/app/v3/services/completeAttempt.server.ts#321) +- `CompleteTaskRunService` when a run has failed (this isn't a bug) + - [`completeAttemptFailed`](/apps/webapp/app/v3/services/completeAttempt.server.ts#352) +- `CompleteTaskRunService` when a run is completed successfully + - [`completeAttemptSuccessfully`](/apps/webapp/app/v3/services/completeAttempt.server.ts#135) +- `CrashTaskRunService` when a run has crashed + - [`call`](/apps/webapp/app/v3/services/crashTaskRun.server.ts#47) +- `ExpireEnqueuedRunService` when a run has expired + - [`call`](/apps/webapp/app/v3/services/expireEnqueuedRun.server.ts#42) + +## 9. RescheduleTaskRunService (when further delaying a delayed run) + +[RescheduleTaskRunService.#call()](/apps/webapp/app/v3/services/rescheduleTaskRun.server.ts#21) + +## 10. Triggering a scheduled run + +Graphile Worker calls this function based on the schedule. We add the schedule data onto the run, and call `TriggerTaskService.call()`. + +[TriggerScheduledRunService.#call()](/apps/webapp/app/v3/services/triggerScheduledTask.server.ts#131) From 98a94a87548a3d9f679ea439e8f06f4897992918 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 19 Sep 2024 19:48:04 +0100 Subject: [PATCH 004/114] Added notes about how triggering currently works --- packages/run-engine/README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/run-engine/README.md b/packages/run-engine/README.md index 4d3f25d1d0..418e70af69 100644 --- a/packages/run-engine/README.md +++ b/packages/run-engine/README.md @@ -47,7 +47,12 @@ It's very important that a run can only be acted on by one process at a time. We These are all the TaskRun mutations happening right now: -## 1. Trigger +## 1. TriggerTaskService + +Directly creates a run if it doesn't exist, either in the `PENDING` or `DELAYED` states. +Enqueues the run. + +[TriggerTaskService.call()](/apps//webapp/app/v3/services/triggerTask.server.ts#246) ## 2. Batch trigger From 1329b6a47a5612377690626bf37603673bae2ac8 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 19 Sep 2024 19:50:20 +0100 Subject: [PATCH 005/114] Details about when triggering happens --- packages/run-engine/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/run-engine/README.md b/packages/run-engine/README.md index 418e70af69..86ccc7991f 100644 --- a/packages/run-engine/README.md +++ b/packages/run-engine/README.md @@ -49,6 +49,14 @@ These are all the TaskRun mutations happening right now: ## 1. TriggerTaskService +This is called from: + +- trigger task API +- `BatchTriggerTaskService` for each item +- `ReplayTaskRunService` +- `TestTaskService` +- `TriggerScheduledTaskService` when the CRON fires + Directly creates a run if it doesn't exist, either in the `PENDING` or `DELAYED` states. Enqueues the run. From 670997e45ed7bf6b5cd120ad856a99e4e2541ad9 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 20 Sep 2024 13:23:43 +0100 Subject: [PATCH 006/114] Lots of notes about waitpoints --- packages/run-engine/README.md | 174 ++++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) diff --git a/packages/run-engine/README.md b/packages/run-engine/README.md index 86ccc7991f..2960756f71 100644 --- a/packages/run-engine/README.md +++ b/packages/run-engine/README.md @@ -25,6 +25,8 @@ The run engine ensures that the run can only transition to valid states. When a run is mutated in any way, we store the state. This data is used for the next step for the run, and also for debugging. +`TaskRunState` is a decent table name. We should have a "description" column which describes the change, this would be purely for internal use but would be very useful for debugging. + ### Run Queue This is used to queue, dequeue, and manage concurrency. It also provides visibility into the concurrency for the env, org, etc. @@ -39,6 +41,178 @@ Heartbeats are used to determine if a run has stopped responding. If a heartbeat Checkpoints allow pausing an executing run and then resuming it later. +## Waitpoints + +A "Waitpoint" is something that prevents a run from continuing: + +- `wait.for()` a future time. +- `triggerAndWait()` until the run is finished. +- `batchTriggerAndWait()` until all runs are finished. +- `wait.forRequest()` wait until a request has been received (not implemented yet). + +They block run execution from continuing until all of them are completed/removed. + +Some of them have data associated with them, e.g. the finished run payload. + +Could a run have multiple at once? That might allow us to support Promise.all wrapped. It would also allow more advanced use cases. + +Could this be how we implement other features like `delay`, `rate limit`, and retries waiting before the next try? + +Could we even open up a direct API/SDK for creating one inside a run (would pause execution)? And then completing one (would continue execution)? It could also be "failed" which the run could act upon differently. + +## Notes from call with Eric + +We could expose the API/SDK for creating/completing Waitpoints. + +> They need to be associated with attempts, because that's what gets continued. And if an attempts fails, we don't want to keep the waitpoints. + +> We should have idempotency keys for `wait.for()` and `wait.until()`, so they wouldn't wait on a second attempt. "Waitpoints" have idempotency keys, and these are used for a `wait.forEvent()` (or whatever we call it). + +> How would debounce use this? When the waitpoint is completed, we would "clear" the "idempotencyKey" which would be the user-provided "debounceKey". It wouldn't literally clear it necessarily. Maybe another column `idempotencyKeyActive` would be set to `false`. Or move the key to another column, which is just for reference. + +> `triggerAndWait`, cancelling a child task run. It would clear the waitpoint `idempotencyKey`, same as above. + +> Copying the output from the run into the waitpoint actually does make sense. It simplifies the API for continuing runs. + +> Inside a run you could wait for another run or runs using the run ID. `const output = await wait.forRunToComplete(runId)`. This would basically just get a run by ID, then wait for it's waitpoint to be completed. This means every run would have a waitpoint associated with it. + +```ts +//inside a run function +import { runs } from "@trigger.dev/sdk/v3"; + +// Loop through all runs with the tag "user_123456" that have completed + +for await (const run of runs.list({ tag: "user_123456" })) { + await wait.forRunToComplete(run.id); +} + +//wait for many runs to complete +await wait.forRunToComplete(runId); +await wait.forRunsToComplete({ tag: "user_123456" }); +``` + +Rate limit inside a task. This is much trickier. + +```ts +//simple time-based rate limit +await wait.forRateLimit(`timed-${payload.user.id}`, { per: { minute: 10 } }); + +const openAiResult = await wait.forRateLimit( + `openai-${payload.user.id}`, + { limit: 100, recharge: { seconds: 2 } }, + (rateLimit, refreshes) => { + const result = await openai.createCompletion({ + model: "gpt-3.5-turbo", + prompt: "What is the meaning of life?", + }); + const tokensUsed = result.tokensUsed; + + await rateLimit.used(tokensUsed); + + return result; + } +); + +//do stuff with openAiResult +``` + +#### `triggerAndWait()` implementation + +Inside the SDK + +```ts +function triggerAndWait_internal(data) { + //if you don't pass in a string, it won't have a "key" + const waitpoint = await createWaitpoint(); + const response = await apiClient.triggerTask({ ...data, waitpointId: waitpoint.id }); + + //...do normal stuff + + // wait for the waitpoint to be completed + // in reality this probably needs to happen inside the runtime + const result = await waitpointCompletion(waitpoint.id); +} +``` + +Pseudo-code for completing a run and completing the waitpoint: + +```ts +function completeRun(tx, data) { + //complete the child run + const run = await tx.taskRun.update({ where: { id: runId }, data, include: { waitpoint } }); + if (run.waitpoint) { + await completeWaitpoint(tx, { id: run.waitpoint.id }); + + //todo in completeWaitpoint it would check if the blocked runs can now continue + //if they have no more blockers then they can continue + + //batchTriggerAndWait with two items + //blocked_by: ["w_1", "w_2"] + //blocked_by: ["w_2"] + //blocked_by: [] then you can continue + } + + const state = await tx.taskRunState.create({ + where: { runId: id }, + data: { runId, status: run.status }, + }); + + const previousState = await tx.taskRunState.findFirst({ where: { runId: runId, latest: true } }); + const waitingOn = previousState.waitingOn?.filter((w) => w !== waitpoint?.id) ?? []; + + if (waitingOn.length === 0) { + } +} +``` + +#### `batchTriggerAndWait()` implementation + +```ts +//todo +``` + +### Example: User-defined waitpoint + +A user's backend code: + +```ts +import { waitpoint } from "@trigger.dev/sdk/v3"; +import type { NextApiRequest, NextApiResponse } from "next"; + +export default async function handler(req: NextApiRequest, res: NextApiResponse<{ id: string }>) { + const userId = req.query.userId; + const isPaying = req.query.isPaying; + + //internal SDK calls, this would be nicer for users to use + const waitpoint = waitpoint(`${userId}/onboarding-completed`); + await waitpoint.complete({ data: { isPaying } }); + + //todo instead this would be a single call + + res.status(200).json(handle); +} +``` + +Inside a user's run + +```ts +export const myTask = task({ + id: "my-task", + run: async (payload) => { + //it doesn't matter if this was completed before the run started + const result = await wait.forPoint<{ isPaying: boolean }>( + `${payload.userId}/onboarding-completed` + ); + }, +}); +``` + +### How would we implement `batchTriggerAndWait`? + +```ts + +``` + ## How does it work? It's very important that a run can only be acted on by one process at a time. We lock runs using RedLock while they're being mutated. This prevents some network-related race conditions like the timing of checkpoints and heartbeats permanently hanging runs. From eef34a15703e4fea823d195a98bae1141380556a Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 20 Sep 2024 13:26:20 +0100 Subject: [PATCH 007/114] Started scaffolding the RunEngine --- packages/run-engine/src/index.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/run-engine/src/index.ts b/packages/run-engine/src/index.ts index 983035481b..c9142eb391 100644 --- a/packages/run-engine/src/index.ts +++ b/packages/run-engine/src/index.ts @@ -15,11 +15,23 @@ export class RunEngine { this.prisma = options.prisma; } - /** Creates a new run with the options, returns metadata */ + /** Triggers one run. + * This doesn't start execution, but it will schedule it for execution. + */ async trigger() { // const result = await this.options.prisma.taskRun.create({}); // return result; } + /** Triggers multiple runs. + * This doesn't start execution, but it will create a batch and schedule them for execution. + */ + async batchTrigger() {} + + /** The run can be added to the queue. When it's pulled from the queue it will be executed. */ + async readyToExecute(runId: string) {} + + /** We want to actually execute the run, this could be a continuation of a previous execution. + * This is called from the queue, when the run has been pulled. */ async execute(runId: string) {} } From 103bf4289bdf2c912b17de34f90b6859a65a6608 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 20 Sep 2024 13:49:47 +0100 Subject: [PATCH 008/114] =?UTF-8?q?Sketch=20of=20Prisma=20waitpoint=20sche?= =?UTF-8?q?ma=20while=20it=E2=80=99s=20fresh=20in=20my=20mind?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/database/prisma/schema.prisma | 40 ++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/packages/database/prisma/schema.prisma b/packages/database/prisma/schema.prisma index 92c370fefa..199ab1f25e 100644 --- a/packages/database/prisma/schema.prisma +++ b/packages/database/prisma/schema.prisma @@ -467,6 +467,7 @@ model Project { alertStorages ProjectAlertStorage[] bulkActionGroups BulkActionGroup[] BackgroundWorkerFile BackgroundWorkerFile[] + waitpoints Waitpoint[] } enum ProjectVersion { @@ -1779,6 +1780,45 @@ enum TaskRunStatus { EXPIRED } +/// A Waitpoint blocks a run from continuing until it's completed +model Waitpoint { + id String @id @default(cuid()) + + type WaitpointType + status WaitpointStatus @default(PENDING) + + idempotencyKey String + userProvidedIdempotencyKey Boolean + + ///if an idempotencyKey is no longer active, we store it here and generate a new one for the idempotencyKey field. This is a workaround because Prisma doesn't support partial indexes. + inactiveIdempotencyKey String? + + output String? + outputType String @default("application/json") + + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) + projectId String + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([projectId, idempotencyKey]) +} + +enum WaitpointType { + RUN + DATETIME + EVENT +} + +enum WaitpointStatus { + PENDING + COMPLETED +} + +///todo add a waitpoint to each run +///todo add a blockedBy array of waitpoint IDs to the run, and attempt. With appropriate indexes for fast search. + model TaskRunTag { id String @id @default(cuid()) name String From 0088b993c22d98521890667ef772c9dfa1c5c24d Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 22 Sep 2024 17:34:30 +0100 Subject: [PATCH 009/114] Got Prisma working with testcontainers --- packages/run-engine/package.json | 9 +- packages/run-engine/src/index.test.ts | 60 +++++ packages/run-engine/tsconfig.json | 1 + packages/run-engine/vitest.config.ts | 8 + pnpm-lock.yaml | 359 +++++++++++++++++++++----- 5 files changed, 374 insertions(+), 63 deletions(-) create mode 100644 packages/run-engine/src/index.test.ts create mode 100644 packages/run-engine/vitest.config.ts diff --git a/packages/run-engine/package.json b/packages/run-engine/package.json index e82b7fb13e..7c6075ae9a 100644 --- a/packages/run-engine/package.json +++ b/packages/run-engine/package.json @@ -10,8 +10,13 @@ "ioredis": "^5.3.2", "typescript": "^4.8.4" }, - "devDependencies": {}, + "devDependencies": { + "@testcontainers/postgresql": "^10.13.1", + "testcontainers": "^10.13.1", + "vitest": "^1.4.0" + }, "scripts": { - "typecheck": "tsc --noEmit" + "typecheck": "tsc --noEmit", + "test": "vitest" } } diff --git a/packages/run-engine/src/index.test.ts b/packages/run-engine/src/index.test.ts new file mode 100644 index 0000000000..6fcccd8601 --- /dev/null +++ b/packages/run-engine/src/index.test.ts @@ -0,0 +1,60 @@ +import { PrismaClient, Prisma } from "@trigger.dev/database"; +import { PostgreSqlContainer } from "@testcontainers/postgresql"; +import path from "path"; +import { execSync } from "child_process"; +import exp from "constants"; + +describe("Placeholder", () => { + it("should pass", () => { + expect(true).toBe(true); + }); + + it( + "should connect and return a query result", + { + timeout: 60_000, + }, + async () => { + const container = await new PostgreSqlContainer().start(); + + // Run migrations + const databasePath = path.resolve(__dirname, "../../database"); + try { + execSync(`npx prisma db push --schema ${databasePath}/prisma/schema.prisma`, { + env: { + ...process.env, + DATABASE_URL: container.getConnectionUri(), + DIRECT_URL: container.getConnectionUri(), + }, + }); + } catch (error) { + expect(error).toBeUndefined(); + } + + // console.log(container.getConnectionUri()); + + const prisma = new PrismaClient({ + datasources: { + db: { + url: container.getConnectionUri(), + }, + }, + }); + prisma.$connect(); + + const user = await prisma.user.create({ + data: { + authenticationMethod: "MAGIC_LINK", + email: "test@example.com", + }, + }); + + const result = await prisma.user.findMany(); + expect(result.length).toEqual(1); + expect(result[0].email).toEqual("test@example.com"); + + await prisma.$disconnect(); + await container.stop(); + } + ); +}); diff --git a/packages/run-engine/tsconfig.json b/packages/run-engine/tsconfig.json index 83c7a33ef1..bd7d79399c 100644 --- a/packages/run-engine/tsconfig.json +++ b/packages/run-engine/tsconfig.json @@ -1,5 +1,6 @@ { "compilerOptions": { + "types": ["vitest/globals"], "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "isolatedModules": true, diff --git a/packages/run-engine/vitest.config.ts b/packages/run-engine/vitest.config.ts new file mode 100644 index 0000000000..4afd926425 --- /dev/null +++ b/packages/run-engine/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["**/*.test.ts"], + globals: true, + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1fc3701b9e..9db834db71 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1248,6 +1248,16 @@ importers: typescript: specifier: ^4.8.4 version: 4.9.5 + devDependencies: + '@testcontainers/postgresql': + specifier: ^10.13.1 + version: 10.13.1 + testcontainers: + specifier: ^10.13.1 + version: 10.13.1 + vitest: + specifier: ^1.4.0 + version: 1.6.0(@types/node@20.14.14) packages/trigger-sdk: dependencies: @@ -2295,7 +2305,7 @@ packages: '@babel/traverse': 7.24.7 '@babel/types': 7.24.7 convert-source-map: 2.0.0 - debug: 4.3.6 + debug: 4.3.7 gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -2435,7 +2445,7 @@ packages: '@babel/core': 7.22.17 '@babel/helper-compilation-targets': 7.22.15 '@babel/helper-plugin-utils': 7.24.0 - debug: 4.3.6 + debug: 4.3.7 lodash.debounce: 4.0.8 resolve: 1.22.8 semver: 6.3.1 @@ -2451,7 +2461,7 @@ packages: '@babel/core': 7.22.17 '@babel/helper-compilation-targets': 7.22.15 '@babel/helper-plugin-utils': 7.24.0 - debug: 4.3.6 + debug: 4.3.7 lodash.debounce: 4.0.8 resolve: 1.22.8 transitivePeerDependencies: @@ -3847,7 +3857,7 @@ packages: '@babel/helper-split-export-declaration': 7.24.7 '@babel/parser': 7.24.7 '@babel/types': 7.24.7 - debug: 4.3.6 + debug: 4.3.7 globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -3861,7 +3871,7 @@ packages: '@babel/parser': 7.25.6 '@babel/template': 7.25.0 '@babel/types': 7.25.6 - debug: 4.3.6 + debug: 4.3.7 globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -3890,6 +3900,10 @@ packages: '@babel/helper-validator-identifier': 7.24.7 to-fast-properties: 2.0.0 + /@balena/dockerignore@1.0.2: + resolution: {integrity: sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==} + dev: true + /@bufbuild/protobuf@1.10.0: resolution: {integrity: sha512-QDdVFLoN93Zjg36NoQPZfsVH9tZew7wKDKyV5qRdj8ntT4wQCOradQjRaTdwMhWUYsgKsvCINKKm87FdEk96Ag==} dev: false @@ -6014,7 +6028,7 @@ packages: engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} dependencies: ajv: 6.12.6 - debug: 4.3.6 + debug: 4.3.7 espree: 9.6.0 globals: 13.19.0 ignore: 5.2.4 @@ -6245,7 +6259,7 @@ packages: deprecated: Use @eslint/config-array instead dependencies: '@humanwhocodes/object-schema': 1.2.1 - debug: 4.3.6 + debug: 4.3.7 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -8097,7 +8111,7 @@ packages: engines: {node: '>=18'} hasBin: true dependencies: - debug: 4.3.6 + debug: 4.3.7 extract-zip: 2.0.1 progress: 2.0.3 proxy-agent: 6.4.0 @@ -14117,6 +14131,14 @@ packages: zod: 3.22.3 dev: false + /@testcontainers/postgresql@10.13.1: + resolution: {integrity: sha512-HAh/3uLAzAhOmzXsOE6hVxkvetczPnX/Zoyt+SgK7QotW98Npr1MDx8OKiaLGTJ8XkIvVvS4Ch6bl+frt4pnkQ==} + dependencies: + testcontainers: 10.13.1 + transitivePeerDependencies: + - supports-color + dev: true + /@testing-library/dom@8.19.1: resolution: {integrity: sha512-P6iIPyYQ+qH8CvGauAqanhVnjrnRe0IZFSYCeGkSRW9q3u8bdVn2NPI+lasFyVsEQn1J/IFmp5Aax41+dAP9wg==} engines: {node: '>=12'} @@ -14407,6 +14429,21 @@ packages: resolution: {integrity: sha512-xFdR6tkm0MWvBfO8xXCSsinYxHcqkQUlcHeSpMC2ukzOb6lwQAfDmW+Qt0AvlGd8HpsS28qKsB+oPeJn9I39jg==} dev: false + /@types/docker-modem@3.0.6: + resolution: {integrity: sha512-yKpAGEuKRSS8wwx0joknWxsmLha78wNMe9R2S3UNsVOkZded8UqOrV8KoeDXoXsjndxwyF3eIhyClGbO1SEhEg==} + dependencies: + '@types/node': 18.19.20 + '@types/ssh2': 1.15.1 + dev: true + + /@types/dockerode@3.3.31: + resolution: {integrity: sha512-42R9eoVqJDSvVspV89g7RwRqfNExgievLNWoHkg7NoWIqAmavIbgQBb4oc0qRtHkxE+I3Xxvqv7qVXFABKPBTg==} + dependencies: + '@types/docker-modem': 3.0.6 + '@types/node': 18.19.20 + '@types/ssh2': 1.15.1 + dev: true + /@types/email-reply-parser@1.4.2: resolution: {integrity: sha512-kmMoK9WMX4zXf3c0D3tkWHDl0E50V2dv6fVirdTQd/mkvE/Jixh0DZAh3kBgpltr1eaWM3W+kAf4A2c2Z2iU2A==} dev: true @@ -14787,6 +14824,25 @@ packages: source-map: 0.6.1 dev: true + /@types/ssh2-streams@0.1.12: + resolution: {integrity: sha512-Sy8tpEmCce4Tq0oSOYdfqaBpA3hDM8SoxoFh5vzFsu2oL+znzGz8oVWW7xb4K920yYMUY+PIG31qZnFMfPWNCg==} + dependencies: + '@types/node': 18.19.20 + dev: true + + /@types/ssh2@0.5.52: + resolution: {integrity: sha512-lbLLlXxdCZOSJMCInKH2+9V/77ET2J6NPQHpFI0kda61Dd1KglJs+fPQBchizmzYSOJBgdTajhPqBO1xxLywvg==} + dependencies: + '@types/node': 18.19.20 + '@types/ssh2-streams': 0.1.12 + dev: true + + /@types/ssh2@1.15.1: + resolution: {integrity: sha512-ZIbEqKAsi5gj35y4P4vkJYly642wIbY6PqoN0xiyQGshKUGXR9WQjF/iF9mXBQ8uBKy3ezfsCkcoHKhd0BzuDA==} + dependencies: + '@types/node': 18.19.20 + dev: true + /@types/statuses@2.0.4: resolution: {integrity: sha512-eqNDvZsCNY49OAXB0Firg/Sc2BgoWsntsLUdybGFOhAfCD6QJ2n9HXUIHGqt5qjrxmMv4wS8WLAw43ZkKcJ8Pw==} dev: false @@ -15596,7 +15652,7 @@ packages: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} dependencies: - debug: 4.3.6 + debug: 4.3.7 transitivePeerDependencies: - supports-color dev: false @@ -15605,7 +15661,7 @@ packages: resolution: {integrity: sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==} engines: {node: '>= 14'} dependencies: - debug: 4.3.6 + debug: 4.3.7 transitivePeerDependencies: - supports-color @@ -15613,7 +15669,7 @@ packages: resolution: {integrity: sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==} engines: {node: '>= 14'} dependencies: - debug: 4.3.6 + debug: 4.3.7 transitivePeerDependencies: - supports-color @@ -15832,6 +15888,19 @@ packages: readable-stream: 3.6.0 dev: true + /archiver-utils@5.0.2: + resolution: {integrity: sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==} + engines: {node: '>= 14'} + dependencies: + glob: 10.3.10 + graceful-fs: 4.2.11 + is-stream: 2.0.1 + lazystream: 1.0.1 + lodash: 4.17.21 + normalize-path: 3.0.0 + readable-stream: 4.5.2 + dev: true + /archiver@5.3.2: resolution: {integrity: sha512-+25nxyyznAXF7Nef3y0EbBeqmGZgeN/BxHX29Rs39djAfaFalmQ89SE6CWyDCHzGL0yt/ycBtNOmGTW0FyGWNw==} engines: {node: '>= 10'} @@ -15845,6 +15914,19 @@ packages: zip-stream: 4.1.1 dev: true + /archiver@7.0.1: + resolution: {integrity: sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==} + engines: {node: '>= 14'} + dependencies: + archiver-utils: 5.0.2 + async: 3.2.6 + buffer-crc32: 1.0.0 + readable-stream: 4.5.2 + readdir-glob: 1.1.3 + tar-stream: 3.1.7 + zip-stream: 6.0.1 + dev: true + /arg@4.1.3: resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==} @@ -16004,7 +16086,6 @@ packages: resolution: {integrity: sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==} dependencies: safer-buffer: 2.1.2 - dev: false /assert-never@1.2.1: resolution: {integrity: sha512-TaTivMB6pYI1kXwrFlEhLeGfOqoDNdTxjCdwRfFFkEA30Eu+k48W34nlok2EYWJfFFzqaEmichdNM7th6M5HNw==} @@ -16045,6 +16126,10 @@ packages: hasBin: true dev: true + /async-lock@1.4.1: + resolution: {integrity: sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ==} + dev: true + /async@0.2.10: resolution: {integrity: sha512-eAkdoKxU6/LkKDBzLpT+t6Ff5EtfSF4wx1WfJiPEEV7WNLnDaRXk0oVysiEPm262roaachGexwUv94WhSgN5TQ==} dev: false @@ -16161,7 +16246,6 @@ packages: /b4a@1.6.6: resolution: {integrity: sha512-5Tk1HLk6b6ctmjIkAcU/Ujv/1WqiDl0F0JdRCR80VsOcUlHcu7pWeWRlOqQLHfDEsVx9YH/aif5AG4ehoCtTmg==} - dev: false /babel-loader@9.1.3(@babel/core@7.24.5)(webpack@5.88.2): resolution: {integrity: sha512-xG3ST4DglodGf8qSwv0MdeWLhrDsw/32QMdTO5T1ZIp9gQur0HkCyFs7Awskr10JKXFXwpAhiCuYX5oGXnRGbw==} @@ -16302,7 +16386,6 @@ packages: /bare-events@2.4.2: resolution: {integrity: sha512-qMKFd2qG/36aA4GwvKq8MxnPgCQAmBWmSyLWsJcbn8v03wvIPQ/hG1Ms8bPzndZxMDoHpxez5VOS+gC9Yi24/Q==} requiresBuild: true - dev: false optional: true /bare-fs@2.3.5: @@ -16312,13 +16395,11 @@ packages: bare-events: 2.4.2 bare-path: 2.1.3 bare-stream: 2.3.0 - dev: false optional: true /bare-os@2.4.4: resolution: {integrity: sha512-z3UiI2yi1mK0sXeRdc4O1Kk8aOa/e+FNWZcTiPB/dfTWyLypuE99LibgRaQki914Jq//yAWylcAt+mknKdixRQ==} requiresBuild: true - dev: false optional: true /bare-path@2.1.3: @@ -16326,7 +16407,6 @@ packages: requiresBuild: true dependencies: bare-os: 2.4.4 - dev: false optional: true /bare-stream@2.3.0: @@ -16335,7 +16415,6 @@ packages: dependencies: b4a: 1.6.6 streamx: 2.20.1 - dev: false optional: true /base64-js@1.5.1: @@ -16360,7 +16439,6 @@ packages: resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==} dependencies: tweetnacl: 0.14.5 - dev: false /better-path-resolve@1.0.0: resolution: {integrity: sha512-pbnl5XzGBdrFU/wT4jqmJVPn2B6UHPBOhzMQkY/SPUPB6QtUXtmBHBIwCbXJol93mOpGMnQyP/+BB19q04xj7g==} @@ -16467,6 +16545,11 @@ packages: /buffer-crc32@0.2.13: resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} + /buffer-crc32@1.0.0: + resolution: {integrity: sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==} + engines: {node: '>=8.0.0'} + dev: true + /buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} @@ -16481,7 +16564,13 @@ packages: dependencies: base64-js: 1.5.1 ieee754: 1.2.1 - dev: false + + /buildcheck@0.0.6: + resolution: {integrity: sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==} + engines: {node: '>=10.0.0'} + requiresBuild: true + dev: true + optional: true /builtins@1.0.3: resolution: {integrity: sha512-uYBjakWipfaO/bXI7E8rq6kpwHRZK5cNYrUv2OzZSI/FvmdMyXJ2tG9dKcjEC5YHmHpUAwsargWIZNWdxb/bnQ==} @@ -16517,7 +16606,6 @@ packages: /byline@5.0.0: resolution: {integrity: sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q==} engines: {node: '>=0.10.0'} - dev: false /bytes@3.0.0: resolution: {integrity: sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==} @@ -16653,7 +16741,7 @@ packages: /capnp-ts@0.7.0: resolution: {integrity: sha512-XKxXAC3HVPv7r674zP0VC3RTXz+/JKhfyw94ljvF80yynK6VkTnqE3jMuN8b3dUVmmc43TjyxjW4KTsmB3c86g==} dependencies: - debug: 4.3.6 + debug: 4.3.7 tslib: 2.6.2 transitivePeerDependencies: - supports-color @@ -17113,6 +17201,17 @@ packages: readable-stream: 3.6.0 dev: true + /compress-commons@6.0.2: + resolution: {integrity: sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==} + engines: {node: '>= 14'} + dependencies: + crc-32: 1.2.2 + crc32-stream: 6.0.0 + is-stream: 2.0.1 + normalize-path: 3.0.0 + readable-stream: 4.5.2 + dev: true + /compressible@2.0.18: resolution: {integrity: sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==} engines: {node: '>= 0.6'} @@ -17299,6 +17398,16 @@ packages: p-event: 5.0.1 dev: true + /cpu-features@0.0.10: + resolution: {integrity: sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==} + engines: {node: '>=10.0.0'} + requiresBuild: true + dependencies: + buildcheck: 0.0.6 + nan: 2.20.0 + dev: true + optional: true + /cpy-cli@5.0.0: resolution: {integrity: sha512-fb+DZYbL9KHc0BC4NYqGRrDIJZPXUmjjtqdw4XRRg8iV8dIfghUX/WiL+q4/B/KFTy3sK6jsbUhBaz0/Hxg7IQ==} engines: {node: '>=16'} @@ -17336,6 +17445,14 @@ packages: readable-stream: 3.6.0 dev: true + /crc32-stream@6.0.0: + resolution: {integrity: sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==} + engines: {node: '>= 14'} + dependencies: + crc-32: 1.2.2 + readable-stream: 4.5.2 + dev: true + /create-require@1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} @@ -17694,7 +17811,6 @@ packages: optional: true dependencies: ms: 2.1.3 - dev: false /decamelize-keys@1.1.1: resolution: {integrity: sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==} @@ -17924,6 +18040,36 @@ packages: /dlv@1.1.3: resolution: {integrity: sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==} + /docker-compose@0.24.8: + resolution: {integrity: sha512-plizRs/Vf15H+GCVxq2EUvyPK7ei9b/cVesHvjnX4xaXjM9spHe2Ytq0BitndFgvTJ3E3NljPNUEl7BAN43iZw==} + engines: {node: '>= 6.0.0'} + dependencies: + yaml: 2.3.1 + dev: true + + /docker-modem@3.0.8: + resolution: {integrity: sha512-f0ReSURdM3pcKPNS30mxOHSbaFLcknGmQjwSfmbcdOw1XWKXVhukM3NJHhr7NpY9BIyyWQb0EBo3KQvvuU5egQ==} + engines: {node: '>= 8.0'} + dependencies: + debug: 4.3.7 + readable-stream: 3.6.0 + split-ca: 1.0.1 + ssh2: 1.16.0 + transitivePeerDependencies: + - supports-color + dev: true + + /dockerode@3.3.5: + resolution: {integrity: sha512-/0YNa3ZDNeLr/tSckmD69+Gq+qVNhvKfAHNeZJBnp7EOP6RGKV8ORrJHkUn20So5wU+xxT7+1n5u8PjHbfjbSA==} + engines: {node: '>= 8.0'} + dependencies: + '@balena/dockerignore': 1.0.2 + docker-modem: 3.0.8 + tar-fs: 2.0.1 + transitivePeerDependencies: + - supports-color + dev: true + /doctrine@2.1.0: resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==} engines: {node: '>=0.10.0'} @@ -18092,7 +18238,7 @@ packages: resolution: {integrity: sha512-9Z0qLB0NIisTRt1DZ/8U2k12RJn8yls/nXMZLn+/N8hANT3TcYjKFKcwbw5zFQiN4NTde3TSY9zb79e1ij6j9Q==} dependencies: '@socket.io/component-emitter': 3.1.0 - debug: 4.3.6 + debug: 4.3.7 engine.io-parser: 5.2.2(patch_hash=e6nctogrhpxoivwiwy37ersfu4) ws: 8.11.0 xmlhttprequest-ssl: 2.0.0 @@ -19224,7 +19370,7 @@ packages: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.6 + debug: 4.3.7 doctrine: 3.0.0 escape-string-regexp: 4.0.0 eslint-scope: 7.2.0 @@ -19508,7 +19654,7 @@ packages: engines: {node: '>= 10.17.0'} hasBin: true dependencies: - debug: 4.3.6 + debug: 4.3.7 get-stream: 5.2.0 yauzl: 2.10.0 optionalDependencies: @@ -19540,7 +19686,6 @@ packages: /fast-fifo@1.3.2: resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} - dev: false /fast-glob@3.3.1: resolution: {integrity: sha512-kNFPyjhh5cKjrUltxs+wFx+ZkbRaxxmZ+X0ZU31SOsxCEtP9VPgtq2teZw1DebupL5GmDaNQ6yKMMVcM41iqDg==} @@ -20099,7 +20244,7 @@ packages: dependencies: basic-ftp: 5.0.3 data-uri-to-buffer: 5.0.1 - debug: 4.3.6 + debug: 4.3.7 fs-extra: 8.1.0 transitivePeerDependencies: - supports-color @@ -20569,7 +20714,7 @@ packages: engines: {node: '>= 14'} dependencies: agent-base: 7.1.0 - debug: 4.3.6 + debug: 4.3.7 transitivePeerDependencies: - supports-color @@ -20578,7 +20723,7 @@ packages: engines: {node: '>= 14'} dependencies: agent-base: 7.1.1 - debug: 4.3.6 + debug: 4.3.7 transitivePeerDependencies: - supports-color dev: false @@ -20597,7 +20742,7 @@ packages: engines: {node: '>= 6'} dependencies: agent-base: 6.0.2 - debug: 4.3.6 + debug: 4.3.7 transitivePeerDependencies: - supports-color dev: false @@ -20607,7 +20752,7 @@ packages: engines: {node: '>= 14'} dependencies: agent-base: 7.1.0 - debug: 4.3.6 + debug: 4.3.7 transitivePeerDependencies: - supports-color dev: false @@ -20617,7 +20762,7 @@ packages: engines: {node: '>= 14'} dependencies: agent-base: 7.1.1 - debug: 4.3.6 + debug: 4.3.7 transitivePeerDependencies: - supports-color @@ -20626,7 +20771,7 @@ packages: engines: {node: '>= 14'} dependencies: agent-base: 7.1.1 - debug: 4.3.6 + debug: 4.3.7 transitivePeerDependencies: - supports-color dev: false @@ -22340,7 +22485,7 @@ packages: resolution: {integrity: sha512-6Mj0yHLdUZjHnOPgr5xfWIMqMWS12zDN6iws9SLuSz76W8jTtAv24MN4/CL7gJrl5vtxGInkkqDv/JIoRsQOvA==} dependencies: '@types/debug': 4.1.12 - debug: 4.3.6 + debug: 4.3.7 decode-named-character-reference: 1.0.2 micromark-core-commonmark: 1.0.6 micromark-factory-space: 1.0.0 @@ -22715,7 +22860,6 @@ packages: /nan@2.20.0: resolution: {integrity: sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==} - dev: false /nano-css@5.3.5(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-vSB9X12bbNu4ALBu7nigJgRViZ6ja3OU7CeuiV1zMIbXOdmkLahgtPmh3GBOlDxbKY0CitqlPdOReGlBLSp+yg==} @@ -23584,7 +23728,7 @@ packages: dependencies: '@tootallnate/quickjs-emscripten': 0.23.0 agent-base: 7.1.0 - debug: 4.3.6 + debug: 4.3.7 get-uri: 6.0.1 http-proxy-agent: 7.0.0 https-proxy-agent: 7.0.1 @@ -23600,7 +23744,7 @@ packages: dependencies: '@tootallnate/quickjs-emscripten': 0.23.0 agent-base: 7.1.1 - debug: 4.3.6 + debug: 4.3.7 get-uri: 6.0.1 http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.5 @@ -24631,6 +24775,11 @@ packages: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} dev: true + /process@0.11.10: + resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} + engines: {node: '>= 0.6.0'} + dev: true + /progress@2.0.3: resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==} engines: {node: '>=0.4.0'} @@ -24674,6 +24823,21 @@ packages: object-assign: 4.1.1 react-is: 16.13.1 + /proper-lockfile@4.1.2: + resolution: {integrity: sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==} + dependencies: + graceful-fs: 4.2.11 + retry: 0.12.0 + signal-exit: 3.0.7 + dev: true + + /properties-reader@2.3.0: + resolution: {integrity: sha512-z597WicA7nDZxK12kZqHr2TcvwNU1GCfA5UwfDY/HDp3hXPoPlb5rlEx9bwGTiJnc0OqbBTkU975jDToth8Gxw==} + engines: {node: '>=14'} + dependencies: + mkdirp: 1.0.4 + dev: true + /property-information@6.2.0: resolution: {integrity: sha512-kma4U7AFCTwpqq5twzC1YVIDXSqg6qQK6JN0smOw8fgRy1OkMi0CYSzFmsy6dnqSenamAtj0CyXMUJ1Mf6oROg==} dev: true @@ -24746,7 +24910,7 @@ packages: engines: {node: '>= 14'} dependencies: agent-base: 7.1.1 - debug: 4.3.6 + debug: 4.3.7 http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.5 lru-cache: 7.18.3 @@ -24862,7 +25026,6 @@ packages: /queue-tick@1.0.1: resolution: {integrity: sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==} - dev: false /quick-lru@4.0.1: resolution: {integrity: sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==} @@ -25498,7 +25661,7 @@ packages: resolution: {integrity: sha512-VIMnQi/Z4HT2Fxuwg5KrY174U1VdUIASQVWXXyqtNRtxSr9IYkn1rsI6Tb6HsrHCmB7gVpNwX6JxPTHcH6IoTA==} engines: {node: '>=6'} dependencies: - graceful-fs: 4.2.10 + graceful-fs: 4.2.11 js-yaml: 3.14.1 pify: 4.0.1 strip-bom: 3.0.0 @@ -25523,6 +25686,17 @@ packages: string_decoder: 1.3.0 util-deprecate: 1.0.2 + /readable-stream@4.5.2: + resolution: {integrity: sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + dependencies: + abort-controller: 3.0.0 + buffer: 6.0.3 + events: 3.3.0 + process: 0.11.10 + string_decoder: 1.3.0 + dev: true + /readdir-glob@1.1.3: resolution: {integrity: sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==} dependencies: @@ -25784,7 +25958,7 @@ packages: remix-auth: ^3.6.0 dependencies: '@remix-run/server-runtime': 2.1.0(typescript@5.2.2) - debug: 4.3.6 + debug: 4.3.7 remix-auth: 3.6.0(@remix-run/react@2.1.0)(@remix-run/server-runtime@2.1.0) transitivePeerDependencies: - supports-color @@ -26504,7 +26678,7 @@ packages: engines: {node: '>=10.0.0'} dependencies: '@socket.io/component-emitter': 3.1.0 - debug: 4.3.6 + debug: 4.3.7 engine.io-client: 6.5.3 socket.io-parser: 4.2.4 transitivePeerDependencies: @@ -26543,7 +26717,7 @@ packages: accepts: 1.3.8 base64id: 2.0.0 cors: 2.8.5 - debug: 4.3.6 + debug: 4.3.7 engine.io: 6.5.4 socket.io-adapter: 2.5.4 socket.io-parser: 4.2.4 @@ -26576,7 +26750,7 @@ packages: accepts: 1.3.8 base64id: 2.0.0 cors: 2.8.5 - debug: 4.3.6 + debug: 4.3.7 engine.io: 6.5.4 socket.io-adapter: 2.5.4 socket.io-parser: 4.2.4 @@ -26591,7 +26765,7 @@ packages: engines: {node: '>= 14'} dependencies: agent-base: 7.1.0 - debug: 4.3.6 + debug: 4.3.7 socks: 2.7.1 transitivePeerDependencies: - supports-color @@ -26602,7 +26776,7 @@ packages: engines: {node: '>= 14'} dependencies: agent-base: 7.1.1 - debug: 4.3.6 + debug: 4.3.7 socks: 2.8.3 transitivePeerDependencies: - supports-color @@ -26703,6 +26877,10 @@ packages: /spdx-license-ids@3.0.12: resolution: {integrity: sha512-rr+VVSXtRhO4OHbXUiAF7xW3Bo9DuuF6C5jH+q/x15j2jniycgKbxU09Hr0WqlSLUs4i4ltHGXqTe7VHclYWyA==} + /split-ca@1.0.1: + resolution: {integrity: sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==} + dev: true + /split2@4.2.0: resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} engines: {node: '>= 10.x'} @@ -26727,6 +26905,25 @@ packages: - supports-color dev: false + /ssh-remote-port-forward@1.0.4: + resolution: {integrity: sha512-x0LV1eVDwjf1gmG7TTnfqIzf+3VPRz7vrNIjX6oYLbeCrf/PeVY6hkT68Mg+q02qXxQhrLjB0jfgvhevoCRmLQ==} + dependencies: + '@types/ssh2': 0.5.52 + ssh2: 1.16.0 + dev: true + + /ssh2@1.16.0: + resolution: {integrity: sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==} + engines: {node: '>=10.16.0'} + requiresBuild: true + dependencies: + asn1: 0.2.6 + bcrypt-pbkdf: 1.0.2 + optionalDependencies: + cpu-features: 0.0.10 + nan: 2.20.0 + dev: true + /sshpk@1.18.0: resolution: {integrity: sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ==} engines: {node: '>=0.10.0'} @@ -26846,7 +27043,6 @@ packages: text-decoder: 1.2.0 optionalDependencies: bare-events: 2.4.2 - dev: false /strict-event-emitter@0.5.1: resolution: {integrity: sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==} @@ -27332,6 +27528,15 @@ packages: resolution: {integrity: sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==} engines: {node: '>=6'} + /tar-fs@2.0.1: + resolution: {integrity: sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA==} + dependencies: + chownr: 1.1.4 + mkdirp-classic: 0.5.3 + pump: 3.0.0 + tar-stream: 2.2.0 + dev: true + /tar-fs@2.1.1: resolution: {integrity: sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==} dependencies: @@ -27349,7 +27554,6 @@ packages: optionalDependencies: bare-fs: 2.3.5 bare-path: 2.1.3 - dev: false /tar-stream@2.2.0: resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} @@ -27368,7 +27572,6 @@ packages: b4a: 1.6.6 fast-fifo: 1.3.2 streamx: 2.20.1 - dev: false /tar@6.1.13: resolution: {integrity: sha512-jdIBIN6LTIe2jqzay/2vtYLlBHa3JF42ot3h1dW8Q0PaAG4v8rm0cvpVePtau5C6OKXGGcgO9q2AMNSWxiLqKw==} @@ -27498,11 +27701,32 @@ packages: commander: 2.20.3 source-map-support: 0.5.21 + /testcontainers@10.13.1: + resolution: {integrity: sha512-JBbOhxmygj/ouH/47GnoVNt+c55Telh/45IjVxEbDoswsLchVmJiuKiw/eF6lE5i7LN+/99xsrSCttI3YRtirg==} + dependencies: + '@balena/dockerignore': 1.0.2 + '@types/dockerode': 3.3.31 + archiver: 7.0.1 + async-lock: 1.4.1 + byline: 5.0.0 + debug: 4.3.7 + docker-compose: 0.24.8 + dockerode: 3.3.5 + get-port: 5.1.1 + proper-lockfile: 4.1.2 + properties-reader: 2.3.0 + ssh-remote-port-forward: 1.0.4 + tar-fs: 3.0.6 + tmp: 0.2.3 + undici: 5.28.4 + transitivePeerDependencies: + - supports-color + dev: true + /text-decoder@1.2.0: resolution: {integrity: sha512-n1yg1mOj9DNpk3NeZOx7T6jchTbyJS3i3cucbNN6FcdPriMZx7NsgrGpWWdWZZGxD7ES1XB+3uoqHMgOKaN+fg==} dependencies: b4a: 1.6.6 - dev: false /text-table@0.2.0: resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==} @@ -27629,6 +27853,11 @@ packages: rimraf: 3.0.2 dev: true + /tmp@0.2.3: + resolution: {integrity: sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==} + engines: {node: '>=14.14'} + dev: true + /to-fast-properties@2.0.0: resolution: {integrity: sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==} engines: {node: '>=4'} @@ -28014,7 +28243,6 @@ packages: /tweetnacl@0.14.5: resolution: {integrity: sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==} - dev: false /type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} @@ -28786,7 +29014,7 @@ packages: hasBin: true dependencies: cac: 6.7.14 - debug: 4.3.6 + debug: 4.3.7 pathe: 1.1.2 picocolors: 1.0.1 vite: 5.2.7(@types/node@20.14.14) @@ -28807,7 +29035,7 @@ packages: hasBin: true dependencies: cac: 6.7.14 - debug: 4.3.6 + debug: 4.3.7 pathe: 1.1.2 tinyrainbow: 1.2.0 vite: 5.2.7(@types/node@20.14.14) @@ -29039,7 +29267,7 @@ packages: dependencies: '@types/node': 20.14.14 esbuild: 0.20.2 - postcss: 8.4.38 + postcss: 8.4.44 rollup: 4.13.2 optionalDependencies: fsevents: 2.3.3 @@ -29190,19 +29418,19 @@ packages: '@vitest/utils': 1.6.0 acorn-walk: 8.3.2 chai: 4.4.1 - debug: 4.3.4 + debug: 4.3.7 execa: 8.0.1 local-pkg: 0.5.0 - magic-string: 0.30.8 - pathe: 1.1.1 - picocolors: 1.0.0 + magic-string: 0.30.11 + pathe: 1.1.2 + picocolors: 1.0.1 std-env: 3.7.0 strip-literal: 2.1.0 - tinybench: 2.6.0 + tinybench: 2.9.0 tinypool: 0.8.3 vite: 5.2.7(@types/node@20.14.14) vite-node: 1.6.0(@types/node@20.14.14) - why-is-node-running: 2.2.2 + why-is-node-running: 2.3.0 transitivePeerDependencies: - less - lightningcss @@ -29862,6 +30090,15 @@ packages: readable-stream: 3.6.0 dev: true + /zip-stream@6.0.1: + resolution: {integrity: sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==} + engines: {node: '>= 14'} + dependencies: + archiver-utils: 5.0.2 + compress-commons: 6.0.2 + readable-stream: 4.5.2 + dev: true + /zod-error@1.5.0: resolution: {integrity: sha512-zzopKZ/skI9iXpqCEPj+iLCKl9b88E43ehcU+sbRoHuwGd9F1IDVGQ70TyO6kmfiRL1g4IXkjsXK+g1gLYl4WQ==} dependencies: From cb4fd95107dc18624a4df0634d24cb0788ecc6e1 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 22 Sep 2024 17:45:38 +0100 Subject: [PATCH 010/114] Use beforeEach/afterEach --- packages/run-engine/package.json | 1 + packages/run-engine/src/index.test.ts | 95 +++++++++++++-------------- pnpm-lock.yaml | 11 ++++ 3 files changed, 58 insertions(+), 49 deletions(-) diff --git a/packages/run-engine/package.json b/packages/run-engine/package.json index 7c6075ae9a..764eea7723 100644 --- a/packages/run-engine/package.json +++ b/packages/run-engine/package.json @@ -12,6 +12,7 @@ }, "devDependencies": { "@testcontainers/postgresql": "^10.13.1", + "@testcontainers/redis": "^10.13.1", "testcontainers": "^10.13.1", "vitest": "^1.4.0" }, diff --git a/packages/run-engine/src/index.test.ts b/packages/run-engine/src/index.test.ts index 6fcccd8601..aa26be0e4b 100644 --- a/packages/run-engine/src/index.test.ts +++ b/packages/run-engine/src/index.test.ts @@ -1,60 +1,57 @@ import { PrismaClient, Prisma } from "@trigger.dev/database"; -import { PostgreSqlContainer } from "@testcontainers/postgresql"; +import { PostgreSqlContainer, StartedPostgreSqlContainer } from "@testcontainers/postgresql"; import path from "path"; import { execSync } from "child_process"; import exp from "constants"; -describe("Placeholder", () => { - it("should pass", () => { - expect(true).toBe(true); - }); - - it( - "should connect and return a query result", - { - timeout: 60_000, - }, - async () => { - const container = await new PostgreSqlContainer().start(); - - // Run migrations - const databasePath = path.resolve(__dirname, "../../database"); - try { - execSync(`npx prisma db push --schema ${databasePath}/prisma/schema.prisma`, { - env: { - ...process.env, - DATABASE_URL: container.getConnectionUri(), - DIRECT_URL: container.getConnectionUri(), - }, - }); - } catch (error) { - expect(error).toBeUndefined(); - } - - // console.log(container.getConnectionUri()); - - const prisma = new PrismaClient({ - datasources: { - db: { - url: container.getConnectionUri(), - }, - }, - }); - prisma.$connect(); +let container: StartedPostgreSqlContainer; +let prisma: PrismaClient; - const user = await prisma.user.create({ - data: { - authenticationMethod: "MAGIC_LINK", - email: "test@example.com", +describe("Placeholder", () => { + beforeEach(async () => { + container = await new PostgreSqlContainer().start(); + + // Run migrations + const databasePath = path.resolve(__dirname, "../../database"); + try { + execSync(`npx prisma db push --schema ${databasePath}/prisma/schema.prisma`, { + env: { + ...process.env, + DATABASE_URL: container.getConnectionUri(), + DIRECT_URL: container.getConnectionUri(), }, }); + } catch (error) { + expect(error).toBeUndefined(); + } - const result = await prisma.user.findMany(); - expect(result.length).toEqual(1); - expect(result[0].email).toEqual("test@example.com"); + // console.log(container.getConnectionUri()); - await prisma.$disconnect(); - await container.stop(); - } - ); + prisma = new PrismaClient({ + datasources: { + db: { + url: container.getConnectionUri(), + }, + }, + }); + prisma.$connect(); + }, 10_000); + + afterEach(async () => { + await prisma?.$disconnect(); + await container?.stop(); + }, 10_000); + + it("should get a user from Postgres", async () => { + const user = await prisma.user.create({ + data: { + authenticationMethod: "MAGIC_LINK", + email: "test@example.com", + }, + }); + + const result = await prisma.user.findMany(); + expect(result.length).toEqual(1); + expect(result[0].email).toEqual("test@example.com"); + }); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9db834db71..c294d1fddb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1252,6 +1252,9 @@ importers: '@testcontainers/postgresql': specifier: ^10.13.1 version: 10.13.1 + '@testcontainers/redis': + specifier: ^10.13.1 + version: 10.13.1 testcontainers: specifier: ^10.13.1 version: 10.13.1 @@ -14139,6 +14142,14 @@ packages: - supports-color dev: true + /@testcontainers/redis@10.13.1: + resolution: {integrity: sha512-pXg15o4oTRaEyb5xryQZUdePtoRId/+3TeU7vnUgDpqOmRacF8/7zL7jqs13uPh1uea6M7a8MDgHQM8j8kXZUg==} + dependencies: + testcontainers: 10.13.1 + transitivePeerDependencies: + - supports-color + dev: true + /@testing-library/dom@8.19.1: resolution: {integrity: sha512-P6iIPyYQ+qH8CvGauAqanhVnjrnRe0IZFSYCeGkSRW9q3u8bdVn2NPI+lasFyVsEQn1J/IFmp5Aax41+dAP9wg==} engines: {node: '>=12'} From c1c8fd2cf72f2c80f28ea06f10c7c949f94c176c Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 22 Sep 2024 18:10:47 +0100 Subject: [PATCH 011/114] Simple Prisma and Redis test --- packages/run-engine/src/index.test.ts | 60 +++++++++++---------------- packages/run-engine/src/test/utils.ts | 49 ++++++++++++++++++++++ 2 files changed, 73 insertions(+), 36 deletions(-) create mode 100644 packages/run-engine/src/test/utils.ts diff --git a/packages/run-engine/src/index.test.ts b/packages/run-engine/src/index.test.ts index aa26be0e4b..8238c0bb61 100644 --- a/packages/run-engine/src/index.test.ts +++ b/packages/run-engine/src/index.test.ts @@ -1,49 +1,33 @@ -import { PrismaClient, Prisma } from "@trigger.dev/database"; -import { PostgreSqlContainer, StartedPostgreSqlContainer } from "@testcontainers/postgresql"; -import path from "path"; -import { execSync } from "child_process"; -import exp from "constants"; +import { StartedPostgreSqlContainer } from "@testcontainers/postgresql"; +import { PrismaClient } from "@trigger.dev/database"; +import { createTestRedisClient, createTestPrismaClient } from "./test/utils"; +import { StartedRedisContainer } from "@testcontainers/redis"; +import { Redis } from "ioredis"; -let container: StartedPostgreSqlContainer; +let postgresContainer: StartedPostgreSqlContainer; let prisma: PrismaClient; +let redisContainer: StartedRedisContainer; +let redis: Redis; describe("Placeholder", () => { beforeEach(async () => { - container = await new PostgreSqlContainer().start(); - - // Run migrations - const databasePath = path.resolve(__dirname, "../../database"); - try { - execSync(`npx prisma db push --schema ${databasePath}/prisma/schema.prisma`, { - env: { - ...process.env, - DATABASE_URL: container.getConnectionUri(), - DIRECT_URL: container.getConnectionUri(), - }, - }); - } catch (error) { - expect(error).toBeUndefined(); - } - - // console.log(container.getConnectionUri()); - - prisma = new PrismaClient({ - datasources: { - db: { - url: container.getConnectionUri(), - }, - }, - }); - prisma.$connect(); - }, 10_000); + const pg = await createTestPrismaClient(); + postgresContainer = pg.container; + prisma = pg.prisma; + const rd = await createTestRedisClient(); + redisContainer = rd.container; + redis = rd.client; + }, 30_000); afterEach(async () => { await prisma?.$disconnect(); - await container?.stop(); + await postgresContainer?.stop(); + await redis?.quit(); + await redisContainer?.stop(); }, 10_000); - it("should get a user from Postgres", async () => { - const user = await prisma.user.create({ + it("Simple connection test", async () => { + await prisma.user.create({ data: { authenticationMethod: "MAGIC_LINK", email: "test@example.com", @@ -53,5 +37,9 @@ describe("Placeholder", () => { const result = await prisma.user.findMany(); expect(result.length).toEqual(1); expect(result[0].email).toEqual("test@example.com"); + + await redis.set("mykey", "value"); + const value = await redis.get("mykey"); + expect(value).toEqual("value"); }); }); diff --git a/packages/run-engine/src/test/utils.ts b/packages/run-engine/src/test/utils.ts new file mode 100644 index 0000000000..a962e250c6 --- /dev/null +++ b/packages/run-engine/src/test/utils.ts @@ -0,0 +1,49 @@ +import { PrismaClient, Prisma } from "@trigger.dev/database"; +import { PostgreSqlContainer, StartedPostgreSqlContainer } from "@testcontainers/postgresql"; +import { RedisContainer } from "@testcontainers/redis"; +import path from "path"; +import { execSync } from "child_process"; +import Redis, { RedisOptions } from "ioredis"; + +export async function createTestPrismaClient() { + const container = await new PostgreSqlContainer().start(); + + // Run migrations + const databasePath = path.resolve(__dirname, "../../../database"); + + execSync(`npx prisma db push --schema ${databasePath}/prisma/schema.prisma`, { + env: { + ...process.env, + DATABASE_URL: container.getConnectionUri(), + DIRECT_URL: container.getConnectionUri(), + }, + }); + + // console.log(container.getConnectionUri()); + + const prisma = new PrismaClient({ + datasources: { + db: { + url: container.getConnectionUri(), + }, + }, + }); + prisma.$connect(); + + return { prisma, container }; +} + +export async function createTestRedisClient() { + const container = await new RedisContainer().start(); + try { + const client = new Redis({ + host: container.getHost(), + port: container.getPort(), + password: container.getPassword(), + }); + return { client, container }; + } catch (e) { + console.error(e); + throw e; + } +} From 3a3043eae3caac9f9dd201e764bf01127268ccb0 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 22 Sep 2024 18:39:10 +0100 Subject: [PATCH 012/114] Return Redis options instead of a client --- packages/run-engine/package.json | 3 ++- packages/run-engine/src/index.test.ts | 10 ++++++---- packages/run-engine/src/test/utils.ts | 21 +++++++++++---------- pnpm-lock.yaml | 3 +++ 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/packages/run-engine/package.json b/packages/run-engine/package.json index 764eea7723..9d3b50b047 100644 --- a/packages/run-engine/package.json +++ b/packages/run-engine/package.json @@ -8,7 +8,8 @@ "@trigger.dev/core": "workspace:*", "@trigger.dev/database": "workspace:*", "ioredis": "^5.3.2", - "typescript": "^4.8.4" + "typescript": "^4.8.4", + "zod": "3.22.3" }, "devDependencies": { "@testcontainers/postgresql": "^10.13.1", diff --git a/packages/run-engine/src/index.test.ts b/packages/run-engine/src/index.test.ts index 8238c0bb61..5d22526fd2 100644 --- a/packages/run-engine/src/index.test.ts +++ b/packages/run-engine/src/index.test.ts @@ -2,12 +2,12 @@ import { StartedPostgreSqlContainer } from "@testcontainers/postgresql"; import { PrismaClient } from "@trigger.dev/database"; import { createTestRedisClient, createTestPrismaClient } from "./test/utils"; import { StartedRedisContainer } from "@testcontainers/redis"; -import { Redis } from "ioredis"; +import { Redis, RedisOptions } from "ioredis"; let postgresContainer: StartedPostgreSqlContainer; let prisma: PrismaClient; let redisContainer: StartedRedisContainer; -let redis: Redis; +let redisOptions: RedisOptions; describe("Placeholder", () => { beforeEach(async () => { @@ -16,13 +16,12 @@ describe("Placeholder", () => { prisma = pg.prisma; const rd = await createTestRedisClient(); redisContainer = rd.container; - redis = rd.client; + redisOptions = rd.options; }, 30_000); afterEach(async () => { await prisma?.$disconnect(); await postgresContainer?.stop(); - await redis?.quit(); await redisContainer?.stop(); }, 10_000); @@ -38,8 +37,11 @@ describe("Placeholder", () => { expect(result.length).toEqual(1); expect(result[0].email).toEqual("test@example.com"); + const redis = new Redis(redisOptions); await redis.set("mykey", "value"); const value = await redis.get("mykey"); expect(value).toEqual("value"); + + await redis.quit(); }); }); diff --git a/packages/run-engine/src/test/utils.ts b/packages/run-engine/src/test/utils.ts index a962e250c6..62e188d5e6 100644 --- a/packages/run-engine/src/test/utils.ts +++ b/packages/run-engine/src/test/utils.ts @@ -1,9 +1,8 @@ -import { PrismaClient, Prisma } from "@trigger.dev/database"; -import { PostgreSqlContainer, StartedPostgreSqlContainer } from "@testcontainers/postgresql"; +import { PostgreSqlContainer } from "@testcontainers/postgresql"; import { RedisContainer } from "@testcontainers/redis"; -import path from "path"; +import { PrismaClient } from "@trigger.dev/database"; import { execSync } from "child_process"; -import Redis, { RedisOptions } from "ioredis"; +import path from "path"; export async function createTestPrismaClient() { const container = await new PostgreSqlContainer().start(); @@ -36,12 +35,14 @@ export async function createTestPrismaClient() { export async function createTestRedisClient() { const container = await new RedisContainer().start(); try { - const client = new Redis({ - host: container.getHost(), - port: container.getPort(), - password: container.getPassword(), - }); - return { client, container }; + return { + options: { + host: container.getHost(), + port: container.getPort(), + password: container.getPassword(), + }, + container, + }; } catch (e) { console.error(e); throw e; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c294d1fddb..dad2df8a4d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1248,6 +1248,9 @@ importers: typescript: specifier: ^4.8.4 version: 4.9.5 + zod: + specifier: 3.22.3 + version: 3.22.3 devDependencies: '@testcontainers/postgresql': specifier: ^10.13.1 From 7836229b97c5e75ad6beacbd5444eb3ba9e1b753 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 22 Sep 2024 18:41:53 +0100 Subject: [PATCH 013/114] Simplified things --- packages/run-engine/src/index.test.ts | 25 ++++++++++++++++--------- packages/run-engine/src/test/utils.ts | 20 +++----------------- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/packages/run-engine/src/index.test.ts b/packages/run-engine/src/index.test.ts index 5d22526fd2..0609d6395e 100644 --- a/packages/run-engine/src/index.test.ts +++ b/packages/run-engine/src/index.test.ts @@ -1,31 +1,34 @@ import { StartedPostgreSqlContainer } from "@testcontainers/postgresql"; import { PrismaClient } from "@trigger.dev/database"; -import { createTestRedisClient, createTestPrismaClient } from "./test/utils"; +import { createRedisContainer, createPostgresContainer } from "./test/utils"; import { StartedRedisContainer } from "@testcontainers/redis"; import { Redis, RedisOptions } from "ioredis"; let postgresContainer: StartedPostgreSqlContainer; -let prisma: PrismaClient; let redisContainer: StartedRedisContainer; -let redisOptions: RedisOptions; describe("Placeholder", () => { beforeEach(async () => { - const pg = await createTestPrismaClient(); + const pg = await createPostgresContainer(); postgresContainer = pg.container; - prisma = pg.prisma; - const rd = await createTestRedisClient(); + const rd = await createRedisContainer(); redisContainer = rd.container; - redisOptions = rd.options; }, 30_000); afterEach(async () => { - await prisma?.$disconnect(); await postgresContainer?.stop(); await redisContainer?.stop(); }, 10_000); it("Simple connection test", async () => { + const prisma = new PrismaClient({ + datasources: { + db: { + url: postgresContainer.getConnectionUri(), + }, + }, + }); + await prisma.user.create({ data: { authenticationMethod: "MAGIC_LINK", @@ -37,7 +40,11 @@ describe("Placeholder", () => { expect(result.length).toEqual(1); expect(result[0].email).toEqual("test@example.com"); - const redis = new Redis(redisOptions); + const redis = new Redis({ + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + }); await redis.set("mykey", "value"); const value = await redis.get("mykey"); expect(value).toEqual("value"); diff --git a/packages/run-engine/src/test/utils.ts b/packages/run-engine/src/test/utils.ts index 62e188d5e6..d1dfc2857c 100644 --- a/packages/run-engine/src/test/utils.ts +++ b/packages/run-engine/src/test/utils.ts @@ -4,7 +4,7 @@ import { PrismaClient } from "@trigger.dev/database"; import { execSync } from "child_process"; import path from "path"; -export async function createTestPrismaClient() { +export async function createPostgresContainer() { const container = await new PostgreSqlContainer().start(); // Run migrations @@ -20,27 +20,13 @@ export async function createTestPrismaClient() { // console.log(container.getConnectionUri()); - const prisma = new PrismaClient({ - datasources: { - db: { - url: container.getConnectionUri(), - }, - }, - }); - prisma.$connect(); - - return { prisma, container }; + return { url: container.getConnectionUri(), container }; } -export async function createTestRedisClient() { +export async function createRedisContainer() { const container = await new RedisContainer().start(); try { return { - options: { - host: container.getHost(), - port: container.getPort(), - password: container.getPassword(), - }, container, }; } catch (e) { From 9a9e8f14865f16d7646f01c44928f95d6dda02d5 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 22 Sep 2024 18:56:44 +0100 Subject: [PATCH 014/114] A very simple FIFO pull-based queue to check the tests working properly --- packages/run-engine/src/index.test.ts | 38 +++++++++++++++++ packages/run-engine/src/simpleQueue.ts | 57 ++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 packages/run-engine/src/simpleQueue.ts diff --git a/packages/run-engine/src/index.test.ts b/packages/run-engine/src/index.test.ts index 0609d6395e..fc0e888c83 100644 --- a/packages/run-engine/src/index.test.ts +++ b/packages/run-engine/src/index.test.ts @@ -3,6 +3,8 @@ import { PrismaClient } from "@trigger.dev/database"; import { createRedisContainer, createPostgresContainer } from "./test/utils"; import { StartedRedisContainer } from "@testcontainers/redis"; import { Redis, RedisOptions } from "ioredis"; +import SimpleQueue from "./simpleQueue"; +import { z } from "zod"; let postgresContainer: StartedPostgreSqlContainer; let redisContainer: StartedRedisContainer; @@ -51,4 +53,40 @@ describe("Placeholder", () => { await redis.quit(); }); + + it("SimpleQueue", async () => { + const queue = new SimpleQueue( + "test", + z.object({ + value: z.number(), + }), + { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + } + ); + + await queue.enqueue("1", { value: 1 }); + await queue.enqueue("2", { value: 2 }, new Date(Date.now() + 100)); + await queue.enqueue("3", { value: 3 }); + + const first = await queue.dequeue(); + expect(first).toEqual({ id: "1", item: { value: 1 } }); + + //we added the second one with a delay + const third = await queue.dequeue(); + expect(third).toEqual({ id: "3", item: { value: 3 } }); + + //wait for 100 ms + await new Promise((resolve) => setTimeout(resolve, 100)); + + const second = await queue.dequeue(); + expect(second).toEqual({ id: "2", item: { value: 2 } }); + + const fourth = await queue.dequeue(); + expect(fourth).toBeNull(); + + await queue.close(); + }); }); diff --git a/packages/run-engine/src/simpleQueue.ts b/packages/run-engine/src/simpleQueue.ts new file mode 100644 index 0000000000..c0373d4f0d --- /dev/null +++ b/packages/run-engine/src/simpleQueue.ts @@ -0,0 +1,57 @@ +import Redis, { RedisOptions } from "ioredis"; +import { z } from "zod"; + +class SimpleQueue { + private redis: Redis; + private keyPrefix: string; + private schema: T; + + constructor(name: string, schema: T, redisOptions: RedisOptions) { + this.redis = new Redis(redisOptions); + this.keyPrefix = `queue:${name}:`; + this.schema = schema; + } + + async enqueue(id: string, item: z.infer, availableAt?: Date): Promise { + const score = availableAt ? availableAt.getTime() : Date.now(); + const serializedItem = JSON.stringify(item); + + await this.redis + .multi() + .zadd(`${this.keyPrefix}queue`, score, id) + .hset(`${this.keyPrefix}items`, id, serializedItem) + .exec(); + } + + async dequeue(): Promise<{ id: string; item: z.infer } | null> { + const now = Date.now(); + + const result = await this.redis + .multi() + .zrangebyscore(`${this.keyPrefix}queue`, "-inf", now, "LIMIT", 0, 1) + .zremrangebyrank(`${this.keyPrefix}queue`, 0, 0) + .exec(); + + if (!result || !result[0][1] || (result[0][1] as string[]).length === 0) { + return null; + } + + const id = (result[0][1] as string[])[0]; + const serializedItem = await this.redis.hget(`${this.keyPrefix}items`, id); + + if (serializedItem) { + await this.redis.hdel(`${this.keyPrefix}items`, id); + const parsedItem = JSON.parse(serializedItem); + const validatedItem = this.schema.parse(parsedItem); + return { id, item: validatedItem }; + } + + return null; + } + + async close(): Promise { + await this.redis.quit(); + } +} + +export default SimpleQueue; From 3d15109a4645e19fcec08fba08e4fe89550fccfe Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 22 Sep 2024 20:16:36 +0100 Subject: [PATCH 015/114] Use vitest extend --- packages/run-engine/src/index.test.ts | 118 ++++++------------ .../{simpleQueue.ts => simpleQueue/index.ts} | 6 +- packages/run-engine/src/test/containerTest.ts | 46 +++++++ 3 files changed, 90 insertions(+), 80 deletions(-) rename packages/run-engine/src/{simpleQueue.ts => simpleQueue/index.ts} (94%) create mode 100644 packages/run-engine/src/test/containerTest.ts diff --git a/packages/run-engine/src/index.test.ts b/packages/run-engine/src/index.test.ts index fc0e888c83..4c731a7be3 100644 --- a/packages/run-engine/src/index.test.ts +++ b/packages/run-engine/src/index.test.ts @@ -1,92 +1,56 @@ -import { StartedPostgreSqlContainer } from "@testcontainers/postgresql"; -import { PrismaClient } from "@trigger.dev/database"; -import { createRedisContainer, createPostgresContainer } from "./test/utils"; -import { StartedRedisContainer } from "@testcontainers/redis"; -import { Redis, RedisOptions } from "ioredis"; -import SimpleQueue from "./simpleQueue"; +import { expect } from "vitest"; import { z } from "zod"; +import { SimpleQueue } from "./simpleQueue"; +import { containerTest } from "./test/containerTest"; + +// Use the extended test +containerTest("Simple connection test", async ({ prisma, redis }) => { + await prisma.user.create({ + data: { + authenticationMethod: "MAGIC_LINK", + email: "test@example.com", + }, + }); -let postgresContainer: StartedPostgreSqlContainer; -let redisContainer: StartedRedisContainer; - -describe("Placeholder", () => { - beforeEach(async () => { - const pg = await createPostgresContainer(); - postgresContainer = pg.container; - const rd = await createRedisContainer(); - redisContainer = rd.container; - }, 30_000); - - afterEach(async () => { - await postgresContainer?.stop(); - await redisContainer?.stop(); - }, 10_000); - - it("Simple connection test", async () => { - const prisma = new PrismaClient({ - datasources: { - db: { - url: postgresContainer.getConnectionUri(), - }, - }, - }); - - await prisma.user.create({ - data: { - authenticationMethod: "MAGIC_LINK", - email: "test@example.com", - }, - }); + const result = await prisma.user.findMany(); + expect(result.length).toEqual(1); + expect(result[0].email).toEqual("test@example.com"); - const result = await prisma.user.findMany(); - expect(result.length).toEqual(1); - expect(result[0].email).toEqual("test@example.com"); + await redis.set("mykey", "value"); + const value = await redis.get("mykey"); + expect(value).toEqual("value"); +}); - const redis = new Redis({ +containerTest("SimpleQueue", async ({ redisContainer }) => { + const queue = new SimpleQueue( + "test", + z.object({ + value: z.number(), + }), + { host: redisContainer.getHost(), port: redisContainer.getPort(), password: redisContainer.getPassword(), - }); - await redis.set("mykey", "value"); - const value = await redis.get("mykey"); - expect(value).toEqual("value"); - - await redis.quit(); - }); + } + ); - it("SimpleQueue", async () => { - const queue = new SimpleQueue( - "test", - z.object({ - value: z.number(), - }), - { - host: redisContainer.getHost(), - port: redisContainer.getPort(), - password: redisContainer.getPassword(), - } - ); + await queue.enqueue("1", { value: 1 }); + await queue.enqueue("2", { value: 2 }, new Date(Date.now() + 100)); + await queue.enqueue("3", { value: 3 }); - await queue.enqueue("1", { value: 1 }); - await queue.enqueue("2", { value: 2 }, new Date(Date.now() + 100)); - await queue.enqueue("3", { value: 3 }); + const first = await queue.dequeue(); + expect(first).toEqual({ id: "1", item: { value: 1 } }); - const first = await queue.dequeue(); - expect(first).toEqual({ id: "1", item: { value: 1 } }); + const third = await queue.dequeue(); + expect(third).toEqual({ id: "3", item: { value: 3 } }); - //we added the second one with a delay - const third = await queue.dequeue(); - expect(third).toEqual({ id: "3", item: { value: 3 } }); + await new Promise((resolve) => setTimeout(resolve, 100)); - //wait for 100 ms - await new Promise((resolve) => setTimeout(resolve, 100)); + const second = await queue.dequeue(); + expect(second).toEqual({ id: "2", item: { value: 2 } }); - const second = await queue.dequeue(); - expect(second).toEqual({ id: "2", item: { value: 2 } }); + const fourth = await queue.dequeue(); + expect(fourth).toBeNull(); - const fourth = await queue.dequeue(); - expect(fourth).toBeNull(); - - await queue.close(); - }); + await queue.close(); }); diff --git a/packages/run-engine/src/simpleQueue.ts b/packages/run-engine/src/simpleQueue/index.ts similarity index 94% rename from packages/run-engine/src/simpleQueue.ts rename to packages/run-engine/src/simpleQueue/index.ts index c0373d4f0d..383d19f756 100644 --- a/packages/run-engine/src/simpleQueue.ts +++ b/packages/run-engine/src/simpleQueue/index.ts @@ -1,12 +1,14 @@ import Redis, { RedisOptions } from "ioredis"; import { z } from "zod"; -class SimpleQueue { +export class SimpleQueue { + name: string; private redis: Redis; private keyPrefix: string; private schema: T; constructor(name: string, schema: T, redisOptions: RedisOptions) { + this.name = name; this.redis = new Redis(redisOptions); this.keyPrefix = `queue:${name}:`; this.schema = schema; @@ -53,5 +55,3 @@ class SimpleQueue { await this.redis.quit(); } } - -export default SimpleQueue; diff --git a/packages/run-engine/src/test/containerTest.ts b/packages/run-engine/src/test/containerTest.ts new file mode 100644 index 0000000000..3d54d8329a --- /dev/null +++ b/packages/run-engine/src/test/containerTest.ts @@ -0,0 +1,46 @@ +import { StartedPostgreSqlContainer } from "@testcontainers/postgresql"; +import { StartedRedisContainer } from "@testcontainers/redis"; +import { PrismaClient } from "@trigger.dev/database"; +import { Redis } from "ioredis"; +import { test } from "vitest"; +import { createPostgresContainer, createRedisContainer } from "./utils"; + +type ContainerTest = { + postgresContainer: StartedPostgreSqlContainer; + redisContainer: StartedRedisContainer; + prisma: PrismaClient; + redis: Redis; +}; + +export const containerTest = test.extend({ + postgresContainer: async ({}, use) => { + const { container } = await createPostgresContainer(); + await use(container); + await container.stop(); + }, + redisContainer: async ({}, use) => { + const { container } = await createRedisContainer(); + await use(container); + await container.stop(); + }, + prisma: async ({ postgresContainer }, use) => { + const prisma = new PrismaClient({ + datasources: { + db: { + url: postgresContainer.getConnectionUri(), + }, + }, + }); + await use(prisma); + await prisma.$disconnect(); + }, + redis: async ({ redisContainer }, use) => { + const redis = new Redis({ + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + }); + await use(redis); + await redis.quit(); + }, +}); From a1769540322d5f9b1931307ce9cb381d60da44c8 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 22 Sep 2024 20:33:15 +0100 Subject: [PATCH 016/114] Separate redis, postgres and combined tests for faster testing --- packages/run-engine/src/index.test.ts | 9 +- packages/run-engine/src/test/containerTest.ts | 89 ++++++++++++------- 2 files changed, 60 insertions(+), 38 deletions(-) diff --git a/packages/run-engine/src/index.test.ts b/packages/run-engine/src/index.test.ts index 4c731a7be3..ea0a7d1e09 100644 --- a/packages/run-engine/src/index.test.ts +++ b/packages/run-engine/src/index.test.ts @@ -1,10 +1,9 @@ import { expect } from "vitest"; import { z } from "zod"; import { SimpleQueue } from "./simpleQueue"; -import { containerTest } from "./test/containerTest"; +import { containerTest, postgresTest, redisTest } from "./test/containerTest"; -// Use the extended test -containerTest("Simple connection test", async ({ prisma, redis }) => { +postgresTest("Prisma create user", { timeout: 15_000 }, async ({ prisma }) => { await prisma.user.create({ data: { authenticationMethod: "MAGIC_LINK", @@ -15,13 +14,15 @@ containerTest("Simple connection test", async ({ prisma, redis }) => { const result = await prisma.user.findMany(); expect(result.length).toEqual(1); expect(result[0].email).toEqual("test@example.com"); +}); +redisTest("Set/get values", async ({ redis }) => { await redis.set("mykey", "value"); const value = await redis.get("mykey"); expect(value).toEqual("value"); }); -containerTest("SimpleQueue", async ({ redisContainer }) => { +redisTest("SimpleQueue enqueue/dequeue", async ({ redisContainer }) => { const queue = new SimpleQueue( "test", z.object({ diff --git a/packages/run-engine/src/test/containerTest.ts b/packages/run-engine/src/test/containerTest.ts index 3d54d8329a..098f784768 100644 --- a/packages/run-engine/src/test/containerTest.ts +++ b/packages/run-engine/src/test/containerTest.ts @@ -2,45 +2,66 @@ import { StartedPostgreSqlContainer } from "@testcontainers/postgresql"; import { StartedRedisContainer } from "@testcontainers/redis"; import { PrismaClient } from "@trigger.dev/database"; import { Redis } from "ioredis"; -import { test } from "vitest"; +import { test, TestAPI } from "vitest"; import { createPostgresContainer, createRedisContainer } from "./utils"; -type ContainerTest = { +type PostgresContext = { postgresContainer: StartedPostgreSqlContainer; - redisContainer: StartedRedisContainer; prisma: PrismaClient; - redis: Redis; }; -export const containerTest = test.extend({ - postgresContainer: async ({}, use) => { - const { container } = await createPostgresContainer(); - await use(container); - await container.stop(); - }, - redisContainer: async ({}, use) => { - const { container } = await createRedisContainer(); - await use(container); - await container.stop(); - }, - prisma: async ({ postgresContainer }, use) => { - const prisma = new PrismaClient({ - datasources: { - db: { - url: postgresContainer.getConnectionUri(), - }, +type RedisContext = { redisContainer: StartedRedisContainer; redis: Redis }; +type ContainerContext = PostgresContext & RedisContext; + +type Use = (value: T) => Promise; + +const postgresContainer = async ({}, use: Use) => { + const { container } = await createPostgresContainer(); + await use(container); + await container.stop(); +}; + +const redisContainer = async ({}, use: Use) => { + const { container } = await createRedisContainer(); + await use(container); + await container.stop(); +}; + +const prisma = async ( + { postgresContainer }: { postgresContainer: StartedPostgreSqlContainer }, + use: Use +) => { + const prisma = new PrismaClient({ + datasources: { + db: { + url: postgresContainer.getConnectionUri(), }, - }); - await use(prisma); - await prisma.$disconnect(); - }, - redis: async ({ redisContainer }, use) => { - const redis = new Redis({ - host: redisContainer.getHost(), - port: redisContainer.getPort(), - password: redisContainer.getPassword(), - }); - await use(redis); - await redis.quit(); - }, + }, + }); + await use(prisma); + await prisma.$disconnect(); +}; + +const redis = async ( + { redisContainer }: { redisContainer: StartedRedisContainer }, + use: Use +) => { + const redis = new Redis({ + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + }); + await use(redis); + await redis.quit(); +}; + +export const containerTest = test.extend({ + postgresContainer, + prisma, + redisContainer, + redis, }); + +export const postgresTest = test.extend({ postgresContainer, prisma }); + +export const redisTest = test.extend({ redisContainer, redis }); From a3cba4ee1c017fecfa78bac6716d3a97e7eee52e Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 23 Sep 2024 14:02:37 +0100 Subject: [PATCH 017/114] Some fixes and test improvements --- packages/run-engine/src/index.test.ts | 38 +----- .../run-engine/src/simpleQueue/index.test.ts | 90 ++++++++++++ packages/run-engine/src/simpleQueue/index.ts | 129 ++++++++++++++---- packages/run-engine/src/test/containerTest.ts | 20 +-- 4 files changed, 204 insertions(+), 73 deletions(-) create mode 100644 packages/run-engine/src/simpleQueue/index.test.ts diff --git a/packages/run-engine/src/index.test.ts b/packages/run-engine/src/index.test.ts index ea0a7d1e09..0101dd00f0 100644 --- a/packages/run-engine/src/index.test.ts +++ b/packages/run-engine/src/index.test.ts @@ -1,7 +1,5 @@ import { expect } from "vitest"; -import { z } from "zod"; -import { SimpleQueue } from "./simpleQueue"; -import { containerTest, postgresTest, redisTest } from "./test/containerTest"; +import { postgresTest, redisTest } from "./test/containerTest"; postgresTest("Prisma create user", { timeout: 15_000 }, async ({ prisma }) => { await prisma.user.create({ @@ -21,37 +19,3 @@ redisTest("Set/get values", async ({ redis }) => { const value = await redis.get("mykey"); expect(value).toEqual("value"); }); - -redisTest("SimpleQueue enqueue/dequeue", async ({ redisContainer }) => { - const queue = new SimpleQueue( - "test", - z.object({ - value: z.number(), - }), - { - host: redisContainer.getHost(), - port: redisContainer.getPort(), - password: redisContainer.getPassword(), - } - ); - - await queue.enqueue("1", { value: 1 }); - await queue.enqueue("2", { value: 2 }, new Date(Date.now() + 100)); - await queue.enqueue("3", { value: 3 }); - - const first = await queue.dequeue(); - expect(first).toEqual({ id: "1", item: { value: 1 } }); - - const third = await queue.dequeue(); - expect(third).toEqual({ id: "3", item: { value: 3 } }); - - await new Promise((resolve) => setTimeout(resolve, 100)); - - const second = await queue.dequeue(); - expect(second).toEqual({ id: "2", item: { value: 2 } }); - - const fourth = await queue.dequeue(); - expect(fourth).toBeNull(); - - await queue.close(); -}); diff --git a/packages/run-engine/src/simpleQueue/index.test.ts b/packages/run-engine/src/simpleQueue/index.test.ts new file mode 100644 index 0000000000..233efebedc --- /dev/null +++ b/packages/run-engine/src/simpleQueue/index.test.ts @@ -0,0 +1,90 @@ +import { expect, it } from "vitest"; +import { z } from "zod"; +import { redisTest } from "../test/containerTest"; +import { SimpleQueue } from "./index"; +import { describe } from "node:test"; + +describe("SimpleQueue", () => { + redisTest("enqueue/dequeue", { timeout: 20_000 }, async ({ redisContainer }) => { + const queue = new SimpleQueue( + "test-1", + z.object({ + value: z.number(), + }), + { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + } + ); + + try { + await queue.enqueue("1", { value: 1 }); + await queue.enqueue("2", { value: 2 }); + + const first = await queue.dequeue(); + expect(first).toEqual({ id: "1", item: { value: 1 } }); + + const second = await queue.dequeue(); + expect(second).toEqual({ id: "2", item: { value: 2 } }); + } finally { + await queue.close(); + } + }); + + redisTest("no items", { timeout: 20_000 }, async ({ redisContainer }) => { + const queue = new SimpleQueue( + "test-2", + z.object({ + value: z.number(), + }), + { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + } + ); + + try { + const missOne = await queue.dequeue(); + expect(missOne).toBeNull(); + + await queue.enqueue("1", { value: 1 }); + const hitOne = await queue.dequeue(); + expect(hitOne).toEqual({ id: "1", item: { value: 1 } }); + + const missTwo = await queue.dequeue(); + expect(missTwo).toBeNull(); + } finally { + await queue.close(); + } + }); + + redisTest("future item", { timeout: 20_000 }, async ({ redisContainer }) => { + const queue = new SimpleQueue( + "test-3", + z.object({ + value: z.number(), + }), + { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + } + ); + + try { + await queue.enqueue("1", { value: 1 }, new Date(Date.now() + 50)); + + const miss = await queue.dequeue(); + expect(miss).toBeNull(); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + const first = await queue.dequeue(); + expect(first).toEqual({ id: "1", item: { value: 1 } }); + } finally { + await queue.close(); + } + }); +}); diff --git a/packages/run-engine/src/simpleQueue/index.ts b/packages/run-engine/src/simpleQueue/index.ts index 383d19f756..37e9d87b47 100644 --- a/packages/run-engine/src/simpleQueue/index.ts +++ b/packages/run-engine/src/simpleQueue/index.ts @@ -4,51 +4,128 @@ import { z } from "zod"; export class SimpleQueue { name: string; private redis: Redis; - private keyPrefix: string; private schema: T; constructor(name: string, schema: T, redisOptions: RedisOptions) { this.name = name; - this.redis = new Redis(redisOptions); - this.keyPrefix = `queue:${name}:`; + this.redis = new Redis({ + ...redisOptions, + keyPrefix: `queue:${name}:`, + retryStrategy(times) { + const delay = Math.min(times * 50, 1000); + return delay; + }, + maxRetriesPerRequest: 3, + }); this.schema = schema; + + this.redis.on("error", (error) => { + console.error(`Redis Error for queue ${this.name}:`, error); + }); + + this.redis.on("connect", () => { + // console.log(`Redis connected for queue ${this.name}`); + }); } async enqueue(id: string, item: z.infer, availableAt?: Date): Promise { - const score = availableAt ? availableAt.getTime() : Date.now(); - const serializedItem = JSON.stringify(item); - - await this.redis - .multi() - .zadd(`${this.keyPrefix}queue`, score, id) - .hset(`${this.keyPrefix}items`, id, serializedItem) - .exec(); + try { + const score = availableAt ? availableAt.getTime() : Date.now(); + const serializedItem = JSON.stringify(item); + + const result = await this.redis + .multi() + .zadd(`queue`, score, id) + .hset(`items`, id, serializedItem) + .exec(); + + if (!result) { + throw new Error("Redis multi command returned null"); + } + + result.forEach((res, index) => { + if (res[0]) { + throw new Error(`Redis operation ${index} failed: ${res[0]}`); + } + }); + } catch (e) { + console.error(`SimpleQueue ${this.name}.enqueue(): error enqueuing`, { + error: e, + id, + item, + }); + throw e; + } } async dequeue(): Promise<{ id: string; item: z.infer } | null> { const now = Date.now(); - const result = await this.redis - .multi() - .zrangebyscore(`${this.keyPrefix}queue`, "-inf", now, "LIMIT", 0, 1) - .zremrangebyrank(`${this.keyPrefix}queue`, 0, 0) - .exec(); + try { + const result = await this.redis + .multi() + .zrangebyscore(`queue`, "-inf", now, "WITHSCORES", "LIMIT", 0, 1) + .exec(); - if (!result || !result[0][1] || (result[0][1] as string[]).length === 0) { - return null; - } + if (!result) { + throw new Error("Redis multi command returned null"); + } + + result.forEach((res, index) => { + if (res[0]) { + throw new Error(`Redis operation ${index} failed: ${res[0]}`); + } + }); + + if (!result[0][1] || (result[0][1] as string[]).length === 0) { + return null; + } - const id = (result[0][1] as string[])[0]; - const serializedItem = await this.redis.hget(`${this.keyPrefix}items`, id); + const [id, score] = result[0][1] as string[]; - if (serializedItem) { - await this.redis.hdel(`${this.keyPrefix}items`, id); + // Check if the item is available now + if (parseInt(score) > now) { + return null; + } + + // Remove the item from the sorted set + await this.redis.zrem(`queue`, id); + + const serializedItem = await this.redis.hget(`items`, id); + + if (!serializedItem) { + console.warn(`Item ${id} not found in hash, might have been deleted`); + return null; + } + + await this.redis.hdel(`items`, id); const parsedItem = JSON.parse(serializedItem); - const validatedItem = this.schema.parse(parsedItem); - return { id, item: validatedItem }; + const validatedItem = this.schema.safeParse(parsedItem); + + if (!validatedItem.success) { + console.error("Invalid item in queue", { + id, + item: parsedItem, + errors: validatedItem.error, + }); + return null; + } + + return { id, item: validatedItem.data }; + } catch (e) { + console.error(`SimpleQueue ${this.name}.dequeue(): error dequeuing`, { error: e }); + throw e; } + } - return null; + async size(): Promise { + try { + const result = await this.redis.zcard(`queue`); + return result; + } catch (e) { + console.error(`SimpleQueue ${this.name}.size(): error getting queue size`, { error: e }); + throw e; + } } async close(): Promise { diff --git a/packages/run-engine/src/test/containerTest.ts b/packages/run-engine/src/test/containerTest.ts index 098f784768..7f7d59cdf3 100644 --- a/packages/run-engine/src/test/containerTest.ts +++ b/packages/run-engine/src/test/containerTest.ts @@ -21,12 +21,6 @@ const postgresContainer = async ({}, use: Use) => { await container.stop(); }; -const redisContainer = async ({}, use: Use) => { - const { container } = await createRedisContainer(); - await use(container); - await container.stop(); -}; - const prisma = async ( { postgresContainer }: { postgresContainer: StartedPostgreSqlContainer }, use: Use @@ -42,6 +36,14 @@ const prisma = async ( await prisma.$disconnect(); }; +export const postgresTest = test.extend({ postgresContainer, prisma }); + +const redisContainer = async ({}, use: Use) => { + const { container } = await createRedisContainer(); + await use(container); + await container.stop(); +}; + const redis = async ( { redisContainer }: { redisContainer: StartedRedisContainer }, use: Use @@ -55,13 +57,11 @@ const redis = async ( await redis.quit(); }; +export const redisTest = test.extend({ redisContainer, redis }); + export const containerTest = test.extend({ postgresContainer, prisma, redisContainer, redis, }); - -export const postgresTest = test.extend({ postgresContainer, prisma }); - -export const redisTest = test.extend({ redisContainer, redis }); From a100ba36b7efc9254be886ecdd450bfa11b8b82f Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 23 Sep 2024 14:10:32 +0100 Subject: [PATCH 018/114] Pass a logger into the queue --- packages/run-engine/src/simpleQueue/index.ts | 31 +++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/packages/run-engine/src/simpleQueue/index.ts b/packages/run-engine/src/simpleQueue/index.ts index 37e9d87b47..6c1bb8351e 100644 --- a/packages/run-engine/src/simpleQueue/index.ts +++ b/packages/run-engine/src/simpleQueue/index.ts @@ -1,12 +1,14 @@ import Redis, { RedisOptions } from "ioredis"; import { z } from "zod"; +import { Logger } from "@trigger.dev/core/logger"; export class SimpleQueue { name: string; private redis: Redis; private schema: T; + private logger: Logger; - constructor(name: string, schema: T, redisOptions: RedisOptions) { + constructor(name: string, schema: T, redisOptions: RedisOptions, logger?: Logger) { this.name = name; this.redis = new Redis({ ...redisOptions, @@ -19,12 +21,14 @@ export class SimpleQueue { }); this.schema = schema; + this.logger = logger ?? new Logger("SimpleQueue", "debug"); + this.redis.on("error", (error) => { - console.error(`Redis Error for queue ${this.name}:`, error); + this.logger.error(`Redis Error for queue ${this.name}:`, { queue: this.name, error }); }); this.redis.on("connect", () => { - // console.log(`Redis connected for queue ${this.name}`); + // this.logger.log(`Redis connected for queue ${this.name}`); }); } @@ -49,7 +53,8 @@ export class SimpleQueue { } }); } catch (e) { - console.error(`SimpleQueue ${this.name}.enqueue(): error enqueuing`, { + this.logger.error(`SimpleQueue ${this.name}.enqueue(): error enqueuing`, { + queue: this.name, error: e, id, item, @@ -94,7 +99,10 @@ export class SimpleQueue { const serializedItem = await this.redis.hget(`items`, id); if (!serializedItem) { - console.warn(`Item ${id} not found in hash, might have been deleted`); + this.logger.warn(`Item ${id} not found in hash, might have been deleted`, { + queue: this.name, + id, + }); return null; } @@ -103,7 +111,8 @@ export class SimpleQueue { const validatedItem = this.schema.safeParse(parsedItem); if (!validatedItem.success) { - console.error("Invalid item in queue", { + this.logger.error("Invalid item in queue", { + queue: this.name, id, item: parsedItem, errors: validatedItem.error, @@ -113,7 +122,10 @@ export class SimpleQueue { return { id, item: validatedItem.data }; } catch (e) { - console.error(`SimpleQueue ${this.name}.dequeue(): error dequeuing`, { error: e }); + this.logger.error(`SimpleQueue ${this.name}.dequeue(): error dequeuing`, { + queue: this.name, + error: e, + }); throw e; } } @@ -123,7 +135,10 @@ export class SimpleQueue { const result = await this.redis.zcard(`queue`); return result; } catch (e) { - console.error(`SimpleQueue ${this.name}.size(): error getting queue size`, { error: e }); + this.logger.error(`SimpleQueue ${this.name}.size(): error getting queue size`, { + queue: this.name, + error: e, + }); throw e; } } From d919ba3e91d87e080824c4cf14cecb1e7809fecf Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 23 Sep 2024 14:19:45 +0100 Subject: [PATCH 019/114] A queue processor that processes items from the given queue as fast as it can --- .../src/simpleQueue/processor.test.ts | 46 +++++++++ .../run-engine/src/simpleQueue/processor.ts | 95 +++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 packages/run-engine/src/simpleQueue/processor.test.ts create mode 100644 packages/run-engine/src/simpleQueue/processor.ts diff --git a/packages/run-engine/src/simpleQueue/processor.test.ts b/packages/run-engine/src/simpleQueue/processor.test.ts new file mode 100644 index 0000000000..557b5654a3 --- /dev/null +++ b/packages/run-engine/src/simpleQueue/processor.test.ts @@ -0,0 +1,46 @@ +import { expect, it } from "vitest"; +import { z } from "zod"; +import { redisTest } from "../test/containerTest"; +import { SimpleQueue } from "./index"; +import { describe } from "node:test"; +import { createQueueProcessor } from "./processor"; + +describe("SimpleQueue processor", () => { + redisTest("Read 5 items", { timeout: 20_000 }, async ({ redisContainer }) => { + const queue = new SimpleQueue( + "test-1", + z.object({ + value: z.number(), + }), + { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + } + ); + + try { + //add 50 items to the queue + let itemCount = 10; + for (let i = 0; i < itemCount; i++) { + await queue.enqueue(i.toString(), { value: i }); + } + + let itemsProcessed = 0; + + const processor = createQueueProcessor(queue, { + onItem: async (id, item) => { + expect(item).toEqual({ value: parseInt(id) }); + itemsProcessed++; + }, + }); + + processor.start(); + await new Promise((resolve) => setTimeout(resolve, 2000)); + expect(itemsProcessed).toEqual(itemCount); + processor.stop(); + } finally { + await queue.close(); + } + }); +}); diff --git a/packages/run-engine/src/simpleQueue/processor.ts b/packages/run-engine/src/simpleQueue/processor.ts new file mode 100644 index 0000000000..5bfb5f1491 --- /dev/null +++ b/packages/run-engine/src/simpleQueue/processor.ts @@ -0,0 +1,95 @@ +import { z } from "zod"; +import { SimpleQueue } from "./index"; + +type QueueProcessorOptions = { + timeout?: number; + retry?: { + delay: { + initial: number; + max: number; + factor: number; + }; + maxAttempts: number; + }; + onItem: (id: string, item: z.infer) => Promise | void; +}; + +export function createQueueProcessor( + queue: SimpleQueue, + options: QueueProcessorOptions +) { + const { + timeout = 1000, + retry = { + delay: { + initial: 1000, + max: 10000, + factor: 2, + }, + maxAttempts: 10, + }, + onItem, + } = options; + + const failures = new Map(); + let isRunning = false; + + async function processQueue() { + if (!isRunning) return; + + const result = await queue.dequeue(); + if (result) { + const { id, item } = result; + try { + await onItem(id, item); + } catch (error) { + console.error("Error processing item:", error); + + const retryCount = failures.get(id) || 0; + if (retryCount >= retry.maxAttempts) { + console.error(`SimpleQueue ${queue.name}: max attempts reached for item ${id}`, { item }); + return; + } + + //requeue with delay + const delay = Math.min( + retry.delay.initial * Math.pow(retry.delay.factor, retryCount), + retry.delay.max + ); + console.log(`SimpleQueue ${queue.name}: requeueing item ${id} in ${delay}ms`, { item }); + await queue.enqueue(id, item, new Date(Date.now() + delay)); + + failures.set(id, retryCount + 1); + } + // Continue processing immediately if still running + if (isRunning) { + setImmediate(processQueue); + } + } else { + // No item found, wait before checking again if still running + if (isRunning) { + setTimeout(processQueue, timeout); + } + } + } + + return { + start: () => { + if (!isRunning) { + console.log("Starting queue processor..."); + isRunning = true; + processQueue(); + } else { + console.log("Queue processor is already running."); + } + }, + stop: () => { + if (isRunning) { + console.log("Stopping queue processor..."); + isRunning = false; + } else { + console.log("Queue processor is already stopped."); + } + }, + }; +} From 968f61313e6bf9424a5f793ec2f38cd9f9a3b29c Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 23 Sep 2024 15:00:14 +0100 Subject: [PATCH 020/114] =?UTF-8?q?Test=20for=20retrying=20an=20item=20tha?= =?UTF-8?q?t=20wasn=E2=80=99t=20processed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/simpleQueue/processor.test.ts | 55 ++++++- .../run-engine/src/simpleQueue/processor.ts | 151 ++++++++++++------ 2 files changed, 151 insertions(+), 55 deletions(-) diff --git a/packages/run-engine/src/simpleQueue/processor.test.ts b/packages/run-engine/src/simpleQueue/processor.test.ts index 557b5654a3..71062e6e26 100644 --- a/packages/run-engine/src/simpleQueue/processor.test.ts +++ b/packages/run-engine/src/simpleQueue/processor.test.ts @@ -4,11 +4,12 @@ import { redisTest } from "../test/containerTest"; import { SimpleQueue } from "./index"; import { describe } from "node:test"; import { createQueueProcessor } from "./processor"; +import { Logger } from "@trigger.dev/core/logger"; describe("SimpleQueue processor", () => { - redisTest("Read 5 items", { timeout: 20_000 }, async ({ redisContainer }) => { + redisTest("Read items", { timeout: 20_000 }, async ({ redisContainer }) => { const queue = new SimpleQueue( - "test-1", + "processor-1", z.object({ value: z.number(), }), @@ -20,7 +21,6 @@ describe("SimpleQueue processor", () => { ); try { - //add 50 items to the queue let itemCount = 10; for (let i = 0; i < itemCount; i++) { await queue.enqueue(i.toString(), { value: i }); @@ -36,11 +36,58 @@ describe("SimpleQueue processor", () => { }); processor.start(); - await new Promise((resolve) => setTimeout(resolve, 2000)); + await new Promise((resolve) => setTimeout(resolve, 200)); expect(itemsProcessed).toEqual(itemCount); processor.stop(); } finally { await queue.close(); } }); + + redisTest("Retrying", { timeout: 20_000 }, async ({ redisContainer }) => { + const queue = new SimpleQueue( + "processor-2", + z.object({ + value: z.number(), + }), + { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + } + ); + + try { + await queue.enqueue("1", { value: 1 }); + + let attempts = 0; + let itemsProcessed = 0; + + const processor = createQueueProcessor(queue, { + retry: { + delay: { + initial: 10, + factor: 1, + }, + maxAttempts: 2, + }, + logger: new Logger("QueueProcessor", "debug"), + onItem: async (id, item) => { + attempts++; + if (attempts === 1) { + throw new Error("Test retry"); + } + expect(item).toEqual({ value: parseInt(id) }); + itemsProcessed++; + }, + }); + + processor.start(); + await new Promise((resolve) => setTimeout(resolve, 2_000)); + expect(itemsProcessed).toEqual(1); + processor.stop(); + } finally { + await queue.close(); + } + }); }); diff --git a/packages/run-engine/src/simpleQueue/processor.ts b/packages/run-engine/src/simpleQueue/processor.ts index 5bfb5f1491..a0c5aa0862 100644 --- a/packages/run-engine/src/simpleQueue/processor.ts +++ b/packages/run-engine/src/simpleQueue/processor.ts @@ -1,35 +1,37 @@ import { z } from "zod"; import { SimpleQueue } from "./index"; +import { Logger } from "@trigger.dev/core/logger"; type QueueProcessorOptions = { timeout?: number; retry?: { - delay: { - initial: number; - max: number; - factor: number; + delay?: { + initial?: number; + max?: number; + factor?: number; }; - maxAttempts: number; + maxAttempts?: number; }; + logger?: Logger; onItem: (id: string, item: z.infer) => Promise | void; }; +const defaultRetryOptions = { + delay: { + initial: 1000, + max: 10000, + factor: 2, + }, + maxAttempts: 10, +}; + export function createQueueProcessor( queue: SimpleQueue, options: QueueProcessorOptions ) { - const { - timeout = 1000, - retry = { - delay: { - initial: 1000, - max: 10000, - factor: 2, - }, - maxAttempts: 10, - }, - onItem, - } = options; + let { timeout = 1000, onItem, logger = new Logger("QueueProcessor", "debug") } = options; + + const retry = deepMerge(defaultRetryOptions, options.retry ?? {}); const failures = new Map(); let isRunning = false; @@ -37,59 +39,106 @@ export function createQueueProcessor( async function processQueue() { if (!isRunning) return; - const result = await queue.dequeue(); - if (result) { - const { id, item } = result; - try { - await onItem(id, item); - } catch (error) { - console.error("Error processing item:", error); - - const retryCount = failures.get(id) || 0; - if (retryCount >= retry.maxAttempts) { - console.error(`SimpleQueue ${queue.name}: max attempts reached for item ${id}`, { item }); - return; - } + try { + const result = await queue.dequeue(); + if (result) { + const { id, item } = result; + try { + await onItem(id, item); + } catch (error) { + logger.warn("Error processing item:", { error, id, item, queue: queue.name }); - //requeue with delay - const delay = Math.min( - retry.delay.initial * Math.pow(retry.delay.factor, retryCount), - retry.delay.max - ); - console.log(`SimpleQueue ${queue.name}: requeueing item ${id} in ${delay}ms`, { item }); - await queue.enqueue(id, item, new Date(Date.now() + delay)); + const retryCount = failures.get(id) || 0; + if (retryCount >= retry.maxAttempts) { + logger.error(`QueueProcessor: max attempts reached for item ${id}`, { + queue: queue.name, + id, + item, + }); + return; + } - failures.set(id, retryCount + 1); - } - // Continue processing immediately if still running - if (isRunning) { - setImmediate(processQueue); - } - } else { - // No item found, wait before checking again if still running - if (isRunning) { - setTimeout(processQueue, timeout); + //requeue with delay + const delay = Math.min( + retry.delay.initial * Math.pow(retry.delay.factor, retryCount), + retry.delay.max + ); + logger.log(`QueueProcessor: requeueing item`, { item, id, delay, queue: queue.name }); + await queue.enqueue(id, item, new Date(Date.now() + delay)); + + failures.set(id, retryCount + 1); + } + // Continue processing immediately if still running + if (isRunning) { + setImmediate(processQueue); + } + } else { + // No item found, wait before checking again if still running + if (isRunning) { + setTimeout(processQueue, timeout); + } } + } catch (error) { + logger.error("Error processing queue:", { error }); + setTimeout(processQueue, timeout); } } return { start: () => { if (!isRunning) { - console.log("Starting queue processor..."); + logger.log("Starting queue processor..."); isRunning = true; processQueue(); } else { - console.log("Queue processor is already running."); + logger.log("Queue processor is already running."); } }, stop: () => { if (isRunning) { - console.log("Stopping queue processor..."); + logger.log("Stopping queue processor..."); isRunning = false; } else { - console.log("Queue processor is already stopped."); + logger.log("Queue processor is already stopped."); } }, }; } + +type DeepPartial = T extends object + ? { + [P in keyof T]?: DeepPartial; + } + : T; + +function isObject(item: unknown): item is Record { + return typeof item === "object" && item !== null && !Array.isArray(item); +} + +function deepMerge(target: T, source: DeepPartial): T { + if (!isObject(target) || !isObject(source)) { + return source as T; + } + + const output = { ...target } as T; + + (Object.keys(source) as Array).forEach((key) => { + if (key in target) { + const targetValue = target[key]; + const sourceValue = source[key]; + + if (isObject(targetValue) && isObject(sourceValue)) { + (output as any)[key] = deepMerge( + targetValue, + sourceValue as DeepPartial + ); + } else if (sourceValue !== undefined) { + (output as any)[key] = sourceValue; + } + } else if (source[key as keyof DeepPartial] !== undefined) { + (output as any)[key] = source[key as keyof DeepPartial]; + } + }); + + return output; +} From 539994c70c888d2d6d592251577ad4e11775ccf9 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 23 Sep 2024 18:42:44 +0100 Subject: [PATCH 021/114] First draft of waitpoints in the Prisma schema --- packages/database/prisma/schema.prisma | 52 ++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/packages/database/prisma/schema.prisma b/packages/database/prisma/schema.prisma index 199ab1f25e..f5975e0535 100644 --- a/packages/database/prisma/schema.prisma +++ b/packages/database/prisma/schema.prisma @@ -468,6 +468,7 @@ model Project { bulkActionGroups BulkActionGroup[] BackgroundWorkerFile BackgroundWorkerFile[] waitpoints Waitpoint[] + taskRunWaitpoints TaskRunWaitpoint[] } enum ProjectVersion { @@ -1650,6 +1651,8 @@ model TaskRun { number Int @default(0) friendlyId String @unique + engine RunEngineVersion @default(V1) + status TaskRunStatus @default(PENDING) idempotencyKey String? @@ -1704,6 +1707,12 @@ model TaskRun { expiredAt DateTime? maxAttempts Int? + ///When this run is finished, the waitpoint will be marked as completed + associatedWaitpoint Waitpoint? + + ///If there are any blocked waitpoints, the run won't be executed + blockedByWaitpoints TaskRunWaitpoint[] + batchItems BatchTaskRunItem[] dependency TaskRunDependency? CheckpointRestoreEvent CheckpointRestoreEvent[] @@ -1780,19 +1789,39 @@ enum TaskRunStatus { EXPIRED } +enum RunEngineVersion { + /// The original version that uses marqs v1 and Graphile + V1 + V2 +} + /// A Waitpoint blocks a run from continuing until it's completed +/// If there's a waitpoint blocking a run, it shouldn't be in the queue model Waitpoint { id String @id @default(cuid()) type WaitpointType status WaitpointStatus @default(PENDING) + /// If it's an Event type waitpoint, this is the event. It can also be provided for the DATETIME type idempotencyKey String userProvidedIdempotencyKey Boolean - ///if an idempotencyKey is no longer active, we store it here and generate a new one for the idempotencyKey field. This is a workaround because Prisma doesn't support partial indexes. + /// If an idempotencyKey is no longer active, we store it here and generate a new one for the idempotencyKey field. + /// This is a workaround because Prisma doesn't support partial indexes. inactiveIdempotencyKey String? + /// If it's a RUN type waitpoint, this is the associated run + completedByTaskRunId String? @unique + completedByTaskRun TaskRun? @relation(fields: [completedByTaskRunId], references: [id], onDelete: SetNull) + + /// If it's a DATETIME type waitpoint, this is the date + completedAfter DateTime? + + /// The runs this waitpoint is blocking + blockingTaskRuns TaskRunWaitpoint[] + + /// When completed, an output can be stored here output String? outputType String @default("application/json") @@ -1816,8 +1845,25 @@ enum WaitpointStatus { COMPLETED } -///todo add a waitpoint to each run -///todo add a blockedBy array of waitpoint IDs to the run, and attempt. With appropriate indexes for fast search. +model TaskRunWaitpoint { + id String @id @default(cuid()) + + taskRun TaskRun @relation(fields: [taskRunId], references: [id]) + taskRunId String + + waitpoint Waitpoint @relation(fields: [waitpointId], references: [id]) + waitpointId String + + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) + projectId String + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([taskRunId, waitpointId]) + @@index([taskRunId]) + @@index([waitpointId]) +} model TaskRunTag { id String @id @default(cuid()) From e8190eb582ef0cc085e8d42f46621aa80dc64bf4 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 23 Sep 2024 18:42:56 +0100 Subject: [PATCH 022/114] Remove the custom logger from the test --- packages/run-engine/src/simpleQueue/processor.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/run-engine/src/simpleQueue/processor.test.ts b/packages/run-engine/src/simpleQueue/processor.test.ts index 71062e6e26..6e27a4d962 100644 --- a/packages/run-engine/src/simpleQueue/processor.test.ts +++ b/packages/run-engine/src/simpleQueue/processor.test.ts @@ -71,7 +71,6 @@ describe("SimpleQueue processor", () => { }, maxAttempts: 2, }, - logger: new Logger("QueueProcessor", "debug"), onItem: async (id, item) => { attempts++; if (attempts === 1) { From 969d6a714dd27ee42510fac9c887ebb79dfddb2b Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 24 Sep 2024 13:40:30 +0100 Subject: [PATCH 023/114] Added a completedAt to Waitpoint --- packages/database/prisma/schema.prisma | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/database/prisma/schema.prisma b/packages/database/prisma/schema.prisma index f5975e0535..8ec991ff19 100644 --- a/packages/database/prisma/schema.prisma +++ b/packages/database/prisma/schema.prisma @@ -1803,6 +1803,8 @@ model Waitpoint { type WaitpointType status WaitpointStatus @default(PENDING) + completedAt DateTime? + /// If it's an Event type waitpoint, this is the event. It can also be provided for the DATETIME type idempotencyKey String userProvidedIdempotencyKey Boolean From 5d63c91663ae34fa54020b603dd6307f12fa15e0 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 24 Sep 2024 15:39:55 +0100 Subject: [PATCH 024/114] Notes on the flow for an execution starting --- packages/run-engine/src/index.ts | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/packages/run-engine/src/index.ts b/packages/run-engine/src/index.ts index c9142eb391..d76c87ed9b 100644 --- a/packages/run-engine/src/index.ts +++ b/packages/run-engine/src/index.ts @@ -29,9 +29,35 @@ export class RunEngine { async batchTrigger() {} /** The run can be added to the queue. When it's pulled from the queue it will be executed. */ - async readyToExecute(runId: string) {} + async prepareForQueue(runId: string) {} /** We want to actually execute the run, this could be a continuation of a previous execution. * This is called from the queue, when the run has been pulled. */ - async execute(runId: string) {} + //todo think more about this, when do we create the attempt? + //todo what does this actually do? + //todo how does it get sent to the worker? DEV and PROD + async prepareForExecution(runId: string) {} + + async prepareForAttempt(runId: string) {} + + async complete(runId: string, completion: any) {} } + +/* +Starting execution flow: + +1. Run id is pulled from a queue +2. Prepare the run for an attempt (returns data to send to the worker) + a. The run is marked as "waiting to start"? + b. Create a TaskRunState with the run id, and the state "waiting to start". + c. Start a heartbeat with the TaskRunState id, in case it never starts. +3. The run is sent to the worker +4. When the worker has received the run, it ask the platform for an attempt +5. The attempt is created + a. The attempt is created + b. The TaskRunState is updated to "EXECUTING" + c. Start a heartbeat with the TaskRunState id. + c. The TaskRun is updated to "EXECUTING" +6. A response is sent back to the worker with the attempt data +7. The code executes... +*/ From 0121687a250e17f590c16e2e75a6e679ecdd7c69 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 24 Sep 2024 16:30:54 +0100 Subject: [PATCH 025/114] Added redlock, moved some files around --- packages/run-engine/package.json | 1 + .../run-engine/src/{ => engine}/index.test.ts | 2 +- packages/run-engine/src/{ => engine}/index.ts | 16 +++++++++--- pnpm-lock.yaml | 25 +++++++++++-------- 4 files changed, 30 insertions(+), 14 deletions(-) rename packages/run-engine/src/{ => engine}/index.test.ts (89%) rename packages/run-engine/src/{ => engine}/index.ts (80%) diff --git a/packages/run-engine/package.json b/packages/run-engine/package.json index 9d3b50b047..94739a7ae3 100644 --- a/packages/run-engine/package.json +++ b/packages/run-engine/package.json @@ -8,6 +8,7 @@ "@trigger.dev/core": "workspace:*", "@trigger.dev/database": "workspace:*", "ioredis": "^5.3.2", + "redlock": "5.0.0-beta.2", "typescript": "^4.8.4", "zod": "3.22.3" }, diff --git a/packages/run-engine/src/index.test.ts b/packages/run-engine/src/engine/index.test.ts similarity index 89% rename from packages/run-engine/src/index.test.ts rename to packages/run-engine/src/engine/index.test.ts index 0101dd00f0..c83fcbc977 100644 --- a/packages/run-engine/src/index.test.ts +++ b/packages/run-engine/src/engine/index.test.ts @@ -1,5 +1,5 @@ import { expect } from "vitest"; -import { postgresTest, redisTest } from "./test/containerTest"; +import { postgresTest, redisTest } from "../test/containerTest"; postgresTest("Prisma create user", { timeout: 15_000 }, async ({ prisma }) => { await prisma.user.create({ diff --git a/packages/run-engine/src/index.ts b/packages/run-engine/src/engine/index.ts similarity index 80% rename from packages/run-engine/src/index.ts rename to packages/run-engine/src/engine/index.ts index d76c87ed9b..319f2d489d 100644 --- a/packages/run-engine/src/index.ts +++ b/packages/run-engine/src/engine/index.ts @@ -1,5 +1,6 @@ import { PrismaClient, Prisma } from "@trigger.dev/database"; -import { type RedisOptions } from "ioredis"; +import { Redis, type RedisOptions } from "ioredis"; +import Redlock from "redlock"; type Options = { prisma: PrismaClient; @@ -10,13 +11,22 @@ type Options = { export class RunEngine { private prisma: PrismaClient; + private redis: Redis; + private redlock: Redlock; constructor(private readonly options: Options) { this.prisma = options.prisma; + this.redis = new Redis(options.redis); + this.redlock = new Redlock([this.redis], { + driftFactor: 0.01, + retryCount: 10, + retryDelay: 200, // time in ms + retryJitter: 200, // time in ms + automaticExtensionThreshold: 500, // time in ms + }); } - /** Triggers one run. - * This doesn't start execution, but it will schedule it for execution. + /** "Triggers" one run, which creates the run */ async trigger() { // const result = await this.options.prisma.taskRun.create({}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dcc54c9a66..ce836cf4a5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1245,6 +1245,9 @@ importers: ioredis: specifier: ^5.3.2 version: 5.3.2 + redlock: + specifier: 5.0.0-beta.2 + version: 5.0.0-beta.2 typescript: specifier: ^4.8.4 version: 4.9.5 @@ -22975,6 +22978,10 @@ packages: resolution: {integrity: sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==} dev: true + /node-abort-controller@3.1.1: + resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} + dev: false + /node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -24464,15 +24471,6 @@ packages: picocolors: 1.0.0 source-map-js: 1.0.2 - /postcss@8.4.38: - resolution: {integrity: sha512-Wglpdk03BSfXkHoQa3b/oulrotAkwrlLDRSOb9D0bN86FdRyE9lppSp33aHNPgBa0JKCoB+drFLZkQoRRYae5A==} - engines: {node: ^10 || ^12 || >=14} - dependencies: - nanoid: 3.3.7 - picocolors: 1.0.1 - source-map-js: 1.2.0 - dev: true - /postcss@8.4.44: resolution: {integrity: sha512-Aweb9unOEpQ3ezu4Q00DPvvM2ZTUitJdNKeP/+uQgr1IBIqu574IaZoURId7BKtWMREwzKa9OgzPzezWGPWFQw==} engines: {node: ^10 || ^12 || >=14} @@ -25720,6 +25718,13 @@ packages: redis-errors: 1.2.0 dev: false + /redlock@5.0.0-beta.2: + resolution: {integrity: sha512-2RDWXg5jgRptDrB1w9O/JgSZC0j7y4SlaXnor93H/UJm/QyDiFgBKNtrh0TI6oCXqYSaSoXxFh6Sd3VtYfhRXw==} + engines: {node: '>=12'} + dependencies: + node-abort-controller: 3.1.1 + dev: false + /reduce-css-calc@2.1.8: resolution: {integrity: sha512-8liAVezDmUcH+tdzoEGrhfbGcP7nOV4NkGE3a74+qqvE7nt9i4sKLGBuZNOnpI4WiGksiNPklZxva80061QiPg==} dependencies: @@ -29190,7 +29195,7 @@ packages: dependencies: '@types/node': 18.11.18 esbuild: 0.20.2 - postcss: 8.4.38 + postcss: 8.4.44 rollup: 4.13.2 optionalDependencies: fsevents: 2.3.3 From 6d4da885a1f08b90e704a373a933799e6fee5cf2 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 24 Sep 2024 16:31:10 +0100 Subject: [PATCH 026/114] Starting point for the TaskRunExecutionSnapshot table --- packages/database/prisma/schema.prisma | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/packages/database/prisma/schema.prisma b/packages/database/prisma/schema.prisma index cd7e4b589f..ecfd110189 100644 --- a/packages/database/prisma/schema.prisma +++ b/packages/database/prisma/schema.prisma @@ -1825,6 +1825,32 @@ enum RunEngineVersion { V2 } +/// Used by the RunEngine during TaskRun execution +/// It has the required information to transactionally progress a run through states, +/// and prevent side effects like heartbeats failing a run that has progressed. +/// It is optimised for performance and is designed to be cleared at some point, +/// so there are no foreign key relationships to other models. +model TaskRunExecutionSnapshot { + id String @id @default(cuid()) + + engine RunEngineVersion @default(V2) + + runId String + runStatus TaskRunStatus + + currentAttemptId String? + currentAttemptStatus TaskAttemptStatus? + + /// When set, we can clear the state + completedAt DateTime? + + /// These are only ever appended, so we don't need updatedAt + createdAt DateTime @default(now()) + + /// Used to get the latest state quickly + @@index([runId, createdAt(sort: Desc)]) +} + /// A Waitpoint blocks a run from continuing until it's completed /// If there's a waitpoint blocking a run, it shouldn't be in the queue model Waitpoint { From 2297d8bb141884ddf209d70223cfe315f60bdfb9 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 24 Sep 2024 17:04:31 +0100 Subject: [PATCH 027/114] Added relationships to TaskRunExecutionSnapshot --- packages/database/prisma/schema.prisma | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/database/prisma/schema.prisma b/packages/database/prisma/schema.prisma index ecfd110189..0f146a15df 100644 --- a/packages/database/prisma/schema.prisma +++ b/packages/database/prisma/schema.prisma @@ -1110,6 +1110,8 @@ model TaskAttempt { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt + executionSnapshot TaskRunExecutionSnapshot[] + @@unique([taskId, number]) } @@ -1716,6 +1718,7 @@ model TaskRun { batchItems BatchTaskRunItem[] dependency TaskRunDependency? CheckpointRestoreEvent CheckpointRestoreEvent[] + executionSnapshot TaskRunExecutionSnapshot[] alerts ProjectAlert[] @@ -1835,10 +1838,14 @@ model TaskRunExecutionSnapshot { engine RunEngineVersion @default(V2) - runId String + runId String + run TaskRun @relation(fields: [runId], references: [id]) + runStatus TaskRunStatus - currentAttemptId String? + currentAttemptId String? + currentAttempt TaskAttempt? @relation(fields: [currentAttemptId], references: [id]) + currentAttemptStatus TaskAttemptStatus? /// When set, we can clear the state From 7f50b367335d367fbeaf588d8dd6afbfd41b1da4 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Wed, 25 Sep 2024 20:08:01 +0100 Subject: [PATCH 028/114] Change some tsconfig --- packages/run-engine/package.json | 3 +++ packages/run-engine/tsconfig.json | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/run-engine/package.json b/packages/run-engine/package.json index 94739a7ae3..3c0fef9709 100644 --- a/packages/run-engine/package.json +++ b/packages/run-engine/package.json @@ -4,7 +4,10 @@ "version": "0.0.1", "main": "./src/index.ts", "types": "./src/index.ts", + "type": "module", "dependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/semantic-conventions": "^1.27.0", "@trigger.dev/core": "workspace:*", "@trigger.dev/database": "workspace:*", "ioredis": "^5.3.2", diff --git a/packages/run-engine/tsconfig.json b/packages/run-engine/tsconfig.json index bd7d79399c..d9349e6dcb 100644 --- a/packages/run-engine/tsconfig.json +++ b/packages/run-engine/tsconfig.json @@ -1,10 +1,13 @@ { "compilerOptions": { + "target": "ES2019", + "lib": ["ES2019", "DOM", "DOM.Iterable"], + "module": "NodeNext", + "moduleResolution": "Node16", "types": ["vitest/globals"], "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "isolatedModules": true, - "moduleResolution": "node", "preserveWatchOutput": true, "skipLibCheck": true, "noEmit": true, From a3961daefd9b2e678bbca614401992c018f7d67f Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Wed, 25 Sep 2024 20:08:15 +0100 Subject: [PATCH 029/114] Moved some things around --- packages/run-engine/src/shared/asyncWorker.ts | 34 +++++++++++++++++++ packages/run-engine/src/shared/index.ts | 32 +++++++++++++++++ .../index.test.ts | 0 .../{simpleQueue => simple-queue}/index.ts | 0 .../processor.test.ts | 0 .../processor.ts | 0 6 files changed, 66 insertions(+) create mode 100644 packages/run-engine/src/shared/asyncWorker.ts create mode 100644 packages/run-engine/src/shared/index.ts rename packages/run-engine/src/{simpleQueue => simple-queue}/index.test.ts (100%) rename packages/run-engine/src/{simpleQueue => simple-queue}/index.ts (100%) rename packages/run-engine/src/{simpleQueue => simple-queue}/processor.test.ts (100%) rename packages/run-engine/src/{simpleQueue => simple-queue}/processor.ts (100%) diff --git a/packages/run-engine/src/shared/asyncWorker.ts b/packages/run-engine/src/shared/asyncWorker.ts new file mode 100644 index 0000000000..016662e1d5 --- /dev/null +++ b/packages/run-engine/src/shared/asyncWorker.ts @@ -0,0 +1,34 @@ +export class AsyncWorker { + private running = false; + private timeout?: NodeJS.Timeout; + + constructor(private readonly fn: () => Promise, private readonly interval: number) {} + + start() { + if (this.running) { + return; + } + + this.running = true; + + this.#run(); + } + + stop() { + this.running = false; + } + + async #run() { + if (!this.running) { + return; + } + + try { + await this.fn(); + } catch (e) { + console.error(e); + } + + this.timeout = setTimeout(this.#run.bind(this), this.interval); + } +} diff --git a/packages/run-engine/src/shared/index.ts b/packages/run-engine/src/shared/index.ts new file mode 100644 index 0000000000..669c90e8e5 --- /dev/null +++ b/packages/run-engine/src/shared/index.ts @@ -0,0 +1,32 @@ +import { Attributes } from "@opentelemetry/api"; +import { Prisma } from "@trigger.dev/database"; + +export type AuthenticatedEnvironment = Prisma.RuntimeEnvironmentGetPayload<{ + include: { project: true; organization: true; orgMember: true }; +}>; + +const SemanticEnvResources = { + ENV_ID: "$trigger.env.id", + ENV_TYPE: "$trigger.env.type", + ENV_SLUG: "$trigger.env.slug", + ORG_ID: "$trigger.org.id", + ORG_SLUG: "$trigger.org.slug", + ORG_TITLE: "$trigger.org.title", + PROJECT_ID: "$trigger.project.id", + PROJECT_NAME: "$trigger.project.name", + USER_ID: "$trigger.user.id", +}; + +export function attributesFromAuthenticatedEnv(env: AuthenticatedEnvironment): Attributes { + return { + [SemanticEnvResources.ENV_ID]: env.id, + [SemanticEnvResources.ENV_TYPE]: env.type, + [SemanticEnvResources.ENV_SLUG]: env.slug, + [SemanticEnvResources.ORG_ID]: env.organizationId, + [SemanticEnvResources.ORG_SLUG]: env.organization.slug, + [SemanticEnvResources.ORG_TITLE]: env.organization.title, + [SemanticEnvResources.PROJECT_ID]: env.projectId, + [SemanticEnvResources.PROJECT_NAME]: env.project.name, + [SemanticEnvResources.USER_ID]: env.orgMember?.userId, + }; +} diff --git a/packages/run-engine/src/simpleQueue/index.test.ts b/packages/run-engine/src/simple-queue/index.test.ts similarity index 100% rename from packages/run-engine/src/simpleQueue/index.test.ts rename to packages/run-engine/src/simple-queue/index.test.ts diff --git a/packages/run-engine/src/simpleQueue/index.ts b/packages/run-engine/src/simple-queue/index.ts similarity index 100% rename from packages/run-engine/src/simpleQueue/index.ts rename to packages/run-engine/src/simple-queue/index.ts diff --git a/packages/run-engine/src/simpleQueue/processor.test.ts b/packages/run-engine/src/simple-queue/processor.test.ts similarity index 100% rename from packages/run-engine/src/simpleQueue/processor.test.ts rename to packages/run-engine/src/simple-queue/processor.test.ts diff --git a/packages/run-engine/src/simpleQueue/processor.ts b/packages/run-engine/src/simple-queue/processor.ts similarity index 100% rename from packages/run-engine/src/simpleQueue/processor.ts rename to packages/run-engine/src/simple-queue/processor.ts From a236149a669b17cfcb64c691f2f8931709487b16 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Wed, 25 Sep 2024 20:08:22 +0100 Subject: [PATCH 030/114] Added some packages --- pnpm-lock.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ce836cf4a5..e70d6d793a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1236,6 +1236,12 @@ importers: packages/run-engine: dependencies: + '@opentelemetry/api': + specifier: ^1.9.0 + version: 1.9.0 + '@opentelemetry/semantic-conventions': + specifier: ^1.27.0 + version: 1.27.0 '@trigger.dev/core': specifier: workspace:* version: link:../core @@ -7856,6 +7862,11 @@ packages: resolution: {integrity: sha512-ZDjMJJQRlyk8A1KZFCc+bCbsyrn1wTwdNt56F7twdfUfnHUZUq77/WfONCj8p72NZOyP7pNTdUWSTYC3GTbuuQ==} engines: {node: '>=14'} + /@opentelemetry/semantic-conventions@1.27.0: + resolution: {integrity: sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==} + engines: {node: '>=14'} + dev: false + /@pkgjs/parseargs@0.11.0: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -24745,7 +24756,7 @@ packages: resolution: {integrity: sha512-cCD7jLTqyPdjEPBo/Xk4Iu8jxjuZgZJ3e/oET3L+ZwOuap/7Cw3dH/TJSsZKs1TQLZ2IHpIlRAKw82ef06kmMw==} engines: {node: ^16 || ^18 || >=20} dependencies: - '@opentelemetry/api': 1.8.0 + '@opentelemetry/api': 1.9.0 tdigest: 0.1.2 dev: false From 416fd885f114d9aa8e7e7be8e473bdafd1ffd7f6 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Wed, 25 Sep 2024 20:08:38 +0100 Subject: [PATCH 031/114] WIP on the RunQueue --- packages/run-engine/src/run-queue/index.ts | 1581 +++++++++++++++++ .../run-engine/src/run-queue/keyProducer.ts | 195 ++ packages/run-engine/src/run-queue/types.ts | 106 ++ 3 files changed, 1882 insertions(+) create mode 100644 packages/run-engine/src/run-queue/index.ts create mode 100644 packages/run-engine/src/run-queue/keyProducer.ts create mode 100644 packages/run-engine/src/run-queue/types.ts diff --git a/packages/run-engine/src/run-queue/index.ts b/packages/run-engine/src/run-queue/index.ts new file mode 100644 index 0000000000..15c2bacc98 --- /dev/null +++ b/packages/run-engine/src/run-queue/index.ts @@ -0,0 +1,1581 @@ +import { + Span, + SpanKind, + SpanOptions, + Tracer, + context, + propagation, + trace, +} from "@opentelemetry/api"; +import { + SEMATTRS_MESSAGE_ID, + SEMATTRS_MESSAGING_OPERATION, + SEMATTRS_MESSAGING_SYSTEM, +} from "@opentelemetry/semantic-conventions"; +import { flattenAttributes } from "@trigger.dev/core/v3"; +import { Redis, type Callback, type RedisOptions, type Result } from "ioredis"; +import { Logger } from "@trigger.dev/core/logger"; +import { AsyncWorker } from "../shared/asyncWorker.js"; +import { + MessagePayload, + QueueCapacities, + RunQueueKeyProducer, + RunQueuePriorityStrategy, +} from "./types.js"; +import { attributesFromAuthenticatedEnv, AuthenticatedEnvironment } from "../shared/index.js"; + +const KEY_PREFIX = "runqueue:"; + +const constants = { + MESSAGE_VISIBILITY_TIMEOUT_QUEUE: "msgVisibilityTimeout", +} as const; + +const SemanticAttributes = { + QUEUE: "runqueue.queue", + PARENT_QUEUE: "runqueue.parentQueue", + RUN_ID: "runqueue.runId", + CONCURRENCY_KEY: "runqueue.concurrencyKey", +}; + +export type RunQueueOptions = { + name: string; + tracer: Tracer; + redis: RedisOptions; + defaultEnvConcurrency: number; + defaultOrgConcurrency: number; + windowSize?: number; + visibilityTimeoutInMs?: number; + workers: number; + keysProducer: RunQueueKeyProducer; + queuePriorityStrategy: RunQueuePriorityStrategy; + envQueuePriorityStrategy: RunQueuePriorityStrategy; + enableRebalancing?: boolean; + verbose?: boolean; + logger: Logger; +}; + +/** + * RunQueue โ€“ the queue that's used to process runs + */ +export class RunQueue { + private logger: Logger; + private redis: Redis; + public keys: RunQueueKeyProducer; + private queuePriorityStrategy: RunQueuePriorityStrategy; + #rebalanceWorkers: Array = []; + + constructor(private readonly options: RunQueueOptions) { + this.redis = new Redis(options.redis); + this.logger = options.logger; + + this.keys = options.keysProducer; + this.queuePriorityStrategy = options.queuePriorityStrategy; + + this.#startRebalanceWorkers(); + this.#registerCommands(); + } + + get name() { + return this.options.name; + } + + get tracer() { + return this.options.tracer; + } + + public async updateQueueConcurrencyLimits( + env: AuthenticatedEnvironment, + queue: string, + concurrency: number + ) { + return this.redis.set(this.keys.queueConcurrencyLimitKey(env, queue), concurrency); + } + + public async removeQueueConcurrencyLimits(env: AuthenticatedEnvironment, queue: string) { + return this.redis.del(this.keys.queueConcurrencyLimitKey(env, queue)); + } + + public async updateEnvConcurrencyLimits(env: AuthenticatedEnvironment) { + await this.#callUpdateGlobalConcurrencyLimits({ + envConcurrencyLimitKey: this.keys.envConcurrencyLimitKey(env), + envConcurrencyLimit: env.maximumConcurrencyLimit, + }); + } + + public async getQueueConcurrencyLimit(env: AuthenticatedEnvironment, queue: string) { + const result = await this.redis.get(this.keys.queueConcurrencyLimitKey(env, queue)); + + return result ? Number(result) : undefined; + } + + public async getEnvConcurrencyLimit(env: AuthenticatedEnvironment) { + const result = await this.redis.get(this.keys.envConcurrencyLimitKey(env)); + + return result ? Number(result) : this.options.defaultEnvConcurrency; + } + + public async lengthOfQueue( + env: AuthenticatedEnvironment, + queue: string, + concurrencyKey?: string + ) { + return this.redis.zcard(this.keys.queueKey(env, queue, concurrencyKey)); + } + + public async oldestMessageInQueue( + env: AuthenticatedEnvironment, + queue: string, + concurrencyKey?: string + ) { + // Get the "score" of the sorted set to get the oldest message score + const result = await this.redis.zrange( + this.keys.queueKey(env, queue, concurrencyKey), + 0, + 0, + "WITHSCORES" + ); + + if (result.length === 0) { + return; + } + + return Number(result[1]); + } + + public async currentConcurrencyOfQueue( + env: AuthenticatedEnvironment, + queue: string, + concurrencyKey?: string + ) { + return this.redis.scard(this.keys.currentConcurrencyKey(env, queue, concurrencyKey)); + } + + public async currentConcurrencyOfEnvironment(env: AuthenticatedEnvironment) { + return this.redis.scard(this.keys.envCurrentConcurrencyKey(env)); + } + + public async enqueueMessage({ + env, + ...message + }: { env: AuthenticatedEnvironment } & MessagePayload) { + return await this.#trace( + "enqueueMessage", + async (span) => { + const { runId, concurrencyKey } = message; + + const queue = this.keys.queueKey(env, message.queue, concurrencyKey); + + const parentQueue = this.keys.envSharedQueueKey(env); + + propagation.inject(context.active(), message); + + span.setAttributes({ + [SemanticAttributes.QUEUE]: queue, + [SemanticAttributes.RUN_ID]: runId, + [SemanticAttributes.CONCURRENCY_KEY]: concurrencyKey, + [SemanticAttributes.PARENT_QUEUE]: parentQueue, + }); + + const messagePayload: MessagePayload = { + ...message, + queue, + }; + + await this.#callEnqueueMessage(messagePayload); + }, + { + kind: SpanKind.PRODUCER, + attributes: { + [SEMATTRS_MESSAGING_OPERATION]: "publish", + [SEMATTRS_MESSAGE_ID]: message.runId, + [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", + ...attributesFromAuthenticatedEnv(env), + }, + } + ); + } + + public async dequeueMessageInEnv(env: AuthenticatedEnvironment) { + return this.#trace( + "dequeueMessageInEnv", + async (span) => { + const parentQueue = this.keys.envSharedQueueKey(env); + + // Read the parent queue for matching queues + const messageQueue = await this.#getRandomQueueFromParentQueue( + parentQueue, + this.options.envQueuePriorityStrategy, + (queue) => this.#calculateMessageQueueCapacities(queue, { checkForDisabled: false }), + env.id + ); + + if (!messageQueue) { + return; + } + + const message = await this.#callDequeueMessage({ + messageQueue, + parentQueue, + concurrencyLimitKey: this.keys.concurrencyLimitKeyFromQueue(messageQueue), + currentConcurrencyKey: this.keys.currentConcurrencyKeyFromQueue(messageQueue), + envConcurrencyLimitKey: this.keys.envConcurrencyLimitKeyFromQueue(messageQueue), + envCurrentConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(messageQueue), + globalConcurrencyTrackerKey: this.keys.globalCurrentConcurrencyKey(messageQueue), + }); + + if (!message) { + return; + } + + span.setAttributes({ + [SEMATTRS_MESSAGE_ID]: message.messageId, + [SemanticAttributes.QUEUE]: message.message.queue, + [SemanticAttributes.RUN_ID]: message.message.runId, + [SemanticAttributes.CONCURRENCY_KEY]: message.message.concurrencyKey, + [SemanticAttributes.PARENT_QUEUE]: message.message.parentQueue, + }); + + return message; + }, + { + kind: SpanKind.CONSUMER, + attributes: { + [SEMATTRS_MESSAGING_OPERATION]: "receive", + [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", + ...attributesFromAuthenticatedEnv(env), + }, + } + ); + } + + public async getSharedQueueDetails() { + const parentQueue = this.keys.sharedQueueKey(); + + const { range } = await this.queuePriorityStrategy.nextCandidateSelection( + parentQueue, + "getSharedQueueDetails" + ); + const queues = await this.#getChildQueuesWithScores(parentQueue, range); + + const queuesWithScores = await this.#calculateQueueScores(queues, (queue) => + this.#calculateMessageQueueCapacities(queue) + ); + + // We need to priority shuffle here to ensure all workers aren't just working on the highest priority queue + const choice = this.queuePriorityStrategy.chooseQueue( + queuesWithScores, + parentQueue, + "getSharedQueueDetails", + range + ); + + return { + selectionId: "getSharedQueueDetails", + queues, + queuesWithScores, + nextRange: range, + queueCount: queues.length, + queueChoice: choice, + }; + } + + /** + * Dequeue a message from the shared queue (this should be used in production environments) + */ + public async dequeueMessageInSharedQueue(consumerId: string) { + return this.#trace( + "dequeueMessageInSharedQueue", + async (span) => { + const parentQueue = this.keys.sharedQueueKey(); + + // Read the parent queue for matching queues + const messageQueue = await this.#getRandomQueueFromParentQueue( + parentQueue, + this.options.queuePriorityStrategy, + (queue) => this.#calculateMessageQueueCapacities(queue, { checkForDisabled: true }), + consumerId + ); + + if (!messageQueue) { + return; + } + + // If the queue includes a concurrency key, we need to remove the ck:concurrencyKey from the queue name + const message = await this.#callDequeueMessage({ + messageQueue, + parentQueue, + concurrencyLimitKey: this.keys.concurrencyLimitKeyFromQueue(messageQueue), + currentConcurrencyKey: this.keys.currentConcurrencyKeyFromQueue(messageQueue), + envConcurrencyLimitKey: this.keys.envConcurrencyLimitKeyFromQueue(messageQueue), + envCurrentConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(messageQueue), + globalConcurrencyTrackerKey: this.keys.globalCurrentConcurrencyKey(messageQueue), + }); + + if (!message) { + return; + } + + span.setAttributes({ + [SEMATTRS_MESSAGE_ID]: message.messageId, + [SemanticAttributes.QUEUE]: message.message.queue, + [SemanticAttributes.RUN_ID]: message.message.runId, + [SemanticAttributes.CONCURRENCY_KEY]: message.message.concurrencyKey, + [SemanticAttributes.PARENT_QUEUE]: message.message.parentQueue, + }); + + return message; + }, + { + kind: SpanKind.CONSUMER, + attributes: { + [SEMATTRS_MESSAGING_OPERATION]: "receive", + [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", + }, + } + ); + } + + public async acknowledgeMessage(messageId: string) { + return this.#trace( + "acknowledgeMessage", + async (span) => { + span.setAttributes({ + [SemanticAttributes.RUN_ID]: messageId, + }); + + const message = await this.#callAcknowledgeMessage({ + messageKey: this.keys.messageKey(messageId), + messageId, + }); + + span.setAttributes({ + [SemanticAttributes.RUN_ID]: messageId, + [SemanticAttributes.QUEUE]: message.queue, + [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, + [SemanticAttributes.PARENT_QUEUE]: message.parentQueue, + }); + }, + { + kind: SpanKind.CONSUMER, + attributes: { + [SEMATTRS_MESSAGING_OPERATION]: "ack", + [SEMATTRS_MESSAGE_ID]: messageId, + [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", + }, + } + ); + } + + public async releaseConcurrency(messageId: string, releaseForRun: boolean = false) { + return this.#trace( + "releaseConcurrency", + async (span) => { + span.setAttributes({ + [SemanticAttributes.MESSAGE_ID]: messageId, + }); + + const message = await this.readMessage(messageId); + + if (!message) { + logger.log(`[${this.name}].releaseConcurrency() message not found`, { + messageId, + releaseForRun, + service: this.name, + }); + return; + } + + span.setAttributes({ + [SemanticAttributes.QUEUE]: message.queue, + [SemanticAttributes.MESSAGE_ID]: message.messageId, + [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, + [SemanticAttributes.PARENT_QUEUE]: message.parentQueue, + }); + + const concurrencyKey = this.keys.currentConcurrencyKeyFromQueue(message.queue); + const envConcurrencyKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); + const orgConcurrencyKey = this.keys.orgCurrentConcurrencyKeyFromQueue(message.queue); + + logger.debug("Calling releaseConcurrency", { + messageId, + queue: message.queue, + concurrencyKey, + envConcurrencyKey, + orgConcurrencyKey, + service: this.name, + releaseForRun, + }); + + return this.redis.releaseConcurrency( + //don't release the for the run, it breaks concurrencyLimits + releaseForRun ? concurrencyKey : "", + envConcurrencyKey, + orgConcurrencyKey, + message.messageId + ); + }, + { + kind: SpanKind.CONSUMER, + attributes: { + [SEMATTRS_MESSAGING_OPERATION]: "releaseConcurrency", + [SEMATTRS_MESSAGE_ID]: messageId, + [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", + }, + } + ); + } + + async #trace( + name: string, + fn: (span: Span) => Promise, + options?: SpanOptions & { sampleRate?: number } + ): Promise { + return this.tracer.startActiveSpan( + name, + { + ...options, + attributes: { + ...options?.attributes, + }, + }, + async (span) => { + try { + return await fn(span); + } catch (e) { + if (e instanceof Error) { + span.recordException(e); + } else { + span.recordException(new Error(String(e))); + } + + throw e; + } finally { + span.end(); + } + } + ); + } + + /** + * Negative acknowledge a message, which will requeue the message + */ + public async nackMessage( + messageId: string, + retryAt: number = Date.now(), + updates?: Record + ) { + return this.#trace( + "nackMessage", + async (span) => { + const message = await this.readMessage(messageId); + + if (!message) { + logger.log(`[${this.name}].nackMessage() message not found`, { + messageId, + retryAt, + updates, + service: this.name, + }); + return; + } + + span.setAttributes({ + [SemanticAttributes.QUEUE]: message.queue, + [SemanticAttributes.MESSAGE_ID]: message.messageId, + [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, + [SemanticAttributes.PARENT_QUEUE]: message.parentQueue, + }); + + if (updates) { + await this.replaceMessage(messageId, updates, retryAt, true); + } + + await this.options.visibilityTimeoutStrategy.cancelHeartbeat(messageId); + + await this.#callNackMessage({ + messageKey: this.keys.messageKey(messageId), + messageQueue: message.queue, + parentQueue: message.parentQueue, + concurrencyKey: this.keys.currentConcurrencyKeyFromQueue(message.queue), + envConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(message.queue), + orgConcurrencyKey: this.keys.orgCurrentConcurrencyKeyFromQueue(message.queue), + visibilityQueue: constants.MESSAGE_VISIBILITY_TIMEOUT_QUEUE, + messageId, + messageScore: retryAt, + }); + + await this.options.subscriber?.messageNacked(message); + }, + { + kind: SpanKind.CONSUMER, + attributes: { + [SEMATTRS_MESSAGING_OPERATION]: "nack", + [SEMATTRS_MESSAGE_ID]: messageId, + [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", + }, + } + ); + } + + // This should increment by the number of seconds, but with a max value of Date.now() + visibilityTimeoutInMs + public async heartbeatMessage(messageId: string) { + await this.options.visibilityTimeoutStrategy.heartbeat(messageId, this.visibilityTimeoutInMs); + } + + get visibilityTimeoutInMs() { + return this.options.visibilityTimeoutInMs ?? 300000; // 5 minutes + } + + async readMessage(messageId: string) { + return this.#trace( + "readMessage", + async (span) => { + const rawMessage = await this.redis.get(this.keys.messageKey(messageId)); + + if (!rawMessage) { + return; + } + + const message = MessagePayload.safeParse(JSON.parse(rawMessage)); + + if (!message.success) { + this.logger.error(`[${this.name}] Failed to parse message`, { + messageId, + error: message.error, + service: this.name, + }); + + return; + } + + return message.data; + }, + { + attributes: { + [SEMATTRS_MESSAGING_OPERATION]: "receive", + [SEMATTRS_MESSAGE_ID]: messageId, + [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", + [SemanticAttributes.RUN_ID]: messageId, + }, + } + ); + } + + async #getRandomQueueFromParentQueue( + parentQueue: string, + queuePriorityStrategy: MarQSQueuePriorityStrategy, + calculateCapacities: (queue: string) => Promise, + consumerId: string + ) { + return this.#trace( + "getRandomQueueFromParentQueue", + async (span) => { + span.setAttribute("consumerId", consumerId); + + const { range } = await queuePriorityStrategy.nextCandidateSelection( + parentQueue, + consumerId + ); + + const queues = await this.#getChildQueuesWithScores(parentQueue, range, span); + span.setAttribute("queueCount", queues.length); + + const queuesWithScores = await this.#calculateQueueScores(queues, calculateCapacities); + span.setAttribute("queuesWithScoresCount", queuesWithScores.length); + + // We need to priority shuffle here to ensure all workers aren't just working on the highest priority queue + const { choice, nextRange } = this.queuePriorityStrategy.chooseQueue( + queuesWithScores, + parentQueue, + consumerId, + range + ); + + span.setAttributes({ + ...flattenAttributes(queues, "runqueue.queues"), + }); + span.setAttributes({ + ...flattenAttributes(queuesWithScores, "runqueue.queuesWithScores"), + }); + span.setAttribute("range.offset", range.offset); + span.setAttribute("range.count", range.count); + span.setAttribute("nextRange.offset", nextRange.offset); + span.setAttribute("nextRange.count", nextRange.count); + + if (this.options.verbose || nextRange.offset > 0) { + if (typeof choice === "string") { + logger.debug(`[${this.name}] getRandomQueueFromParentQueue`, { + queues, + queuesWithScores, + range, + nextRange, + queueCount: queues.length, + queuesWithScoresCount: queuesWithScores.length, + queueChoice: choice, + consumerId, + }); + } else { + logger.debug(`[${this.name}] getRandomQueueFromParentQueue`, { + queues, + queuesWithScores, + range, + nextRange, + queueCount: queues.length, + queuesWithScoresCount: queuesWithScores.length, + noQueueChoice: true, + consumerId, + }); + } + } + + if (typeof choice !== "string") { + span.setAttribute("noQueueChoice", true); + + return; + } else { + span.setAttribute("queueChoice", choice); + + return choice; + } + }, + { + kind: SpanKind.CONSUMER, + attributes: { + [SEMATTRS_MESSAGING_OPERATION]: "receive", + [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", + [SemanticAttributes.PARENT_QUEUE]: parentQueue, + }, + } + ); + } + + // Calculate the weights of the queues based on the age and the capacity + async #calculateQueueScores( + queues: Array<{ value: string; score: number }>, + calculateCapacities: (queue: string) => Promise + ) { + const now = Date.now(); + + const queueScores = await Promise.all( + queues.map(async (queue) => { + return { + queue: queue.value, + capacities: await calculateCapacities(queue.value), + age: now - queue.score, + size: await this.redis.zcard(queue.value), + }; + }) + ); + + return queueScores; + } + + async #calculateMessageQueueCapacities(queue: string, options?: { checkForDisabled?: boolean }) { + return await this.#callCalculateMessageCapacities({ + currentConcurrencyKey: this.keys.currentConcurrencyKeyFromQueue(queue), + currentEnvConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(queue), + concurrencyLimitKey: this.keys.concurrencyLimitKeyFromQueue(queue), + envConcurrencyLimitKey: this.keys.envConcurrencyLimitKeyFromQueue(queue), + disabledConcurrencyLimitKey: options?.checkForDisabled + ? this.keys.disabledConcurrencyLimitKeyFromQueue(queue) + : undefined, + }); + } + + async #getChildQueuesWithScores( + key: string, + range: QueueRange, + span?: Span + ): Promise> { + const valuesWithScores = await this.redis.zrangebyscore( + key, + "-inf", + Date.now(), + "WITHSCORES", + "LIMIT", + range.offset, + range.count + ); + + span?.setAttribute("zrangebyscore.valuesWithScores.rawLength", valuesWithScores.length); + span?.setAttributes({ + ...flattenAttributes(valuesWithScores, "zrangebyscore.valuesWithScores.rawValues"), + }); + + const result: Array<{ value: string; score: number }> = []; + + for (let i = 0; i < valuesWithScores.length; i += 2) { + result.push({ + value: valuesWithScores[i], + score: Number(valuesWithScores[i + 1]), + }); + } + + return result; + } + + #startRebalanceWorkers() { + if (!this.options.enableRebalancing) { + return; + } + + // Start a new worker to rebalance parent queues periodically + for (let i = 0; i < this.options.workers; i++) { + const worker = new AsyncWorker(this.#rebalanceParentQueues.bind(this), 60_000); + + this.#rebalanceWorkers.push(worker); + + worker.start(); + } + } + + queueConcurrencyScanStream( + count: number = 100, + onEndCallback?: () => void, + onErrorCallback?: (error: Error) => void + ) { + const pattern = this.keys.queueCurrentConcurrencyScanPattern(); + + logger.debug("Starting queue concurrency scan stream", { + pattern, + component: "runqueue", + operation: "queueConcurrencyScanStream", + service: this.name, + count, + }); + + const redis = this.redis.duplicate(); + + const stream = redis.scanStream({ + match: pattern, + type: "set", + count, + }); + + stream.on("end", () => { + onEndCallback?.(); + redis.quit(); + }); + + stream.on("error", (error) => { + onErrorCallback?.(error); + redis.quit(); + }); + + return { stream, redis }; + } + + async #rebalanceParentQueues() { + return await new Promise((resolve, reject) => { + // Scan for sorted sets with the parent queue pattern + const pattern = this.keys.sharedQueueScanPattern(); + const redis = this.redis.duplicate(); + const stream = redis.scanStream({ + match: pattern, + type: "zset", + count: 100, + }); + + logger.debug("Streaming parent queues based on pattern", { + pattern, + component: "runqueue", + operation: "rebalanceParentQueues", + service: this.name, + }); + + stream.on("data", async (keys) => { + const uniqueKeys = Array.from(new Set(keys)); + + if (uniqueKeys.length === 0) { + return; + } + + stream.pause(); + + logger.debug("Rebalancing parent queues", { + component: "runqueue", + operation: "rebalanceParentQueues", + parentQueues: uniqueKeys, + service: this.name, + }); + + Promise.all( + uniqueKeys.map(async (key) => this.#rebalanceParentQueue(this.keys.stripKeyPrefix(key))) + ).finally(() => { + stream.resume(); + }); + }); + + stream.on("end", () => { + redis.quit().finally(() => { + resolve(); + }); + }); + + stream.on("error", (e) => { + redis.quit().finally(() => { + reject(e); + }); + }); + }); + } + + // Parent queue is a sorted set, the values of which are queue keys and the scores are is the oldest message in the queue + // We need to scan the parent queue and rebalance the queues based on the oldest message in the queue + async #rebalanceParentQueue(parentQueue: string) { + return await new Promise((resolve, reject) => { + const redis = this.redis.duplicate(); + + const stream = redis.zscanStream(parentQueue, { + match: "*", + count: 100, + }); + + stream.on("data", async (childQueues) => { + stream.pause(); + + // childQueues is a flat array but of the form [queue1, score1, queue2, score2, ...], we want to group them into pairs + const childQueuesWithScores: Record = {}; + + for (let i = 0; i < childQueues.length; i += 2) { + childQueuesWithScores[childQueues[i]] = childQueues[i + 1]; + } + + this.logger.debug("Rebalancing child queues", { + parentQueue, + childQueuesWithScores, + component: "runqueue", + operation: "rebalanceParentQueues", + service: this.name, + }); + + await Promise.all( + Object.entries(childQueuesWithScores).map(async ([childQueue, currentScore]) => + this.#callRebalanceParentQueueChild({ parentQueue, childQueue, currentScore }) + ) + ).finally(() => { + stream.resume(); + }); + }); + + stream.on("end", () => { + redis.quit().finally(() => { + resolve(); + }); + }); + + stream.on("error", (e) => { + redis.quit().finally(() => { + reject(e); + }); + }); + }); + } + + async #callEnqueueMessage(message: MessagePayload) { + const concurrencyKey = this.keys.currentConcurrencyKeyFromQueue(message.queue); + const envConcurrencyKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); + const taskConcurrencyTrackerKey = this.keys.currentTaskIdentifierKey({ + orgId: message.orgId, + projectId: message.projectId, + environmentId: message.environmentId, + taskIdentifier: message.taskIdentifier, + }); + const envConcurrencyTrackerKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); + const globalConcurrencyTrackerKey = this.keys.globalCurrentConcurrencyKey( + message.environmentType + ); + + this.logger.debug("Calling enqueueMessage", { + messagePayload: message, + concurrencyKey, + envConcurrencyKey, + service: this.name, + }); + + return this.redis.enqueueMessage( + message.queue, + message.parentQueue, + this.keys.messageKey(message.runId), + concurrencyKey, + envConcurrencyKey, + taskConcurrencyTrackerKey, + envConcurrencyTrackerKey, + globalConcurrencyTrackerKey, + message.queue, + message.runId, + JSON.stringify(message), + String(message.timestamp) + ); + } + + async #callDequeueMessage({ + messageQueue, + parentQueue, + concurrencyLimitKey, + envConcurrencyLimitKey, + currentConcurrencyKey, + envCurrentConcurrencyKey, + globalConcurrencyTrackerKey, + }: { + messageQueue: string; + parentQueue: string; + concurrencyLimitKey: string; + envConcurrencyLimitKey: string; + currentConcurrencyKey: string; + envCurrentConcurrencyKey: string; + globalConcurrencyTrackerKey: string; + }) { + const result = await this.redis.dequeueMessage( + //keys + messageQueue, + parentQueue, + concurrencyLimitKey, + envConcurrencyLimitKey, + currentConcurrencyKey, + envCurrentConcurrencyKey, + globalConcurrencyTrackerKey, + //args + messageQueue, + String(Date.now()), + String(this.options.defaultEnvConcurrency) + ); + + if (!result) { + return; + } + + this.logger.debug("Dequeue message result", { + result, + service: this.name, + }); + + if (result.length !== 2) { + this.logger.error("Invalid dequeue message result", { + result, + service: this.name, + }); + return; + } + + const [messageId, messageScore] = result; + + //read message + const message = await this.readMessage(messageId); + + if (!message) { + this.logger.error(`Dequeued then failed to read message. This is unrecoverable.`, { + messageId, + messageScore, + service: this.name, + }); + return; + } + + //update task concurrency + await this.redis.sadd(this.keys.currentTaskIdentifierKey(message), messageId); + + return { + messageId, + messageScore, + message, + }; + } + + async #callAcknowledgeMessage({ + parentQueue, + messageKey, + messageQueue, + concurrencyKey, + envConcurrencyKey, + messageId, + }: { + parentQueue: string; + messageKey: string; + messageQueue: string; + concurrencyKey: string; + envConcurrencyKey: string; + messageId: string; + }) { + this.logger.debug("Calling acknowledgeMessage", { + messageKey, + messageQueue, + concurrencyKey, + envConcurrencyKey, + messageId, + parentQueue, + service: this.name, + }); + + return this.redis.acknowledgeMessage( + parentQueue, + messageKey, + messageQueue, + concurrencyKey, + envConcurrencyKey, + messageId, + messageQueue + ); + } + + async #callNackMessage({ + messageKey, + messageQueue, + parentQueue, + concurrencyKey, + envConcurrencyKey, + orgConcurrencyKey, + visibilityQueue, + messageId, + messageScore, + }: { + messageKey: string; + messageQueue: string; + parentQueue: string; + concurrencyKey: string; + envConcurrencyKey: string; + orgConcurrencyKey: string; + visibilityQueue: string; + messageId: string; + messageScore: number; + }) { + this.logger.debug("Calling nackMessage", { + messageKey, + messageQueue, + parentQueue, + concurrencyKey, + envConcurrencyKey, + orgConcurrencyKey, + visibilityQueue, + messageId, + messageScore, + service: this.name, + }); + + return this.redis.nackMessage( + messageKey, + messageQueue, + parentQueue, + concurrencyKey, + envConcurrencyKey, + orgConcurrencyKey, + visibilityQueue, + messageQueue, + messageId, + String(Date.now()), + String(messageScore) + ); + } + + async #callCalculateMessageCapacities({ + currentConcurrencyKey, + currentEnvConcurrencyKey, + + concurrencyLimitKey, + envConcurrencyLimitKey, + + disabledConcurrencyLimitKey, + }: { + currentConcurrencyKey: string; + currentEnvConcurrencyKey: string; + + concurrencyLimitKey: string; + envConcurrencyLimitKey: string; + + disabledConcurrencyLimitKey: string | undefined; + }): Promise { + const capacities = disabledConcurrencyLimitKey + ? await this.redis.calculateMessageQueueCapacitiesWithDisabling( + currentConcurrencyKey, + currentEnvConcurrencyKey, + concurrencyLimitKey, + envConcurrencyLimitKey, + disabledConcurrencyLimitKey, + String(this.options.defaultEnvConcurrency) + ) + : await this.redis.calculateMessageQueueCapacities( + currentConcurrencyKey, + currentEnvConcurrencyKey, + concurrencyLimitKey, + envConcurrencyLimitKey, + String(this.options.defaultEnvConcurrency) + ); + + const queueCurrent = Number(capacities[0]); + const envLimit = Number(capacities[3]); + const isOrgEnabled = Boolean(capacities[5]); + const queueLimit = capacities[1] + ? Number(capacities[1]) + : Math.min(envLimit, isOrgEnabled ? Infinity : 0); + const envCurrent = Number(capacities[2]); + const orgCurrent = Number(capacities[4]); + + return { + queue: { current: queueCurrent, limit: queueLimit }, + env: { current: envCurrent, limit: envLimit }, + }; + } + + #callUpdateGlobalConcurrencyLimits({ + envConcurrencyLimitKey, + envConcurrencyLimit, + }: { + envConcurrencyLimitKey: string; + envConcurrencyLimit: number; + }) { + return this.redis.updateGlobalConcurrencyLimits( + envConcurrencyLimitKey, + String(envConcurrencyLimit) + ); + } + + async #callRebalanceParentQueueChild({ + parentQueue, + childQueue, + currentScore, + }: { + parentQueue: string; + childQueue: string; + currentScore: string; + }) { + const rebalanceResult = await this.redis.rebalanceParentQueueChild( + childQueue, + parentQueue, + childQueue, + currentScore + ); + + if (rebalanceResult) { + this.logger.debug("Rebalanced parent queue child", { + parentQueue, + childQueue, + currentScore, + rebalanceResult, + operation: "rebalanceParentQueueChild", + service: this.name, + }); + } + + return rebalanceResult; + } + + #registerCommands() { + this.redis.defineCommand("enqueueMessage", { + numberOfKeys: 8, + lua: ` +local queue = KEYS[1] +local parentQueue = KEYS[2] +local messageKey = KEYS[3] +local concurrencyKey = KEYS[4] +local envCurrentConcurrencyKey = KEYS[5] +local taskConcurrencyTrackerKey = KEYS[6] +local envConcurrencyTrackerKey = KEYS[7] +local globalConcurrencyTrackerKey = KEYS[8] + +local queueName = ARGV[1] +local messageId = ARGV[2] +local messageData = ARGV[3] +local messageScore = ARGV[4] + +-- Write the message to the message key +redis.call('SET', messageKey, messageData) + +-- Add the message to the queue +redis.call('ZADD', queue, messageScore, messageId) + +-- Rebalance the parent queue +local earliestMessage = redis.call('ZRANGE', queue, 0, 0, 'WITHSCORES') +if #earliestMessage == 0 then + redis.call('ZREM', parentQueue, queueName) +else + redis.call('ZADD', parentQueue, earliestMessage[2], queueName) +end + +-- Update the concurrency keys +redis.call('SREM', concurrencyKey, messageId) +redis.call('SREM', envCurrentConcurrencyKey, messageId) + +-- Update concurrency tracking (remove) +redis.call('SREM', taskConcurrencyTrackerKey, messageId) +redis.call('SREM', envConcurrencyTrackerKey, messageId) +redis.call('SREM', globalConcurrencyTrackerKey, messageId) + `, + }); + + this.redis.defineCommand("dequeueMessage", { + numberOfKeys: 7, + lua: ` +local childQueue = KEYS[1] +local parentQueue = KEYS[2] +local concurrencyLimitKey = KEYS[3] +local envConcurrencyLimitKey = KEYS[4] +local currentConcurrencyKey = KEYS[5] +local envCurrentConcurrencyKey = KEYS[6] +local globalConcurrencyTrackerKey = KEYS[7] + +local childQueueName = ARGV[1] +local currentTime = tonumber(ARGV[2]) +local defaultEnvConcurrencyLimit = ARGV[3] + +-- Check current env concurrency against the limit +local envCurrentConcurrency = tonumber(redis.call('SCARD', envCurrentConcurrencyKey) or '0') +local envConcurrencyLimit = tonumber(redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit) + +if envCurrentConcurrency >= envConcurrencyLimit then + return nil +end + +-- Check current queue concurrency against the limit +local currentConcurrency = tonumber(redis.call('SCARD', currentConcurrencyKey) or '0') +local concurrencyLimit = tonumber(redis.call('GET', concurrencyLimitKey) or '1000000') + +-- Check condition only if concurrencyLimit exists +if currentConcurrency >= concurrencyLimit then + return nil +end + +-- Attempt to dequeue the next message +local messages = redis.call('ZRANGEBYSCORE', childQueue, '-inf', currentTime, 'WITHSCORES', 'LIMIT', 0, 1) + +if #messages == 0 then + return nil +end + +local messageId = messages[1] +local messageScore = tonumber(messages[2]) + +-- Update concurrency +redis.call('ZREM', childQueue, messageId) +redis.call('SADD', currentConcurrencyKey, messageId) +redis.call('SADD', envCurrentConcurrencyKey, messageId) +redis.call('SADD', taskConcurrencyKey, messageId) + +-- Rebalance the parent queue +local earliestMessage = redis.call('ZRANGE', childQueue, 0, 0, 'WITHSCORES') +if #earliestMessage == 0 then + redis.call('ZREM', parentQueue, childQueueName) +else + redis.call('ZADD', parentQueue, earliestMessage[2], childQueueName) +end + +return {messageId, messageScore} -- Return message details + `, + }); + + this.redis.defineCommand("acknowledgeMessage", { + numberOfKeys: 7, + lua: ` +-- Keys: parentQueue, messageKey, messageQueue, visibilityQueue, concurrencyKey, envCurrentConcurrencyKey, orgCurrentConcurrencyKey +local parentQueue = KEYS[1] +local messageKey = KEYS[2] +local messageQueue = KEYS[3] +local visibilityQueue = KEYS[4] +local concurrencyKey = KEYS[5] +local envCurrentConcurrencyKey = KEYS[6] +local orgCurrentConcurrencyKey = KEYS[7] + +-- Args: messageId, messageQueueName +local messageId = ARGV[1] +local messageQueueName = ARGV[2] + +-- Remove the message from the message key +redis.call('DEL', messageKey) + +-- Remove the message from the queue +redis.call('ZREM', messageQueue, messageId) + +-- Rebalance the parent queue +local earliestMessage = redis.call('ZRANGE', messageQueue, 0, 0, 'WITHSCORES') +if #earliestMessage == 0 then + redis.call('ZREM', parentQueue, messageQueueName) +else + redis.call('ZADD', parentQueue, earliestMessage[2], messageQueueName) +end + +-- Remove the message from the timeout queue (deprecated, will eventually remove this) +redis.call('ZREM', visibilityQueue, messageId) + +-- Update the concurrency keys +redis.call('SREM', concurrencyKey, messageId) +redis.call('SREM', envCurrentConcurrencyKey, messageId) +redis.call('SREM', orgCurrentConcurrencyKey, messageId) +`, + }); + + this.redis.defineCommand("nackMessage", { + numberOfKeys: 7, + lua: ` +-- Keys: childQueueKey, parentQueueKey, visibilityQueue, concurrencyKey, envConcurrencyKey, orgConcurrencyKey, messageId +local messageKey = KEYS[1] +local childQueueKey = KEYS[2] +local parentQueueKey = KEYS[3] +local concurrencyKey = KEYS[4] +local envConcurrencyKey = KEYS[5] +local orgConcurrencyKey = KEYS[6] +local visibilityQueue = KEYS[7] + +-- Args: childQueueName, messageId, currentTime, messageScore +local childQueueName = ARGV[1] +local messageId = ARGV[2] +local currentTime = tonumber(ARGV[3]) +local messageScore = tonumber(ARGV[4]) + +-- Update the concurrency keys +redis.call('SREM', concurrencyKey, messageId) +redis.call('SREM', envConcurrencyKey, messageId) +redis.call('SREM', orgConcurrencyKey, messageId) + +-- Check to see if the message is still in the visibilityQueue +local messageVisibility = tonumber(redis.call('ZSCORE', visibilityQueue, messageId)) or 0 + +if messageVisibility > 0 then +-- Remove the message from the timeout queue (deprecated, will eventually remove this) + redis.call('ZREM', visibilityQueue, messageId) +end + +-- Enqueue the message into the queue +redis.call('ZADD', childQueueKey, messageScore, messageId) + +-- Rebalance the parent queue +local earliestMessage = redis.call('ZRANGE', childQueueKey, 0, 0, 'WITHSCORES') +if #earliestMessage == 0 then + redis.call('ZREM', parentQueueKey, childQueueName) +else + redis.call('ZADD', parentQueueKey, earliestMessage[2], childQueueName) +end +`, + }); + + this.redis.defineCommand("releaseConcurrency", { + numberOfKeys: 3, + lua: ` +local concurrencyKey = KEYS[1] +local envCurrentConcurrencyKey = KEYS[2] +local orgCurrentConcurrencyKey = KEYS[3] + +local messageId = ARGV[1] + +-- Update the concurrency keys +if concurrencyKey ~= "" then + redis.call('SREM', concurrencyKey, messageId) +end +redis.call('SREM', envCurrentConcurrencyKey, messageId) +redis.call('SREM', orgCurrentConcurrencyKey, messageId) +`, + }); + + this.redis.defineCommand("calculateMessageQueueCapacitiesWithDisabling", { + numberOfKeys: 5, + lua: ` +-- Keys +local currentConcurrencyKey = KEYS[1] +local currentEnvConcurrencyKey = KEYS[2] +local concurrencyLimitKey = KEYS[4] +local envConcurrencyLimitKey = KEYS[5] +local disabledConcurrencyLimitKey = KEYS[7] + +-- Args +local defaultEnvConcurrencyLimit = tonumber(ARGV[1]) + +-- Check if disabledConcurrencyLimitKey exists +local orgIsEnabled +if redis.call('EXISTS', disabledConcurrencyLimitKey) == 1 then + orgIsEnabled = false +else + orgIsEnabled = true +end + +local currentEnvConcurrency = tonumber(redis.call('SCARD', currentEnvConcurrencyKey) or '0') +local envConcurrencyLimit = tonumber(redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit) + +local currentConcurrency = tonumber(redis.call('SCARD', currentConcurrencyKey) or '0') +local concurrencyLimit = redis.call('GET', concurrencyLimitKey) + +-- Return current capacity and concurrency limits for the queue, env, org +return { currentConcurrency, concurrencyLimit, currentEnvConcurrency, envConcurrencyLimit, currentOrgConcurrency, orgIsEnabled } + `, + }); + + this.redis.defineCommand("calculateMessageQueueCapacities", { + numberOfKeys: 6, + lua: ` +-- Keys: +local currentConcurrencyKey = KEYS[1] +local currentEnvConcurrencyKey = KEYS[2] +local concurrencyLimitKey = KEYS[4] +local envConcurrencyLimitKey = KEYS[5] + +-- Args +local defaultEnvConcurrencyLimit = tonumber(ARGV[1]) + +local currentEnvConcurrency = tonumber(redis.call('SCARD', currentEnvConcurrencyKey) or '0') +local envConcurrencyLimit = tonumber(redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit) + +local currentConcurrency = tonumber(redis.call('SCARD', currentConcurrencyKey) or '0') +local concurrencyLimit = redis.call('GET', concurrencyLimitKey) + +-- Return current capacity and concurrency limits for the queue, env, org +return { currentConcurrency, concurrencyLimit, currentEnvConcurrency, envConcurrencyLimit, currentOrgConcurrency, true } + `, + }); + + this.redis.defineCommand("updateGlobalConcurrencyLimits", { + numberOfKeys: 2, + lua: ` +-- Keys: envConcurrencyLimitKey, orgConcurrencyLimitKey +local envConcurrencyLimitKey = KEYS[1] +local orgConcurrencyLimitKey = KEYS[2] + +-- Args: envConcurrencyLimit, orgConcurrencyLimit +local envConcurrencyLimit = ARGV[1] +local orgConcurrencyLimit = ARGV[2] + +redis.call('SET', envConcurrencyLimitKey, envConcurrencyLimit) +redis.call('SET', orgConcurrencyLimitKey, orgConcurrencyLimit) + `, + }); + + this.redis.defineCommand("rebalanceParentQueueChild", { + numberOfKeys: 2, + lua: ` +-- Keys: childQueueKey, parentQueueKey +local childQueueKey = KEYS[1] +local parentQueueKey = KEYS[2] + +-- Args: childQueueName, currentScore +local childQueueName = ARGV[1] +local currentScore = ARGV[2] + +-- Rebalance the parent queue +local earliestMessage = redis.call('ZRANGE', childQueueKey, 0, 0, 'WITHSCORES') +if #earliestMessage == 0 then + redis.call('ZREM', parentQueueKey, childQueueName) + + -- Return true because the parent queue was rebalanced + return true +else + -- If the earliest message is different, update the parent queue and return true, else return false + if earliestMessage[2] == currentScore then + return false + end + + redis.call('ZADD', parentQueueKey, earliestMessage[2], childQueueName) + + return earliestMessage[2] +end +`, + }); + } +} + +declare module "ioredis" { + interface RedisCommander { + enqueueMessage( + //keys + queue: string, + parentQueue: string, + messageKey: string, + concurrencyKey: string, + envConcurrencyKey: string, + taskConcurrencyTrackerKey: string, + environmentConcurrencyTrackerKey: string, + environmentTypeConcurrencyTrackerKey: string, + //args + queueName: string, + messageId: string, + messageData: string, + messageScore: string, + callback?: Callback + ): Result; + + dequeueMessage( + //keys + childQueue: string, + parentQueue: string, + concurrencyLimitKey: string, + envConcurrencyLimitKey: string, + currentConcurrencyKey: string, + envCurrentConcurrencyKey: string, + globalCurrentConcurrencyKey: string, + //args + childQueueName: string, + currentTime: string, + defaultEnvConcurrencyLimit: string, + callback?: Callback<[string, string]> + ): Result<[string, string] | null, Context>; + + acknowledgeMessage( + parentQueue: string, + messageKey: string, + messageQueue: string, + visibilityQueue: string, + concurrencyKey: string, + envConcurrencyKey: string, + orgConcurrencyKey: string, + messageId: string, + messageQueueName: string, + callback?: Callback + ): Result; + + nackMessage( + messageKey: string, + childQueueKey: string, + parentQueueKey: string, + concurrencyKey: string, + envConcurrencyKey: string, + orgConcurrencyKey: string, + visibilityQueue: string, + childQueueName: string, + messageId: string, + currentTime: string, + messageScore: string, + callback?: Callback + ): Result; + + releaseConcurrency( + concurrencyKey: string, + envConcurrencyKey: string, + orgConcurrencyKey: string, + messageId: string, + callback?: Callback + ): Result; + + calculateMessageQueueCapacities( + currentConcurrencyKey: string, + currentEnvConcurrencyKey: string, + concurrencyLimitKey: string, + envConcurrencyLimitKey: string, + defaultEnvConcurrencyLimit: string, + callback?: Callback + ): Result<[number, number, number, number, number, boolean], Context>; + + calculateMessageQueueCapacitiesWithDisabling( + currentConcurrencyKey: string, + currentEnvConcurrencyKey: string, + concurrencyLimitKey: string, + envConcurrencyLimitKey: string, + disabledConcurrencyLimitKey: string, + defaultEnvConcurrencyLimit: string, + callback?: Callback + ): Result<[number, number, number, number, number, boolean], Context>; + + updateGlobalConcurrencyLimits( + envConcurrencyLimitKey: string, + envConcurrencyLimit: string, + callback?: Callback + ): Result; + + rebalanceParentQueueChild( + childQueueKey: string, + parentQueueKey: string, + childQueueName: string, + currentScore: string, + callback?: Callback + ): Result; + } +} + +// Only allow alphanumeric characters, underscores, hyphens, and slashes (and only the first 128 characters) +export function sanitizeQueueName(queueName: string) { + return queueName.replace(/[^a-zA-Z0-9_\-\/]/g, "").substring(0, 128); +} diff --git a/packages/run-engine/src/run-queue/keyProducer.ts b/packages/run-engine/src/run-queue/keyProducer.ts new file mode 100644 index 0000000000..df3c56da24 --- /dev/null +++ b/packages/run-engine/src/run-queue/keyProducer.ts @@ -0,0 +1,195 @@ +import { RuntimeEnvironmentType } from "@trigger.dev/database"; +import { AuthenticatedEnvironment } from "../shared/index.js"; +import { RunQueueKeyProducer } from "./types.js"; + +const constants = { + SHARED_QUEUE: "sharedQueue", + CURRENT_CONCURRENCY_PART: "currentConcurrency", + CONCURRENCY_LIMIT_PART: "concurrency", + DISABLED_CONCURRENCY_LIMIT_PART: "disabledConcurrency", + ENV_PART: "env", + ENV_TYPE_PART: "envType", + ORG_PART: "org", + PROJECT_PART: "proj", + QUEUE_PART: "queue", + CONCURRENCY_KEY_PART: "ck", + TASK_PART: "task", + MESSAGE_PART: "message", +} as const; + +//org:${orgId}:proj:${projId}:envType:${envType}:env:${envId}queue:${queue}:ck:${concurrencyKey}:currentConcurrency + +export class RunQueueShortKeyProducer implements RunQueueKeyProducer { + constructor(private _prefix: string) {} + + sharedQueueScanPattern() { + return `${this._prefix}*${constants.SHARED_QUEUE}`; + } + + queueCurrentConcurrencyScanPattern() { + return `${this._prefix}${constants.ORG_PART}:*:${constants.PROJECT_PART}:*:${constants.ENV_TYPE_PART}:*:${constants.ENV_PART}:*:${constants.QUEUE_PART}:*:${constants.CURRENT_CONCURRENCY_PART}`; + } + + stripKeyPrefix(key: string): string { + if (key.startsWith(this._prefix)) { + return key.slice(this._prefix.length); + } + + return key; + } + + queueConcurrencyLimitKey(env: AuthenticatedEnvironment, queue: string) { + return [this.queueKey(env, queue), constants.CONCURRENCY_LIMIT_PART].join(":"); + } + + envConcurrencyLimitKey(env: AuthenticatedEnvironment) { + return [this.envKeySection(env.id), constants.CONCURRENCY_LIMIT_PART].join(":"); + } + + queueKey(env: AuthenticatedEnvironment, queue: string, concurrencyKey?: string) { + return [ + this.orgKeySection(env.organizationId), + this.projKeySection(env.projectId), + this.envTypeKeySection(env.type), + this.envKeySection(env.id), + this.queueSection(queue), + ] + .concat(concurrencyKey ? this.concurrencyKeySection(concurrencyKey) : []) + .join(":"); + } + + envSharedQueueKey(env: AuthenticatedEnvironment) { + if (env.type === "DEVELOPMENT") { + return [ + this.orgKeySection(env.organizationId), + this.projKeySection(env.projectId), + this.envKeySection(env.id), + constants.SHARED_QUEUE, + ].join(":"); + } + + return this.sharedQueueKey(); + } + + sharedQueueKey(): string { + return constants.SHARED_QUEUE; + } + + concurrencyLimitKeyFromQueue(queue: string) { + const concurrencyQueueName = queue.replace(/:ck:.+$/, ""); + return `${concurrencyQueueName}:${constants.CONCURRENCY_LIMIT_PART}`; + } + + currentConcurrencyKeyFromQueue(queue: string) { + return `${queue}:${constants.CURRENT_CONCURRENCY_PART}`; + } + + //orgs:${orgId}:proj:${projectId}:task:${taskIdentifier}:env:${envId}:currentConcurrency + currentTaskIdentifierKey({ + orgId, + projectId, + taskIdentifier, + environmentId, + }: { + orgId: string; + projectId: string; + taskIdentifier: string; + environmentId: string; + }) { + return [ + this.orgKeySection(orgId), + this.projKeySection(projectId), + this.taskIdentifierSection(taskIdentifier), + environmentId ? this.envKeySection(environmentId) : undefined, + constants.CURRENT_CONCURRENCY_PART, + ] + .filter(Boolean) + .join(":"); + } + + currentConcurrencyKey( + env: AuthenticatedEnvironment, + queue: string, + concurrencyKey?: string + ): string { + return [this.queueKey(env, queue, concurrencyKey), constants.CURRENT_CONCURRENCY_PART].join( + ":" + ); + } + + disabledConcurrencyLimitKeyFromQueue(queue: string) { + const { orgId } = this.extractComponentsFromQueue(queue); + return `${constants.ORG_PART}:${orgId}:${constants.DISABLED_CONCURRENCY_LIMIT_PART}`; + } + + envConcurrencyLimitKeyFromQueue(queue: string) { + const { envId } = this.extractComponentsFromQueue(queue); + return `${constants.ENV_PART}:${envId}:${constants.CONCURRENCY_LIMIT_PART}`; + } + + envCurrentConcurrencyKeyFromQueue(queue: string) { + const { envId } = this.extractComponentsFromQueue(queue); + return `${constants.ENV_PART}:${envId}:${constants.CURRENT_CONCURRENCY_PART}`; + } + + envCurrentConcurrencyKey(env: AuthenticatedEnvironment): string { + return [this.envKeySection(env.id), constants.CURRENT_CONCURRENCY_PART].join(":"); + } + + globalCurrentConcurrencyKey(queue: string): string { + return queue.replace(/:env:.+$/, ":*"); + } + + messageKey(messageId: string) { + return `${constants.MESSAGE_PART}:${messageId}`; + } + + private envKeySection(envId: string) { + return `${constants.ENV_PART}:${envId}`; + } + + private envTypeKeySection(envType: RuntimeEnvironmentType) { + return `${constants.ENV_TYPE_PART}:${envType === "DEVELOPMENT" ? "dev" : "deployed"}`; + } + + private projKeySection(projId: string) { + return `${constants.PROJECT_PART}:${projId}`; + } + + private orgKeySection(orgId: string) { + return `${constants.ORG_PART}:${orgId}`; + } + + private queueSection(queue: string) { + return `${constants.QUEUE_PART}:${queue}`; + } + + private concurrencyKeySection(concurrencyKey: string) { + return `${constants.CONCURRENCY_KEY_PART}:${concurrencyKey}`; + } + + private taskIdentifierSection(taskIdentifier: string) { + return `${constants.TASK_PART}:${taskIdentifier}`; + } + + private extractComponentsFromQueue(queue: string) { + const parts = this.normalizeQueue(queue).split(":"); + return { + orgId: parts[1], + projectId: parts[3], + envType: parts[5], + envId: parts[7], + queue: parts[9], + concurrencyKey: parts.at(11), + }; + } + + // This removes the leading prefix from the queue name if it exists + private normalizeQueue(queue: string) { + if (queue.startsWith(this._prefix)) { + return queue.slice(this._prefix.length); + } + + return queue; + } +} diff --git a/packages/run-engine/src/run-queue/types.ts b/packages/run-engine/src/run-queue/types.ts new file mode 100644 index 0000000000..ea2bc2f042 --- /dev/null +++ b/packages/run-engine/src/run-queue/types.ts @@ -0,0 +1,106 @@ +import { z } from "zod"; +import { AuthenticatedEnvironment } from "../shared/index.js"; +import { RuntimeEnvironmentType } from "@trigger.dev/database"; +import { env } from "process"; + +export const MessagePayload = z.object({ + version: z.literal("1"), + runId: z.string(), + taskIdentifier: z.string(), + orgId: z.string(), + projectId: z.string(), + environmentId: z.string(), + environmentType: z.nativeEnum(RuntimeEnvironmentType), + queue: z.string(), + timestamp: z.number(), + parentQueue: z.string(), + concurrencyKey: z.string().optional(), +}); + +export type MessagePayload = z.infer; + +export type QueueCapacity = { + current: number; + limit: number; +}; + +export type QueueCapacities = { + queue: QueueCapacity; + env: QueueCapacity; +}; + +export type QueueWithScores = { + queue: string; + capacities: QueueCapacities; + age: number; + size: number; +}; + +export type QueueRange = { offset: number; count: number }; + +export interface RunQueueKeyProducer { + queueConcurrencyLimitKey(env: AuthenticatedEnvironment, queue: string): string; + envConcurrencyLimitKey(env: AuthenticatedEnvironment): string; + queueKey(env: AuthenticatedEnvironment, queue: string, concurrencyKey?: string): string; + envSharedQueueKey(env: AuthenticatedEnvironment): string; + sharedQueueKey(): string; + sharedQueueScanPattern(): string; + queueCurrentConcurrencyScanPattern(): string; + concurrencyLimitKeyFromQueue(queue: string): string; + currentConcurrencyKeyFromQueue(queue: string): string; + currentConcurrencyKey( + env: AuthenticatedEnvironment, + queue: string, + concurrencyKey?: string + ): string; + currentTaskIdentifierKey({ + orgId, + projectId, + taskIdentifier, + environmentId, + }: { + orgId: string; + projectId: string; + taskIdentifier: string; + environmentId: string; + }): string; + disabledConcurrencyLimitKeyFromQueue(queue: string): string; + envConcurrencyLimitKeyFromQueue(queue: string): string; + envCurrentConcurrencyKeyFromQueue(queue: string): string; + envCurrentConcurrencyKey(env: AuthenticatedEnvironment): string; + messageKey(messageId: string): string; + globalCurrentConcurrencyKey(queue: string): string; + stripKeyPrefix(key: string): string; +} + +export type PriorityStrategyChoice = string | { abort: true }; + +export interface RunQueuePriorityStrategy { + /** + * chooseQueue is called to select the next queue to process a message from + * + * @param queues + * @param parentQueue + * @param consumerId + * + * @returns The queue to process the message from, or an object with `abort: true` if no queue is available + */ + chooseQueue( + queues: Array, + parentQueue: string, + consumerId: string, + previousRange: QueueRange + ): { choice: PriorityStrategyChoice; nextRange: QueueRange }; + + /** + * This function is called to get the next candidate selection for the queue + * The `range` is used to select the set of queues that will be considered for the next selection (passed to chooseQueue) + * The `selectionId` is used to identify the selection and should be passed to chooseQueue + * + * @param parentQueue The parent queue that holds the candidate queues + * @param consumerId The consumerId that is making the request + * + * @returns The scores and the selectionId for the next candidate selection + */ + nextCandidateSelection(parentQueue: string, consumerId: string): Promise<{ range: QueueRange }>; +} From c3de286a9082d51148dde70af028a0db802b7c81 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 26 Sep 2024 13:36:02 +0100 Subject: [PATCH 032/114] Fix for some imports --- packages/run-engine/src/engine/index.test.ts | 2 +- packages/run-engine/src/simple-queue/index.test.ts | 8 ++++---- packages/run-engine/src/simple-queue/processor.test.ts | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/run-engine/src/engine/index.test.ts b/packages/run-engine/src/engine/index.test.ts index c83fcbc977..cf139f8c29 100644 --- a/packages/run-engine/src/engine/index.test.ts +++ b/packages/run-engine/src/engine/index.test.ts @@ -1,5 +1,5 @@ import { expect } from "vitest"; -import { postgresTest, redisTest } from "../test/containerTest"; +import { postgresTest, redisTest } from "../test/containerTest.js"; postgresTest("Prisma create user", { timeout: 15_000 }, async ({ prisma }) => { await prisma.user.create({ diff --git a/packages/run-engine/src/simple-queue/index.test.ts b/packages/run-engine/src/simple-queue/index.test.ts index 233efebedc..85918bec81 100644 --- a/packages/run-engine/src/simple-queue/index.test.ts +++ b/packages/run-engine/src/simple-queue/index.test.ts @@ -1,8 +1,8 @@ -import { expect, it } from "vitest"; -import { z } from "zod"; -import { redisTest } from "../test/containerTest"; -import { SimpleQueue } from "./index"; import { describe } from "node:test"; +import { expect } from "vitest"; +import { z } from "zod"; +import { redisTest } from "../test/containerTest.js"; +import { SimpleQueue } from "./index.js"; describe("SimpleQueue", () => { redisTest("enqueue/dequeue", { timeout: 20_000 }, async ({ redisContainer }) => { diff --git a/packages/run-engine/src/simple-queue/processor.test.ts b/packages/run-engine/src/simple-queue/processor.test.ts index 6e27a4d962..2904f8a5d5 100644 --- a/packages/run-engine/src/simple-queue/processor.test.ts +++ b/packages/run-engine/src/simple-queue/processor.test.ts @@ -1,9 +1,9 @@ import { expect, it } from "vitest"; import { z } from "zod"; -import { redisTest } from "../test/containerTest"; -import { SimpleQueue } from "./index"; +import { redisTest } from "../test/containerTest.js"; +import { SimpleQueue } from "./index.js"; import { describe } from "node:test"; -import { createQueueProcessor } from "./processor"; +import { createQueueProcessor } from "./processor.js"; import { Logger } from "@trigger.dev/core/logger"; describe("SimpleQueue processor", () => { From 08adc145394465b47d1c6b89cba4a676e42fae61 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 26 Sep 2024 13:36:19 +0100 Subject: [PATCH 033/114] Key producer with some tests --- .../src/run-queue/keyProducer.test.ts | 330 ++++++++++++++++++ .../run-engine/src/run-queue/keyProducer.ts | 93 ++--- packages/run-engine/src/run-queue/types.ts | 10 +- packages/run-engine/src/shared/index.ts | 23 +- 4 files changed, 406 insertions(+), 50 deletions(-) create mode 100644 packages/run-engine/src/run-queue/keyProducer.test.ts diff --git a/packages/run-engine/src/run-queue/keyProducer.test.ts b/packages/run-engine/src/run-queue/keyProducer.test.ts new file mode 100644 index 0000000000..f37e58cc09 --- /dev/null +++ b/packages/run-engine/src/run-queue/keyProducer.test.ts @@ -0,0 +1,330 @@ +import { expect, it } from "vitest"; +import { z } from "zod"; +import { redisTest } from "../test/containerTest.js"; +import { describe } from "node:test"; +import { RunQueueShortKeyProducer } from "./keyProducer.js"; + +describe("KeyProducer", () => { + it("sharedQueueScanPattern", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const pattern = keyProducer.sharedQueueScanPattern(); + expect(pattern).toBe("test:*sharedQueue"); + }); + + it("queueCurrentConcurrencyScanPattern", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const pattern = keyProducer.queueCurrentConcurrencyScanPattern(); + expect(pattern).toBe("test:{org:*}:proj:*:envType:*:env:*:queue:*:currentConcurrency"); + }); + + it("stripKeyPrefix", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.stripKeyPrefix("test:abc"); + expect(key).toBe("abc"); + }); + + it("queueConcurrencyLimitKey deployed", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.queueConcurrencyLimitKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + expect(key).toBe( + "{org:o1234}:proj:p1234:envType:deployed:env:e1234:queue:task/task-name:concurrency" + ); + }); + + it("queueConcurrencyLimitKey dev", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.queueConcurrencyLimitKey( + { + id: "e1234", + type: "DEVELOPMENT", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + expect(key).toBe( + "{org:o1234}:proj:p1234:envType:dev:env:e1234:queue:task/task-name:concurrency" + ); + }); + + it("envConcurrencyLimitKey", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.envConcurrencyLimitKey({ + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }); + expect(key).toBe("{org:o1234}:proj:p1234:envType:deployed:env:e1234:concurrency"); + }); + + it("queueKey (no concurrency)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + expect(key).toBe("{org:o1234}:proj:p1234:envType:deployed:env:e1234:queue:task/task-name"); + }); + + it("queueKey (w concurrency)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name", + "c1234" + ); + expect(key).toBe( + "{org:o1234}:proj:p1234:envType:deployed:env:e1234:queue:task/task-name:ck:c1234" + ); + }); + + it("envSharedQueueKey dev", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.envSharedQueueKey({ + id: "e1234", + type: "DEVELOPMENT", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }); + expect(key).toBe("{org:o1234}:proj:p1234:envType:dev:env:e1234:sharedQueue"); + }); + + it("envSharedQueueKey deployed", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.envSharedQueueKey({ + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }); + expect(key).toBe("sharedQueue"); + }); + + it("sharedQueueKey", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.sharedQueueKey(); + expect(key).toBe("sharedQueue"); + }); + + it("concurrencyLimitKeyFromQueue (w concurrency)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name", + "c1234" + ); + const key = keyProducer.concurrencyLimitKeyFromQueue(queueKey); + expect(key).toBe( + "{org:o1234}:proj:p1234:envType:deployed:env:e1234:queue:task/task-name:concurrency" + ); + }); + + it("concurrencyLimitKeyFromQueue (no concurrency)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + const key = keyProducer.concurrencyLimitKeyFromQueue(queueKey); + expect(key).toBe( + "{org:o1234}:proj:p1234:envType:deployed:env:e1234:queue:task/task-name:concurrency" + ); + }); + + it("currentConcurrencyKeyFromQueue (w concurrency)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name", + "c1234" + ); + const key = keyProducer.currentConcurrencyKeyFromQueue(queueKey); + expect(key).toBe( + "{org:o1234}:proj:p1234:envType:deployed:env:e1234:queue:task/task-name:ck:c1234:currentConcurrency" + ); + }); + + it("currentConcurrencyKeyFromQueue (no concurrency)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + const key = keyProducer.currentConcurrencyKeyFromQueue(queueKey); + expect(key).toBe( + "{org:o1234}:proj:p1234:envType:deployed:env:e1234:queue:task/task-name:currentConcurrency" + ); + }); + + it("currentConcurrencyKey (w concurrency)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.currentConcurrencyKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name", + "c1234" + ); + expect(key).toBe( + "{org:o1234}:proj:p1234:envType:deployed:env:e1234:queue:task/task-name:ck:c1234:currentConcurrency" + ); + }); + + it("currentConcurrencyKey (no concurrency)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.currentConcurrencyKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + expect(key).toBe( + "{org:o1234}:proj:p1234:envType:deployed:env:e1234:queue:task/task-name:currentConcurrency" + ); + }); + + it("currentTaskIdentifierKey", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.currentTaskIdentifierKey({ + orgId: "o1234", + projectId: "p1234", + taskIdentifier: "task/task-name", + environmentId: "e1234", + }); + expect(key).toBe("{org:o1234}:proj:p1234:task:task/task-name:env:e1234:currentConcurrency"); + }); + + it("disabledConcurrencyLimitKeyFromQueue", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + const key = keyProducer.disabledConcurrencyLimitKeyFromQueue(queueKey); + expect(key).toBe("{org:o1234}:disabledConcurrency"); + }); + + it("envConcurrencyLimitKeyFromQueue", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + const key = keyProducer.envConcurrencyLimitKeyFromQueue(queueKey); + expect(key).toBe("{org:o1234}:env:e1234:concurrency"); + }); + + it("envCurrentConcurrencyKeyFromQueue", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + const key = keyProducer.envCurrentConcurrencyKeyFromQueue(queueKey); + expect(key).toBe("{org:o1234}:env:e1234:currentConcurrency"); + }); + + it("envCurrentConcurrencyKey", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.envCurrentConcurrencyKey({ + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }); + expect(key).toBe("{org:o1234}:env:e1234:currentConcurrency"); + }); + + it("globalCurrentConcurrencyKey", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + const key = keyProducer.globalCurrentConcurrencyKey(queueKey); + expect(key).toBe("currentConcurrency"); + }); +}); diff --git a/packages/run-engine/src/run-queue/keyProducer.ts b/packages/run-engine/src/run-queue/keyProducer.ts index df3c56da24..4a7e790972 100644 --- a/packages/run-engine/src/run-queue/keyProducer.ts +++ b/packages/run-engine/src/run-queue/keyProducer.ts @@ -17,8 +17,6 @@ const constants = { MESSAGE_PART: "message", } as const; -//org:${orgId}:proj:${projId}:envType:${envType}:env:${envId}queue:${queue}:ck:${concurrencyKey}:currentConcurrency - export class RunQueueShortKeyProducer implements RunQueueKeyProducer { constructor(private _prefix: string) {} @@ -27,7 +25,7 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { } queueCurrentConcurrencyScanPattern() { - return `${this._prefix}${constants.ORG_PART}:*:${constants.PROJECT_PART}:*:${constants.ENV_TYPE_PART}:*:${constants.ENV_PART}:*:${constants.QUEUE_PART}:*:${constants.CURRENT_CONCURRENCY_PART}`; + return `${this._prefix}{${constants.ORG_PART}:*}:${constants.PROJECT_PART}:*:${constants.ENV_TYPE_PART}:*:${constants.ENV_PART}:*:${constants.QUEUE_PART}:*:${constants.CURRENT_CONCURRENCY_PART}`; } stripKeyPrefix(key: string): string { @@ -43,13 +41,19 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { } envConcurrencyLimitKey(env: AuthenticatedEnvironment) { - return [this.envKeySection(env.id), constants.CONCURRENCY_LIMIT_PART].join(":"); + return [ + this.orgKeySection(env.organization.id), + this.projKeySection(env.project.id), + this.envTypeKeySection(env.type), + this.envKeySection(env.id), + constants.CONCURRENCY_LIMIT_PART, + ].join(":"); } queueKey(env: AuthenticatedEnvironment, queue: string, concurrencyKey?: string) { return [ - this.orgKeySection(env.organizationId), - this.projKeySection(env.projectId), + this.orgKeySection(env.organization.id), + this.projKeySection(env.project.id), this.envTypeKeySection(env.type), this.envKeySection(env.id), this.queueSection(queue), @@ -61,8 +65,9 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { envSharedQueueKey(env: AuthenticatedEnvironment) { if (env.type === "DEVELOPMENT") { return [ - this.orgKeySection(env.organizationId), - this.projKeySection(env.projectId), + this.orgKeySection(env.organization.id), + this.projKeySection(env.project.id), + this.envTypeKeySection(env.type), this.envKeySection(env.id), constants.SHARED_QUEUE, ].join(":"); @@ -84,7 +89,16 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { return `${queue}:${constants.CURRENT_CONCURRENCY_PART}`; } - //orgs:${orgId}:proj:${projectId}:task:${taskIdentifier}:env:${envId}:currentConcurrency + currentConcurrencyKey( + env: AuthenticatedEnvironment, + queue: string, + concurrencyKey?: string + ): string { + return [this.queueKey(env, queue, concurrencyKey), constants.CURRENT_CONCURRENCY_PART].join( + ":" + ); + } + currentTaskIdentifierKey({ orgId, projectId, @@ -100,48 +114,57 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { this.orgKeySection(orgId), this.projKeySection(projectId), this.taskIdentifierSection(taskIdentifier), - environmentId ? this.envKeySection(environmentId) : undefined, + this.envKeySection(environmentId), constants.CURRENT_CONCURRENCY_PART, ] .filter(Boolean) .join(":"); } - currentConcurrencyKey( - env: AuthenticatedEnvironment, - queue: string, - concurrencyKey?: string - ): string { - return [this.queueKey(env, queue, concurrencyKey), constants.CURRENT_CONCURRENCY_PART].join( - ":" - ); - } - disabledConcurrencyLimitKeyFromQueue(queue: string) { const { orgId } = this.extractComponentsFromQueue(queue); - return `${constants.ORG_PART}:${orgId}:${constants.DISABLED_CONCURRENCY_LIMIT_PART}`; + return `{${constants.ORG_PART}:${orgId}}:${constants.DISABLED_CONCURRENCY_LIMIT_PART}`; } envConcurrencyLimitKeyFromQueue(queue: string) { - const { envId } = this.extractComponentsFromQueue(queue); - return `${constants.ENV_PART}:${envId}:${constants.CONCURRENCY_LIMIT_PART}`; + const { orgId, envId } = this.extractComponentsFromQueue(queue); + return `{${constants.ORG_PART}:${orgId}}:${constants.ENV_PART}:${envId}:${constants.CONCURRENCY_LIMIT_PART}`; } envCurrentConcurrencyKeyFromQueue(queue: string) { - const { envId } = this.extractComponentsFromQueue(queue); - return `${constants.ENV_PART}:${envId}:${constants.CURRENT_CONCURRENCY_PART}`; + const { orgId, envId } = this.extractComponentsFromQueue(queue); + return `{${constants.ORG_PART}:${orgId}}:${constants.ENV_PART}:${envId}:${constants.CURRENT_CONCURRENCY_PART}`; } envCurrentConcurrencyKey(env: AuthenticatedEnvironment): string { - return [this.envKeySection(env.id), constants.CURRENT_CONCURRENCY_PART].join(":"); + return [ + this.orgKeySection(env.organization.id), + this.envKeySection(env.id), + constants.CURRENT_CONCURRENCY_PART, + ].join(":"); } + //todo think about this globalCurrentConcurrencyKey(queue: string): string { return queue.replace(/:env:.+$/, ":*"); } - messageKey(messageId: string) { - return `${constants.MESSAGE_PART}:${messageId}`; + messageKey(orgId: string, messageId: string) { + return [this.orgKeySection(orgId), `${constants.MESSAGE_PART}:${messageId}`] + .filter(Boolean) + .join(":"); + } + + extractComponentsFromQueue(queue: string) { + const parts = this.normalizeQueue(queue).split(":"); + return { + orgId: parts[1].replace("{", "").replace("}", ""), + projectId: parts[3], + envType: parts[5], + envId: parts[7], + queue: parts[9], + concurrencyKey: parts.at(11), + }; } private envKeySection(envId: string) { @@ -157,7 +180,7 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { } private orgKeySection(orgId: string) { - return `${constants.ORG_PART}:${orgId}`; + return `{${constants.ORG_PART}:${orgId}}`; } private queueSection(queue: string) { @@ -172,18 +195,6 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { return `${constants.TASK_PART}:${taskIdentifier}`; } - private extractComponentsFromQueue(queue: string) { - const parts = this.normalizeQueue(queue).split(":"); - return { - orgId: parts[1], - projectId: parts[3], - envType: parts[5], - envId: parts[7], - queue: parts[9], - concurrencyKey: parts.at(11), - }; - } - // This removes the leading prefix from the queue name if it exists private normalizeQueue(queue: string) { if (queue.startsWith(this._prefix)) { diff --git a/packages/run-engine/src/run-queue/types.ts b/packages/run-engine/src/run-queue/types.ts index ea2bc2f042..5d522232f2 100644 --- a/packages/run-engine/src/run-queue/types.ts +++ b/packages/run-engine/src/run-queue/types.ts @@ -68,9 +68,17 @@ export interface RunQueueKeyProducer { envConcurrencyLimitKeyFromQueue(queue: string): string; envCurrentConcurrencyKeyFromQueue(queue: string): string; envCurrentConcurrencyKey(env: AuthenticatedEnvironment): string; - messageKey(messageId: string): string; + messageKey(orgId: string, messageId: string): string; globalCurrentConcurrencyKey(queue: string): string; stripKeyPrefix(key: string): string; + extractComponentsFromQueue(queue: string): { + orgId: string; + projectId: string; + envType: string; + envId: string; + queue: string; + concurrencyKey: string | undefined; + }; } export type PriorityStrategyChoice = string | { abort: true }; diff --git a/packages/run-engine/src/shared/index.ts b/packages/run-engine/src/shared/index.ts index 669c90e8e5..b7393714a7 100644 --- a/packages/run-engine/src/shared/index.ts +++ b/packages/run-engine/src/shared/index.ts @@ -1,10 +1,22 @@ import { Attributes } from "@opentelemetry/api"; import { Prisma } from "@trigger.dev/database"; -export type AuthenticatedEnvironment = Prisma.RuntimeEnvironmentGetPayload<{ +type EnvironmentWithExtras = Prisma.RuntimeEnvironmentGetPayload<{ include: { project: true; organization: true; orgMember: true }; }>; +export type AuthenticatedEnvironment = { + id: EnvironmentWithExtras["id"]; + type: EnvironmentWithExtras["type"]; + maximumConcurrencyLimit: EnvironmentWithExtras["maximumConcurrencyLimit"]; + project: { + id: EnvironmentWithExtras["project"]["id"]; + }; + organization: { + id: EnvironmentWithExtras["organization"]["id"]; + }; +}; + const SemanticEnvResources = { ENV_ID: "$trigger.env.id", ENV_TYPE: "$trigger.env.type", @@ -21,12 +33,7 @@ export function attributesFromAuthenticatedEnv(env: AuthenticatedEnvironment): A return { [SemanticEnvResources.ENV_ID]: env.id, [SemanticEnvResources.ENV_TYPE]: env.type, - [SemanticEnvResources.ENV_SLUG]: env.slug, - [SemanticEnvResources.ORG_ID]: env.organizationId, - [SemanticEnvResources.ORG_SLUG]: env.organization.slug, - [SemanticEnvResources.ORG_TITLE]: env.organization.title, - [SemanticEnvResources.PROJECT_ID]: env.projectId, - [SemanticEnvResources.PROJECT_NAME]: env.project.name, - [SemanticEnvResources.USER_ID]: env.orgMember?.userId, + [SemanticEnvResources.ORG_ID]: env.organization.id, + [SemanticEnvResources.PROJECT_ID]: env.project.id, }; } From 8d13edcabac8f366ed55018513c9fd44a27affed Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 26 Sep 2024 14:09:40 +0100 Subject: [PATCH 034/114] =?UTF-8?q?Removed=20the=20nv=20type=20from=20the?= =?UTF-8?q?=20keys=E2=80=A6=20it=E2=80=99s=20not=20useful=20to=20do=20glob?= =?UTF-8?q?al=20queries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/run-engine/src/run-queue/index.ts | 316 ++++++++---------- .../src/run-queue/keyProducer.test.ts | 73 +--- .../run-engine/src/run-queue/keyProducer.ts | 22 +- packages/run-engine/src/run-queue/types.ts | 2 - 4 files changed, 158 insertions(+), 255 deletions(-) diff --git a/packages/run-engine/src/run-queue/index.ts b/packages/run-engine/src/run-queue/index.ts index 15c2bacc98..cce98ec276 100644 --- a/packages/run-engine/src/run-queue/index.ts +++ b/packages/run-engine/src/run-queue/index.ts @@ -19,6 +19,7 @@ import { AsyncWorker } from "../shared/asyncWorker.js"; import { MessagePayload, QueueCapacities, + QueueRange, RunQueueKeyProducer, RunQueuePriorityStrategy, } from "./types.js"; @@ -35,6 +36,7 @@ const SemanticAttributes = { PARENT_QUEUE: "runqueue.parentQueue", RUN_ID: "runqueue.runId", CONCURRENCY_KEY: "runqueue.concurrencyKey", + ORG_ID: "runqueue.orgId", }; export type RunQueueOptions = { @@ -44,7 +46,6 @@ export type RunQueueOptions = { defaultEnvConcurrency: number; defaultOrgConcurrency: number; windowSize?: number; - visibilityTimeoutInMs?: number; workers: number; keysProducer: RunQueueKeyProducer; queuePriorityStrategy: RunQueuePriorityStrategy; @@ -220,7 +221,6 @@ export class RunQueue { currentConcurrencyKey: this.keys.currentConcurrencyKeyFromQueue(messageQueue), envConcurrencyLimitKey: this.keys.envConcurrencyLimitKeyFromQueue(messageQueue), envCurrentConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(messageQueue), - globalConcurrencyTrackerKey: this.keys.globalCurrentConcurrencyKey(messageQueue), }); if (!message) { @@ -308,7 +308,6 @@ export class RunQueue { currentConcurrencyKey: this.keys.currentConcurrencyKeyFromQueue(messageQueue), envConcurrencyLimitKey: this.keys.envConcurrencyLimitKeyFromQueue(messageQueue), envCurrentConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(messageQueue), - globalConcurrencyTrackerKey: this.keys.globalCurrentConcurrencyKey(messageQueue), }); if (!message) { @@ -335,25 +334,32 @@ export class RunQueue { ); } - public async acknowledgeMessage(messageId: string) { + /** + * Acknowledge a message, which will: + * - remove all data from the queue + * - release all concurrency + * This is done when the run is in a final state. + * @param orgId + * @param messageId + */ + public async acknowledgeMessage(orgId: string, messageId: string) { return this.#trace( "acknowledgeMessage", async (span) => { - span.setAttributes({ - [SemanticAttributes.RUN_ID]: messageId, - }); - - const message = await this.#callAcknowledgeMessage({ - messageKey: this.keys.messageKey(messageId), - messageId, - }); - - span.setAttributes({ - [SemanticAttributes.RUN_ID]: messageId, - [SemanticAttributes.QUEUE]: message.queue, - [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, - [SemanticAttributes.PARENT_QUEUE]: message.parentQueue, - }); + // span.setAttributes({ + // [SemanticAttributes.RUN_ID]: messageId, + // [SemanticAttributes.ORG_ID]: orgId, + // }); + // const message = await this.#callAcknowledgeMessage({ + // messageKey: this.keys.messageKey(orgId, messageId), + // messageId, + // }); + // span.setAttributes({ + // [SemanticAttributes.RUN_ID]: messageId, + // [SemanticAttributes.QUEUE]: message.queue, + // [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, + // [SemanticAttributes.PARENT_QUEUE]: message.parentQueue, + // }); }, { kind: SpanKind.CONSUMER, @@ -366,53 +372,102 @@ export class RunQueue { ); } + /** + * Negative acknowledge a message, which will requeue the message + */ + public async nackMessage( + messageId: string, + retryAt: number = Date.now(), + updates?: Record + ) { + return this.#trace( + "nackMessage", + async (span) => { + // const message = await this.readMessage(messageId); + // if (!message) { + // logger.log(`[${this.name}].nackMessage() message not found`, { + // messageId, + // retryAt, + // updates, + // service: this.name, + // }); + // return; + // } + // span.setAttributes({ + // [SemanticAttributes.QUEUE]: message.queue, + // [SemanticAttributes.MESSAGE_ID]: message.messageId, + // [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, + // [SemanticAttributes.PARENT_QUEUE]: message.parentQueue, + // }); + // if (updates) { + // await this.replaceMessage(messageId, updates, retryAt, true); + // } + // await this.options.visibilityTimeoutStrategy.cancelHeartbeat(messageId); + // await this.#callNackMessage({ + // messageKey: this.keys.messageKey(messageId), + // messageQueue: message.queue, + // parentQueue: message.parentQueue, + // concurrencyKey: this.keys.currentConcurrencyKeyFromQueue(message.queue), + // envConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(message.queue), + // orgConcurrencyKey: this.keys.orgCurrentConcurrencyKeyFromQueue(message.queue), + // visibilityQueue: constants.MESSAGE_VISIBILITY_TIMEOUT_QUEUE, + // messageId, + // messageScore: retryAt, + // }); + // await this.options.subscriber?.messageNacked(message); + }, + { + kind: SpanKind.CONSUMER, + attributes: { + [SEMATTRS_MESSAGING_OPERATION]: "nack", + [SEMATTRS_MESSAGE_ID]: messageId, + [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", + }, + } + ); + } + public async releaseConcurrency(messageId: string, releaseForRun: boolean = false) { return this.#trace( "releaseConcurrency", async (span) => { - span.setAttributes({ - [SemanticAttributes.MESSAGE_ID]: messageId, - }); - - const message = await this.readMessage(messageId); - - if (!message) { - logger.log(`[${this.name}].releaseConcurrency() message not found`, { - messageId, - releaseForRun, - service: this.name, - }); - return; - } - - span.setAttributes({ - [SemanticAttributes.QUEUE]: message.queue, - [SemanticAttributes.MESSAGE_ID]: message.messageId, - [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, - [SemanticAttributes.PARENT_QUEUE]: message.parentQueue, - }); - - const concurrencyKey = this.keys.currentConcurrencyKeyFromQueue(message.queue); - const envConcurrencyKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); - const orgConcurrencyKey = this.keys.orgCurrentConcurrencyKeyFromQueue(message.queue); - - logger.debug("Calling releaseConcurrency", { - messageId, - queue: message.queue, - concurrencyKey, - envConcurrencyKey, - orgConcurrencyKey, - service: this.name, - releaseForRun, - }); - - return this.redis.releaseConcurrency( - //don't release the for the run, it breaks concurrencyLimits - releaseForRun ? concurrencyKey : "", - envConcurrencyKey, - orgConcurrencyKey, - message.messageId - ); + // span.setAttributes({ + // [SemanticAttributes.MESSAGE_ID]: messageId, + // }); + // const message = await this.readMessage(messageId); + // if (!message) { + // logger.log(`[${this.name}].releaseConcurrency() message not found`, { + // messageId, + // releaseForRun, + // service: this.name, + // }); + // return; + // } + // span.setAttributes({ + // [SemanticAttributes.QUEUE]: message.queue, + // [SemanticAttributes.MESSAGE_ID]: message.messageId, + // [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, + // [SemanticAttributes.PARENT_QUEUE]: message.parentQueue, + // }); + // const concurrencyKey = this.keys.currentConcurrencyKeyFromQueue(message.queue); + // const envConcurrencyKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); + // const orgConcurrencyKey = this.keys.orgCurrentConcurrencyKeyFromQueue(message.queue); + // logger.debug("Calling releaseConcurrency", { + // messageId, + // queue: message.queue, + // concurrencyKey, + // envConcurrencyKey, + // orgConcurrencyKey, + // service: this.name, + // releaseForRun, + // }); + // return this.redis.releaseConcurrency( + // //don't release the for the run, it breaks concurrencyLimits + // releaseForRun ? concurrencyKey : "", + // envConcurrencyKey, + // orgConcurrencyKey, + // message.messageId + // ); }, { kind: SpanKind.CONSUMER, @@ -456,81 +511,11 @@ export class RunQueue { ); } - /** - * Negative acknowledge a message, which will requeue the message - */ - public async nackMessage( - messageId: string, - retryAt: number = Date.now(), - updates?: Record - ) { - return this.#trace( - "nackMessage", - async (span) => { - const message = await this.readMessage(messageId); - - if (!message) { - logger.log(`[${this.name}].nackMessage() message not found`, { - messageId, - retryAt, - updates, - service: this.name, - }); - return; - } - - span.setAttributes({ - [SemanticAttributes.QUEUE]: message.queue, - [SemanticAttributes.MESSAGE_ID]: message.messageId, - [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, - [SemanticAttributes.PARENT_QUEUE]: message.parentQueue, - }); - - if (updates) { - await this.replaceMessage(messageId, updates, retryAt, true); - } - - await this.options.visibilityTimeoutStrategy.cancelHeartbeat(messageId); - - await this.#callNackMessage({ - messageKey: this.keys.messageKey(messageId), - messageQueue: message.queue, - parentQueue: message.parentQueue, - concurrencyKey: this.keys.currentConcurrencyKeyFromQueue(message.queue), - envConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(message.queue), - orgConcurrencyKey: this.keys.orgCurrentConcurrencyKeyFromQueue(message.queue), - visibilityQueue: constants.MESSAGE_VISIBILITY_TIMEOUT_QUEUE, - messageId, - messageScore: retryAt, - }); - - await this.options.subscriber?.messageNacked(message); - }, - { - kind: SpanKind.CONSUMER, - attributes: { - [SEMATTRS_MESSAGING_OPERATION]: "nack", - [SEMATTRS_MESSAGE_ID]: messageId, - [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", - }, - } - ); - } - - // This should increment by the number of seconds, but with a max value of Date.now() + visibilityTimeoutInMs - public async heartbeatMessage(messageId: string) { - await this.options.visibilityTimeoutStrategy.heartbeat(messageId, this.visibilityTimeoutInMs); - } - - get visibilityTimeoutInMs() { - return this.options.visibilityTimeoutInMs ?? 300000; // 5 minutes - } - - async readMessage(messageId: string) { + async readMessage(orgId: string, messageId: string) { return this.#trace( "readMessage", async (span) => { - const rawMessage = await this.redis.get(this.keys.messageKey(messageId)); + const rawMessage = await this.redis.get(this.keys.messageKey(orgId, messageId)); if (!rawMessage) { return; @@ -563,7 +548,7 @@ export class RunQueue { async #getRandomQueueFromParentQueue( parentQueue: string, - queuePriorityStrategy: MarQSQueuePriorityStrategy, + queuePriorityStrategy: RunQueuePriorityStrategy, calculateCapacities: (queue: string) => Promise, consumerId: string ) { @@ -604,7 +589,7 @@ export class RunQueue { if (this.options.verbose || nextRange.offset > 0) { if (typeof choice === "string") { - logger.debug(`[${this.name}] getRandomQueueFromParentQueue`, { + this.logger.debug(`[${this.name}] getRandomQueueFromParentQueue`, { queues, queuesWithScores, range, @@ -615,7 +600,7 @@ export class RunQueue { consumerId, }); } else { - logger.debug(`[${this.name}] getRandomQueueFromParentQueue`, { + this.logger.debug(`[${this.name}] getRandomQueueFromParentQueue`, { queues, queuesWithScores, range, @@ -736,7 +721,7 @@ export class RunQueue { ) { const pattern = this.keys.queueCurrentConcurrencyScanPattern(); - logger.debug("Starting queue concurrency scan stream", { + this.logger.debug("Starting queue concurrency scan stream", { pattern, component: "runqueue", operation: "queueConcurrencyScanStream", @@ -776,7 +761,7 @@ export class RunQueue { count: 100, }); - logger.debug("Streaming parent queues based on pattern", { + this.logger.debug("Streaming parent queues based on pattern", { pattern, component: "runqueue", operation: "rebalanceParentQueues", @@ -792,7 +777,7 @@ export class RunQueue { stream.pause(); - logger.debug("Rebalancing parent queues", { + this.logger.debug("Rebalancing parent queues", { component: "runqueue", operation: "rebalanceParentQueues", parentQueues: uniqueKeys, @@ -882,9 +867,6 @@ export class RunQueue { taskIdentifier: message.taskIdentifier, }); const envConcurrencyTrackerKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); - const globalConcurrencyTrackerKey = this.keys.globalCurrentConcurrencyKey( - message.environmentType - ); this.logger.debug("Calling enqueueMessage", { messagePayload: message, @@ -896,12 +878,11 @@ export class RunQueue { return this.redis.enqueueMessage( message.queue, message.parentQueue, - this.keys.messageKey(message.runId), + this.keys.messageKey(message.orgId, message.runId), concurrencyKey, envConcurrencyKey, taskConcurrencyTrackerKey, envConcurrencyTrackerKey, - globalConcurrencyTrackerKey, message.queue, message.runId, JSON.stringify(message), @@ -916,7 +897,6 @@ export class RunQueue { envConcurrencyLimitKey, currentConcurrencyKey, envCurrentConcurrencyKey, - globalConcurrencyTrackerKey, }: { messageQueue: string; parentQueue: string; @@ -924,7 +904,6 @@ export class RunQueue { envConcurrencyLimitKey: string; currentConcurrencyKey: string; envCurrentConcurrencyKey: string; - globalConcurrencyTrackerKey: string; }) { const result = await this.redis.dequeueMessage( //keys @@ -934,7 +913,6 @@ export class RunQueue { envConcurrencyLimitKey, currentConcurrencyKey, envCurrentConcurrencyKey, - globalConcurrencyTrackerKey, //args messageQueue, String(Date.now()), @@ -961,7 +939,8 @@ export class RunQueue { const [messageId, messageScore] = result; //read message - const message = await this.readMessage(messageId); + const { orgId } = this.keys.extractComponentsFromQueue(messageQueue); + const message = await this.readMessage(orgId, messageId); if (!message) { this.logger.error(`Dequeued then failed to read message. This is unrecoverable.`, { @@ -983,39 +962,27 @@ export class RunQueue { } async #callAcknowledgeMessage({ - parentQueue, messageKey, - messageQueue, - concurrencyKey, - envConcurrencyKey, messageId, }: { - parentQueue: string; messageKey: string; - messageQueue: string; - concurrencyKey: string; - envConcurrencyKey: string; messageId: string; }) { this.logger.debug("Calling acknowledgeMessage", { messageKey, - messageQueue, - concurrencyKey, - envConcurrencyKey, messageId, - parentQueue, service: this.name, }); - return this.redis.acknowledgeMessage( - parentQueue, - messageKey, - messageQueue, - concurrencyKey, - envConcurrencyKey, - messageId, - messageQueue - ); + // return this.redis.acknowledgeMessage( + // parentQueue, + // messageKey, + // messageQueue, + // concurrencyKey, + // envConcurrencyKey, + // messageId, + // messageQueue + // ); } async #callNackMessage({ @@ -1161,7 +1128,7 @@ export class RunQueue { #registerCommands() { this.redis.defineCommand("enqueueMessage", { - numberOfKeys: 8, + numberOfKeys: 7, lua: ` local queue = KEYS[1] local parentQueue = KEYS[2] @@ -1170,7 +1137,6 @@ local concurrencyKey = KEYS[4] local envCurrentConcurrencyKey = KEYS[5] local taskConcurrencyTrackerKey = KEYS[6] local envConcurrencyTrackerKey = KEYS[7] -local globalConcurrencyTrackerKey = KEYS[8] local queueName = ARGV[1] local messageId = ARGV[2] @@ -1198,12 +1164,11 @@ redis.call('SREM', envCurrentConcurrencyKey, messageId) -- Update concurrency tracking (remove) redis.call('SREM', taskConcurrencyTrackerKey, messageId) redis.call('SREM', envConcurrencyTrackerKey, messageId) -redis.call('SREM', globalConcurrencyTrackerKey, messageId) `, }); this.redis.defineCommand("dequeueMessage", { - numberOfKeys: 7, + numberOfKeys: 6, lua: ` local childQueue = KEYS[1] local parentQueue = KEYS[2] @@ -1211,7 +1176,6 @@ local concurrencyLimitKey = KEYS[3] local envConcurrencyLimitKey = KEYS[4] local currentConcurrencyKey = KEYS[5] local envCurrentConcurrencyKey = KEYS[6] -local globalConcurrencyTrackerKey = KEYS[7] local childQueueName = ARGV[1] local currentTime = tonumber(ARGV[2]) @@ -1479,7 +1443,6 @@ declare module "ioredis" { envConcurrencyKey: string, taskConcurrencyTrackerKey: string, environmentConcurrencyTrackerKey: string, - environmentTypeConcurrencyTrackerKey: string, //args queueName: string, messageId: string, @@ -1496,7 +1459,6 @@ declare module "ioredis" { envConcurrencyLimitKey: string, currentConcurrencyKey: string, envCurrentConcurrencyKey: string, - globalCurrentConcurrencyKey: string, //args childQueueName: string, currentTime: string, diff --git a/packages/run-engine/src/run-queue/keyProducer.test.ts b/packages/run-engine/src/run-queue/keyProducer.test.ts index f37e58cc09..1008424350 100644 --- a/packages/run-engine/src/run-queue/keyProducer.test.ts +++ b/packages/run-engine/src/run-queue/keyProducer.test.ts @@ -14,7 +14,7 @@ describe("KeyProducer", () => { it("queueCurrentConcurrencyScanPattern", () => { const keyProducer = new RunQueueShortKeyProducer("test:"); const pattern = keyProducer.queueCurrentConcurrencyScanPattern(); - expect(pattern).toBe("test:{org:*}:proj:*:envType:*:env:*:queue:*:currentConcurrency"); + expect(pattern).toBe("test:{org:*}:proj:*:env:*:queue:*:currentConcurrency"); }); it("stripKeyPrefix", () => { @@ -23,7 +23,7 @@ describe("KeyProducer", () => { expect(key).toBe("abc"); }); - it("queueConcurrencyLimitKey deployed", () => { + it("queueConcurrencyLimitKey", () => { const keyProducer = new RunQueueShortKeyProducer("test:"); const key = keyProducer.queueConcurrencyLimitKey( { @@ -35,26 +35,7 @@ describe("KeyProducer", () => { }, "task/task-name" ); - expect(key).toBe( - "{org:o1234}:proj:p1234:envType:deployed:env:e1234:queue:task/task-name:concurrency" - ); - }); - - it("queueConcurrencyLimitKey dev", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const key = keyProducer.queueConcurrencyLimitKey( - { - id: "e1234", - type: "DEVELOPMENT", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }, - "task/task-name" - ); - expect(key).toBe( - "{org:o1234}:proj:p1234:envType:dev:env:e1234:queue:task/task-name:concurrency" - ); + expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:concurrency"); }); it("envConcurrencyLimitKey", () => { @@ -66,7 +47,7 @@ describe("KeyProducer", () => { project: { id: "p1234" }, organization: { id: "o1234" }, }); - expect(key).toBe("{org:o1234}:proj:p1234:envType:deployed:env:e1234:concurrency"); + expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:concurrency"); }); it("queueKey (no concurrency)", () => { @@ -81,7 +62,7 @@ describe("KeyProducer", () => { }, "task/task-name" ); - expect(key).toBe("{org:o1234}:proj:p1234:envType:deployed:env:e1234:queue:task/task-name"); + expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name"); }); it("queueKey (w concurrency)", () => { @@ -97,9 +78,7 @@ describe("KeyProducer", () => { "task/task-name", "c1234" ); - expect(key).toBe( - "{org:o1234}:proj:p1234:envType:deployed:env:e1234:queue:task/task-name:ck:c1234" - ); + expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:ck:c1234"); }); it("envSharedQueueKey dev", () => { @@ -111,7 +90,7 @@ describe("KeyProducer", () => { project: { id: "p1234" }, organization: { id: "o1234" }, }); - expect(key).toBe("{org:o1234}:proj:p1234:envType:dev:env:e1234:sharedQueue"); + expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:sharedQueue"); }); it("envSharedQueueKey deployed", () => { @@ -146,9 +125,7 @@ describe("KeyProducer", () => { "c1234" ); const key = keyProducer.concurrencyLimitKeyFromQueue(queueKey); - expect(key).toBe( - "{org:o1234}:proj:p1234:envType:deployed:env:e1234:queue:task/task-name:concurrency" - ); + expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:concurrency"); }); it("concurrencyLimitKeyFromQueue (no concurrency)", () => { @@ -164,9 +141,7 @@ describe("KeyProducer", () => { "task/task-name" ); const key = keyProducer.concurrencyLimitKeyFromQueue(queueKey); - expect(key).toBe( - "{org:o1234}:proj:p1234:envType:deployed:env:e1234:queue:task/task-name:concurrency" - ); + expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:concurrency"); }); it("currentConcurrencyKeyFromQueue (w concurrency)", () => { @@ -184,7 +159,7 @@ describe("KeyProducer", () => { ); const key = keyProducer.currentConcurrencyKeyFromQueue(queueKey); expect(key).toBe( - "{org:o1234}:proj:p1234:envType:deployed:env:e1234:queue:task/task-name:ck:c1234:currentConcurrency" + "{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:ck:c1234:currentConcurrency" ); }); @@ -201,9 +176,7 @@ describe("KeyProducer", () => { "task/task-name" ); const key = keyProducer.currentConcurrencyKeyFromQueue(queueKey); - expect(key).toBe( - "{org:o1234}:proj:p1234:envType:deployed:env:e1234:queue:task/task-name:currentConcurrency" - ); + expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:currentConcurrency"); }); it("currentConcurrencyKey (w concurrency)", () => { @@ -220,7 +193,7 @@ describe("KeyProducer", () => { "c1234" ); expect(key).toBe( - "{org:o1234}:proj:p1234:envType:deployed:env:e1234:queue:task/task-name:ck:c1234:currentConcurrency" + "{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:ck:c1234:currentConcurrency" ); }); @@ -236,9 +209,9 @@ describe("KeyProducer", () => { }, "task/task-name" ); - expect(key).toBe( - "{org:o1234}:proj:p1234:envType:deployed:env:e1234:queue:task/task-name:currentConcurrency" - ); + + //todo it's pointless having envType:deployed in here I think + expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:currentConcurrency"); }); it("currentTaskIdentifierKey", () => { @@ -311,20 +284,4 @@ describe("KeyProducer", () => { }); expect(key).toBe("{org:o1234}:env:e1234:currentConcurrency"); }); - - it("globalCurrentConcurrencyKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const queueKey = keyProducer.queueKey( - { - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }, - "task/task-name" - ); - const key = keyProducer.globalCurrentConcurrencyKey(queueKey); - expect(key).toBe("currentConcurrency"); - }); }); diff --git a/packages/run-engine/src/run-queue/keyProducer.ts b/packages/run-engine/src/run-queue/keyProducer.ts index 4a7e790972..54098cd9ce 100644 --- a/packages/run-engine/src/run-queue/keyProducer.ts +++ b/packages/run-engine/src/run-queue/keyProducer.ts @@ -8,7 +8,6 @@ const constants = { CONCURRENCY_LIMIT_PART: "concurrency", DISABLED_CONCURRENCY_LIMIT_PART: "disabledConcurrency", ENV_PART: "env", - ENV_TYPE_PART: "envType", ORG_PART: "org", PROJECT_PART: "proj", QUEUE_PART: "queue", @@ -25,7 +24,7 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { } queueCurrentConcurrencyScanPattern() { - return `${this._prefix}{${constants.ORG_PART}:*}:${constants.PROJECT_PART}:*:${constants.ENV_TYPE_PART}:*:${constants.ENV_PART}:*:${constants.QUEUE_PART}:*:${constants.CURRENT_CONCURRENCY_PART}`; + return `${this._prefix}{${constants.ORG_PART}:*}:${constants.PROJECT_PART}:*:${constants.ENV_PART}:*:${constants.QUEUE_PART}:*:${constants.CURRENT_CONCURRENCY_PART}`; } stripKeyPrefix(key: string): string { @@ -44,7 +43,6 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { return [ this.orgKeySection(env.organization.id), this.projKeySection(env.project.id), - this.envTypeKeySection(env.type), this.envKeySection(env.id), constants.CONCURRENCY_LIMIT_PART, ].join(":"); @@ -54,7 +52,6 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { return [ this.orgKeySection(env.organization.id), this.projKeySection(env.project.id), - this.envTypeKeySection(env.type), this.envKeySection(env.id), this.queueSection(queue), ] @@ -67,7 +64,6 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { return [ this.orgKeySection(env.organization.id), this.projKeySection(env.project.id), - this.envTypeKeySection(env.type), this.envKeySection(env.id), constants.SHARED_QUEUE, ].join(":"); @@ -144,11 +140,6 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { ].join(":"); } - //todo think about this - globalCurrentConcurrencyKey(queue: string): string { - return queue.replace(/:env:.+$/, ":*"); - } - messageKey(orgId: string, messageId: string) { return [this.orgKeySection(orgId), `${constants.MESSAGE_PART}:${messageId}`] .filter(Boolean) @@ -160,10 +151,9 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { return { orgId: parts[1].replace("{", "").replace("}", ""), projectId: parts[3], - envType: parts[5], - envId: parts[7], - queue: parts[9], - concurrencyKey: parts.at(11), + envId: parts[5], + queue: parts[7], + concurrencyKey: parts.at(9), }; } @@ -171,10 +161,6 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { return `${constants.ENV_PART}:${envId}`; } - private envTypeKeySection(envType: RuntimeEnvironmentType) { - return `${constants.ENV_TYPE_PART}:${envType === "DEVELOPMENT" ? "dev" : "deployed"}`; - } - private projKeySection(projId: string) { return `${constants.PROJECT_PART}:${projId}`; } diff --git a/packages/run-engine/src/run-queue/types.ts b/packages/run-engine/src/run-queue/types.ts index 5d522232f2..79d75a78fb 100644 --- a/packages/run-engine/src/run-queue/types.ts +++ b/packages/run-engine/src/run-queue/types.ts @@ -69,12 +69,10 @@ export interface RunQueueKeyProducer { envCurrentConcurrencyKeyFromQueue(queue: string): string; envCurrentConcurrencyKey(env: AuthenticatedEnvironment): string; messageKey(orgId: string, messageId: string): string; - globalCurrentConcurrencyKey(queue: string): string; stripKeyPrefix(key: string): string; extractComponentsFromQueue(queue: string): { orgId: string; projectId: string; - envType: string; envId: string; queue: string; concurrencyKey: string | undefined; From 0fa6bbf43bd7e68fa0f3e1e6b7eb108d44ed770e Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 26 Sep 2024 14:44:52 +0100 Subject: [PATCH 035/114] Passing unit tests for all the public key producer functions --- .../src/run-queue/keyProducer.test.ts | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/packages/run-engine/src/run-queue/keyProducer.test.ts b/packages/run-engine/src/run-queue/keyProducer.test.ts index 1008424350..0121098f05 100644 --- a/packages/run-engine/src/run-queue/keyProducer.test.ts +++ b/packages/run-engine/src/run-queue/keyProducer.test.ts @@ -284,4 +284,55 @@ describe("KeyProducer", () => { }); expect(key).toBe("{org:o1234}:env:e1234:currentConcurrency"); }); + + it("messageKey", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const key = keyProducer.messageKey("o1234", "m1234"); + expect(key).toBe("{org:o1234}:message:m1234"); + }); + + it("extractComponentsFromQueue (no concurrencyKey)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + const components = keyProducer.extractComponentsFromQueue(queueKey); + expect(components).toEqual({ + orgId: "o1234", + projectId: "p1234", + envId: "e1234", + queue: "task/task-name", + concurrencyKey: undefined, + }); + }); + + it("extractComponentsFromQueue (w concurrencyKey)", () => { + const keyProducer = new RunQueueShortKeyProducer("test:"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name", + "c1234" + ); + const components = keyProducer.extractComponentsFromQueue(queueKey); + expect(components).toEqual({ + orgId: "o1234", + projectId: "p1234", + envId: "e1234", + queue: "task/task-name", + concurrencyKey: "c1234", + }); + }); }); From 07e76d795f5f8ddb1eae5bfbc4e0ca25664b8fdf Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 26 Sep 2024 17:31:33 +0100 Subject: [PATCH 036/114] Some basic tests passing for the RunQueue --- .../run-engine/src/run-queue/index.test.ts | 106 ++++++++++++++++ packages/run-engine/src/run-queue/index.ts | 43 +++---- .../simpleWeightedPriorityStrategy.ts | 119 ++++++++++++++++++ 3 files changed, 240 insertions(+), 28 deletions(-) create mode 100644 packages/run-engine/src/run-queue/index.test.ts create mode 100644 packages/run-engine/src/run-queue/simpleWeightedPriorityStrategy.ts diff --git a/packages/run-engine/src/run-queue/index.test.ts b/packages/run-engine/src/run-queue/index.test.ts new file mode 100644 index 0000000000..05a586a6ea --- /dev/null +++ b/packages/run-engine/src/run-queue/index.test.ts @@ -0,0 +1,106 @@ +import { + Span, + SpanKind, + SpanOptions, + Tracer, + context, + propagation, + trace, +} from "@opentelemetry/api"; +import { describe } from "node:test"; +import { redisTest } from "../test/containerTest.js"; +import { RunQueue } from "./index.js"; +import { RunQueueShortKeyProducer } from "./keyProducer.js"; +import { SimpleWeightedChoiceStrategy } from "./simpleWeightedPriorityStrategy.js"; +import { logger } from "@trigger.dev/core/v3"; +import { Logger } from "@trigger.dev/core/logger"; + +const testOptions = { + name: "rq", + tracer: trace.getTracer("rq"), + keysProducer: new RunQueueShortKeyProducer("rq:"), + queuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 36 }), + envQueuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 12 }), + workers: 1, + defaultEnvConcurrency: 10, + enableRebalancing: false, + logger: new Logger("RunQueue", "debug"), +}; + +const authenticatedEnv = { + id: "e1234", + type: "PRODUCTION" as const, + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, +}; + +describe("RunQueue", () => { + redisTest( + "Get/set Queue concurrency limit", + { timeout: 5_000 }, + async ({ redisContainer, redis }) => { + const queue = new RunQueue({ + ...testOptions, + redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, + }); + + try { + //initial value + const initial = await queue.getQueueConcurrencyLimit(authenticatedEnv, "task/my-task"); + expect(initial).toBe(undefined); + + //set 20 + const result = await queue.updateQueueConcurrencyLimits( + authenticatedEnv, + "task/my-task", + 20 + ); + expect(result).toBe("OK"); + + //get 20 + const updated = await queue.getQueueConcurrencyLimit(authenticatedEnv, "task/my-task"); + expect(updated).toBe(20); + + //remove + const result2 = await queue.removeQueueConcurrencyLimits(authenticatedEnv, "task/my-task"); + expect(result2).toBe(1); + + //get undefined + const removed = await queue.getQueueConcurrencyLimit(authenticatedEnv, "task/my-task"); + expect(removed).toBe(undefined); + } finally { + await queue.quit(); + } + } + ); + + redisTest( + "Update env concurrency limits", + { timeout: 5_000 }, + async ({ redisContainer, redis }) => { + const queue = new RunQueue({ + ...testOptions, + redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, + }); + + try { + //initial value + const initial = await queue.getEnvConcurrencyLimit(authenticatedEnv); + expect(initial).toBe(10); + + //set 20 + await queue.updateEnvConcurrencyLimits({ + ...authenticatedEnv, + maximumConcurrencyLimit: 20, + }); + + //get 20 + const updated = await queue.getEnvConcurrencyLimit(authenticatedEnv); + expect(updated).toBe(20); + } finally { + await queue.quit(); + } + } + ); +}); diff --git a/packages/run-engine/src/run-queue/index.ts b/packages/run-engine/src/run-queue/index.ts index cce98ec276..9c273d5ae5 100644 --- a/packages/run-engine/src/run-queue/index.ts +++ b/packages/run-engine/src/run-queue/index.ts @@ -1,21 +1,14 @@ -import { - Span, - SpanKind, - SpanOptions, - Tracer, - context, - propagation, - trace, -} from "@opentelemetry/api"; +import { context, propagation, Span, SpanKind, SpanOptions, Tracer } from "@opentelemetry/api"; import { SEMATTRS_MESSAGE_ID, SEMATTRS_MESSAGING_OPERATION, SEMATTRS_MESSAGING_SYSTEM, } from "@opentelemetry/semantic-conventions"; +import { Logger } from "@trigger.dev/core/logger"; import { flattenAttributes } from "@trigger.dev/core/v3"; import { Redis, type Callback, type RedisOptions, type Result } from "ioredis"; -import { Logger } from "@trigger.dev/core/logger"; import { AsyncWorker } from "../shared/asyncWorker.js"; +import { attributesFromAuthenticatedEnv, AuthenticatedEnvironment } from "../shared/index.js"; import { MessagePayload, QueueCapacities, @@ -23,13 +16,6 @@ import { RunQueueKeyProducer, RunQueuePriorityStrategy, } from "./types.js"; -import { attributesFromAuthenticatedEnv, AuthenticatedEnvironment } from "../shared/index.js"; - -const KEY_PREFIX = "runqueue:"; - -const constants = { - MESSAGE_VISIBILITY_TIMEOUT_QUEUE: "msgVisibilityTimeout", -} as const; const SemanticAttributes = { QUEUE: "runqueue.queue", @@ -44,7 +30,6 @@ export type RunQueueOptions = { tracer: Tracer; redis: RedisOptions; defaultEnvConcurrency: number; - defaultOrgConcurrency: number; windowSize?: number; workers: number; keysProducer: RunQueueKeyProducer; @@ -96,6 +81,12 @@ export class RunQueue { return this.redis.del(this.keys.queueConcurrencyLimitKey(env, queue)); } + public async getQueueConcurrencyLimit(env: AuthenticatedEnvironment, queue: string) { + const result = await this.redis.get(this.keys.queueConcurrencyLimitKey(env, queue)); + + return result ? Number(result) : undefined; + } + public async updateEnvConcurrencyLimits(env: AuthenticatedEnvironment) { await this.#callUpdateGlobalConcurrencyLimits({ envConcurrencyLimitKey: this.keys.envConcurrencyLimitKey(env), @@ -103,12 +94,6 @@ export class RunQueue { }); } - public async getQueueConcurrencyLimit(env: AuthenticatedEnvironment, queue: string) { - const result = await this.redis.get(this.keys.queueConcurrencyLimitKey(env, queue)); - - return result ? Number(result) : undefined; - } - public async getEnvConcurrencyLimit(env: AuthenticatedEnvironment) { const result = await this.redis.get(this.keys.envConcurrencyLimitKey(env)); @@ -546,6 +531,11 @@ export class RunQueue { ); } + async quit() { + await Promise.all(this.#rebalanceWorkers.map((worker) => worker.stop())); + await this.redis.quit(); + } + async #getRandomQueueFromParentQueue( parentQueue: string, queuePriorityStrategy: RunQueuePriorityStrategy, @@ -1384,18 +1374,15 @@ return { currentConcurrency, concurrencyLimit, currentEnvConcurrency, envConcurr }); this.redis.defineCommand("updateGlobalConcurrencyLimits", { - numberOfKeys: 2, + numberOfKeys: 1, lua: ` -- Keys: envConcurrencyLimitKey, orgConcurrencyLimitKey local envConcurrencyLimitKey = KEYS[1] -local orgConcurrencyLimitKey = KEYS[2] -- Args: envConcurrencyLimit, orgConcurrencyLimit local envConcurrencyLimit = ARGV[1] -local orgConcurrencyLimit = ARGV[2] redis.call('SET', envConcurrencyLimitKey, envConcurrencyLimit) -redis.call('SET', orgConcurrencyLimitKey, orgConcurrencyLimit) `, }); diff --git a/packages/run-engine/src/run-queue/simpleWeightedPriorityStrategy.ts b/packages/run-engine/src/run-queue/simpleWeightedPriorityStrategy.ts new file mode 100644 index 0000000000..b85019e449 --- /dev/null +++ b/packages/run-engine/src/run-queue/simpleWeightedPriorityStrategy.ts @@ -0,0 +1,119 @@ +import { + RunQueuePriorityStrategy, + PriorityStrategyChoice, + QueueRange, + QueueWithScores, +} from "./types.js"; + +export type SimpleWeightedChoiceStrategyOptions = { + queueSelectionCount: number; + randomSeed?: string; + excludeEnvCapacity?: boolean; +}; + +export class SimpleWeightedChoiceStrategy implements RunQueuePriorityStrategy { + private _nextRangesByParentQueue: Map = new Map(); + + constructor(private options: SimpleWeightedChoiceStrategyOptions) {} + + private nextRangeForParentQueue(parentQueue: string, consumerId: string): QueueRange { + return ( + this._nextRangesByParentQueue.get(`${consumerId}:${parentQueue}`) ?? { + offset: 0, + count: this.options.queueSelectionCount, + } + ); + } + + chooseQueue( + queues: QueueWithScores[], + parentQueue: string, + consumerId: string, + previousRange: QueueRange + ): { choice: PriorityStrategyChoice; nextRange: QueueRange } { + const filteredQueues = filterQueuesAtCapacity(queues); + + if (queues.length === this.options.queueSelectionCount) { + const nextRange: QueueRange = { + offset: previousRange.offset + this.options.queueSelectionCount, + count: this.options.queueSelectionCount, + }; + + // If all queues are at capacity, and we were passed the max number of queues, then we will slide the window "to the right" + this._nextRangesByParentQueue.set(`${consumerId}:${parentQueue}`, nextRange); + } else { + this._nextRangesByParentQueue.delete(`${consumerId}:${parentQueue}`); + } + + if (filteredQueues.length === 0) { + return { + choice: { abort: true }, + nextRange: this.nextRangeForParentQueue(parentQueue, consumerId), + }; + } + + const queueWeights = this.#calculateQueueWeights(filteredQueues); + + const choice = weightedRandomChoice(queueWeights); + + return { + choice, + nextRange: this.nextRangeForParentQueue(parentQueue, consumerId), + }; + } + + async nextCandidateSelection( + parentQueue: string, + consumerId: string + ): Promise<{ range: QueueRange }> { + return { + range: this.nextRangeForParentQueue(parentQueue, consumerId), + }; + } + + #calculateQueueWeights(queues: QueueWithScores[]) { + const avgQueueSize = queues.reduce((acc, { size }) => acc + size, 0) / queues.length; + const avgMessageAge = queues.reduce((acc, { age }) => acc + age, 0) / queues.length; + + return queues.map(({ capacities, age, queue, size }) => { + let totalWeight = 1; + + if (size > avgQueueSize) { + totalWeight += Math.min(size / avgQueueSize, 4); + } + + if (age > avgMessageAge) { + totalWeight += Math.min(age / avgMessageAge, 4); + } + + return { + queue, + totalWeight: age, + }; + }); + } +} + +function filterQueuesAtCapacity(queues: QueueWithScores[]) { + return queues.filter( + (queue) => + queue.capacities.queue.current < queue.capacities.queue.limit && + queue.capacities.env.current < queue.capacities.env.limit + ); +} + +function weightedRandomChoice(queues: Array<{ queue: string; totalWeight: number }>) { + const totalWeight = queues.reduce((acc, queue) => acc + queue.totalWeight, 0); + let randomNum = Math.random() * totalWeight; + + for (const queue of queues) { + if (randomNum < queue.totalWeight) { + return queue.queue; + } + + randomNum -= queue.totalWeight; + } + + // If we get here, we should just return a random queue + return queues[Math.floor(Math.random() * queues.length)].queue; +} From 8f30533d40260d99ee01f32fb263d2b936e631ad Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 26 Sep 2024 17:50:44 +0100 Subject: [PATCH 037/114] Simple enqueue test working --- .../run-engine/src/run-queue/index.test.ts | 61 +++++++++++++++++++ packages/run-engine/src/run-queue/index.ts | 7 ++- 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/packages/run-engine/src/run-queue/index.test.ts b/packages/run-engine/src/run-queue/index.test.ts index 05a586a6ea..f6f4a40633 100644 --- a/packages/run-engine/src/run-queue/index.test.ts +++ b/packages/run-engine/src/run-queue/index.test.ts @@ -14,6 +14,7 @@ import { RunQueueShortKeyProducer } from "./keyProducer.js"; import { SimpleWeightedChoiceStrategy } from "./simpleWeightedPriorityStrategy.js"; import { logger } from "@trigger.dev/core/v3"; import { Logger } from "@trigger.dev/core/logger"; +import { MessagePayload } from "./types.js"; const testOptions = { name: "rq", @@ -35,6 +36,19 @@ const authenticatedEnv = { organization: { id: "o1234" }, }; +const message: MessagePayload = { + version: "1" as const, + runId: "r1234", + taskIdentifier: "task/my-task", + orgId: "o1234", + projectId: "p1234", + environmentId: "e1234", + environmentType: "PRODUCTION", + queue: "task/my-task", + timestamp: Date.now(), + parentQueue: "parentQueue", +}; + describe("RunQueue", () => { redisTest( "Get/set Queue concurrency limit", @@ -103,4 +117,51 @@ describe("RunQueue", () => { } } ); + + redisTest( + "Enqueue a message (no concurrency key)", + { timeout: 5_000 }, + async ({ redisContainer, redis }) => { + const queue = new RunQueue({ + ...testOptions, + redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, + }); + + try { + const result = await queue.lengthOfQueue(authenticatedEnv, message.queue); + expect(result).toBe(0); + + const oldestScore = await queue.oldestMessageInQueue(authenticatedEnv, message.queue); + expect(oldestScore).toBe(undefined); + + await queue.enqueueMessage({ env: authenticatedEnv, message }); + + const result2 = await queue.lengthOfQueue(authenticatedEnv, message.queue); + expect(result2).toBe(1); + + const oldestScore2 = await queue.oldestMessageInQueue(authenticatedEnv, message.queue); + expect(oldestScore2).toBe(message.timestamp); + } finally { + await queue.quit(); + } + } + ); + + //todo after enqueuing + // redisTest("Get length of a queue (zero)", { timeout: 5_000 }, async ({ redisContainer, redis }) => { + // const queue = new RunQueue({ + // ...testOptions, + // redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, + // }); + + // try { + // const result = await queue.lengthOfQueue(authenticatedEnv, "task/my-task"); + // expect(result).toBe(0); + + // const result2 = await queue.lengthOfQueue(authenticatedEnv, "task/my-task", "user_12345"); + // expect(result2).toBe(0); + // } finally { + // await queue.quit(); + // } + // }); }); diff --git a/packages/run-engine/src/run-queue/index.ts b/packages/run-engine/src/run-queue/index.ts index 9c273d5ae5..3051f879e7 100644 --- a/packages/run-engine/src/run-queue/index.ts +++ b/packages/run-engine/src/run-queue/index.ts @@ -142,8 +142,11 @@ export class RunQueue { public async enqueueMessage({ env, - ...message - }: { env: AuthenticatedEnvironment } & MessagePayload) { + message, + }: { + env: AuthenticatedEnvironment; + message: MessagePayload; + }) { return await this.#trace( "enqueueMessage", async (span) => { From ba8822f86a4dc57d4525168de6936ed1b46eed3e Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 26 Sep 2024 18:16:43 +0100 Subject: [PATCH 038/114] Enqueue and dequeue for dev is working --- .../run-engine/src/run-queue/index.test.ts | 82 +++++++++++-------- packages/run-engine/src/run-queue/index.ts | 45 +++++----- packages/run-engine/src/run-queue/types.ts | 1 - 3 files changed, 69 insertions(+), 59 deletions(-) diff --git a/packages/run-engine/src/run-queue/index.test.ts b/packages/run-engine/src/run-queue/index.test.ts index f6f4a40633..f4f5d7792c 100644 --- a/packages/run-engine/src/run-queue/index.test.ts +++ b/packages/run-engine/src/run-queue/index.test.ts @@ -28,7 +28,7 @@ const testOptions = { logger: new Logger("RunQueue", "debug"), }; -const authenticatedEnv = { +const authenticatedEnvProd = { id: "e1234", type: "PRODUCTION" as const, maximumConcurrencyLimit: 10, @@ -36,7 +36,15 @@ const authenticatedEnv = { organization: { id: "o1234" }, }; -const message: MessagePayload = { +const authenticatedEnvDev = { + id: "e1234", + type: "DEVELOPMENT" as const, + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, +}; + +const messageProd: MessagePayload = { version: "1" as const, runId: "r1234", taskIdentifier: "task/my-task", @@ -46,7 +54,18 @@ const message: MessagePayload = { environmentType: "PRODUCTION", queue: "task/my-task", timestamp: Date.now(), - parentQueue: "parentQueue", +}; + +const messageDev: MessagePayload = { + version: "1" as const, + runId: "r4321", + taskIdentifier: "task/my-task", + orgId: "o1234", + projectId: "p1234", + environmentId: "e4321", + environmentType: "DEVELOPMENT", + queue: "task/my-task", + timestamp: Date.now(), }; describe("RunQueue", () => { @@ -61,27 +80,30 @@ describe("RunQueue", () => { try { //initial value - const initial = await queue.getQueueConcurrencyLimit(authenticatedEnv, "task/my-task"); + const initial = await queue.getQueueConcurrencyLimit(authenticatedEnvProd, "task/my-task"); expect(initial).toBe(undefined); //set 20 const result = await queue.updateQueueConcurrencyLimits( - authenticatedEnv, + authenticatedEnvProd, "task/my-task", 20 ); expect(result).toBe("OK"); //get 20 - const updated = await queue.getQueueConcurrencyLimit(authenticatedEnv, "task/my-task"); + const updated = await queue.getQueueConcurrencyLimit(authenticatedEnvProd, "task/my-task"); expect(updated).toBe(20); //remove - const result2 = await queue.removeQueueConcurrencyLimits(authenticatedEnv, "task/my-task"); + const result2 = await queue.removeQueueConcurrencyLimits( + authenticatedEnvProd, + "task/my-task" + ); expect(result2).toBe(1); //get undefined - const removed = await queue.getQueueConcurrencyLimit(authenticatedEnv, "task/my-task"); + const removed = await queue.getQueueConcurrencyLimit(authenticatedEnvProd, "task/my-task"); expect(removed).toBe(undefined); } finally { await queue.quit(); @@ -100,17 +122,17 @@ describe("RunQueue", () => { try { //initial value - const initial = await queue.getEnvConcurrencyLimit(authenticatedEnv); + const initial = await queue.getEnvConcurrencyLimit(authenticatedEnvProd); expect(initial).toBe(10); //set 20 await queue.updateEnvConcurrencyLimits({ - ...authenticatedEnv, + ...authenticatedEnvProd, maximumConcurrencyLimit: 20, }); //get 20 - const updated = await queue.getEnvConcurrencyLimit(authenticatedEnv); + const updated = await queue.getEnvConcurrencyLimit(authenticatedEnvProd); expect(updated).toBe(20); } finally { await queue.quit(); @@ -119,7 +141,7 @@ describe("RunQueue", () => { ); redisTest( - "Enqueue a message (no concurrency key)", + "Enqueue/Dequeue a message in env (no concurrency key)", { timeout: 5_000 }, async ({ redisContainer, redis }) => { const queue = new RunQueue({ @@ -128,40 +150,28 @@ describe("RunQueue", () => { }); try { - const result = await queue.lengthOfQueue(authenticatedEnv, message.queue); + const result = await queue.lengthOfQueue(authenticatedEnvDev, messageDev.queue); expect(result).toBe(0); - const oldestScore = await queue.oldestMessageInQueue(authenticatedEnv, message.queue); + const oldestScore = await queue.oldestMessageInQueue(authenticatedEnvDev, messageDev.queue); expect(oldestScore).toBe(undefined); - await queue.enqueueMessage({ env: authenticatedEnv, message }); + await queue.enqueueMessage({ env: authenticatedEnvDev, message: messageDev }); - const result2 = await queue.lengthOfQueue(authenticatedEnv, message.queue); + const result2 = await queue.lengthOfQueue(authenticatedEnvDev, messageDev.queue); expect(result2).toBe(1); - const oldestScore2 = await queue.oldestMessageInQueue(authenticatedEnv, message.queue); - expect(oldestScore2).toBe(message.timestamp); + const oldestScore2 = await queue.oldestMessageInQueue( + authenticatedEnvDev, + messageDev.queue + ); + expect(oldestScore2).toBe(messageDev.timestamp); + + const dequeued = await queue.dequeueMessageInEnv(authenticatedEnvDev); + expect(dequeued?.messageId).toEqual(messageDev.runId); } finally { await queue.quit(); } } ); - - //todo after enqueuing - // redisTest("Get length of a queue (zero)", { timeout: 5_000 }, async ({ redisContainer, redis }) => { - // const queue = new RunQueue({ - // ...testOptions, - // redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, - // }); - - // try { - // const result = await queue.lengthOfQueue(authenticatedEnv, "task/my-task"); - // expect(result).toBe(0); - - // const result2 = await queue.lengthOfQueue(authenticatedEnv, "task/my-task", "user_12345"); - // expect(result2).toBe(0); - // } finally { - // await queue.quit(); - // } - // }); }); diff --git a/packages/run-engine/src/run-queue/index.ts b/packages/run-engine/src/run-queue/index.ts index 3051f879e7..48e7312083 100644 --- a/packages/run-engine/src/run-queue/index.ts +++ b/packages/run-engine/src/run-queue/index.ts @@ -170,7 +170,7 @@ export class RunQueue { queue, }; - await this.#callEnqueueMessage(messagePayload); + await this.#callEnqueueMessage(messagePayload, parentQueue); }, { kind: SpanKind.PRODUCER, @@ -199,6 +199,10 @@ export class RunQueue { ); if (!messageQueue) { + this.logger.log("No message queue found", { + parentQueue, + }); + return; } @@ -220,7 +224,7 @@ export class RunQueue { [SemanticAttributes.QUEUE]: message.message.queue, [SemanticAttributes.RUN_ID]: message.message.runId, [SemanticAttributes.CONCURRENCY_KEY]: message.message.concurrencyKey, - [SemanticAttributes.PARENT_QUEUE]: message.message.parentQueue, + [SemanticAttributes.PARENT_QUEUE]: parentQueue, }); return message; @@ -307,7 +311,7 @@ export class RunQueue { [SemanticAttributes.QUEUE]: message.message.queue, [SemanticAttributes.RUN_ID]: message.message.runId, [SemanticAttributes.CONCURRENCY_KEY]: message.message.concurrencyKey, - [SemanticAttributes.PARENT_QUEUE]: message.message.parentQueue, + [SemanticAttributes.PARENT_QUEUE]: parentQueue, }); return message; @@ -850,7 +854,7 @@ export class RunQueue { }); } - async #callEnqueueMessage(message: MessagePayload) { + async #callEnqueueMessage(message: MessagePayload, parentQueue: string) { const concurrencyKey = this.keys.currentConcurrencyKeyFromQueue(message.queue); const envConcurrencyKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); const taskConcurrencyTrackerKey = this.keys.currentTaskIdentifierKey({ @@ -870,7 +874,7 @@ export class RunQueue { return this.redis.enqueueMessage( message.queue, - message.parentQueue, + parentQueue, this.keys.messageKey(message.orgId, message.runId), concurrencyKey, envConcurrencyKey, @@ -1030,18 +1034,14 @@ export class RunQueue { async #callCalculateMessageCapacities({ currentConcurrencyKey, currentEnvConcurrencyKey, - concurrencyLimitKey, envConcurrencyLimitKey, - disabledConcurrencyLimitKey, }: { currentConcurrencyKey: string; currentEnvConcurrencyKey: string; - concurrencyLimitKey: string; envConcurrencyLimitKey: string; - disabledConcurrencyLimitKey: string | undefined; }): Promise { const capacities = disabledConcurrencyLimitKey @@ -1063,12 +1063,11 @@ export class RunQueue { const queueCurrent = Number(capacities[0]); const envLimit = Number(capacities[3]); - const isOrgEnabled = Boolean(capacities[5]); + const isOrgEnabled = Boolean(capacities[4]); const queueLimit = capacities[1] ? Number(capacities[1]) : Math.min(envLimit, isOrgEnabled ? Infinity : 0); const envCurrent = Number(capacities[2]); - const orgCurrent = Number(capacities[4]); return { queue: { current: queueCurrent, limit: queueLimit }, @@ -1205,7 +1204,9 @@ local messageScore = tonumber(messages[2]) redis.call('ZREM', childQueue, messageId) redis.call('SADD', currentConcurrencyKey, messageId) redis.call('SADD', envCurrentConcurrencyKey, messageId) -redis.call('SADD', taskConcurrencyKey, messageId) + +-- todo add this in here +-- redis.call('SADD', taskConcurrencyKey, messageId) -- Rebalance the parent queue local earliestMessage = redis.call('ZRANGE', childQueue, 0, 0, 'WITHSCORES') @@ -1327,9 +1328,9 @@ redis.call('SREM', orgCurrentConcurrencyKey, messageId) -- Keys local currentConcurrencyKey = KEYS[1] local currentEnvConcurrencyKey = KEYS[2] -local concurrencyLimitKey = KEYS[4] -local envConcurrencyLimitKey = KEYS[5] -local disabledConcurrencyLimitKey = KEYS[7] +local concurrencyLimitKey = KEYS[3] +local envConcurrencyLimitKey = KEYS[4] +local disabledConcurrencyLimitKey = KEYS[5] -- Args local defaultEnvConcurrencyLimit = tonumber(ARGV[1]) @@ -1349,18 +1350,18 @@ local currentConcurrency = tonumber(redis.call('SCARD', currentConcurrencyKey) o local concurrencyLimit = redis.call('GET', concurrencyLimitKey) -- Return current capacity and concurrency limits for the queue, env, org -return { currentConcurrency, concurrencyLimit, currentEnvConcurrency, envConcurrencyLimit, currentOrgConcurrency, orgIsEnabled } +return { currentConcurrency, concurrencyLimit, currentEnvConcurrency, envConcurrencyLimit, orgIsEnabled } `, }); this.redis.defineCommand("calculateMessageQueueCapacities", { - numberOfKeys: 6, + numberOfKeys: 4, lua: ` -- Keys: local currentConcurrencyKey = KEYS[1] local currentEnvConcurrencyKey = KEYS[2] -local concurrencyLimitKey = KEYS[4] -local envConcurrencyLimitKey = KEYS[5] +local concurrencyLimitKey = KEYS[3] +local envConcurrencyLimitKey = KEYS[4] -- Args local defaultEnvConcurrencyLimit = tonumber(ARGV[1]) @@ -1372,7 +1373,7 @@ local currentConcurrency = tonumber(redis.call('SCARD', currentConcurrencyKey) o local concurrencyLimit = redis.call('GET', concurrencyLimitKey) -- Return current capacity and concurrency limits for the queue, env, org -return { currentConcurrency, concurrencyLimit, currentEnvConcurrency, envConcurrencyLimit, currentOrgConcurrency, true } +return { currentConcurrency, concurrencyLimit, currentEnvConcurrency, envConcurrencyLimit, true } `, }); @@ -1499,7 +1500,7 @@ declare module "ioredis" { envConcurrencyLimitKey: string, defaultEnvConcurrencyLimit: string, callback?: Callback - ): Result<[number, number, number, number, number, boolean], Context>; + ): Result<[number, number, number, number, boolean], Context>; calculateMessageQueueCapacitiesWithDisabling( currentConcurrencyKey: string, @@ -1509,7 +1510,7 @@ declare module "ioredis" { disabledConcurrencyLimitKey: string, defaultEnvConcurrencyLimit: string, callback?: Callback - ): Result<[number, number, number, number, number, boolean], Context>; + ): Result<[number, number, number, number, boolean], Context>; updateGlobalConcurrencyLimits( envConcurrencyLimitKey: string, diff --git a/packages/run-engine/src/run-queue/types.ts b/packages/run-engine/src/run-queue/types.ts index 79d75a78fb..4c30880ec1 100644 --- a/packages/run-engine/src/run-queue/types.ts +++ b/packages/run-engine/src/run-queue/types.ts @@ -13,7 +13,6 @@ export const MessagePayload = z.object({ environmentType: z.nativeEnum(RuntimeEnvironmentType), queue: z.string(), timestamp: z.number(), - parentQueue: z.string(), concurrencyKey: z.string().optional(), }); From 63639d96dbb91af3dca353b93107443711c6ddf7 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 26 Sep 2024 18:51:53 +0100 Subject: [PATCH 039/114] =?UTF-8?q?Don=E2=80=99t=20log=20everything=20duri?= =?UTF-8?q?ng=20the=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/run-engine/src/run-queue/index.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/run-engine/src/run-queue/index.test.ts b/packages/run-engine/src/run-queue/index.test.ts index f4f5d7792c..88f235e981 100644 --- a/packages/run-engine/src/run-queue/index.test.ts +++ b/packages/run-engine/src/run-queue/index.test.ts @@ -25,7 +25,7 @@ const testOptions = { workers: 1, defaultEnvConcurrency: 10, enableRebalancing: false, - logger: new Logger("RunQueue", "debug"), + logger: new Logger("RunQueue", "warn"), }; const authenticatedEnvProd = { From c06731a607dc89c72947d0a7f83f778cf1c49350 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 26 Sep 2024 18:55:38 +0100 Subject: [PATCH 040/114] Enqueuing/dequeuing from the shared queue is working --- .../run-engine/src/run-queue/index.test.ts | 59 +++++++++++++++---- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/packages/run-engine/src/run-queue/index.test.ts b/packages/run-engine/src/run-queue/index.test.ts index 88f235e981..537f51081c 100644 --- a/packages/run-engine/src/run-queue/index.test.ts +++ b/packages/run-engine/src/run-queue/index.test.ts @@ -1,19 +1,10 @@ -import { - Span, - SpanKind, - SpanOptions, - Tracer, - context, - propagation, - trace, -} from "@opentelemetry/api"; +import { trace } from "@opentelemetry/api"; +import { Logger } from "@trigger.dev/core/logger"; import { describe } from "node:test"; import { redisTest } from "../test/containerTest.js"; import { RunQueue } from "./index.js"; import { RunQueueShortKeyProducer } from "./keyProducer.js"; import { SimpleWeightedChoiceStrategy } from "./simpleWeightedPriorityStrategy.js"; -import { logger } from "@trigger.dev/core/v3"; -import { Logger } from "@trigger.dev/core/logger"; import { MessagePayload } from "./types.js"; const testOptions = { @@ -141,7 +132,7 @@ describe("RunQueue", () => { ); redisTest( - "Enqueue/Dequeue a message in env (no concurrency key)", + "Enqueue/Dequeue a message in env (DEV run, no concurrency key)", { timeout: 5_000 }, async ({ redisContainer, redis }) => { const queue = new RunQueue({ @@ -169,6 +160,50 @@ describe("RunQueue", () => { const dequeued = await queue.dequeueMessageInEnv(authenticatedEnvDev); expect(dequeued?.messageId).toEqual(messageDev.runId); + + const dequeued2 = await queue.dequeueMessageInEnv(authenticatedEnvDev); + expect(dequeued2).toBe(undefined); + } finally { + await queue.quit(); + } + } + ); + + redisTest( + "Enqueue/Dequeue a message from the shared queue (PROD run, no concurrency key)", + { timeout: 5_000 }, + async ({ redisContainer, redis }) => { + const queue = new RunQueue({ + ...testOptions, + redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, + }); + + try { + const result = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); + expect(result).toBe(0); + + const oldestScore = await queue.oldestMessageInQueue( + authenticatedEnvProd, + messageProd.queue + ); + expect(oldestScore).toBe(undefined); + + await queue.enqueueMessage({ env: authenticatedEnvProd, message: messageProd }); + + const result2 = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); + expect(result2).toBe(1); + + const oldestScore2 = await queue.oldestMessageInQueue( + authenticatedEnvProd, + messageProd.queue + ); + expect(oldestScore2).toBe(messageProd.timestamp); + + const dequeued = await queue.dequeueMessageInSharedQueue("test_12345"); + expect(dequeued?.messageId).toEqual(messageProd.runId); + + const dequeued2 = await queue.dequeueMessageInEnv(authenticatedEnvDev); + expect(dequeued2).toBe(undefined); } finally { await queue.quit(); } From 2b8ec705cd5d05b14cc22b606d454fff49fa498f Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 26 Sep 2024 19:17:07 +0100 Subject: [PATCH 041/114] Tests for getting a shared queue --- .../run-engine/src/run-queue/index.test.ts | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/packages/run-engine/src/run-queue/index.test.ts b/packages/run-engine/src/run-queue/index.test.ts index 537f51081c..2c50d8f51e 100644 --- a/packages/run-engine/src/run-queue/index.test.ts +++ b/packages/run-engine/src/run-queue/index.test.ts @@ -6,6 +6,7 @@ import { RunQueue } from "./index.js"; import { RunQueueShortKeyProducer } from "./keyProducer.js"; import { SimpleWeightedChoiceStrategy } from "./simpleWeightedPriorityStrategy.js"; import { MessagePayload } from "./types.js"; +import { abort } from "node:process"; const testOptions = { name: "rq", @@ -209,4 +210,30 @@ describe("RunQueue", () => { } } ); + + redisTest("Get shared queue details", { timeout: 5_000 }, async ({ redisContainer, redis }) => { + const queue = new RunQueue({ + ...testOptions, + redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, + }); + + try { + const result = await queue.getSharedQueueDetails(); + expect(result.selectionId).toBe("getSharedQueueDetails"); + expect(result.queueCount).toBe(0); + expect(result.queueChoice.choice).toStrictEqual({ abort: true }); + + await queue.enqueueMessage({ env: authenticatedEnvProd, message: messageProd }); + + const result2 = await queue.getSharedQueueDetails(); + expect(result2.selectionId).toBe("getSharedQueueDetails"); + expect(result2.queueCount).toBe(1); + expect(result2.queues[0].score).toBe(messageProd.timestamp); + expect(result2.queueChoice.choice).toBe( + "{org:o1234}:proj:p1234:env:e1234:queue:task/my-task" + ); + } finally { + await queue.quit(); + } + }); }); From 87364f1d44aac6d08dd1008f6621491ad0eeab51 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 27 Sep 2024 12:04:50 +0100 Subject: [PATCH 042/114] The key producer sharedQueue can now be named, to allow multiple separate queues --- .../src/run-queue/keyProducer.test.ts | 57 +++++++++---------- .../run-engine/src/run-queue/keyProducer.ts | 16 ++++-- 2 files changed, 39 insertions(+), 34 deletions(-) diff --git a/packages/run-engine/src/run-queue/keyProducer.test.ts b/packages/run-engine/src/run-queue/keyProducer.test.ts index 0121098f05..71d193abff 100644 --- a/packages/run-engine/src/run-queue/keyProducer.test.ts +++ b/packages/run-engine/src/run-queue/keyProducer.test.ts @@ -6,25 +6,25 @@ import { RunQueueShortKeyProducer } from "./keyProducer.js"; describe("KeyProducer", () => { it("sharedQueueScanPattern", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const pattern = keyProducer.sharedQueueScanPattern(); - expect(pattern).toBe("test:*sharedQueue"); + expect(pattern).toBe("test:*sharedQueue:main"); }); it("queueCurrentConcurrencyScanPattern", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const pattern = keyProducer.queueCurrentConcurrencyScanPattern(); expect(pattern).toBe("test:{org:*}:proj:*:env:*:queue:*:currentConcurrency"); }); it("stripKeyPrefix", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const key = keyProducer.stripKeyPrefix("test:abc"); expect(key).toBe("abc"); }); it("queueConcurrencyLimitKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const key = keyProducer.queueConcurrencyLimitKey( { id: "e1234", @@ -39,7 +39,7 @@ describe("KeyProducer", () => { }); it("envConcurrencyLimitKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const key = keyProducer.envConcurrencyLimitKey({ id: "e1234", type: "PRODUCTION", @@ -51,7 +51,7 @@ describe("KeyProducer", () => { }); it("queueKey (no concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const key = keyProducer.queueKey( { id: "e1234", @@ -66,7 +66,7 @@ describe("KeyProducer", () => { }); it("queueKey (w concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const key = keyProducer.queueKey( { id: "e1234", @@ -82,7 +82,7 @@ describe("KeyProducer", () => { }); it("envSharedQueueKey dev", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const key = keyProducer.envSharedQueueKey({ id: "e1234", type: "DEVELOPMENT", @@ -90,11 +90,11 @@ describe("KeyProducer", () => { project: { id: "p1234" }, organization: { id: "o1234" }, }); - expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:sharedQueue"); + expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:sharedQueue:main"); }); it("envSharedQueueKey deployed", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const key = keyProducer.envSharedQueueKey({ id: "e1234", type: "PRODUCTION", @@ -102,17 +102,17 @@ describe("KeyProducer", () => { project: { id: "p1234" }, organization: { id: "o1234" }, }); - expect(key).toBe("sharedQueue"); + expect(key).toBe("sharedQueue:main"); }); it("sharedQueueKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const key = keyProducer.sharedQueueKey(); - expect(key).toBe("sharedQueue"); + expect(key).toBe("sharedQueue:main"); }); it("concurrencyLimitKeyFromQueue (w concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const queueKey = keyProducer.queueKey( { id: "e1234", @@ -129,7 +129,7 @@ describe("KeyProducer", () => { }); it("concurrencyLimitKeyFromQueue (no concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const queueKey = keyProducer.queueKey( { id: "e1234", @@ -145,7 +145,7 @@ describe("KeyProducer", () => { }); it("currentConcurrencyKeyFromQueue (w concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const queueKey = keyProducer.queueKey( { id: "e1234", @@ -164,7 +164,7 @@ describe("KeyProducer", () => { }); it("currentConcurrencyKeyFromQueue (no concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const queueKey = keyProducer.queueKey( { id: "e1234", @@ -180,7 +180,7 @@ describe("KeyProducer", () => { }); it("currentConcurrencyKey (w concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const key = keyProducer.currentConcurrencyKey( { id: "e1234", @@ -198,7 +198,7 @@ describe("KeyProducer", () => { }); it("currentConcurrencyKey (no concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const key = keyProducer.currentConcurrencyKey( { id: "e1234", @@ -210,12 +210,11 @@ describe("KeyProducer", () => { "task/task-name" ); - //todo it's pointless having envType:deployed in here I think expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:currentConcurrency"); }); it("currentTaskIdentifierKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const key = keyProducer.currentTaskIdentifierKey({ orgId: "o1234", projectId: "p1234", @@ -226,7 +225,7 @@ describe("KeyProducer", () => { }); it("disabledConcurrencyLimitKeyFromQueue", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const queueKey = keyProducer.queueKey( { id: "e1234", @@ -242,7 +241,7 @@ describe("KeyProducer", () => { }); it("envConcurrencyLimitKeyFromQueue", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const queueKey = keyProducer.queueKey( { id: "e1234", @@ -258,7 +257,7 @@ describe("KeyProducer", () => { }); it("envCurrentConcurrencyKeyFromQueue", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const queueKey = keyProducer.queueKey( { id: "e1234", @@ -274,7 +273,7 @@ describe("KeyProducer", () => { }); it("envCurrentConcurrencyKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const key = keyProducer.envCurrentConcurrencyKey({ id: "e1234", type: "PRODUCTION", @@ -286,13 +285,13 @@ describe("KeyProducer", () => { }); it("messageKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const key = keyProducer.messageKey("o1234", "m1234"); expect(key).toBe("{org:o1234}:message:m1234"); }); it("extractComponentsFromQueue (no concurrencyKey)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const queueKey = keyProducer.queueKey( { id: "e1234", @@ -314,7 +313,7 @@ describe("KeyProducer", () => { }); it("extractComponentsFromQueue (w concurrencyKey)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const queueKey = keyProducer.queueKey( { id: "e1234", diff --git a/packages/run-engine/src/run-queue/keyProducer.ts b/packages/run-engine/src/run-queue/keyProducer.ts index 54098cd9ce..e7311a77ec 100644 --- a/packages/run-engine/src/run-queue/keyProducer.ts +++ b/packages/run-engine/src/run-queue/keyProducer.ts @@ -3,7 +3,6 @@ import { AuthenticatedEnvironment } from "../shared/index.js"; import { RunQueueKeyProducer } from "./types.js"; const constants = { - SHARED_QUEUE: "sharedQueue", CURRENT_CONCURRENCY_PART: "currentConcurrency", CONCURRENCY_LIMIT_PART: "concurrency", DISABLED_CONCURRENCY_LIMIT_PART: "disabledConcurrency", @@ -17,10 +16,17 @@ const constants = { } as const; export class RunQueueShortKeyProducer implements RunQueueKeyProducer { - constructor(private _prefix: string) {} + sharedQueue: string; + + constructor( + private _prefix: string, + sharedQueue: string + ) { + this.sharedQueue = `sharedQueue:${sharedQueue}`; + } sharedQueueScanPattern() { - return `${this._prefix}*${constants.SHARED_QUEUE}`; + return `${this._prefix}*${this.sharedQueue}`; } queueCurrentConcurrencyScanPattern() { @@ -65,7 +71,7 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { this.orgKeySection(env.organization.id), this.projKeySection(env.project.id), this.envKeySection(env.id), - constants.SHARED_QUEUE, + this.sharedQueue, ].join(":"); } @@ -73,7 +79,7 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { } sharedQueueKey(): string { - return constants.SHARED_QUEUE; + return this.sharedQueue; } concurrencyLimitKeyFromQueue(queue: string) { From 4265e6dba97c5c14ba7ab9a2819c258f6d399f17 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 27 Sep 2024 12:06:09 +0100 Subject: [PATCH 043/114] The key producer uses the name of the queue as the input --- packages/run-engine/src/run-queue/index.test.ts | 5 ++++- packages/run-engine/src/run-queue/index.ts | 7 +++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/run-engine/src/run-queue/index.test.ts b/packages/run-engine/src/run-queue/index.test.ts index 2c50d8f51e..21149241b6 100644 --- a/packages/run-engine/src/run-queue/index.test.ts +++ b/packages/run-engine/src/run-queue/index.test.ts @@ -11,7 +11,6 @@ import { abort } from "node:process"; const testOptions = { name: "rq", tracer: trace.getTracer("rq"), - keysProducer: new RunQueueShortKeyProducer("rq:"), queuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 36 }), envQueuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 12 }), workers: 1, @@ -162,6 +161,8 @@ describe("RunQueue", () => { const dequeued = await queue.dequeueMessageInEnv(authenticatedEnvDev); expect(dequeued?.messageId).toEqual(messageDev.runId); + //todo check all the currentConcurrency values + const dequeued2 = await queue.dequeueMessageInEnv(authenticatedEnvDev); expect(dequeued2).toBe(undefined); } finally { @@ -203,6 +204,8 @@ describe("RunQueue", () => { const dequeued = await queue.dequeueMessageInSharedQueue("test_12345"); expect(dequeued?.messageId).toEqual(messageProd.runId); + //todo check all the currentConcurrency values + const dequeued2 = await queue.dequeueMessageInEnv(authenticatedEnvDev); expect(dequeued2).toBe(undefined); } finally { diff --git a/packages/run-engine/src/run-queue/index.ts b/packages/run-engine/src/run-queue/index.ts index 48e7312083..d969c37fcf 100644 --- a/packages/run-engine/src/run-queue/index.ts +++ b/packages/run-engine/src/run-queue/index.ts @@ -16,6 +16,7 @@ import { RunQueueKeyProducer, RunQueuePriorityStrategy, } from "./types.js"; +import { RunQueueShortKeyProducer } from "./keyProducer.js"; const SemanticAttributes = { QUEUE: "runqueue.queue", @@ -32,7 +33,6 @@ export type RunQueueOptions = { defaultEnvConcurrency: number; windowSize?: number; workers: number; - keysProducer: RunQueueKeyProducer; queuePriorityStrategy: RunQueuePriorityStrategy; envQueuePriorityStrategy: RunQueuePriorityStrategy; enableRebalancing?: boolean; @@ -54,7 +54,7 @@ export class RunQueue { this.redis = new Redis(options.redis); this.logger = options.logger; - this.keys = options.keysProducer; + this.keys = new RunQueueShortKeyProducer("rq:", options.name); this.queuePriorityStrategy = options.queuePriorityStrategy; this.#startRebalanceWorkers(); @@ -933,6 +933,7 @@ export class RunQueue { return; } + //todo refactor to get the message at the same time, and do the task concurrency update const [messageId, messageScore] = result; //read message @@ -1152,8 +1153,6 @@ end -- Update the concurrency keys redis.call('SREM', concurrencyKey, messageId) redis.call('SREM', envCurrentConcurrencyKey, messageId) - --- Update concurrency tracking (remove) redis.call('SREM', taskConcurrencyTrackerKey, messageId) redis.call('SREM', envConcurrencyTrackerKey, messageId) `, From e69c127d8116843514d02b70476bcdb576bad078 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 27 Sep 2024 14:51:51 +0100 Subject: [PATCH 044/114] Extra info in the Prisma schema --- packages/database/prisma/schema.prisma | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/database/prisma/schema.prisma b/packages/database/prisma/schema.prisma index 0f146a15df..2556acfcce 100644 --- a/packages/database/prisma/schema.prisma +++ b/packages/database/prisma/schema.prisma @@ -1854,6 +1854,13 @@ model TaskRunExecutionSnapshot { /// These are only ever appended, so we don't need updatedAt createdAt DateTime @default(now()) + ///todo machine spec? + + ///worker + + /// For debugging + description String + /// Used to get the latest state quickly @@index([runId, createdAt(sort: Desc)]) } From 5de384faeb62054368eb094ac318536ad2e62cf9 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 27 Sep 2024 15:44:24 +0100 Subject: [PATCH 045/114] Dequeuing a message gets the payload and sets the task concurrency all in one Lua script --- .../run-engine/src/run-queue/index.test.ts | 2 + packages/run-engine/src/run-queue/index.ts | 62 +++++++++++++------ .../src/run-queue/keyProducer.test.ts | 19 +++--- .../run-engine/src/run-queue/keyProducer.ts | 39 +++++------- packages/run-engine/src/run-queue/types.ts | 14 +---- 5 files changed, 76 insertions(+), 60 deletions(-) diff --git a/packages/run-engine/src/run-queue/index.test.ts b/packages/run-engine/src/run-queue/index.test.ts index 21149241b6..9c230aba37 100644 --- a/packages/run-engine/src/run-queue/index.test.ts +++ b/packages/run-engine/src/run-queue/index.test.ts @@ -160,6 +160,7 @@ describe("RunQueue", () => { const dequeued = await queue.dequeueMessageInEnv(authenticatedEnvDev); expect(dequeued?.messageId).toEqual(messageDev.runId); + expect(dequeued?.message.orgId).toEqual(messageDev.orgId); //todo check all the currentConcurrency values @@ -203,6 +204,7 @@ describe("RunQueue", () => { const dequeued = await queue.dequeueMessageInSharedQueue("test_12345"); expect(dequeued?.messageId).toEqual(messageProd.runId); + expect(dequeued?.message.orgId).toEqual(messageProd.orgId); //todo check all the currentConcurrency values diff --git a/packages/run-engine/src/run-queue/index.ts b/packages/run-engine/src/run-queue/index.ts index d969c37fcf..18fca41543 100644 --- a/packages/run-engine/src/run-queue/index.ts +++ b/packages/run-engine/src/run-queue/index.ts @@ -213,6 +213,9 @@ export class RunQueue { currentConcurrencyKey: this.keys.currentConcurrencyKeyFromQueue(messageQueue), envConcurrencyLimitKey: this.keys.envConcurrencyLimitKeyFromQueue(messageQueue), envCurrentConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(messageQueue), + messageKeyPrefix: this.keys.messageKeyPrefixFromQueue(messageQueue), + taskCurrentConcurrentKeyPrefix: + this.keys.taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(messageQueue), }); if (!message) { @@ -300,6 +303,9 @@ export class RunQueue { currentConcurrencyKey: this.keys.currentConcurrencyKeyFromQueue(messageQueue), envConcurrencyLimitKey: this.keys.envConcurrencyLimitKeyFromQueue(messageQueue), envCurrentConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(messageQueue), + messageKeyPrefix: this.keys.messageKeyPrefixFromQueue(messageQueue), + taskCurrentConcurrentKeyPrefix: + this.keys.taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(messageQueue), }); if (!message) { @@ -857,12 +863,10 @@ export class RunQueue { async #callEnqueueMessage(message: MessagePayload, parentQueue: string) { const concurrencyKey = this.keys.currentConcurrencyKeyFromQueue(message.queue); const envConcurrencyKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); - const taskConcurrencyTrackerKey = this.keys.currentTaskIdentifierKey({ - orgId: message.orgId, - projectId: message.projectId, - environmentId: message.environmentId, - taskIdentifier: message.taskIdentifier, - }); + const taskConcurrencyTrackerKey = this.keys.taskIdentifierCurrentConcurrencyKeyFromQueue( + message.queue, + message.taskIdentifier + ); const envConcurrencyTrackerKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); this.logger.debug("Calling enqueueMessage", { @@ -894,6 +898,8 @@ export class RunQueue { envConcurrencyLimitKey, currentConcurrencyKey, envCurrentConcurrencyKey, + messageKeyPrefix, + taskCurrentConcurrentKeyPrefix, }: { messageQueue: string; parentQueue: string; @@ -901,6 +907,8 @@ export class RunQueue { envConcurrencyLimitKey: string; currentConcurrencyKey: string; envCurrentConcurrencyKey: string; + messageKeyPrefix: string; + taskCurrentConcurrentKeyPrefix: string; }) { const result = await this.redis.dequeueMessage( //keys @@ -910,6 +918,8 @@ export class RunQueue { envConcurrencyLimitKey, currentConcurrencyKey, envCurrentConcurrencyKey, + messageKeyPrefix, + taskCurrentConcurrentKeyPrefix, //args messageQueue, String(Date.now()), @@ -925,7 +935,7 @@ export class RunQueue { service: this.name, }); - if (result.length !== 2) { + if (result.length !== 3) { this.logger.error("Invalid dequeue message result", { result, service: this.name, @@ -933,24 +943,21 @@ export class RunQueue { return; } - //todo refactor to get the message at the same time, and do the task concurrency update - const [messageId, messageScore] = result; + const [messageId, messageScore, rawMessage] = result; //read message - const { orgId } = this.keys.extractComponentsFromQueue(messageQueue); - const message = await this.readMessage(orgId, messageId); - - if (!message) { - this.logger.error(`Dequeued then failed to read message. This is unrecoverable.`, { + const parsedMessage = MessagePayload.safeParse(JSON.parse(rawMessage)); + if (!parsedMessage.success) { + this.logger.error(`[${this.name}] Failed to parse message`, { messageId, - messageScore, + error: parsedMessage.error, service: this.name, }); + return; } - //update task concurrency - await this.redis.sadd(this.keys.currentTaskIdentifierKey(message), messageId); + const message = parsedMessage.data; return { messageId, @@ -1159,7 +1166,7 @@ redis.call('SREM', envConcurrencyTrackerKey, messageId) }); this.redis.defineCommand("dequeueMessage", { - numberOfKeys: 6, + numberOfKeys: 8, lua: ` local childQueue = KEYS[1] local parentQueue = KEYS[2] @@ -1167,6 +1174,8 @@ local concurrencyLimitKey = KEYS[3] local envConcurrencyLimitKey = KEYS[4] local currentConcurrencyKey = KEYS[5] local envCurrentConcurrencyKey = KEYS[6] +local messageKeyPrefix = KEYS[7] +local taskCurrentConcurrentKeyPrefix = KEYS[8] local childQueueName = ARGV[1] local currentTime = tonumber(ARGV[2]) @@ -1199,10 +1208,21 @@ end local messageId = messages[1] local messageScore = tonumber(messages[2]) +-- Get the message payload +local messageKey = messageKeyPrefix .. messageId +local messagePayload = redis.call('GET', messageKey) + +-- Parse JSON payload and extract taskIdentifier +local taskIdentifier = cjson.decode(messagePayload).taskIdentifier + +-- Perform SADD with taskIdentifier and messageId +local taskConcurrencyKey = taskCurrentConcurrentKeyPrefix .. taskIdentifier + -- Update concurrency redis.call('ZREM', childQueue, messageId) redis.call('SADD', currentConcurrencyKey, messageId) redis.call('SADD', envCurrentConcurrencyKey, messageId) +redis.call('SADD', taskConcurrencyKey, messageId) -- todo add this in here -- redis.call('SADD', taskConcurrencyKey, messageId) @@ -1215,7 +1235,7 @@ else redis.call('ZADD', parentQueue, earliestMessage[2], childQueueName) end -return {messageId, messageScore} -- Return message details +return {messageId, messageScore, messagePayload} -- Return message details `, }); @@ -1449,12 +1469,14 @@ declare module "ioredis" { envConcurrencyLimitKey: string, currentConcurrencyKey: string, envCurrentConcurrencyKey: string, + messageKeyPrefix: string, + taskCurrentConcurrentKeyPrefix: string, //args childQueueName: string, currentTime: string, defaultEnvConcurrencyLimit: string, callback?: Callback<[string, string]> - ): Result<[string, string] | null, Context>; + ): Result<[string, string, string] | null, Context>; acknowledgeMessage( parentQueue: string, diff --git a/packages/run-engine/src/run-queue/keyProducer.test.ts b/packages/run-engine/src/run-queue/keyProducer.test.ts index 71d193abff..c99840f92b 100644 --- a/packages/run-engine/src/run-queue/keyProducer.test.ts +++ b/packages/run-engine/src/run-queue/keyProducer.test.ts @@ -215,13 +215,18 @@ describe("KeyProducer", () => { it("currentTaskIdentifierKey", () => { const keyProducer = new RunQueueShortKeyProducer("test:", "main"); - const key = keyProducer.currentTaskIdentifierKey({ - orgId: "o1234", - projectId: "p1234", - taskIdentifier: "task/task-name", - environmentId: "e1234", - }); - expect(key).toBe("{org:o1234}:proj:p1234:task:task/task-name:env:e1234:currentConcurrency"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + const key = keyProducer.taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(queueKey); + expect(key).toBe("{org:o1234}:proj:p1234:task:"); }); it("disabledConcurrencyLimitKeyFromQueue", () => { diff --git a/packages/run-engine/src/run-queue/keyProducer.ts b/packages/run-engine/src/run-queue/keyProducer.ts index e7311a77ec..5e52792e77 100644 --- a/packages/run-engine/src/run-queue/keyProducer.ts +++ b/packages/run-engine/src/run-queue/keyProducer.ts @@ -101,28 +101,6 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { ); } - currentTaskIdentifierKey({ - orgId, - projectId, - taskIdentifier, - environmentId, - }: { - orgId: string; - projectId: string; - taskIdentifier: string; - environmentId: string; - }) { - return [ - this.orgKeySection(orgId), - this.projKeySection(projectId), - this.taskIdentifierSection(taskIdentifier), - this.envKeySection(environmentId), - constants.CURRENT_CONCURRENCY_PART, - ] - .filter(Boolean) - .join(":"); - } - disabledConcurrencyLimitKeyFromQueue(queue: string) { const { orgId } = this.extractComponentsFromQueue(queue); return `{${constants.ORG_PART}:${orgId}}:${constants.DISABLED_CONCURRENCY_LIMIT_PART}`; @@ -146,6 +124,23 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { ].join(":"); } + taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(queue: string) { + const { orgId, projectId } = this.extractComponentsFromQueue(queue); + + return `${[this.orgKeySection(orgId), this.projKeySection(projectId), constants.TASK_PART] + .filter(Boolean) + .join(":")}:`; + } + + taskIdentifierCurrentConcurrencyKeyFromQueue(queue: string, taskIdentifier: string) { + return `${this.taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(queue)}:${taskIdentifier}`; + } + + messageKeyPrefixFromQueue(queue: string) { + const { orgId } = this.extractComponentsFromQueue(queue); + return `${this.orgKeySection(orgId)}:${constants.MESSAGE_PART}:`; + } + messageKey(orgId: string, messageId: string) { return [this.orgKeySection(orgId), `${constants.MESSAGE_PART}:${messageId}`] .filter(Boolean) diff --git a/packages/run-engine/src/run-queue/types.ts b/packages/run-engine/src/run-queue/types.ts index 4c30880ec1..8f98c17530 100644 --- a/packages/run-engine/src/run-queue/types.ts +++ b/packages/run-engine/src/run-queue/types.ts @@ -52,21 +52,13 @@ export interface RunQueueKeyProducer { queue: string, concurrencyKey?: string ): string; - currentTaskIdentifierKey({ - orgId, - projectId, - taskIdentifier, - environmentId, - }: { - orgId: string; - projectId: string; - taskIdentifier: string; - environmentId: string; - }): string; + taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(queue: string): string; + taskIdentifierCurrentConcurrencyKeyFromQueue(queue: string, taskIdentifier: string): string; disabledConcurrencyLimitKeyFromQueue(queue: string): string; envConcurrencyLimitKeyFromQueue(queue: string): string; envCurrentConcurrencyKeyFromQueue(queue: string): string; envCurrentConcurrencyKey(env: AuthenticatedEnvironment): string; + messageKeyPrefixFromQueue(queue: string): string; messageKey(orgId: string, messageId: string): string; stripKeyPrefix(key: string): string; extractComponentsFromQueue(queue: string): { From dc2d89e98ec7a0ae46ad30f1a37f3ca53d0ac328 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 27 Sep 2024 15:57:11 +0100 Subject: [PATCH 046/114] Adding more keys so we can read the concurrency from the queue --- .../src/run-queue/keyProducer.test.ts | 53 ++++++++++++++++++- .../run-engine/src/run-queue/keyProducer.ts | 29 +++++++++- packages/run-engine/src/run-queue/types.ts | 24 ++++++--- 3 files changed, 98 insertions(+), 8 deletions(-) diff --git a/packages/run-engine/src/run-queue/keyProducer.test.ts b/packages/run-engine/src/run-queue/keyProducer.test.ts index c99840f92b..5780e4d91b 100644 --- a/packages/run-engine/src/run-queue/keyProducer.test.ts +++ b/packages/run-engine/src/run-queue/keyProducer.test.ts @@ -213,7 +213,7 @@ describe("KeyProducer", () => { expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:currentConcurrency"); }); - it("currentTaskIdentifierKey", () => { + it("taskIdentifierCurrentConcurrencyKeyPrefixFromQueue", () => { const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const queueKey = keyProducer.queueKey( { @@ -229,6 +229,57 @@ describe("KeyProducer", () => { expect(key).toBe("{org:o1234}:proj:p1234:task:"); }); + it("taskIdentifierCurrentConcurrencyKeyFromQueue", () => { + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const queueKey = keyProducer.queueKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task/task-name" + ); + const key = keyProducer.taskIdentifierCurrentConcurrencyKeyFromQueue(queueKey, "task-name"); + expect(key).toBe("{org:o1234}:proj:p1234:task:task-name"); + }); + + it("taskIdentifierCurrentConcurrencyKey", () => { + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const key = keyProducer.taskIdentifierCurrentConcurrencyKey( + { + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }, + "task-name" + ); + expect(key).toBe("{org:o1234}:proj:p1234:task:task-name"); + }); + + it("projectCurrentConcurrencyKey", () => { + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const key = keyProducer.projectCurrentConcurrencyKey({ + id: "e1234", + type: "PRODUCTION", + maximumConcurrencyLimit: 10, + project: { id: "p1234" }, + organization: { id: "o1234" }, + }); + expect(key).toBe("{org:o1234}:proj:p1234:currentConcurrency"); + }); + + it("projectCurrentConcurrencyKeyFromQueue", () => { + const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const key = keyProducer.projectCurrentConcurrencyKeyFromQueue( + "{org:o1234}:proj:p1234:currentConcurrency" + ); + expect(key).toBe("{org:o1234}:proj:p1234:currentConcurrency"); + }); + it("disabledConcurrencyLimitKeyFromQueue", () => { const keyProducer = new RunQueueShortKeyProducer("test:", "main"); const queueKey = keyProducer.queueKey( diff --git a/packages/run-engine/src/run-queue/keyProducer.ts b/packages/run-engine/src/run-queue/keyProducer.ts index 5e52792e77..becea0a38e 100644 --- a/packages/run-engine/src/run-queue/keyProducer.ts +++ b/packages/run-engine/src/run-queue/keyProducer.ts @@ -133,7 +133,34 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { } taskIdentifierCurrentConcurrencyKeyFromQueue(queue: string, taskIdentifier: string) { - return `${this.taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(queue)}:${taskIdentifier}`; + return `${this.taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(queue)}${taskIdentifier}`; + } + + taskIdentifierCurrentConcurrencyKey( + env: AuthenticatedEnvironment, + taskIdentifier: string + ): string { + return [ + this.orgKeySection(env.organization.id), + this.projKeySection(env.project.id), + constants.TASK_PART, + taskIdentifier, + ].join(":"); + } + + projectCurrentConcurrencyKey(env: AuthenticatedEnvironment): string { + return [ + this.orgKeySection(env.organization.id), + this.projKeySection(env.project.id), + constants.CURRENT_CONCURRENCY_PART, + ].join(":"); + } + + projectCurrentConcurrencyKeyFromQueue(queue: string): string { + const { orgId, projectId } = this.extractComponentsFromQueue(queue); + return `${this.orgKeySection(orgId)}:${this.projKeySection(projectId)}:${ + constants.CURRENT_CONCURRENCY_PART + }`; } messageKeyPrefixFromQueue(queue: string) { diff --git a/packages/run-engine/src/run-queue/types.ts b/packages/run-engine/src/run-queue/types.ts index 8f98c17530..fc742ff9d9 100644 --- a/packages/run-engine/src/run-queue/types.ts +++ b/packages/run-engine/src/run-queue/types.ts @@ -38,13 +38,13 @@ export type QueueWithScores = { export type QueueRange = { offset: number; count: number }; export interface RunQueueKeyProducer { - queueConcurrencyLimitKey(env: AuthenticatedEnvironment, queue: string): string; - envConcurrencyLimitKey(env: AuthenticatedEnvironment): string; - queueKey(env: AuthenticatedEnvironment, queue: string, concurrencyKey?: string): string; envSharedQueueKey(env: AuthenticatedEnvironment): string; sharedQueueKey(): string; sharedQueueScanPattern(): string; queueCurrentConcurrencyScanPattern(): string; + //queue + queueKey(env: AuthenticatedEnvironment, queue: string, concurrencyKey?: string): string; + queueConcurrencyLimitKey(env: AuthenticatedEnvironment, queue: string): string; concurrencyLimitKeyFromQueue(queue: string): string; currentConcurrencyKeyFromQueue(queue: string): string; currentConcurrencyKey( @@ -52,14 +52,26 @@ export interface RunQueueKeyProducer { queue: string, concurrencyKey?: string ): string; - taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(queue: string): string; - taskIdentifierCurrentConcurrencyKeyFromQueue(queue: string, taskIdentifier: string): string; disabledConcurrencyLimitKeyFromQueue(queue: string): string; + //env oncurrency + envCurrentConcurrencyKey(env: AuthenticatedEnvironment): string; + envConcurrencyLimitKey(env: AuthenticatedEnvironment): string; envConcurrencyLimitKeyFromQueue(queue: string): string; envCurrentConcurrencyKeyFromQueue(queue: string): string; - envCurrentConcurrencyKey(env: AuthenticatedEnvironment): string; + //task concurrency + taskIdentifierCurrentConcurrencyKey( + env: AuthenticatedEnvironment, + taskIdentifier: string + ): string; + taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(queue: string): string; + taskIdentifierCurrentConcurrencyKeyFromQueue(queue: string, taskIdentifier: string): string; + //project concurrency + projectCurrentConcurrencyKey(env: AuthenticatedEnvironment): string; + projectCurrentConcurrencyKeyFromQueue(queue: string): string; + //message payload messageKeyPrefixFromQueue(queue: string): string; messageKey(orgId: string, messageId: string): string; + //utils stripKeyPrefix(key: string): string; extractComponentsFromQueue(queue: string): { orgId: string; From 8af3003c2328dc614809a6d3cfabeaa3c22e04c5 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 27 Sep 2024 16:12:31 +0100 Subject: [PATCH 047/114] Setting the concurrency with dequeue and enquque is working --- .../run-engine/src/run-queue/index.test.ts | 37 +++- packages/run-engine/src/run-queue/index.ts | 172 ++++++++---------- 2 files changed, 112 insertions(+), 97 deletions(-) diff --git a/packages/run-engine/src/run-queue/index.test.ts b/packages/run-engine/src/run-queue/index.test.ts index 9c230aba37..4cb9ee27d1 100644 --- a/packages/run-engine/src/run-queue/index.test.ts +++ b/packages/run-engine/src/run-queue/index.test.ts @@ -141,28 +141,63 @@ describe("RunQueue", () => { }); try { + //initial queue length const result = await queue.lengthOfQueue(authenticatedEnvDev, messageDev.queue); expect(result).toBe(0); + //initial oldest message const oldestScore = await queue.oldestMessageInQueue(authenticatedEnvDev, messageDev.queue); expect(oldestScore).toBe(undefined); + //enqueue message await queue.enqueueMessage({ env: authenticatedEnvDev, message: messageDev }); + //queue length const result2 = await queue.lengthOfQueue(authenticatedEnvDev, messageDev.queue); expect(result2).toBe(1); + //oldest message const oldestScore2 = await queue.oldestMessageInQueue( authenticatedEnvDev, messageDev.queue ); expect(oldestScore2).toBe(messageDev.timestamp); + //concurrencies + const queueConcurrency = await queue.currentConcurrencyOfQueue( + authenticatedEnvDev, + messageDev.queue + ); + expect(queueConcurrency).toBe(0); + const envConcurrency = await queue.currentConcurrencyOfEnvironment(authenticatedEnvDev); + expect(envConcurrency).toBe(0); + const projectConcurrency = await queue.currentConcurrencyOfProject(authenticatedEnvDev); + expect(projectConcurrency).toBe(0); + const taskConcurrency = await queue.currentConcurrencyOfTask( + authenticatedEnvDev, + messageDev.taskIdentifier + ); + expect(taskConcurrency).toBe(0); + const dequeued = await queue.dequeueMessageInEnv(authenticatedEnvDev); expect(dequeued?.messageId).toEqual(messageDev.runId); expect(dequeued?.message.orgId).toEqual(messageDev.orgId); - //todo check all the currentConcurrency values + //concurrencies + const queueConcurrency2 = await queue.currentConcurrencyOfQueue( + authenticatedEnvDev, + messageDev.queue + ); + expect(queueConcurrency2).toBe(1); + const envConcurrency2 = await queue.currentConcurrencyOfEnvironment(authenticatedEnvDev); + expect(envConcurrency2).toBe(1); + const projectConcurrency2 = await queue.currentConcurrencyOfProject(authenticatedEnvDev); + expect(projectConcurrency2).toBe(1); + const taskConcurrency2 = await queue.currentConcurrencyOfTask( + authenticatedEnvDev, + messageDev.taskIdentifier + ); + expect(taskConcurrency2).toBe(1); const dequeued2 = await queue.dequeueMessageInEnv(authenticatedEnvDev); expect(dequeued2).toBe(undefined); diff --git a/packages/run-engine/src/run-queue/index.ts b/packages/run-engine/src/run-queue/index.ts index 18fca41543..fb322e68a8 100644 --- a/packages/run-engine/src/run-queue/index.ts +++ b/packages/run-engine/src/run-queue/index.ts @@ -140,6 +140,14 @@ export class RunQueue { return this.redis.scard(this.keys.envCurrentConcurrencyKey(env)); } + public async currentConcurrencyOfProject(env: AuthenticatedEnvironment) { + return this.redis.scard(this.keys.projectCurrentConcurrencyKey(env)); + } + + public async currentConcurrencyOfTask(env: AuthenticatedEnvironment, taskIdentifier: string) { + return this.redis.scard(this.keys.taskIdentifierCurrentConcurrencyKey(env, taskIdentifier)); + } + public async enqueueMessage({ env, message, @@ -199,7 +207,7 @@ export class RunQueue { ); if (!messageQueue) { - this.logger.log("No message queue found", { + this.logger.debug("No message queue found", { parentQueue, }); @@ -213,6 +221,8 @@ export class RunQueue { currentConcurrencyKey: this.keys.currentConcurrencyKeyFromQueue(messageQueue), envConcurrencyLimitKey: this.keys.envConcurrencyLimitKeyFromQueue(messageQueue), envCurrentConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(messageQueue), + projectCurrentConcurrencyKey: + this.keys.projectCurrentConcurrencyKeyFromQueue(messageQueue), messageKeyPrefix: this.keys.messageKeyPrefixFromQueue(messageQueue), taskCurrentConcurrentKeyPrefix: this.keys.taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(messageQueue), @@ -303,6 +313,8 @@ export class RunQueue { currentConcurrencyKey: this.keys.currentConcurrencyKeyFromQueue(messageQueue), envConcurrencyLimitKey: this.keys.envConcurrencyLimitKeyFromQueue(messageQueue), envCurrentConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(messageQueue), + projectCurrentConcurrencyKey: + this.keys.projectCurrentConcurrencyKeyFromQueue(messageQueue), messageKeyPrefix: this.keys.messageKeyPrefixFromQueue(messageQueue), taskCurrentConcurrentKeyPrefix: this.keys.taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(messageQueue), @@ -478,6 +490,47 @@ export class RunQueue { ); } + queueConcurrencyScanStream( + count: number = 100, + onEndCallback?: () => void, + onErrorCallback?: (error: Error) => void + ) { + const pattern = this.keys.queueCurrentConcurrencyScanPattern(); + + this.logger.debug("Starting queue concurrency scan stream", { + pattern, + component: "runqueue", + operation: "queueConcurrencyScanStream", + service: this.name, + count, + }); + + const redis = this.redis.duplicate(); + + const stream = redis.scanStream({ + match: pattern, + type: "set", + count, + }); + + stream.on("end", () => { + onEndCallback?.(); + redis.quit(); + }); + + stream.on("error", (error) => { + onErrorCallback?.(error); + redis.quit(); + }); + + return { stream, redis }; + } + + async quit() { + await Promise.all(this.#rebalanceWorkers.map((worker) => worker.stop())); + await this.redis.quit(); + } + async #trace( name: string, fn: (span: Span) => Promise, @@ -509,46 +562,6 @@ export class RunQueue { ); } - async readMessage(orgId: string, messageId: string) { - return this.#trace( - "readMessage", - async (span) => { - const rawMessage = await this.redis.get(this.keys.messageKey(orgId, messageId)); - - if (!rawMessage) { - return; - } - - const message = MessagePayload.safeParse(JSON.parse(rawMessage)); - - if (!message.success) { - this.logger.error(`[${this.name}] Failed to parse message`, { - messageId, - error: message.error, - service: this.name, - }); - - return; - } - - return message.data; - }, - { - attributes: { - [SEMATTRS_MESSAGING_OPERATION]: "receive", - [SEMATTRS_MESSAGE_ID]: messageId, - [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", - [SemanticAttributes.RUN_ID]: messageId, - }, - } - ); - } - - async quit() { - await Promise.all(this.#rebalanceWorkers.map((worker) => worker.stop())); - await this.redis.quit(); - } - async #getRandomQueueFromParentQueue( parentQueue: string, queuePriorityStrategy: RunQueuePriorityStrategy, @@ -717,42 +730,6 @@ export class RunQueue { } } - queueConcurrencyScanStream( - count: number = 100, - onEndCallback?: () => void, - onErrorCallback?: (error: Error) => void - ) { - const pattern = this.keys.queueCurrentConcurrencyScanPattern(); - - this.logger.debug("Starting queue concurrency scan stream", { - pattern, - component: "runqueue", - operation: "queueConcurrencyScanStream", - service: this.name, - count, - }); - - const redis = this.redis.duplicate(); - - const stream = redis.scanStream({ - match: pattern, - type: "set", - count, - }); - - stream.on("end", () => { - onEndCallback?.(); - redis.quit(); - }); - - stream.on("error", (error) => { - onErrorCallback?.(error); - redis.quit(); - }); - - return { stream, redis }; - } - async #rebalanceParentQueues() { return await new Promise((resolve, reject) => { // Scan for sorted sets with the parent queue pattern @@ -863,11 +840,11 @@ export class RunQueue { async #callEnqueueMessage(message: MessagePayload, parentQueue: string) { const concurrencyKey = this.keys.currentConcurrencyKeyFromQueue(message.queue); const envConcurrencyKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); - const taskConcurrencyTrackerKey = this.keys.taskIdentifierCurrentConcurrencyKeyFromQueue( + const taskConcurrencyKey = this.keys.taskIdentifierCurrentConcurrencyKeyFromQueue( message.queue, message.taskIdentifier ); - const envConcurrencyTrackerKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); + const projectConcurrencyKey = this.keys.projectCurrentConcurrencyKeyFromQueue(message.queue); this.logger.debug("Calling enqueueMessage", { messagePayload: message, @@ -882,8 +859,8 @@ export class RunQueue { this.keys.messageKey(message.orgId, message.runId), concurrencyKey, envConcurrencyKey, - taskConcurrencyTrackerKey, - envConcurrencyTrackerKey, + taskConcurrencyKey, + projectConcurrencyKey, message.queue, message.runId, JSON.stringify(message), @@ -898,6 +875,7 @@ export class RunQueue { envConcurrencyLimitKey, currentConcurrencyKey, envCurrentConcurrencyKey, + projectCurrentConcurrencyKey, messageKeyPrefix, taskCurrentConcurrentKeyPrefix, }: { @@ -907,6 +885,7 @@ export class RunQueue { envConcurrencyLimitKey: string; currentConcurrencyKey: string; envCurrentConcurrencyKey: string; + projectCurrentConcurrencyKey: string; messageKeyPrefix: string; taskCurrentConcurrentKeyPrefix: string; }) { @@ -918,6 +897,7 @@ export class RunQueue { envConcurrencyLimitKey, currentConcurrencyKey, envCurrentConcurrencyKey, + projectCurrentConcurrencyKey, messageKeyPrefix, taskCurrentConcurrentKeyPrefix, //args @@ -1134,9 +1114,9 @@ local queue = KEYS[1] local parentQueue = KEYS[2] local messageKey = KEYS[3] local concurrencyKey = KEYS[4] -local envCurrentConcurrencyKey = KEYS[5] -local taskConcurrencyTrackerKey = KEYS[6] -local envConcurrencyTrackerKey = KEYS[7] +local envConcurrencyKey = KEYS[5] +local taskConcurrencyKey = KEYS[6] +local projectConcurrencyKey = KEYS[7] local queueName = ARGV[1] local messageId = ARGV[2] @@ -1159,14 +1139,14 @@ end -- Update the concurrency keys redis.call('SREM', concurrencyKey, messageId) -redis.call('SREM', envCurrentConcurrencyKey, messageId) -redis.call('SREM', taskConcurrencyTrackerKey, messageId) -redis.call('SREM', envConcurrencyTrackerKey, messageId) +redis.call('SREM', envConcurrencyKey, messageId) +redis.call('SREM', taskConcurrencyKey, messageId) +redis.call('SREM', projectConcurrencyKey, messageId) `, }); this.redis.defineCommand("dequeueMessage", { - numberOfKeys: 8, + numberOfKeys: 9, lua: ` local childQueue = KEYS[1] local parentQueue = KEYS[2] @@ -1174,8 +1154,9 @@ local concurrencyLimitKey = KEYS[3] local envConcurrencyLimitKey = KEYS[4] local currentConcurrencyKey = KEYS[5] local envCurrentConcurrencyKey = KEYS[6] -local messageKeyPrefix = KEYS[7] -local taskCurrentConcurrentKeyPrefix = KEYS[8] +local projectConcurrencyKey = KEYS[7] +local messageKeyPrefix = KEYS[8] +local taskCurrentConcurrentKeyPrefix = KEYS[9] local childQueueName = ARGV[1] local currentTime = tonumber(ARGV[2]) @@ -1222,11 +1203,9 @@ local taskConcurrencyKey = taskCurrentConcurrentKeyPrefix .. taskIdentifier redis.call('ZREM', childQueue, messageId) redis.call('SADD', currentConcurrencyKey, messageId) redis.call('SADD', envCurrentConcurrencyKey, messageId) +redis.call('SADD', projectConcurrencyKey, messageId) redis.call('SADD', taskConcurrencyKey, messageId) --- todo add this in here --- redis.call('SADD', taskConcurrencyKey, messageId) - -- Rebalance the parent queue local earliestMessage = redis.call('ZRANGE', childQueue, 0, 0, 'WITHSCORES') if #earliestMessage == 0 then @@ -1451,8 +1430,8 @@ declare module "ioredis" { messageKey: string, concurrencyKey: string, envConcurrencyKey: string, - taskConcurrencyTrackerKey: string, - environmentConcurrencyTrackerKey: string, + taskConcurrencyKey: string, + projectConcurrencyKey: string, //args queueName: string, messageId: string, @@ -1468,7 +1447,8 @@ declare module "ioredis" { concurrencyLimitKey: string, envConcurrencyLimitKey: string, currentConcurrencyKey: string, - envCurrentConcurrencyKey: string, + envConcurrencyKey: string, + projectConcurrencyKey: string, messageKeyPrefix: string, taskCurrentConcurrentKeyPrefix: string, //args From d2ed8cebd8491adf1196850cd362e39684e87e58 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 27 Sep 2024 17:23:39 +0100 Subject: [PATCH 048/114] Improved the tests and fixed some bugs --- .../run-engine/src/run-queue/index.test.ts | 52 +++++++- packages/run-engine/src/run-queue/index.ts | 125 ++++++++++++++---- packages/run-engine/src/run-queue/types.ts | 13 +- 3 files changed, 154 insertions(+), 36 deletions(-) diff --git a/packages/run-engine/src/run-queue/index.test.ts b/packages/run-engine/src/run-queue/index.test.ts index 4cb9ee27d1..60932f0fd6 100644 --- a/packages/run-engine/src/run-queue/index.test.ts +++ b/packages/run-engine/src/run-queue/index.test.ts @@ -5,7 +5,7 @@ import { redisTest } from "../test/containerTest.js"; import { RunQueue } from "./index.js"; import { RunQueueShortKeyProducer } from "./keyProducer.js"; import { SimpleWeightedChoiceStrategy } from "./simpleWeightedPriorityStrategy.js"; -import { MessagePayload } from "./types.js"; +import { InputPayload } from "./types.js"; import { abort } from "node:process"; const testOptions = { @@ -35,8 +35,7 @@ const authenticatedEnvDev = { organization: { id: "o1234" }, }; -const messageProd: MessagePayload = { - version: "1" as const, +const messageProd: InputPayload = { runId: "r1234", taskIdentifier: "task/my-task", orgId: "o1234", @@ -47,8 +46,7 @@ const messageProd: MessagePayload = { timestamp: Date.now(), }; -const messageDev: MessagePayload = { - version: "1" as const, +const messageDev: InputPayload = { runId: "r4321", taskIdentifier: "task/my-task", orgId: "o1234", @@ -182,6 +180,10 @@ describe("RunQueue", () => { const dequeued = await queue.dequeueMessageInEnv(authenticatedEnvDev); expect(dequeued?.messageId).toEqual(messageDev.runId); expect(dequeued?.message.orgId).toEqual(messageDev.orgId); + expect(dequeued?.message.version).toEqual("1"); + expect(dequeued?.message.parentQueue).toEqual( + "{org:o1234}:proj:p1234:env:e1234:sharedQueue:rq" + ); //concurrencies const queueConcurrency2 = await queue.currentConcurrencyOfQueue( @@ -217,31 +219,69 @@ describe("RunQueue", () => { }); try { + //initial queue length const result = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); expect(result).toBe(0); + //initial oldest message const oldestScore = await queue.oldestMessageInQueue( authenticatedEnvProd, messageProd.queue ); expect(oldestScore).toBe(undefined); + //enqueue message await queue.enqueueMessage({ env: authenticatedEnvProd, message: messageProd }); + //queue length const result2 = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); expect(result2).toBe(1); + //oldest message const oldestScore2 = await queue.oldestMessageInQueue( authenticatedEnvProd, messageProd.queue ); expect(oldestScore2).toBe(messageProd.timestamp); + //concurrencies + const queueConcurrency = await queue.currentConcurrencyOfQueue( + authenticatedEnvProd, + messageProd.queue + ); + expect(queueConcurrency).toBe(0); + const envConcurrency = await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd); + expect(envConcurrency).toBe(0); + const projectConcurrency = await queue.currentConcurrencyOfProject(authenticatedEnvProd); + expect(projectConcurrency).toBe(0); + const taskConcurrency = await queue.currentConcurrencyOfTask( + authenticatedEnvProd, + messageProd.taskIdentifier + ); + expect(taskConcurrency).toBe(0); + + //dequeue const dequeued = await queue.dequeueMessageInSharedQueue("test_12345"); expect(dequeued?.messageId).toEqual(messageProd.runId); expect(dequeued?.message.orgId).toEqual(messageProd.orgId); + expect(dequeued?.message.version).toEqual("1"); + expect(dequeued?.message.parentQueue).toEqual("sharedQueue:rq"); - //todo check all the currentConcurrency values + //concurrencies + const queueConcurrency2 = await queue.currentConcurrencyOfQueue( + authenticatedEnvProd, + messageProd.queue + ); + expect(queueConcurrency2).toBe(1); + const envConcurrency2 = await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd); + expect(envConcurrency2).toBe(1); + const projectConcurrency2 = await queue.currentConcurrencyOfProject(authenticatedEnvProd); + expect(projectConcurrency2).toBe(1); + const taskConcurrency2 = await queue.currentConcurrencyOfTask( + authenticatedEnvProd, + messageProd.taskIdentifier + ); + expect(taskConcurrency2).toBe(1); const dequeued2 = await queue.dequeueMessageInEnv(authenticatedEnvDev); expect(dequeued2).toBe(undefined); diff --git a/packages/run-engine/src/run-queue/index.ts b/packages/run-engine/src/run-queue/index.ts index fb322e68a8..a3569fbaac 100644 --- a/packages/run-engine/src/run-queue/index.ts +++ b/packages/run-engine/src/run-queue/index.ts @@ -10,7 +10,8 @@ import { Redis, type Callback, type RedisOptions, type Result } from "ioredis"; import { AsyncWorker } from "../shared/asyncWorker.js"; import { attributesFromAuthenticatedEnv, AuthenticatedEnvironment } from "../shared/index.js"; import { - MessagePayload, + InputPayload, + OutputPayload, QueueCapacities, QueueRange, RunQueueKeyProducer, @@ -153,7 +154,7 @@ export class RunQueue { message, }: { env: AuthenticatedEnvironment; - message: MessagePayload; + message: InputPayload; }) { return await this.#trace( "enqueueMessage", @@ -173,9 +174,11 @@ export class RunQueue { [SemanticAttributes.PARENT_QUEUE]: parentQueue, }); - const messagePayload: MessagePayload = { + const messagePayload: OutputPayload = { ...message, + version: "1", queue, + parentQueue, }; await this.#callEnqueueMessage(messagePayload, parentQueue); @@ -349,27 +352,42 @@ export class RunQueue { * - remove all data from the queue * - release all concurrency * This is done when the run is in a final state. - * @param orgId * @param messageId */ public async acknowledgeMessage(orgId: string, messageId: string) { return this.#trace( "acknowledgeMessage", async (span) => { - // span.setAttributes({ - // [SemanticAttributes.RUN_ID]: messageId, - // [SemanticAttributes.ORG_ID]: orgId, - // }); - // const message = await this.#callAcknowledgeMessage({ + const message = await this.#readMessage(orgId, messageId); + + if (!message) { + this.logger.log(`[${this.name}].acknowledgeMessage() message not found`, { + messageId, + service: this.name, + }); + return; + } + + span.setAttributes({ + [SemanticAttributes.QUEUE]: message.queue, + [SemanticAttributes.ORG_ID]: message.orgId, + [SemanticAttributes.RUN_ID]: messageId, + [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, + }); + + // await this.#callAcknowledgeMessage({ + // parentQueue: message.parentQueue, // messageKey: this.keys.messageKey(orgId, messageId), + // messageQueue: message.queue, + // concurrencyKey: this.keys.currentConcurrencyKeyFromQueue(message.queue), + // envConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(message.queue), + // projectConcurrencyKey: this.keys.projectCurrentConcurrencyKeyFromQueue(message.queue), + // taskConcurrencyKey: this.keys.taskIdentifierCurrentConcurrencyKeyFromQueue( + // message.queue, + // message.taskIdentifier + // ), // messageId, // }); - // span.setAttributes({ - // [SemanticAttributes.RUN_ID]: messageId, - // [SemanticAttributes.QUEUE]: message.queue, - // [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, - // [SemanticAttributes.PARENT_QUEUE]: message.parentQueue, - // }); }, { kind: SpanKind.CONSUMER, @@ -562,6 +580,41 @@ export class RunQueue { ); } + async #readMessage(orgId: string, messageId: string) { + return this.#trace( + "readMessage", + async (span) => { + const rawMessage = await this.redis.get(this.keys.messageKey(orgId, messageId)); + + if (!rawMessage) { + return; + } + + const message = OutputPayload.safeParse(JSON.parse(rawMessage)); + + if (!message.success) { + this.logger.error(`[${this.name}] Failed to parse message`, { + messageId, + error: message.error, + service: this.name, + }); + + return; + } + + return message.data; + }, + { + attributes: { + [SEMATTRS_MESSAGING_OPERATION]: "receive", + [SEMATTRS_MESSAGE_ID]: messageId, + [SEMATTRS_MESSAGING_SYSTEM]: "marqs", + [SemanticAttributes.RUN_ID]: messageId, + }, + } + ); + } + async #getRandomQueueFromParentQueue( parentQueue: string, queuePriorityStrategy: RunQueuePriorityStrategy, @@ -837,7 +890,7 @@ export class RunQueue { }); } - async #callEnqueueMessage(message: MessagePayload, parentQueue: string) { + async #callEnqueueMessage(message: OutputPayload, parentQueue: string) { const concurrencyKey = this.keys.currentConcurrencyKeyFromQueue(message.queue); const envConcurrencyKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); const taskConcurrencyKey = this.keys.taskIdentifierCurrentConcurrencyKeyFromQueue( @@ -926,7 +979,7 @@ export class RunQueue { const [messageId, messageScore, rawMessage] = result; //read message - const parsedMessage = MessagePayload.safeParse(JSON.parse(rawMessage)); + const parsedMessage = OutputPayload.safeParse(JSON.parse(rawMessage)); if (!parsedMessage.success) { this.logger.error(`[${this.name}] Failed to parse message`, { messageId, @@ -947,27 +1000,47 @@ export class RunQueue { } async #callAcknowledgeMessage({ + parentQueue, messageKey, + messageQueue, + visibilityQueue, + concurrencyKey, + envConcurrencyKey, + orgConcurrencyKey, messageId, }: { + parentQueue: string; messageKey: string; + messageQueue: string; + visibilityQueue: string; + concurrencyKey: string; + envConcurrencyKey: string; + orgConcurrencyKey: string; messageId: string; }) { this.logger.debug("Calling acknowledgeMessage", { messageKey, + messageQueue, + visibilityQueue, + concurrencyKey, + envConcurrencyKey, + orgConcurrencyKey, messageId, + parentQueue, service: this.name, }); - // return this.redis.acknowledgeMessage( - // parentQueue, - // messageKey, - // messageQueue, - // concurrencyKey, - // envConcurrencyKey, - // messageId, - // messageQueue - // ); + return this.redis.acknowledgeMessage( + parentQueue, + messageKey, + messageQueue, + visibilityQueue, + concurrencyKey, + envConcurrencyKey, + orgConcurrencyKey, + messageId, + messageQueue + ); } async #callNackMessage({ diff --git a/packages/run-engine/src/run-queue/types.ts b/packages/run-engine/src/run-queue/types.ts index fc742ff9d9..74e8d48451 100644 --- a/packages/run-engine/src/run-queue/types.ts +++ b/packages/run-engine/src/run-queue/types.ts @@ -2,9 +2,9 @@ import { z } from "zod"; import { AuthenticatedEnvironment } from "../shared/index.js"; import { RuntimeEnvironmentType } from "@trigger.dev/database"; import { env } from "process"; +import { version } from "os"; -export const MessagePayload = z.object({ - version: z.literal("1"), +export const InputPayload = z.object({ runId: z.string(), taskIdentifier: z.string(), orgId: z.string(), @@ -12,11 +12,16 @@ export const MessagePayload = z.object({ environmentId: z.string(), environmentType: z.nativeEnum(RuntimeEnvironmentType), queue: z.string(), - timestamp: z.number(), concurrencyKey: z.string().optional(), + timestamp: z.number(), }); +export type InputPayload = z.infer; -export type MessagePayload = z.infer; +export const OutputPayload = InputPayload.extend({ + version: z.literal("1"), + parentQueue: z.string(), +}); +export type OutputPayload = z.infer; export type QueueCapacity = { current: number; From a6175a7b763de059d136500902b2230b90aeabf4 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 27 Sep 2024 18:34:28 +0100 Subject: [PATCH 049/114] Acking is resetting the concurrencies --- .../run-engine/src/run-queue/index.test.ts | 42 +++++++++++ packages/run-engine/src/run-queue/index.ts | 75 +++++++++---------- 2 files changed, 77 insertions(+), 40 deletions(-) diff --git a/packages/run-engine/src/run-queue/index.test.ts b/packages/run-engine/src/run-queue/index.test.ts index 60932f0fd6..5c96530d05 100644 --- a/packages/run-engine/src/run-queue/index.test.ts +++ b/packages/run-engine/src/run-queue/index.test.ts @@ -283,6 +283,10 @@ describe("RunQueue", () => { ); expect(taskConcurrency2).toBe(1); + //queue length + const length2 = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); + expect(length2).toBe(0); + const dequeued2 = await queue.dequeueMessageInEnv(authenticatedEnvDev); expect(dequeued2).toBe(undefined); } finally { @@ -316,4 +320,42 @@ describe("RunQueue", () => { await queue.quit(); } }); + + redisTest("Acking", { timeout: 5_000 }, async ({ redisContainer, redis }) => { + const queue = new RunQueue({ + ...testOptions, + redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, + }); + + try { + await queue.enqueueMessage({ env: authenticatedEnvProd, message: messageProd }); + + const message = await queue.dequeueMessageInSharedQueue("test_12345"); + expect(message).toBeDefined(); + + await queue.acknowledgeMessage(message!.message.orgId, message!.messageId); + + //concurrencies + const queueConcurrency = await queue.currentConcurrencyOfQueue( + authenticatedEnvProd, + messageProd.queue + ); + expect(queueConcurrency).toBe(0); + const envConcurrency = await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd); + expect(envConcurrency).toBe(0); + const projectConcurrency = await queue.currentConcurrencyOfProject(authenticatedEnvProd); + expect(projectConcurrency).toBe(0); + const taskConcurrency = await queue.currentConcurrencyOfTask( + authenticatedEnvProd, + messageProd.taskIdentifier + ); + expect(taskConcurrency).toBe(0); + + //dequeue + const message2 = await queue.dequeueMessageInSharedQueue("test_12345"); + expect(message2).toBeUndefined(); + } finally { + await queue.quit(); + } + }); }); diff --git a/packages/run-engine/src/run-queue/index.ts b/packages/run-engine/src/run-queue/index.ts index a3569fbaac..007542915c 100644 --- a/packages/run-engine/src/run-queue/index.ts +++ b/packages/run-engine/src/run-queue/index.ts @@ -375,19 +375,19 @@ export class RunQueue { [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, }); - // await this.#callAcknowledgeMessage({ - // parentQueue: message.parentQueue, - // messageKey: this.keys.messageKey(orgId, messageId), - // messageQueue: message.queue, - // concurrencyKey: this.keys.currentConcurrencyKeyFromQueue(message.queue), - // envConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(message.queue), - // projectConcurrencyKey: this.keys.projectCurrentConcurrencyKeyFromQueue(message.queue), - // taskConcurrencyKey: this.keys.taskIdentifierCurrentConcurrencyKeyFromQueue( - // message.queue, - // message.taskIdentifier - // ), - // messageId, - // }); + await this.#callAcknowledgeMessage({ + messageId, + messageQueue: message.queue, + parentQueue: message.parentQueue, + messageKey: this.keys.messageKey(orgId, messageId), + concurrencyKey: this.keys.currentConcurrencyKeyFromQueue(message.queue), + envConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(message.queue), + taskConcurrencyKey: this.keys.taskIdentifierCurrentConcurrencyKeyFromQueue( + message.queue, + message.taskIdentifier + ), + projectConcurrencyKey: this.keys.projectCurrentConcurrencyKeyFromQueue(message.queue), + }); }, { kind: SpanKind.CONSUMER, @@ -1000,31 +1000,31 @@ export class RunQueue { } async #callAcknowledgeMessage({ + messageId, parentQueue, messageKey, messageQueue, - visibilityQueue, concurrencyKey, envConcurrencyKey, - orgConcurrencyKey, - messageId, + taskConcurrencyKey, + projectConcurrencyKey, }: { parentQueue: string; messageKey: string; messageQueue: string; - visibilityQueue: string; concurrencyKey: string; envConcurrencyKey: string; - orgConcurrencyKey: string; + taskConcurrencyKey: string; + projectConcurrencyKey: string; messageId: string; }) { this.logger.debug("Calling acknowledgeMessage", { messageKey, messageQueue, - visibilityQueue, concurrencyKey, envConcurrencyKey, - orgConcurrencyKey, + projectConcurrencyKey, + taskConcurrencyKey, messageId, parentQueue, service: this.name, @@ -1034,12 +1034,11 @@ export class RunQueue { parentQueue, messageKey, messageQueue, - visibilityQueue, concurrencyKey, envConcurrencyKey, - orgConcurrencyKey, - messageId, - messageQueue + projectConcurrencyKey, + taskConcurrencyKey, + messageId ); } @@ -1294,18 +1293,17 @@ return {messageId, messageScore, messagePayload} -- Return message details this.redis.defineCommand("acknowledgeMessage", { numberOfKeys: 7, lua: ` --- Keys: parentQueue, messageKey, messageQueue, visibilityQueue, concurrencyKey, envCurrentConcurrencyKey, orgCurrentConcurrencyKey +-- Keys: local parentQueue = KEYS[1] local messageKey = KEYS[2] local messageQueue = KEYS[3] -local visibilityQueue = KEYS[4] -local concurrencyKey = KEYS[5] -local envCurrentConcurrencyKey = KEYS[6] -local orgCurrentConcurrencyKey = KEYS[7] +local concurrencyKey = KEYS[4] +local envCurrentConcurrencyKey = KEYS[5] +local projectCurrentConcurrencyKey = KEYS[6] +local taskCurrentConcurrencyKey = KEYS[7] --- Args: messageId, messageQueueName +-- Args: local messageId = ARGV[1] -local messageQueueName = ARGV[2] -- Remove the message from the message key redis.call('DEL', messageKey) @@ -1316,18 +1314,16 @@ redis.call('ZREM', messageQueue, messageId) -- Rebalance the parent queue local earliestMessage = redis.call('ZRANGE', messageQueue, 0, 0, 'WITHSCORES') if #earliestMessage == 0 then - redis.call('ZREM', parentQueue, messageQueueName) + redis.call('ZREM', parentQueue, messageQueue) else - redis.call('ZADD', parentQueue, earliestMessage[2], messageQueueName) + redis.call('ZADD', parentQueue, earliestMessage[2], messageQueue) end --- Remove the message from the timeout queue (deprecated, will eventually remove this) -redis.call('ZREM', visibilityQueue, messageId) - -- Update the concurrency keys redis.call('SREM', concurrencyKey, messageId) redis.call('SREM', envCurrentConcurrencyKey, messageId) -redis.call('SREM', orgCurrentConcurrencyKey, messageId) +redis.call('SREM', projectCurrentConcurrencyKey, messageId) +redis.call('SREM', taskCurrentConcurrencyKey, messageId) `, }); @@ -1535,12 +1531,11 @@ declare module "ioredis" { parentQueue: string, messageKey: string, messageQueue: string, - visibilityQueue: string, concurrencyKey: string, envConcurrencyKey: string, - orgConcurrencyKey: string, + projectConcurrencyKey: string, + taskConcurrencyKey: string, messageId: string, - messageQueueName: string, callback?: Callback ): Result; From 58daf853803d75033a6f12af4545c15242af5369 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 28 Sep 2024 18:39:02 +0100 Subject: [PATCH 050/114] Check the key has been removed after acking --- packages/run-engine/src/run-queue/index.test.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/run-engine/src/run-queue/index.test.ts b/packages/run-engine/src/run-queue/index.test.ts index 5c96530d05..3627e27d45 100644 --- a/packages/run-engine/src/run-queue/index.test.ts +++ b/packages/run-engine/src/run-queue/index.test.ts @@ -333,6 +333,11 @@ describe("RunQueue", () => { const message = await queue.dequeueMessageInSharedQueue("test_12345"); expect(message).toBeDefined(); + //check the message is gone + const key = queue.keys.messageKey(message!.message.orgId, message!.messageId); + const exists = await redis.exists(key); + expect(exists).toBe(1); + await queue.acknowledgeMessage(message!.message.orgId, message!.messageId); //concurrencies @@ -351,6 +356,11 @@ describe("RunQueue", () => { ); expect(taskConcurrency).toBe(0); + //check the message is gone + const key2 = queue.keys.messageKey(message!.message.orgId, message!.messageId); + const exists2 = await redis.exists(key); + expect(exists2).toBe(0); + //dequeue const message2 = await queue.dequeueMessageInSharedQueue("test_12345"); expect(message2).toBeUndefined(); From 69b5c559f3dff40bbd9ffabb5b435d0f04674f1b Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 29 Sep 2024 14:10:53 +0100 Subject: [PATCH 051/114] Nacking is working --- .../run-engine/src/run-queue/index.test.ts | 64 +++++- packages/run-engine/src/run-queue/index.ts | 191 +++++++----------- 2 files changed, 140 insertions(+), 115 deletions(-) diff --git a/packages/run-engine/src/run-queue/index.test.ts b/packages/run-engine/src/run-queue/index.test.ts index 3627e27d45..cfaaedadab 100644 --- a/packages/run-engine/src/run-queue/index.test.ts +++ b/packages/run-engine/src/run-queue/index.test.ts @@ -357,7 +357,6 @@ describe("RunQueue", () => { expect(taskConcurrency).toBe(0); //check the message is gone - const key2 = queue.keys.messageKey(message!.message.orgId, message!.messageId); const exists2 = await redis.exists(key); expect(exists2).toBe(0); @@ -368,4 +367,67 @@ describe("RunQueue", () => { await queue.quit(); } }); + + redisTest("Nacking", { timeout: 5_000 }, async ({ redisContainer, redis }) => { + const queue = new RunQueue({ + ...testOptions, + redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, + }); + + try { + await queue.enqueueMessage({ env: authenticatedEnvProd, message: messageProd }); + + const message = await queue.dequeueMessageInSharedQueue("test_12345"); + expect(message).toBeDefined(); + + //check the message is there + const key = queue.keys.messageKey(message!.message.orgId, message!.messageId); + const exists = await redis.exists(key); + expect(exists).toBe(1); + + //concurrencies + const queueConcurrency = await queue.currentConcurrencyOfQueue( + authenticatedEnvProd, + messageProd.queue + ); + expect(queueConcurrency).toBe(1); + const envConcurrency = await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd); + expect(envConcurrency).toBe(1); + const projectConcurrency = await queue.currentConcurrencyOfProject(authenticatedEnvProd); + expect(projectConcurrency).toBe(1); + const taskConcurrency = await queue.currentConcurrencyOfTask( + authenticatedEnvProd, + messageProd.taskIdentifier + ); + expect(taskConcurrency).toBe(1); + + await queue.nackMessage(message!.message.orgId, message!.messageId); + + //concurrencies + const queueConcurrency2 = await queue.currentConcurrencyOfQueue( + authenticatedEnvProd, + messageProd.queue + ); + expect(queueConcurrency2).toBe(0); + const envConcurrency2 = await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd); + expect(envConcurrency2).toBe(0); + const projectConcurrency2 = await queue.currentConcurrencyOfProject(authenticatedEnvProd); + expect(projectConcurrency2).toBe(0); + const taskConcurrency2 = await queue.currentConcurrencyOfTask( + authenticatedEnvProd, + messageProd.taskIdentifier + ); + expect(taskConcurrency2).toBe(0); + + //check the message is there + const exists2 = await redis.exists(key); + expect(exists2).toBe(1); + + //dequeue + const message2 = await queue.dequeueMessageInSharedQueue("test_12345"); + expect(message2?.messageId).toBe(messageProd.runId); + } finally { + await queue.quit(); + } + }); }); diff --git a/packages/run-engine/src/run-queue/index.ts b/packages/run-engine/src/run-queue/index.ts index 007542915c..dbd50af5ec 100644 --- a/packages/run-engine/src/run-queue/index.ts +++ b/packages/run-engine/src/run-queue/index.ts @@ -401,48 +401,71 @@ export class RunQueue { } /** - * Negative acknowledge a message, which will requeue the message + * Negative acknowledge a message, which will requeue the message (with an optional future date) */ - public async nackMessage( - messageId: string, - retryAt: number = Date.now(), - updates?: Record - ) { + public async nackMessage(orgId: string, messageId: string, retryAt: number = Date.now()) { return this.#trace( "nackMessage", async (span) => { - // const message = await this.readMessage(messageId); - // if (!message) { - // logger.log(`[${this.name}].nackMessage() message not found`, { - // messageId, - // retryAt, - // updates, - // service: this.name, - // }); - // return; - // } - // span.setAttributes({ - // [SemanticAttributes.QUEUE]: message.queue, - // [SemanticAttributes.MESSAGE_ID]: message.messageId, - // [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, - // [SemanticAttributes.PARENT_QUEUE]: message.parentQueue, - // }); - // if (updates) { - // await this.replaceMessage(messageId, updates, retryAt, true); - // } - // await this.options.visibilityTimeoutStrategy.cancelHeartbeat(messageId); - // await this.#callNackMessage({ - // messageKey: this.keys.messageKey(messageId), - // messageQueue: message.queue, - // parentQueue: message.parentQueue, - // concurrencyKey: this.keys.currentConcurrencyKeyFromQueue(message.queue), - // envConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(message.queue), - // orgConcurrencyKey: this.keys.orgCurrentConcurrencyKeyFromQueue(message.queue), - // visibilityQueue: constants.MESSAGE_VISIBILITY_TIMEOUT_QUEUE, - // messageId, - // messageScore: retryAt, - // }); - // await this.options.subscriber?.messageNacked(message); + const message = await this.#readMessage(orgId, messageId); + if (!message) { + this.logger.log(`[${this.name}].nackMessage() message not found`, { + orgId, + messageId, + retryAt, + service: this.name, + }); + return; + } + + span.setAttributes({ + [SemanticAttributes.QUEUE]: message.queue, + [SemanticAttributes.RUN_ID]: messageId, + [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, + [SemanticAttributes.PARENT_QUEUE]: message.parentQueue, + }); + + const messageKey = this.keys.messageKey(orgId, messageId); + const messageQueue = message.queue; + const parentQueue = message.parentQueue; + const concurrencyKey = this.keys.currentConcurrencyKeyFromQueue(message.queue); + const envConcurrencyKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); + const taskConcurrencyKey = this.keys.taskIdentifierCurrentConcurrencyKeyFromQueue( + message.queue, + message.taskIdentifier + ); + const projectConcurrencyKey = this.keys.projectCurrentConcurrencyKeyFromQueue( + message.queue + ); + + const messageScore = retryAt; + + this.logger.debug("Calling nackMessage", { + messageKey, + messageQueue, + parentQueue, + concurrencyKey, + envConcurrencyKey, + projectConcurrencyKey, + taskConcurrencyKey, + messageId, + messageScore, + service: this.name, + }); + + await this.redis.nackMessage( + //keys + messageKey, + messageQueue, + parentQueue, + concurrencyKey, + envConcurrencyKey, + projectConcurrencyKey, + taskConcurrencyKey, + //args + messageId, + String(messageScore) + ); }, { kind: SpanKind.CONSUMER, @@ -1042,55 +1065,6 @@ export class RunQueue { ); } - async #callNackMessage({ - messageKey, - messageQueue, - parentQueue, - concurrencyKey, - envConcurrencyKey, - orgConcurrencyKey, - visibilityQueue, - messageId, - messageScore, - }: { - messageKey: string; - messageQueue: string; - parentQueue: string; - concurrencyKey: string; - envConcurrencyKey: string; - orgConcurrencyKey: string; - visibilityQueue: string; - messageId: string; - messageScore: number; - }) { - this.logger.debug("Calling nackMessage", { - messageKey, - messageQueue, - parentQueue, - concurrencyKey, - envConcurrencyKey, - orgConcurrencyKey, - visibilityQueue, - messageId, - messageScore, - service: this.name, - }); - - return this.redis.nackMessage( - messageKey, - messageQueue, - parentQueue, - concurrencyKey, - envConcurrencyKey, - orgConcurrencyKey, - visibilityQueue, - messageQueue, - messageId, - String(Date.now()), - String(messageScore) - ); - } - async #callCalculateMessageCapacities({ currentConcurrencyKey, currentEnvConcurrencyKey, @@ -1330,43 +1304,34 @@ redis.call('SREM', taskCurrentConcurrencyKey, messageId) this.redis.defineCommand("nackMessage", { numberOfKeys: 7, lua: ` --- Keys: childQueueKey, parentQueueKey, visibilityQueue, concurrencyKey, envConcurrencyKey, orgConcurrencyKey, messageId +-- Keys: local messageKey = KEYS[1] -local childQueueKey = KEYS[2] +local messageQueueKey = KEYS[2] local parentQueueKey = KEYS[3] local concurrencyKey = KEYS[4] local envConcurrencyKey = KEYS[5] -local orgConcurrencyKey = KEYS[6] -local visibilityQueue = KEYS[7] +local projectConcurrencyKey = KEYS[6] +local taskConcurrencyKey = KEYS[7] --- Args: childQueueName, messageId, currentTime, messageScore -local childQueueName = ARGV[1] -local messageId = ARGV[2] -local currentTime = tonumber(ARGV[3]) -local messageScore = tonumber(ARGV[4]) +-- Args: +local messageId = ARGV[1] +local messageScore = tonumber(ARGV[2]) -- Update the concurrency keys redis.call('SREM', concurrencyKey, messageId) redis.call('SREM', envConcurrencyKey, messageId) -redis.call('SREM', orgConcurrencyKey, messageId) - --- Check to see if the message is still in the visibilityQueue -local messageVisibility = tonumber(redis.call('ZSCORE', visibilityQueue, messageId)) or 0 - -if messageVisibility > 0 then --- Remove the message from the timeout queue (deprecated, will eventually remove this) - redis.call('ZREM', visibilityQueue, messageId) -end +redis.call('SREM', projectConcurrencyKey, messageId) +redis.call('SREM', taskConcurrencyKey, messageId) -- Enqueue the message into the queue -redis.call('ZADD', childQueueKey, messageScore, messageId) +redis.call('ZADD', messageQueueKey, messageScore, messageId) -- Rebalance the parent queue -local earliestMessage = redis.call('ZRANGE', childQueueKey, 0, 0, 'WITHSCORES') +local earliestMessage = redis.call('ZRANGE', messageQueueKey, 0, 0, 'WITHSCORES') if #earliestMessage == 0 then - redis.call('ZREM', parentQueueKey, childQueueName) + redis.call('ZREM', parentQueueKey, messageQueueKey) else - redis.call('ZADD', parentQueueKey, earliestMessage[2], childQueueName) + redis.call('ZADD', parentQueueKey, earliestMessage[2], messageQueueKey) end `, }); @@ -1541,15 +1506,13 @@ declare module "ioredis" { nackMessage( messageKey: string, - childQueueKey: string, + messageQueue: string, parentQueueKey: string, concurrencyKey: string, envConcurrencyKey: string, - orgConcurrencyKey: string, - visibilityQueue: string, - childQueueName: string, + projectConcurrencyKey: string, + taskConcurrencyKey: string, messageId: string, - currentTime: string, messageScore: string, callback?: Callback ): Result; From 4309819fb84d864319c7a9b5c78f3f7c6a7b25e4 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 29 Sep 2024 14:52:26 +0100 Subject: [PATCH 052/114] Changed the package to CommonJS + Node10 so it works with Redlock --- packages/run-engine/package.json | 1 - packages/run-engine/tsconfig.json | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/run-engine/package.json b/packages/run-engine/package.json index 3c0fef9709..ad252b22fe 100644 --- a/packages/run-engine/package.json +++ b/packages/run-engine/package.json @@ -4,7 +4,6 @@ "version": "0.0.1", "main": "./src/index.ts", "types": "./src/index.ts", - "type": "module", "dependencies": { "@opentelemetry/api": "^1.9.0", "@opentelemetry/semantic-conventions": "^1.27.0", diff --git a/packages/run-engine/tsconfig.json b/packages/run-engine/tsconfig.json index d9349e6dcb..47472e2963 100644 --- a/packages/run-engine/tsconfig.json +++ b/packages/run-engine/tsconfig.json @@ -2,8 +2,10 @@ "compilerOptions": { "target": "ES2019", "lib": ["ES2019", "DOM", "DOM.Iterable"], - "module": "NodeNext", - "moduleResolution": "Node16", + "module": "CommonJS", + "moduleResolution": "Node10", + "moduleDetection": "force", + "verbatimModuleSyntax": false, "types": ["vitest/globals"], "esModuleInterop": true, "forceConsistentCasingInFileNames": true, From dbbc81d71e212dc9a61eb7d2abf0e9267f2f938a Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 29 Sep 2024 15:44:18 +0100 Subject: [PATCH 053/114] Moved the database, otel and emails packages to be in internal-packages --- .gitmodules | 4 ++-- apps/webapp/tsconfig.json | 12 ++++++------ {packages => internal-packages}/database/.env | 0 {packages => internal-packages}/database/.gitignore | 0 .../database/.infisical.json | 0 .../database/CHANGELOG.md | 0 {packages => internal-packages}/database/README.md | 0 .../database/package.json | 0 .../migrations/20221206131204_init/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20221212112045_api_connections/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20221219171221_support_output_step/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20221221164650_added_istest_to_run/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230105175909_make_ts_a_string/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230124224734_remove_retry/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230210141917_add_templates/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230307161815_add_stopping_status/migration.sql | 0 .../20230308120746_add_deployment_logs/migration.sql | 0 .../20230308151627_add_log_number/migration.sql | 0 .../migration.sql | 0 .../20230309110029_add_log_polls_model/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230309112008_add_poll_number/migration.sql | 0 .../migration.sql | 0 .../20230312103325_add_key_value_items/migration.sql | 0 .../20230312132000_more_kv_types/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230328125714_add_endpoints_model/migration.sql | 0 .../migration.sql | 0 .../20230329120440_add_job_models/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230329154731_add_executions/migration.sql | 0 .../20230330115131_add_tasjs/migration.sql | 0 .../20230330123046_add_waiting_status/migration.sql | 0 .../20230330125238_rename_finished_at/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230331091517_add_noop_to_tasks/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230406092923_add_job_connections/migration.sql | 0 .../20230418092025_add_uses_local_auth/migration.sql | 0 .../migration.sql | 0 .../20230419141305_add_http_sources/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230425094302_add_job_event_rule/migration.sql | 0 .../migration.sql | 0 .../20230425103833_add_job_alias_model/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230427131511_add_subtasks/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230504090710_add_shadow_to_job/migration.sql | 0 .../migration.sql | 0 .../20230504200916_add_redact_to_tasks/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230505091221_create_queue_model/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230512145548_add_run_connections/migration.sql | 0 .../20230512162858_add_start_position/migration.sql | 0 .../20230512163048_move_start_position/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230515151445_add_trigger_sources/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230525111943_project_slug_added/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230605160520_add_style_to_tasks/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230612150500_add_task_attempts/migration.sql | 0 .../20230613091640_event_example_table/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230704094425_add_pk_api_key/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migrations/20230724074140_changed/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20230925174509_add_callback_url/migration.sql | 0 .../20230927160010_add_data_migrations/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20231115142936_add_webhook_source/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20240116163753_add_task_run_model/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20240131105237_project_deleted_at/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20240212174757_project_version/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20240213154458_add_image_details/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20240425122814_add_alert_schema/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20240507113449_add_alert_storage/migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../database/prisma/migrations/migration_lock.toml | 0 .../database/prisma/schema.prisma | 0 .../database/src/index.ts | 0 .../database/tsconfig.json | 0 {packages => internal-packages}/emails/.gitignore | 0 {packages => internal-packages}/emails/README.md | 0 .../emails/emails/alert-attempt-failure.tsx | 0 .../emails/emails/alert-run-failure.tsx | 0 .../emails/emails/components/BasePath.tsx | 0 .../emails/emails/components/Footer.tsx | 0 .../emails/emails/components/Image.tsx | 0 .../emails/emails/components/styles.ts | 0 .../emails/emails/deployment-failure.tsx | 0 .../emails/emails/deployment-success.tsx | 0 .../emails/emails/invite.tsx | 0 .../emails/emails/magic-link.tsx | 0 .../emails/emails/welcome.tsx | 0 {packages => internal-packages}/emails/package.json | 0 {packages => internal-packages}/emails/src/index.tsx | 0 {packages => internal-packages}/emails/tsconfig.json | 0 .../otlp-importer/.gitignore | 0 .../otlp-importer/CHANGELOG.md | 0 .../otlp-importer/LICENSE | 0 .../otlp-importer/README.md | 0 .../otlp-importer/jest.config.js | 0 .../otlp-importer/package.json | 0 {packages => internal-packages}/otlp-importer/protos | 0 .../otlp-importer/scripts/generate-protos.mjs | 0 .../otlp-importer/scripts/generate-protos.sh | 0 .../otlp-importer/scripts/submodule.mjs | 0 .../otlp-importer/scripts/utils.mjs | 0 .../proto/collector/logs/v1/logs_service.ts | 0 .../proto/collector/metrics/v1/metrics_service.ts | 0 .../proto/collector/trace/v1/trace_service.ts | 0 .../opentelemetry/proto/common/v1/common.ts | 0 .../generated/opentelemetry/proto/logs/v1/logs.ts | 0 .../opentelemetry/proto/metrics/v1/metrics.ts | 0 .../opentelemetry/proto/resource/v1/resource.ts | 0 .../generated/opentelemetry/proto/trace/v1/trace.ts | 0 .../otlp-importer/src/index.ts | 0 .../otlp-importer/tsconfig.json | 0 .../otlp-importer/tsup.config.ts | 0 {packages => internal-packages}/run-engine/README.md | 0 .../run-engine/package.json | 2 +- .../run-engine/src/engine/index.test.ts | 0 .../run-engine/src/engine/index.ts | 2 +- .../run-engine/src/run-queue/index.test.ts | 0 .../run-engine/src/run-queue/index.ts | 0 .../run-engine/src/run-queue/keyProducer.test.ts | 0 .../run-engine/src/run-queue/keyProducer.ts | 2 +- .../src/run-queue/simpleWeightedPriorityStrategy.ts | 0 .../run-engine/src/run-queue/types.ts | 2 +- .../run-engine/src/shared/asyncWorker.ts | 0 .../run-engine/src/shared/index.ts | 2 +- .../run-engine/src/simple-queue/index.test.ts | 0 .../run-engine/src/simple-queue/index.ts | 0 .../run-engine/src/simple-queue/processor.test.ts | 0 .../run-engine/src/simple-queue/processor.ts | 0 .../run-engine/src/test/containerTest.ts | 2 +- .../run-engine/src/test/utils.ts | 2 +- .../run-engine/tsconfig.json | 4 ++-- .../run-engine/vitest.config.ts | 0 pnpm-workspace.yaml | 1 + 627 files changed, 18 insertions(+), 17 deletions(-) rename {packages => internal-packages}/database/.env (100%) rename {packages => internal-packages}/database/.gitignore (100%) rename {packages => internal-packages}/database/.infisical.json (100%) rename {packages => internal-packages}/database/CHANGELOG.md (100%) rename {packages => internal-packages}/database/README.md (100%) rename {packages => internal-packages}/database/package.json (100%) rename {packages => internal-packages}/database/prisma/migrations/20221206131204_init/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221207113401_user_organization_workflow/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221209205520_add_runtime_environment/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221212112045_api_connections/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221212134846_api_optional_external_id/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221212171032_api_external_id_integer/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221212175847_api_connection_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221215112631_add_workflow_triggers/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221216105935_add_custom_event_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221216110114_add_custom_event_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221216125550_added_runtime_environment_title/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221216135920_add_workflow_run_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221216140358_rename_data_to_input/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221216140505_add_context_to_workflow_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221216140541_add_datetime_to_workflow_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221216151235_remove_title_column_from_env/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221219142856_add_workflow_run_step_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221219171221_support_output_step/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221220095618_add_error_to_workflow_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221220100932_add_timestamps_to_run_and_steps/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221220101020_make_started_at_optional/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221220205030_add_connection_slot_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221221113722_add_registered_webhook_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221221114754_update_registered_webhook_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221221124220_add_secret_to_registered_webhook/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221221164650_added_istest_to_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221222153652_refactor_schema_generic_pub_sub_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221223132913_more_pub_sub_refactoring/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221223142352_added_organization_and_key_to_external_source/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221223143123_added_organization_and_key_index/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221223144952_extracted_event_rules_by_environment/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221223145049_add_unique_constract_to_event_rule/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221223150238_add_event_rule_to_workflow_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221223150407_add_trigger_metadata_to_event_rule/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221223152905_add_secret_to_external_source/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221223155859_add_service_to_trigger_event/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221223162136_added_timestamps_to_trigger_event/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221227182358_rename_rule_to_filter/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221227191746_renamed_processed_to_dispatch_for_events/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221228101710_add_service_for_external_source/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221228155121_add_external_service_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221228175137_triggerevent_added_is_test/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221229095721_add_integration_requests/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221229100124_add_waiting_for_connection_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221229101944_added_workflow_service_eventnames/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221229113827_add_fetching_to_integration_request/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20221229121050_add_integration_responses/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230102095903_connection_added_authentication_method/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230102100144_connection_authentication_config/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230102142756_add_durable_delay_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230102175304_connection_removed_delete_cascades/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230103145652_add_idempotency_key_to_workflow_run_steps/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230104144824_add_interrupted_status_to_workflow_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230104150555_add_attempt_count_to_workflow_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230104160649_add_interruption_step/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230105114046_add_disconnection_type/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230105115145_remove_interruption_type/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230105115649_add_disconnected_state_to_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230105115719_remove_interrupted_state_from_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230105173643_add_timestamp_to_steps/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230105175909_make_ts_a_string/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230109195958_response_changed_output_context/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230114154045_add_scheduler_external_source/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230114171501_add_cancelled_external_source_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230115194010_add_scheduler_source_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230115194253_add_scheduler_uniq_index/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230115194338_schedule_should_be_json/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230119164847_add_schema_for_disabling_and_archiving/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230119174326_move_new_datetime_fields_to_workflow/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230119175445_add_enabled_to_event_rule/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230119183126_removed_archived_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230120010921_add_trigger_ttl_in_seconds/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230120014606_add_timed_out_status_for_workflow_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230120014641_add_timed_out_columns/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230120225200_add_fetch_request_step_type/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230120234147_add_fetch_request_models/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230124224344_add_retry_to_fetch_request/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230124224734_remove_retry/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230125105732_add_manual_registration_to_external_source/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230127121743_add_slack_interaction_trigger_type/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230127123917_add_internal_source_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230203110022_add_run_once_step_type/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230203150225_add_json_schema_to_workflows/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230209171333_add_github_app_authorizations/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230209171703_add_status_to_github_app_authorization/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230209171921_split_an_auth_attempt_with_auth_table/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230210111316_add_installaton_details_to_authorization/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230210111519_change_access_token_url_column_name/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230210141917_add_templates/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230210142709_modify_templates_schema/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230210150029_add_repo_data_to_org_templates/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230210153657_add_template_id_to_auth_attempt/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230210175708_add_repository_id_to_org_template/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230213113228_add_status_to_org_template/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230213142107_add_documentation_to_templates/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230213154802_simplify_org_template_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230214101645_integration_request_added_version/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230214163434_add_run_local_docs_to_templates/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230215101806_workflowrunstep_displayproperties/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230216100218_add_cloud_waitlist_to_user/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230216131220_add_account_type_to_app_auth/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230216132133_add_account_name_to_auth/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230216184843_add_org_template_to_workflows/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230224144605_add_is_live_to_templates/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230302104940_add_metadata_to_workflows/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230303085314_setup_github_app_models_for_cloud/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230303085914_add_redirect_to_auth_attempt/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230303092424_remove_oauth_token_columns/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230303101108_add_timestamps_to_attempts/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230303103808_add_timestamps_to_auths/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230303123529_add_token_to_github_app_auth/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230303131806_add_authorization_id_to_attempt/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230305201223_add_repository_project/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230305201615_make_repo_project_name_unique_and_add_org/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230306101403_modify_docker_columns/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230306101518_remove_docker_columns/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230306101927_add_deployments_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230306103508_added_status_text_to_projects/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230306114828_added_error_to_deployments/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230306115630_added_build_duration_to_deployments/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230306121727_added_vm_stuff_to_projects/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230306122326_added_current_deployment/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230306132827_added_more_columns_to_deployment/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230306132935_make_build_id_unique/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230306140931_added_back_in_docker_columns/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230306150053_added_projects_to_workflows/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230307094704_added_more_states_to_deployment_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230307104034_added_stopping_columns/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230307160247_add_version_column_to_deployment/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230307161815_add_stopping_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230308120746_add_deployment_logs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230308151627_add_log_number/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230309091702_add_latest_log_dates/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230309110029_add_log_polls_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230309110318_add_next_poll_scheduled_at/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230309110358_make_next_poll_scheduled_at_optional/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230309111003_add_created_at_to_polls/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230309112008_add_poll_number/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230310121352_add_preparing_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230312103325_add_key_value_items/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230312132000_more_kv_types/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230313100030_added_feature_cloud_to_users/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230313143913_added_latest_commit_to_projects/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230313173750_added_hosted_waitlist_bool/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230322093004_add_current_environments/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230328125714_add_endpoints_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230328130438_remove_creator_from_endpoints/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230329120440_add_job_models/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230329121955_add_trigger_json_to_job_version/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230329123037_simplified_job_models/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230329125135_create_event_log_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230329131748_add_deliver_at_to_event_logs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230329132829_changed_dispatched_at_to_delivered_at/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230329133809_rename_endpoint_name_to_slug/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230329154731_add_executions/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230330115131_add_tasjs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230330123046_add_waiting_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230330125238_rename_finished_at/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230330125349_rename_finished_at_on_executions/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230330132602_use_string_instead_of_big_int/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230331091517_add_noop_to_tasks/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230331150231_use_ulid_instead_of_cuid_and_ts_for_tasks/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230404094247_add_key_and_icon_to_tasks/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230404094406_make_display_key_optional/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230404095457_rename_display_properties_to_elements/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230404101137_add_elements_to_executions/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230404203551_remove_source_from_events/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230406092923_add_job_connections/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230418092025_add_uses_local_auth/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230419140902_apiconnection_secrets/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230419141305_add_http_sources/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230419141524_make_webhooks_specific_to_an_endpoint/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230419141651_add_timestamps_to_http_sources/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230419155523_api_connection_attempt/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230419175107_remove_status_from_http_source/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230419183033_add_ready_to_job_instance/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230421162946_add_interactive_to_http_source/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230421164041_add_webhook_deliveries/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230421164059_remove_source_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230421171344_add_connection_to_http_sources/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230423183231_api_connection_attempt_title/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230425094302_add_job_event_rule/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230425094839_only_one_event_rule_per_job_instance/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230425103833_add_job_alias_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230425104022_add_env_to_job_alias/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230425104911_add_version_to_job_alias/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230425122235_add_resuming_tasks_to_event_rules/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230425133327_make_task_event_rules_one_to_one/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230425162308_can_have_more_than_one_event_rule_on_an_instance/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230425162613_add_action_identifier_to_event_rule/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230425162923_remove_uniq_on_action_identifier/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230426093630_api_connection_attempt_added_security_code/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230426150713_add_support_for_projects/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230426151056_users_are_now_org_members/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230426162129_scope_jobs_to_projects/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230426182130_api_connection_expires_at/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230427131511_add_subtasks/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230428092036_add_slug_to_api_connections/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230428092422_rename_api_connection_models/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230428135439_secret_reference_key_unique/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230428141846_secret_reference_multiple_api_connections/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230503094127_add_trigger_variants/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230503113400_add_slug_to_trigger_variants/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230503172156_rename_create_execution_to_create_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230504090710_add_shadow_to_job/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230504152611_rename_shadow_to_internal/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230504200916_add_redact_to_tasks/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230505085501_add_queue_columns_to_job/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230505085546_move_queue_columns_to_job_instance/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230505085931_add_queued_job_run_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230505091221_create_queue_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230505092917_add_queued_at_to_job_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230511101816_remove_job_trigger_variants/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230512085413_schema_redesign_for_new_system/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230512123150_add_dynamic_triggers/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230512145548_add_run_connections/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230512162858_add_start_position/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230512163048_move_start_position/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230515101628_add_prepare_to_job_version/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230515102310_remove_prepare_from_job_version/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230515110730_add_back_prepare_to_job_version/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230515111507_add_prepared_to_job_version/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230515151445_add_trigger_sources/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230515152217_move_params_to_trigger_sources/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230516121549_remove_http_source_table/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230516122003_add_secret_reference_to_trigger_source/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230516154239_add_integration_identifier_to_clients/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230516155545_add_integration_auth_method/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230516163116_add_scopes_to_connections/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230516171911_add_description_to_clients/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230517105559_rename_job_connections_to_job_integrations/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230517105823_rename_connection_metadata/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230517153909_add_dynamic_trigger_to_trigger_sources/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230517154918_add_dynamic_trigger_to_job_triggers/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230518134021_polymorphic_event_dispatching/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230519133020_add_dynamic_registrations/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230522085325_make_secret_reference_provider_an_enum/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230522114830_add_manual_to_event_dispatcher/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230522131952_create_schedule_sources_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230522140136_added_event_id_to_event_record/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230523132431_add_type_to_dynamic_triggers/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230523135129_add_metadata_to_schedules/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230525103409_add_env_to_external_accounts/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230525110458_add_external_account_to_events/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230525110838_add_external_account_to_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230525111943_project_slug_added/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230525123241_add_external_account_to_schedule_sources/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230525124804_add_external_account_to_trigger_sources/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230525134604_add_missing_connections/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230525141420_add_waiting_on_missing_connections_statys/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230530135120_task_added_connection_key/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230531174211_user_marketing_emails/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230601165125_add_org_member_invite/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230601170659_invite_unique_orgid_email_constraint/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230601170752_invite_remove_orgmember/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230602110258_added_user_confirmed_basic_details/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230602155844_keep_runtimeenvironment_when_orgmember_deleted/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230605123159_renamed_credentials_reference_to_custom_client_reference/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230605144132_add_elements_to_job_versions/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230605155714_add_run_connection_to_tasks/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230605160520_add_style_to_tasks/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230606081054_add_job_run_execution/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230606081441_add_preprocess_job_execution_reason/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230606081946_add_preprocess_runs_to_job_version/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230606082119_add_preprocess_to_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230606082719_add_preprocess_status_to_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230606120402_add_task_to_run_execution/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230606122534_improve_run_execution_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230606132031_removed_http_responses_from_executions/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230608102125_scope_job_versions_to_environments/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230608151823_scope_trigger_source_to_envs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230609095323_rename_elements_to_properties/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230609150207_add_graphile_job_id_to_executions/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230609150822_add_graphile_job_id_string/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230612150500_add_task_attempts/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230613091640_event_example_table/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230613092902_event_example_payload/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230613104234_event_example_added_name_icon/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230614103739_add_deploy_hook_identifier_to_endpoints/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230614110359_rename_deploy_to_index/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230614122553_create_endpoint_index_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230614125945_add_source_data_to_endpoint_index/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230614135014_change_endpoint_source_enum/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230614141902_added_api_to_endpoint_index_source/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230615152126_revamp_integration_schema/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230616083056_add_integration_auth_method/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230616084406_remove_metadata_and_add_icon_to_integration/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230616093240_api_integration_vote_added/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230616094008_updating_run_connections_to_work_with_new_integrations/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230616102332_make_trigger_source_integrations_required/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230616103552_add_api_identifier_to_integrations/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230616104748_add_integration_definition/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230616104937_remove_api_identifier/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230616105239_remove_duplicate_identifier_from_auth_method/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230619100936_add_operation_to_task/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230627212239_add_source_context_to_event_record/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230628100705_add_metadata_to_dynamic_trigger_registrations/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230628102426_add_dynamic_source_columns_to_trigger_source/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230630093541_integration_added_setup_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230630160813_integration_definition_packagename_description_help/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230703152107_add_account_identifier_to_missing_connections/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230704094159_add_auto_enable_to_envs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230704094425_add_pk_api_key/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230705151109_add_cloud_invitation_stuff/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230705152702_remove_is_cloud_activated/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230707101156_add_timestamps_to_jobs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230707105750_add_source_registration_job_to_dynamic_triggers/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230707122412_add_source_registration_job_to_trigger_sources/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230707145540_add_auth_identifier_to_user_table/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230707145604_make_auth_identifier_unique/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230708193746_add_output_properties_to_tasks/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230711175328_added_canceled_run_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230712075144_added_canceled_to_task_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230712132529_added_relationship_between_schedule_source_and_dynamic_trigger/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230716054029_is_retry_for_job_run_execution/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230721124527_add_icon_to_integration_definition/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230724074140_changed/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230725161151_pk_api_key_unique_and_non_null/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230731142627_remove_user_access_token/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230731145332_add_version_to_secret_store/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230814131639_added_cancelled_at_column_to_the_event_record_table/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230821123033_trigger_source_option_added_for_multiple_event_dimensions/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230821132604_remove_trigger_source_event_table/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230822130655_add_status_to_job_version/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230823124049_add_deleted_at_to_jobs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230824191603_added_trigger_source_metadata/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230904145326_add_execution_columns_to_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230904205457_add_max_run_execution_time_to_orgs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230919124531_add_resolver_auth_source/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230919141017_added_job_run_status_record_table/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230919150351_add_unresolved_auth_job_run_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230922205611_add_invalid_payload_run_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230925174509_add_callback_url/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230927160010_add_data_migrations/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20230929100348_add_yielded_executions_to_job_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231003092741_add_version_to_endpoint/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231005064823_add_job_run_internal/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231010115840_endpoint_index_status_added/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231010120458_endpoint_index_data_and_stats_are_now_optional/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231010135433_endpoint_index_added_error_column/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231011104302_add_run_chunk_column/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231011134840_add_auto_yielded_executions/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231011141302_add_location_to_auto_yield_executions/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231011145406_change_run_chunk_execution_limit_default/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231011213532_add_auto_yield_threshold_settings_to_endpoints/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231013083144_add_next_event_timestamp_to_schedule_source/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231019123406_add_force_yield_immediately_to_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231020145127_add_sdk_version_to_endpoints/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231023144342_added_event_record_payload_type_default_is_json/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231023173456_added_trigger_http_endpoint_table/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231024122540_http_endpoint_scoped_to_project_not_environment/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231024122736_runtime_environment_shortcodes/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231025091939_trigger_http_endpoint_environment_created/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231025144821_set_endpoint_run_chunk_execution_limit_from_60ms_to_60000ms/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231026100653_added_skip_triggering_runs_to_trigger_http_endpoint_environment/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231026103218_add_endpoint_relation_to_http_endpoint_environment/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231026103614_trigger_http_endpoint_environment_removed_the_environment_constraint/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231026104408_trigger_http_endpoint_environment_added_the_environment_constraint_back_in/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231026165235_trigger_http_endpoint_environment_added_source/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231028193441_prepare_for_invoke_trigger/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231101105021_add_child_execution_mode_to_tasks/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231101202056_trigger_http_endpoint_environment_added_created_at_updated_at/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231102143530_added_relationships_between_event_record_and_trigger_http_endpoint_trigger_http_endpoint_environment/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231102171207_job_version_added_trigger_link_and_trigger_help/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231107134830_add_context_to_tasks/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231109122305_add_external_account_to_dispatcher/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231113151412_add_output_is_undefined/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231115134828_add_events_schema_and_tables/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231115142936_add_webhook_source/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231116113235_add_unique_index_on_job_run_sub/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231117145312_add_additional_run_statuses/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231120163155_add_webhook_environment/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231121144353_make_job_run_number_optional/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231121154359_add_job_counter_table/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231121154545_seed_job_counter_tables/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231121155237_change_kv_value_to_bytes/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231122091927_add_webhook_request_delivery/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231122105932_add_env_to_webhook_delivery/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231122151126_add_delivery_numbers/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231122210707_add_concurrency_limit_tables_and_columns/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231122212600_make_job_queues_optional/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231123113308_remove_concurrency_group_from_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231123115015_add_concurrency_limit_group_id_to_run_executions/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231124131123_add_referral_source_and_company_size/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231204163703_add_composite_index_to_triggerdotdev_events_to_speed_up_organization_queries/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231206205233_add_execution_failure_count_to_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20231214103115_add_tunnel_id_to_runtime_environment/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240111142421_created_personal_access_token_and_authorization_code_tables/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240111143407_encrypt_the_personal_access_token_in_the_database/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240111144844_encrypted_token_now_json_and_added_obfuscated_token_column/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240111151921_added_hashed_token_column_which_will_be_used_for_search/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240115160657_add_external_ref_to_projects/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240116115734_add_background_worker_models/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240116162443_added_organization_runs_enabled_column/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240116163753_add_task_run_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240116165157_add_task_identifier_to_task_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240118155050_add_completion_columns_to_task_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240118155439_change_payload_output_type_defaults/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240118160024_make_task_run_output_optional/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240123093539_add_content_hash_to_background_worker/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240123193905_restructure_task_runs_with_attempts/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240124104013_add_friendly_id_to_v3_tables/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240124122331_convert_task_attempt_error_to_json/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240126130139_add_locked_to_version_to_task_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240126204658_add_parent_attempt_to_task_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240129140944_endpoint_deleted_at_column/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240129161848_endpoint_nullable_slug_instead_of_deletedat_column/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240130165343_add_composite_index_to_job_run_for_job_id_and_created_at/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240130205109_add_trace_context_to_task_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240131105237_project_deleted_at/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240202115155_added_job_run_index_back_in_using_prisma_schema/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240206112723_organization_deleted_at/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240206133516_integration_connections_can_be_disabled/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240207174021_add_unified_task_event_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240207195749_move_event_data_to_json/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240208124114_changes_to_the_task_event_schema/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240208133710_use_bigint_for_event_durations/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240209110749_convert_task_fields_to_optional/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240209121123_add_task_run_trace_columns/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240209153602_add_worker_columns_to_task_event/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240212174757_project_version/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240213134217_added_sdk_version_and_cli_version_to_background_worker/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240213141507_add_task_queues_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240213154458_add_image_details/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240214172901_add_concurrency_key_to_task_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240214173404_add_queue_options_to_task_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240214174325_associate_task_runs_with_task_queues/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240214175158_add_queue_properties_to_events/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240214180324_remove_task_queue_relationship_from_task_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240214180709_add_queue_relationship_to_task_run_attempts/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240215113146_added_task_run_counter/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240215113618_added_number_to_task_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240220165410_add_attempt_number_to_task_events/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240221140555_add_queue_and_retry_config_to_background_worker_tasks/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240223115106_batch_trigger_models/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240223121156_added_environment_variable_and_environment_variable_value_tables/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240224132039_change_image_details_unique_constraint/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240227142146_environment_variable_added_friendly_id/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240227144742_task_run_added_is_test/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240227170811_add_is_cancelled_to_task_events/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240228114913_add_checkpoint_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240228161621_add_run_is_test_to_task_events/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240229141613_add_batch_id_to_task_events/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240305111659_add_deployment_models/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240305154054_use_external_build_data_json_column/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240307095223_add_content_hash_to_worker_deployments/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240307104333_move_image_details_to_deployments/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240308203644_add_deployment_error_columns/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240311135706_add_triggered_by_to_deployments/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240312095501_add_status_to_task_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240312095826_add_interrupted_status_to_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240312105844_add_system_failure_status_to_task_run_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240312125252_add_status_to_batch_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240312131122_add_task_run_attempt_to_batch_run_items/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240313110150_add_environment_to_task_run_attempts/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240316174721_add_run_and_metadata_to_checkpoint/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240318135831_add_checkpoint_restore_event/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240318170823_add_image_ref_to_checkpoint/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240319120645_convert_start_time_to_nanoseconds_since_epoch/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240319121124_what_migration_is_this/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240320100720_add_timed_out_status_to_deployments/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240322165042_organization_v3_enabled_defaults_to_false/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240322172035_add_checkpoint_event_to_dependencies/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240325224419_add_output_type_to_task_events/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240326145956_add_payload_columns_to_task_event/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240327121557_add_machine_config_to_worker_task/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240329142454_add_concurrency_limit_columns/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240402105424_set_default_env_concurrency_limit_to_5/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240404150051_add_crashed_task_run_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240409090907_add_waiting_for_deploy_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240411135457_task_schedules_for_v3/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240411145517_added_trigger_source_to_background_worker_task/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240412151157_add_operational_task_schedule_columns/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240415134559_create_runtime_env_session_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240415135132_make_attempt_dependencies_an_array/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240415143325_remove_attempt_unique_constraint_from_batch_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240416100646_add_cron_description_column/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240417083233_make_schedule_columns_more_generic/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240417142604_dont_delete_task_runs_if_schedules_are_deleted/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240418092931_make_idempotency_key_optional/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240418093153_add_idempotency_key_to_task_event/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240418100819_make_batch_task_run_idempotency_key_optional/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240418114019_make_batch_run_item_run_id_non_unique/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240424210540_change_on_delete_ref_action_on_run_execution/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240425114134_task_run_compound_index_for_project_id_created_at_and_task_identifier/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240425122814_add_alert_schema/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240425131147_add_enabled_flag_to_alert_channels/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240426095144_add_deployment_success_alert_type/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240426095622_add_deduplication_key_to_alert_channels/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240426102405_add_unique_deduplication_index_to_alert_channels/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240428142050_add_models_for_slack_integration/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240428150144_org_integration_non_optional_fields/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240430101936_add_lazy_attempt_support_flag_to_workers/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240430110419_task_run_indexes_projectid_task_identifier_and_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240430110717_task_run_compound_index_projectid_task_identifier_and_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240507113449_add_alert_storage/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240517105021_add_environment_types_to_alert_channel/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240517105224_remove_test_alert_type/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240517135206_created_bulk_action_group_and_bulk_action_item_for_canceling_and_replaying_in_bulk/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240517164246_bulk_action_item_source_run_id_is_required/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240517164924_bulk_action_item_added_failed_state_with_error/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240520112812_create_deferred_scheduled_event_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240522130825_org_v2_enabled_flag_defaults_to_false/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240522134117_organization_added_has_requested_v3_column/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240523081426_added_task_run_number_counter_with_environment_as_well_as_task_identifier/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240523135511_added_task_event_trace_id_index/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240606090155_add_v2_marqs_enabled_flag_on_org/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240610183406_added_schedule_instances_limit/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240610195009_add_an_index_on_span_id/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240611104018_add_started_at_to_task_runs/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240611113047_change_schedules_limit_to_maximum_schedules_limit/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240611115843_add_usage_duration_columns/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240611133630_added_timezone_to_task_schedule_defaults_to_null_which_is_utc/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240611143911_add_cost_in_cents_on_task_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240612091006_add_machine_presets_data_to_task_events/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240612171759_add_task_run_base_cost/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240612175820_add_usage_cost_in_cents_to_task_event/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240613082558_add_machine_preset_to_task_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240613115526_delete_task_schedule_timezone_column/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240613115623_task_schedule_timezone_column_defaults_to_utc/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240625095006_add_run_id_index_to_task_events/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240629082837_add_task_run_delay_changes/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240630204935_add_ttl_schema_changes/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240702131302_add_max_attempts_to_task_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240704170301_scope_idempotency_key_to_task_identifier/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240704205712_add_schedule_id_index_to_task_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240707190354_add_built_at_to_worker_deployment/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240717101035_add_task_schedule_type_dynamic_and_static/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240720101649_added_task_run_tag_removed_task_tag/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240723104125_task_run_tag_name_id_index/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240729101628_task_run_span_id_index/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240801175428_added_task_run_completed_at_column/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240804143817_add_task_run_logs_deleted_at_column/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240806163040_task_run_completed_at_index/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240809153150_background_worker_task_add_index_for_quick_lookup_of_task_identifiers/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240809213411_add_checkpoint_attempt_number_column/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240810090402_add_background_worker_file_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240811185335_improve_background_worker_file_model/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240815123344_add_project_alert_type_task_run_and_migrate_existing_alerts/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240815125616_migrate_alerts_from_attempt_to_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240821135542_job_run_index_for_organization_id_and_created_at/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240821141347_job_run_index_version_id/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240821142006_batch_task_run_item_index_task_run_attempt_id/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240821142353_batch_task_run_item_index_task_run_id/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240821145232_task_index_parent_id/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240905134437_add_builder_project_id_to_projects/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240909141925_task_run_attempt_index_on_task_run_id/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240916155127_added_job_run_event_id_index/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240920085046_add_task_hierarchy_columns_without_parent_task_run_id_index/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240920085226_add_parent_task_run_id_index_concurrently/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240924125845_add_root_task_run_id_index/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240924130558_add_parent_span_id_to_task_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240925092304_add_metadata_and_output_to_task_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240925205409_add_error_to_task_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240926093535_add_seed_metadata_columns_to_task_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20240926110455_add_parent_span_id_index_to_the_task_run_table/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/migration_lock.toml (100%) rename {packages => internal-packages}/database/prisma/schema.prisma (100%) rename {packages => internal-packages}/database/src/index.ts (100%) rename {packages => internal-packages}/database/tsconfig.json (100%) rename {packages => internal-packages}/emails/.gitignore (100%) rename {packages => internal-packages}/emails/README.md (100%) rename {packages => internal-packages}/emails/emails/alert-attempt-failure.tsx (100%) rename {packages => internal-packages}/emails/emails/alert-run-failure.tsx (100%) rename {packages => internal-packages}/emails/emails/components/BasePath.tsx (100%) rename {packages => internal-packages}/emails/emails/components/Footer.tsx (100%) rename {packages => internal-packages}/emails/emails/components/Image.tsx (100%) rename {packages => internal-packages}/emails/emails/components/styles.ts (100%) rename {packages => internal-packages}/emails/emails/deployment-failure.tsx (100%) rename {packages => internal-packages}/emails/emails/deployment-success.tsx (100%) rename {packages => internal-packages}/emails/emails/invite.tsx (100%) rename {packages => internal-packages}/emails/emails/magic-link.tsx (100%) rename {packages => internal-packages}/emails/emails/welcome.tsx (100%) rename {packages => internal-packages}/emails/package.json (100%) rename {packages => internal-packages}/emails/src/index.tsx (100%) rename {packages => internal-packages}/emails/tsconfig.json (100%) rename {packages => internal-packages}/otlp-importer/.gitignore (100%) rename {packages => internal-packages}/otlp-importer/CHANGELOG.md (100%) rename {packages => internal-packages}/otlp-importer/LICENSE (100%) rename {packages => internal-packages}/otlp-importer/README.md (100%) rename {packages => internal-packages}/otlp-importer/jest.config.js (100%) rename {packages => internal-packages}/otlp-importer/package.json (100%) rename {packages => internal-packages}/otlp-importer/protos (100%) rename {packages => internal-packages}/otlp-importer/scripts/generate-protos.mjs (100%) rename {packages => internal-packages}/otlp-importer/scripts/generate-protos.sh (100%) rename {packages => internal-packages}/otlp-importer/scripts/submodule.mjs (100%) rename {packages => internal-packages}/otlp-importer/scripts/utils.mjs (100%) rename {packages => internal-packages}/otlp-importer/src/generated/opentelemetry/proto/collector/logs/v1/logs_service.ts (100%) rename {packages => internal-packages}/otlp-importer/src/generated/opentelemetry/proto/collector/metrics/v1/metrics_service.ts (100%) rename {packages => internal-packages}/otlp-importer/src/generated/opentelemetry/proto/collector/trace/v1/trace_service.ts (100%) rename {packages => internal-packages}/otlp-importer/src/generated/opentelemetry/proto/common/v1/common.ts (100%) rename {packages => internal-packages}/otlp-importer/src/generated/opentelemetry/proto/logs/v1/logs.ts (100%) rename {packages => internal-packages}/otlp-importer/src/generated/opentelemetry/proto/metrics/v1/metrics.ts (100%) rename {packages => internal-packages}/otlp-importer/src/generated/opentelemetry/proto/resource/v1/resource.ts (100%) rename {packages => internal-packages}/otlp-importer/src/generated/opentelemetry/proto/trace/v1/trace.ts (100%) rename {packages => internal-packages}/otlp-importer/src/index.ts (100%) rename {packages => internal-packages}/otlp-importer/tsconfig.json (100%) rename {packages => internal-packages}/otlp-importer/tsup.config.ts (100%) rename {packages => internal-packages}/run-engine/README.md (100%) rename {packages => internal-packages}/run-engine/package.json (94%) rename {packages => internal-packages}/run-engine/src/engine/index.test.ts (100%) rename {packages => internal-packages}/run-engine/src/engine/index.ts (97%) rename {packages => internal-packages}/run-engine/src/run-queue/index.test.ts (100%) rename {packages => internal-packages}/run-engine/src/run-queue/index.ts (100%) rename {packages => internal-packages}/run-engine/src/run-queue/keyProducer.test.ts (100%) rename {packages => internal-packages}/run-engine/src/run-queue/keyProducer.ts (98%) rename {packages => internal-packages}/run-engine/src/run-queue/simpleWeightedPriorityStrategy.ts (100%) rename {packages => internal-packages}/run-engine/src/run-queue/types.ts (98%) rename {packages => internal-packages}/run-engine/src/shared/asyncWorker.ts (100%) rename {packages => internal-packages}/run-engine/src/shared/index.ts (96%) rename {packages => internal-packages}/run-engine/src/simple-queue/index.test.ts (100%) rename {packages => internal-packages}/run-engine/src/simple-queue/index.ts (100%) rename {packages => internal-packages}/run-engine/src/simple-queue/processor.test.ts (100%) rename {packages => internal-packages}/run-engine/src/simple-queue/processor.ts (100%) rename {packages => internal-packages}/run-engine/src/test/containerTest.ts (97%) rename {packages => internal-packages}/run-engine/src/test/utils.ts (94%) rename {packages => internal-packages}/run-engine/tsconfig.json (82%) rename {packages => internal-packages}/run-engine/vitest.config.ts (100%) diff --git a/.gitmodules b/.gitmodules index a284e2e666..ecf08cb1a4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "packages/otlp-importer/protos"] - path = packages/otlp-importer/protos +[submodule "internal-packages/otlp-importer/protos"] + path = internal-packages/otlp-importer/protos url = https://github.com/open-telemetry/opentelemetry-proto.git diff --git a/apps/webapp/tsconfig.json b/apps/webapp/tsconfig.json index f2825d7e25..16eb40c47a 100644 --- a/apps/webapp/tsconfig.json +++ b/apps/webapp/tsconfig.json @@ -24,14 +24,14 @@ "@trigger.dev/sdk/*": ["../../packages/trigger-sdk/src/*"], "@trigger.dev/core": ["../../packages/core/src/index"], "@trigger.dev/core/*": ["../../packages/core/src/*"], - "@trigger.dev/database": ["../../packages/database/src/index"], - "@trigger.dev/database/*": ["../../packages/database/src/*"], + "@trigger.dev/database": ["../../internal-packages/database/src/index"], + "@trigger.dev/database/*": ["../../internal-packages/database/src/*"], "@trigger.dev/yalt": ["../../packages/yalt/src/index"], "@trigger.dev/yalt/*": ["../../packages/yalt/src/*"], - "@trigger.dev/otlp-importer": ["../../packages/otlp-importer/src/index"], - "@trigger.dev/otlp-importer/*": ["../../packages/otlp-importer/src/*"], - "emails": ["../../packages/emails/src/index"], - "emails/*": ["../../packages/emails/src/*"] + "@trigger.dev/otlp-importer": ["../../internal-packages/otlp-importer/src/index"], + "@trigger.dev/otlp-importer/*": ["../../internal-packages/otlp-importer/src/*"], + "emails": ["../../internal-packages/emails/src/index"], + "emails/*": ["../../internal-packages/emails/src/*"] }, "noEmit": true } diff --git a/packages/database/.env b/internal-packages/database/.env similarity index 100% rename from packages/database/.env rename to internal-packages/database/.env diff --git a/packages/database/.gitignore b/internal-packages/database/.gitignore similarity index 100% rename from packages/database/.gitignore rename to internal-packages/database/.gitignore diff --git a/packages/database/.infisical.json b/internal-packages/database/.infisical.json similarity index 100% rename from packages/database/.infisical.json rename to internal-packages/database/.infisical.json diff --git a/packages/database/CHANGELOG.md b/internal-packages/database/CHANGELOG.md similarity index 100% rename from packages/database/CHANGELOG.md rename to internal-packages/database/CHANGELOG.md diff --git a/packages/database/README.md b/internal-packages/database/README.md similarity index 100% rename from packages/database/README.md rename to internal-packages/database/README.md diff --git a/packages/database/package.json b/internal-packages/database/package.json similarity index 100% rename from packages/database/package.json rename to internal-packages/database/package.json diff --git a/packages/database/prisma/migrations/20221206131204_init/migration.sql b/internal-packages/database/prisma/migrations/20221206131204_init/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221206131204_init/migration.sql rename to internal-packages/database/prisma/migrations/20221206131204_init/migration.sql diff --git a/packages/database/prisma/migrations/20221207113401_user_organization_workflow/migration.sql b/internal-packages/database/prisma/migrations/20221207113401_user_organization_workflow/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221207113401_user_organization_workflow/migration.sql rename to internal-packages/database/prisma/migrations/20221207113401_user_organization_workflow/migration.sql diff --git a/packages/database/prisma/migrations/20221209205520_add_runtime_environment/migration.sql b/internal-packages/database/prisma/migrations/20221209205520_add_runtime_environment/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221209205520_add_runtime_environment/migration.sql rename to internal-packages/database/prisma/migrations/20221209205520_add_runtime_environment/migration.sql diff --git a/packages/database/prisma/migrations/20221212112045_api_connections/migration.sql b/internal-packages/database/prisma/migrations/20221212112045_api_connections/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221212112045_api_connections/migration.sql rename to internal-packages/database/prisma/migrations/20221212112045_api_connections/migration.sql diff --git a/packages/database/prisma/migrations/20221212134846_api_optional_external_id/migration.sql b/internal-packages/database/prisma/migrations/20221212134846_api_optional_external_id/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221212134846_api_optional_external_id/migration.sql rename to internal-packages/database/prisma/migrations/20221212134846_api_optional_external_id/migration.sql diff --git a/packages/database/prisma/migrations/20221212171032_api_external_id_integer/migration.sql b/internal-packages/database/prisma/migrations/20221212171032_api_external_id_integer/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221212171032_api_external_id_integer/migration.sql rename to internal-packages/database/prisma/migrations/20221212171032_api_external_id_integer/migration.sql diff --git a/packages/database/prisma/migrations/20221212175847_api_connection_status/migration.sql b/internal-packages/database/prisma/migrations/20221212175847_api_connection_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221212175847_api_connection_status/migration.sql rename to internal-packages/database/prisma/migrations/20221212175847_api_connection_status/migration.sql diff --git a/packages/database/prisma/migrations/20221215112631_add_workflow_triggers/migration.sql b/internal-packages/database/prisma/migrations/20221215112631_add_workflow_triggers/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221215112631_add_workflow_triggers/migration.sql rename to internal-packages/database/prisma/migrations/20221215112631_add_workflow_triggers/migration.sql diff --git a/packages/database/prisma/migrations/20221216105935_add_custom_event_model/migration.sql b/internal-packages/database/prisma/migrations/20221216105935_add_custom_event_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221216105935_add_custom_event_model/migration.sql rename to internal-packages/database/prisma/migrations/20221216105935_add_custom_event_model/migration.sql diff --git a/packages/database/prisma/migrations/20221216110114_add_custom_event_status/migration.sql b/internal-packages/database/prisma/migrations/20221216110114_add_custom_event_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221216110114_add_custom_event_status/migration.sql rename to internal-packages/database/prisma/migrations/20221216110114_add_custom_event_status/migration.sql diff --git a/packages/database/prisma/migrations/20221216125550_added_runtime_environment_title/migration.sql b/internal-packages/database/prisma/migrations/20221216125550_added_runtime_environment_title/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221216125550_added_runtime_environment_title/migration.sql rename to internal-packages/database/prisma/migrations/20221216125550_added_runtime_environment_title/migration.sql diff --git a/packages/database/prisma/migrations/20221216135920_add_workflow_run_model/migration.sql b/internal-packages/database/prisma/migrations/20221216135920_add_workflow_run_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221216135920_add_workflow_run_model/migration.sql rename to internal-packages/database/prisma/migrations/20221216135920_add_workflow_run_model/migration.sql diff --git a/packages/database/prisma/migrations/20221216140358_rename_data_to_input/migration.sql b/internal-packages/database/prisma/migrations/20221216140358_rename_data_to_input/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221216140358_rename_data_to_input/migration.sql rename to internal-packages/database/prisma/migrations/20221216140358_rename_data_to_input/migration.sql diff --git a/packages/database/prisma/migrations/20221216140505_add_context_to_workflow_run/migration.sql b/internal-packages/database/prisma/migrations/20221216140505_add_context_to_workflow_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221216140505_add_context_to_workflow_run/migration.sql rename to internal-packages/database/prisma/migrations/20221216140505_add_context_to_workflow_run/migration.sql diff --git a/packages/database/prisma/migrations/20221216140541_add_datetime_to_workflow_run/migration.sql b/internal-packages/database/prisma/migrations/20221216140541_add_datetime_to_workflow_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221216140541_add_datetime_to_workflow_run/migration.sql rename to internal-packages/database/prisma/migrations/20221216140541_add_datetime_to_workflow_run/migration.sql diff --git a/packages/database/prisma/migrations/20221216151235_remove_title_column_from_env/migration.sql b/internal-packages/database/prisma/migrations/20221216151235_remove_title_column_from_env/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221216151235_remove_title_column_from_env/migration.sql rename to internal-packages/database/prisma/migrations/20221216151235_remove_title_column_from_env/migration.sql diff --git a/packages/database/prisma/migrations/20221219142856_add_workflow_run_step_model/migration.sql b/internal-packages/database/prisma/migrations/20221219142856_add_workflow_run_step_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221219142856_add_workflow_run_step_model/migration.sql rename to internal-packages/database/prisma/migrations/20221219142856_add_workflow_run_step_model/migration.sql diff --git a/packages/database/prisma/migrations/20221219171221_support_output_step/migration.sql b/internal-packages/database/prisma/migrations/20221219171221_support_output_step/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221219171221_support_output_step/migration.sql rename to internal-packages/database/prisma/migrations/20221219171221_support_output_step/migration.sql diff --git a/packages/database/prisma/migrations/20221220095618_add_error_to_workflow_run/migration.sql b/internal-packages/database/prisma/migrations/20221220095618_add_error_to_workflow_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221220095618_add_error_to_workflow_run/migration.sql rename to internal-packages/database/prisma/migrations/20221220095618_add_error_to_workflow_run/migration.sql diff --git a/packages/database/prisma/migrations/20221220100932_add_timestamps_to_run_and_steps/migration.sql b/internal-packages/database/prisma/migrations/20221220100932_add_timestamps_to_run_and_steps/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221220100932_add_timestamps_to_run_and_steps/migration.sql rename to internal-packages/database/prisma/migrations/20221220100932_add_timestamps_to_run_and_steps/migration.sql diff --git a/packages/database/prisma/migrations/20221220101020_make_started_at_optional/migration.sql b/internal-packages/database/prisma/migrations/20221220101020_make_started_at_optional/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221220101020_make_started_at_optional/migration.sql rename to internal-packages/database/prisma/migrations/20221220101020_make_started_at_optional/migration.sql diff --git a/packages/database/prisma/migrations/20221220205030_add_connection_slot_model/migration.sql b/internal-packages/database/prisma/migrations/20221220205030_add_connection_slot_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221220205030_add_connection_slot_model/migration.sql rename to internal-packages/database/prisma/migrations/20221220205030_add_connection_slot_model/migration.sql diff --git a/packages/database/prisma/migrations/20221221113722_add_registered_webhook_model/migration.sql b/internal-packages/database/prisma/migrations/20221221113722_add_registered_webhook_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221221113722_add_registered_webhook_model/migration.sql rename to internal-packages/database/prisma/migrations/20221221113722_add_registered_webhook_model/migration.sql diff --git a/packages/database/prisma/migrations/20221221114754_update_registered_webhook_model/migration.sql b/internal-packages/database/prisma/migrations/20221221114754_update_registered_webhook_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221221114754_update_registered_webhook_model/migration.sql rename to internal-packages/database/prisma/migrations/20221221114754_update_registered_webhook_model/migration.sql diff --git a/packages/database/prisma/migrations/20221221124220_add_secret_to_registered_webhook/migration.sql b/internal-packages/database/prisma/migrations/20221221124220_add_secret_to_registered_webhook/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221221124220_add_secret_to_registered_webhook/migration.sql rename to internal-packages/database/prisma/migrations/20221221124220_add_secret_to_registered_webhook/migration.sql diff --git a/packages/database/prisma/migrations/20221221164650_added_istest_to_run/migration.sql b/internal-packages/database/prisma/migrations/20221221164650_added_istest_to_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221221164650_added_istest_to_run/migration.sql rename to internal-packages/database/prisma/migrations/20221221164650_added_istest_to_run/migration.sql diff --git a/packages/database/prisma/migrations/20221222153652_refactor_schema_generic_pub_sub_model/migration.sql b/internal-packages/database/prisma/migrations/20221222153652_refactor_schema_generic_pub_sub_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221222153652_refactor_schema_generic_pub_sub_model/migration.sql rename to internal-packages/database/prisma/migrations/20221222153652_refactor_schema_generic_pub_sub_model/migration.sql diff --git a/packages/database/prisma/migrations/20221223132913_more_pub_sub_refactoring/migration.sql b/internal-packages/database/prisma/migrations/20221223132913_more_pub_sub_refactoring/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221223132913_more_pub_sub_refactoring/migration.sql rename to internal-packages/database/prisma/migrations/20221223132913_more_pub_sub_refactoring/migration.sql diff --git a/packages/database/prisma/migrations/20221223142352_added_organization_and_key_to_external_source/migration.sql b/internal-packages/database/prisma/migrations/20221223142352_added_organization_and_key_to_external_source/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221223142352_added_organization_and_key_to_external_source/migration.sql rename to internal-packages/database/prisma/migrations/20221223142352_added_organization_and_key_to_external_source/migration.sql diff --git a/packages/database/prisma/migrations/20221223143123_added_organization_and_key_index/migration.sql b/internal-packages/database/prisma/migrations/20221223143123_added_organization_and_key_index/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221223143123_added_organization_and_key_index/migration.sql rename to internal-packages/database/prisma/migrations/20221223143123_added_organization_and_key_index/migration.sql diff --git a/packages/database/prisma/migrations/20221223144952_extracted_event_rules_by_environment/migration.sql b/internal-packages/database/prisma/migrations/20221223144952_extracted_event_rules_by_environment/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221223144952_extracted_event_rules_by_environment/migration.sql rename to internal-packages/database/prisma/migrations/20221223144952_extracted_event_rules_by_environment/migration.sql diff --git a/packages/database/prisma/migrations/20221223145049_add_unique_constract_to_event_rule/migration.sql b/internal-packages/database/prisma/migrations/20221223145049_add_unique_constract_to_event_rule/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221223145049_add_unique_constract_to_event_rule/migration.sql rename to internal-packages/database/prisma/migrations/20221223145049_add_unique_constract_to_event_rule/migration.sql diff --git a/packages/database/prisma/migrations/20221223150238_add_event_rule_to_workflow_runs/migration.sql b/internal-packages/database/prisma/migrations/20221223150238_add_event_rule_to_workflow_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221223150238_add_event_rule_to_workflow_runs/migration.sql rename to internal-packages/database/prisma/migrations/20221223150238_add_event_rule_to_workflow_runs/migration.sql diff --git a/packages/database/prisma/migrations/20221223150407_add_trigger_metadata_to_event_rule/migration.sql b/internal-packages/database/prisma/migrations/20221223150407_add_trigger_metadata_to_event_rule/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221223150407_add_trigger_metadata_to_event_rule/migration.sql rename to internal-packages/database/prisma/migrations/20221223150407_add_trigger_metadata_to_event_rule/migration.sql diff --git a/packages/database/prisma/migrations/20221223152905_add_secret_to_external_source/migration.sql b/internal-packages/database/prisma/migrations/20221223152905_add_secret_to_external_source/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221223152905_add_secret_to_external_source/migration.sql rename to internal-packages/database/prisma/migrations/20221223152905_add_secret_to_external_source/migration.sql diff --git a/packages/database/prisma/migrations/20221223155859_add_service_to_trigger_event/migration.sql b/internal-packages/database/prisma/migrations/20221223155859_add_service_to_trigger_event/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221223155859_add_service_to_trigger_event/migration.sql rename to internal-packages/database/prisma/migrations/20221223155859_add_service_to_trigger_event/migration.sql diff --git a/packages/database/prisma/migrations/20221223162136_added_timestamps_to_trigger_event/migration.sql b/internal-packages/database/prisma/migrations/20221223162136_added_timestamps_to_trigger_event/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221223162136_added_timestamps_to_trigger_event/migration.sql rename to internal-packages/database/prisma/migrations/20221223162136_added_timestamps_to_trigger_event/migration.sql diff --git a/packages/database/prisma/migrations/20221227182358_rename_rule_to_filter/migration.sql b/internal-packages/database/prisma/migrations/20221227182358_rename_rule_to_filter/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221227182358_rename_rule_to_filter/migration.sql rename to internal-packages/database/prisma/migrations/20221227182358_rename_rule_to_filter/migration.sql diff --git a/packages/database/prisma/migrations/20221227191746_renamed_processed_to_dispatch_for_events/migration.sql b/internal-packages/database/prisma/migrations/20221227191746_renamed_processed_to_dispatch_for_events/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221227191746_renamed_processed_to_dispatch_for_events/migration.sql rename to internal-packages/database/prisma/migrations/20221227191746_renamed_processed_to_dispatch_for_events/migration.sql diff --git a/packages/database/prisma/migrations/20221228101710_add_service_for_external_source/migration.sql b/internal-packages/database/prisma/migrations/20221228101710_add_service_for_external_source/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221228101710_add_service_for_external_source/migration.sql rename to internal-packages/database/prisma/migrations/20221228101710_add_service_for_external_source/migration.sql diff --git a/packages/database/prisma/migrations/20221228155121_add_external_service_model/migration.sql b/internal-packages/database/prisma/migrations/20221228155121_add_external_service_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221228155121_add_external_service_model/migration.sql rename to internal-packages/database/prisma/migrations/20221228155121_add_external_service_model/migration.sql diff --git a/packages/database/prisma/migrations/20221228175137_triggerevent_added_is_test/migration.sql b/internal-packages/database/prisma/migrations/20221228175137_triggerevent_added_is_test/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221228175137_triggerevent_added_is_test/migration.sql rename to internal-packages/database/prisma/migrations/20221228175137_triggerevent_added_is_test/migration.sql diff --git a/packages/database/prisma/migrations/20221229095721_add_integration_requests/migration.sql b/internal-packages/database/prisma/migrations/20221229095721_add_integration_requests/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221229095721_add_integration_requests/migration.sql rename to internal-packages/database/prisma/migrations/20221229095721_add_integration_requests/migration.sql diff --git a/packages/database/prisma/migrations/20221229100124_add_waiting_for_connection_status/migration.sql b/internal-packages/database/prisma/migrations/20221229100124_add_waiting_for_connection_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221229100124_add_waiting_for_connection_status/migration.sql rename to internal-packages/database/prisma/migrations/20221229100124_add_waiting_for_connection_status/migration.sql diff --git a/packages/database/prisma/migrations/20221229101944_added_workflow_service_eventnames/migration.sql b/internal-packages/database/prisma/migrations/20221229101944_added_workflow_service_eventnames/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221229101944_added_workflow_service_eventnames/migration.sql rename to internal-packages/database/prisma/migrations/20221229101944_added_workflow_service_eventnames/migration.sql diff --git a/packages/database/prisma/migrations/20221229113827_add_fetching_to_integration_request/migration.sql b/internal-packages/database/prisma/migrations/20221229113827_add_fetching_to_integration_request/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221229113827_add_fetching_to_integration_request/migration.sql rename to internal-packages/database/prisma/migrations/20221229113827_add_fetching_to_integration_request/migration.sql diff --git a/packages/database/prisma/migrations/20221229121050_add_integration_responses/migration.sql b/internal-packages/database/prisma/migrations/20221229121050_add_integration_responses/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20221229121050_add_integration_responses/migration.sql rename to internal-packages/database/prisma/migrations/20221229121050_add_integration_responses/migration.sql diff --git a/packages/database/prisma/migrations/20230102095903_connection_added_authentication_method/migration.sql b/internal-packages/database/prisma/migrations/20230102095903_connection_added_authentication_method/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230102095903_connection_added_authentication_method/migration.sql rename to internal-packages/database/prisma/migrations/20230102095903_connection_added_authentication_method/migration.sql diff --git a/packages/database/prisma/migrations/20230102100144_connection_authentication_config/migration.sql b/internal-packages/database/prisma/migrations/20230102100144_connection_authentication_config/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230102100144_connection_authentication_config/migration.sql rename to internal-packages/database/prisma/migrations/20230102100144_connection_authentication_config/migration.sql diff --git a/packages/database/prisma/migrations/20230102142756_add_durable_delay_model/migration.sql b/internal-packages/database/prisma/migrations/20230102142756_add_durable_delay_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230102142756_add_durable_delay_model/migration.sql rename to internal-packages/database/prisma/migrations/20230102142756_add_durable_delay_model/migration.sql diff --git a/packages/database/prisma/migrations/20230102175304_connection_removed_delete_cascades/migration.sql b/internal-packages/database/prisma/migrations/20230102175304_connection_removed_delete_cascades/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230102175304_connection_removed_delete_cascades/migration.sql rename to internal-packages/database/prisma/migrations/20230102175304_connection_removed_delete_cascades/migration.sql diff --git a/packages/database/prisma/migrations/20230103145652_add_idempotency_key_to_workflow_run_steps/migration.sql b/internal-packages/database/prisma/migrations/20230103145652_add_idempotency_key_to_workflow_run_steps/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230103145652_add_idempotency_key_to_workflow_run_steps/migration.sql rename to internal-packages/database/prisma/migrations/20230103145652_add_idempotency_key_to_workflow_run_steps/migration.sql diff --git a/packages/database/prisma/migrations/20230104144824_add_interrupted_status_to_workflow_runs/migration.sql b/internal-packages/database/prisma/migrations/20230104144824_add_interrupted_status_to_workflow_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230104144824_add_interrupted_status_to_workflow_runs/migration.sql rename to internal-packages/database/prisma/migrations/20230104144824_add_interrupted_status_to_workflow_runs/migration.sql diff --git a/packages/database/prisma/migrations/20230104150555_add_attempt_count_to_workflow_run/migration.sql b/internal-packages/database/prisma/migrations/20230104150555_add_attempt_count_to_workflow_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230104150555_add_attempt_count_to_workflow_run/migration.sql rename to internal-packages/database/prisma/migrations/20230104150555_add_attempt_count_to_workflow_run/migration.sql diff --git a/packages/database/prisma/migrations/20230104160649_add_interruption_step/migration.sql b/internal-packages/database/prisma/migrations/20230104160649_add_interruption_step/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230104160649_add_interruption_step/migration.sql rename to internal-packages/database/prisma/migrations/20230104160649_add_interruption_step/migration.sql diff --git a/packages/database/prisma/migrations/20230105114046_add_disconnection_type/migration.sql b/internal-packages/database/prisma/migrations/20230105114046_add_disconnection_type/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230105114046_add_disconnection_type/migration.sql rename to internal-packages/database/prisma/migrations/20230105114046_add_disconnection_type/migration.sql diff --git a/packages/database/prisma/migrations/20230105115145_remove_interruption_type/migration.sql b/internal-packages/database/prisma/migrations/20230105115145_remove_interruption_type/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230105115145_remove_interruption_type/migration.sql rename to internal-packages/database/prisma/migrations/20230105115145_remove_interruption_type/migration.sql diff --git a/packages/database/prisma/migrations/20230105115649_add_disconnected_state_to_runs/migration.sql b/internal-packages/database/prisma/migrations/20230105115649_add_disconnected_state_to_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230105115649_add_disconnected_state_to_runs/migration.sql rename to internal-packages/database/prisma/migrations/20230105115649_add_disconnected_state_to_runs/migration.sql diff --git a/packages/database/prisma/migrations/20230105115719_remove_interrupted_state_from_runs/migration.sql b/internal-packages/database/prisma/migrations/20230105115719_remove_interrupted_state_from_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230105115719_remove_interrupted_state_from_runs/migration.sql rename to internal-packages/database/prisma/migrations/20230105115719_remove_interrupted_state_from_runs/migration.sql diff --git a/packages/database/prisma/migrations/20230105173643_add_timestamp_to_steps/migration.sql b/internal-packages/database/prisma/migrations/20230105173643_add_timestamp_to_steps/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230105173643_add_timestamp_to_steps/migration.sql rename to internal-packages/database/prisma/migrations/20230105173643_add_timestamp_to_steps/migration.sql diff --git a/packages/database/prisma/migrations/20230105175909_make_ts_a_string/migration.sql b/internal-packages/database/prisma/migrations/20230105175909_make_ts_a_string/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230105175909_make_ts_a_string/migration.sql rename to internal-packages/database/prisma/migrations/20230105175909_make_ts_a_string/migration.sql diff --git a/packages/database/prisma/migrations/20230109195958_response_changed_output_context/migration.sql b/internal-packages/database/prisma/migrations/20230109195958_response_changed_output_context/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230109195958_response_changed_output_context/migration.sql rename to internal-packages/database/prisma/migrations/20230109195958_response_changed_output_context/migration.sql diff --git a/packages/database/prisma/migrations/20230114154045_add_scheduler_external_source/migration.sql b/internal-packages/database/prisma/migrations/20230114154045_add_scheduler_external_source/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230114154045_add_scheduler_external_source/migration.sql rename to internal-packages/database/prisma/migrations/20230114154045_add_scheduler_external_source/migration.sql diff --git a/packages/database/prisma/migrations/20230114171501_add_cancelled_external_source_status/migration.sql b/internal-packages/database/prisma/migrations/20230114171501_add_cancelled_external_source_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230114171501_add_cancelled_external_source_status/migration.sql rename to internal-packages/database/prisma/migrations/20230114171501_add_cancelled_external_source_status/migration.sql diff --git a/packages/database/prisma/migrations/20230115194010_add_scheduler_source_model/migration.sql b/internal-packages/database/prisma/migrations/20230115194010_add_scheduler_source_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230115194010_add_scheduler_source_model/migration.sql rename to internal-packages/database/prisma/migrations/20230115194010_add_scheduler_source_model/migration.sql diff --git a/packages/database/prisma/migrations/20230115194253_add_scheduler_uniq_index/migration.sql b/internal-packages/database/prisma/migrations/20230115194253_add_scheduler_uniq_index/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230115194253_add_scheduler_uniq_index/migration.sql rename to internal-packages/database/prisma/migrations/20230115194253_add_scheduler_uniq_index/migration.sql diff --git a/packages/database/prisma/migrations/20230115194338_schedule_should_be_json/migration.sql b/internal-packages/database/prisma/migrations/20230115194338_schedule_should_be_json/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230115194338_schedule_should_be_json/migration.sql rename to internal-packages/database/prisma/migrations/20230115194338_schedule_should_be_json/migration.sql diff --git a/packages/database/prisma/migrations/20230119164847_add_schema_for_disabling_and_archiving/migration.sql b/internal-packages/database/prisma/migrations/20230119164847_add_schema_for_disabling_and_archiving/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230119164847_add_schema_for_disabling_and_archiving/migration.sql rename to internal-packages/database/prisma/migrations/20230119164847_add_schema_for_disabling_and_archiving/migration.sql diff --git a/packages/database/prisma/migrations/20230119174326_move_new_datetime_fields_to_workflow/migration.sql b/internal-packages/database/prisma/migrations/20230119174326_move_new_datetime_fields_to_workflow/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230119174326_move_new_datetime_fields_to_workflow/migration.sql rename to internal-packages/database/prisma/migrations/20230119174326_move_new_datetime_fields_to_workflow/migration.sql diff --git a/packages/database/prisma/migrations/20230119175445_add_enabled_to_event_rule/migration.sql b/internal-packages/database/prisma/migrations/20230119175445_add_enabled_to_event_rule/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230119175445_add_enabled_to_event_rule/migration.sql rename to internal-packages/database/prisma/migrations/20230119175445_add_enabled_to_event_rule/migration.sql diff --git a/packages/database/prisma/migrations/20230119183126_removed_archived_status/migration.sql b/internal-packages/database/prisma/migrations/20230119183126_removed_archived_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230119183126_removed_archived_status/migration.sql rename to internal-packages/database/prisma/migrations/20230119183126_removed_archived_status/migration.sql diff --git a/packages/database/prisma/migrations/20230120010921_add_trigger_ttl_in_seconds/migration.sql b/internal-packages/database/prisma/migrations/20230120010921_add_trigger_ttl_in_seconds/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230120010921_add_trigger_ttl_in_seconds/migration.sql rename to internal-packages/database/prisma/migrations/20230120010921_add_trigger_ttl_in_seconds/migration.sql diff --git a/packages/database/prisma/migrations/20230120014606_add_timed_out_status_for_workflow_runs/migration.sql b/internal-packages/database/prisma/migrations/20230120014606_add_timed_out_status_for_workflow_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230120014606_add_timed_out_status_for_workflow_runs/migration.sql rename to internal-packages/database/prisma/migrations/20230120014606_add_timed_out_status_for_workflow_runs/migration.sql diff --git a/packages/database/prisma/migrations/20230120014641_add_timed_out_columns/migration.sql b/internal-packages/database/prisma/migrations/20230120014641_add_timed_out_columns/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230120014641_add_timed_out_columns/migration.sql rename to internal-packages/database/prisma/migrations/20230120014641_add_timed_out_columns/migration.sql diff --git a/packages/database/prisma/migrations/20230120225200_add_fetch_request_step_type/migration.sql b/internal-packages/database/prisma/migrations/20230120225200_add_fetch_request_step_type/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230120225200_add_fetch_request_step_type/migration.sql rename to internal-packages/database/prisma/migrations/20230120225200_add_fetch_request_step_type/migration.sql diff --git a/packages/database/prisma/migrations/20230120234147_add_fetch_request_models/migration.sql b/internal-packages/database/prisma/migrations/20230120234147_add_fetch_request_models/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230120234147_add_fetch_request_models/migration.sql rename to internal-packages/database/prisma/migrations/20230120234147_add_fetch_request_models/migration.sql diff --git a/packages/database/prisma/migrations/20230124224344_add_retry_to_fetch_request/migration.sql b/internal-packages/database/prisma/migrations/20230124224344_add_retry_to_fetch_request/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230124224344_add_retry_to_fetch_request/migration.sql rename to internal-packages/database/prisma/migrations/20230124224344_add_retry_to_fetch_request/migration.sql diff --git a/packages/database/prisma/migrations/20230124224734_remove_retry/migration.sql b/internal-packages/database/prisma/migrations/20230124224734_remove_retry/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230124224734_remove_retry/migration.sql rename to internal-packages/database/prisma/migrations/20230124224734_remove_retry/migration.sql diff --git a/packages/database/prisma/migrations/20230125105732_add_manual_registration_to_external_source/migration.sql b/internal-packages/database/prisma/migrations/20230125105732_add_manual_registration_to_external_source/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230125105732_add_manual_registration_to_external_source/migration.sql rename to internal-packages/database/prisma/migrations/20230125105732_add_manual_registration_to_external_source/migration.sql diff --git a/packages/database/prisma/migrations/20230127121743_add_slack_interaction_trigger_type/migration.sql b/internal-packages/database/prisma/migrations/20230127121743_add_slack_interaction_trigger_type/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230127121743_add_slack_interaction_trigger_type/migration.sql rename to internal-packages/database/prisma/migrations/20230127121743_add_slack_interaction_trigger_type/migration.sql diff --git a/packages/database/prisma/migrations/20230127123917_add_internal_source_model/migration.sql b/internal-packages/database/prisma/migrations/20230127123917_add_internal_source_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230127123917_add_internal_source_model/migration.sql rename to internal-packages/database/prisma/migrations/20230127123917_add_internal_source_model/migration.sql diff --git a/packages/database/prisma/migrations/20230203110022_add_run_once_step_type/migration.sql b/internal-packages/database/prisma/migrations/20230203110022_add_run_once_step_type/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230203110022_add_run_once_step_type/migration.sql rename to internal-packages/database/prisma/migrations/20230203110022_add_run_once_step_type/migration.sql diff --git a/packages/database/prisma/migrations/20230203150225_add_json_schema_to_workflows/migration.sql b/internal-packages/database/prisma/migrations/20230203150225_add_json_schema_to_workflows/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230203150225_add_json_schema_to_workflows/migration.sql rename to internal-packages/database/prisma/migrations/20230203150225_add_json_schema_to_workflows/migration.sql diff --git a/packages/database/prisma/migrations/20230209171333_add_github_app_authorizations/migration.sql b/internal-packages/database/prisma/migrations/20230209171333_add_github_app_authorizations/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230209171333_add_github_app_authorizations/migration.sql rename to internal-packages/database/prisma/migrations/20230209171333_add_github_app_authorizations/migration.sql diff --git a/packages/database/prisma/migrations/20230209171703_add_status_to_github_app_authorization/migration.sql b/internal-packages/database/prisma/migrations/20230209171703_add_status_to_github_app_authorization/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230209171703_add_status_to_github_app_authorization/migration.sql rename to internal-packages/database/prisma/migrations/20230209171703_add_status_to_github_app_authorization/migration.sql diff --git a/packages/database/prisma/migrations/20230209171921_split_an_auth_attempt_with_auth_table/migration.sql b/internal-packages/database/prisma/migrations/20230209171921_split_an_auth_attempt_with_auth_table/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230209171921_split_an_auth_attempt_with_auth_table/migration.sql rename to internal-packages/database/prisma/migrations/20230209171921_split_an_auth_attempt_with_auth_table/migration.sql diff --git a/packages/database/prisma/migrations/20230210111316_add_installaton_details_to_authorization/migration.sql b/internal-packages/database/prisma/migrations/20230210111316_add_installaton_details_to_authorization/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230210111316_add_installaton_details_to_authorization/migration.sql rename to internal-packages/database/prisma/migrations/20230210111316_add_installaton_details_to_authorization/migration.sql diff --git a/packages/database/prisma/migrations/20230210111519_change_access_token_url_column_name/migration.sql b/internal-packages/database/prisma/migrations/20230210111519_change_access_token_url_column_name/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230210111519_change_access_token_url_column_name/migration.sql rename to internal-packages/database/prisma/migrations/20230210111519_change_access_token_url_column_name/migration.sql diff --git a/packages/database/prisma/migrations/20230210141917_add_templates/migration.sql b/internal-packages/database/prisma/migrations/20230210141917_add_templates/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230210141917_add_templates/migration.sql rename to internal-packages/database/prisma/migrations/20230210141917_add_templates/migration.sql diff --git a/packages/database/prisma/migrations/20230210142709_modify_templates_schema/migration.sql b/internal-packages/database/prisma/migrations/20230210142709_modify_templates_schema/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230210142709_modify_templates_schema/migration.sql rename to internal-packages/database/prisma/migrations/20230210142709_modify_templates_schema/migration.sql diff --git a/packages/database/prisma/migrations/20230210150029_add_repo_data_to_org_templates/migration.sql b/internal-packages/database/prisma/migrations/20230210150029_add_repo_data_to_org_templates/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230210150029_add_repo_data_to_org_templates/migration.sql rename to internal-packages/database/prisma/migrations/20230210150029_add_repo_data_to_org_templates/migration.sql diff --git a/packages/database/prisma/migrations/20230210153657_add_template_id_to_auth_attempt/migration.sql b/internal-packages/database/prisma/migrations/20230210153657_add_template_id_to_auth_attempt/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230210153657_add_template_id_to_auth_attempt/migration.sql rename to internal-packages/database/prisma/migrations/20230210153657_add_template_id_to_auth_attempt/migration.sql diff --git a/packages/database/prisma/migrations/20230210175708_add_repository_id_to_org_template/migration.sql b/internal-packages/database/prisma/migrations/20230210175708_add_repository_id_to_org_template/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230210175708_add_repository_id_to_org_template/migration.sql rename to internal-packages/database/prisma/migrations/20230210175708_add_repository_id_to_org_template/migration.sql diff --git a/packages/database/prisma/migrations/20230213113228_add_status_to_org_template/migration.sql b/internal-packages/database/prisma/migrations/20230213113228_add_status_to_org_template/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230213113228_add_status_to_org_template/migration.sql rename to internal-packages/database/prisma/migrations/20230213113228_add_status_to_org_template/migration.sql diff --git a/packages/database/prisma/migrations/20230213142107_add_documentation_to_templates/migration.sql b/internal-packages/database/prisma/migrations/20230213142107_add_documentation_to_templates/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230213142107_add_documentation_to_templates/migration.sql rename to internal-packages/database/prisma/migrations/20230213142107_add_documentation_to_templates/migration.sql diff --git a/packages/database/prisma/migrations/20230213154802_simplify_org_template_status/migration.sql b/internal-packages/database/prisma/migrations/20230213154802_simplify_org_template_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230213154802_simplify_org_template_status/migration.sql rename to internal-packages/database/prisma/migrations/20230213154802_simplify_org_template_status/migration.sql diff --git a/packages/database/prisma/migrations/20230214101645_integration_request_added_version/migration.sql b/internal-packages/database/prisma/migrations/20230214101645_integration_request_added_version/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230214101645_integration_request_added_version/migration.sql rename to internal-packages/database/prisma/migrations/20230214101645_integration_request_added_version/migration.sql diff --git a/packages/database/prisma/migrations/20230214163434_add_run_local_docs_to_templates/migration.sql b/internal-packages/database/prisma/migrations/20230214163434_add_run_local_docs_to_templates/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230214163434_add_run_local_docs_to_templates/migration.sql rename to internal-packages/database/prisma/migrations/20230214163434_add_run_local_docs_to_templates/migration.sql diff --git a/packages/database/prisma/migrations/20230215101806_workflowrunstep_displayproperties/migration.sql b/internal-packages/database/prisma/migrations/20230215101806_workflowrunstep_displayproperties/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230215101806_workflowrunstep_displayproperties/migration.sql rename to internal-packages/database/prisma/migrations/20230215101806_workflowrunstep_displayproperties/migration.sql diff --git a/packages/database/prisma/migrations/20230216100218_add_cloud_waitlist_to_user/migration.sql b/internal-packages/database/prisma/migrations/20230216100218_add_cloud_waitlist_to_user/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230216100218_add_cloud_waitlist_to_user/migration.sql rename to internal-packages/database/prisma/migrations/20230216100218_add_cloud_waitlist_to_user/migration.sql diff --git a/packages/database/prisma/migrations/20230216131220_add_account_type_to_app_auth/migration.sql b/internal-packages/database/prisma/migrations/20230216131220_add_account_type_to_app_auth/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230216131220_add_account_type_to_app_auth/migration.sql rename to internal-packages/database/prisma/migrations/20230216131220_add_account_type_to_app_auth/migration.sql diff --git a/packages/database/prisma/migrations/20230216132133_add_account_name_to_auth/migration.sql b/internal-packages/database/prisma/migrations/20230216132133_add_account_name_to_auth/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230216132133_add_account_name_to_auth/migration.sql rename to internal-packages/database/prisma/migrations/20230216132133_add_account_name_to_auth/migration.sql diff --git a/packages/database/prisma/migrations/20230216184843_add_org_template_to_workflows/migration.sql b/internal-packages/database/prisma/migrations/20230216184843_add_org_template_to_workflows/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230216184843_add_org_template_to_workflows/migration.sql rename to internal-packages/database/prisma/migrations/20230216184843_add_org_template_to_workflows/migration.sql diff --git a/packages/database/prisma/migrations/20230224144605_add_is_live_to_templates/migration.sql b/internal-packages/database/prisma/migrations/20230224144605_add_is_live_to_templates/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230224144605_add_is_live_to_templates/migration.sql rename to internal-packages/database/prisma/migrations/20230224144605_add_is_live_to_templates/migration.sql diff --git a/packages/database/prisma/migrations/20230302104940_add_metadata_to_workflows/migration.sql b/internal-packages/database/prisma/migrations/20230302104940_add_metadata_to_workflows/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230302104940_add_metadata_to_workflows/migration.sql rename to internal-packages/database/prisma/migrations/20230302104940_add_metadata_to_workflows/migration.sql diff --git a/packages/database/prisma/migrations/20230303085314_setup_github_app_models_for_cloud/migration.sql b/internal-packages/database/prisma/migrations/20230303085314_setup_github_app_models_for_cloud/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230303085314_setup_github_app_models_for_cloud/migration.sql rename to internal-packages/database/prisma/migrations/20230303085314_setup_github_app_models_for_cloud/migration.sql diff --git a/packages/database/prisma/migrations/20230303085914_add_redirect_to_auth_attempt/migration.sql b/internal-packages/database/prisma/migrations/20230303085914_add_redirect_to_auth_attempt/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230303085914_add_redirect_to_auth_attempt/migration.sql rename to internal-packages/database/prisma/migrations/20230303085914_add_redirect_to_auth_attempt/migration.sql diff --git a/packages/database/prisma/migrations/20230303092424_remove_oauth_token_columns/migration.sql b/internal-packages/database/prisma/migrations/20230303092424_remove_oauth_token_columns/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230303092424_remove_oauth_token_columns/migration.sql rename to internal-packages/database/prisma/migrations/20230303092424_remove_oauth_token_columns/migration.sql diff --git a/packages/database/prisma/migrations/20230303101108_add_timestamps_to_attempts/migration.sql b/internal-packages/database/prisma/migrations/20230303101108_add_timestamps_to_attempts/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230303101108_add_timestamps_to_attempts/migration.sql rename to internal-packages/database/prisma/migrations/20230303101108_add_timestamps_to_attempts/migration.sql diff --git a/packages/database/prisma/migrations/20230303103808_add_timestamps_to_auths/migration.sql b/internal-packages/database/prisma/migrations/20230303103808_add_timestamps_to_auths/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230303103808_add_timestamps_to_auths/migration.sql rename to internal-packages/database/prisma/migrations/20230303103808_add_timestamps_to_auths/migration.sql diff --git a/packages/database/prisma/migrations/20230303123529_add_token_to_github_app_auth/migration.sql b/internal-packages/database/prisma/migrations/20230303123529_add_token_to_github_app_auth/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230303123529_add_token_to_github_app_auth/migration.sql rename to internal-packages/database/prisma/migrations/20230303123529_add_token_to_github_app_auth/migration.sql diff --git a/packages/database/prisma/migrations/20230303131806_add_authorization_id_to_attempt/migration.sql b/internal-packages/database/prisma/migrations/20230303131806_add_authorization_id_to_attempt/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230303131806_add_authorization_id_to_attempt/migration.sql rename to internal-packages/database/prisma/migrations/20230303131806_add_authorization_id_to_attempt/migration.sql diff --git a/packages/database/prisma/migrations/20230305201223_add_repository_project/migration.sql b/internal-packages/database/prisma/migrations/20230305201223_add_repository_project/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230305201223_add_repository_project/migration.sql rename to internal-packages/database/prisma/migrations/20230305201223_add_repository_project/migration.sql diff --git a/packages/database/prisma/migrations/20230305201615_make_repo_project_name_unique_and_add_org/migration.sql b/internal-packages/database/prisma/migrations/20230305201615_make_repo_project_name_unique_and_add_org/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230305201615_make_repo_project_name_unique_and_add_org/migration.sql rename to internal-packages/database/prisma/migrations/20230305201615_make_repo_project_name_unique_and_add_org/migration.sql diff --git a/packages/database/prisma/migrations/20230306101403_modify_docker_columns/migration.sql b/internal-packages/database/prisma/migrations/20230306101403_modify_docker_columns/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230306101403_modify_docker_columns/migration.sql rename to internal-packages/database/prisma/migrations/20230306101403_modify_docker_columns/migration.sql diff --git a/packages/database/prisma/migrations/20230306101518_remove_docker_columns/migration.sql b/internal-packages/database/prisma/migrations/20230306101518_remove_docker_columns/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230306101518_remove_docker_columns/migration.sql rename to internal-packages/database/prisma/migrations/20230306101518_remove_docker_columns/migration.sql diff --git a/packages/database/prisma/migrations/20230306101927_add_deployments_model/migration.sql b/internal-packages/database/prisma/migrations/20230306101927_add_deployments_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230306101927_add_deployments_model/migration.sql rename to internal-packages/database/prisma/migrations/20230306101927_add_deployments_model/migration.sql diff --git a/packages/database/prisma/migrations/20230306103508_added_status_text_to_projects/migration.sql b/internal-packages/database/prisma/migrations/20230306103508_added_status_text_to_projects/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230306103508_added_status_text_to_projects/migration.sql rename to internal-packages/database/prisma/migrations/20230306103508_added_status_text_to_projects/migration.sql diff --git a/packages/database/prisma/migrations/20230306114828_added_error_to_deployments/migration.sql b/internal-packages/database/prisma/migrations/20230306114828_added_error_to_deployments/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230306114828_added_error_to_deployments/migration.sql rename to internal-packages/database/prisma/migrations/20230306114828_added_error_to_deployments/migration.sql diff --git a/packages/database/prisma/migrations/20230306115630_added_build_duration_to_deployments/migration.sql b/internal-packages/database/prisma/migrations/20230306115630_added_build_duration_to_deployments/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230306115630_added_build_duration_to_deployments/migration.sql rename to internal-packages/database/prisma/migrations/20230306115630_added_build_duration_to_deployments/migration.sql diff --git a/packages/database/prisma/migrations/20230306121727_added_vm_stuff_to_projects/migration.sql b/internal-packages/database/prisma/migrations/20230306121727_added_vm_stuff_to_projects/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230306121727_added_vm_stuff_to_projects/migration.sql rename to internal-packages/database/prisma/migrations/20230306121727_added_vm_stuff_to_projects/migration.sql diff --git a/packages/database/prisma/migrations/20230306122326_added_current_deployment/migration.sql b/internal-packages/database/prisma/migrations/20230306122326_added_current_deployment/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230306122326_added_current_deployment/migration.sql rename to internal-packages/database/prisma/migrations/20230306122326_added_current_deployment/migration.sql diff --git a/packages/database/prisma/migrations/20230306132827_added_more_columns_to_deployment/migration.sql b/internal-packages/database/prisma/migrations/20230306132827_added_more_columns_to_deployment/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230306132827_added_more_columns_to_deployment/migration.sql rename to internal-packages/database/prisma/migrations/20230306132827_added_more_columns_to_deployment/migration.sql diff --git a/packages/database/prisma/migrations/20230306132935_make_build_id_unique/migration.sql b/internal-packages/database/prisma/migrations/20230306132935_make_build_id_unique/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230306132935_make_build_id_unique/migration.sql rename to internal-packages/database/prisma/migrations/20230306132935_make_build_id_unique/migration.sql diff --git a/packages/database/prisma/migrations/20230306140931_added_back_in_docker_columns/migration.sql b/internal-packages/database/prisma/migrations/20230306140931_added_back_in_docker_columns/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230306140931_added_back_in_docker_columns/migration.sql rename to internal-packages/database/prisma/migrations/20230306140931_added_back_in_docker_columns/migration.sql diff --git a/packages/database/prisma/migrations/20230306150053_added_projects_to_workflows/migration.sql b/internal-packages/database/prisma/migrations/20230306150053_added_projects_to_workflows/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230306150053_added_projects_to_workflows/migration.sql rename to internal-packages/database/prisma/migrations/20230306150053_added_projects_to_workflows/migration.sql diff --git a/packages/database/prisma/migrations/20230307094704_added_more_states_to_deployment_status/migration.sql b/internal-packages/database/prisma/migrations/20230307094704_added_more_states_to_deployment_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230307094704_added_more_states_to_deployment_status/migration.sql rename to internal-packages/database/prisma/migrations/20230307094704_added_more_states_to_deployment_status/migration.sql diff --git a/packages/database/prisma/migrations/20230307104034_added_stopping_columns/migration.sql b/internal-packages/database/prisma/migrations/20230307104034_added_stopping_columns/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230307104034_added_stopping_columns/migration.sql rename to internal-packages/database/prisma/migrations/20230307104034_added_stopping_columns/migration.sql diff --git a/packages/database/prisma/migrations/20230307160247_add_version_column_to_deployment/migration.sql b/internal-packages/database/prisma/migrations/20230307160247_add_version_column_to_deployment/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230307160247_add_version_column_to_deployment/migration.sql rename to internal-packages/database/prisma/migrations/20230307160247_add_version_column_to_deployment/migration.sql diff --git a/packages/database/prisma/migrations/20230307161815_add_stopping_status/migration.sql b/internal-packages/database/prisma/migrations/20230307161815_add_stopping_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230307161815_add_stopping_status/migration.sql rename to internal-packages/database/prisma/migrations/20230307161815_add_stopping_status/migration.sql diff --git a/packages/database/prisma/migrations/20230308120746_add_deployment_logs/migration.sql b/internal-packages/database/prisma/migrations/20230308120746_add_deployment_logs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230308120746_add_deployment_logs/migration.sql rename to internal-packages/database/prisma/migrations/20230308120746_add_deployment_logs/migration.sql diff --git a/packages/database/prisma/migrations/20230308151627_add_log_number/migration.sql b/internal-packages/database/prisma/migrations/20230308151627_add_log_number/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230308151627_add_log_number/migration.sql rename to internal-packages/database/prisma/migrations/20230308151627_add_log_number/migration.sql diff --git a/packages/database/prisma/migrations/20230309091702_add_latest_log_dates/migration.sql b/internal-packages/database/prisma/migrations/20230309091702_add_latest_log_dates/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230309091702_add_latest_log_dates/migration.sql rename to internal-packages/database/prisma/migrations/20230309091702_add_latest_log_dates/migration.sql diff --git a/packages/database/prisma/migrations/20230309110029_add_log_polls_model/migration.sql b/internal-packages/database/prisma/migrations/20230309110029_add_log_polls_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230309110029_add_log_polls_model/migration.sql rename to internal-packages/database/prisma/migrations/20230309110029_add_log_polls_model/migration.sql diff --git a/packages/database/prisma/migrations/20230309110318_add_next_poll_scheduled_at/migration.sql b/internal-packages/database/prisma/migrations/20230309110318_add_next_poll_scheduled_at/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230309110318_add_next_poll_scheduled_at/migration.sql rename to internal-packages/database/prisma/migrations/20230309110318_add_next_poll_scheduled_at/migration.sql diff --git a/packages/database/prisma/migrations/20230309110358_make_next_poll_scheduled_at_optional/migration.sql b/internal-packages/database/prisma/migrations/20230309110358_make_next_poll_scheduled_at_optional/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230309110358_make_next_poll_scheduled_at_optional/migration.sql rename to internal-packages/database/prisma/migrations/20230309110358_make_next_poll_scheduled_at_optional/migration.sql diff --git a/packages/database/prisma/migrations/20230309111003_add_created_at_to_polls/migration.sql b/internal-packages/database/prisma/migrations/20230309111003_add_created_at_to_polls/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230309111003_add_created_at_to_polls/migration.sql rename to internal-packages/database/prisma/migrations/20230309111003_add_created_at_to_polls/migration.sql diff --git a/packages/database/prisma/migrations/20230309112008_add_poll_number/migration.sql b/internal-packages/database/prisma/migrations/20230309112008_add_poll_number/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230309112008_add_poll_number/migration.sql rename to internal-packages/database/prisma/migrations/20230309112008_add_poll_number/migration.sql diff --git a/packages/database/prisma/migrations/20230310121352_add_preparing_status/migration.sql b/internal-packages/database/prisma/migrations/20230310121352_add_preparing_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230310121352_add_preparing_status/migration.sql rename to internal-packages/database/prisma/migrations/20230310121352_add_preparing_status/migration.sql diff --git a/packages/database/prisma/migrations/20230312103325_add_key_value_items/migration.sql b/internal-packages/database/prisma/migrations/20230312103325_add_key_value_items/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230312103325_add_key_value_items/migration.sql rename to internal-packages/database/prisma/migrations/20230312103325_add_key_value_items/migration.sql diff --git a/packages/database/prisma/migrations/20230312132000_more_kv_types/migration.sql b/internal-packages/database/prisma/migrations/20230312132000_more_kv_types/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230312132000_more_kv_types/migration.sql rename to internal-packages/database/prisma/migrations/20230312132000_more_kv_types/migration.sql diff --git a/packages/database/prisma/migrations/20230313100030_added_feature_cloud_to_users/migration.sql b/internal-packages/database/prisma/migrations/20230313100030_added_feature_cloud_to_users/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230313100030_added_feature_cloud_to_users/migration.sql rename to internal-packages/database/prisma/migrations/20230313100030_added_feature_cloud_to_users/migration.sql diff --git a/packages/database/prisma/migrations/20230313143913_added_latest_commit_to_projects/migration.sql b/internal-packages/database/prisma/migrations/20230313143913_added_latest_commit_to_projects/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230313143913_added_latest_commit_to_projects/migration.sql rename to internal-packages/database/prisma/migrations/20230313143913_added_latest_commit_to_projects/migration.sql diff --git a/packages/database/prisma/migrations/20230313173750_added_hosted_waitlist_bool/migration.sql b/internal-packages/database/prisma/migrations/20230313173750_added_hosted_waitlist_bool/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230313173750_added_hosted_waitlist_bool/migration.sql rename to internal-packages/database/prisma/migrations/20230313173750_added_hosted_waitlist_bool/migration.sql diff --git a/packages/database/prisma/migrations/20230322093004_add_current_environments/migration.sql b/internal-packages/database/prisma/migrations/20230322093004_add_current_environments/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230322093004_add_current_environments/migration.sql rename to internal-packages/database/prisma/migrations/20230322093004_add_current_environments/migration.sql diff --git a/packages/database/prisma/migrations/20230328125714_add_endpoints_model/migration.sql b/internal-packages/database/prisma/migrations/20230328125714_add_endpoints_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230328125714_add_endpoints_model/migration.sql rename to internal-packages/database/prisma/migrations/20230328125714_add_endpoints_model/migration.sql diff --git a/packages/database/prisma/migrations/20230328130438_remove_creator_from_endpoints/migration.sql b/internal-packages/database/prisma/migrations/20230328130438_remove_creator_from_endpoints/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230328130438_remove_creator_from_endpoints/migration.sql rename to internal-packages/database/prisma/migrations/20230328130438_remove_creator_from_endpoints/migration.sql diff --git a/packages/database/prisma/migrations/20230329120440_add_job_models/migration.sql b/internal-packages/database/prisma/migrations/20230329120440_add_job_models/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230329120440_add_job_models/migration.sql rename to internal-packages/database/prisma/migrations/20230329120440_add_job_models/migration.sql diff --git a/packages/database/prisma/migrations/20230329121955_add_trigger_json_to_job_version/migration.sql b/internal-packages/database/prisma/migrations/20230329121955_add_trigger_json_to_job_version/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230329121955_add_trigger_json_to_job_version/migration.sql rename to internal-packages/database/prisma/migrations/20230329121955_add_trigger_json_to_job_version/migration.sql diff --git a/packages/database/prisma/migrations/20230329123037_simplified_job_models/migration.sql b/internal-packages/database/prisma/migrations/20230329123037_simplified_job_models/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230329123037_simplified_job_models/migration.sql rename to internal-packages/database/prisma/migrations/20230329123037_simplified_job_models/migration.sql diff --git a/packages/database/prisma/migrations/20230329125135_create_event_log_model/migration.sql b/internal-packages/database/prisma/migrations/20230329125135_create_event_log_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230329125135_create_event_log_model/migration.sql rename to internal-packages/database/prisma/migrations/20230329125135_create_event_log_model/migration.sql diff --git a/packages/database/prisma/migrations/20230329131748_add_deliver_at_to_event_logs/migration.sql b/internal-packages/database/prisma/migrations/20230329131748_add_deliver_at_to_event_logs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230329131748_add_deliver_at_to_event_logs/migration.sql rename to internal-packages/database/prisma/migrations/20230329131748_add_deliver_at_to_event_logs/migration.sql diff --git a/packages/database/prisma/migrations/20230329132829_changed_dispatched_at_to_delivered_at/migration.sql b/internal-packages/database/prisma/migrations/20230329132829_changed_dispatched_at_to_delivered_at/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230329132829_changed_dispatched_at_to_delivered_at/migration.sql rename to internal-packages/database/prisma/migrations/20230329132829_changed_dispatched_at_to_delivered_at/migration.sql diff --git a/packages/database/prisma/migrations/20230329133809_rename_endpoint_name_to_slug/migration.sql b/internal-packages/database/prisma/migrations/20230329133809_rename_endpoint_name_to_slug/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230329133809_rename_endpoint_name_to_slug/migration.sql rename to internal-packages/database/prisma/migrations/20230329133809_rename_endpoint_name_to_slug/migration.sql diff --git a/packages/database/prisma/migrations/20230329154731_add_executions/migration.sql b/internal-packages/database/prisma/migrations/20230329154731_add_executions/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230329154731_add_executions/migration.sql rename to internal-packages/database/prisma/migrations/20230329154731_add_executions/migration.sql diff --git a/packages/database/prisma/migrations/20230330115131_add_tasjs/migration.sql b/internal-packages/database/prisma/migrations/20230330115131_add_tasjs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230330115131_add_tasjs/migration.sql rename to internal-packages/database/prisma/migrations/20230330115131_add_tasjs/migration.sql diff --git a/packages/database/prisma/migrations/20230330123046_add_waiting_status/migration.sql b/internal-packages/database/prisma/migrations/20230330123046_add_waiting_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230330123046_add_waiting_status/migration.sql rename to internal-packages/database/prisma/migrations/20230330123046_add_waiting_status/migration.sql diff --git a/packages/database/prisma/migrations/20230330125238_rename_finished_at/migration.sql b/internal-packages/database/prisma/migrations/20230330125238_rename_finished_at/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230330125238_rename_finished_at/migration.sql rename to internal-packages/database/prisma/migrations/20230330125238_rename_finished_at/migration.sql diff --git a/packages/database/prisma/migrations/20230330125349_rename_finished_at_on_executions/migration.sql b/internal-packages/database/prisma/migrations/20230330125349_rename_finished_at_on_executions/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230330125349_rename_finished_at_on_executions/migration.sql rename to internal-packages/database/prisma/migrations/20230330125349_rename_finished_at_on_executions/migration.sql diff --git a/packages/database/prisma/migrations/20230330132602_use_string_instead_of_big_int/migration.sql b/internal-packages/database/prisma/migrations/20230330132602_use_string_instead_of_big_int/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230330132602_use_string_instead_of_big_int/migration.sql rename to internal-packages/database/prisma/migrations/20230330132602_use_string_instead_of_big_int/migration.sql diff --git a/packages/database/prisma/migrations/20230331091517_add_noop_to_tasks/migration.sql b/internal-packages/database/prisma/migrations/20230331091517_add_noop_to_tasks/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230331091517_add_noop_to_tasks/migration.sql rename to internal-packages/database/prisma/migrations/20230331091517_add_noop_to_tasks/migration.sql diff --git a/packages/database/prisma/migrations/20230331150231_use_ulid_instead_of_cuid_and_ts_for_tasks/migration.sql b/internal-packages/database/prisma/migrations/20230331150231_use_ulid_instead_of_cuid_and_ts_for_tasks/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230331150231_use_ulid_instead_of_cuid_and_ts_for_tasks/migration.sql rename to internal-packages/database/prisma/migrations/20230331150231_use_ulid_instead_of_cuid_and_ts_for_tasks/migration.sql diff --git a/packages/database/prisma/migrations/20230404094247_add_key_and_icon_to_tasks/migration.sql b/internal-packages/database/prisma/migrations/20230404094247_add_key_and_icon_to_tasks/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230404094247_add_key_and_icon_to_tasks/migration.sql rename to internal-packages/database/prisma/migrations/20230404094247_add_key_and_icon_to_tasks/migration.sql diff --git a/packages/database/prisma/migrations/20230404094406_make_display_key_optional/migration.sql b/internal-packages/database/prisma/migrations/20230404094406_make_display_key_optional/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230404094406_make_display_key_optional/migration.sql rename to internal-packages/database/prisma/migrations/20230404094406_make_display_key_optional/migration.sql diff --git a/packages/database/prisma/migrations/20230404095457_rename_display_properties_to_elements/migration.sql b/internal-packages/database/prisma/migrations/20230404095457_rename_display_properties_to_elements/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230404095457_rename_display_properties_to_elements/migration.sql rename to internal-packages/database/prisma/migrations/20230404095457_rename_display_properties_to_elements/migration.sql diff --git a/packages/database/prisma/migrations/20230404101137_add_elements_to_executions/migration.sql b/internal-packages/database/prisma/migrations/20230404101137_add_elements_to_executions/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230404101137_add_elements_to_executions/migration.sql rename to internal-packages/database/prisma/migrations/20230404101137_add_elements_to_executions/migration.sql diff --git a/packages/database/prisma/migrations/20230404203551_remove_source_from_events/migration.sql b/internal-packages/database/prisma/migrations/20230404203551_remove_source_from_events/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230404203551_remove_source_from_events/migration.sql rename to internal-packages/database/prisma/migrations/20230404203551_remove_source_from_events/migration.sql diff --git a/packages/database/prisma/migrations/20230406092923_add_job_connections/migration.sql b/internal-packages/database/prisma/migrations/20230406092923_add_job_connections/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230406092923_add_job_connections/migration.sql rename to internal-packages/database/prisma/migrations/20230406092923_add_job_connections/migration.sql diff --git a/packages/database/prisma/migrations/20230418092025_add_uses_local_auth/migration.sql b/internal-packages/database/prisma/migrations/20230418092025_add_uses_local_auth/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230418092025_add_uses_local_auth/migration.sql rename to internal-packages/database/prisma/migrations/20230418092025_add_uses_local_auth/migration.sql diff --git a/packages/database/prisma/migrations/20230419140902_apiconnection_secrets/migration.sql b/internal-packages/database/prisma/migrations/20230419140902_apiconnection_secrets/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230419140902_apiconnection_secrets/migration.sql rename to internal-packages/database/prisma/migrations/20230419140902_apiconnection_secrets/migration.sql diff --git a/packages/database/prisma/migrations/20230419141305_add_http_sources/migration.sql b/internal-packages/database/prisma/migrations/20230419141305_add_http_sources/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230419141305_add_http_sources/migration.sql rename to internal-packages/database/prisma/migrations/20230419141305_add_http_sources/migration.sql diff --git a/packages/database/prisma/migrations/20230419141524_make_webhooks_specific_to_an_endpoint/migration.sql b/internal-packages/database/prisma/migrations/20230419141524_make_webhooks_specific_to_an_endpoint/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230419141524_make_webhooks_specific_to_an_endpoint/migration.sql rename to internal-packages/database/prisma/migrations/20230419141524_make_webhooks_specific_to_an_endpoint/migration.sql diff --git a/packages/database/prisma/migrations/20230419141651_add_timestamps_to_http_sources/migration.sql b/internal-packages/database/prisma/migrations/20230419141651_add_timestamps_to_http_sources/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230419141651_add_timestamps_to_http_sources/migration.sql rename to internal-packages/database/prisma/migrations/20230419141651_add_timestamps_to_http_sources/migration.sql diff --git a/packages/database/prisma/migrations/20230419155523_api_connection_attempt/migration.sql b/internal-packages/database/prisma/migrations/20230419155523_api_connection_attempt/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230419155523_api_connection_attempt/migration.sql rename to internal-packages/database/prisma/migrations/20230419155523_api_connection_attempt/migration.sql diff --git a/packages/database/prisma/migrations/20230419175107_remove_status_from_http_source/migration.sql b/internal-packages/database/prisma/migrations/20230419175107_remove_status_from_http_source/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230419175107_remove_status_from_http_source/migration.sql rename to internal-packages/database/prisma/migrations/20230419175107_remove_status_from_http_source/migration.sql diff --git a/packages/database/prisma/migrations/20230419183033_add_ready_to_job_instance/migration.sql b/internal-packages/database/prisma/migrations/20230419183033_add_ready_to_job_instance/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230419183033_add_ready_to_job_instance/migration.sql rename to internal-packages/database/prisma/migrations/20230419183033_add_ready_to_job_instance/migration.sql diff --git a/packages/database/prisma/migrations/20230421162946_add_interactive_to_http_source/migration.sql b/internal-packages/database/prisma/migrations/20230421162946_add_interactive_to_http_source/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230421162946_add_interactive_to_http_source/migration.sql rename to internal-packages/database/prisma/migrations/20230421162946_add_interactive_to_http_source/migration.sql diff --git a/packages/database/prisma/migrations/20230421164041_add_webhook_deliveries/migration.sql b/internal-packages/database/prisma/migrations/20230421164041_add_webhook_deliveries/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230421164041_add_webhook_deliveries/migration.sql rename to internal-packages/database/prisma/migrations/20230421164041_add_webhook_deliveries/migration.sql diff --git a/packages/database/prisma/migrations/20230421164059_remove_source_status/migration.sql b/internal-packages/database/prisma/migrations/20230421164059_remove_source_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230421164059_remove_source_status/migration.sql rename to internal-packages/database/prisma/migrations/20230421164059_remove_source_status/migration.sql diff --git a/packages/database/prisma/migrations/20230421171344_add_connection_to_http_sources/migration.sql b/internal-packages/database/prisma/migrations/20230421171344_add_connection_to_http_sources/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230421171344_add_connection_to_http_sources/migration.sql rename to internal-packages/database/prisma/migrations/20230421171344_add_connection_to_http_sources/migration.sql diff --git a/packages/database/prisma/migrations/20230423183231_api_connection_attempt_title/migration.sql b/internal-packages/database/prisma/migrations/20230423183231_api_connection_attempt_title/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230423183231_api_connection_attempt_title/migration.sql rename to internal-packages/database/prisma/migrations/20230423183231_api_connection_attempt_title/migration.sql diff --git a/packages/database/prisma/migrations/20230425094302_add_job_event_rule/migration.sql b/internal-packages/database/prisma/migrations/20230425094302_add_job_event_rule/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230425094302_add_job_event_rule/migration.sql rename to internal-packages/database/prisma/migrations/20230425094302_add_job_event_rule/migration.sql diff --git a/packages/database/prisma/migrations/20230425094839_only_one_event_rule_per_job_instance/migration.sql b/internal-packages/database/prisma/migrations/20230425094839_only_one_event_rule_per_job_instance/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230425094839_only_one_event_rule_per_job_instance/migration.sql rename to internal-packages/database/prisma/migrations/20230425094839_only_one_event_rule_per_job_instance/migration.sql diff --git a/packages/database/prisma/migrations/20230425103833_add_job_alias_model/migration.sql b/internal-packages/database/prisma/migrations/20230425103833_add_job_alias_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230425103833_add_job_alias_model/migration.sql rename to internal-packages/database/prisma/migrations/20230425103833_add_job_alias_model/migration.sql diff --git a/packages/database/prisma/migrations/20230425104022_add_env_to_job_alias/migration.sql b/internal-packages/database/prisma/migrations/20230425104022_add_env_to_job_alias/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230425104022_add_env_to_job_alias/migration.sql rename to internal-packages/database/prisma/migrations/20230425104022_add_env_to_job_alias/migration.sql diff --git a/packages/database/prisma/migrations/20230425104911_add_version_to_job_alias/migration.sql b/internal-packages/database/prisma/migrations/20230425104911_add_version_to_job_alias/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230425104911_add_version_to_job_alias/migration.sql rename to internal-packages/database/prisma/migrations/20230425104911_add_version_to_job_alias/migration.sql diff --git a/packages/database/prisma/migrations/20230425122235_add_resuming_tasks_to_event_rules/migration.sql b/internal-packages/database/prisma/migrations/20230425122235_add_resuming_tasks_to_event_rules/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230425122235_add_resuming_tasks_to_event_rules/migration.sql rename to internal-packages/database/prisma/migrations/20230425122235_add_resuming_tasks_to_event_rules/migration.sql diff --git a/packages/database/prisma/migrations/20230425133327_make_task_event_rules_one_to_one/migration.sql b/internal-packages/database/prisma/migrations/20230425133327_make_task_event_rules_one_to_one/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230425133327_make_task_event_rules_one_to_one/migration.sql rename to internal-packages/database/prisma/migrations/20230425133327_make_task_event_rules_one_to_one/migration.sql diff --git a/packages/database/prisma/migrations/20230425162308_can_have_more_than_one_event_rule_on_an_instance/migration.sql b/internal-packages/database/prisma/migrations/20230425162308_can_have_more_than_one_event_rule_on_an_instance/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230425162308_can_have_more_than_one_event_rule_on_an_instance/migration.sql rename to internal-packages/database/prisma/migrations/20230425162308_can_have_more_than_one_event_rule_on_an_instance/migration.sql diff --git a/packages/database/prisma/migrations/20230425162613_add_action_identifier_to_event_rule/migration.sql b/internal-packages/database/prisma/migrations/20230425162613_add_action_identifier_to_event_rule/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230425162613_add_action_identifier_to_event_rule/migration.sql rename to internal-packages/database/prisma/migrations/20230425162613_add_action_identifier_to_event_rule/migration.sql diff --git a/packages/database/prisma/migrations/20230425162923_remove_uniq_on_action_identifier/migration.sql b/internal-packages/database/prisma/migrations/20230425162923_remove_uniq_on_action_identifier/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230425162923_remove_uniq_on_action_identifier/migration.sql rename to internal-packages/database/prisma/migrations/20230425162923_remove_uniq_on_action_identifier/migration.sql diff --git a/packages/database/prisma/migrations/20230426093630_api_connection_attempt_added_security_code/migration.sql b/internal-packages/database/prisma/migrations/20230426093630_api_connection_attempt_added_security_code/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230426093630_api_connection_attempt_added_security_code/migration.sql rename to internal-packages/database/prisma/migrations/20230426093630_api_connection_attempt_added_security_code/migration.sql diff --git a/packages/database/prisma/migrations/20230426150713_add_support_for_projects/migration.sql b/internal-packages/database/prisma/migrations/20230426150713_add_support_for_projects/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230426150713_add_support_for_projects/migration.sql rename to internal-packages/database/prisma/migrations/20230426150713_add_support_for_projects/migration.sql diff --git a/packages/database/prisma/migrations/20230426151056_users_are_now_org_members/migration.sql b/internal-packages/database/prisma/migrations/20230426151056_users_are_now_org_members/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230426151056_users_are_now_org_members/migration.sql rename to internal-packages/database/prisma/migrations/20230426151056_users_are_now_org_members/migration.sql diff --git a/packages/database/prisma/migrations/20230426162129_scope_jobs_to_projects/migration.sql b/internal-packages/database/prisma/migrations/20230426162129_scope_jobs_to_projects/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230426162129_scope_jobs_to_projects/migration.sql rename to internal-packages/database/prisma/migrations/20230426162129_scope_jobs_to_projects/migration.sql diff --git a/packages/database/prisma/migrations/20230426182130_api_connection_expires_at/migration.sql b/internal-packages/database/prisma/migrations/20230426182130_api_connection_expires_at/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230426182130_api_connection_expires_at/migration.sql rename to internal-packages/database/prisma/migrations/20230426182130_api_connection_expires_at/migration.sql diff --git a/packages/database/prisma/migrations/20230427131511_add_subtasks/migration.sql b/internal-packages/database/prisma/migrations/20230427131511_add_subtasks/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230427131511_add_subtasks/migration.sql rename to internal-packages/database/prisma/migrations/20230427131511_add_subtasks/migration.sql diff --git a/packages/database/prisma/migrations/20230428092036_add_slug_to_api_connections/migration.sql b/internal-packages/database/prisma/migrations/20230428092036_add_slug_to_api_connections/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230428092036_add_slug_to_api_connections/migration.sql rename to internal-packages/database/prisma/migrations/20230428092036_add_slug_to_api_connections/migration.sql diff --git a/packages/database/prisma/migrations/20230428092422_rename_api_connection_models/migration.sql b/internal-packages/database/prisma/migrations/20230428092422_rename_api_connection_models/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230428092422_rename_api_connection_models/migration.sql rename to internal-packages/database/prisma/migrations/20230428092422_rename_api_connection_models/migration.sql diff --git a/packages/database/prisma/migrations/20230428135439_secret_reference_key_unique/migration.sql b/internal-packages/database/prisma/migrations/20230428135439_secret_reference_key_unique/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230428135439_secret_reference_key_unique/migration.sql rename to internal-packages/database/prisma/migrations/20230428135439_secret_reference_key_unique/migration.sql diff --git a/packages/database/prisma/migrations/20230428141846_secret_reference_multiple_api_connections/migration.sql b/internal-packages/database/prisma/migrations/20230428141846_secret_reference_multiple_api_connections/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230428141846_secret_reference_multiple_api_connections/migration.sql rename to internal-packages/database/prisma/migrations/20230428141846_secret_reference_multiple_api_connections/migration.sql diff --git a/packages/database/prisma/migrations/20230503094127_add_trigger_variants/migration.sql b/internal-packages/database/prisma/migrations/20230503094127_add_trigger_variants/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230503094127_add_trigger_variants/migration.sql rename to internal-packages/database/prisma/migrations/20230503094127_add_trigger_variants/migration.sql diff --git a/packages/database/prisma/migrations/20230503113400_add_slug_to_trigger_variants/migration.sql b/internal-packages/database/prisma/migrations/20230503113400_add_slug_to_trigger_variants/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230503113400_add_slug_to_trigger_variants/migration.sql rename to internal-packages/database/prisma/migrations/20230503113400_add_slug_to_trigger_variants/migration.sql diff --git a/packages/database/prisma/migrations/20230503172156_rename_create_execution_to_create_run/migration.sql b/internal-packages/database/prisma/migrations/20230503172156_rename_create_execution_to_create_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230503172156_rename_create_execution_to_create_run/migration.sql rename to internal-packages/database/prisma/migrations/20230503172156_rename_create_execution_to_create_run/migration.sql diff --git a/packages/database/prisma/migrations/20230504090710_add_shadow_to_job/migration.sql b/internal-packages/database/prisma/migrations/20230504090710_add_shadow_to_job/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230504090710_add_shadow_to_job/migration.sql rename to internal-packages/database/prisma/migrations/20230504090710_add_shadow_to_job/migration.sql diff --git a/packages/database/prisma/migrations/20230504152611_rename_shadow_to_internal/migration.sql b/internal-packages/database/prisma/migrations/20230504152611_rename_shadow_to_internal/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230504152611_rename_shadow_to_internal/migration.sql rename to internal-packages/database/prisma/migrations/20230504152611_rename_shadow_to_internal/migration.sql diff --git a/packages/database/prisma/migrations/20230504200916_add_redact_to_tasks/migration.sql b/internal-packages/database/prisma/migrations/20230504200916_add_redact_to_tasks/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230504200916_add_redact_to_tasks/migration.sql rename to internal-packages/database/prisma/migrations/20230504200916_add_redact_to_tasks/migration.sql diff --git a/packages/database/prisma/migrations/20230505085501_add_queue_columns_to_job/migration.sql b/internal-packages/database/prisma/migrations/20230505085501_add_queue_columns_to_job/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230505085501_add_queue_columns_to_job/migration.sql rename to internal-packages/database/prisma/migrations/20230505085501_add_queue_columns_to_job/migration.sql diff --git a/packages/database/prisma/migrations/20230505085546_move_queue_columns_to_job_instance/migration.sql b/internal-packages/database/prisma/migrations/20230505085546_move_queue_columns_to_job_instance/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230505085546_move_queue_columns_to_job_instance/migration.sql rename to internal-packages/database/prisma/migrations/20230505085546_move_queue_columns_to_job_instance/migration.sql diff --git a/packages/database/prisma/migrations/20230505085931_add_queued_job_run_status/migration.sql b/internal-packages/database/prisma/migrations/20230505085931_add_queued_job_run_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230505085931_add_queued_job_run_status/migration.sql rename to internal-packages/database/prisma/migrations/20230505085931_add_queued_job_run_status/migration.sql diff --git a/packages/database/prisma/migrations/20230505091221_create_queue_model/migration.sql b/internal-packages/database/prisma/migrations/20230505091221_create_queue_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230505091221_create_queue_model/migration.sql rename to internal-packages/database/prisma/migrations/20230505091221_create_queue_model/migration.sql diff --git a/packages/database/prisma/migrations/20230505092917_add_queued_at_to_job_runs/migration.sql b/internal-packages/database/prisma/migrations/20230505092917_add_queued_at_to_job_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230505092917_add_queued_at_to_job_runs/migration.sql rename to internal-packages/database/prisma/migrations/20230505092917_add_queued_at_to_job_runs/migration.sql diff --git a/packages/database/prisma/migrations/20230511101816_remove_job_trigger_variants/migration.sql b/internal-packages/database/prisma/migrations/20230511101816_remove_job_trigger_variants/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230511101816_remove_job_trigger_variants/migration.sql rename to internal-packages/database/prisma/migrations/20230511101816_remove_job_trigger_variants/migration.sql diff --git a/packages/database/prisma/migrations/20230512085413_schema_redesign_for_new_system/migration.sql b/internal-packages/database/prisma/migrations/20230512085413_schema_redesign_for_new_system/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230512085413_schema_redesign_for_new_system/migration.sql rename to internal-packages/database/prisma/migrations/20230512085413_schema_redesign_for_new_system/migration.sql diff --git a/packages/database/prisma/migrations/20230512123150_add_dynamic_triggers/migration.sql b/internal-packages/database/prisma/migrations/20230512123150_add_dynamic_triggers/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230512123150_add_dynamic_triggers/migration.sql rename to internal-packages/database/prisma/migrations/20230512123150_add_dynamic_triggers/migration.sql diff --git a/packages/database/prisma/migrations/20230512145548_add_run_connections/migration.sql b/internal-packages/database/prisma/migrations/20230512145548_add_run_connections/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230512145548_add_run_connections/migration.sql rename to internal-packages/database/prisma/migrations/20230512145548_add_run_connections/migration.sql diff --git a/packages/database/prisma/migrations/20230512162858_add_start_position/migration.sql b/internal-packages/database/prisma/migrations/20230512162858_add_start_position/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230512162858_add_start_position/migration.sql rename to internal-packages/database/prisma/migrations/20230512162858_add_start_position/migration.sql diff --git a/packages/database/prisma/migrations/20230512163048_move_start_position/migration.sql b/internal-packages/database/prisma/migrations/20230512163048_move_start_position/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230512163048_move_start_position/migration.sql rename to internal-packages/database/prisma/migrations/20230512163048_move_start_position/migration.sql diff --git a/packages/database/prisma/migrations/20230515101628_add_prepare_to_job_version/migration.sql b/internal-packages/database/prisma/migrations/20230515101628_add_prepare_to_job_version/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230515101628_add_prepare_to_job_version/migration.sql rename to internal-packages/database/prisma/migrations/20230515101628_add_prepare_to_job_version/migration.sql diff --git a/packages/database/prisma/migrations/20230515102310_remove_prepare_from_job_version/migration.sql b/internal-packages/database/prisma/migrations/20230515102310_remove_prepare_from_job_version/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230515102310_remove_prepare_from_job_version/migration.sql rename to internal-packages/database/prisma/migrations/20230515102310_remove_prepare_from_job_version/migration.sql diff --git a/packages/database/prisma/migrations/20230515110730_add_back_prepare_to_job_version/migration.sql b/internal-packages/database/prisma/migrations/20230515110730_add_back_prepare_to_job_version/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230515110730_add_back_prepare_to_job_version/migration.sql rename to internal-packages/database/prisma/migrations/20230515110730_add_back_prepare_to_job_version/migration.sql diff --git a/packages/database/prisma/migrations/20230515111507_add_prepared_to_job_version/migration.sql b/internal-packages/database/prisma/migrations/20230515111507_add_prepared_to_job_version/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230515111507_add_prepared_to_job_version/migration.sql rename to internal-packages/database/prisma/migrations/20230515111507_add_prepared_to_job_version/migration.sql diff --git a/packages/database/prisma/migrations/20230515151445_add_trigger_sources/migration.sql b/internal-packages/database/prisma/migrations/20230515151445_add_trigger_sources/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230515151445_add_trigger_sources/migration.sql rename to internal-packages/database/prisma/migrations/20230515151445_add_trigger_sources/migration.sql diff --git a/packages/database/prisma/migrations/20230515152217_move_params_to_trigger_sources/migration.sql b/internal-packages/database/prisma/migrations/20230515152217_move_params_to_trigger_sources/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230515152217_move_params_to_trigger_sources/migration.sql rename to internal-packages/database/prisma/migrations/20230515152217_move_params_to_trigger_sources/migration.sql diff --git a/packages/database/prisma/migrations/20230516121549_remove_http_source_table/migration.sql b/internal-packages/database/prisma/migrations/20230516121549_remove_http_source_table/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230516121549_remove_http_source_table/migration.sql rename to internal-packages/database/prisma/migrations/20230516121549_remove_http_source_table/migration.sql diff --git a/packages/database/prisma/migrations/20230516122003_add_secret_reference_to_trigger_source/migration.sql b/internal-packages/database/prisma/migrations/20230516122003_add_secret_reference_to_trigger_source/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230516122003_add_secret_reference_to_trigger_source/migration.sql rename to internal-packages/database/prisma/migrations/20230516122003_add_secret_reference_to_trigger_source/migration.sql diff --git a/packages/database/prisma/migrations/20230516154239_add_integration_identifier_to_clients/migration.sql b/internal-packages/database/prisma/migrations/20230516154239_add_integration_identifier_to_clients/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230516154239_add_integration_identifier_to_clients/migration.sql rename to internal-packages/database/prisma/migrations/20230516154239_add_integration_identifier_to_clients/migration.sql diff --git a/packages/database/prisma/migrations/20230516155545_add_integration_auth_method/migration.sql b/internal-packages/database/prisma/migrations/20230516155545_add_integration_auth_method/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230516155545_add_integration_auth_method/migration.sql rename to internal-packages/database/prisma/migrations/20230516155545_add_integration_auth_method/migration.sql diff --git a/packages/database/prisma/migrations/20230516163116_add_scopes_to_connections/migration.sql b/internal-packages/database/prisma/migrations/20230516163116_add_scopes_to_connections/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230516163116_add_scopes_to_connections/migration.sql rename to internal-packages/database/prisma/migrations/20230516163116_add_scopes_to_connections/migration.sql diff --git a/packages/database/prisma/migrations/20230516171911_add_description_to_clients/migration.sql b/internal-packages/database/prisma/migrations/20230516171911_add_description_to_clients/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230516171911_add_description_to_clients/migration.sql rename to internal-packages/database/prisma/migrations/20230516171911_add_description_to_clients/migration.sql diff --git a/packages/database/prisma/migrations/20230517105559_rename_job_connections_to_job_integrations/migration.sql b/internal-packages/database/prisma/migrations/20230517105559_rename_job_connections_to_job_integrations/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230517105559_rename_job_connections_to_job_integrations/migration.sql rename to internal-packages/database/prisma/migrations/20230517105559_rename_job_connections_to_job_integrations/migration.sql diff --git a/packages/database/prisma/migrations/20230517105823_rename_connection_metadata/migration.sql b/internal-packages/database/prisma/migrations/20230517105823_rename_connection_metadata/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230517105823_rename_connection_metadata/migration.sql rename to internal-packages/database/prisma/migrations/20230517105823_rename_connection_metadata/migration.sql diff --git a/packages/database/prisma/migrations/20230517153909_add_dynamic_trigger_to_trigger_sources/migration.sql b/internal-packages/database/prisma/migrations/20230517153909_add_dynamic_trigger_to_trigger_sources/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230517153909_add_dynamic_trigger_to_trigger_sources/migration.sql rename to internal-packages/database/prisma/migrations/20230517153909_add_dynamic_trigger_to_trigger_sources/migration.sql diff --git a/packages/database/prisma/migrations/20230517154918_add_dynamic_trigger_to_job_triggers/migration.sql b/internal-packages/database/prisma/migrations/20230517154918_add_dynamic_trigger_to_job_triggers/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230517154918_add_dynamic_trigger_to_job_triggers/migration.sql rename to internal-packages/database/prisma/migrations/20230517154918_add_dynamic_trigger_to_job_triggers/migration.sql diff --git a/packages/database/prisma/migrations/20230518134021_polymorphic_event_dispatching/migration.sql b/internal-packages/database/prisma/migrations/20230518134021_polymorphic_event_dispatching/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230518134021_polymorphic_event_dispatching/migration.sql rename to internal-packages/database/prisma/migrations/20230518134021_polymorphic_event_dispatching/migration.sql diff --git a/packages/database/prisma/migrations/20230519133020_add_dynamic_registrations/migration.sql b/internal-packages/database/prisma/migrations/20230519133020_add_dynamic_registrations/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230519133020_add_dynamic_registrations/migration.sql rename to internal-packages/database/prisma/migrations/20230519133020_add_dynamic_registrations/migration.sql diff --git a/packages/database/prisma/migrations/20230522085325_make_secret_reference_provider_an_enum/migration.sql b/internal-packages/database/prisma/migrations/20230522085325_make_secret_reference_provider_an_enum/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230522085325_make_secret_reference_provider_an_enum/migration.sql rename to internal-packages/database/prisma/migrations/20230522085325_make_secret_reference_provider_an_enum/migration.sql diff --git a/packages/database/prisma/migrations/20230522114830_add_manual_to_event_dispatcher/migration.sql b/internal-packages/database/prisma/migrations/20230522114830_add_manual_to_event_dispatcher/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230522114830_add_manual_to_event_dispatcher/migration.sql rename to internal-packages/database/prisma/migrations/20230522114830_add_manual_to_event_dispatcher/migration.sql diff --git a/packages/database/prisma/migrations/20230522131952_create_schedule_sources_model/migration.sql b/internal-packages/database/prisma/migrations/20230522131952_create_schedule_sources_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230522131952_create_schedule_sources_model/migration.sql rename to internal-packages/database/prisma/migrations/20230522131952_create_schedule_sources_model/migration.sql diff --git a/packages/database/prisma/migrations/20230522140136_added_event_id_to_event_record/migration.sql b/internal-packages/database/prisma/migrations/20230522140136_added_event_id_to_event_record/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230522140136_added_event_id_to_event_record/migration.sql rename to internal-packages/database/prisma/migrations/20230522140136_added_event_id_to_event_record/migration.sql diff --git a/packages/database/prisma/migrations/20230523132431_add_type_to_dynamic_triggers/migration.sql b/internal-packages/database/prisma/migrations/20230523132431_add_type_to_dynamic_triggers/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230523132431_add_type_to_dynamic_triggers/migration.sql rename to internal-packages/database/prisma/migrations/20230523132431_add_type_to_dynamic_triggers/migration.sql diff --git a/packages/database/prisma/migrations/20230523135129_add_metadata_to_schedules/migration.sql b/internal-packages/database/prisma/migrations/20230523135129_add_metadata_to_schedules/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230523135129_add_metadata_to_schedules/migration.sql rename to internal-packages/database/prisma/migrations/20230523135129_add_metadata_to_schedules/migration.sql diff --git a/packages/database/prisma/migrations/20230525103409_add_env_to_external_accounts/migration.sql b/internal-packages/database/prisma/migrations/20230525103409_add_env_to_external_accounts/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230525103409_add_env_to_external_accounts/migration.sql rename to internal-packages/database/prisma/migrations/20230525103409_add_env_to_external_accounts/migration.sql diff --git a/packages/database/prisma/migrations/20230525110458_add_external_account_to_events/migration.sql b/internal-packages/database/prisma/migrations/20230525110458_add_external_account_to_events/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230525110458_add_external_account_to_events/migration.sql rename to internal-packages/database/prisma/migrations/20230525110458_add_external_account_to_events/migration.sql diff --git a/packages/database/prisma/migrations/20230525110838_add_external_account_to_runs/migration.sql b/internal-packages/database/prisma/migrations/20230525110838_add_external_account_to_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230525110838_add_external_account_to_runs/migration.sql rename to internal-packages/database/prisma/migrations/20230525110838_add_external_account_to_runs/migration.sql diff --git a/packages/database/prisma/migrations/20230525111943_project_slug_added/migration.sql b/internal-packages/database/prisma/migrations/20230525111943_project_slug_added/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230525111943_project_slug_added/migration.sql rename to internal-packages/database/prisma/migrations/20230525111943_project_slug_added/migration.sql diff --git a/packages/database/prisma/migrations/20230525123241_add_external_account_to_schedule_sources/migration.sql b/internal-packages/database/prisma/migrations/20230525123241_add_external_account_to_schedule_sources/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230525123241_add_external_account_to_schedule_sources/migration.sql rename to internal-packages/database/prisma/migrations/20230525123241_add_external_account_to_schedule_sources/migration.sql diff --git a/packages/database/prisma/migrations/20230525124804_add_external_account_to_trigger_sources/migration.sql b/internal-packages/database/prisma/migrations/20230525124804_add_external_account_to_trigger_sources/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230525124804_add_external_account_to_trigger_sources/migration.sql rename to internal-packages/database/prisma/migrations/20230525124804_add_external_account_to_trigger_sources/migration.sql diff --git a/packages/database/prisma/migrations/20230525134604_add_missing_connections/migration.sql b/internal-packages/database/prisma/migrations/20230525134604_add_missing_connections/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230525134604_add_missing_connections/migration.sql rename to internal-packages/database/prisma/migrations/20230525134604_add_missing_connections/migration.sql diff --git a/packages/database/prisma/migrations/20230525141420_add_waiting_on_missing_connections_statys/migration.sql b/internal-packages/database/prisma/migrations/20230525141420_add_waiting_on_missing_connections_statys/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230525141420_add_waiting_on_missing_connections_statys/migration.sql rename to internal-packages/database/prisma/migrations/20230525141420_add_waiting_on_missing_connections_statys/migration.sql diff --git a/packages/database/prisma/migrations/20230530135120_task_added_connection_key/migration.sql b/internal-packages/database/prisma/migrations/20230530135120_task_added_connection_key/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230530135120_task_added_connection_key/migration.sql rename to internal-packages/database/prisma/migrations/20230530135120_task_added_connection_key/migration.sql diff --git a/packages/database/prisma/migrations/20230531174211_user_marketing_emails/migration.sql b/internal-packages/database/prisma/migrations/20230531174211_user_marketing_emails/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230531174211_user_marketing_emails/migration.sql rename to internal-packages/database/prisma/migrations/20230531174211_user_marketing_emails/migration.sql diff --git a/packages/database/prisma/migrations/20230601165125_add_org_member_invite/migration.sql b/internal-packages/database/prisma/migrations/20230601165125_add_org_member_invite/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230601165125_add_org_member_invite/migration.sql rename to internal-packages/database/prisma/migrations/20230601165125_add_org_member_invite/migration.sql diff --git a/packages/database/prisma/migrations/20230601170659_invite_unique_orgid_email_constraint/migration.sql b/internal-packages/database/prisma/migrations/20230601170659_invite_unique_orgid_email_constraint/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230601170659_invite_unique_orgid_email_constraint/migration.sql rename to internal-packages/database/prisma/migrations/20230601170659_invite_unique_orgid_email_constraint/migration.sql diff --git a/packages/database/prisma/migrations/20230601170752_invite_remove_orgmember/migration.sql b/internal-packages/database/prisma/migrations/20230601170752_invite_remove_orgmember/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230601170752_invite_remove_orgmember/migration.sql rename to internal-packages/database/prisma/migrations/20230601170752_invite_remove_orgmember/migration.sql diff --git a/packages/database/prisma/migrations/20230602110258_added_user_confirmed_basic_details/migration.sql b/internal-packages/database/prisma/migrations/20230602110258_added_user_confirmed_basic_details/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230602110258_added_user_confirmed_basic_details/migration.sql rename to internal-packages/database/prisma/migrations/20230602110258_added_user_confirmed_basic_details/migration.sql diff --git a/packages/database/prisma/migrations/20230602155844_keep_runtimeenvironment_when_orgmember_deleted/migration.sql b/internal-packages/database/prisma/migrations/20230602155844_keep_runtimeenvironment_when_orgmember_deleted/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230602155844_keep_runtimeenvironment_when_orgmember_deleted/migration.sql rename to internal-packages/database/prisma/migrations/20230602155844_keep_runtimeenvironment_when_orgmember_deleted/migration.sql diff --git a/packages/database/prisma/migrations/20230605123159_renamed_credentials_reference_to_custom_client_reference/migration.sql b/internal-packages/database/prisma/migrations/20230605123159_renamed_credentials_reference_to_custom_client_reference/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230605123159_renamed_credentials_reference_to_custom_client_reference/migration.sql rename to internal-packages/database/prisma/migrations/20230605123159_renamed_credentials_reference_to_custom_client_reference/migration.sql diff --git a/packages/database/prisma/migrations/20230605144132_add_elements_to_job_versions/migration.sql b/internal-packages/database/prisma/migrations/20230605144132_add_elements_to_job_versions/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230605144132_add_elements_to_job_versions/migration.sql rename to internal-packages/database/prisma/migrations/20230605144132_add_elements_to_job_versions/migration.sql diff --git a/packages/database/prisma/migrations/20230605155714_add_run_connection_to_tasks/migration.sql b/internal-packages/database/prisma/migrations/20230605155714_add_run_connection_to_tasks/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230605155714_add_run_connection_to_tasks/migration.sql rename to internal-packages/database/prisma/migrations/20230605155714_add_run_connection_to_tasks/migration.sql diff --git a/packages/database/prisma/migrations/20230605160520_add_style_to_tasks/migration.sql b/internal-packages/database/prisma/migrations/20230605160520_add_style_to_tasks/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230605160520_add_style_to_tasks/migration.sql rename to internal-packages/database/prisma/migrations/20230605160520_add_style_to_tasks/migration.sql diff --git a/packages/database/prisma/migrations/20230606081054_add_job_run_execution/migration.sql b/internal-packages/database/prisma/migrations/20230606081054_add_job_run_execution/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230606081054_add_job_run_execution/migration.sql rename to internal-packages/database/prisma/migrations/20230606081054_add_job_run_execution/migration.sql diff --git a/packages/database/prisma/migrations/20230606081441_add_preprocess_job_execution_reason/migration.sql b/internal-packages/database/prisma/migrations/20230606081441_add_preprocess_job_execution_reason/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230606081441_add_preprocess_job_execution_reason/migration.sql rename to internal-packages/database/prisma/migrations/20230606081441_add_preprocess_job_execution_reason/migration.sql diff --git a/packages/database/prisma/migrations/20230606081946_add_preprocess_runs_to_job_version/migration.sql b/internal-packages/database/prisma/migrations/20230606081946_add_preprocess_runs_to_job_version/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230606081946_add_preprocess_runs_to_job_version/migration.sql rename to internal-packages/database/prisma/migrations/20230606081946_add_preprocess_runs_to_job_version/migration.sql diff --git a/packages/database/prisma/migrations/20230606082119_add_preprocess_to_runs/migration.sql b/internal-packages/database/prisma/migrations/20230606082119_add_preprocess_to_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230606082119_add_preprocess_to_runs/migration.sql rename to internal-packages/database/prisma/migrations/20230606082119_add_preprocess_to_runs/migration.sql diff --git a/packages/database/prisma/migrations/20230606082719_add_preprocess_status_to_runs/migration.sql b/internal-packages/database/prisma/migrations/20230606082719_add_preprocess_status_to_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230606082719_add_preprocess_status_to_runs/migration.sql rename to internal-packages/database/prisma/migrations/20230606082719_add_preprocess_status_to_runs/migration.sql diff --git a/packages/database/prisma/migrations/20230606120402_add_task_to_run_execution/migration.sql b/internal-packages/database/prisma/migrations/20230606120402_add_task_to_run_execution/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230606120402_add_task_to_run_execution/migration.sql rename to internal-packages/database/prisma/migrations/20230606120402_add_task_to_run_execution/migration.sql diff --git a/packages/database/prisma/migrations/20230606122534_improve_run_execution_model/migration.sql b/internal-packages/database/prisma/migrations/20230606122534_improve_run_execution_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230606122534_improve_run_execution_model/migration.sql rename to internal-packages/database/prisma/migrations/20230606122534_improve_run_execution_model/migration.sql diff --git a/packages/database/prisma/migrations/20230606132031_removed_http_responses_from_executions/migration.sql b/internal-packages/database/prisma/migrations/20230606132031_removed_http_responses_from_executions/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230606132031_removed_http_responses_from_executions/migration.sql rename to internal-packages/database/prisma/migrations/20230606132031_removed_http_responses_from_executions/migration.sql diff --git a/packages/database/prisma/migrations/20230608102125_scope_job_versions_to_environments/migration.sql b/internal-packages/database/prisma/migrations/20230608102125_scope_job_versions_to_environments/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230608102125_scope_job_versions_to_environments/migration.sql rename to internal-packages/database/prisma/migrations/20230608102125_scope_job_versions_to_environments/migration.sql diff --git a/packages/database/prisma/migrations/20230608151823_scope_trigger_source_to_envs/migration.sql b/internal-packages/database/prisma/migrations/20230608151823_scope_trigger_source_to_envs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230608151823_scope_trigger_source_to_envs/migration.sql rename to internal-packages/database/prisma/migrations/20230608151823_scope_trigger_source_to_envs/migration.sql diff --git a/packages/database/prisma/migrations/20230609095323_rename_elements_to_properties/migration.sql b/internal-packages/database/prisma/migrations/20230609095323_rename_elements_to_properties/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230609095323_rename_elements_to_properties/migration.sql rename to internal-packages/database/prisma/migrations/20230609095323_rename_elements_to_properties/migration.sql diff --git a/packages/database/prisma/migrations/20230609150207_add_graphile_job_id_to_executions/migration.sql b/internal-packages/database/prisma/migrations/20230609150207_add_graphile_job_id_to_executions/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230609150207_add_graphile_job_id_to_executions/migration.sql rename to internal-packages/database/prisma/migrations/20230609150207_add_graphile_job_id_to_executions/migration.sql diff --git a/packages/database/prisma/migrations/20230609150822_add_graphile_job_id_string/migration.sql b/internal-packages/database/prisma/migrations/20230609150822_add_graphile_job_id_string/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230609150822_add_graphile_job_id_string/migration.sql rename to internal-packages/database/prisma/migrations/20230609150822_add_graphile_job_id_string/migration.sql diff --git a/packages/database/prisma/migrations/20230612150500_add_task_attempts/migration.sql b/internal-packages/database/prisma/migrations/20230612150500_add_task_attempts/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230612150500_add_task_attempts/migration.sql rename to internal-packages/database/prisma/migrations/20230612150500_add_task_attempts/migration.sql diff --git a/packages/database/prisma/migrations/20230613091640_event_example_table/migration.sql b/internal-packages/database/prisma/migrations/20230613091640_event_example_table/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230613091640_event_example_table/migration.sql rename to internal-packages/database/prisma/migrations/20230613091640_event_example_table/migration.sql diff --git a/packages/database/prisma/migrations/20230613092902_event_example_payload/migration.sql b/internal-packages/database/prisma/migrations/20230613092902_event_example_payload/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230613092902_event_example_payload/migration.sql rename to internal-packages/database/prisma/migrations/20230613092902_event_example_payload/migration.sql diff --git a/packages/database/prisma/migrations/20230613104234_event_example_added_name_icon/migration.sql b/internal-packages/database/prisma/migrations/20230613104234_event_example_added_name_icon/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230613104234_event_example_added_name_icon/migration.sql rename to internal-packages/database/prisma/migrations/20230613104234_event_example_added_name_icon/migration.sql diff --git a/packages/database/prisma/migrations/20230614103739_add_deploy_hook_identifier_to_endpoints/migration.sql b/internal-packages/database/prisma/migrations/20230614103739_add_deploy_hook_identifier_to_endpoints/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230614103739_add_deploy_hook_identifier_to_endpoints/migration.sql rename to internal-packages/database/prisma/migrations/20230614103739_add_deploy_hook_identifier_to_endpoints/migration.sql diff --git a/packages/database/prisma/migrations/20230614110359_rename_deploy_to_index/migration.sql b/internal-packages/database/prisma/migrations/20230614110359_rename_deploy_to_index/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230614110359_rename_deploy_to_index/migration.sql rename to internal-packages/database/prisma/migrations/20230614110359_rename_deploy_to_index/migration.sql diff --git a/packages/database/prisma/migrations/20230614122553_create_endpoint_index_model/migration.sql b/internal-packages/database/prisma/migrations/20230614122553_create_endpoint_index_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230614122553_create_endpoint_index_model/migration.sql rename to internal-packages/database/prisma/migrations/20230614122553_create_endpoint_index_model/migration.sql diff --git a/packages/database/prisma/migrations/20230614125945_add_source_data_to_endpoint_index/migration.sql b/internal-packages/database/prisma/migrations/20230614125945_add_source_data_to_endpoint_index/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230614125945_add_source_data_to_endpoint_index/migration.sql rename to internal-packages/database/prisma/migrations/20230614125945_add_source_data_to_endpoint_index/migration.sql diff --git a/packages/database/prisma/migrations/20230614135014_change_endpoint_source_enum/migration.sql b/internal-packages/database/prisma/migrations/20230614135014_change_endpoint_source_enum/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230614135014_change_endpoint_source_enum/migration.sql rename to internal-packages/database/prisma/migrations/20230614135014_change_endpoint_source_enum/migration.sql diff --git a/packages/database/prisma/migrations/20230614141902_added_api_to_endpoint_index_source/migration.sql b/internal-packages/database/prisma/migrations/20230614141902_added_api_to_endpoint_index_source/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230614141902_added_api_to_endpoint_index_source/migration.sql rename to internal-packages/database/prisma/migrations/20230614141902_added_api_to_endpoint_index_source/migration.sql diff --git a/packages/database/prisma/migrations/20230615152126_revamp_integration_schema/migration.sql b/internal-packages/database/prisma/migrations/20230615152126_revamp_integration_schema/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230615152126_revamp_integration_schema/migration.sql rename to internal-packages/database/prisma/migrations/20230615152126_revamp_integration_schema/migration.sql diff --git a/packages/database/prisma/migrations/20230616083056_add_integration_auth_method/migration.sql b/internal-packages/database/prisma/migrations/20230616083056_add_integration_auth_method/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230616083056_add_integration_auth_method/migration.sql rename to internal-packages/database/prisma/migrations/20230616083056_add_integration_auth_method/migration.sql diff --git a/packages/database/prisma/migrations/20230616084406_remove_metadata_and_add_icon_to_integration/migration.sql b/internal-packages/database/prisma/migrations/20230616084406_remove_metadata_and_add_icon_to_integration/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230616084406_remove_metadata_and_add_icon_to_integration/migration.sql rename to internal-packages/database/prisma/migrations/20230616084406_remove_metadata_and_add_icon_to_integration/migration.sql diff --git a/packages/database/prisma/migrations/20230616093240_api_integration_vote_added/migration.sql b/internal-packages/database/prisma/migrations/20230616093240_api_integration_vote_added/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230616093240_api_integration_vote_added/migration.sql rename to internal-packages/database/prisma/migrations/20230616093240_api_integration_vote_added/migration.sql diff --git a/packages/database/prisma/migrations/20230616094008_updating_run_connections_to_work_with_new_integrations/migration.sql b/internal-packages/database/prisma/migrations/20230616094008_updating_run_connections_to_work_with_new_integrations/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230616094008_updating_run_connections_to_work_with_new_integrations/migration.sql rename to internal-packages/database/prisma/migrations/20230616094008_updating_run_connections_to_work_with_new_integrations/migration.sql diff --git a/packages/database/prisma/migrations/20230616102332_make_trigger_source_integrations_required/migration.sql b/internal-packages/database/prisma/migrations/20230616102332_make_trigger_source_integrations_required/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230616102332_make_trigger_source_integrations_required/migration.sql rename to internal-packages/database/prisma/migrations/20230616102332_make_trigger_source_integrations_required/migration.sql diff --git a/packages/database/prisma/migrations/20230616103552_add_api_identifier_to_integrations/migration.sql b/internal-packages/database/prisma/migrations/20230616103552_add_api_identifier_to_integrations/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230616103552_add_api_identifier_to_integrations/migration.sql rename to internal-packages/database/prisma/migrations/20230616103552_add_api_identifier_to_integrations/migration.sql diff --git a/packages/database/prisma/migrations/20230616104748_add_integration_definition/migration.sql b/internal-packages/database/prisma/migrations/20230616104748_add_integration_definition/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230616104748_add_integration_definition/migration.sql rename to internal-packages/database/prisma/migrations/20230616104748_add_integration_definition/migration.sql diff --git a/packages/database/prisma/migrations/20230616104937_remove_api_identifier/migration.sql b/internal-packages/database/prisma/migrations/20230616104937_remove_api_identifier/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230616104937_remove_api_identifier/migration.sql rename to internal-packages/database/prisma/migrations/20230616104937_remove_api_identifier/migration.sql diff --git a/packages/database/prisma/migrations/20230616105239_remove_duplicate_identifier_from_auth_method/migration.sql b/internal-packages/database/prisma/migrations/20230616105239_remove_duplicate_identifier_from_auth_method/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230616105239_remove_duplicate_identifier_from_auth_method/migration.sql rename to internal-packages/database/prisma/migrations/20230616105239_remove_duplicate_identifier_from_auth_method/migration.sql diff --git a/packages/database/prisma/migrations/20230619100936_add_operation_to_task/migration.sql b/internal-packages/database/prisma/migrations/20230619100936_add_operation_to_task/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230619100936_add_operation_to_task/migration.sql rename to internal-packages/database/prisma/migrations/20230619100936_add_operation_to_task/migration.sql diff --git a/packages/database/prisma/migrations/20230627212239_add_source_context_to_event_record/migration.sql b/internal-packages/database/prisma/migrations/20230627212239_add_source_context_to_event_record/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230627212239_add_source_context_to_event_record/migration.sql rename to internal-packages/database/prisma/migrations/20230627212239_add_source_context_to_event_record/migration.sql diff --git a/packages/database/prisma/migrations/20230628100705_add_metadata_to_dynamic_trigger_registrations/migration.sql b/internal-packages/database/prisma/migrations/20230628100705_add_metadata_to_dynamic_trigger_registrations/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230628100705_add_metadata_to_dynamic_trigger_registrations/migration.sql rename to internal-packages/database/prisma/migrations/20230628100705_add_metadata_to_dynamic_trigger_registrations/migration.sql diff --git a/packages/database/prisma/migrations/20230628102426_add_dynamic_source_columns_to_trigger_source/migration.sql b/internal-packages/database/prisma/migrations/20230628102426_add_dynamic_source_columns_to_trigger_source/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230628102426_add_dynamic_source_columns_to_trigger_source/migration.sql rename to internal-packages/database/prisma/migrations/20230628102426_add_dynamic_source_columns_to_trigger_source/migration.sql diff --git a/packages/database/prisma/migrations/20230630093541_integration_added_setup_status/migration.sql b/internal-packages/database/prisma/migrations/20230630093541_integration_added_setup_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230630093541_integration_added_setup_status/migration.sql rename to internal-packages/database/prisma/migrations/20230630093541_integration_added_setup_status/migration.sql diff --git a/packages/database/prisma/migrations/20230630160813_integration_definition_packagename_description_help/migration.sql b/internal-packages/database/prisma/migrations/20230630160813_integration_definition_packagename_description_help/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230630160813_integration_definition_packagename_description_help/migration.sql rename to internal-packages/database/prisma/migrations/20230630160813_integration_definition_packagename_description_help/migration.sql diff --git a/packages/database/prisma/migrations/20230703152107_add_account_identifier_to_missing_connections/migration.sql b/internal-packages/database/prisma/migrations/20230703152107_add_account_identifier_to_missing_connections/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230703152107_add_account_identifier_to_missing_connections/migration.sql rename to internal-packages/database/prisma/migrations/20230703152107_add_account_identifier_to_missing_connections/migration.sql diff --git a/packages/database/prisma/migrations/20230704094159_add_auto_enable_to_envs/migration.sql b/internal-packages/database/prisma/migrations/20230704094159_add_auto_enable_to_envs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230704094159_add_auto_enable_to_envs/migration.sql rename to internal-packages/database/prisma/migrations/20230704094159_add_auto_enable_to_envs/migration.sql diff --git a/packages/database/prisma/migrations/20230704094425_add_pk_api_key/migration.sql b/internal-packages/database/prisma/migrations/20230704094425_add_pk_api_key/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230704094425_add_pk_api_key/migration.sql rename to internal-packages/database/prisma/migrations/20230704094425_add_pk_api_key/migration.sql diff --git a/packages/database/prisma/migrations/20230705151109_add_cloud_invitation_stuff/migration.sql b/internal-packages/database/prisma/migrations/20230705151109_add_cloud_invitation_stuff/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230705151109_add_cloud_invitation_stuff/migration.sql rename to internal-packages/database/prisma/migrations/20230705151109_add_cloud_invitation_stuff/migration.sql diff --git a/packages/database/prisma/migrations/20230705152702_remove_is_cloud_activated/migration.sql b/internal-packages/database/prisma/migrations/20230705152702_remove_is_cloud_activated/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230705152702_remove_is_cloud_activated/migration.sql rename to internal-packages/database/prisma/migrations/20230705152702_remove_is_cloud_activated/migration.sql diff --git a/packages/database/prisma/migrations/20230707101156_add_timestamps_to_jobs/migration.sql b/internal-packages/database/prisma/migrations/20230707101156_add_timestamps_to_jobs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230707101156_add_timestamps_to_jobs/migration.sql rename to internal-packages/database/prisma/migrations/20230707101156_add_timestamps_to_jobs/migration.sql diff --git a/packages/database/prisma/migrations/20230707105750_add_source_registration_job_to_dynamic_triggers/migration.sql b/internal-packages/database/prisma/migrations/20230707105750_add_source_registration_job_to_dynamic_triggers/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230707105750_add_source_registration_job_to_dynamic_triggers/migration.sql rename to internal-packages/database/prisma/migrations/20230707105750_add_source_registration_job_to_dynamic_triggers/migration.sql diff --git a/packages/database/prisma/migrations/20230707122412_add_source_registration_job_to_trigger_sources/migration.sql b/internal-packages/database/prisma/migrations/20230707122412_add_source_registration_job_to_trigger_sources/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230707122412_add_source_registration_job_to_trigger_sources/migration.sql rename to internal-packages/database/prisma/migrations/20230707122412_add_source_registration_job_to_trigger_sources/migration.sql diff --git a/packages/database/prisma/migrations/20230707145540_add_auth_identifier_to_user_table/migration.sql b/internal-packages/database/prisma/migrations/20230707145540_add_auth_identifier_to_user_table/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230707145540_add_auth_identifier_to_user_table/migration.sql rename to internal-packages/database/prisma/migrations/20230707145540_add_auth_identifier_to_user_table/migration.sql diff --git a/packages/database/prisma/migrations/20230707145604_make_auth_identifier_unique/migration.sql b/internal-packages/database/prisma/migrations/20230707145604_make_auth_identifier_unique/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230707145604_make_auth_identifier_unique/migration.sql rename to internal-packages/database/prisma/migrations/20230707145604_make_auth_identifier_unique/migration.sql diff --git a/packages/database/prisma/migrations/20230708193746_add_output_properties_to_tasks/migration.sql b/internal-packages/database/prisma/migrations/20230708193746_add_output_properties_to_tasks/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230708193746_add_output_properties_to_tasks/migration.sql rename to internal-packages/database/prisma/migrations/20230708193746_add_output_properties_to_tasks/migration.sql diff --git a/packages/database/prisma/migrations/20230711175328_added_canceled_run_status/migration.sql b/internal-packages/database/prisma/migrations/20230711175328_added_canceled_run_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230711175328_added_canceled_run_status/migration.sql rename to internal-packages/database/prisma/migrations/20230711175328_added_canceled_run_status/migration.sql diff --git a/packages/database/prisma/migrations/20230712075144_added_canceled_to_task_status/migration.sql b/internal-packages/database/prisma/migrations/20230712075144_added_canceled_to_task_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230712075144_added_canceled_to_task_status/migration.sql rename to internal-packages/database/prisma/migrations/20230712075144_added_canceled_to_task_status/migration.sql diff --git a/packages/database/prisma/migrations/20230712132529_added_relationship_between_schedule_source_and_dynamic_trigger/migration.sql b/internal-packages/database/prisma/migrations/20230712132529_added_relationship_between_schedule_source_and_dynamic_trigger/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230712132529_added_relationship_between_schedule_source_and_dynamic_trigger/migration.sql rename to internal-packages/database/prisma/migrations/20230712132529_added_relationship_between_schedule_source_and_dynamic_trigger/migration.sql diff --git a/packages/database/prisma/migrations/20230716054029_is_retry_for_job_run_execution/migration.sql b/internal-packages/database/prisma/migrations/20230716054029_is_retry_for_job_run_execution/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230716054029_is_retry_for_job_run_execution/migration.sql rename to internal-packages/database/prisma/migrations/20230716054029_is_retry_for_job_run_execution/migration.sql diff --git a/packages/database/prisma/migrations/20230721124527_add_icon_to_integration_definition/migration.sql b/internal-packages/database/prisma/migrations/20230721124527_add_icon_to_integration_definition/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230721124527_add_icon_to_integration_definition/migration.sql rename to internal-packages/database/prisma/migrations/20230721124527_add_icon_to_integration_definition/migration.sql diff --git a/packages/database/prisma/migrations/20230724074140_changed/migration.sql b/internal-packages/database/prisma/migrations/20230724074140_changed/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230724074140_changed/migration.sql rename to internal-packages/database/prisma/migrations/20230724074140_changed/migration.sql diff --git a/packages/database/prisma/migrations/20230725161151_pk_api_key_unique_and_non_null/migration.sql b/internal-packages/database/prisma/migrations/20230725161151_pk_api_key_unique_and_non_null/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230725161151_pk_api_key_unique_and_non_null/migration.sql rename to internal-packages/database/prisma/migrations/20230725161151_pk_api_key_unique_and_non_null/migration.sql diff --git a/packages/database/prisma/migrations/20230731142627_remove_user_access_token/migration.sql b/internal-packages/database/prisma/migrations/20230731142627_remove_user_access_token/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230731142627_remove_user_access_token/migration.sql rename to internal-packages/database/prisma/migrations/20230731142627_remove_user_access_token/migration.sql diff --git a/packages/database/prisma/migrations/20230731145332_add_version_to_secret_store/migration.sql b/internal-packages/database/prisma/migrations/20230731145332_add_version_to_secret_store/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230731145332_add_version_to_secret_store/migration.sql rename to internal-packages/database/prisma/migrations/20230731145332_add_version_to_secret_store/migration.sql diff --git a/packages/database/prisma/migrations/20230814131639_added_cancelled_at_column_to_the_event_record_table/migration.sql b/internal-packages/database/prisma/migrations/20230814131639_added_cancelled_at_column_to_the_event_record_table/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230814131639_added_cancelled_at_column_to_the_event_record_table/migration.sql rename to internal-packages/database/prisma/migrations/20230814131639_added_cancelled_at_column_to_the_event_record_table/migration.sql diff --git a/packages/database/prisma/migrations/20230821123033_trigger_source_option_added_for_multiple_event_dimensions/migration.sql b/internal-packages/database/prisma/migrations/20230821123033_trigger_source_option_added_for_multiple_event_dimensions/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230821123033_trigger_source_option_added_for_multiple_event_dimensions/migration.sql rename to internal-packages/database/prisma/migrations/20230821123033_trigger_source_option_added_for_multiple_event_dimensions/migration.sql diff --git a/packages/database/prisma/migrations/20230821132604_remove_trigger_source_event_table/migration.sql b/internal-packages/database/prisma/migrations/20230821132604_remove_trigger_source_event_table/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230821132604_remove_trigger_source_event_table/migration.sql rename to internal-packages/database/prisma/migrations/20230821132604_remove_trigger_source_event_table/migration.sql diff --git a/packages/database/prisma/migrations/20230822130655_add_status_to_job_version/migration.sql b/internal-packages/database/prisma/migrations/20230822130655_add_status_to_job_version/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230822130655_add_status_to_job_version/migration.sql rename to internal-packages/database/prisma/migrations/20230822130655_add_status_to_job_version/migration.sql diff --git a/packages/database/prisma/migrations/20230823124049_add_deleted_at_to_jobs/migration.sql b/internal-packages/database/prisma/migrations/20230823124049_add_deleted_at_to_jobs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230823124049_add_deleted_at_to_jobs/migration.sql rename to internal-packages/database/prisma/migrations/20230823124049_add_deleted_at_to_jobs/migration.sql diff --git a/packages/database/prisma/migrations/20230824191603_added_trigger_source_metadata/migration.sql b/internal-packages/database/prisma/migrations/20230824191603_added_trigger_source_metadata/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230824191603_added_trigger_source_metadata/migration.sql rename to internal-packages/database/prisma/migrations/20230824191603_added_trigger_source_metadata/migration.sql diff --git a/packages/database/prisma/migrations/20230904145326_add_execution_columns_to_runs/migration.sql b/internal-packages/database/prisma/migrations/20230904145326_add_execution_columns_to_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230904145326_add_execution_columns_to_runs/migration.sql rename to internal-packages/database/prisma/migrations/20230904145326_add_execution_columns_to_runs/migration.sql diff --git a/packages/database/prisma/migrations/20230904205457_add_max_run_execution_time_to_orgs/migration.sql b/internal-packages/database/prisma/migrations/20230904205457_add_max_run_execution_time_to_orgs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230904205457_add_max_run_execution_time_to_orgs/migration.sql rename to internal-packages/database/prisma/migrations/20230904205457_add_max_run_execution_time_to_orgs/migration.sql diff --git a/packages/database/prisma/migrations/20230919124531_add_resolver_auth_source/migration.sql b/internal-packages/database/prisma/migrations/20230919124531_add_resolver_auth_source/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230919124531_add_resolver_auth_source/migration.sql rename to internal-packages/database/prisma/migrations/20230919124531_add_resolver_auth_source/migration.sql diff --git a/packages/database/prisma/migrations/20230919141017_added_job_run_status_record_table/migration.sql b/internal-packages/database/prisma/migrations/20230919141017_added_job_run_status_record_table/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230919141017_added_job_run_status_record_table/migration.sql rename to internal-packages/database/prisma/migrations/20230919141017_added_job_run_status_record_table/migration.sql diff --git a/packages/database/prisma/migrations/20230919150351_add_unresolved_auth_job_run_status/migration.sql b/internal-packages/database/prisma/migrations/20230919150351_add_unresolved_auth_job_run_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230919150351_add_unresolved_auth_job_run_status/migration.sql rename to internal-packages/database/prisma/migrations/20230919150351_add_unresolved_auth_job_run_status/migration.sql diff --git a/packages/database/prisma/migrations/20230922205611_add_invalid_payload_run_status/migration.sql b/internal-packages/database/prisma/migrations/20230922205611_add_invalid_payload_run_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230922205611_add_invalid_payload_run_status/migration.sql rename to internal-packages/database/prisma/migrations/20230922205611_add_invalid_payload_run_status/migration.sql diff --git a/packages/database/prisma/migrations/20230925174509_add_callback_url/migration.sql b/internal-packages/database/prisma/migrations/20230925174509_add_callback_url/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230925174509_add_callback_url/migration.sql rename to internal-packages/database/prisma/migrations/20230925174509_add_callback_url/migration.sql diff --git a/packages/database/prisma/migrations/20230927160010_add_data_migrations/migration.sql b/internal-packages/database/prisma/migrations/20230927160010_add_data_migrations/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230927160010_add_data_migrations/migration.sql rename to internal-packages/database/prisma/migrations/20230927160010_add_data_migrations/migration.sql diff --git a/packages/database/prisma/migrations/20230929100348_add_yielded_executions_to_job_run/migration.sql b/internal-packages/database/prisma/migrations/20230929100348_add_yielded_executions_to_job_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20230929100348_add_yielded_executions_to_job_run/migration.sql rename to internal-packages/database/prisma/migrations/20230929100348_add_yielded_executions_to_job_run/migration.sql diff --git a/packages/database/prisma/migrations/20231003092741_add_version_to_endpoint/migration.sql b/internal-packages/database/prisma/migrations/20231003092741_add_version_to_endpoint/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231003092741_add_version_to_endpoint/migration.sql rename to internal-packages/database/prisma/migrations/20231003092741_add_version_to_endpoint/migration.sql diff --git a/packages/database/prisma/migrations/20231005064823_add_job_run_internal/migration.sql b/internal-packages/database/prisma/migrations/20231005064823_add_job_run_internal/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231005064823_add_job_run_internal/migration.sql rename to internal-packages/database/prisma/migrations/20231005064823_add_job_run_internal/migration.sql diff --git a/packages/database/prisma/migrations/20231010115840_endpoint_index_status_added/migration.sql b/internal-packages/database/prisma/migrations/20231010115840_endpoint_index_status_added/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231010115840_endpoint_index_status_added/migration.sql rename to internal-packages/database/prisma/migrations/20231010115840_endpoint_index_status_added/migration.sql diff --git a/packages/database/prisma/migrations/20231010120458_endpoint_index_data_and_stats_are_now_optional/migration.sql b/internal-packages/database/prisma/migrations/20231010120458_endpoint_index_data_and_stats_are_now_optional/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231010120458_endpoint_index_data_and_stats_are_now_optional/migration.sql rename to internal-packages/database/prisma/migrations/20231010120458_endpoint_index_data_and_stats_are_now_optional/migration.sql diff --git a/packages/database/prisma/migrations/20231010135433_endpoint_index_added_error_column/migration.sql b/internal-packages/database/prisma/migrations/20231010135433_endpoint_index_added_error_column/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231010135433_endpoint_index_added_error_column/migration.sql rename to internal-packages/database/prisma/migrations/20231010135433_endpoint_index_added_error_column/migration.sql diff --git a/packages/database/prisma/migrations/20231011104302_add_run_chunk_column/migration.sql b/internal-packages/database/prisma/migrations/20231011104302_add_run_chunk_column/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231011104302_add_run_chunk_column/migration.sql rename to internal-packages/database/prisma/migrations/20231011104302_add_run_chunk_column/migration.sql diff --git a/packages/database/prisma/migrations/20231011134840_add_auto_yielded_executions/migration.sql b/internal-packages/database/prisma/migrations/20231011134840_add_auto_yielded_executions/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231011134840_add_auto_yielded_executions/migration.sql rename to internal-packages/database/prisma/migrations/20231011134840_add_auto_yielded_executions/migration.sql diff --git a/packages/database/prisma/migrations/20231011141302_add_location_to_auto_yield_executions/migration.sql b/internal-packages/database/prisma/migrations/20231011141302_add_location_to_auto_yield_executions/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231011141302_add_location_to_auto_yield_executions/migration.sql rename to internal-packages/database/prisma/migrations/20231011141302_add_location_to_auto_yield_executions/migration.sql diff --git a/packages/database/prisma/migrations/20231011145406_change_run_chunk_execution_limit_default/migration.sql b/internal-packages/database/prisma/migrations/20231011145406_change_run_chunk_execution_limit_default/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231011145406_change_run_chunk_execution_limit_default/migration.sql rename to internal-packages/database/prisma/migrations/20231011145406_change_run_chunk_execution_limit_default/migration.sql diff --git a/packages/database/prisma/migrations/20231011213532_add_auto_yield_threshold_settings_to_endpoints/migration.sql b/internal-packages/database/prisma/migrations/20231011213532_add_auto_yield_threshold_settings_to_endpoints/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231011213532_add_auto_yield_threshold_settings_to_endpoints/migration.sql rename to internal-packages/database/prisma/migrations/20231011213532_add_auto_yield_threshold_settings_to_endpoints/migration.sql diff --git a/packages/database/prisma/migrations/20231013083144_add_next_event_timestamp_to_schedule_source/migration.sql b/internal-packages/database/prisma/migrations/20231013083144_add_next_event_timestamp_to_schedule_source/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231013083144_add_next_event_timestamp_to_schedule_source/migration.sql rename to internal-packages/database/prisma/migrations/20231013083144_add_next_event_timestamp_to_schedule_source/migration.sql diff --git a/packages/database/prisma/migrations/20231019123406_add_force_yield_immediately_to_runs/migration.sql b/internal-packages/database/prisma/migrations/20231019123406_add_force_yield_immediately_to_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231019123406_add_force_yield_immediately_to_runs/migration.sql rename to internal-packages/database/prisma/migrations/20231019123406_add_force_yield_immediately_to_runs/migration.sql diff --git a/packages/database/prisma/migrations/20231020145127_add_sdk_version_to_endpoints/migration.sql b/internal-packages/database/prisma/migrations/20231020145127_add_sdk_version_to_endpoints/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231020145127_add_sdk_version_to_endpoints/migration.sql rename to internal-packages/database/prisma/migrations/20231020145127_add_sdk_version_to_endpoints/migration.sql diff --git a/packages/database/prisma/migrations/20231023144342_added_event_record_payload_type_default_is_json/migration.sql b/internal-packages/database/prisma/migrations/20231023144342_added_event_record_payload_type_default_is_json/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231023144342_added_event_record_payload_type_default_is_json/migration.sql rename to internal-packages/database/prisma/migrations/20231023144342_added_event_record_payload_type_default_is_json/migration.sql diff --git a/packages/database/prisma/migrations/20231023173456_added_trigger_http_endpoint_table/migration.sql b/internal-packages/database/prisma/migrations/20231023173456_added_trigger_http_endpoint_table/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231023173456_added_trigger_http_endpoint_table/migration.sql rename to internal-packages/database/prisma/migrations/20231023173456_added_trigger_http_endpoint_table/migration.sql diff --git a/packages/database/prisma/migrations/20231024122540_http_endpoint_scoped_to_project_not_environment/migration.sql b/internal-packages/database/prisma/migrations/20231024122540_http_endpoint_scoped_to_project_not_environment/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231024122540_http_endpoint_scoped_to_project_not_environment/migration.sql rename to internal-packages/database/prisma/migrations/20231024122540_http_endpoint_scoped_to_project_not_environment/migration.sql diff --git a/packages/database/prisma/migrations/20231024122736_runtime_environment_shortcodes/migration.sql b/internal-packages/database/prisma/migrations/20231024122736_runtime_environment_shortcodes/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231024122736_runtime_environment_shortcodes/migration.sql rename to internal-packages/database/prisma/migrations/20231024122736_runtime_environment_shortcodes/migration.sql diff --git a/packages/database/prisma/migrations/20231025091939_trigger_http_endpoint_environment_created/migration.sql b/internal-packages/database/prisma/migrations/20231025091939_trigger_http_endpoint_environment_created/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231025091939_trigger_http_endpoint_environment_created/migration.sql rename to internal-packages/database/prisma/migrations/20231025091939_trigger_http_endpoint_environment_created/migration.sql diff --git a/packages/database/prisma/migrations/20231025144821_set_endpoint_run_chunk_execution_limit_from_60ms_to_60000ms/migration.sql b/internal-packages/database/prisma/migrations/20231025144821_set_endpoint_run_chunk_execution_limit_from_60ms_to_60000ms/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231025144821_set_endpoint_run_chunk_execution_limit_from_60ms_to_60000ms/migration.sql rename to internal-packages/database/prisma/migrations/20231025144821_set_endpoint_run_chunk_execution_limit_from_60ms_to_60000ms/migration.sql diff --git a/packages/database/prisma/migrations/20231026100653_added_skip_triggering_runs_to_trigger_http_endpoint_environment/migration.sql b/internal-packages/database/prisma/migrations/20231026100653_added_skip_triggering_runs_to_trigger_http_endpoint_environment/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231026100653_added_skip_triggering_runs_to_trigger_http_endpoint_environment/migration.sql rename to internal-packages/database/prisma/migrations/20231026100653_added_skip_triggering_runs_to_trigger_http_endpoint_environment/migration.sql diff --git a/packages/database/prisma/migrations/20231026103218_add_endpoint_relation_to_http_endpoint_environment/migration.sql b/internal-packages/database/prisma/migrations/20231026103218_add_endpoint_relation_to_http_endpoint_environment/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231026103218_add_endpoint_relation_to_http_endpoint_environment/migration.sql rename to internal-packages/database/prisma/migrations/20231026103218_add_endpoint_relation_to_http_endpoint_environment/migration.sql diff --git a/packages/database/prisma/migrations/20231026103614_trigger_http_endpoint_environment_removed_the_environment_constraint/migration.sql b/internal-packages/database/prisma/migrations/20231026103614_trigger_http_endpoint_environment_removed_the_environment_constraint/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231026103614_trigger_http_endpoint_environment_removed_the_environment_constraint/migration.sql rename to internal-packages/database/prisma/migrations/20231026103614_trigger_http_endpoint_environment_removed_the_environment_constraint/migration.sql diff --git a/packages/database/prisma/migrations/20231026104408_trigger_http_endpoint_environment_added_the_environment_constraint_back_in/migration.sql b/internal-packages/database/prisma/migrations/20231026104408_trigger_http_endpoint_environment_added_the_environment_constraint_back_in/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231026104408_trigger_http_endpoint_environment_added_the_environment_constraint_back_in/migration.sql rename to internal-packages/database/prisma/migrations/20231026104408_trigger_http_endpoint_environment_added_the_environment_constraint_back_in/migration.sql diff --git a/packages/database/prisma/migrations/20231026165235_trigger_http_endpoint_environment_added_source/migration.sql b/internal-packages/database/prisma/migrations/20231026165235_trigger_http_endpoint_environment_added_source/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231026165235_trigger_http_endpoint_environment_added_source/migration.sql rename to internal-packages/database/prisma/migrations/20231026165235_trigger_http_endpoint_environment_added_source/migration.sql diff --git a/packages/database/prisma/migrations/20231028193441_prepare_for_invoke_trigger/migration.sql b/internal-packages/database/prisma/migrations/20231028193441_prepare_for_invoke_trigger/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231028193441_prepare_for_invoke_trigger/migration.sql rename to internal-packages/database/prisma/migrations/20231028193441_prepare_for_invoke_trigger/migration.sql diff --git a/packages/database/prisma/migrations/20231101105021_add_child_execution_mode_to_tasks/migration.sql b/internal-packages/database/prisma/migrations/20231101105021_add_child_execution_mode_to_tasks/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231101105021_add_child_execution_mode_to_tasks/migration.sql rename to internal-packages/database/prisma/migrations/20231101105021_add_child_execution_mode_to_tasks/migration.sql diff --git a/packages/database/prisma/migrations/20231101202056_trigger_http_endpoint_environment_added_created_at_updated_at/migration.sql b/internal-packages/database/prisma/migrations/20231101202056_trigger_http_endpoint_environment_added_created_at_updated_at/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231101202056_trigger_http_endpoint_environment_added_created_at_updated_at/migration.sql rename to internal-packages/database/prisma/migrations/20231101202056_trigger_http_endpoint_environment_added_created_at_updated_at/migration.sql diff --git a/packages/database/prisma/migrations/20231102143530_added_relationships_between_event_record_and_trigger_http_endpoint_trigger_http_endpoint_environment/migration.sql b/internal-packages/database/prisma/migrations/20231102143530_added_relationships_between_event_record_and_trigger_http_endpoint_trigger_http_endpoint_environment/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231102143530_added_relationships_between_event_record_and_trigger_http_endpoint_trigger_http_endpoint_environment/migration.sql rename to internal-packages/database/prisma/migrations/20231102143530_added_relationships_between_event_record_and_trigger_http_endpoint_trigger_http_endpoint_environment/migration.sql diff --git a/packages/database/prisma/migrations/20231102171207_job_version_added_trigger_link_and_trigger_help/migration.sql b/internal-packages/database/prisma/migrations/20231102171207_job_version_added_trigger_link_and_trigger_help/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231102171207_job_version_added_trigger_link_and_trigger_help/migration.sql rename to internal-packages/database/prisma/migrations/20231102171207_job_version_added_trigger_link_and_trigger_help/migration.sql diff --git a/packages/database/prisma/migrations/20231107134830_add_context_to_tasks/migration.sql b/internal-packages/database/prisma/migrations/20231107134830_add_context_to_tasks/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231107134830_add_context_to_tasks/migration.sql rename to internal-packages/database/prisma/migrations/20231107134830_add_context_to_tasks/migration.sql diff --git a/packages/database/prisma/migrations/20231109122305_add_external_account_to_dispatcher/migration.sql b/internal-packages/database/prisma/migrations/20231109122305_add_external_account_to_dispatcher/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231109122305_add_external_account_to_dispatcher/migration.sql rename to internal-packages/database/prisma/migrations/20231109122305_add_external_account_to_dispatcher/migration.sql diff --git a/packages/database/prisma/migrations/20231113151412_add_output_is_undefined/migration.sql b/internal-packages/database/prisma/migrations/20231113151412_add_output_is_undefined/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231113151412_add_output_is_undefined/migration.sql rename to internal-packages/database/prisma/migrations/20231113151412_add_output_is_undefined/migration.sql diff --git a/packages/database/prisma/migrations/20231115134828_add_events_schema_and_tables/migration.sql b/internal-packages/database/prisma/migrations/20231115134828_add_events_schema_and_tables/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231115134828_add_events_schema_and_tables/migration.sql rename to internal-packages/database/prisma/migrations/20231115134828_add_events_schema_and_tables/migration.sql diff --git a/packages/database/prisma/migrations/20231115142936_add_webhook_source/migration.sql b/internal-packages/database/prisma/migrations/20231115142936_add_webhook_source/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231115142936_add_webhook_source/migration.sql rename to internal-packages/database/prisma/migrations/20231115142936_add_webhook_source/migration.sql diff --git a/packages/database/prisma/migrations/20231116113235_add_unique_index_on_job_run_sub/migration.sql b/internal-packages/database/prisma/migrations/20231116113235_add_unique_index_on_job_run_sub/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231116113235_add_unique_index_on_job_run_sub/migration.sql rename to internal-packages/database/prisma/migrations/20231116113235_add_unique_index_on_job_run_sub/migration.sql diff --git a/packages/database/prisma/migrations/20231117145312_add_additional_run_statuses/migration.sql b/internal-packages/database/prisma/migrations/20231117145312_add_additional_run_statuses/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231117145312_add_additional_run_statuses/migration.sql rename to internal-packages/database/prisma/migrations/20231117145312_add_additional_run_statuses/migration.sql diff --git a/packages/database/prisma/migrations/20231120163155_add_webhook_environment/migration.sql b/internal-packages/database/prisma/migrations/20231120163155_add_webhook_environment/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231120163155_add_webhook_environment/migration.sql rename to internal-packages/database/prisma/migrations/20231120163155_add_webhook_environment/migration.sql diff --git a/packages/database/prisma/migrations/20231121144353_make_job_run_number_optional/migration.sql b/internal-packages/database/prisma/migrations/20231121144353_make_job_run_number_optional/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231121144353_make_job_run_number_optional/migration.sql rename to internal-packages/database/prisma/migrations/20231121144353_make_job_run_number_optional/migration.sql diff --git a/packages/database/prisma/migrations/20231121154359_add_job_counter_table/migration.sql b/internal-packages/database/prisma/migrations/20231121154359_add_job_counter_table/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231121154359_add_job_counter_table/migration.sql rename to internal-packages/database/prisma/migrations/20231121154359_add_job_counter_table/migration.sql diff --git a/packages/database/prisma/migrations/20231121154545_seed_job_counter_tables/migration.sql b/internal-packages/database/prisma/migrations/20231121154545_seed_job_counter_tables/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231121154545_seed_job_counter_tables/migration.sql rename to internal-packages/database/prisma/migrations/20231121154545_seed_job_counter_tables/migration.sql diff --git a/packages/database/prisma/migrations/20231121155237_change_kv_value_to_bytes/migration.sql b/internal-packages/database/prisma/migrations/20231121155237_change_kv_value_to_bytes/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231121155237_change_kv_value_to_bytes/migration.sql rename to internal-packages/database/prisma/migrations/20231121155237_change_kv_value_to_bytes/migration.sql diff --git a/packages/database/prisma/migrations/20231122091927_add_webhook_request_delivery/migration.sql b/internal-packages/database/prisma/migrations/20231122091927_add_webhook_request_delivery/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231122091927_add_webhook_request_delivery/migration.sql rename to internal-packages/database/prisma/migrations/20231122091927_add_webhook_request_delivery/migration.sql diff --git a/packages/database/prisma/migrations/20231122105932_add_env_to_webhook_delivery/migration.sql b/internal-packages/database/prisma/migrations/20231122105932_add_env_to_webhook_delivery/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231122105932_add_env_to_webhook_delivery/migration.sql rename to internal-packages/database/prisma/migrations/20231122105932_add_env_to_webhook_delivery/migration.sql diff --git a/packages/database/prisma/migrations/20231122151126_add_delivery_numbers/migration.sql b/internal-packages/database/prisma/migrations/20231122151126_add_delivery_numbers/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231122151126_add_delivery_numbers/migration.sql rename to internal-packages/database/prisma/migrations/20231122151126_add_delivery_numbers/migration.sql diff --git a/packages/database/prisma/migrations/20231122210707_add_concurrency_limit_tables_and_columns/migration.sql b/internal-packages/database/prisma/migrations/20231122210707_add_concurrency_limit_tables_and_columns/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231122210707_add_concurrency_limit_tables_and_columns/migration.sql rename to internal-packages/database/prisma/migrations/20231122210707_add_concurrency_limit_tables_and_columns/migration.sql diff --git a/packages/database/prisma/migrations/20231122212600_make_job_queues_optional/migration.sql b/internal-packages/database/prisma/migrations/20231122212600_make_job_queues_optional/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231122212600_make_job_queues_optional/migration.sql rename to internal-packages/database/prisma/migrations/20231122212600_make_job_queues_optional/migration.sql diff --git a/packages/database/prisma/migrations/20231123113308_remove_concurrency_group_from_run/migration.sql b/internal-packages/database/prisma/migrations/20231123113308_remove_concurrency_group_from_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231123113308_remove_concurrency_group_from_run/migration.sql rename to internal-packages/database/prisma/migrations/20231123113308_remove_concurrency_group_from_run/migration.sql diff --git a/packages/database/prisma/migrations/20231123115015_add_concurrency_limit_group_id_to_run_executions/migration.sql b/internal-packages/database/prisma/migrations/20231123115015_add_concurrency_limit_group_id_to_run_executions/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231123115015_add_concurrency_limit_group_id_to_run_executions/migration.sql rename to internal-packages/database/prisma/migrations/20231123115015_add_concurrency_limit_group_id_to_run_executions/migration.sql diff --git a/packages/database/prisma/migrations/20231124131123_add_referral_source_and_company_size/migration.sql b/internal-packages/database/prisma/migrations/20231124131123_add_referral_source_and_company_size/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231124131123_add_referral_source_and_company_size/migration.sql rename to internal-packages/database/prisma/migrations/20231124131123_add_referral_source_and_company_size/migration.sql diff --git a/packages/database/prisma/migrations/20231204163703_add_composite_index_to_triggerdotdev_events_to_speed_up_organization_queries/migration.sql b/internal-packages/database/prisma/migrations/20231204163703_add_composite_index_to_triggerdotdev_events_to_speed_up_organization_queries/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231204163703_add_composite_index_to_triggerdotdev_events_to_speed_up_organization_queries/migration.sql rename to internal-packages/database/prisma/migrations/20231204163703_add_composite_index_to_triggerdotdev_events_to_speed_up_organization_queries/migration.sql diff --git a/packages/database/prisma/migrations/20231206205233_add_execution_failure_count_to_runs/migration.sql b/internal-packages/database/prisma/migrations/20231206205233_add_execution_failure_count_to_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231206205233_add_execution_failure_count_to_runs/migration.sql rename to internal-packages/database/prisma/migrations/20231206205233_add_execution_failure_count_to_runs/migration.sql diff --git a/packages/database/prisma/migrations/20231214103115_add_tunnel_id_to_runtime_environment/migration.sql b/internal-packages/database/prisma/migrations/20231214103115_add_tunnel_id_to_runtime_environment/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20231214103115_add_tunnel_id_to_runtime_environment/migration.sql rename to internal-packages/database/prisma/migrations/20231214103115_add_tunnel_id_to_runtime_environment/migration.sql diff --git a/packages/database/prisma/migrations/20240111142421_created_personal_access_token_and_authorization_code_tables/migration.sql b/internal-packages/database/prisma/migrations/20240111142421_created_personal_access_token_and_authorization_code_tables/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240111142421_created_personal_access_token_and_authorization_code_tables/migration.sql rename to internal-packages/database/prisma/migrations/20240111142421_created_personal_access_token_and_authorization_code_tables/migration.sql diff --git a/packages/database/prisma/migrations/20240111143407_encrypt_the_personal_access_token_in_the_database/migration.sql b/internal-packages/database/prisma/migrations/20240111143407_encrypt_the_personal_access_token_in_the_database/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240111143407_encrypt_the_personal_access_token_in_the_database/migration.sql rename to internal-packages/database/prisma/migrations/20240111143407_encrypt_the_personal_access_token_in_the_database/migration.sql diff --git a/packages/database/prisma/migrations/20240111144844_encrypted_token_now_json_and_added_obfuscated_token_column/migration.sql b/internal-packages/database/prisma/migrations/20240111144844_encrypted_token_now_json_and_added_obfuscated_token_column/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240111144844_encrypted_token_now_json_and_added_obfuscated_token_column/migration.sql rename to internal-packages/database/prisma/migrations/20240111144844_encrypted_token_now_json_and_added_obfuscated_token_column/migration.sql diff --git a/packages/database/prisma/migrations/20240111151921_added_hashed_token_column_which_will_be_used_for_search/migration.sql b/internal-packages/database/prisma/migrations/20240111151921_added_hashed_token_column_which_will_be_used_for_search/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240111151921_added_hashed_token_column_which_will_be_used_for_search/migration.sql rename to internal-packages/database/prisma/migrations/20240111151921_added_hashed_token_column_which_will_be_used_for_search/migration.sql diff --git a/packages/database/prisma/migrations/20240115160657_add_external_ref_to_projects/migration.sql b/internal-packages/database/prisma/migrations/20240115160657_add_external_ref_to_projects/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240115160657_add_external_ref_to_projects/migration.sql rename to internal-packages/database/prisma/migrations/20240115160657_add_external_ref_to_projects/migration.sql diff --git a/packages/database/prisma/migrations/20240116115734_add_background_worker_models/migration.sql b/internal-packages/database/prisma/migrations/20240116115734_add_background_worker_models/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240116115734_add_background_worker_models/migration.sql rename to internal-packages/database/prisma/migrations/20240116115734_add_background_worker_models/migration.sql diff --git a/packages/database/prisma/migrations/20240116162443_added_organization_runs_enabled_column/migration.sql b/internal-packages/database/prisma/migrations/20240116162443_added_organization_runs_enabled_column/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240116162443_added_organization_runs_enabled_column/migration.sql rename to internal-packages/database/prisma/migrations/20240116162443_added_organization_runs_enabled_column/migration.sql diff --git a/packages/database/prisma/migrations/20240116163753_add_task_run_model/migration.sql b/internal-packages/database/prisma/migrations/20240116163753_add_task_run_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240116163753_add_task_run_model/migration.sql rename to internal-packages/database/prisma/migrations/20240116163753_add_task_run_model/migration.sql diff --git a/packages/database/prisma/migrations/20240116165157_add_task_identifier_to_task_runs/migration.sql b/internal-packages/database/prisma/migrations/20240116165157_add_task_identifier_to_task_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240116165157_add_task_identifier_to_task_runs/migration.sql rename to internal-packages/database/prisma/migrations/20240116165157_add_task_identifier_to_task_runs/migration.sql diff --git a/packages/database/prisma/migrations/20240118155050_add_completion_columns_to_task_runs/migration.sql b/internal-packages/database/prisma/migrations/20240118155050_add_completion_columns_to_task_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240118155050_add_completion_columns_to_task_runs/migration.sql rename to internal-packages/database/prisma/migrations/20240118155050_add_completion_columns_to_task_runs/migration.sql diff --git a/packages/database/prisma/migrations/20240118155439_change_payload_output_type_defaults/migration.sql b/internal-packages/database/prisma/migrations/20240118155439_change_payload_output_type_defaults/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240118155439_change_payload_output_type_defaults/migration.sql rename to internal-packages/database/prisma/migrations/20240118155439_change_payload_output_type_defaults/migration.sql diff --git a/packages/database/prisma/migrations/20240118160024_make_task_run_output_optional/migration.sql b/internal-packages/database/prisma/migrations/20240118160024_make_task_run_output_optional/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240118160024_make_task_run_output_optional/migration.sql rename to internal-packages/database/prisma/migrations/20240118160024_make_task_run_output_optional/migration.sql diff --git a/packages/database/prisma/migrations/20240123093539_add_content_hash_to_background_worker/migration.sql b/internal-packages/database/prisma/migrations/20240123093539_add_content_hash_to_background_worker/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240123093539_add_content_hash_to_background_worker/migration.sql rename to internal-packages/database/prisma/migrations/20240123093539_add_content_hash_to_background_worker/migration.sql diff --git a/packages/database/prisma/migrations/20240123193905_restructure_task_runs_with_attempts/migration.sql b/internal-packages/database/prisma/migrations/20240123193905_restructure_task_runs_with_attempts/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240123193905_restructure_task_runs_with_attempts/migration.sql rename to internal-packages/database/prisma/migrations/20240123193905_restructure_task_runs_with_attempts/migration.sql diff --git a/packages/database/prisma/migrations/20240124104013_add_friendly_id_to_v3_tables/migration.sql b/internal-packages/database/prisma/migrations/20240124104013_add_friendly_id_to_v3_tables/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240124104013_add_friendly_id_to_v3_tables/migration.sql rename to internal-packages/database/prisma/migrations/20240124104013_add_friendly_id_to_v3_tables/migration.sql diff --git a/packages/database/prisma/migrations/20240124122331_convert_task_attempt_error_to_json/migration.sql b/internal-packages/database/prisma/migrations/20240124122331_convert_task_attempt_error_to_json/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240124122331_convert_task_attempt_error_to_json/migration.sql rename to internal-packages/database/prisma/migrations/20240124122331_convert_task_attempt_error_to_json/migration.sql diff --git a/packages/database/prisma/migrations/20240126130139_add_locked_to_version_to_task_runs/migration.sql b/internal-packages/database/prisma/migrations/20240126130139_add_locked_to_version_to_task_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240126130139_add_locked_to_version_to_task_runs/migration.sql rename to internal-packages/database/prisma/migrations/20240126130139_add_locked_to_version_to_task_runs/migration.sql diff --git a/packages/database/prisma/migrations/20240126204658_add_parent_attempt_to_task_runs/migration.sql b/internal-packages/database/prisma/migrations/20240126204658_add_parent_attempt_to_task_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240126204658_add_parent_attempt_to_task_runs/migration.sql rename to internal-packages/database/prisma/migrations/20240126204658_add_parent_attempt_to_task_runs/migration.sql diff --git a/packages/database/prisma/migrations/20240129140944_endpoint_deleted_at_column/migration.sql b/internal-packages/database/prisma/migrations/20240129140944_endpoint_deleted_at_column/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240129140944_endpoint_deleted_at_column/migration.sql rename to internal-packages/database/prisma/migrations/20240129140944_endpoint_deleted_at_column/migration.sql diff --git a/packages/database/prisma/migrations/20240129161848_endpoint_nullable_slug_instead_of_deletedat_column/migration.sql b/internal-packages/database/prisma/migrations/20240129161848_endpoint_nullable_slug_instead_of_deletedat_column/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240129161848_endpoint_nullable_slug_instead_of_deletedat_column/migration.sql rename to internal-packages/database/prisma/migrations/20240129161848_endpoint_nullable_slug_instead_of_deletedat_column/migration.sql diff --git a/packages/database/prisma/migrations/20240130165343_add_composite_index_to_job_run_for_job_id_and_created_at/migration.sql b/internal-packages/database/prisma/migrations/20240130165343_add_composite_index_to_job_run_for_job_id_and_created_at/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240130165343_add_composite_index_to_job_run_for_job_id_and_created_at/migration.sql rename to internal-packages/database/prisma/migrations/20240130165343_add_composite_index_to_job_run_for_job_id_and_created_at/migration.sql diff --git a/packages/database/prisma/migrations/20240130205109_add_trace_context_to_task_run/migration.sql b/internal-packages/database/prisma/migrations/20240130205109_add_trace_context_to_task_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240130205109_add_trace_context_to_task_run/migration.sql rename to internal-packages/database/prisma/migrations/20240130205109_add_trace_context_to_task_run/migration.sql diff --git a/packages/database/prisma/migrations/20240131105237_project_deleted_at/migration.sql b/internal-packages/database/prisma/migrations/20240131105237_project_deleted_at/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240131105237_project_deleted_at/migration.sql rename to internal-packages/database/prisma/migrations/20240131105237_project_deleted_at/migration.sql diff --git a/packages/database/prisma/migrations/20240202115155_added_job_run_index_back_in_using_prisma_schema/migration.sql b/internal-packages/database/prisma/migrations/20240202115155_added_job_run_index_back_in_using_prisma_schema/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240202115155_added_job_run_index_back_in_using_prisma_schema/migration.sql rename to internal-packages/database/prisma/migrations/20240202115155_added_job_run_index_back_in_using_prisma_schema/migration.sql diff --git a/packages/database/prisma/migrations/20240206112723_organization_deleted_at/migration.sql b/internal-packages/database/prisma/migrations/20240206112723_organization_deleted_at/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240206112723_organization_deleted_at/migration.sql rename to internal-packages/database/prisma/migrations/20240206112723_organization_deleted_at/migration.sql diff --git a/packages/database/prisma/migrations/20240206133516_integration_connections_can_be_disabled/migration.sql b/internal-packages/database/prisma/migrations/20240206133516_integration_connections_can_be_disabled/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240206133516_integration_connections_can_be_disabled/migration.sql rename to internal-packages/database/prisma/migrations/20240206133516_integration_connections_can_be_disabled/migration.sql diff --git a/packages/database/prisma/migrations/20240207174021_add_unified_task_event_model/migration.sql b/internal-packages/database/prisma/migrations/20240207174021_add_unified_task_event_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240207174021_add_unified_task_event_model/migration.sql rename to internal-packages/database/prisma/migrations/20240207174021_add_unified_task_event_model/migration.sql diff --git a/packages/database/prisma/migrations/20240207195749_move_event_data_to_json/migration.sql b/internal-packages/database/prisma/migrations/20240207195749_move_event_data_to_json/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240207195749_move_event_data_to_json/migration.sql rename to internal-packages/database/prisma/migrations/20240207195749_move_event_data_to_json/migration.sql diff --git a/packages/database/prisma/migrations/20240208124114_changes_to_the_task_event_schema/migration.sql b/internal-packages/database/prisma/migrations/20240208124114_changes_to_the_task_event_schema/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240208124114_changes_to_the_task_event_schema/migration.sql rename to internal-packages/database/prisma/migrations/20240208124114_changes_to_the_task_event_schema/migration.sql diff --git a/packages/database/prisma/migrations/20240208133710_use_bigint_for_event_durations/migration.sql b/internal-packages/database/prisma/migrations/20240208133710_use_bigint_for_event_durations/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240208133710_use_bigint_for_event_durations/migration.sql rename to internal-packages/database/prisma/migrations/20240208133710_use_bigint_for_event_durations/migration.sql diff --git a/packages/database/prisma/migrations/20240209110749_convert_task_fields_to_optional/migration.sql b/internal-packages/database/prisma/migrations/20240209110749_convert_task_fields_to_optional/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240209110749_convert_task_fields_to_optional/migration.sql rename to internal-packages/database/prisma/migrations/20240209110749_convert_task_fields_to_optional/migration.sql diff --git a/packages/database/prisma/migrations/20240209121123_add_task_run_trace_columns/migration.sql b/internal-packages/database/prisma/migrations/20240209121123_add_task_run_trace_columns/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240209121123_add_task_run_trace_columns/migration.sql rename to internal-packages/database/prisma/migrations/20240209121123_add_task_run_trace_columns/migration.sql diff --git a/packages/database/prisma/migrations/20240209153602_add_worker_columns_to_task_event/migration.sql b/internal-packages/database/prisma/migrations/20240209153602_add_worker_columns_to_task_event/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240209153602_add_worker_columns_to_task_event/migration.sql rename to internal-packages/database/prisma/migrations/20240209153602_add_worker_columns_to_task_event/migration.sql diff --git a/packages/database/prisma/migrations/20240212174757_project_version/migration.sql b/internal-packages/database/prisma/migrations/20240212174757_project_version/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240212174757_project_version/migration.sql rename to internal-packages/database/prisma/migrations/20240212174757_project_version/migration.sql diff --git a/packages/database/prisma/migrations/20240213134217_added_sdk_version_and_cli_version_to_background_worker/migration.sql b/internal-packages/database/prisma/migrations/20240213134217_added_sdk_version_and_cli_version_to_background_worker/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240213134217_added_sdk_version_and_cli_version_to_background_worker/migration.sql rename to internal-packages/database/prisma/migrations/20240213134217_added_sdk_version_and_cli_version_to_background_worker/migration.sql diff --git a/packages/database/prisma/migrations/20240213141507_add_task_queues_model/migration.sql b/internal-packages/database/prisma/migrations/20240213141507_add_task_queues_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240213141507_add_task_queues_model/migration.sql rename to internal-packages/database/prisma/migrations/20240213141507_add_task_queues_model/migration.sql diff --git a/packages/database/prisma/migrations/20240213154458_add_image_details/migration.sql b/internal-packages/database/prisma/migrations/20240213154458_add_image_details/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240213154458_add_image_details/migration.sql rename to internal-packages/database/prisma/migrations/20240213154458_add_image_details/migration.sql diff --git a/packages/database/prisma/migrations/20240214172901_add_concurrency_key_to_task_run/migration.sql b/internal-packages/database/prisma/migrations/20240214172901_add_concurrency_key_to_task_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240214172901_add_concurrency_key_to_task_run/migration.sql rename to internal-packages/database/prisma/migrations/20240214172901_add_concurrency_key_to_task_run/migration.sql diff --git a/packages/database/prisma/migrations/20240214173404_add_queue_options_to_task_run/migration.sql b/internal-packages/database/prisma/migrations/20240214173404_add_queue_options_to_task_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240214173404_add_queue_options_to_task_run/migration.sql rename to internal-packages/database/prisma/migrations/20240214173404_add_queue_options_to_task_run/migration.sql diff --git a/packages/database/prisma/migrations/20240214174325_associate_task_runs_with_task_queues/migration.sql b/internal-packages/database/prisma/migrations/20240214174325_associate_task_runs_with_task_queues/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240214174325_associate_task_runs_with_task_queues/migration.sql rename to internal-packages/database/prisma/migrations/20240214174325_associate_task_runs_with_task_queues/migration.sql diff --git a/packages/database/prisma/migrations/20240214175158_add_queue_properties_to_events/migration.sql b/internal-packages/database/prisma/migrations/20240214175158_add_queue_properties_to_events/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240214175158_add_queue_properties_to_events/migration.sql rename to internal-packages/database/prisma/migrations/20240214175158_add_queue_properties_to_events/migration.sql diff --git a/packages/database/prisma/migrations/20240214180324_remove_task_queue_relationship_from_task_runs/migration.sql b/internal-packages/database/prisma/migrations/20240214180324_remove_task_queue_relationship_from_task_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240214180324_remove_task_queue_relationship_from_task_runs/migration.sql rename to internal-packages/database/prisma/migrations/20240214180324_remove_task_queue_relationship_from_task_runs/migration.sql diff --git a/packages/database/prisma/migrations/20240214180709_add_queue_relationship_to_task_run_attempts/migration.sql b/internal-packages/database/prisma/migrations/20240214180709_add_queue_relationship_to_task_run_attempts/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240214180709_add_queue_relationship_to_task_run_attempts/migration.sql rename to internal-packages/database/prisma/migrations/20240214180709_add_queue_relationship_to_task_run_attempts/migration.sql diff --git a/packages/database/prisma/migrations/20240215113146_added_task_run_counter/migration.sql b/internal-packages/database/prisma/migrations/20240215113146_added_task_run_counter/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240215113146_added_task_run_counter/migration.sql rename to internal-packages/database/prisma/migrations/20240215113146_added_task_run_counter/migration.sql diff --git a/packages/database/prisma/migrations/20240215113618_added_number_to_task_run/migration.sql b/internal-packages/database/prisma/migrations/20240215113618_added_number_to_task_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240215113618_added_number_to_task_run/migration.sql rename to internal-packages/database/prisma/migrations/20240215113618_added_number_to_task_run/migration.sql diff --git a/packages/database/prisma/migrations/20240220165410_add_attempt_number_to_task_events/migration.sql b/internal-packages/database/prisma/migrations/20240220165410_add_attempt_number_to_task_events/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240220165410_add_attempt_number_to_task_events/migration.sql rename to internal-packages/database/prisma/migrations/20240220165410_add_attempt_number_to_task_events/migration.sql diff --git a/packages/database/prisma/migrations/20240221140555_add_queue_and_retry_config_to_background_worker_tasks/migration.sql b/internal-packages/database/prisma/migrations/20240221140555_add_queue_and_retry_config_to_background_worker_tasks/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240221140555_add_queue_and_retry_config_to_background_worker_tasks/migration.sql rename to internal-packages/database/prisma/migrations/20240221140555_add_queue_and_retry_config_to_background_worker_tasks/migration.sql diff --git a/packages/database/prisma/migrations/20240223115106_batch_trigger_models/migration.sql b/internal-packages/database/prisma/migrations/20240223115106_batch_trigger_models/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240223115106_batch_trigger_models/migration.sql rename to internal-packages/database/prisma/migrations/20240223115106_batch_trigger_models/migration.sql diff --git a/packages/database/prisma/migrations/20240223121156_added_environment_variable_and_environment_variable_value_tables/migration.sql b/internal-packages/database/prisma/migrations/20240223121156_added_environment_variable_and_environment_variable_value_tables/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240223121156_added_environment_variable_and_environment_variable_value_tables/migration.sql rename to internal-packages/database/prisma/migrations/20240223121156_added_environment_variable_and_environment_variable_value_tables/migration.sql diff --git a/packages/database/prisma/migrations/20240224132039_change_image_details_unique_constraint/migration.sql b/internal-packages/database/prisma/migrations/20240224132039_change_image_details_unique_constraint/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240224132039_change_image_details_unique_constraint/migration.sql rename to internal-packages/database/prisma/migrations/20240224132039_change_image_details_unique_constraint/migration.sql diff --git a/packages/database/prisma/migrations/20240227142146_environment_variable_added_friendly_id/migration.sql b/internal-packages/database/prisma/migrations/20240227142146_environment_variable_added_friendly_id/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240227142146_environment_variable_added_friendly_id/migration.sql rename to internal-packages/database/prisma/migrations/20240227142146_environment_variable_added_friendly_id/migration.sql diff --git a/packages/database/prisma/migrations/20240227144742_task_run_added_is_test/migration.sql b/internal-packages/database/prisma/migrations/20240227144742_task_run_added_is_test/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240227144742_task_run_added_is_test/migration.sql rename to internal-packages/database/prisma/migrations/20240227144742_task_run_added_is_test/migration.sql diff --git a/packages/database/prisma/migrations/20240227170811_add_is_cancelled_to_task_events/migration.sql b/internal-packages/database/prisma/migrations/20240227170811_add_is_cancelled_to_task_events/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240227170811_add_is_cancelled_to_task_events/migration.sql rename to internal-packages/database/prisma/migrations/20240227170811_add_is_cancelled_to_task_events/migration.sql diff --git a/packages/database/prisma/migrations/20240228114913_add_checkpoint_model/migration.sql b/internal-packages/database/prisma/migrations/20240228114913_add_checkpoint_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240228114913_add_checkpoint_model/migration.sql rename to internal-packages/database/prisma/migrations/20240228114913_add_checkpoint_model/migration.sql diff --git a/packages/database/prisma/migrations/20240228161621_add_run_is_test_to_task_events/migration.sql b/internal-packages/database/prisma/migrations/20240228161621_add_run_is_test_to_task_events/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240228161621_add_run_is_test_to_task_events/migration.sql rename to internal-packages/database/prisma/migrations/20240228161621_add_run_is_test_to_task_events/migration.sql diff --git a/packages/database/prisma/migrations/20240229141613_add_batch_id_to_task_events/migration.sql b/internal-packages/database/prisma/migrations/20240229141613_add_batch_id_to_task_events/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240229141613_add_batch_id_to_task_events/migration.sql rename to internal-packages/database/prisma/migrations/20240229141613_add_batch_id_to_task_events/migration.sql diff --git a/packages/database/prisma/migrations/20240305111659_add_deployment_models/migration.sql b/internal-packages/database/prisma/migrations/20240305111659_add_deployment_models/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240305111659_add_deployment_models/migration.sql rename to internal-packages/database/prisma/migrations/20240305111659_add_deployment_models/migration.sql diff --git a/packages/database/prisma/migrations/20240305154054_use_external_build_data_json_column/migration.sql b/internal-packages/database/prisma/migrations/20240305154054_use_external_build_data_json_column/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240305154054_use_external_build_data_json_column/migration.sql rename to internal-packages/database/prisma/migrations/20240305154054_use_external_build_data_json_column/migration.sql diff --git a/packages/database/prisma/migrations/20240307095223_add_content_hash_to_worker_deployments/migration.sql b/internal-packages/database/prisma/migrations/20240307095223_add_content_hash_to_worker_deployments/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240307095223_add_content_hash_to_worker_deployments/migration.sql rename to internal-packages/database/prisma/migrations/20240307095223_add_content_hash_to_worker_deployments/migration.sql diff --git a/packages/database/prisma/migrations/20240307104333_move_image_details_to_deployments/migration.sql b/internal-packages/database/prisma/migrations/20240307104333_move_image_details_to_deployments/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240307104333_move_image_details_to_deployments/migration.sql rename to internal-packages/database/prisma/migrations/20240307104333_move_image_details_to_deployments/migration.sql diff --git a/packages/database/prisma/migrations/20240308203644_add_deployment_error_columns/migration.sql b/internal-packages/database/prisma/migrations/20240308203644_add_deployment_error_columns/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240308203644_add_deployment_error_columns/migration.sql rename to internal-packages/database/prisma/migrations/20240308203644_add_deployment_error_columns/migration.sql diff --git a/packages/database/prisma/migrations/20240311135706_add_triggered_by_to_deployments/migration.sql b/internal-packages/database/prisma/migrations/20240311135706_add_triggered_by_to_deployments/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240311135706_add_triggered_by_to_deployments/migration.sql rename to internal-packages/database/prisma/migrations/20240311135706_add_triggered_by_to_deployments/migration.sql diff --git a/packages/database/prisma/migrations/20240312095501_add_status_to_task_runs/migration.sql b/internal-packages/database/prisma/migrations/20240312095501_add_status_to_task_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240312095501_add_status_to_task_runs/migration.sql rename to internal-packages/database/prisma/migrations/20240312095501_add_status_to_task_runs/migration.sql diff --git a/packages/database/prisma/migrations/20240312095826_add_interrupted_status_to_runs/migration.sql b/internal-packages/database/prisma/migrations/20240312095826_add_interrupted_status_to_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240312095826_add_interrupted_status_to_runs/migration.sql rename to internal-packages/database/prisma/migrations/20240312095826_add_interrupted_status_to_runs/migration.sql diff --git a/packages/database/prisma/migrations/20240312105844_add_system_failure_status_to_task_run_status/migration.sql b/internal-packages/database/prisma/migrations/20240312105844_add_system_failure_status_to_task_run_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240312105844_add_system_failure_status_to_task_run_status/migration.sql rename to internal-packages/database/prisma/migrations/20240312105844_add_system_failure_status_to_task_run_status/migration.sql diff --git a/packages/database/prisma/migrations/20240312125252_add_status_to_batch_run/migration.sql b/internal-packages/database/prisma/migrations/20240312125252_add_status_to_batch_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240312125252_add_status_to_batch_run/migration.sql rename to internal-packages/database/prisma/migrations/20240312125252_add_status_to_batch_run/migration.sql diff --git a/packages/database/prisma/migrations/20240312131122_add_task_run_attempt_to_batch_run_items/migration.sql b/internal-packages/database/prisma/migrations/20240312131122_add_task_run_attempt_to_batch_run_items/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240312131122_add_task_run_attempt_to_batch_run_items/migration.sql rename to internal-packages/database/prisma/migrations/20240312131122_add_task_run_attempt_to_batch_run_items/migration.sql diff --git a/packages/database/prisma/migrations/20240313110150_add_environment_to_task_run_attempts/migration.sql b/internal-packages/database/prisma/migrations/20240313110150_add_environment_to_task_run_attempts/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240313110150_add_environment_to_task_run_attempts/migration.sql rename to internal-packages/database/prisma/migrations/20240313110150_add_environment_to_task_run_attempts/migration.sql diff --git a/packages/database/prisma/migrations/20240316174721_add_run_and_metadata_to_checkpoint/migration.sql b/internal-packages/database/prisma/migrations/20240316174721_add_run_and_metadata_to_checkpoint/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240316174721_add_run_and_metadata_to_checkpoint/migration.sql rename to internal-packages/database/prisma/migrations/20240316174721_add_run_and_metadata_to_checkpoint/migration.sql diff --git a/packages/database/prisma/migrations/20240318135831_add_checkpoint_restore_event/migration.sql b/internal-packages/database/prisma/migrations/20240318135831_add_checkpoint_restore_event/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240318135831_add_checkpoint_restore_event/migration.sql rename to internal-packages/database/prisma/migrations/20240318135831_add_checkpoint_restore_event/migration.sql diff --git a/packages/database/prisma/migrations/20240318170823_add_image_ref_to_checkpoint/migration.sql b/internal-packages/database/prisma/migrations/20240318170823_add_image_ref_to_checkpoint/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240318170823_add_image_ref_to_checkpoint/migration.sql rename to internal-packages/database/prisma/migrations/20240318170823_add_image_ref_to_checkpoint/migration.sql diff --git a/packages/database/prisma/migrations/20240319120645_convert_start_time_to_nanoseconds_since_epoch/migration.sql b/internal-packages/database/prisma/migrations/20240319120645_convert_start_time_to_nanoseconds_since_epoch/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240319120645_convert_start_time_to_nanoseconds_since_epoch/migration.sql rename to internal-packages/database/prisma/migrations/20240319120645_convert_start_time_to_nanoseconds_since_epoch/migration.sql diff --git a/packages/database/prisma/migrations/20240319121124_what_migration_is_this/migration.sql b/internal-packages/database/prisma/migrations/20240319121124_what_migration_is_this/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240319121124_what_migration_is_this/migration.sql rename to internal-packages/database/prisma/migrations/20240319121124_what_migration_is_this/migration.sql diff --git a/packages/database/prisma/migrations/20240320100720_add_timed_out_status_to_deployments/migration.sql b/internal-packages/database/prisma/migrations/20240320100720_add_timed_out_status_to_deployments/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240320100720_add_timed_out_status_to_deployments/migration.sql rename to internal-packages/database/prisma/migrations/20240320100720_add_timed_out_status_to_deployments/migration.sql diff --git a/packages/database/prisma/migrations/20240322165042_organization_v3_enabled_defaults_to_false/migration.sql b/internal-packages/database/prisma/migrations/20240322165042_organization_v3_enabled_defaults_to_false/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240322165042_organization_v3_enabled_defaults_to_false/migration.sql rename to internal-packages/database/prisma/migrations/20240322165042_organization_v3_enabled_defaults_to_false/migration.sql diff --git a/packages/database/prisma/migrations/20240322172035_add_checkpoint_event_to_dependencies/migration.sql b/internal-packages/database/prisma/migrations/20240322172035_add_checkpoint_event_to_dependencies/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240322172035_add_checkpoint_event_to_dependencies/migration.sql rename to internal-packages/database/prisma/migrations/20240322172035_add_checkpoint_event_to_dependencies/migration.sql diff --git a/packages/database/prisma/migrations/20240325224419_add_output_type_to_task_events/migration.sql b/internal-packages/database/prisma/migrations/20240325224419_add_output_type_to_task_events/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240325224419_add_output_type_to_task_events/migration.sql rename to internal-packages/database/prisma/migrations/20240325224419_add_output_type_to_task_events/migration.sql diff --git a/packages/database/prisma/migrations/20240326145956_add_payload_columns_to_task_event/migration.sql b/internal-packages/database/prisma/migrations/20240326145956_add_payload_columns_to_task_event/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240326145956_add_payload_columns_to_task_event/migration.sql rename to internal-packages/database/prisma/migrations/20240326145956_add_payload_columns_to_task_event/migration.sql diff --git a/packages/database/prisma/migrations/20240327121557_add_machine_config_to_worker_task/migration.sql b/internal-packages/database/prisma/migrations/20240327121557_add_machine_config_to_worker_task/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240327121557_add_machine_config_to_worker_task/migration.sql rename to internal-packages/database/prisma/migrations/20240327121557_add_machine_config_to_worker_task/migration.sql diff --git a/packages/database/prisma/migrations/20240329142454_add_concurrency_limit_columns/migration.sql b/internal-packages/database/prisma/migrations/20240329142454_add_concurrency_limit_columns/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240329142454_add_concurrency_limit_columns/migration.sql rename to internal-packages/database/prisma/migrations/20240329142454_add_concurrency_limit_columns/migration.sql diff --git a/packages/database/prisma/migrations/20240402105424_set_default_env_concurrency_limit_to_5/migration.sql b/internal-packages/database/prisma/migrations/20240402105424_set_default_env_concurrency_limit_to_5/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240402105424_set_default_env_concurrency_limit_to_5/migration.sql rename to internal-packages/database/prisma/migrations/20240402105424_set_default_env_concurrency_limit_to_5/migration.sql diff --git a/packages/database/prisma/migrations/20240404150051_add_crashed_task_run_status/migration.sql b/internal-packages/database/prisma/migrations/20240404150051_add_crashed_task_run_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240404150051_add_crashed_task_run_status/migration.sql rename to internal-packages/database/prisma/migrations/20240404150051_add_crashed_task_run_status/migration.sql diff --git a/packages/database/prisma/migrations/20240409090907_add_waiting_for_deploy_status/migration.sql b/internal-packages/database/prisma/migrations/20240409090907_add_waiting_for_deploy_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240409090907_add_waiting_for_deploy_status/migration.sql rename to internal-packages/database/prisma/migrations/20240409090907_add_waiting_for_deploy_status/migration.sql diff --git a/packages/database/prisma/migrations/20240411135457_task_schedules_for_v3/migration.sql b/internal-packages/database/prisma/migrations/20240411135457_task_schedules_for_v3/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240411135457_task_schedules_for_v3/migration.sql rename to internal-packages/database/prisma/migrations/20240411135457_task_schedules_for_v3/migration.sql diff --git a/packages/database/prisma/migrations/20240411145517_added_trigger_source_to_background_worker_task/migration.sql b/internal-packages/database/prisma/migrations/20240411145517_added_trigger_source_to_background_worker_task/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240411145517_added_trigger_source_to_background_worker_task/migration.sql rename to internal-packages/database/prisma/migrations/20240411145517_added_trigger_source_to_background_worker_task/migration.sql diff --git a/packages/database/prisma/migrations/20240412151157_add_operational_task_schedule_columns/migration.sql b/internal-packages/database/prisma/migrations/20240412151157_add_operational_task_schedule_columns/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240412151157_add_operational_task_schedule_columns/migration.sql rename to internal-packages/database/prisma/migrations/20240412151157_add_operational_task_schedule_columns/migration.sql diff --git a/packages/database/prisma/migrations/20240415134559_create_runtime_env_session_model/migration.sql b/internal-packages/database/prisma/migrations/20240415134559_create_runtime_env_session_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240415134559_create_runtime_env_session_model/migration.sql rename to internal-packages/database/prisma/migrations/20240415134559_create_runtime_env_session_model/migration.sql diff --git a/packages/database/prisma/migrations/20240415135132_make_attempt_dependencies_an_array/migration.sql b/internal-packages/database/prisma/migrations/20240415135132_make_attempt_dependencies_an_array/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240415135132_make_attempt_dependencies_an_array/migration.sql rename to internal-packages/database/prisma/migrations/20240415135132_make_attempt_dependencies_an_array/migration.sql diff --git a/packages/database/prisma/migrations/20240415143325_remove_attempt_unique_constraint_from_batch_runs/migration.sql b/internal-packages/database/prisma/migrations/20240415143325_remove_attempt_unique_constraint_from_batch_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240415143325_remove_attempt_unique_constraint_from_batch_runs/migration.sql rename to internal-packages/database/prisma/migrations/20240415143325_remove_attempt_unique_constraint_from_batch_runs/migration.sql diff --git a/packages/database/prisma/migrations/20240416100646_add_cron_description_column/migration.sql b/internal-packages/database/prisma/migrations/20240416100646_add_cron_description_column/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240416100646_add_cron_description_column/migration.sql rename to internal-packages/database/prisma/migrations/20240416100646_add_cron_description_column/migration.sql diff --git a/packages/database/prisma/migrations/20240417083233_make_schedule_columns_more_generic/migration.sql b/internal-packages/database/prisma/migrations/20240417083233_make_schedule_columns_more_generic/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240417083233_make_schedule_columns_more_generic/migration.sql rename to internal-packages/database/prisma/migrations/20240417083233_make_schedule_columns_more_generic/migration.sql diff --git a/packages/database/prisma/migrations/20240417142604_dont_delete_task_runs_if_schedules_are_deleted/migration.sql b/internal-packages/database/prisma/migrations/20240417142604_dont_delete_task_runs_if_schedules_are_deleted/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240417142604_dont_delete_task_runs_if_schedules_are_deleted/migration.sql rename to internal-packages/database/prisma/migrations/20240417142604_dont_delete_task_runs_if_schedules_are_deleted/migration.sql diff --git a/packages/database/prisma/migrations/20240418092931_make_idempotency_key_optional/migration.sql b/internal-packages/database/prisma/migrations/20240418092931_make_idempotency_key_optional/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240418092931_make_idempotency_key_optional/migration.sql rename to internal-packages/database/prisma/migrations/20240418092931_make_idempotency_key_optional/migration.sql diff --git a/packages/database/prisma/migrations/20240418093153_add_idempotency_key_to_task_event/migration.sql b/internal-packages/database/prisma/migrations/20240418093153_add_idempotency_key_to_task_event/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240418093153_add_idempotency_key_to_task_event/migration.sql rename to internal-packages/database/prisma/migrations/20240418093153_add_idempotency_key_to_task_event/migration.sql diff --git a/packages/database/prisma/migrations/20240418100819_make_batch_task_run_idempotency_key_optional/migration.sql b/internal-packages/database/prisma/migrations/20240418100819_make_batch_task_run_idempotency_key_optional/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240418100819_make_batch_task_run_idempotency_key_optional/migration.sql rename to internal-packages/database/prisma/migrations/20240418100819_make_batch_task_run_idempotency_key_optional/migration.sql diff --git a/packages/database/prisma/migrations/20240418114019_make_batch_run_item_run_id_non_unique/migration.sql b/internal-packages/database/prisma/migrations/20240418114019_make_batch_run_item_run_id_non_unique/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240418114019_make_batch_run_item_run_id_non_unique/migration.sql rename to internal-packages/database/prisma/migrations/20240418114019_make_batch_run_item_run_id_non_unique/migration.sql diff --git a/packages/database/prisma/migrations/20240424210540_change_on_delete_ref_action_on_run_execution/migration.sql b/internal-packages/database/prisma/migrations/20240424210540_change_on_delete_ref_action_on_run_execution/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240424210540_change_on_delete_ref_action_on_run_execution/migration.sql rename to internal-packages/database/prisma/migrations/20240424210540_change_on_delete_ref_action_on_run_execution/migration.sql diff --git a/packages/database/prisma/migrations/20240425114134_task_run_compound_index_for_project_id_created_at_and_task_identifier/migration.sql b/internal-packages/database/prisma/migrations/20240425114134_task_run_compound_index_for_project_id_created_at_and_task_identifier/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240425114134_task_run_compound_index_for_project_id_created_at_and_task_identifier/migration.sql rename to internal-packages/database/prisma/migrations/20240425114134_task_run_compound_index_for_project_id_created_at_and_task_identifier/migration.sql diff --git a/packages/database/prisma/migrations/20240425122814_add_alert_schema/migration.sql b/internal-packages/database/prisma/migrations/20240425122814_add_alert_schema/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240425122814_add_alert_schema/migration.sql rename to internal-packages/database/prisma/migrations/20240425122814_add_alert_schema/migration.sql diff --git a/packages/database/prisma/migrations/20240425131147_add_enabled_flag_to_alert_channels/migration.sql b/internal-packages/database/prisma/migrations/20240425131147_add_enabled_flag_to_alert_channels/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240425131147_add_enabled_flag_to_alert_channels/migration.sql rename to internal-packages/database/prisma/migrations/20240425131147_add_enabled_flag_to_alert_channels/migration.sql diff --git a/packages/database/prisma/migrations/20240426095144_add_deployment_success_alert_type/migration.sql b/internal-packages/database/prisma/migrations/20240426095144_add_deployment_success_alert_type/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240426095144_add_deployment_success_alert_type/migration.sql rename to internal-packages/database/prisma/migrations/20240426095144_add_deployment_success_alert_type/migration.sql diff --git a/packages/database/prisma/migrations/20240426095622_add_deduplication_key_to_alert_channels/migration.sql b/internal-packages/database/prisma/migrations/20240426095622_add_deduplication_key_to_alert_channels/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240426095622_add_deduplication_key_to_alert_channels/migration.sql rename to internal-packages/database/prisma/migrations/20240426095622_add_deduplication_key_to_alert_channels/migration.sql diff --git a/packages/database/prisma/migrations/20240426102405_add_unique_deduplication_index_to_alert_channels/migration.sql b/internal-packages/database/prisma/migrations/20240426102405_add_unique_deduplication_index_to_alert_channels/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240426102405_add_unique_deduplication_index_to_alert_channels/migration.sql rename to internal-packages/database/prisma/migrations/20240426102405_add_unique_deduplication_index_to_alert_channels/migration.sql diff --git a/packages/database/prisma/migrations/20240428142050_add_models_for_slack_integration/migration.sql b/internal-packages/database/prisma/migrations/20240428142050_add_models_for_slack_integration/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240428142050_add_models_for_slack_integration/migration.sql rename to internal-packages/database/prisma/migrations/20240428142050_add_models_for_slack_integration/migration.sql diff --git a/packages/database/prisma/migrations/20240428150144_org_integration_non_optional_fields/migration.sql b/internal-packages/database/prisma/migrations/20240428150144_org_integration_non_optional_fields/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240428150144_org_integration_non_optional_fields/migration.sql rename to internal-packages/database/prisma/migrations/20240428150144_org_integration_non_optional_fields/migration.sql diff --git a/packages/database/prisma/migrations/20240430101936_add_lazy_attempt_support_flag_to_workers/migration.sql b/internal-packages/database/prisma/migrations/20240430101936_add_lazy_attempt_support_flag_to_workers/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240430101936_add_lazy_attempt_support_flag_to_workers/migration.sql rename to internal-packages/database/prisma/migrations/20240430101936_add_lazy_attempt_support_flag_to_workers/migration.sql diff --git a/packages/database/prisma/migrations/20240430110419_task_run_indexes_projectid_task_identifier_and_status/migration.sql b/internal-packages/database/prisma/migrations/20240430110419_task_run_indexes_projectid_task_identifier_and_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240430110419_task_run_indexes_projectid_task_identifier_and_status/migration.sql rename to internal-packages/database/prisma/migrations/20240430110419_task_run_indexes_projectid_task_identifier_and_status/migration.sql diff --git a/packages/database/prisma/migrations/20240430110717_task_run_compound_index_projectid_task_identifier_and_status/migration.sql b/internal-packages/database/prisma/migrations/20240430110717_task_run_compound_index_projectid_task_identifier_and_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240430110717_task_run_compound_index_projectid_task_identifier_and_status/migration.sql rename to internal-packages/database/prisma/migrations/20240430110717_task_run_compound_index_projectid_task_identifier_and_status/migration.sql diff --git a/packages/database/prisma/migrations/20240507113449_add_alert_storage/migration.sql b/internal-packages/database/prisma/migrations/20240507113449_add_alert_storage/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240507113449_add_alert_storage/migration.sql rename to internal-packages/database/prisma/migrations/20240507113449_add_alert_storage/migration.sql diff --git a/packages/database/prisma/migrations/20240517105021_add_environment_types_to_alert_channel/migration.sql b/internal-packages/database/prisma/migrations/20240517105021_add_environment_types_to_alert_channel/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240517105021_add_environment_types_to_alert_channel/migration.sql rename to internal-packages/database/prisma/migrations/20240517105021_add_environment_types_to_alert_channel/migration.sql diff --git a/packages/database/prisma/migrations/20240517105224_remove_test_alert_type/migration.sql b/internal-packages/database/prisma/migrations/20240517105224_remove_test_alert_type/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240517105224_remove_test_alert_type/migration.sql rename to internal-packages/database/prisma/migrations/20240517105224_remove_test_alert_type/migration.sql diff --git a/packages/database/prisma/migrations/20240517135206_created_bulk_action_group_and_bulk_action_item_for_canceling_and_replaying_in_bulk/migration.sql b/internal-packages/database/prisma/migrations/20240517135206_created_bulk_action_group_and_bulk_action_item_for_canceling_and_replaying_in_bulk/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240517135206_created_bulk_action_group_and_bulk_action_item_for_canceling_and_replaying_in_bulk/migration.sql rename to internal-packages/database/prisma/migrations/20240517135206_created_bulk_action_group_and_bulk_action_item_for_canceling_and_replaying_in_bulk/migration.sql diff --git a/packages/database/prisma/migrations/20240517164246_bulk_action_item_source_run_id_is_required/migration.sql b/internal-packages/database/prisma/migrations/20240517164246_bulk_action_item_source_run_id_is_required/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240517164246_bulk_action_item_source_run_id_is_required/migration.sql rename to internal-packages/database/prisma/migrations/20240517164246_bulk_action_item_source_run_id_is_required/migration.sql diff --git a/packages/database/prisma/migrations/20240517164924_bulk_action_item_added_failed_state_with_error/migration.sql b/internal-packages/database/prisma/migrations/20240517164924_bulk_action_item_added_failed_state_with_error/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240517164924_bulk_action_item_added_failed_state_with_error/migration.sql rename to internal-packages/database/prisma/migrations/20240517164924_bulk_action_item_added_failed_state_with_error/migration.sql diff --git a/packages/database/prisma/migrations/20240520112812_create_deferred_scheduled_event_model/migration.sql b/internal-packages/database/prisma/migrations/20240520112812_create_deferred_scheduled_event_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240520112812_create_deferred_scheduled_event_model/migration.sql rename to internal-packages/database/prisma/migrations/20240520112812_create_deferred_scheduled_event_model/migration.sql diff --git a/packages/database/prisma/migrations/20240522130825_org_v2_enabled_flag_defaults_to_false/migration.sql b/internal-packages/database/prisma/migrations/20240522130825_org_v2_enabled_flag_defaults_to_false/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240522130825_org_v2_enabled_flag_defaults_to_false/migration.sql rename to internal-packages/database/prisma/migrations/20240522130825_org_v2_enabled_flag_defaults_to_false/migration.sql diff --git a/packages/database/prisma/migrations/20240522134117_organization_added_has_requested_v3_column/migration.sql b/internal-packages/database/prisma/migrations/20240522134117_organization_added_has_requested_v3_column/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240522134117_organization_added_has_requested_v3_column/migration.sql rename to internal-packages/database/prisma/migrations/20240522134117_organization_added_has_requested_v3_column/migration.sql diff --git a/packages/database/prisma/migrations/20240523081426_added_task_run_number_counter_with_environment_as_well_as_task_identifier/migration.sql b/internal-packages/database/prisma/migrations/20240523081426_added_task_run_number_counter_with_environment_as_well_as_task_identifier/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240523081426_added_task_run_number_counter_with_environment_as_well_as_task_identifier/migration.sql rename to internal-packages/database/prisma/migrations/20240523081426_added_task_run_number_counter_with_environment_as_well_as_task_identifier/migration.sql diff --git a/packages/database/prisma/migrations/20240523135511_added_task_event_trace_id_index/migration.sql b/internal-packages/database/prisma/migrations/20240523135511_added_task_event_trace_id_index/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240523135511_added_task_event_trace_id_index/migration.sql rename to internal-packages/database/prisma/migrations/20240523135511_added_task_event_trace_id_index/migration.sql diff --git a/packages/database/prisma/migrations/20240606090155_add_v2_marqs_enabled_flag_on_org/migration.sql b/internal-packages/database/prisma/migrations/20240606090155_add_v2_marqs_enabled_flag_on_org/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240606090155_add_v2_marqs_enabled_flag_on_org/migration.sql rename to internal-packages/database/prisma/migrations/20240606090155_add_v2_marqs_enabled_flag_on_org/migration.sql diff --git a/packages/database/prisma/migrations/20240610183406_added_schedule_instances_limit/migration.sql b/internal-packages/database/prisma/migrations/20240610183406_added_schedule_instances_limit/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240610183406_added_schedule_instances_limit/migration.sql rename to internal-packages/database/prisma/migrations/20240610183406_added_schedule_instances_limit/migration.sql diff --git a/packages/database/prisma/migrations/20240610195009_add_an_index_on_span_id/migration.sql b/internal-packages/database/prisma/migrations/20240610195009_add_an_index_on_span_id/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240610195009_add_an_index_on_span_id/migration.sql rename to internal-packages/database/prisma/migrations/20240610195009_add_an_index_on_span_id/migration.sql diff --git a/packages/database/prisma/migrations/20240611104018_add_started_at_to_task_runs/migration.sql b/internal-packages/database/prisma/migrations/20240611104018_add_started_at_to_task_runs/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240611104018_add_started_at_to_task_runs/migration.sql rename to internal-packages/database/prisma/migrations/20240611104018_add_started_at_to_task_runs/migration.sql diff --git a/packages/database/prisma/migrations/20240611113047_change_schedules_limit_to_maximum_schedules_limit/migration.sql b/internal-packages/database/prisma/migrations/20240611113047_change_schedules_limit_to_maximum_schedules_limit/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240611113047_change_schedules_limit_to_maximum_schedules_limit/migration.sql rename to internal-packages/database/prisma/migrations/20240611113047_change_schedules_limit_to_maximum_schedules_limit/migration.sql diff --git a/packages/database/prisma/migrations/20240611115843_add_usage_duration_columns/migration.sql b/internal-packages/database/prisma/migrations/20240611115843_add_usage_duration_columns/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240611115843_add_usage_duration_columns/migration.sql rename to internal-packages/database/prisma/migrations/20240611115843_add_usage_duration_columns/migration.sql diff --git a/packages/database/prisma/migrations/20240611133630_added_timezone_to_task_schedule_defaults_to_null_which_is_utc/migration.sql b/internal-packages/database/prisma/migrations/20240611133630_added_timezone_to_task_schedule_defaults_to_null_which_is_utc/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240611133630_added_timezone_to_task_schedule_defaults_to_null_which_is_utc/migration.sql rename to internal-packages/database/prisma/migrations/20240611133630_added_timezone_to_task_schedule_defaults_to_null_which_is_utc/migration.sql diff --git a/packages/database/prisma/migrations/20240611143911_add_cost_in_cents_on_task_run/migration.sql b/internal-packages/database/prisma/migrations/20240611143911_add_cost_in_cents_on_task_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240611143911_add_cost_in_cents_on_task_run/migration.sql rename to internal-packages/database/prisma/migrations/20240611143911_add_cost_in_cents_on_task_run/migration.sql diff --git a/packages/database/prisma/migrations/20240612091006_add_machine_presets_data_to_task_events/migration.sql b/internal-packages/database/prisma/migrations/20240612091006_add_machine_presets_data_to_task_events/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240612091006_add_machine_presets_data_to_task_events/migration.sql rename to internal-packages/database/prisma/migrations/20240612091006_add_machine_presets_data_to_task_events/migration.sql diff --git a/packages/database/prisma/migrations/20240612171759_add_task_run_base_cost/migration.sql b/internal-packages/database/prisma/migrations/20240612171759_add_task_run_base_cost/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240612171759_add_task_run_base_cost/migration.sql rename to internal-packages/database/prisma/migrations/20240612171759_add_task_run_base_cost/migration.sql diff --git a/packages/database/prisma/migrations/20240612175820_add_usage_cost_in_cents_to_task_event/migration.sql b/internal-packages/database/prisma/migrations/20240612175820_add_usage_cost_in_cents_to_task_event/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240612175820_add_usage_cost_in_cents_to_task_event/migration.sql rename to internal-packages/database/prisma/migrations/20240612175820_add_usage_cost_in_cents_to_task_event/migration.sql diff --git a/packages/database/prisma/migrations/20240613082558_add_machine_preset_to_task_run/migration.sql b/internal-packages/database/prisma/migrations/20240613082558_add_machine_preset_to_task_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240613082558_add_machine_preset_to_task_run/migration.sql rename to internal-packages/database/prisma/migrations/20240613082558_add_machine_preset_to_task_run/migration.sql diff --git a/packages/database/prisma/migrations/20240613115526_delete_task_schedule_timezone_column/migration.sql b/internal-packages/database/prisma/migrations/20240613115526_delete_task_schedule_timezone_column/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240613115526_delete_task_schedule_timezone_column/migration.sql rename to internal-packages/database/prisma/migrations/20240613115526_delete_task_schedule_timezone_column/migration.sql diff --git a/packages/database/prisma/migrations/20240613115623_task_schedule_timezone_column_defaults_to_utc/migration.sql b/internal-packages/database/prisma/migrations/20240613115623_task_schedule_timezone_column_defaults_to_utc/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240613115623_task_schedule_timezone_column_defaults_to_utc/migration.sql rename to internal-packages/database/prisma/migrations/20240613115623_task_schedule_timezone_column_defaults_to_utc/migration.sql diff --git a/packages/database/prisma/migrations/20240625095006_add_run_id_index_to_task_events/migration.sql b/internal-packages/database/prisma/migrations/20240625095006_add_run_id_index_to_task_events/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240625095006_add_run_id_index_to_task_events/migration.sql rename to internal-packages/database/prisma/migrations/20240625095006_add_run_id_index_to_task_events/migration.sql diff --git a/packages/database/prisma/migrations/20240629082837_add_task_run_delay_changes/migration.sql b/internal-packages/database/prisma/migrations/20240629082837_add_task_run_delay_changes/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240629082837_add_task_run_delay_changes/migration.sql rename to internal-packages/database/prisma/migrations/20240629082837_add_task_run_delay_changes/migration.sql diff --git a/packages/database/prisma/migrations/20240630204935_add_ttl_schema_changes/migration.sql b/internal-packages/database/prisma/migrations/20240630204935_add_ttl_schema_changes/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240630204935_add_ttl_schema_changes/migration.sql rename to internal-packages/database/prisma/migrations/20240630204935_add_ttl_schema_changes/migration.sql diff --git a/packages/database/prisma/migrations/20240702131302_add_max_attempts_to_task_run/migration.sql b/internal-packages/database/prisma/migrations/20240702131302_add_max_attempts_to_task_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240702131302_add_max_attempts_to_task_run/migration.sql rename to internal-packages/database/prisma/migrations/20240702131302_add_max_attempts_to_task_run/migration.sql diff --git a/packages/database/prisma/migrations/20240704170301_scope_idempotency_key_to_task_identifier/migration.sql b/internal-packages/database/prisma/migrations/20240704170301_scope_idempotency_key_to_task_identifier/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240704170301_scope_idempotency_key_to_task_identifier/migration.sql rename to internal-packages/database/prisma/migrations/20240704170301_scope_idempotency_key_to_task_identifier/migration.sql diff --git a/packages/database/prisma/migrations/20240704205712_add_schedule_id_index_to_task_run/migration.sql b/internal-packages/database/prisma/migrations/20240704205712_add_schedule_id_index_to_task_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240704205712_add_schedule_id_index_to_task_run/migration.sql rename to internal-packages/database/prisma/migrations/20240704205712_add_schedule_id_index_to_task_run/migration.sql diff --git a/packages/database/prisma/migrations/20240707190354_add_built_at_to_worker_deployment/migration.sql b/internal-packages/database/prisma/migrations/20240707190354_add_built_at_to_worker_deployment/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240707190354_add_built_at_to_worker_deployment/migration.sql rename to internal-packages/database/prisma/migrations/20240707190354_add_built_at_to_worker_deployment/migration.sql diff --git a/packages/database/prisma/migrations/20240717101035_add_task_schedule_type_dynamic_and_static/migration.sql b/internal-packages/database/prisma/migrations/20240717101035_add_task_schedule_type_dynamic_and_static/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240717101035_add_task_schedule_type_dynamic_and_static/migration.sql rename to internal-packages/database/prisma/migrations/20240717101035_add_task_schedule_type_dynamic_and_static/migration.sql diff --git a/packages/database/prisma/migrations/20240720101649_added_task_run_tag_removed_task_tag/migration.sql b/internal-packages/database/prisma/migrations/20240720101649_added_task_run_tag_removed_task_tag/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240720101649_added_task_run_tag_removed_task_tag/migration.sql rename to internal-packages/database/prisma/migrations/20240720101649_added_task_run_tag_removed_task_tag/migration.sql diff --git a/packages/database/prisma/migrations/20240723104125_task_run_tag_name_id_index/migration.sql b/internal-packages/database/prisma/migrations/20240723104125_task_run_tag_name_id_index/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240723104125_task_run_tag_name_id_index/migration.sql rename to internal-packages/database/prisma/migrations/20240723104125_task_run_tag_name_id_index/migration.sql diff --git a/packages/database/prisma/migrations/20240729101628_task_run_span_id_index/migration.sql b/internal-packages/database/prisma/migrations/20240729101628_task_run_span_id_index/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240729101628_task_run_span_id_index/migration.sql rename to internal-packages/database/prisma/migrations/20240729101628_task_run_span_id_index/migration.sql diff --git a/packages/database/prisma/migrations/20240801175428_added_task_run_completed_at_column/migration.sql b/internal-packages/database/prisma/migrations/20240801175428_added_task_run_completed_at_column/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240801175428_added_task_run_completed_at_column/migration.sql rename to internal-packages/database/prisma/migrations/20240801175428_added_task_run_completed_at_column/migration.sql diff --git a/packages/database/prisma/migrations/20240804143817_add_task_run_logs_deleted_at_column/migration.sql b/internal-packages/database/prisma/migrations/20240804143817_add_task_run_logs_deleted_at_column/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240804143817_add_task_run_logs_deleted_at_column/migration.sql rename to internal-packages/database/prisma/migrations/20240804143817_add_task_run_logs_deleted_at_column/migration.sql diff --git a/packages/database/prisma/migrations/20240806163040_task_run_completed_at_index/migration.sql b/internal-packages/database/prisma/migrations/20240806163040_task_run_completed_at_index/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240806163040_task_run_completed_at_index/migration.sql rename to internal-packages/database/prisma/migrations/20240806163040_task_run_completed_at_index/migration.sql diff --git a/packages/database/prisma/migrations/20240809153150_background_worker_task_add_index_for_quick_lookup_of_task_identifiers/migration.sql b/internal-packages/database/prisma/migrations/20240809153150_background_worker_task_add_index_for_quick_lookup_of_task_identifiers/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240809153150_background_worker_task_add_index_for_quick_lookup_of_task_identifiers/migration.sql rename to internal-packages/database/prisma/migrations/20240809153150_background_worker_task_add_index_for_quick_lookup_of_task_identifiers/migration.sql diff --git a/packages/database/prisma/migrations/20240809213411_add_checkpoint_attempt_number_column/migration.sql b/internal-packages/database/prisma/migrations/20240809213411_add_checkpoint_attempt_number_column/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240809213411_add_checkpoint_attempt_number_column/migration.sql rename to internal-packages/database/prisma/migrations/20240809213411_add_checkpoint_attempt_number_column/migration.sql diff --git a/packages/database/prisma/migrations/20240810090402_add_background_worker_file_model/migration.sql b/internal-packages/database/prisma/migrations/20240810090402_add_background_worker_file_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240810090402_add_background_worker_file_model/migration.sql rename to internal-packages/database/prisma/migrations/20240810090402_add_background_worker_file_model/migration.sql diff --git a/packages/database/prisma/migrations/20240811185335_improve_background_worker_file_model/migration.sql b/internal-packages/database/prisma/migrations/20240811185335_improve_background_worker_file_model/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240811185335_improve_background_worker_file_model/migration.sql rename to internal-packages/database/prisma/migrations/20240811185335_improve_background_worker_file_model/migration.sql diff --git a/packages/database/prisma/migrations/20240815123344_add_project_alert_type_task_run_and_migrate_existing_alerts/migration.sql b/internal-packages/database/prisma/migrations/20240815123344_add_project_alert_type_task_run_and_migrate_existing_alerts/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240815123344_add_project_alert_type_task_run_and_migrate_existing_alerts/migration.sql rename to internal-packages/database/prisma/migrations/20240815123344_add_project_alert_type_task_run_and_migrate_existing_alerts/migration.sql diff --git a/packages/database/prisma/migrations/20240815125616_migrate_alerts_from_attempt_to_run/migration.sql b/internal-packages/database/prisma/migrations/20240815125616_migrate_alerts_from_attempt_to_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240815125616_migrate_alerts_from_attempt_to_run/migration.sql rename to internal-packages/database/prisma/migrations/20240815125616_migrate_alerts_from_attempt_to_run/migration.sql diff --git a/packages/database/prisma/migrations/20240821135542_job_run_index_for_organization_id_and_created_at/migration.sql b/internal-packages/database/prisma/migrations/20240821135542_job_run_index_for_organization_id_and_created_at/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240821135542_job_run_index_for_organization_id_and_created_at/migration.sql rename to internal-packages/database/prisma/migrations/20240821135542_job_run_index_for_organization_id_and_created_at/migration.sql diff --git a/packages/database/prisma/migrations/20240821141347_job_run_index_version_id/migration.sql b/internal-packages/database/prisma/migrations/20240821141347_job_run_index_version_id/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240821141347_job_run_index_version_id/migration.sql rename to internal-packages/database/prisma/migrations/20240821141347_job_run_index_version_id/migration.sql diff --git a/packages/database/prisma/migrations/20240821142006_batch_task_run_item_index_task_run_attempt_id/migration.sql b/internal-packages/database/prisma/migrations/20240821142006_batch_task_run_item_index_task_run_attempt_id/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240821142006_batch_task_run_item_index_task_run_attempt_id/migration.sql rename to internal-packages/database/prisma/migrations/20240821142006_batch_task_run_item_index_task_run_attempt_id/migration.sql diff --git a/packages/database/prisma/migrations/20240821142353_batch_task_run_item_index_task_run_id/migration.sql b/internal-packages/database/prisma/migrations/20240821142353_batch_task_run_item_index_task_run_id/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240821142353_batch_task_run_item_index_task_run_id/migration.sql rename to internal-packages/database/prisma/migrations/20240821142353_batch_task_run_item_index_task_run_id/migration.sql diff --git a/packages/database/prisma/migrations/20240821145232_task_index_parent_id/migration.sql b/internal-packages/database/prisma/migrations/20240821145232_task_index_parent_id/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240821145232_task_index_parent_id/migration.sql rename to internal-packages/database/prisma/migrations/20240821145232_task_index_parent_id/migration.sql diff --git a/packages/database/prisma/migrations/20240905134437_add_builder_project_id_to_projects/migration.sql b/internal-packages/database/prisma/migrations/20240905134437_add_builder_project_id_to_projects/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240905134437_add_builder_project_id_to_projects/migration.sql rename to internal-packages/database/prisma/migrations/20240905134437_add_builder_project_id_to_projects/migration.sql diff --git a/packages/database/prisma/migrations/20240909141925_task_run_attempt_index_on_task_run_id/migration.sql b/internal-packages/database/prisma/migrations/20240909141925_task_run_attempt_index_on_task_run_id/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240909141925_task_run_attempt_index_on_task_run_id/migration.sql rename to internal-packages/database/prisma/migrations/20240909141925_task_run_attempt_index_on_task_run_id/migration.sql diff --git a/packages/database/prisma/migrations/20240916155127_added_job_run_event_id_index/migration.sql b/internal-packages/database/prisma/migrations/20240916155127_added_job_run_event_id_index/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240916155127_added_job_run_event_id_index/migration.sql rename to internal-packages/database/prisma/migrations/20240916155127_added_job_run_event_id_index/migration.sql diff --git a/packages/database/prisma/migrations/20240920085046_add_task_hierarchy_columns_without_parent_task_run_id_index/migration.sql b/internal-packages/database/prisma/migrations/20240920085046_add_task_hierarchy_columns_without_parent_task_run_id_index/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240920085046_add_task_hierarchy_columns_without_parent_task_run_id_index/migration.sql rename to internal-packages/database/prisma/migrations/20240920085046_add_task_hierarchy_columns_without_parent_task_run_id_index/migration.sql diff --git a/packages/database/prisma/migrations/20240920085226_add_parent_task_run_id_index_concurrently/migration.sql b/internal-packages/database/prisma/migrations/20240920085226_add_parent_task_run_id_index_concurrently/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240920085226_add_parent_task_run_id_index_concurrently/migration.sql rename to internal-packages/database/prisma/migrations/20240920085226_add_parent_task_run_id_index_concurrently/migration.sql diff --git a/packages/database/prisma/migrations/20240924125845_add_root_task_run_id_index/migration.sql b/internal-packages/database/prisma/migrations/20240924125845_add_root_task_run_id_index/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240924125845_add_root_task_run_id_index/migration.sql rename to internal-packages/database/prisma/migrations/20240924125845_add_root_task_run_id_index/migration.sql diff --git a/packages/database/prisma/migrations/20240924130558_add_parent_span_id_to_task_run/migration.sql b/internal-packages/database/prisma/migrations/20240924130558_add_parent_span_id_to_task_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240924130558_add_parent_span_id_to_task_run/migration.sql rename to internal-packages/database/prisma/migrations/20240924130558_add_parent_span_id_to_task_run/migration.sql diff --git a/packages/database/prisma/migrations/20240925092304_add_metadata_and_output_to_task_run/migration.sql b/internal-packages/database/prisma/migrations/20240925092304_add_metadata_and_output_to_task_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240925092304_add_metadata_and_output_to_task_run/migration.sql rename to internal-packages/database/prisma/migrations/20240925092304_add_metadata_and_output_to_task_run/migration.sql diff --git a/packages/database/prisma/migrations/20240925205409_add_error_to_task_run/migration.sql b/internal-packages/database/prisma/migrations/20240925205409_add_error_to_task_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240925205409_add_error_to_task_run/migration.sql rename to internal-packages/database/prisma/migrations/20240925205409_add_error_to_task_run/migration.sql diff --git a/packages/database/prisma/migrations/20240926093535_add_seed_metadata_columns_to_task_run/migration.sql b/internal-packages/database/prisma/migrations/20240926093535_add_seed_metadata_columns_to_task_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240926093535_add_seed_metadata_columns_to_task_run/migration.sql rename to internal-packages/database/prisma/migrations/20240926093535_add_seed_metadata_columns_to_task_run/migration.sql diff --git a/packages/database/prisma/migrations/20240926110455_add_parent_span_id_index_to_the_task_run_table/migration.sql b/internal-packages/database/prisma/migrations/20240926110455_add_parent_span_id_index_to_the_task_run_table/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20240926110455_add_parent_span_id_index_to_the_task_run_table/migration.sql rename to internal-packages/database/prisma/migrations/20240926110455_add_parent_span_id_index_to_the_task_run_table/migration.sql diff --git a/packages/database/prisma/migrations/migration_lock.toml b/internal-packages/database/prisma/migrations/migration_lock.toml similarity index 100% rename from packages/database/prisma/migrations/migration_lock.toml rename to internal-packages/database/prisma/migrations/migration_lock.toml diff --git a/packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma similarity index 100% rename from packages/database/prisma/schema.prisma rename to internal-packages/database/prisma/schema.prisma diff --git a/packages/database/src/index.ts b/internal-packages/database/src/index.ts similarity index 100% rename from packages/database/src/index.ts rename to internal-packages/database/src/index.ts diff --git a/packages/database/tsconfig.json b/internal-packages/database/tsconfig.json similarity index 100% rename from packages/database/tsconfig.json rename to internal-packages/database/tsconfig.json diff --git a/packages/emails/.gitignore b/internal-packages/emails/.gitignore similarity index 100% rename from packages/emails/.gitignore rename to internal-packages/emails/.gitignore diff --git a/packages/emails/README.md b/internal-packages/emails/README.md similarity index 100% rename from packages/emails/README.md rename to internal-packages/emails/README.md diff --git a/packages/emails/emails/alert-attempt-failure.tsx b/internal-packages/emails/emails/alert-attempt-failure.tsx similarity index 100% rename from packages/emails/emails/alert-attempt-failure.tsx rename to internal-packages/emails/emails/alert-attempt-failure.tsx diff --git a/packages/emails/emails/alert-run-failure.tsx b/internal-packages/emails/emails/alert-run-failure.tsx similarity index 100% rename from packages/emails/emails/alert-run-failure.tsx rename to internal-packages/emails/emails/alert-run-failure.tsx diff --git a/packages/emails/emails/components/BasePath.tsx b/internal-packages/emails/emails/components/BasePath.tsx similarity index 100% rename from packages/emails/emails/components/BasePath.tsx rename to internal-packages/emails/emails/components/BasePath.tsx diff --git a/packages/emails/emails/components/Footer.tsx b/internal-packages/emails/emails/components/Footer.tsx similarity index 100% rename from packages/emails/emails/components/Footer.tsx rename to internal-packages/emails/emails/components/Footer.tsx diff --git a/packages/emails/emails/components/Image.tsx b/internal-packages/emails/emails/components/Image.tsx similarity index 100% rename from packages/emails/emails/components/Image.tsx rename to internal-packages/emails/emails/components/Image.tsx diff --git a/packages/emails/emails/components/styles.ts b/internal-packages/emails/emails/components/styles.ts similarity index 100% rename from packages/emails/emails/components/styles.ts rename to internal-packages/emails/emails/components/styles.ts diff --git a/packages/emails/emails/deployment-failure.tsx b/internal-packages/emails/emails/deployment-failure.tsx similarity index 100% rename from packages/emails/emails/deployment-failure.tsx rename to internal-packages/emails/emails/deployment-failure.tsx diff --git a/packages/emails/emails/deployment-success.tsx b/internal-packages/emails/emails/deployment-success.tsx similarity index 100% rename from packages/emails/emails/deployment-success.tsx rename to internal-packages/emails/emails/deployment-success.tsx diff --git a/packages/emails/emails/invite.tsx b/internal-packages/emails/emails/invite.tsx similarity index 100% rename from packages/emails/emails/invite.tsx rename to internal-packages/emails/emails/invite.tsx diff --git a/packages/emails/emails/magic-link.tsx b/internal-packages/emails/emails/magic-link.tsx similarity index 100% rename from packages/emails/emails/magic-link.tsx rename to internal-packages/emails/emails/magic-link.tsx diff --git a/packages/emails/emails/welcome.tsx b/internal-packages/emails/emails/welcome.tsx similarity index 100% rename from packages/emails/emails/welcome.tsx rename to internal-packages/emails/emails/welcome.tsx diff --git a/packages/emails/package.json b/internal-packages/emails/package.json similarity index 100% rename from packages/emails/package.json rename to internal-packages/emails/package.json diff --git a/packages/emails/src/index.tsx b/internal-packages/emails/src/index.tsx similarity index 100% rename from packages/emails/src/index.tsx rename to internal-packages/emails/src/index.tsx diff --git a/packages/emails/tsconfig.json b/internal-packages/emails/tsconfig.json similarity index 100% rename from packages/emails/tsconfig.json rename to internal-packages/emails/tsconfig.json diff --git a/packages/otlp-importer/.gitignore b/internal-packages/otlp-importer/.gitignore similarity index 100% rename from packages/otlp-importer/.gitignore rename to internal-packages/otlp-importer/.gitignore diff --git a/packages/otlp-importer/CHANGELOG.md b/internal-packages/otlp-importer/CHANGELOG.md similarity index 100% rename from packages/otlp-importer/CHANGELOG.md rename to internal-packages/otlp-importer/CHANGELOG.md diff --git a/packages/otlp-importer/LICENSE b/internal-packages/otlp-importer/LICENSE similarity index 100% rename from packages/otlp-importer/LICENSE rename to internal-packages/otlp-importer/LICENSE diff --git a/packages/otlp-importer/README.md b/internal-packages/otlp-importer/README.md similarity index 100% rename from packages/otlp-importer/README.md rename to internal-packages/otlp-importer/README.md diff --git a/packages/otlp-importer/jest.config.js b/internal-packages/otlp-importer/jest.config.js similarity index 100% rename from packages/otlp-importer/jest.config.js rename to internal-packages/otlp-importer/jest.config.js diff --git a/packages/otlp-importer/package.json b/internal-packages/otlp-importer/package.json similarity index 100% rename from packages/otlp-importer/package.json rename to internal-packages/otlp-importer/package.json diff --git a/packages/otlp-importer/protos b/internal-packages/otlp-importer/protos similarity index 100% rename from packages/otlp-importer/protos rename to internal-packages/otlp-importer/protos diff --git a/packages/otlp-importer/scripts/generate-protos.mjs b/internal-packages/otlp-importer/scripts/generate-protos.mjs similarity index 100% rename from packages/otlp-importer/scripts/generate-protos.mjs rename to internal-packages/otlp-importer/scripts/generate-protos.mjs diff --git a/packages/otlp-importer/scripts/generate-protos.sh b/internal-packages/otlp-importer/scripts/generate-protos.sh similarity index 100% rename from packages/otlp-importer/scripts/generate-protos.sh rename to internal-packages/otlp-importer/scripts/generate-protos.sh diff --git a/packages/otlp-importer/scripts/submodule.mjs b/internal-packages/otlp-importer/scripts/submodule.mjs similarity index 100% rename from packages/otlp-importer/scripts/submodule.mjs rename to internal-packages/otlp-importer/scripts/submodule.mjs diff --git a/packages/otlp-importer/scripts/utils.mjs b/internal-packages/otlp-importer/scripts/utils.mjs similarity index 100% rename from packages/otlp-importer/scripts/utils.mjs rename to internal-packages/otlp-importer/scripts/utils.mjs diff --git a/packages/otlp-importer/src/generated/opentelemetry/proto/collector/logs/v1/logs_service.ts b/internal-packages/otlp-importer/src/generated/opentelemetry/proto/collector/logs/v1/logs_service.ts similarity index 100% rename from packages/otlp-importer/src/generated/opentelemetry/proto/collector/logs/v1/logs_service.ts rename to internal-packages/otlp-importer/src/generated/opentelemetry/proto/collector/logs/v1/logs_service.ts diff --git a/packages/otlp-importer/src/generated/opentelemetry/proto/collector/metrics/v1/metrics_service.ts b/internal-packages/otlp-importer/src/generated/opentelemetry/proto/collector/metrics/v1/metrics_service.ts similarity index 100% rename from packages/otlp-importer/src/generated/opentelemetry/proto/collector/metrics/v1/metrics_service.ts rename to internal-packages/otlp-importer/src/generated/opentelemetry/proto/collector/metrics/v1/metrics_service.ts diff --git a/packages/otlp-importer/src/generated/opentelemetry/proto/collector/trace/v1/trace_service.ts b/internal-packages/otlp-importer/src/generated/opentelemetry/proto/collector/trace/v1/trace_service.ts similarity index 100% rename from packages/otlp-importer/src/generated/opentelemetry/proto/collector/trace/v1/trace_service.ts rename to internal-packages/otlp-importer/src/generated/opentelemetry/proto/collector/trace/v1/trace_service.ts diff --git a/packages/otlp-importer/src/generated/opentelemetry/proto/common/v1/common.ts b/internal-packages/otlp-importer/src/generated/opentelemetry/proto/common/v1/common.ts similarity index 100% rename from packages/otlp-importer/src/generated/opentelemetry/proto/common/v1/common.ts rename to internal-packages/otlp-importer/src/generated/opentelemetry/proto/common/v1/common.ts diff --git a/packages/otlp-importer/src/generated/opentelemetry/proto/logs/v1/logs.ts b/internal-packages/otlp-importer/src/generated/opentelemetry/proto/logs/v1/logs.ts similarity index 100% rename from packages/otlp-importer/src/generated/opentelemetry/proto/logs/v1/logs.ts rename to internal-packages/otlp-importer/src/generated/opentelemetry/proto/logs/v1/logs.ts diff --git a/packages/otlp-importer/src/generated/opentelemetry/proto/metrics/v1/metrics.ts b/internal-packages/otlp-importer/src/generated/opentelemetry/proto/metrics/v1/metrics.ts similarity index 100% rename from packages/otlp-importer/src/generated/opentelemetry/proto/metrics/v1/metrics.ts rename to internal-packages/otlp-importer/src/generated/opentelemetry/proto/metrics/v1/metrics.ts diff --git a/packages/otlp-importer/src/generated/opentelemetry/proto/resource/v1/resource.ts b/internal-packages/otlp-importer/src/generated/opentelemetry/proto/resource/v1/resource.ts similarity index 100% rename from packages/otlp-importer/src/generated/opentelemetry/proto/resource/v1/resource.ts rename to internal-packages/otlp-importer/src/generated/opentelemetry/proto/resource/v1/resource.ts diff --git a/packages/otlp-importer/src/generated/opentelemetry/proto/trace/v1/trace.ts b/internal-packages/otlp-importer/src/generated/opentelemetry/proto/trace/v1/trace.ts similarity index 100% rename from packages/otlp-importer/src/generated/opentelemetry/proto/trace/v1/trace.ts rename to internal-packages/otlp-importer/src/generated/opentelemetry/proto/trace/v1/trace.ts diff --git a/packages/otlp-importer/src/index.ts b/internal-packages/otlp-importer/src/index.ts similarity index 100% rename from packages/otlp-importer/src/index.ts rename to internal-packages/otlp-importer/src/index.ts diff --git a/packages/otlp-importer/tsconfig.json b/internal-packages/otlp-importer/tsconfig.json similarity index 100% rename from packages/otlp-importer/tsconfig.json rename to internal-packages/otlp-importer/tsconfig.json diff --git a/packages/otlp-importer/tsup.config.ts b/internal-packages/otlp-importer/tsup.config.ts similarity index 100% rename from packages/otlp-importer/tsup.config.ts rename to internal-packages/otlp-importer/tsup.config.ts diff --git a/packages/run-engine/README.md b/internal-packages/run-engine/README.md similarity index 100% rename from packages/run-engine/README.md rename to internal-packages/run-engine/README.md diff --git a/packages/run-engine/package.json b/internal-packages/run-engine/package.json similarity index 94% rename from packages/run-engine/package.json rename to internal-packages/run-engine/package.json index ad252b22fe..2762976dbd 100644 --- a/packages/run-engine/package.json +++ b/internal-packages/run-engine/package.json @@ -1,5 +1,5 @@ { - "name": "@trigger.dev/run-engine", + "name": "@internal/run-engine", "private": true, "version": "0.0.1", "main": "./src/index.ts", diff --git a/packages/run-engine/src/engine/index.test.ts b/internal-packages/run-engine/src/engine/index.test.ts similarity index 100% rename from packages/run-engine/src/engine/index.test.ts rename to internal-packages/run-engine/src/engine/index.test.ts diff --git a/packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts similarity index 97% rename from packages/run-engine/src/engine/index.ts rename to internal-packages/run-engine/src/engine/index.ts index 319f2d489d..51a408e1ac 100644 --- a/packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -1,4 +1,4 @@ -import { PrismaClient, Prisma } from "@trigger.dev/database"; +import { PrismaClient, Prisma } from "../../../database/src"; import { Redis, type RedisOptions } from "ioredis"; import Redlock from "redlock"; diff --git a/packages/run-engine/src/run-queue/index.test.ts b/internal-packages/run-engine/src/run-queue/index.test.ts similarity index 100% rename from packages/run-engine/src/run-queue/index.test.ts rename to internal-packages/run-engine/src/run-queue/index.test.ts diff --git a/packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts similarity index 100% rename from packages/run-engine/src/run-queue/index.ts rename to internal-packages/run-engine/src/run-queue/index.ts diff --git a/packages/run-engine/src/run-queue/keyProducer.test.ts b/internal-packages/run-engine/src/run-queue/keyProducer.test.ts similarity index 100% rename from packages/run-engine/src/run-queue/keyProducer.test.ts rename to internal-packages/run-engine/src/run-queue/keyProducer.test.ts diff --git a/packages/run-engine/src/run-queue/keyProducer.ts b/internal-packages/run-engine/src/run-queue/keyProducer.ts similarity index 98% rename from packages/run-engine/src/run-queue/keyProducer.ts rename to internal-packages/run-engine/src/run-queue/keyProducer.ts index becea0a38e..ff6a75ee48 100644 --- a/packages/run-engine/src/run-queue/keyProducer.ts +++ b/internal-packages/run-engine/src/run-queue/keyProducer.ts @@ -1,4 +1,4 @@ -import { RuntimeEnvironmentType } from "@trigger.dev/database"; +import { RuntimeEnvironmentType } from "../../../database/src/index.js"; import { AuthenticatedEnvironment } from "../shared/index.js"; import { RunQueueKeyProducer } from "./types.js"; diff --git a/packages/run-engine/src/run-queue/simpleWeightedPriorityStrategy.ts b/internal-packages/run-engine/src/run-queue/simpleWeightedPriorityStrategy.ts similarity index 100% rename from packages/run-engine/src/run-queue/simpleWeightedPriorityStrategy.ts rename to internal-packages/run-engine/src/run-queue/simpleWeightedPriorityStrategy.ts diff --git a/packages/run-engine/src/run-queue/types.ts b/internal-packages/run-engine/src/run-queue/types.ts similarity index 98% rename from packages/run-engine/src/run-queue/types.ts rename to internal-packages/run-engine/src/run-queue/types.ts index 74e8d48451..1502fed560 100644 --- a/packages/run-engine/src/run-queue/types.ts +++ b/internal-packages/run-engine/src/run-queue/types.ts @@ -1,6 +1,6 @@ import { z } from "zod"; import { AuthenticatedEnvironment } from "../shared/index.js"; -import { RuntimeEnvironmentType } from "@trigger.dev/database"; +import { RuntimeEnvironmentType } from "../../../database/src/index.js"; import { env } from "process"; import { version } from "os"; diff --git a/packages/run-engine/src/shared/asyncWorker.ts b/internal-packages/run-engine/src/shared/asyncWorker.ts similarity index 100% rename from packages/run-engine/src/shared/asyncWorker.ts rename to internal-packages/run-engine/src/shared/asyncWorker.ts diff --git a/packages/run-engine/src/shared/index.ts b/internal-packages/run-engine/src/shared/index.ts similarity index 96% rename from packages/run-engine/src/shared/index.ts rename to internal-packages/run-engine/src/shared/index.ts index b7393714a7..7bef31b404 100644 --- a/packages/run-engine/src/shared/index.ts +++ b/internal-packages/run-engine/src/shared/index.ts @@ -1,5 +1,5 @@ import { Attributes } from "@opentelemetry/api"; -import { Prisma } from "@trigger.dev/database"; +import { Prisma } from "../../../database/src"; type EnvironmentWithExtras = Prisma.RuntimeEnvironmentGetPayload<{ include: { project: true; organization: true; orgMember: true }; diff --git a/packages/run-engine/src/simple-queue/index.test.ts b/internal-packages/run-engine/src/simple-queue/index.test.ts similarity index 100% rename from packages/run-engine/src/simple-queue/index.test.ts rename to internal-packages/run-engine/src/simple-queue/index.test.ts diff --git a/packages/run-engine/src/simple-queue/index.ts b/internal-packages/run-engine/src/simple-queue/index.ts similarity index 100% rename from packages/run-engine/src/simple-queue/index.ts rename to internal-packages/run-engine/src/simple-queue/index.ts diff --git a/packages/run-engine/src/simple-queue/processor.test.ts b/internal-packages/run-engine/src/simple-queue/processor.test.ts similarity index 100% rename from packages/run-engine/src/simple-queue/processor.test.ts rename to internal-packages/run-engine/src/simple-queue/processor.test.ts diff --git a/packages/run-engine/src/simple-queue/processor.ts b/internal-packages/run-engine/src/simple-queue/processor.ts similarity index 100% rename from packages/run-engine/src/simple-queue/processor.ts rename to internal-packages/run-engine/src/simple-queue/processor.ts diff --git a/packages/run-engine/src/test/containerTest.ts b/internal-packages/run-engine/src/test/containerTest.ts similarity index 97% rename from packages/run-engine/src/test/containerTest.ts rename to internal-packages/run-engine/src/test/containerTest.ts index 7f7d59cdf3..18e5e15d0f 100644 --- a/packages/run-engine/src/test/containerTest.ts +++ b/internal-packages/run-engine/src/test/containerTest.ts @@ -1,6 +1,6 @@ import { StartedPostgreSqlContainer } from "@testcontainers/postgresql"; import { StartedRedisContainer } from "@testcontainers/redis"; -import { PrismaClient } from "@trigger.dev/database"; +import { PrismaClient } from "../../../database/src"; import { Redis } from "ioredis"; import { test, TestAPI } from "vitest"; import { createPostgresContainer, createRedisContainer } from "./utils"; diff --git a/packages/run-engine/src/test/utils.ts b/internal-packages/run-engine/src/test/utils.ts similarity index 94% rename from packages/run-engine/src/test/utils.ts rename to internal-packages/run-engine/src/test/utils.ts index d1dfc2857c..e998f7afa9 100644 --- a/packages/run-engine/src/test/utils.ts +++ b/internal-packages/run-engine/src/test/utils.ts @@ -1,6 +1,6 @@ import { PostgreSqlContainer } from "@testcontainers/postgresql"; import { RedisContainer } from "@testcontainers/redis"; -import { PrismaClient } from "@trigger.dev/database"; +import { PrismaClient } from "../../../database/src"; import { execSync } from "child_process"; import path from "path"; diff --git a/packages/run-engine/tsconfig.json b/internal-packages/run-engine/tsconfig.json similarity index 82% rename from packages/run-engine/tsconfig.json rename to internal-packages/run-engine/tsconfig.json index 47472e2963..bfd15dde0d 100644 --- a/packages/run-engine/tsconfig.json +++ b/internal-packages/run-engine/tsconfig.json @@ -17,8 +17,8 @@ "paths": { "@trigger.dev/core": ["../../packages/core/src/index"], "@trigger.dev/core/*": ["../../packages/core/src/*"], - "@trigger.dev/database": ["../../packages/database/src/index"], - "@trigger.dev/database/*": ["../../packages/database/src/*"] + "@trigger.dev/database": ["../database/src/index"], + "@trigger.dev/database/*": ["../database/src/*"] } }, "exclude": ["node_modules"] diff --git a/packages/run-engine/vitest.config.ts b/internal-packages/run-engine/vitest.config.ts similarity index 100% rename from packages/run-engine/vitest.config.ts rename to internal-packages/run-engine/vitest.config.ts diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index b6e3cda0b0..a97140d11e 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,5 +1,6 @@ packages: - "packages/*" + - "internal-packages/*" - "apps/**" - "references/*" - "docs" From 82a0867af2b01e7e7cee6788b749373adc9bc658 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 29 Sep 2024 21:08:43 +0100 Subject: [PATCH 054/114] Moved some Prisma code to the database package --- apps/webapp/app/db.server.ts | 68 ++++++------------- internal-packages/database/src/index.ts | 1 + internal-packages/database/src/transaction.ts | 58 ++++++++++++++++ 3 files changed, 80 insertions(+), 47 deletions(-) create mode 100644 internal-packages/database/src/transaction.ts diff --git a/apps/webapp/app/db.server.ts b/apps/webapp/app/db.server.ts index 8af02c7897..90942662d3 100644 --- a/apps/webapp/app/db.server.ts +++ b/apps/webapp/app/db.server.ts @@ -1,39 +1,24 @@ -import { Prisma, PrismaClient } from "@trigger.dev/database"; +import { + Prisma, + PrismaClient, + PrismaClientOrTransaction, + PrismaReplicaClient, + PrismaTransactionClient, + PrismaTransactionOptions, +} from "@trigger.dev/database"; import invariant from "tiny-invariant"; import { z } from "zod"; import { env } from "./env.server"; import { logger } from "./services/logger.server"; import { isValidDatabaseUrl } from "./utils/db"; import { singleton } from "./utils/singleton"; +import { $transaction as transac } from "@trigger.dev/database"; -export type PrismaTransactionClient = Omit< - PrismaClient, - "$connect" | "$disconnect" | "$on" | "$transaction" | "$use" | "$extends" ->; - -export type PrismaClientOrTransaction = PrismaClient | PrismaTransactionClient; - -function isTransactionClient(prisma: PrismaClientOrTransaction): prisma is PrismaTransactionClient { - return !("$transaction" in prisma); -} - -function isPrismaKnownError(error: unknown): error is Prisma.PrismaClientKnownRequestError { - return ( - typeof error === "object" && error !== null && "code" in error && typeof error.code === "string" - ); -} - -export type PrismaTransactionOptions = { - /** The maximum amount of time (in ms) Prisma Client will wait to acquire a transaction from the database. The default value is 2000ms. */ - maxWait?: number; - - /** The maximum amount of time (in ms) the interactive transaction can run before being canceled and rolled back. The default value is 5000ms. */ - timeout?: number; - - /** Sets the transaction isolation level. By default this is set to the value currently configured in your database. */ - isolationLevel?: Prisma.TransactionIsolationLevel; - - swallowPrismaErrors?: boolean; +export type { + PrismaTransactionClient, + PrismaClientOrTransaction, + PrismaTransactionOptions, + PrismaReplicaClient, }; export async function $transaction( @@ -41,14 +26,10 @@ export async function $transaction( fn: (prisma: PrismaTransactionClient) => Promise, options?: PrismaTransactionOptions ): Promise { - if (isTransactionClient(prisma)) { - return fn(prisma); - } - - try { - return await (prisma as PrismaClient).$transaction(fn, options); - } catch (error) { - if (isPrismaKnownError(error)) { + return transac( + prisma, + fn, + (error) => { logger.error("prisma.$transaction error", { code: error.code, meta: error.meta, @@ -56,22 +37,15 @@ export async function $transaction( message: error.message, name: error.name, }); - - if (options?.swallowPrismaErrors) { - return; - } - } - - throw error; - } + }, + options + ); } export { Prisma }; export const prisma = singleton("prisma", getClient); -export type PrismaReplicaClient = Omit; - export const $replica: PrismaReplicaClient = singleton( "replica", () => getReplicaClient() ?? prisma diff --git a/internal-packages/database/src/index.ts b/internal-packages/database/src/index.ts index fcab163dab..cb0f9d4eb9 100644 --- a/internal-packages/database/src/index.ts +++ b/internal-packages/database/src/index.ts @@ -1 +1,2 @@ export * from "@prisma/client"; +export * from "./transaction"; diff --git a/internal-packages/database/src/transaction.ts b/internal-packages/database/src/transaction.ts new file mode 100644 index 0000000000..017e222c65 --- /dev/null +++ b/internal-packages/database/src/transaction.ts @@ -0,0 +1,58 @@ +import { Prisma, PrismaClient } from "@prisma/client"; + +export type PrismaTransactionClient = Omit< + PrismaClient, + "$connect" | "$disconnect" | "$on" | "$transaction" | "$use" | "$extends" +>; + +export type PrismaClientOrTransaction = PrismaClient | PrismaTransactionClient; + +export type PrismaReplicaClient = Omit; + +function isTransactionClient(prisma: PrismaClientOrTransaction): prisma is PrismaTransactionClient { + return !("$transaction" in prisma); +} + +function isPrismaKnownError(error: unknown): error is Prisma.PrismaClientKnownRequestError { + return ( + typeof error === "object" && error !== null && "code" in error && typeof error.code === "string" + ); +} + +export type PrismaTransactionOptions = { + /** The maximum amount of time (in ms) Prisma Client will wait to acquire a transaction from the database. The default value is 2000ms. */ + maxWait?: number; + + /** The maximum amount of time (in ms) the interactive transaction can run before being canceled and rolled back. The default value is 5000ms. */ + timeout?: number; + + /** Sets the transaction isolation level. By default this is set to the value currently configured in your database. */ + isolationLevel?: Prisma.TransactionIsolationLevel; + + swallowPrismaErrors?: boolean; +}; + +export async function $transaction( + prisma: PrismaClientOrTransaction, + fn: (prisma: PrismaTransactionClient) => Promise, + prismaError: (error: Prisma.PrismaClientKnownRequestError) => void, + options?: PrismaTransactionOptions +): Promise { + if (isTransactionClient(prisma)) { + return fn(prisma); + } + + try { + return await (prisma as PrismaClient).$transaction(fn, options); + } catch (error) { + if (isPrismaKnownError(error)) { + prismaError(error); + + if (options?.swallowPrismaErrors) { + return; + } + } + + throw error; + } +} From a20e180c629e3a7d406c6eb065491e94c96ae1cd Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 29 Sep 2024 21:08:56 +0100 Subject: [PATCH 055/114] Started using the RunEngine for triggering --- .../app/v3/services/triggerTaskV2.server.ts | 580 ++++++++++++++++++ apps/webapp/package.json | 1 + apps/webapp/tsconfig.json | 4 +- .../run-engine/src/engine/index.ts | 236 ++++++- internal-packages/run-engine/src/index.ts | 1 + .../run-engine/src/run-queue/index.ts | 34 +- .../run-engine/src/run-queue/types.ts | 18 +- .../run-engine/src/shared/index.ts | 16 +- 8 files changed, 850 insertions(+), 40 deletions(-) create mode 100644 apps/webapp/app/v3/services/triggerTaskV2.server.ts create mode 100644 internal-packages/run-engine/src/index.ts diff --git a/apps/webapp/app/v3/services/triggerTaskV2.server.ts b/apps/webapp/app/v3/services/triggerTaskV2.server.ts new file mode 100644 index 0000000000..8207849927 --- /dev/null +++ b/apps/webapp/app/v3/services/triggerTaskV2.server.ts @@ -0,0 +1,580 @@ +import { + IOPacket, + QueueOptions, + SemanticInternalAttributes, + TriggerTaskRequestBody, + packetRequiresOffloading, +} from "@trigger.dev/core/v3"; +import { env } from "~/env.server"; +import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { autoIncrementCounter } from "~/services/autoIncrementCounter.server"; +import { workerQueue } from "~/services/worker.server"; +import { marqs, sanitizeQueueName } from "~/v3/marqs/index.server"; +import { eventRepository } from "../eventRepository.server"; +import { generateFriendlyId } from "../friendlyIdentifiers"; +import { uploadToObjectStore } from "../r2.server"; +import { startActiveSpan } from "../tracer.server"; +import { getEntitlement } from "~/services/platform.v3.server"; +import { BaseService, ServiceValidationError } from "./baseService.server"; +import { logger } from "~/services/logger.server"; +import { isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus"; +import { createTag, MAX_TAGS_PER_RUN } from "~/models/taskRunTag.server"; +import { findCurrentWorkerFromEnvironment } from "../models/workerDeployment.server"; +import { handleMetadataPacket } from "~/utils/packets"; +import { RunEngine } from "@internal/run-engine"; +import { prisma } from "~/db.server"; + +export type TriggerTaskServiceOptions = { + idempotencyKey?: string; + triggerVersion?: string; + traceContext?: Record; + spanParentAsLink?: boolean; + parentAsLinkType?: "replay" | "trigger"; + batchId?: string; + customIcon?: string; +}; + +export class OutOfEntitlementError extends Error { + constructor() { + super("You can't trigger a task because you have run out of credits."); + } +} + +const engine = new RunEngine({ + prisma, + redis: { + port: env.REDIS_PORT, + host: env.REDIS_HOST, + username: env.REDIS_USERNAME, + password: env.REDIS_PASSWORD, + enableAutoPipelining: true, + ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }), + }, +}); + +export class TriggerTaskService extends BaseService { + public async call( + taskId: string, + environment: AuthenticatedEnvironment, + body: TriggerTaskRequestBody, + options: TriggerTaskServiceOptions = {} + ) { + return await this.traceWithEnv("call()", environment, async (span) => { + span.setAttribute("taskId", taskId); + + const idempotencyKey = options.idempotencyKey ?? body.options?.idempotencyKey; + const delayUntil = await parseDelay(body.options?.delay); + + const ttl = + typeof body.options?.ttl === "number" + ? stringifyDuration(body.options?.ttl) + : body.options?.ttl ?? (environment.type === "DEVELOPMENT" ? "10m" : undefined); + + const existingRun = idempotencyKey + ? await this._prisma.taskRun.findUnique({ + where: { + runtimeEnvironmentId_taskIdentifier_idempotencyKey: { + runtimeEnvironmentId: environment.id, + idempotencyKey, + taskIdentifier: taskId, + }, + }, + }) + : undefined; + + if (existingRun) { + span.setAttribute("runId", existingRun.friendlyId); + + return existingRun; + } + + if (environment.type !== "DEVELOPMENT") { + const result = await getEntitlement(environment.organizationId); + if (result && result.hasAccess === false) { + throw new OutOfEntitlementError(); + } + } + + if ( + body.options?.tags && + typeof body.options.tags !== "string" && + body.options.tags.length > MAX_TAGS_PER_RUN + ) { + throw new ServiceValidationError( + `Runs can only have ${MAX_TAGS_PER_RUN} tags, you're trying to set ${body.options.tags.length}.` + ); + } + + const runFriendlyId = generateFriendlyId("run"); + + const payloadPacket = await this.#handlePayloadPacket( + body.payload, + body.options?.payloadType ?? "application/json", + runFriendlyId, + environment + ); + + const metadataPacket = body.options?.metadata + ? handleMetadataPacket( + body.options?.metadata, + body.options?.metadataType ?? "application/json" + ) + : undefined; + + const dependentAttempt = body.options?.dependentAttempt + ? await this._prisma.taskRunAttempt.findUnique({ + where: { friendlyId: body.options.dependentAttempt }, + include: { + taskRun: { + select: { + id: true, + status: true, + taskIdentifier: true, + rootTaskRunId: true, + depth: true, + }, + }, + }, + }) + : undefined; + + if ( + dependentAttempt && + (isFinalAttemptStatus(dependentAttempt.status) || + isFinalRunStatus(dependentAttempt.taskRun.status)) + ) { + logger.debug("Dependent attempt or run is in a terminal state", { + dependentAttempt: dependentAttempt, + }); + + if (isFinalAttemptStatus(dependentAttempt.status)) { + throw new ServiceValidationError( + `Cannot trigger ${taskId} as the parent attempt has a status of ${dependentAttempt.status}` + ); + } else { + throw new ServiceValidationError( + `Cannot trigger ${taskId} as the parent run has a status of ${dependentAttempt.taskRun.status}` + ); + } + } + + const parentAttempt = body.options?.parentAttempt + ? await this._prisma.taskRunAttempt.findUnique({ + where: { friendlyId: body.options.parentAttempt }, + include: { + taskRun: { + select: { + id: true, + status: true, + taskIdentifier: true, + rootTaskRunId: true, + depth: true, + }, + }, + }, + }) + : undefined; + + const dependentBatchRun = body.options?.dependentBatch + ? await this._prisma.batchTaskRun.findUnique({ + where: { friendlyId: body.options.dependentBatch }, + include: { + dependentTaskAttempt: { + include: { + taskRun: { + select: { + id: true, + status: true, + taskIdentifier: true, + rootTaskRunId: true, + depth: true, + }, + }, + }, + }, + }, + }) + : undefined; + + if ( + dependentBatchRun && + dependentBatchRun.dependentTaskAttempt && + (isFinalAttemptStatus(dependentBatchRun.dependentTaskAttempt.status) || + isFinalRunStatus(dependentBatchRun.dependentTaskAttempt.taskRun.status)) + ) { + logger.debug("Dependent batch run task attempt or run has been canceled", { + dependentBatchRunId: dependentBatchRun.id, + status: dependentBatchRun.status, + attempt: dependentBatchRun.dependentTaskAttempt, + }); + + if (isFinalAttemptStatus(dependentBatchRun.dependentTaskAttempt.status)) { + throw new ServiceValidationError( + `Cannot trigger ${taskId} as the parent attempt has a status of ${dependentBatchRun.dependentTaskAttempt.status}` + ); + } else { + throw new ServiceValidationError( + `Cannot trigger ${taskId} as the parent run has a status of ${dependentBatchRun.dependentTaskAttempt.taskRun.status}` + ); + } + } + + const parentBatchRun = body.options?.parentBatch + ? await this._prisma.batchTaskRun.findUnique({ + where: { friendlyId: body.options.parentBatch }, + include: { + dependentTaskAttempt: { + include: { + taskRun: { + select: { + id: true, + status: true, + taskIdentifier: true, + rootTaskRunId: true, + }, + }, + }, + }, + }, + }) + : undefined; + + return await eventRepository.traceEvent( + taskId, + { + context: options.traceContext, + spanParentAsLink: options.spanParentAsLink, + parentAsLinkType: options.parentAsLinkType, + kind: "SERVER", + environment, + taskSlug: taskId, + attributes: { + properties: { + [SemanticInternalAttributes.SHOW_ACTIONS]: true, + }, + style: { + icon: options.customIcon ?? "task", + }, + runIsTest: body.options?.test ?? false, + batchId: options.batchId, + idempotencyKey, + }, + incomplete: true, + immediate: true, + }, + async (event, traceContext, traceparent) => { + const run = await autoIncrementCounter.incrementInTransaction( + `v3-run:${environment.id}:${taskId}`, + async (num, tx) => { + const lockedToBackgroundWorker = body.options?.lockToVersion + ? await tx.backgroundWorker.findUnique({ + where: { + projectId_runtimeEnvironmentId_version: { + projectId: environment.projectId, + runtimeEnvironmentId: environment.id, + version: body.options?.lockToVersion, + }, + }, + }) + : undefined; + + let queueName = sanitizeQueueName( + await this.#getQueueName(taskId, environment, body.options?.queue?.name) + ); + + // Check that the queuename is not an empty string + if (!queueName) { + queueName = sanitizeQueueName(`task/${taskId}`); + } + + event.setAttribute("queueName", queueName); + span.setAttribute("queueName", queueName); + + //upsert tags + let tagIds: string[] = []; + const bodyTags = + typeof body.options?.tags === "string" ? [body.options.tags] : body.options?.tags; + if (bodyTags && bodyTags.length > 0) { + for (const tag of bodyTags) { + const tagRecord = await createTag({ + tag, + projectId: environment.projectId, + }); + if (tagRecord) { + tagIds.push(tagRecord.id); + } + } + } + + const depth = dependentAttempt + ? dependentAttempt.taskRun.depth + 1 + : parentAttempt + ? parentAttempt.taskRun.depth + 1 + : dependentBatchRun?.dependentTaskAttempt + ? dependentBatchRun.dependentTaskAttempt.taskRun.depth + 1 + : 0; + + event.setAttribute("runId", runFriendlyId); + span.setAttribute("runId", runFriendlyId); + + const taskRun = await engine.trigger( + { + number: num, + friendlyId: runFriendlyId, + environment: environment, + idempotencyKey, + taskIdentifier: taskId, + payload: payloadPacket.data ?? "", + payloadType: payloadPacket.dataType, + context: body.context, + traceContext: traceContext, + traceId: event.traceId, + spanId: event.spanId, + parentSpanId: + options.parentAsLinkType === "replay" ? undefined : traceparent?.spanId, + lockedToVersionId: lockedToBackgroundWorker?.id, + concurrencyKey: body.options?.concurrencyKey, + queueName, + queue: body.options?.queue, + isTest: body.options?.test ?? false, + delayUntil, + queuedAt: delayUntil ? undefined : new Date(), + maxAttempts: body.options?.maxAttempts, + ttl, + tags: tagIds, + parentTaskRunId: parentAttempt?.taskRun.id, + parentTaskRunAttemptId: parentAttempt?.id, + rootTaskRunId: parentAttempt?.taskRun.rootTaskRunId ?? parentAttempt?.taskRun.id, + batchId: dependentBatchRun?.id ?? parentBatchRun?.id, + resumeParentOnCompletion: !!(dependentAttempt ?? dependentBatchRun), + depth, + metadata: metadataPacket?.data, + metadataType: metadataPacket?.dataType, + seedMetadata: metadataPacket?.data, + seedMetadataType: metadataPacket?.dataType, + isWait: true, + }, + this._prisma + ); + + return taskRun; + }, + async (_, tx) => { + const counter = await tx.taskRunNumberCounter.findUnique({ + where: { + taskIdentifier_environmentId: { + taskIdentifier: taskId, + environmentId: environment.id, + }, + }, + select: { lastNumber: true }, + }); + + return counter?.lastNumber; + }, + this._prisma + ); + + return run; + } + ); + }); + } + + async #getQueueName(taskId: string, environment: AuthenticatedEnvironment, queueName?: string) { + if (queueName) { + return queueName; + } + + const defaultQueueName = `task/${taskId}`; + + const worker = await findCurrentWorkerFromEnvironment(environment); + + if (!worker) { + logger.debug("Failed to get queue name: No worker found", { + taskId, + environmentId: environment.id, + }); + + return defaultQueueName; + } + + const task = await this._prisma.backgroundWorkerTask.findUnique({ + where: { + workerId_slug: { + workerId: worker.id, + slug: taskId, + }, + }, + }); + + if (!task) { + console.log("Failed to get queue name: No task found", { + taskId, + environmentId: environment.id, + }); + + return defaultQueueName; + } + + const queueConfig = QueueOptions.optional().nullable().safeParse(task.queueConfig); + + if (!queueConfig.success) { + console.log("Failed to get queue name: Invalid queue config", { + taskId, + environmentId: environment.id, + queueConfig: task.queueConfig, + }); + + return defaultQueueName; + } + + return queueConfig.data?.name ?? defaultQueueName; + } + + async #handlePayloadPacket( + payload: any, + payloadType: string, + pathPrefix: string, + environment: AuthenticatedEnvironment + ) { + return await startActiveSpan("handlePayloadPacket()", async (span) => { + const packet = this.#createPayloadPacket(payload, payloadType); + + if (!packet.data) { + return packet; + } + + const { needsOffloading, size } = packetRequiresOffloading( + packet, + env.TASK_PAYLOAD_OFFLOAD_THRESHOLD + ); + + if (!needsOffloading) { + return packet; + } + + const filename = `${pathPrefix}/payload.json`; + + await uploadToObjectStore(filename, packet.data, packet.dataType, environment); + + return { + data: filename, + dataType: "application/store", + }; + }); + } + + #createPayloadPacket(payload: any, payloadType: string): IOPacket { + if (payloadType === "application/json") { + return { data: JSON.stringify(payload), dataType: "application/json" }; + } + + if (typeof payload === "string") { + return { data: payload, dataType: payloadType }; + } + + return { dataType: payloadType }; + } +} + +export async function parseDelay(value?: string | Date): Promise { + if (!value) { + return; + } + + if (value instanceof Date) { + return value; + } + + try { + const date = new Date(value); + + // Check if the date is valid + if (isNaN(date.getTime())) { + return parseNaturalLanguageDuration(value); + } + + if (date.getTime() <= Date.now()) { + return; + } + + return date; + } catch (error) { + return parseNaturalLanguageDuration(value); + } +} + +export function parseNaturalLanguageDuration(duration: string): Date | undefined { + const regexPattern = /^(\d+w)?(\d+d)?(\d+h)?(\d+m)?(\d+s)?$/; + + const result: Date = new Date(); + let hasMatch = false; + + const elements = duration.match(regexPattern); + if (elements) { + if (elements[1]) { + const weeks = Number(elements[1].slice(0, -1)); + if (weeks >= 0) { + result.setDate(result.getDate() + 7 * weeks); + hasMatch = true; + } + } + if (elements[2]) { + const days = Number(elements[2].slice(0, -1)); + if (days >= 0) { + result.setDate(result.getDate() + days); + hasMatch = true; + } + } + if (elements[3]) { + const hours = Number(elements[3].slice(0, -1)); + if (hours >= 0) { + result.setHours(result.getHours() + hours); + hasMatch = true; + } + } + if (elements[4]) { + const minutes = Number(elements[4].slice(0, -1)); + if (minutes >= 0) { + result.setMinutes(result.getMinutes() + minutes); + hasMatch = true; + } + } + if (elements[5]) { + const seconds = Number(elements[5].slice(0, -1)); + if (seconds >= 0) { + result.setSeconds(result.getSeconds() + seconds); + hasMatch = true; + } + } + } + + if (hasMatch) { + return result; + } + + return undefined; +} + +function stringifyDuration(seconds: number): string | undefined { + if (seconds <= 0) { + return; + } + + const units = { + w: Math.floor(seconds / 604800), + d: Math.floor((seconds % 604800) / 86400), + h: Math.floor((seconds % 86400) / 3600), + m: Math.floor((seconds % 3600) / 60), + s: Math.floor(seconds % 60), + }; + + // Filter the units having non-zero values and join them + const result: string = Object.entries(units) + .filter(([unit, val]) => val != 0) + .map(([unit, val]) => `${val}${unit}`) + .join(""); + + return result; +} diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 553d12c73d..be150ae46e 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -49,6 +49,7 @@ "@electric-sql/react": "^0.3.5", "@headlessui/react": "^1.7.8", "@heroicons/react": "^2.0.12", + "@internal/run-engine": "workspace:*", "@internationalized/date": "^3.5.1", "@lezer/highlight": "^1.1.6", "@opentelemetry/api": "1.9.0", diff --git a/apps/webapp/tsconfig.json b/apps/webapp/tsconfig.json index 16eb40c47a..6b44d0f7c9 100644 --- a/apps/webapp/tsconfig.json +++ b/apps/webapp/tsconfig.json @@ -31,7 +31,9 @@ "@trigger.dev/otlp-importer": ["../../internal-packages/otlp-importer/src/index"], "@trigger.dev/otlp-importer/*": ["../../internal-packages/otlp-importer/src/*"], "emails": ["../../internal-packages/emails/src/index"], - "emails/*": ["../../internal-packages/emails/src/*"] + "emails/*": ["../../internal-packages/emails/src/*"], + "@internal/run-engine": ["../../internal-packages/run-engine/src/index"], + "@internal/run-engine/*": ["../../internal-packages/run-engine/src/*"] }, "noEmit": true } diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 51a408e1ac..a232e286af 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -1,17 +1,54 @@ -import { PrismaClient, Prisma } from "../../../database/src"; +import { PrismaClient, Prisma, PrismaClientOrTransaction } from "@trigger.dev/database"; import { Redis, type RedisOptions } from "ioredis"; import Redlock from "redlock"; +import { AuthenticatedEnvironment, MinimalAuthenticatedEnvironment } from "../shared"; +import { QueueOptions } from "@trigger.dev/core/v3"; +import { RunQueue } from "../run-queue"; type Options = { - prisma: PrismaClient; redis: RedisOptions; - //todo - // queue: RunQueue; + prisma: PrismaClientOrTransaction; +}; + +type TriggerParams = { + friendlyId: string; + number: number; + environment: MinimalAuthenticatedEnvironment; + idempotencyKey?: string; + taskIdentifier: string; + payload: string; + payloadType: string; + context: any; + traceContext: Record; + traceId: string; + spanId: string; + parentSpanId?: string; + lockedToVersionId?: string; + concurrencyKey?: string; + queueName: string; + queue?: QueueOptions; + isTest: boolean; + delayUntil?: Date; + queuedAt?: Date; + maxAttempts?: number; + ttl?: string; + tags: string[]; + parentTaskRunId?: string; + parentTaskRunAttemptId?: string; + rootTaskRunId?: string; + batchId?: string; + resumeParentOnCompletion: boolean; + depth: number; + metadata?: string; + metadataType?: string; + seedMetadata?: string; + seedMetadataType?: string; + isWait: boolean; }; export class RunEngine { - private prisma: PrismaClient; private redis: Redis; + private prisma: PrismaClientOrTransaction; private redlock: Redlock; constructor(private readonly options: Options) { @@ -28,9 +65,192 @@ export class RunEngine { /** "Triggers" one run, which creates the run */ - async trigger() { - // const result = await this.options.prisma.taskRun.create({}); - // return result; + async trigger( + { + friendlyId, + number, + environment, + idempotencyKey, + taskIdentifier, + payload, + payloadType, + context, + traceContext, + traceId, + spanId, + parentSpanId, + lockedToVersionId, + concurrencyKey, + queueName, + queue, + isTest, + delayUntil, + queuedAt, + maxAttempts, + ttl, + tags, + parentTaskRunId, + parentTaskRunAttemptId, + rootTaskRunId, + batchId, + resumeParentOnCompletion, + depth, + metadata, + metadataType, + seedMetadata, + seedMetadataType, + isWait, + }: TriggerParams, + tx?: PrismaClientOrTransaction + ) { + //todo create a waitable + const prisma = tx ?? this.prisma; + + //todo attach waitable to the run + + const taskRun = await prisma.taskRun.create({ + data: { + status: delayUntil ? "DELAYED" : "PENDING", + number, + friendlyId, + runtimeEnvironmentId: environment.id, + projectId: environment.project.id, + idempotencyKey, + taskIdentifier, + payload, + payloadType, + context, + traceContext, + traceId, + spanId, + parentSpanId, + lockedToVersionId, + concurrencyKey, + queue: queueName, + isTest, + delayUntil, + queuedAt, + maxAttempts, + ttl, + tags: + tags.length === 0 + ? undefined + : { + connect: tags.map((id) => ({ id })), + }, + parentTaskRunId, + parentTaskRunAttemptId, + rootTaskRunId, + batchId, + resumeParentOnCompletion, + depth, + metadata, + metadataType, + seedMetadata, + seedMetadataType, + }, + }); + + await this.redlock.using([taskRun.id], 5000, async (signal) => { + if (signal.aborted) { + throw signal.error; + } + + if (isWait) { + //todo block the parentTaskRun with this runId + } + + if (dependentAttempt) { + await prisma.taskRunDependency.create({ + data: { + taskRunId: taskRun.id, + dependentAttemptId: dependentAttempt.id, + }, + }); + } else if (dependentBatchRun) { + await prisma.taskRunDependency.create({ + data: { + taskRunId: taskRun.id, + dependentBatchRunId: dependentBatchRun.id, + }, + }); + } + + if (queue) { + const concurrencyLimit = + typeof body.options.queue.concurrencyLimit === "number" + ? Math.max(0, body.options.queue.concurrencyLimit) + : undefined; + + let taskQueue = await prisma.taskQueue.findFirst({ + where: { + runtimeEnvironmentId: environment.id, + name: queueName, + }, + }); + + if (taskQueue) { + taskQueue = await prisma.taskQueue.update({ + where: { + id: taskQueue.id, + }, + data: { + concurrencyLimit, + rateLimit: body.options.queue.rateLimit, + }, + }); + } else { + taskQueue = await prisma.taskQueue.create({ + data: { + friendlyId: generateFriendlyId("queue"), + name: queueName, + concurrencyLimit, + runtimeEnvironmentId: environment.id, + projectId: environment.projectId, + rateLimit: body.options.queue.rateLimit, + type: "NAMED", + }, + }); + } + + if (typeof taskQueue.concurrencyLimit === "number") { + await marqs?.updateQueueConcurrencyLimits( + environment, + taskQueue.name, + taskQueue.concurrencyLimit + ); + } else { + await marqs?.removeQueueConcurrencyLimits(environment, taskQueue.name); + } + } + + if (taskRun.delayUntil) { + //todo create an additional WaitPoint + + await workerQueue.enqueue( + "v3.enqueueDelayedRun", + { runId: taskRun.id }, + { tx, runAt: delayUntil, jobKey: `v3.enqueueDelayedRun.${taskRun.id}` } + ); + } + + if (!taskRun.delayUntil && taskRun.ttl) { + const expireAt = parseNaturalLanguageDuration(taskRun.ttl); + + if (expireAt) { + await workerQueue.enqueue( + "v3.expireRun", + { runId: taskRun.id }, + { tx, runAt: expireAt, jobKey: `v3.expireRun.${taskRun.id}` } + ); + } + } + }); + + return taskRun; + //todo waitpoints + //todo enqueue + //todo release concurrency? } /** Triggers multiple runs. diff --git a/internal-packages/run-engine/src/index.ts b/internal-packages/run-engine/src/index.ts new file mode 100644 index 0000000000..b71175be2a --- /dev/null +++ b/internal-packages/run-engine/src/index.ts @@ -0,0 +1 @@ +export { RunEngine } from "./engine/index"; diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index dbd50af5ec..37b4d4f62d 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -8,7 +8,10 @@ import { Logger } from "@trigger.dev/core/logger"; import { flattenAttributes } from "@trigger.dev/core/v3"; import { Redis, type Callback, type RedisOptions, type Result } from "ioredis"; import { AsyncWorker } from "../shared/asyncWorker.js"; -import { attributesFromAuthenticatedEnv, AuthenticatedEnvironment } from "../shared/index.js"; +import { + attributesFromAuthenticatedEnv, + MinimalAuthenticatedEnvironment, +} from "../shared/index.js"; import { InputPayload, OutputPayload, @@ -71,38 +74,38 @@ export class RunQueue { } public async updateQueueConcurrencyLimits( - env: AuthenticatedEnvironment, + env: MinimalAuthenticatedEnvironment, queue: string, concurrency: number ) { return this.redis.set(this.keys.queueConcurrencyLimitKey(env, queue), concurrency); } - public async removeQueueConcurrencyLimits(env: AuthenticatedEnvironment, queue: string) { + public async removeQueueConcurrencyLimits(env: MinimalAuthenticatedEnvironment, queue: string) { return this.redis.del(this.keys.queueConcurrencyLimitKey(env, queue)); } - public async getQueueConcurrencyLimit(env: AuthenticatedEnvironment, queue: string) { + public async getQueueConcurrencyLimit(env: MinimalAuthenticatedEnvironment, queue: string) { const result = await this.redis.get(this.keys.queueConcurrencyLimitKey(env, queue)); return result ? Number(result) : undefined; } - public async updateEnvConcurrencyLimits(env: AuthenticatedEnvironment) { + public async updateEnvConcurrencyLimits(env: MinimalAuthenticatedEnvironment) { await this.#callUpdateGlobalConcurrencyLimits({ envConcurrencyLimitKey: this.keys.envConcurrencyLimitKey(env), envConcurrencyLimit: env.maximumConcurrencyLimit, }); } - public async getEnvConcurrencyLimit(env: AuthenticatedEnvironment) { + public async getEnvConcurrencyLimit(env: MinimalAuthenticatedEnvironment) { const result = await this.redis.get(this.keys.envConcurrencyLimitKey(env)); return result ? Number(result) : this.options.defaultEnvConcurrency; } public async lengthOfQueue( - env: AuthenticatedEnvironment, + env: MinimalAuthenticatedEnvironment, queue: string, concurrencyKey?: string ) { @@ -110,7 +113,7 @@ export class RunQueue { } public async oldestMessageInQueue( - env: AuthenticatedEnvironment, + env: MinimalAuthenticatedEnvironment, queue: string, concurrencyKey?: string ) { @@ -130,22 +133,25 @@ export class RunQueue { } public async currentConcurrencyOfQueue( - env: AuthenticatedEnvironment, + env: MinimalAuthenticatedEnvironment, queue: string, concurrencyKey?: string ) { return this.redis.scard(this.keys.currentConcurrencyKey(env, queue, concurrencyKey)); } - public async currentConcurrencyOfEnvironment(env: AuthenticatedEnvironment) { + public async currentConcurrencyOfEnvironment(env: MinimalAuthenticatedEnvironment) { return this.redis.scard(this.keys.envCurrentConcurrencyKey(env)); } - public async currentConcurrencyOfProject(env: AuthenticatedEnvironment) { + public async currentConcurrencyOfProject(env: MinimalAuthenticatedEnvironment) { return this.redis.scard(this.keys.projectCurrentConcurrencyKey(env)); } - public async currentConcurrencyOfTask(env: AuthenticatedEnvironment, taskIdentifier: string) { + public async currentConcurrencyOfTask( + env: MinimalAuthenticatedEnvironment, + taskIdentifier: string + ) { return this.redis.scard(this.keys.taskIdentifierCurrentConcurrencyKey(env, taskIdentifier)); } @@ -153,7 +159,7 @@ export class RunQueue { env, message, }: { - env: AuthenticatedEnvironment; + env: MinimalAuthenticatedEnvironment; message: InputPayload; }) { return await this.#trace( @@ -195,7 +201,7 @@ export class RunQueue { ); } - public async dequeueMessageInEnv(env: AuthenticatedEnvironment) { + public async dequeueMessageInEnv(env: MinimalAuthenticatedEnvironment) { return this.#trace( "dequeueMessageInEnv", async (span) => { diff --git a/internal-packages/run-engine/src/run-queue/types.ts b/internal-packages/run-engine/src/run-queue/types.ts index 1502fed560..eaf49e5491 100644 --- a/internal-packages/run-engine/src/run-queue/types.ts +++ b/internal-packages/run-engine/src/run-queue/types.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import { AuthenticatedEnvironment } from "../shared/index.js"; +import { MinimalAuthenticatedEnvironment } from "../shared/index.js"; import { RuntimeEnvironmentType } from "../../../database/src/index.js"; import { env } from "process"; import { version } from "os"; @@ -43,35 +43,35 @@ export type QueueWithScores = { export type QueueRange = { offset: number; count: number }; export interface RunQueueKeyProducer { - envSharedQueueKey(env: AuthenticatedEnvironment): string; + envSharedQueueKey(env: MinimalAuthenticatedEnvironment): string; sharedQueueKey(): string; sharedQueueScanPattern(): string; queueCurrentConcurrencyScanPattern(): string; //queue - queueKey(env: AuthenticatedEnvironment, queue: string, concurrencyKey?: string): string; - queueConcurrencyLimitKey(env: AuthenticatedEnvironment, queue: string): string; + queueKey(env: MinimalAuthenticatedEnvironment, queue: string, concurrencyKey?: string): string; + queueConcurrencyLimitKey(env: MinimalAuthenticatedEnvironment, queue: string): string; concurrencyLimitKeyFromQueue(queue: string): string; currentConcurrencyKeyFromQueue(queue: string): string; currentConcurrencyKey( - env: AuthenticatedEnvironment, + env: MinimalAuthenticatedEnvironment, queue: string, concurrencyKey?: string ): string; disabledConcurrencyLimitKeyFromQueue(queue: string): string; //env oncurrency - envCurrentConcurrencyKey(env: AuthenticatedEnvironment): string; - envConcurrencyLimitKey(env: AuthenticatedEnvironment): string; + envCurrentConcurrencyKey(env: MinimalAuthenticatedEnvironment): string; + envConcurrencyLimitKey(env: MinimalAuthenticatedEnvironment): string; envConcurrencyLimitKeyFromQueue(queue: string): string; envCurrentConcurrencyKeyFromQueue(queue: string): string; //task concurrency taskIdentifierCurrentConcurrencyKey( - env: AuthenticatedEnvironment, + env: MinimalAuthenticatedEnvironment, taskIdentifier: string ): string; taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(queue: string): string; taskIdentifierCurrentConcurrencyKeyFromQueue(queue: string, taskIdentifier: string): string; //project concurrency - projectCurrentConcurrencyKey(env: AuthenticatedEnvironment): string; + projectCurrentConcurrencyKey(env: MinimalAuthenticatedEnvironment): string; projectCurrentConcurrencyKeyFromQueue(queue: string): string; //message payload messageKeyPrefixFromQueue(queue: string): string; diff --git a/internal-packages/run-engine/src/shared/index.ts b/internal-packages/run-engine/src/shared/index.ts index 7bef31b404..6bd3e304e3 100644 --- a/internal-packages/run-engine/src/shared/index.ts +++ b/internal-packages/run-engine/src/shared/index.ts @@ -1,19 +1,19 @@ import { Attributes } from "@opentelemetry/api"; import { Prisma } from "../../../database/src"; -type EnvironmentWithExtras = Prisma.RuntimeEnvironmentGetPayload<{ +export type AuthenticatedEnvironment = Prisma.RuntimeEnvironmentGetPayload<{ include: { project: true; organization: true; orgMember: true }; }>; -export type AuthenticatedEnvironment = { - id: EnvironmentWithExtras["id"]; - type: EnvironmentWithExtras["type"]; - maximumConcurrencyLimit: EnvironmentWithExtras["maximumConcurrencyLimit"]; +export type MinimalAuthenticatedEnvironment = { + id: AuthenticatedEnvironment["id"]; + type: AuthenticatedEnvironment["type"]; + maximumConcurrencyLimit: AuthenticatedEnvironment["maximumConcurrencyLimit"]; project: { - id: EnvironmentWithExtras["project"]["id"]; + id: AuthenticatedEnvironment["project"]["id"]; }; organization: { - id: EnvironmentWithExtras["organization"]["id"]; + id: AuthenticatedEnvironment["organization"]["id"]; }; }; @@ -29,7 +29,7 @@ const SemanticEnvResources = { USER_ID: "$trigger.user.id", }; -export function attributesFromAuthenticatedEnv(env: AuthenticatedEnvironment): Attributes { +export function attributesFromAuthenticatedEnv(env: MinimalAuthenticatedEnvironment): Attributes { return { [SemanticEnvResources.ENV_ID]: env.id, [SemanticEnvResources.ENV_TYPE]: env.type, From f7d3c41feb8df387b255c40b951b162ac8189f8e Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 29 Sep 2024 18:33:49 -0700 Subject: [PATCH 056/114] Progress on run engine triggering, first waitpoint code --- apps/webapp/app/v3/friendlyIdentifiers.ts | 8 +- internal-packages/run-engine/package.json | 1 + .../run-engine/src/engine/index.ts | 74 +- .../run-engine/src/engine/waitpoint.ts | 50 + packages/core/package.json | 1 + packages/core/src/v3/apps/friendlyId.ts | 7 + packages/core/src/v3/apps/index.ts | 1 + pnpm-lock.yaml | 994 +++++++++--------- 8 files changed, 610 insertions(+), 526 deletions(-) create mode 100644 internal-packages/run-engine/src/engine/waitpoint.ts create mode 100644 packages/core/src/v3/apps/friendlyId.ts diff --git a/apps/webapp/app/v3/friendlyIdentifiers.ts b/apps/webapp/app/v3/friendlyIdentifiers.ts index 1036edf297..b545fb3431 100644 --- a/apps/webapp/app/v3/friendlyIdentifiers.ts +++ b/apps/webapp/app/v3/friendlyIdentifiers.ts @@ -1,7 +1 @@ -import { customAlphabet } from "nanoid"; - -const idGenerator = customAlphabet("123456789abcdefghijkmnopqrstuvwxyz", 21); - -export function generateFriendlyId(prefix: string, size?: number) { - return `${prefix}_${idGenerator(size)}`; -} +export { generateFriendlyId } from "@trigger.dev/core/v3/apps"; diff --git a/internal-packages/run-engine/package.json b/internal-packages/run-engine/package.json index 2762976dbd..10e2f95a18 100644 --- a/internal-packages/run-engine/package.json +++ b/internal-packages/run-engine/package.json @@ -10,6 +10,7 @@ "@trigger.dev/core": "workspace:*", "@trigger.dev/database": "workspace:*", "ioredis": "^5.3.2", + "nanoid": "^3.3.4", "redlock": "5.0.0-beta.2", "typescript": "^4.8.4", "zod": "3.22.3" diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index a232e286af..2dc17fd322 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -1,13 +1,16 @@ -import { PrismaClient, Prisma, PrismaClientOrTransaction } from "@trigger.dev/database"; +import { QueueOptions } from "@trigger.dev/core/v3"; +import { PrismaClientOrTransaction } from "@trigger.dev/database"; import { Redis, type RedisOptions } from "ioredis"; import Redlock from "redlock"; -import { AuthenticatedEnvironment, MinimalAuthenticatedEnvironment } from "../shared"; -import { QueueOptions } from "@trigger.dev/core/v3"; import { RunQueue } from "../run-queue"; +import { MinimalAuthenticatedEnvironment } from "../shared"; +import { blockRunWithWaitpoint, createRunAssociatedWaitpoint } from "./waitpoint"; +import { generateFriendlyId } from "@trigger.dev/core/v3/apps"; type Options = { redis: RedisOptions; prisma: PrismaClientOrTransaction; + queue: RunQueue; }; type TriggerParams = { @@ -50,6 +53,7 @@ export class RunEngine { private redis: Redis; private prisma: PrismaClientOrTransaction; private redlock: Redlock; + private runQueue: RunQueue; constructor(private readonly options: Options) { this.prisma = options.prisma; @@ -61,6 +65,19 @@ export class RunEngine { retryJitter: 200, // time in ms automaticExtensionThreshold: 500, // time in ms }); + + //todo change the way dequeuing works so you have to pass in the name of the shared queue? + + this.runQueue = new RunQueue({ + name: "rq", + tracer: trace.getTracer("rq"), + queuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 36 }), + envQueuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 12 }), + workers: 1, + defaultEnvConcurrency: 10, + enableRebalancing: false, + logger: new Logger("RunQueue", "warn"), + }); } /** "Triggers" one run, which creates the run @@ -103,11 +120,9 @@ export class RunEngine { }: TriggerParams, tx?: PrismaClientOrTransaction ) { - //todo create a waitable const prisma = tx ?? this.prisma; - //todo attach waitable to the run - + //create run const taskRun = await prisma.taskRun.create({ data: { status: delayUntil ? "DELAYED" : "PENDING", @@ -156,30 +171,24 @@ export class RunEngine { throw signal.error; } - if (isWait) { - //todo block the parentTaskRun with this runId - } + //create associated waitpoint (this completes when the run completes) + const associatedWaitpoint = await createRunAssociatedWaitpoint(prisma, { + projectId: environment.project.id, + completedByTaskRunId: taskRun.id, + }); - if (dependentAttempt) { - await prisma.taskRunDependency.create({ - data: { - taskRunId: taskRun.id, - dependentAttemptId: dependentAttempt.id, - }, - }); - } else if (dependentBatchRun) { - await prisma.taskRunDependency.create({ - data: { - taskRunId: taskRun.id, - dependentBatchRunId: dependentBatchRun.id, - }, + if (isWait && parentTaskRunId) { + //this will block the parent run from continuing until this waitpoint is completed (and removed) + await blockRunWithWaitpoint(prisma, { + runId: parentTaskRunId, + waitpoint: associatedWaitpoint, }); } if (queue) { const concurrencyLimit = - typeof body.options.queue.concurrencyLimit === "number" - ? Math.max(0, body.options.queue.concurrencyLimit) + typeof queue.concurrencyLimit === "number" + ? Math.max(0, queue.concurrencyLimit) : undefined; let taskQueue = await prisma.taskQueue.findFirst({ @@ -196,7 +205,7 @@ export class RunEngine { }, data: { concurrencyLimit, - rateLimit: body.options.queue.rateLimit, + rateLimit: queue.rateLimit, }, }); } else { @@ -206,28 +215,27 @@ export class RunEngine { name: queueName, concurrencyLimit, runtimeEnvironmentId: environment.id, - projectId: environment.projectId, - rateLimit: body.options.queue.rateLimit, + projectId: environment.project.id, + rateLimit: queue.rateLimit, type: "NAMED", }, }); } if (typeof taskQueue.concurrencyLimit === "number") { - await marqs?.updateQueueConcurrencyLimits( + await this.runQueue.updateQueueConcurrencyLimits( environment, taskQueue.name, taskQueue.concurrencyLimit ); } else { - await marqs?.removeQueueConcurrencyLimits(environment, taskQueue.name); + await this.runQueue.removeQueueConcurrencyLimits(environment, taskQueue.name); } } if (taskRun.delayUntil) { - //todo create an additional WaitPoint - - await workerQueue.enqueue( + //todo create a WaitPoint + const delayWaitpoint = await workerQueue.enqueue( "v3.enqueueDelayedRun", { runId: taskRun.id }, { tx, runAt: delayUntil, jobKey: `v3.enqueueDelayedRun.${taskRun.id}` } @@ -245,6 +253,8 @@ export class RunEngine { ); } } + + await this.runQueue.enqueueMessage({ env: environment, message: {} }); }); return taskRun; diff --git a/internal-packages/run-engine/src/engine/waitpoint.ts b/internal-packages/run-engine/src/engine/waitpoint.ts new file mode 100644 index 0000000000..4dae24783a --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpoint.ts @@ -0,0 +1,50 @@ +import { PrismaClientOrTransaction, Waitpoint } from "@trigger.dev/database"; +import { nanoid } from "nanoid"; + +export async function createRunAssociatedWaitpoint( + prisma: PrismaClientOrTransaction, + { projectId, completedByTaskRunId }: { projectId: string; completedByTaskRunId: string } +) { + return prisma.waitpoint.create({ + data: { + type: "RUN", + status: "PENDING", + idempotencyKey: nanoid(24), + userProvidedIdempotencyKey: false, + projectId, + completedByTaskRunId, + }, + }); +} + +export async function createDateTimeWaitpoint( + prisma: PrismaClientOrTransaction, + { projectId, completedAfter }: { projectId: string; completedAfter: Date } +) { + return prisma.waitpoint.create({ + data: { + type: "DATETIME", + status: "PENDING", + idempotencyKey: nanoid(24), + userProvidedIdempotencyKey: false, + projectId, + completedAfter, + }, + }); +} + +export async function blockRunWithWaitpoint( + prisma: PrismaClientOrTransaction, + { runId, waitpoint }: { runId: string; waitpoint: Waitpoint } +) { + return prisma.taskRunWaitpoint.create({ + data: { + taskRunId: runId, + waitpointId: waitpoint.id, + projectId: waitpoint.projectId, + }, + }); +} + +/** Any runs blocked by this waitpoint will get continued (if no other waitpoints exist) */ +export function completeWaitpoint(id: string) {} diff --git a/packages/core/package.json b/packages/core/package.json index c6accd5765..4dc7613521 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -188,6 +188,7 @@ "@opentelemetry/semantic-conventions": "1.25.1", "execa": "^8.0.1", "humanize-duration": "^3.27.3", + "nanoid": "^3.3.4", "socket.io-client": "4.7.5", "superjson": "^2.2.1", "zod": "3.22.3", diff --git a/packages/core/src/v3/apps/friendlyId.ts b/packages/core/src/v3/apps/friendlyId.ts new file mode 100644 index 0000000000..1036edf297 --- /dev/null +++ b/packages/core/src/v3/apps/friendlyId.ts @@ -0,0 +1,7 @@ +import { customAlphabet } from "nanoid"; + +const idGenerator = customAlphabet("123456789abcdefghijkmnopqrstuvwxyz", 21); + +export function generateFriendlyId(prefix: string, size?: number) { + return `${prefix}_${idGenerator(size)}`; +} diff --git a/packages/core/src/v3/apps/index.ts b/packages/core/src/v3/apps/index.ts index 9c9b429ed1..73c24ae0e7 100644 --- a/packages/core/src/v3/apps/index.ts +++ b/packages/core/src/v3/apps/index.ts @@ -5,3 +5,4 @@ export * from "./checkpoints.js"; export * from "./http.js"; export * from "./provider.js"; export * from "./isExecaChildProcess.js"; +export * from "./friendlyId.js"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 025af9709b..ad27a307c0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -37,7 +37,7 @@ importers: version: 1.37.0 '@trigger.dev/database': specifier: workspace:* - version: link:packages/database + version: link:internal-packages/database '@types/node': specifier: 20.14.14 version: 20.14.14 @@ -240,6 +240,9 @@ importers: '@heroicons/react': specifier: ^2.0.12 version: 2.0.13(react@18.2.0) + '@internal/run-engine': + specifier: workspace:* + version: link:../../internal-packages/run-engine '@internationalized/date': specifier: ^3.5.1 version: 3.5.1 @@ -377,10 +380,10 @@ importers: version: link:../../packages/core '@trigger.dev/database': specifier: workspace:* - version: link:../../packages/database + version: link:../../internal-packages/database '@trigger.dev/otlp-importer': specifier: workspace:* - version: link:../../packages/otlp-importer + version: link:../../internal-packages/otlp-importer '@trigger.dev/platform': specifier: 1.0.12 version: 1.0.12 @@ -437,7 +440,7 @@ importers: version: 16.4.5 emails: specifier: workspace:* - version: link:../../packages/emails + version: link:../../internal-packages/emails evt: specifier: ^2.4.13 version: 2.4.13 @@ -820,6 +823,118 @@ importers: docs: {} + internal-packages/database: + dependencies: + '@prisma/client': + specifier: 5.4.1 + version: 5.4.1(prisma@5.4.1) + typescript: + specifier: ^4.8.4 + version: 4.9.5 + devDependencies: + prisma: + specifier: 5.4.1 + version: 5.4.1 + + internal-packages/emails: + dependencies: + '@react-email/components': + specifier: 0.0.16 + version: 0.0.16(@types/react@18.2.69)(react@18.3.1) + '@react-email/render': + specifier: ^0.0.12 + version: 0.0.12 + react: + specifier: ^18.2.0 + version: 18.3.1 + react-email: + specifier: ^2.1.1 + version: 2.1.2(eslint@8.45.0) + resend: + specifier: ^3.2.0 + version: 3.2.0 + tiny-invariant: + specifier: ^1.2.0 + version: 1.3.1 + zod: + specifier: 3.22.3 + version: 3.22.3 + devDependencies: + '@types/node': + specifier: ^18 + version: 18.19.20 + '@types/react': + specifier: 18.2.69 + version: 18.2.69 + typescript: + specifier: ^4.9.4 + version: 4.9.5 + + internal-packages/otlp-importer: + dependencies: + long: + specifier: ^5.2.3 + version: 5.2.3 + protobufjs: + specifier: ^7.2.6 + version: 7.3.2 + devDependencies: + '@types/node': + specifier: ^20 + version: 20.14.14 + rimraf: + specifier: ^3.0.2 + version: 3.0.2 + ts-proto: + specifier: ^1.167.3 + version: 1.167.3 + typescript: + specifier: ^5.5.0 + version: 5.5.4 + + internal-packages/run-engine: + dependencies: + '@opentelemetry/api': + specifier: ^1.9.0 + version: 1.9.0 + '@opentelemetry/semantic-conventions': + specifier: ^1.27.0 + version: 1.27.0 + '@trigger.dev/core': + specifier: workspace:* + version: link:../../packages/core + '@trigger.dev/database': + specifier: workspace:* + version: link:../database + ioredis: + specifier: ^5.3.2 + version: 5.3.2 + nanoid: + specifier: ^3.3.4 + version: 3.3.7 + redlock: + specifier: 5.0.0-beta.2 + version: 5.0.0-beta.2 + typescript: + specifier: ^4.8.4 + version: 4.9.5 + zod: + specifier: 3.22.3 + version: 3.22.3 + devDependencies: + '@testcontainers/postgresql': + specifier: ^10.13.1 + version: 10.13.1 + '@testcontainers/redis': + specifier: ^10.13.1 + version: 10.13.1 + testcontainers: + specifier: ^10.13.1 + version: 10.13.1 + vitest: + specifier: ^1.4.0 + version: 1.6.0(@types/node@20.14.14) + packages/build: dependencies: '@trigger.dev/core': @@ -1109,6 +1224,9 @@ importers: humanize-duration: specifier: ^3.27.3 version: 3.27.3 + nanoid: + specifier: ^3.3.4 + version: 3.3.7 socket.io-client: specifier: 4.7.5 version: 4.7.5 @@ -1165,115 +1283,6 @@ importers: specifier: ^1.6.0 version: 1.6.0(@types/node@20.14.14) - packages/database: - dependencies: - '@prisma/client': - specifier: 5.4.1 - version: 5.4.1(prisma@5.4.1) - typescript: - specifier: ^4.8.4 - version: 4.9.5 - devDependencies: - prisma: - specifier: 5.4.1 - version: 5.4.1 - - packages/emails: - dependencies: - '@react-email/components': - specifier: 0.0.16 - version: 0.0.16(@types/react@18.2.69)(react@18.2.0) - '@react-email/render': - specifier: ^0.0.12 - version: 0.0.12 - react: - specifier: ^18.2.0 - version: 18.2.0 - react-email: - specifier: ^2.1.1 - version: 2.1.2(eslint@8.45.0) - resend: - specifier: ^3.2.0 - version: 3.2.0 - tiny-invariant: - specifier: ^1.2.0 - version: 1.3.1 - zod: - specifier: 3.22.3 - version: 3.22.3 - devDependencies: - '@types/node': - specifier: ^18 - version: 18.17.1 - '@types/react': - specifier: 18.2.69 - version: 18.2.69 - typescript: - specifier: ^4.9.4 - version: 4.9.5 - - packages/otlp-importer: - dependencies: - long: - specifier: ^5.2.3 - version: 5.2.3 - protobufjs: - specifier: ^7.2.6 - version: 7.2.6 - devDependencies: - '@types/node': - specifier: ^20 - version: 20.14.14 - rimraf: - specifier: ^3.0.2 - version: 3.0.2 - ts-proto: - specifier: ^1.167.3 - version: 1.167.3 - typescript: - specifier: ^5.5.0 - version: 5.5.4 - - packages/run-engine: - dependencies: - '@opentelemetry/api': - specifier: ^1.9.0 - version: 1.9.0 - '@opentelemetry/semantic-conventions': - specifier: ^1.27.0 - version: 1.27.0 - '@trigger.dev/core': - specifier: workspace:* - version: link:../core - '@trigger.dev/database': - specifier: workspace:* - version: link:../database - ioredis: - specifier: ^5.3.2 - version: 5.3.2 - redlock: - specifier: 5.0.0-beta.2 - version: 5.0.0-beta.2 - typescript: - specifier: ^4.8.4 - version: 4.9.5 - zod: - specifier: 3.22.3 - version: 3.22.3 - devDependencies: - '@testcontainers/postgresql': - specifier: ^10.13.1 - version: 10.13.1 - '@testcontainers/redis': - specifier: ^10.13.1 - version: 10.13.1 - testcontainers: - specifier: ^10.13.1 - version: 10.13.1 - vitest: - specifier: ^1.4.0 - version: 1.6.0(@types/node@20.14.14) - packages/trigger-sdk: dependencies: '@opentelemetry/api': @@ -2782,7 +2791,7 @@ packages: engines: {node: '>=6.0.0'} hasBin: true dependencies: - '@babel/types': 7.24.7 + '@babel/types': 7.25.6 dev: false /@babel/parser@7.24.5: @@ -6099,15 +6108,15 @@ packages: - '@types/react' dev: false - /@floating-ui/react-dom@2.0.9(react-dom@18.2.0)(react@18.2.0): + /@floating-ui/react-dom@2.0.9(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-q0umO0+LQK4+p6aGyvzASqKbKOJcAHJ7ycE9CuUvfx3s9zTHWmGJTPOIlM/hmSBfUfg/XfY5YhLBLR/LHwShQQ==} peerDependencies: react: '>=16.8.0' react-dom: '>=16.8.0' dependencies: '@floating-ui/dom': 1.6.5 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) dev: false /@floating-ui/utils@0.2.2: @@ -8209,7 +8218,7 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false - /@radix-ui/react-arrow@1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0): + /@radix-ui/react-arrow@1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-wSP+pHsB/jQRaL6voubsQ/ZlrGBHHrOjmBnr19hxYgtS0WvAFwZhK2WP/YY5yF9uKECCEEDGxuLxq1NBK51wFA==} peerDependencies: '@types/react': '*' @@ -8223,14 +8232,14 @@ packages: optional: true dependencies: '@babel/runtime': 7.24.5 - '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) '@types/react': 18.3.1 '@types/react-dom': 18.2.7 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) dev: false - /@radix-ui/react-collapsible@1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0): + /@radix-ui/react-collapsible@1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-UBmVDkmR6IvDsloHVN+3rtx4Mi5TFvylYXpluuv0f37dtaz3H99bp8No0LGXRigVpl3UAT4l9j6bIchh42S/Gg==} peerDependencies: '@types/react': '*' @@ -8245,17 +8254,17 @@ packages: dependencies: '@babel/runtime': 7.24.5 '@radix-ui/primitive': 1.0.1 - '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-context': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-id': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-presence': 1.0.1(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.3.1)(react@18.2.0) + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-context': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-id': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-presence': 1.0.1(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.3.1)(react@18.3.1) '@types/react': 18.3.1 '@types/react-dom': 18.2.7 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) dev: false /@radix-ui/react-collection@1.0.2(react-dom@18.2.0)(react@18.2.0): @@ -8297,7 +8306,7 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false - /@radix-ui/react-collection@1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0): + /@radix-ui/react-collection@1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-3SzW+0PW7yBBoQlT8wNcGtaxaD0XSu0uLUFgrtHY08Acx05TaHaOmVLR73c0j/cqpDy53KBMO7s0dx2wmOIDIA==} peerDependencies: '@types/react': '*' @@ -8311,14 +8320,14 @@ packages: optional: true dependencies: '@babel/runtime': 7.24.5 - '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-context': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-slot': 1.0.2(@types/react@18.3.1)(react@18.2.0) + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-context': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-slot': 1.0.2(@types/react@18.3.1)(react@18.3.1) '@types/react': 18.3.1 '@types/react-dom': 18.2.7 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) dev: false /@radix-ui/react-compose-refs@1.0.0(react@18.2.0): @@ -8344,7 +8353,21 @@ packages: react: 18.2.0 dev: false - /@radix-ui/react-compose-refs@1.0.1(@types/react@18.3.1)(react@18.2.0): + /@radix-ui/react-compose-refs@1.0.1(@types/react@18.2.69)(react@18.3.1): + resolution: {integrity: sha512-fDSBgd44FKHa1FRMU59qBMPFcl2PZE+2nmqunj+BWFyYYjnhIDWL2ItDs3rrbJDQOtzt5nIebLCQc4QRfz6LJw==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.24.5 + '@types/react': 18.2.69 + react: 18.3.1 + dev: false + + /@radix-ui/react-compose-refs@1.0.1(@types/react@18.3.1)(react@18.3.1): resolution: {integrity: sha512-fDSBgd44FKHa1FRMU59qBMPFcl2PZE+2nmqunj+BWFyYYjnhIDWL2ItDs3rrbJDQOtzt5nIebLCQc4QRfz6LJw==} peerDependencies: '@types/react': '*' @@ -8355,7 +8378,7 @@ packages: dependencies: '@babel/runtime': 7.24.5 '@types/react': 18.3.1 - react: 18.2.0 + react: 18.3.1 dev: false /@radix-ui/react-compose-refs@1.1.0(@types/react@18.2.69)(react@18.2.0): @@ -8394,7 +8417,7 @@ packages: react: 18.2.0 dev: false - /@radix-ui/react-context@1.0.1(@types/react@18.3.1)(react@18.2.0): + /@radix-ui/react-context@1.0.1(@types/react@18.3.1)(react@18.3.1): resolution: {integrity: sha512-ebbrdFoYTcuZ0v4wG5tedGnp9tzcV8awzsxYph7gXUyvnNLuTIcCk1q17JEbnVhXAKG9oX3KtchwiMIAYp9NLg==} peerDependencies: '@types/react': '*' @@ -8405,7 +8428,7 @@ packages: dependencies: '@babel/runtime': 7.24.5 '@types/react': 18.3.1 - react: 18.2.0 + react: 18.3.1 dev: false /@radix-ui/react-dialog@1.0.3(@types/react@18.2.69)(react-dom@18.2.0)(react@18.2.0): @@ -8492,7 +8515,7 @@ packages: react: 18.2.0 dev: false - /@radix-ui/react-direction@1.0.1(@types/react@18.3.1)(react@18.2.0): + /@radix-ui/react-direction@1.0.1(@types/react@18.3.1)(react@18.3.1): resolution: {integrity: sha512-RXcvnXgyvYvBEOhCBuddKecVkoMiI10Jcm5cTI7abJRAHYfFxeu+FBQs/DvdxSYucxR5mna0dNsL6QFlds5TMA==} peerDependencies: '@types/react': '*' @@ -8503,7 +8526,7 @@ packages: dependencies: '@babel/runtime': 7.24.5 '@types/react': 18.3.1 - react: 18.2.0 + react: 18.3.1 dev: false /@radix-ui/react-dismissable-layer@1.0.3(react-dom@18.2.0)(react@18.2.0): @@ -8547,7 +8570,7 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false - /@radix-ui/react-dismissable-layer@1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0): + /@radix-ui/react-dismissable-layer@1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-7UpBa/RKMoHJYjie1gkF1DlK8l1fdU/VKDpoS3rCCo8YBJR294GwcEHyxHw72yvphJ7ld0AXEcSLAzY2F/WyCg==} peerDependencies: '@types/react': '*' @@ -8562,17 +8585,17 @@ packages: dependencies: '@babel/runtime': 7.24.5 '@radix-ui/primitive': 1.0.1 - '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-use-escape-keydown': 1.0.3(@types/react@18.3.1)(react@18.2.0) + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-use-escape-keydown': 1.0.3(@types/react@18.3.1)(react@18.3.1) '@types/react': 18.3.1 '@types/react-dom': 18.2.7 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) dev: false - /@radix-ui/react-dismissable-layer@1.0.5(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0): + /@radix-ui/react-dismissable-layer@1.0.5(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-aJeDjQhywg9LBu2t/At58hCvr7pEm0o2Ke1x33B+MhjNmmZ17sy4KImo0KPLgsnc/zN7GPdce8Cnn0SWvwZO7g==} peerDependencies: '@types/react': '*' @@ -8587,14 +8610,14 @@ packages: dependencies: '@babel/runtime': 7.24.5 '@radix-ui/primitive': 1.0.1 - '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-use-escape-keydown': 1.0.3(@types/react@18.3.1)(react@18.2.0) + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-use-escape-keydown': 1.0.3(@types/react@18.3.1)(react@18.3.1) '@types/react': 18.3.1 '@types/react-dom': 18.2.7 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) dev: false /@radix-ui/react-focus-guards@1.0.0(react@18.2.0): @@ -8620,7 +8643,7 @@ packages: react: 18.2.0 dev: false - /@radix-ui/react-focus-guards@1.0.1(@types/react@18.3.1)(react@18.2.0): + /@radix-ui/react-focus-guards@1.0.1(@types/react@18.3.1)(react@18.3.1): resolution: {integrity: sha512-Rect2dWbQ8waGzhMavsIbmSVCgYxkXLxxR3ZvCX79JOglzdEy4JXMb98lq4hPxUbLr77nP0UOGf4rcMU+s1pUA==} peerDependencies: '@types/react': '*' @@ -8631,7 +8654,7 @@ packages: dependencies: '@babel/runtime': 7.24.5 '@types/react': 18.3.1 - react: 18.2.0 + react: 18.3.1 dev: false /@radix-ui/react-focus-scope@1.0.2(react-dom@18.2.0)(react@18.2.0): @@ -8671,7 +8694,7 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false - /@radix-ui/react-focus-scope@1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0): + /@radix-ui/react-focus-scope@1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-sL04Mgvf+FmyvZeYfNu1EPAaaxD+aw7cYeIB9L9Fvq8+urhltTRaEo5ysKOpHuKPclsZcSUMKlN05x4u+CINpA==} peerDependencies: '@types/react': '*' @@ -8685,13 +8708,13 @@ packages: optional: true dependencies: '@babel/runtime': 7.24.5 - '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.3.1)(react@18.2.0) + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.3.1)(react@18.3.1) '@types/react': 18.3.1 '@types/react-dom': 18.2.7 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) dev: false /@radix-ui/react-id@1.0.0(react@18.2.0): @@ -8719,7 +8742,7 @@ packages: react: 18.2.0 dev: false - /@radix-ui/react-id@1.0.1(@types/react@18.3.1)(react@18.2.0): + /@radix-ui/react-id@1.0.1(@types/react@18.3.1)(react@18.3.1): resolution: {integrity: sha512-tI7sT/kqYp8p96yGWY1OAnLHrqDgzHefRBKQ2YAkBS5ja7QLcZ9Z/uY7bEjPUatf8RomoXM8/1sMj1IJaE5UzQ==} peerDependencies: '@types/react': '*' @@ -8729,9 +8752,9 @@ packages: optional: true dependencies: '@babel/runtime': 7.24.5 - '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.3.1)(react@18.2.0) + '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.3.1)(react@18.3.1) '@types/react': 18.3.1 - react: 18.2.0 + react: 18.3.1 dev: false /@radix-ui/react-label@2.0.1(react-dom@18.2.0)(react@18.2.0): @@ -8774,7 +8797,7 @@ packages: - '@types/react' dev: false - /@radix-ui/react-popover@1.0.7(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0): + /@radix-ui/react-popover@1.0.7(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-shtvVnlsxT6faMnK/a7n0wptwBD23xc1Z5mdrtKLwVEfsEMXodS0r5s0/g5P0hX//EKYZS2sxUjqfzlg52ZSnQ==} peerDependencies: '@types/react': '*' @@ -8789,24 +8812,24 @@ packages: dependencies: '@babel/runtime': 7.24.5 '@radix-ui/primitive': 1.0.1 - '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-context': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-dismissable-layer': 1.0.5(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-focus-guards': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-focus-scope': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-id': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-popper': 1.1.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-portal': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-presence': 1.0.1(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-slot': 1.0.2(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.3.1)(react@18.2.0) + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-context': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-dismissable-layer': 1.0.5(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-focus-guards': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-focus-scope': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-id': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-popper': 1.1.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-portal': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-presence': 1.0.1(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-slot': 1.0.2(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.3.1)(react@18.3.1) '@types/react': 18.3.1 '@types/react-dom': 18.2.7 aria-hidden: 1.2.3 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - react-remove-scroll: 2.5.5(@types/react@18.3.1)(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) + react-remove-scroll: 2.5.5(@types/react@18.3.1)(react@18.3.1) dev: false /@radix-ui/react-popper@1.1.1(@types/react@18.2.69)(react-dom@18.2.0)(react@18.2.0): @@ -8832,7 +8855,7 @@ packages: - '@types/react' dev: false - /@radix-ui/react-popper@1.1.2(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0): + /@radix-ui/react-popper@1.1.2(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-1CnGGfFi/bbqtJZZ0P/NQY20xdG3E0LALJaLUEoKwPLwl6PPPfbeiCqMVQnhoFRAxjJj4RpBRJzDmUgsex2tSg==} peerDependencies: '@types/react': '*' @@ -8846,23 +8869,23 @@ packages: optional: true dependencies: '@babel/runtime': 7.24.5 - '@floating-ui/react-dom': 2.0.9(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-arrow': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-context': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-use-rect': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-use-size': 1.0.1(@types/react@18.3.1)(react@18.2.0) + '@floating-ui/react-dom': 2.0.9(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-arrow': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-context': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-use-rect': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-use-size': 1.0.1(@types/react@18.3.1)(react@18.3.1) '@radix-ui/rect': 1.0.1 '@types/react': 18.3.1 '@types/react-dom': 18.2.7 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) dev: false - /@radix-ui/react-popper@1.1.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0): + /@radix-ui/react-popper@1.1.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-cKpopj/5RHZWjrbF2846jBNacjQVwkP068DfmgrNJXpvVWrOvlAmE9xSiy5OqeE+Gi8D9fP+oDhUnPqNMY8/5w==} peerDependencies: '@types/react': '*' @@ -8876,20 +8899,20 @@ packages: optional: true dependencies: '@babel/runtime': 7.24.5 - '@floating-ui/react-dom': 2.0.9(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-arrow': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-context': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-use-rect': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-use-size': 1.0.1(@types/react@18.3.1)(react@18.2.0) + '@floating-ui/react-dom': 2.0.9(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-arrow': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-context': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-use-rect': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-use-size': 1.0.1(@types/react@18.3.1)(react@18.3.1) '@radix-ui/rect': 1.0.1 '@types/react': 18.3.1 '@types/react-dom': 18.2.7 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) dev: false /@radix-ui/react-portal@1.0.2(react-dom@18.2.0)(react@18.2.0): @@ -8925,7 +8948,7 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false - /@radix-ui/react-portal@1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0): + /@radix-ui/react-portal@1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-xLYZeHrWoPmA5mEKEfZZevoVRK/Q43GfzRXkWV6qawIWWK8t6ifIiLQdd7rmQ4Vk1bmI21XhqF9BN3jWf+phpA==} peerDependencies: '@types/react': '*' @@ -8939,14 +8962,14 @@ packages: optional: true dependencies: '@babel/runtime': 7.24.5 - '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) '@types/react': 18.3.1 '@types/react-dom': 18.2.7 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) dev: false - /@radix-ui/react-portal@1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0): + /@radix-ui/react-portal@1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-Qki+C/EuGUVCQTOTD5vzJzJuMUlewbzuKyUy+/iHM2uwGiru9gZeBJtHAPKAEkB5KWGi9mP/CHKcY0wt1aW45Q==} peerDependencies: '@types/react': '*' @@ -8960,11 +8983,11 @@ packages: optional: true dependencies: '@babel/runtime': 7.24.5 - '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) '@types/react': 18.3.1 '@types/react-dom': 18.2.7 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) dev: false /@radix-ui/react-presence@1.0.0(react-dom@18.2.0)(react@18.2.0): @@ -9002,7 +9025,7 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false - /@radix-ui/react-presence@1.0.1(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0): + /@radix-ui/react-presence@1.0.1(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-UXLW4UAbIY5ZjcvzjfRFo5gxva8QirC9hF7wRE4U5gz+TP0DbRk+//qyuAQ1McDxBt1xNMBTaciFGvEmJvAZCg==} peerDependencies: '@types/react': '*' @@ -9016,12 +9039,12 @@ packages: optional: true dependencies: '@babel/runtime': 7.24.5 - '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.3.1)(react@18.2.0) + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.3.1)(react@18.3.1) '@types/react': 18.3.1 '@types/react-dom': 18.2.7 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) dev: false /@radix-ui/react-primitive@1.0.2(react-dom@18.2.0)(react@18.2.0): @@ -9057,7 +9080,7 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false - /@radix-ui/react-primitive@1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0): + /@radix-ui/react-primitive@1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-yi58uVyoAcK/Nq1inRY56ZSjKypBNKTa/1mcL8qdl6oJeEaDbOldlzrGn7P6Q3Id5d+SYNGc5AJgc4vGhjs5+g==} peerDependencies: '@types/react': '*' @@ -9071,11 +9094,11 @@ packages: optional: true dependencies: '@babel/runtime': 7.24.5 - '@radix-ui/react-slot': 1.0.2(@types/react@18.3.1)(react@18.2.0) + '@radix-ui/react-slot': 1.0.2(@types/react@18.3.1)(react@18.3.1) '@types/react': 18.3.1 '@types/react-dom': 18.2.7 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) dev: false /@radix-ui/react-radio-group@1.1.3(@types/react-dom@18.2.7)(@types/react@18.2.69)(react-dom@18.2.0)(react@18.2.0): @@ -9157,7 +9180,7 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false - /@radix-ui/react-roving-focus@1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0): + /@radix-ui/react-roving-focus@1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-2mUg5Mgcu001VkGy+FfzZyzbmuUWzgWkj3rvv4yu+mLw03+mTzbxZHvfcGyFp2b8EkQeMkpRQ5FiA2Vr2O6TeQ==} peerDependencies: '@types/react': '*' @@ -9172,18 +9195,18 @@ packages: dependencies: '@babel/runtime': 7.24.5 '@radix-ui/primitive': 1.0.1 - '@radix-ui/react-collection': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-context': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-direction': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-id': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.3.1)(react@18.2.0) + '@radix-ui/react-collection': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-context': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-direction': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-id': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.3.1)(react@18.3.1) '@types/react': 18.3.1 '@types/react-dom': 18.2.7 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) dev: false /@radix-ui/react-select@1.2.1(@types/react@18.2.69)(react-dom@18.2.0)(react@18.2.0): @@ -9276,7 +9299,7 @@ packages: react: 18.2.0 dev: false - /@radix-ui/react-slot@1.0.2(@types/react@18.3.1)(react@18.2.0): + /@radix-ui/react-slot@1.0.2(@types/react@18.2.69)(react@18.3.1): resolution: {integrity: sha512-YeTpuq4deV+6DusvVUW4ivBgnkHwECUu0BiN43L5UCDFgdhsRUWAghhTF5MbvNTPzmiFOx90asDSUjWuCNapwg==} peerDependencies: '@types/react': '*' @@ -9286,9 +9309,24 @@ packages: optional: true dependencies: '@babel/runtime': 7.24.5 - '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.2.0) + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.2.69)(react@18.3.1) + '@types/react': 18.2.69 + react: 18.3.1 + dev: false + + /@radix-ui/react-slot@1.0.2(@types/react@18.3.1)(react@18.3.1): + resolution: {integrity: sha512-YeTpuq4deV+6DusvVUW4ivBgnkHwECUu0BiN43L5UCDFgdhsRUWAghhTF5MbvNTPzmiFOx90asDSUjWuCNapwg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.24.5 + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.3.1) '@types/react': 18.3.1 - react: 18.2.0 + react: 18.3.1 dev: false /@radix-ui/react-switch@1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.69)(react-dom@18.2.0)(react@18.2.0): @@ -9337,7 +9375,7 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false - /@radix-ui/react-toggle-group@1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0): + /@radix-ui/react-toggle-group@1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-Uaj/M/cMyiyT9Bx6fOZO0SAG4Cls0GptBWiBmBxofmDbNVnYYoyRWj/2M/6VCi/7qcXFWnHhRUfdfZFvvkuu8A==} peerDependencies: '@types/react': '*' @@ -9352,19 +9390,19 @@ packages: dependencies: '@babel/runtime': 7.24.5 '@radix-ui/primitive': 1.0.1 - '@radix-ui/react-context': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-direction': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-roving-focus': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-toggle': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.3.1)(react@18.2.0) + '@radix-ui/react-context': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-direction': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-roving-focus': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-toggle': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.3.1)(react@18.3.1) '@types/react': 18.3.1 '@types/react-dom': 18.2.7 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) dev: false - /@radix-ui/react-toggle@1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0): + /@radix-ui/react-toggle@1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-Pkqg3+Bc98ftZGsl60CLANXQBBQ4W3mTFS9EJvNxKMZ7magklKV69/id1mlAlOFDDfHvlCms0fx8fA4CMKDJHg==} peerDependencies: '@types/react': '*' @@ -9379,12 +9417,12 @@ packages: dependencies: '@babel/runtime': 7.24.5 '@radix-ui/primitive': 1.0.1 - '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.3.1)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.3.1)(react@18.3.1) '@types/react': 18.3.1 '@types/react-dom': 18.2.7 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) dev: false /@radix-ui/react-tooltip@1.0.5(@types/react@18.2.69)(react-dom@18.2.0)(react@18.2.0): @@ -9412,7 +9450,7 @@ packages: - '@types/react' dev: false - /@radix-ui/react-tooltip@1.0.6(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0): + /@radix-ui/react-tooltip@1.0.6(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-DmNFOiwEc2UDigsYj6clJENma58OelxD24O4IODoZ+3sQc3Zb+L8w1EP+y9laTuKCLAysPw4fD6/v0j4KNV8rg==} peerDependencies: '@types/react': '*' @@ -9427,21 +9465,21 @@ packages: dependencies: '@babel/runtime': 7.24.5 '@radix-ui/primitive': 1.0.1 - '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-context': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-dismissable-layer': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-id': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-popper': 1.1.2(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-portal': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-presence': 1.0.1(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-slot': 1.0.2(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-visually-hidden': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-context': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-dismissable-layer': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-id': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-popper': 1.1.2(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-portal': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-presence': 1.0.1(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-slot': 1.0.2(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-visually-hidden': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) '@types/react': 18.3.1 '@types/react-dom': 18.2.7 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) dev: false /@radix-ui/react-use-callback-ref@1.0.0(react@18.2.0): @@ -9467,7 +9505,7 @@ packages: react: 18.2.0 dev: false - /@radix-ui/react-use-callback-ref@1.0.1(@types/react@18.3.1)(react@18.2.0): + /@radix-ui/react-use-callback-ref@1.0.1(@types/react@18.3.1)(react@18.3.1): resolution: {integrity: sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==} peerDependencies: '@types/react': '*' @@ -9478,7 +9516,7 @@ packages: dependencies: '@babel/runtime': 7.24.5 '@types/react': 18.3.1 - react: 18.2.0 + react: 18.3.1 dev: false /@radix-ui/react-use-controllable-state@1.0.0(react@18.2.0): @@ -9506,7 +9544,7 @@ packages: react: 18.2.0 dev: false - /@radix-ui/react-use-controllable-state@1.0.1(@types/react@18.3.1)(react@18.2.0): + /@radix-ui/react-use-controllable-state@1.0.1(@types/react@18.3.1)(react@18.3.1): resolution: {integrity: sha512-Svl5GY5FQeN758fWKrjM6Qb7asvXeiZltlT4U2gVfl8Gx5UAv2sMR0LWo8yhsIZh2oQ0eFdZ59aoOOMV7b47VA==} peerDependencies: '@types/react': '*' @@ -9516,9 +9554,9 @@ packages: optional: true dependencies: '@babel/runtime': 7.24.5 - '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.3.1)(react@18.2.0) + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.3.1)(react@18.3.1) '@types/react': 18.3.1 - react: 18.2.0 + react: 18.3.1 dev: false /@radix-ui/react-use-escape-keydown@1.0.2(react@18.2.0): @@ -9546,7 +9584,7 @@ packages: react: 18.2.0 dev: false - /@radix-ui/react-use-escape-keydown@1.0.3(@types/react@18.3.1)(react@18.2.0): + /@radix-ui/react-use-escape-keydown@1.0.3(@types/react@18.3.1)(react@18.3.1): resolution: {integrity: sha512-vyL82j40hcFicA+M4Ex7hVkB9vHgSse1ZWomAqV2Je3RleKGO5iM8KMOEtfoSB0PnIelMd2lATjTGMYqN5ylTg==} peerDependencies: '@types/react': '*' @@ -9556,9 +9594,9 @@ packages: optional: true dependencies: '@babel/runtime': 7.24.5 - '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.3.1)(react@18.2.0) + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.3.1)(react@18.3.1) '@types/react': 18.3.1 - react: 18.2.0 + react: 18.3.1 dev: false /@radix-ui/react-use-layout-effect@1.0.0(react@18.2.0): @@ -9584,7 +9622,7 @@ packages: react: 18.2.0 dev: false - /@radix-ui/react-use-layout-effect@1.0.1(@types/react@18.3.1)(react@18.2.0): + /@radix-ui/react-use-layout-effect@1.0.1(@types/react@18.3.1)(react@18.3.1): resolution: {integrity: sha512-v/5RegiJWYdoCvMnITBkNNx6bCj20fiaJnWtRkU18yITptraXjffz5Qbn05uOiQnOvi+dbkznkoaMltz1GnszQ==} peerDependencies: '@types/react': '*' @@ -9595,7 +9633,7 @@ packages: dependencies: '@babel/runtime': 7.24.5 '@types/react': 18.3.1 - react: 18.2.0 + react: 18.3.1 dev: false /@radix-ui/react-use-previous@1.0.0(react@18.2.0): @@ -9631,7 +9669,7 @@ packages: react: 18.2.0 dev: false - /@radix-ui/react-use-rect@1.0.1(@types/react@18.3.1)(react@18.2.0): + /@radix-ui/react-use-rect@1.0.1(@types/react@18.3.1)(react@18.3.1): resolution: {integrity: sha512-Cq5DLuSiuYVKNU8orzJMbl15TXilTnJKUCltMVQg53BQOF1/C5toAaGrowkgksdBQ9H+SRL23g0HDmg9tvmxXw==} peerDependencies: '@types/react': '*' @@ -9643,7 +9681,7 @@ packages: '@babel/runtime': 7.24.5 '@radix-ui/rect': 1.0.1 '@types/react': 18.3.1 - react: 18.2.0 + react: 18.3.1 dev: false /@radix-ui/react-use-size@1.0.0(react@18.2.0): @@ -9671,7 +9709,7 @@ packages: react: 18.2.0 dev: false - /@radix-ui/react-use-size@1.0.1(@types/react@18.3.1)(react@18.2.0): + /@radix-ui/react-use-size@1.0.1(@types/react@18.3.1)(react@18.3.1): resolution: {integrity: sha512-ibay+VqrgcaI6veAojjofPATwledXiSmX+C0KrBk/xgpX9rBzPV3OsfwlhQdUOFbh+LKQorLYT+xTXW9V8yd0g==} peerDependencies: '@types/react': '*' @@ -9681,9 +9719,9 @@ packages: optional: true dependencies: '@babel/runtime': 7.24.5 - '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.3.1)(react@18.2.0) + '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.3.1)(react@18.3.1) '@types/react': 18.3.1 - react: 18.2.0 + react: 18.3.1 dev: false /@radix-ui/react-visually-hidden@1.0.2(react-dom@18.2.0)(react@18.2.0): @@ -9698,7 +9736,7 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false - /@radix-ui/react-visually-hidden@1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0): + /@radix-ui/react-visually-hidden@1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-D4w41yN5YRKtu464TLnByKzMDG/JlMPHtfZgQAu9v6mNakUqGUI9vUrfQKz8NK41VMm/xbZbh76NUTVtIYqOMA==} peerDependencies: '@types/react': '*' @@ -9712,11 +9750,11 @@ packages: optional: true dependencies: '@babel/runtime': 7.24.5 - '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) '@types/react': 18.3.1 '@types/react-dom': 18.2.7 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) dev: false /@radix-ui/rect@1.0.0: @@ -11082,38 +11120,38 @@ packages: react: 19.0.0-rc.0 dev: false - /@react-email/body@0.0.7(react@18.2.0): + /@react-email/body@0.0.7(react@18.3.1): resolution: {integrity: sha512-vjJ5P1MUNWV0KNivaEWA6MGj/I3c764qQJMsKjCHlW6mkFJ4SXbm2OlQFtKAb++Bj8LDqBlnE6oW77bWcMc0NA==} peerDependencies: react: 18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/body@0.0.8(react@18.2.0): + /@react-email/body@0.0.8(react@18.3.1): resolution: {integrity: sha512-gqdkNYlIaIw0OdpWu8KjIcQSIFvx7t2bZpXVxMMvBS859Ia1+1X3b5RNbjI3S1ZqLddUf7owOHkO4MiXGE+nxg==} peerDependencies: react: ^18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/button@0.0.14(react@18.2.0): + /@react-email/button@0.0.14(react@18.3.1): resolution: {integrity: sha512-SMk40moGcAvkHIALX4XercQlK0PNeeEIam6OXHw68ea9WtzzqVwiK4pzLY0iiMI9B4xWHcaS2lCPf3cKbQBf1Q==} engines: {node: '>=18.0.0'} peerDependencies: react: 18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/button@0.0.15(react@18.2.0): + /@react-email/button@0.0.15(react@18.3.1): resolution: {integrity: sha512-9Zi6SO3E8PoHYDfcJTecImiHLyitYWmIRs0HE3Ogra60ZzlWP2EXu+AZqwQnhXuq+9pbgwBWNWxB5YPetNPTNA==} engines: {node: '>=18.0.0'} peerDependencies: react: ^18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false /@react-email/button@0.0.17(react@19.0.0-rc.0): @@ -11125,24 +11163,24 @@ packages: react: 19.0.0-rc.0 dev: false - /@react-email/code-block@0.0.3(react@18.2.0): + /@react-email/code-block@0.0.3(react@18.3.1): resolution: {integrity: sha512-nxhl7WjjM2cOYtl0boBZfSObTrUCz2LbarcMyHkTVAsA9rbjbtWAQF7jmlefXJusk3Uol5l2c8hTh2lHLlHTRQ==} engines: {node: '>=18.0.0'} peerDependencies: react: 18.2.0 dependencies: prismjs: 1.29.0 - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/code-block@0.0.4(react@18.2.0): + /@react-email/code-block@0.0.4(react@18.3.1): resolution: {integrity: sha512-xjVLi/9dFNJ70N7hYme+21eQWa3b9/kgp4V+FKQJkQCuIMobxPRCIGM5jKD/0Vo2OqrE5chYv/dkg/aP8a8sPg==} engines: {node: '>=18.0.0'} peerDependencies: react: ^18.2.0 dependencies: prismjs: 1.29.0 - react: 18.2.0 + react: 18.3.1 dev: false /@react-email/code-block@0.0.8(react@19.0.0-rc.0): @@ -11155,22 +11193,22 @@ packages: react: 19.0.0-rc.0 dev: false - /@react-email/code-inline@0.0.1(react@18.2.0): + /@react-email/code-inline@0.0.1(react@18.3.1): resolution: {integrity: sha512-SeZKTB9Q4+TUafzeUm/8tGK3dFgywUHb1od/BrAiJCo/im65aT+oJfggJLjK2jCdSsus8odcK2kReeM3/FCNTQ==} engines: {node: '>=18.0.0'} peerDependencies: react: 18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/code-inline@0.0.2(react@18.2.0): + /@react-email/code-inline@0.0.2(react@18.3.1): resolution: {integrity: sha512-0cmgbbibFeOJl0q04K9jJlPDuJ+SEiX/OG6m3Ko7UOkG3TqjRD8Dtvkij6jNDVfUh/zESpqJCP2CxrCLLMUjdA==} engines: {node: '>=18.0.0'} peerDependencies: react: ^18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false /@react-email/code-inline@0.0.4(react@19.0.0-rc.0): @@ -11182,13 +11220,13 @@ packages: react: 19.0.0-rc.0 dev: false - /@react-email/column@0.0.10(react@18.2.0): + /@react-email/column@0.0.10(react@18.3.1): resolution: {integrity: sha512-MnP8Mnwipr0X3XtdD6jMLckb0sI5/IlS6Kl/2F6/rsSWBJy5Gg6nizlekTdkwDmy0kNSe3/1nGU0Zqo98pl63Q==} engines: {node: '>=18.0.0'} peerDependencies: react: ^18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false /@react-email/column@0.0.12(react@19.0.0-rc.0): @@ -11200,73 +11238,73 @@ packages: react: 19.0.0-rc.0 dev: false - /@react-email/column@0.0.9(react@18.2.0): + /@react-email/column@0.0.9(react@18.3.1): resolution: {integrity: sha512-1ekqNBgmbS6m97/sUFOnVvQtLYljUWamw8Y44VId95v6SjiJ4ca+hMcdOteHWBH67xkRofEOWTvqDRea5SBV8w==} engines: {node: '>=18.0.0'} peerDependencies: react: 18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/components@0.0.16(@types/react@18.2.69)(react@18.2.0): + /@react-email/components@0.0.16(@types/react@18.2.69)(react@18.3.1): resolution: {integrity: sha512-1WATpMSH03cRvhfNjGl/Up3seZJOzN9KLzlk3Q9g/cqNhZEJ7HYxoZM4AQKAI0V3ttXzzxKv8Oj+AZQLHDiICA==} engines: {node: '>=18.0.0'} peerDependencies: react: 18.2.0 dependencies: - '@react-email/body': 0.0.7(react@18.2.0) - '@react-email/button': 0.0.14(react@18.2.0) - '@react-email/code-block': 0.0.3(react@18.2.0) - '@react-email/code-inline': 0.0.1(react@18.2.0) - '@react-email/column': 0.0.9(react@18.2.0) - '@react-email/container': 0.0.11(react@18.2.0) - '@react-email/font': 0.0.5(react@18.2.0) - '@react-email/head': 0.0.7(react@18.2.0) - '@react-email/heading': 0.0.11(@types/react@18.2.69)(react@18.2.0) - '@react-email/hr': 0.0.7(react@18.2.0) - '@react-email/html': 0.0.7(react@18.2.0) - '@react-email/img': 0.0.7(react@18.2.0) - '@react-email/link': 0.0.7(react@18.2.0) - '@react-email/markdown': 0.0.9(react@18.2.0) - '@react-email/preview': 0.0.8(react@18.2.0) + '@react-email/body': 0.0.7(react@18.3.1) + '@react-email/button': 0.0.14(react@18.3.1) + '@react-email/code-block': 0.0.3(react@18.3.1) + '@react-email/code-inline': 0.0.1(react@18.3.1) + '@react-email/column': 0.0.9(react@18.3.1) + '@react-email/container': 0.0.11(react@18.3.1) + '@react-email/font': 0.0.5(react@18.3.1) + '@react-email/head': 0.0.7(react@18.3.1) + '@react-email/heading': 0.0.11(@types/react@18.2.69)(react@18.3.1) + '@react-email/hr': 0.0.7(react@18.3.1) + '@react-email/html': 0.0.7(react@18.3.1) + '@react-email/img': 0.0.7(react@18.3.1) + '@react-email/link': 0.0.7(react@18.3.1) + '@react-email/markdown': 0.0.9(react@18.3.1) + '@react-email/preview': 0.0.8(react@18.3.1) '@react-email/render': 0.0.12 - '@react-email/row': 0.0.7(react@18.2.0) - '@react-email/section': 0.0.11(react@18.2.0) - '@react-email/tailwind': 0.0.15(react@18.2.0) - '@react-email/text': 0.0.7(react@18.2.0) - react: 18.2.0 + '@react-email/row': 0.0.7(react@18.3.1) + '@react-email/section': 0.0.11(react@18.3.1) + '@react-email/tailwind': 0.0.15(react@18.3.1) + '@react-email/text': 0.0.7(react@18.3.1) + react: 18.3.1 transitivePeerDependencies: - '@types/react' dev: false - /@react-email/components@0.0.17(@types/react@18.3.1)(react@18.2.0): + /@react-email/components@0.0.17(@types/react@18.3.1)(react@18.3.1): resolution: {integrity: sha512-x5gGQaK0QchbwHvUrCBVnE8GCWdO5osTVuTSA54Fwzels6ZDeNTHEYRx9gI3Nwcf/dkoVYkVH4rzWST0SF0MLA==} engines: {node: '>=18.0.0'} peerDependencies: react: ^18.2.0 dependencies: - '@react-email/body': 0.0.8(react@18.2.0) - '@react-email/button': 0.0.15(react@18.2.0) - '@react-email/code-block': 0.0.4(react@18.2.0) - '@react-email/code-inline': 0.0.2(react@18.2.0) - '@react-email/column': 0.0.10(react@18.2.0) - '@react-email/container': 0.0.12(react@18.2.0) - '@react-email/font': 0.0.6(react@18.2.0) - '@react-email/head': 0.0.8(react@18.2.0) - '@react-email/heading': 0.0.12(@types/react@18.3.1)(react@18.2.0) - '@react-email/hr': 0.0.8(react@18.2.0) - '@react-email/html': 0.0.8(react@18.2.0) - '@react-email/img': 0.0.8(react@18.2.0) - '@react-email/link': 0.0.8(react@18.2.0) - '@react-email/markdown': 0.0.10(react@18.2.0) - '@react-email/preview': 0.0.9(react@18.2.0) + '@react-email/body': 0.0.8(react@18.3.1) + '@react-email/button': 0.0.15(react@18.3.1) + '@react-email/code-block': 0.0.4(react@18.3.1) + '@react-email/code-inline': 0.0.2(react@18.3.1) + '@react-email/column': 0.0.10(react@18.3.1) + '@react-email/container': 0.0.12(react@18.3.1) + '@react-email/font': 0.0.6(react@18.3.1) + '@react-email/head': 0.0.8(react@18.3.1) + '@react-email/heading': 0.0.12(@types/react@18.3.1)(react@18.3.1) + '@react-email/hr': 0.0.8(react@18.3.1) + '@react-email/html': 0.0.8(react@18.3.1) + '@react-email/img': 0.0.8(react@18.3.1) + '@react-email/link': 0.0.8(react@18.3.1) + '@react-email/markdown': 0.0.10(react@18.3.1) + '@react-email/preview': 0.0.9(react@18.3.1) '@react-email/render': 0.0.13 - '@react-email/row': 0.0.8(react@18.2.0) - '@react-email/section': 0.0.12(react@18.2.0) - '@react-email/tailwind': 0.0.16(react@18.2.0) - '@react-email/text': 0.0.8(react@18.2.0) - react: 18.2.0 + '@react-email/row': 0.0.8(react@18.3.1) + '@react-email/section': 0.0.12(react@18.3.1) + '@react-email/tailwind': 0.0.16(react@18.3.1) + '@react-email/text': 0.0.8(react@18.3.1) + react: 18.3.1 transitivePeerDependencies: - '@types/react' dev: false @@ -11302,22 +11340,22 @@ packages: - react-dom dev: false - /@react-email/container@0.0.11(react@18.2.0): + /@react-email/container@0.0.11(react@18.3.1): resolution: {integrity: sha512-jzl/EHs0ClXIRFamfH+NR/cqv4GsJJscqRhdYtnWYuRAsWpKBM1muycrrPqIVhWvWi6sFHInWTt07jX+bDc3SQ==} engines: {node: '>=18.0.0'} peerDependencies: react: 18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/container@0.0.12(react@18.2.0): + /@react-email/container@0.0.12(react@18.3.1): resolution: {integrity: sha512-HFu8Pu5COPFfeZxSL+wKv/TV5uO/sp4zQ0XkRCdnGkj/xoq0lqOHVDL4yC2Pu6fxXF/9C3PHDA++5uEYV5WVJw==} engines: {node: '>=18.0.0'} peerDependencies: react: ^18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false /@react-email/container@0.0.14(react@19.0.0-rc.0): @@ -11329,20 +11367,20 @@ packages: react: 19.0.0-rc.0 dev: false - /@react-email/font@0.0.5(react@18.2.0): + /@react-email/font@0.0.5(react@18.3.1): resolution: {integrity: sha512-if/qKYmH3rJ2egQJoKbV8SfKCPavu+ikUq/naT/UkCr8Q0lkk309tRA0x7fXG/WeIrmcipjMzFRGTm2TxTecDw==} peerDependencies: react: 18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/font@0.0.6(react@18.2.0): + /@react-email/font@0.0.6(react@18.3.1): resolution: {integrity: sha512-sZZFvEZ4U3vNCAZ8wXqIO3DuGJR2qE/8m2fEH+tdqwa532zGO3zW+UlCTg0b9455wkJSzEBeaWik0IkNvjXzxw==} peerDependencies: react: ^18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false /@react-email/font@0.0.8(react@19.0.0-rc.0): @@ -11362,44 +11400,44 @@ packages: react: 19.0.0-rc.0 dev: false - /@react-email/head@0.0.7(react@18.2.0): + /@react-email/head@0.0.7(react@18.3.1): resolution: {integrity: sha512-IcXL4jc0H1qzAXJCD9ajcRFBQdbUHkjKJyiUeogpaYSVZSq6cVDWQuGaI23TA9k+pI2TFeQimogUFb3Kgeeudw==} engines: {node: '>=18.0.0'} peerDependencies: react: 18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/head@0.0.8(react@18.2.0): + /@react-email/head@0.0.8(react@18.3.1): resolution: {integrity: sha512-8/NI0gtQmLIilAe6rebK1TWw3IXHxtrR02rInkQq8yQ7zKbYbzx7Q/FhmsJgAk+uYh2Er/KhgYJ0sHZyDhfMTQ==} engines: {node: '>=18.0.0'} peerDependencies: react: ^18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/heading@0.0.11(@types/react@18.2.69)(react@18.2.0): + /@react-email/heading@0.0.11(@types/react@18.2.69)(react@18.3.1): resolution: {integrity: sha512-EF5ZtRCxhHPw3m+8iibKKg0RAvAeHj1AP68sjU7s6+J+kvRgllr/E972Wi5Y8UvcIGossCvpX1WrSMDzeB4puA==} engines: {node: '>=18.0.0'} peerDependencies: react: 18.2.0 dependencies: - '@radix-ui/react-slot': 1.0.2(@types/react@18.2.69)(react@18.2.0) - react: 18.2.0 + '@radix-ui/react-slot': 1.0.2(@types/react@18.2.69)(react@18.3.1) + react: 18.3.1 transitivePeerDependencies: - '@types/react' dev: false - /@react-email/heading@0.0.12(@types/react@18.3.1)(react@18.2.0): + /@react-email/heading@0.0.12(@types/react@18.3.1)(react@18.3.1): resolution: {integrity: sha512-eB7mpnAvDmwvQLoPuwEiPRH4fPXWe6ltz6Ptbry2BlI88F0a2k11Ghb4+sZHBqg7vVw/MKbqEgtLqr3QJ/KfCQ==} engines: {node: '>=18.0.0'} peerDependencies: react: ^18.2.0 dependencies: - '@radix-ui/react-slot': 1.0.2(@types/react@18.3.1)(react@18.2.0) - react: 18.2.0 + '@radix-ui/react-slot': 1.0.2(@types/react@18.3.1)(react@18.3.1) + react: 18.3.1 transitivePeerDependencies: - '@types/react' dev: false @@ -11422,22 +11460,22 @@ packages: react: 19.0.0-rc.0 dev: false - /@react-email/hr@0.0.7(react@18.2.0): + /@react-email/hr@0.0.7(react@18.3.1): resolution: {integrity: sha512-8suK0M/deXHt0DBSeKhSC4bnCBCBm37xk6KJh9M0/FIKlvdltQBem52YUiuqVl1XLB87Y6v6tvspn3SZ9fuxEA==} engines: {node: '>=18.0.0'} peerDependencies: react: 18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/hr@0.0.8(react@18.2.0): + /@react-email/hr@0.0.8(react@18.3.1): resolution: {integrity: sha512-JLVvpCg2wYKEB+n/PGCggWG9fRU5e4lxsGdpK5SDLsCL0ic3OLKSpHMfeE+ZSuw0GixAVVQN7F64PVJHQkd4MQ==} engines: {node: '>=18.0.0'} peerDependencies: react: ^18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false /@react-email/html@0.0.10(react@19.0.0-rc.0): @@ -11449,22 +11487,22 @@ packages: react: 19.0.0-rc.0 dev: false - /@react-email/html@0.0.7(react@18.2.0): + /@react-email/html@0.0.7(react@18.3.1): resolution: {integrity: sha512-oy7OoRtoOKApVI/5Lz1OZptMKmMYJu9Xn6+lOmdBQchAuSdQtWJqxhrSj/iI/mm8HZWo6MZEQ6SFpfOuf8/P6Q==} engines: {node: '>=18.0.0'} peerDependencies: react: 18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/html@0.0.8(react@18.2.0): + /@react-email/html@0.0.8(react@18.3.1): resolution: {integrity: sha512-arII3wBNLpeJtwyIJXPaILm5BPKhA+nvdC1F9QkuKcOBJv2zXctn8XzPqyGqDfdplV692ulNJP7XY55YqbKp6w==} engines: {node: '>=18.0.0'} peerDependencies: react: ^18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false /@react-email/img@0.0.10(react@19.0.0-rc.0): @@ -11476,22 +11514,22 @@ packages: react: 19.0.0-rc.0 dev: false - /@react-email/img@0.0.7(react@18.2.0): + /@react-email/img@0.0.7(react@18.3.1): resolution: {integrity: sha512-up9tM2/dJ24u/CFjcvioKbyGuPw1yeJg605QA7VkrygEhd0CoQEjjgumfugpJ+VJgIt4ZjT9xMVCK5QWTIWoaA==} engines: {node: '>=18.0.0'} peerDependencies: react: 18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/img@0.0.8(react@18.2.0): + /@react-email/img@0.0.8(react@18.3.1): resolution: {integrity: sha512-jx/rPuKo31tV18fu7P5rRqelaH5wkhg83Dq7uLwJpfqhbi4KFBGeBfD0Y3PiLPPoh+WvYf+Adv9W2ghNW8nOMQ==} engines: {node: '>=18.0.0'} peerDependencies: react: ^18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false /@react-email/link@0.0.10(react@19.0.0-rc.0): @@ -11503,32 +11541,32 @@ packages: react: 19.0.0-rc.0 dev: false - /@react-email/link@0.0.7(react@18.2.0): + /@react-email/link@0.0.7(react@18.3.1): resolution: {integrity: sha512-hXPChT3ZMyKnUSA60BLEMD2maEgyB2A37yg5bASbLMrXmsExHi6/IS1h2XiUPLDK4KqH5KFaFxi2cdNo1JOKwA==} engines: {node: '>=18.0.0'} peerDependencies: react: 18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/link@0.0.8(react@18.2.0): + /@react-email/link@0.0.8(react@18.3.1): resolution: {integrity: sha512-nVikuTi8WJHa6Baad4VuRUbUCa/7EtZ1Qy73TRejaCHn+vhetc39XGqHzKLNh+Z/JFL8Hv9g+4AgG16o2R0ogQ==} engines: {node: '>=18.0.0'} peerDependencies: react: ^18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/markdown@0.0.10(react@18.2.0): + /@react-email/markdown@0.0.10(react@18.3.1): resolution: {integrity: sha512-MH0xO+NJ4IuJcx9nyxbgGKAMXyudFjCZ0A2GQvuWajemW9qy2hgnJ3mW3/z5lwcenG+JPn7JyO/iZpizQ7u1tA==} engines: {node: '>=18.0.0'} peerDependencies: react: ^18.2.0 dependencies: - md-to-react-email: 5.0.2(react@18.2.0) - react: 18.2.0 + md-to-react-email: 5.0.2(react@18.3.1) + react: 18.3.1 dev: false /@react-email/markdown@0.0.12(react@19.0.0-rc.0): @@ -11541,14 +11579,14 @@ packages: react: 19.0.0-rc.0 dev: false - /@react-email/markdown@0.0.9(react@18.2.0): + /@react-email/markdown@0.0.9(react@18.3.1): resolution: {integrity: sha512-t//19Zz+W5svKqrSrqoOLpf6dq70jbwYxX8Z+NEMi4LqylklccOaYAyKrkYyulfZwhW7KDH9d2wjVk5jfUABxQ==} engines: {node: '>=18.0.0'} peerDependencies: react: 18.2.0 dependencies: - md-to-react-email: 5.0.2(react@18.2.0) - react: 18.2.0 + md-to-react-email: 5.0.2(react@18.3.1) + react: 18.3.1 dev: false /@react-email/preview@0.0.11(react@19.0.0-rc.0): @@ -11560,22 +11598,22 @@ packages: react: 19.0.0-rc.0 dev: false - /@react-email/preview@0.0.8(react@18.2.0): + /@react-email/preview@0.0.8(react@18.3.1): resolution: {integrity: sha512-Jm0KUYBZQd2w0s2QRMQy0zfHdo3Ns+9bYSE1OybjknlvhANirjuZw9E5KfWgdzO7PyrRtB1OBOQD8//Obc4uIQ==} engines: {node: '>=18.0.0'} peerDependencies: react: 18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/preview@0.0.9(react@18.2.0): + /@react-email/preview@0.0.9(react@18.3.1): resolution: {integrity: sha512-2fyAA/zzZYfYmxfyn3p2YOIU30klyA6Dq4ytyWq4nfzQWWglt5hNDE0cMhObvRtfjM9ghMSVtoELAb0MWiF/kw==} engines: {node: '>=18.0.0'} peerDependencies: react: ^18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false /@react-email/render@0.0.12: @@ -11621,40 +11659,40 @@ packages: react: 19.0.0-rc.0 dev: false - /@react-email/row@0.0.7(react@18.2.0): + /@react-email/row@0.0.7(react@18.3.1): resolution: {integrity: sha512-h7pwrLVGk5CIx7Ai/oPxBgCCAGY7BEpCUQ7FCzi4+eThcs5IdjSwDPefLEkwaFS8KZc56UNwTAH92kNq5B7blg==} engines: {node: '>=18.0.0'} peerDependencies: react: 18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/row@0.0.8(react@18.2.0): + /@react-email/row@0.0.8(react@18.3.1): resolution: {integrity: sha512-JsB6pxs/ZyjYpEML3nbwJRGAerjcN/Pa/QG48XUwnT/MioDWrUuyQuefw+CwCrSUZ2P1IDrv2tUD3/E3xzcoKw==} engines: {node: '>=18.0.0'} peerDependencies: react: ^18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/section@0.0.11(react@18.2.0): + /@react-email/section@0.0.11(react@18.3.1): resolution: {integrity: sha512-3bZ/DuvX1julATI7oqYza6pOtWZgLJDBaa62LFFEvYjisyN+k6lrP2KOucPsDKu2DOkUzlQgK0FOm6VQJX+C0w==} engines: {node: '>=18.0.0'} peerDependencies: react: 18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/section@0.0.12(react@18.2.0): + /@react-email/section@0.0.12(react@18.3.1): resolution: {integrity: sha512-UCD/N/BeOTN4h3VZBUaFdiSem6HnpuxD1Q51TdBFnqeNqS5hBomp8LWJJ9s4gzwHWk1XPdNfLA3I/fJwulJshg==} engines: {node: '>=18.0.0'} peerDependencies: react: ^18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false /@react-email/section@0.0.14(react@19.0.0-rc.0): @@ -11666,22 +11704,22 @@ packages: react: 19.0.0-rc.0 dev: false - /@react-email/tailwind@0.0.15(react@18.2.0): + /@react-email/tailwind@0.0.15(react@18.3.1): resolution: {integrity: sha512-TE3NQ7VKhhvv3Zv0Z1NtoV6AF7aOWiG4juVezMZw1hZCG0mkN6iXC63u23vPQi12y6xCp20ZUHfg67kQeDSP/g==} engines: {node: '>=18.0.0'} peerDependencies: react: 18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/tailwind@0.0.16(react@18.2.0): + /@react-email/tailwind@0.0.16(react@18.3.1): resolution: {integrity: sha512-uMifPxCEHaHLhpS1kVCMGyTeEL+aMYzHT4bgj8CkgCiBoF9wNNfIVMUlHGzHUTv4ZTEPaMfZgC/Hi8RqzL/Ogw==} engines: {node: '>=18.0.0'} peerDependencies: react: ^18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false /@react-email/tailwind@0.1.0(react@19.0.0-rc.0): @@ -11702,22 +11740,22 @@ packages: react: 19.0.0-rc.0 dev: false - /@react-email/text@0.0.7(react@18.2.0): + /@react-email/text@0.0.7(react@18.3.1): resolution: {integrity: sha512-eHCx0mdllGcgK9X7wiLKjNZCBRfxRVNjD3NNYRmOc3Icbl8M9JHriJIfxBuGCmGg2UAORK5P3KmaLQ8b99/pbA==} engines: {node: '>=18.0.0'} peerDependencies: react: 18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false - /@react-email/text@0.0.8(react@18.2.0): + /@react-email/text@0.0.8(react@18.3.1): resolution: {integrity: sha512-uvN2TNWMrfC9wv/LLmMLbbEN1GrMWZb9dBK14eYxHHAEHCeyvGb5ePZZ2MPyzO7Y5yTC+vFEnCEr76V+hWMxCQ==} engines: {node: '>=18.0.0'} peerDependencies: react: ^18.2.0 dependencies: - react: 18.2.0 + react: 18.3.1 dev: false /@react-spring/rafz@9.7.4: @@ -14215,7 +14253,7 @@ packages: dependencies: '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.4.1) '@opentelemetry/instrumentation': 0.52.1(@opentelemetry/api@1.4.1) - '@opentelemetry/semantic-conventions': 1.25.1 + '@opentelemetry/semantic-conventions': 1.27.0 '@traceloop/ai-semantic-conventions': 0.10.0 js-tiktoken: 1.0.14 tslib: 2.6.2 @@ -14703,7 +14741,7 @@ packages: dependencies: '@types/prop-types': 15.7.5 '@types/scheduler': 0.16.2 - csstype: 3.1.1 + csstype: 3.1.3 /@types/react@18.3.1: resolution: {integrity: sha512-V0kuGBX3+prX+DQ/7r2qsv1NsdfnCLnTgnRJ1pYnxykBhGMz+qj+box5lq7XsO5mtZsBqpjwwTu/7wszPfMBcw==} @@ -15097,7 +15135,7 @@ packages: chalk: 4.1.2 css-what: 5.1.0 cssesc: 3.0.0 - csstype: 3.1.1 + csstype: 3.1.3 deep-object-diff: 1.1.9 deepmerge: 4.3.1 media-query-parser: 2.0.2 @@ -15259,7 +15297,7 @@ packages: cli-truncate: 3.1.0 diff: 5.1.0 loupe: 2.3.7 - picocolors: 1.0.0 + picocolors: 1.0.1 pretty-format: 27.5.1 dev: true @@ -16346,7 +16384,7 @@ packages: resolution: {integrity: sha512-fdRxJkQ9MUSEi4jH2DcV3FAPFktk0wefilxrwNyUuWpoWawQGN7G7cB+fOYTtFfI6XNkFgwqJ/D3G18BoJJ/jg==} engines: {node: '>= 10.0.0'} dependencies: - '@babel/types': 7.24.7 + '@babel/types': 7.25.6 dev: false /bail@2.0.2: @@ -17560,7 +17598,6 @@ packages: /csstype@3.1.3: resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==} - dev: false /csv-generate@3.4.3: resolution: {integrity: sha512-w/T+rqR0vwvHqWs/1ZyMDWtHHSJaN06klRqJXBEpDJaM/+dZkso0OKh1VcuuYvK3XM53KysVNq8Ko/epCK8wOw==} @@ -18068,7 +18105,7 @@ packages: resolution: {integrity: sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==} dependencies: '@babel/runtime': 7.24.5 - csstype: 3.1.1 + csstype: 3.1.3 dev: false /dom-serializer@2.0.0: @@ -19975,7 +20012,7 @@ packages: '@emotion/is-prop-valid': 0.8.8 dev: false - /framer-motion@10.17.4(react-dom@18.2.0)(react@18.2.0): + /framer-motion@10.17.4(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-CYBSs6cWfzcasAX8aofgKFZootmkQtR4qxbfTOksBLny/lbUfkGbQAFOS3qnl6Uau1N9y8tUpI7mVIrHgkFjLQ==} peerDependencies: react: ^18.0.0 @@ -19986,8 +20023,8 @@ packages: react-dom: optional: true dependencies: - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) tslib: 2.6.2 optionalDependencies: '@emotion/is-prop-valid': 0.8.8 @@ -22009,13 +22046,13 @@ packages: remove-accents: 0.5.0 dev: false - /md-to-react-email@5.0.2(react@18.2.0): + /md-to-react-email@5.0.2(react@18.3.1): resolution: {integrity: sha512-x6kkpdzIzUhecda/yahltfEl53mH26QdWu4abUF9+S0Jgam8P//Ciro8cdhyMHnT5MQUJYrIbO6ORM2UxPiNNA==} peerDependencies: react: 18.x dependencies: marked: 7.0.4 - react: 18.2.0 + react: 18.3.1 dev: false /md-to-react-email@5.0.2(react@19.0.0-rc.0): @@ -22862,6 +22899,7 @@ packages: resolution: {integrity: sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + dev: false /nanoid@3.3.7: resolution: {integrity: sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==} @@ -22902,7 +22940,7 @@ packages: engines: {node: '>=10'} dev: true - /next@14.1.0(react-dom@18.2.0)(react@18.2.0): + /next@14.1.0(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-wlzrsbfeSU48YQBjZhDzOwhWhGsy+uQycR8bHAOt1LY1bn3zZEcDyHQOEoN3aWzQ8LHCAJ1nqrWCc9XF2+O45Q==} engines: {node: '>=18.17.0'} hasBin: true @@ -22923,9 +22961,9 @@ packages: caniuse-lite: 1.0.30001655 graceful-fs: 4.2.11 postcss: 8.4.31 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - styled-jsx: 5.1.1(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) + styled-jsx: 5.1.1(react@18.3.1) optionalDependencies: '@next/swc-darwin-arm64': 14.1.0 '@next/swc-darwin-x64': 14.1.0 @@ -24461,7 +24499,7 @@ packages: resolution: {integrity: sha512-cbI+jaqIeu/VGqXEarWkRCCffhjgXc0qjBtXpqJhTBohMUjUQnbBr0xqX3vEKudc4iviTewcJo5ajcec5+wdJw==} engines: {node: ^10 || ^12 || >=14} dependencies: - nanoid: 3.3.6 + nanoid: 3.3.7 picocolors: 1.0.0 source-map-js: 1.0.2 dev: true @@ -24479,8 +24517,8 @@ packages: engines: {node: ^10 || ^12 || >=14} dependencies: nanoid: 3.3.7 - picocolors: 1.0.0 - source-map-js: 1.0.2 + picocolors: 1.0.1 + source-map-js: 1.2.0 /postcss@8.4.44: resolution: {integrity: sha512-Aweb9unOEpQ3ezu4Q00DPvvM2ZTUitJdNKeP/+uQgr1IBIqu574IaZoURId7BKtWMREwzKa9OgzPzezWGPWFQw==} @@ -24672,14 +24710,14 @@ packages: /printable-characters@1.0.42: resolution: {integrity: sha512-dKp+C4iXWK4vVYZmYSd0KBH5F/h1HoZRsbJ82AVKRO3PEo8L4lBS/vLwhVtpwwuYcoIsVY+1JYKR268yn480uQ==} - /prism-react-renderer@2.1.0(react@18.2.0): + /prism-react-renderer@2.1.0(react@18.3.1): resolution: {integrity: sha512-I5cvXHjA1PVGbGm1MsWCpvBCRrYyxEri0MC7/JbfIfYfcXAxHyO5PaUjs3A8H5GW6kJcLhTHxxMaOZZpRZD2iQ==} peerDependencies: react: '>=16.0.0' dependencies: '@types/prismjs': 1.26.0 clsx: 1.2.1 - react: 18.2.0 + react: 18.3.1 dev: false /prism-react-renderer@2.3.1(react@18.2.0): @@ -24814,24 +24852,6 @@ packages: resolution: {integrity: sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==} dev: false - /protobufjs@7.2.6: - resolution: {integrity: sha512-dgJaEDDL6x8ASUZ1YqWciTRrdOuYNzoOf27oHNfdyvKqHr5i0FV7FSLU+aIeFjyFgVxrpTOtQUi0BLLBymZaBw==} - engines: {node: '>=12.0.0'} - requiresBuild: true - dependencies: - '@protobufjs/aspromise': 1.1.2 - '@protobufjs/base64': 1.1.2 - '@protobufjs/codegen': 2.0.4 - '@protobufjs/eventemitter': 1.1.0 - '@protobufjs/fetch': 1.1.0 - '@protobufjs/float': 1.0.2 - '@protobufjs/inquire': 1.1.0 - '@protobufjs/path': 1.1.2 - '@protobufjs/pool': 1.1.0 - '@protobufjs/utf8': 1.1.0 - '@types/node': 18.19.20 - long: 5.2.3 - /protobufjs@7.3.2: resolution: {integrity: sha512-RXyHaACeqXeqAKGLDl68rQKbmObRsTIn4TYVUUug1KfS47YWCo5MacGITEryugIgZqORCvJWEk4l449POg5Txg==} engines: {node: '>=12.0.0'} @@ -25190,12 +25210,12 @@ packages: dependencies: '@babel/parser': 7.24.1 '@radix-ui/colors': 1.0.1 - '@radix-ui/react-collapsible': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-popover': 1.0.7(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-slot': 1.0.2(@types/react@18.3.1)(react@18.2.0) - '@radix-ui/react-toggle-group': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@radix-ui/react-tooltip': 1.0.6(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.2.0) - '@react-email/components': 0.0.17(@types/react@18.3.1)(react@18.2.0) + '@radix-ui/react-collapsible': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-popover': 1.0.7(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-slot': 1.0.2(@types/react@18.3.1)(react@18.3.1) + '@radix-ui/react-toggle-group': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@radix-ui/react-tooltip': 1.0.6(@types/react-dom@18.2.7)(@types/react@18.3.1)(react-dom@18.2.0)(react@18.3.1) + '@react-email/components': 0.0.17(@types/react@18.3.1)(react@18.3.1) '@react-email/render': 0.0.13 '@swc/core': 1.3.101 '@types/react': 18.3.1 @@ -25211,21 +25231,21 @@ packages: esbuild: 0.19.11 eslint-config-prettier: 9.0.0(eslint@8.45.0) eslint-config-turbo: 1.10.12(eslint@8.45.0) - framer-motion: 10.17.4(react-dom@18.2.0)(react@18.2.0) + framer-motion: 10.17.4(react-dom@18.2.0)(react@18.3.1) glob: 10.3.4 log-symbols: 4.1.0 mime-types: 2.1.35 - next: 14.1.0(react-dom@18.2.0)(react@18.2.0) + next: 14.1.0(react-dom@18.2.0)(react@18.3.1) normalize-path: 3.0.0 ora: 5.4.1 postcss: 8.4.35 - prism-react-renderer: 2.1.0(react@18.2.0) - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + prism-react-renderer: 2.1.0(react@18.3.1) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) shelljs: 0.8.5 socket.io: 4.7.3 socket.io-client: 4.7.3 - sonner: 1.3.1(react-dom@18.2.0)(react@18.2.0) + sonner: 1.3.1(react-dom@18.2.0)(react@18.3.1) source-map-js: 1.0.2 stacktrace-parser: 0.1.10 tailwind-merge: 2.2.0 @@ -25343,7 +25363,7 @@ packages: tslib: 2.6.2 dev: false - /react-remove-scroll-bar@2.3.4(@types/react@18.3.1)(react@18.2.0): + /react-remove-scroll-bar@2.3.4(@types/react@18.3.1)(react@18.3.1): resolution: {integrity: sha512-63C4YQBUt0m6ALadE9XV56hV8BgJWDmmTPY758iIJjfQKt2nYwoUrPk0LXRXcB/yIj82T1/Ixfdpdk68LwIB0A==} engines: {node: '>=10'} peerDependencies: @@ -25354,8 +25374,8 @@ packages: optional: true dependencies: '@types/react': 18.3.1 - react: 18.2.0 - react-style-singleton: 2.2.1(@types/react@18.3.1)(react@18.2.0) + react: 18.3.1 + react-style-singleton: 2.2.1(@types/react@18.3.1)(react@18.3.1) tslib: 2.6.2 dev: false @@ -25378,7 +25398,7 @@ packages: use-sidecar: 1.1.2(@types/react@18.2.69)(react@18.2.0) dev: false - /react-remove-scroll@2.5.5(@types/react@18.3.1)(react@18.2.0): + /react-remove-scroll@2.5.5(@types/react@18.3.1)(react@18.3.1): resolution: {integrity: sha512-ImKhrzJJsyXJfBZ4bzu8Bwpka14c/fQt0k+cyFp/PBhTfyDnU5hjOtM4AG/0AMyy8oKzOTR0lDgJIM7pYXI0kw==} engines: {node: '>=10'} peerDependencies: @@ -25389,12 +25409,12 @@ packages: optional: true dependencies: '@types/react': 18.3.1 - react: 18.2.0 - react-remove-scroll-bar: 2.3.4(@types/react@18.3.1)(react@18.2.0) - react-style-singleton: 2.2.1(@types/react@18.3.1)(react@18.2.0) + react: 18.3.1 + react-remove-scroll-bar: 2.3.4(@types/react@18.3.1)(react@18.3.1) + react-style-singleton: 2.2.1(@types/react@18.3.1)(react@18.3.1) tslib: 2.6.2 - use-callback-ref: 1.3.0(@types/react@18.3.1)(react@18.2.0) - use-sidecar: 1.1.2(@types/react@18.3.1)(react@18.2.0) + use-callback-ref: 1.3.0(@types/react@18.3.1)(react@18.3.1) + use-sidecar: 1.1.2(@types/react@18.3.1)(react@18.3.1) dev: false /react-resizable-panels@2.0.9(react-dom@18.2.0)(react@18.2.0): @@ -25489,7 +25509,7 @@ packages: tslib: 2.6.2 dev: false - /react-style-singleton@2.2.1(@types/react@18.3.1)(react@18.2.0): + /react-style-singleton@2.2.1(@types/react@18.3.1)(react@18.3.1): resolution: {integrity: sha512-ZWj0fHEMyWkHzKYUr2Bs/4zU6XLmq9HsgBURm7g5pAVfyn49DgUiNgY2d4lXRlYSiCif9YBGpQleewkcqddc7g==} engines: {node: '>=10'} peerDependencies: @@ -25502,7 +25522,7 @@ packages: '@types/react': 18.3.1 get-nonce: 1.0.1 invariant: 2.2.4 - react: 18.2.0 + react: 18.3.1 tslib: 2.6.2 dev: false @@ -26783,14 +26803,14 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false - /sonner@1.3.1(react-dom@18.2.0)(react@18.2.0): + /sonner@1.3.1(react-dom@18.2.0)(react@18.3.1): resolution: {integrity: sha512-+rOAO56b2eI3q5BtgljERSn2umRk63KFIvgb2ohbZ5X+Eb5u+a/7/0ZgswYqgBMg8dyl7n6OXd9KasA8QF9ToA==} peerDependencies: react: ^18.0.0 react-dom: ^18.0.0 dependencies: - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) + react: 18.3.1 + react-dom: 18.2.0(react@18.3.1) dev: false /source-map-js@1.0.2: @@ -27225,7 +27245,7 @@ packages: react: 19.0.0-rc.0 dev: false - /styled-jsx@5.1.1(react@18.2.0): + /styled-jsx@5.1.1(react@18.3.1): resolution: {integrity: sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw==} engines: {node: '>= 12.0.0'} peerDependencies: @@ -27239,7 +27259,7 @@ packages: optional: true dependencies: client-only: 0.0.1 - react: 18.2.0 + react: 18.3.1 dev: false /stylis@4.3.0: @@ -27991,7 +28011,7 @@ packages: resolution: {integrity: sha512-TYyJ7+H+7Jsqawdv+mfsEpZPTIj9siDHS6EMCzG/z3b/PZiphsX+mWtqFfFVe5/N0Th6V3elK9lQqjnrgTOfrg==} dependencies: long: 5.2.3 - protobufjs: 7.2.6 + protobufjs: 7.3.2 dev: true /ts-proto@1.167.3: @@ -27999,7 +28019,7 @@ packages: hasBin: true dependencies: case-anything: 2.1.13 - protobufjs: 7.2.6 + protobufjs: 7.3.2 ts-poet: 6.6.0 ts-proto-descriptors: 1.15.0 dev: true @@ -28714,7 +28734,7 @@ packages: tslib: 2.6.2 dev: false - /use-callback-ref@1.3.0(@types/react@18.3.1)(react@18.2.0): + /use-callback-ref@1.3.0(@types/react@18.3.1)(react@18.3.1): resolution: {integrity: sha512-3FT9PRuRdbB9HfXhEq35u4oZkvpJ5kuYbpqhCfmiZyReuRgpnhDlbr2ZEnnuS0RrJAPn6l23xjFg9kpDM+Ms7w==} engines: {node: '>=10'} peerDependencies: @@ -28725,7 +28745,7 @@ packages: optional: true dependencies: '@types/react': 18.3.1 - react: 18.2.0 + react: 18.3.1 tslib: 2.6.2 dev: false @@ -28758,7 +28778,7 @@ packages: tslib: 2.6.2 dev: false - /use-sidecar@1.1.2(@types/react@18.3.1)(react@18.2.0): + /use-sidecar@1.1.2(@types/react@18.3.1)(react@18.3.1): resolution: {integrity: sha512-epTbsLuzZ7lPClpz2TyryBfztm7m+28DlEv2ZCQ3MDr5ssiwyOwGH/e5F9CkfWjJ1t4clvI58yF822/GUkjjhw==} engines: {node: '>=10'} peerDependencies: @@ -28770,7 +28790,7 @@ packages: dependencies: '@types/react': 18.3.1 detect-node-es: 1.1.0 - react: 18.2.0 + react: 18.3.1 tslib: 2.6.2 dev: false @@ -28920,10 +28940,10 @@ packages: hasBin: true dependencies: cac: 6.7.14 - debug: 4.3.6 + debug: 4.3.7 mlly: 1.7.1 pathe: 1.1.2 - picocolors: 1.0.0 + picocolors: 1.0.1 source-map: 0.6.1 source-map-support: 0.5.21 vite: 4.4.9(@types/node@18.11.18) @@ -28944,10 +28964,10 @@ packages: hasBin: true dependencies: cac: 6.7.14 - debug: 4.3.6 + debug: 4.3.7 mlly: 1.7.1 pathe: 1.1.2 - picocolors: 1.0.0 + picocolors: 1.0.1 source-map: 0.6.1 source-map-support: 0.5.21 vite: 4.4.9(@types/node@18.19.20) @@ -29170,7 +29190,7 @@ packages: dependencies: '@types/node': 18.19.20 esbuild: 0.18.11 - postcss: 8.4.29 + postcss: 8.4.44 rollup: 3.29.1 optionalDependencies: fsevents: 2.3.3 From c196ca6824f4c4cb845fea317dfb7667770f9d8d Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 29 Sep 2024 18:41:33 -0700 Subject: [PATCH 057/114] Create a delay waitpoint --- .../run-engine/src/engine/index.ts | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 2dc17fd322..17f5087609 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -4,8 +4,13 @@ import { Redis, type RedisOptions } from "ioredis"; import Redlock from "redlock"; import { RunQueue } from "../run-queue"; import { MinimalAuthenticatedEnvironment } from "../shared"; -import { blockRunWithWaitpoint, createRunAssociatedWaitpoint } from "./waitpoint"; +import { + blockRunWithWaitpoint, + createDateTimeWaitpoint, + createRunAssociatedWaitpoint, +} from "./waitpoint"; import { generateFriendlyId } from "@trigger.dev/core/v3/apps"; +import { run } from "node:test"; type Options = { redis: RedisOptions; @@ -234,8 +239,14 @@ export class RunEngine { } if (taskRun.delayUntil) { - //todo create a WaitPoint - const delayWaitpoint = await workerQueue.enqueue( + const delayWaitpoint = await createDateTimeWaitpoint(prisma, { + projectId: environment.project.id, + completedAfter: taskRun.delayUntil, + }); + + //waitpoint + + await workerQueue.enqueue( "v3.enqueueDelayedRun", { runId: taskRun.id }, { tx, runAt: delayUntil, jobKey: `v3.enqueueDelayedRun.${taskRun.id}` } @@ -254,7 +265,20 @@ export class RunEngine { } } - await this.runQueue.enqueueMessage({ env: environment, message: {} }); + await this.runQueue.enqueueMessage({ + env: environment, + message: { + runId: taskRun.id, + taskIdentifier: taskRun.taskIdentifier, + orgId: environment.organization.id, + projectId: environment.project.id, + environmentId: environment.id, + environmentType: environment.type, + queue: taskRun.queue, + concurrencyKey: taskRun.concurrencyKey ?? undefined, + timestamp: Date.now(), + }, + }); }); return taskRun; From cf90b1261392c8090dda5fbf40ef8045bb94cf0e Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 30 Sep 2024 10:44:51 -0700 Subject: [PATCH 058/114] Moved ZodWorker to an internal package so it can be used in the run engine as well as the webapp --- internal-packages/zod-worker/package.json | 25 +++++++ .../zod-worker/src/index.ts | 65 +++++++++++-------- .../zod-worker/src}/pgListen.server.ts | 18 +++-- internal-packages/zod-worker/src/types.ts | 11 ++++ internal-packages/zod-worker/tsconfig.json | 25 +++++++ 5 files changed, 113 insertions(+), 31 deletions(-) create mode 100644 internal-packages/zod-worker/package.json rename apps/webapp/app/platform/zodWorker.server.ts => internal-packages/zod-worker/src/index.ts (92%) rename {apps/webapp/app/services/db => internal-packages/zod-worker/src}/pgListen.server.ts (86%) create mode 100644 internal-packages/zod-worker/src/types.ts create mode 100644 internal-packages/zod-worker/tsconfig.json diff --git a/internal-packages/zod-worker/package.json b/internal-packages/zod-worker/package.json new file mode 100644 index 0000000000..3c6d0c1f9b --- /dev/null +++ b/internal-packages/zod-worker/package.json @@ -0,0 +1,25 @@ +{ + "name": "@internal/zod-worker", + "private": true, + "version": "0.0.1", + "main": "./src/index.ts", + "types": "./src/index.ts", + "dependencies": { + "@opentelemetry/api": "^1.9.0", + "@trigger.dev/core": "workspace:*", + "@trigger.dev/database": "workspace:*", + "graphile-worker": "0.16.6", + "lodash.omit": "^4.5.0", + "typescript": "^4.8.4", + "zod": "3.22.3" + }, + "devDependencies": { + "@types/lodash.omit": "^4.5.7", + "@types/pg": "8.6.6", + "vitest": "^1.4.0" + }, + "scripts": { + "typecheck": "tsc --noEmit", + "test": "vitest" + } +} diff --git a/apps/webapp/app/platform/zodWorker.server.ts b/internal-packages/zod-worker/src/index.ts similarity index 92% rename from apps/webapp/app/platform/zodWorker.server.ts rename to internal-packages/zod-worker/src/index.ts index b231a4acad..ae1bc07229 100644 --- a/apps/webapp/app/platform/zodWorker.server.ts +++ b/internal-packages/zod-worker/src/index.ts @@ -1,4 +1,5 @@ import { SpanKind, SpanStatusCode, trace } from "@opentelemetry/api"; +import { flattenAttributes } from "@trigger.dev/core/v3"; import type { CronItem, CronItemOptions, @@ -17,14 +18,15 @@ import { makeWorkerUtils, parseCronItems, } from "graphile-worker"; - -import { flattenAttributes } from "@trigger.dev/core/v3"; import omit from "lodash.omit"; import { z } from "zod"; -import { $replica, PrismaClient, PrismaClientOrTransaction } from "~/db.server"; -import { env } from "~/env.server"; -import { PgListenService } from "~/services/db/pgListen.server"; -import { workerLogger as logger } from "~/services/logger.server"; +import { Logger } from "@trigger.dev/core/logger"; +import { + PrismaClient, + PrismaClientOrTransaction, + PrismaReplicaClient, +} from "@trigger.dev/database"; +import { PgListenService } from "./pgListen.server"; const tracer = trace.getTracer("zodWorker", "3.0.0.dp.1"); @@ -123,6 +125,7 @@ export type ZodWorkerOptions = { name: string; runnerOptions: RunnerOptions; prisma: PrismaClient; + replica: PrismaReplicaClient; schema: TMessageCatalog; tasks: ZodTasks; recurringTasks?: ZodRecurringTasks; @@ -130,12 +133,15 @@ export type ZodWorkerOptions = { reporter?: ZodWorkerReporter; shutdownTimeoutInMs?: number; rateLimiter?: ZodWorkerRateLimiter; + verboseLogging?: boolean; + logger: Logger; }; export class ZodWorker { #name: string; #schema: TMessageCatalog; #prisma: PrismaClient; + #replica: PrismaReplicaClient; #runnerOptions: RunnerOptions; #tasks: ZodTasks; #recurringTasks?: ZodRecurringTasks; @@ -146,11 +152,14 @@ export class ZodWorker { #shutdownTimeoutInMs?: number; #shuttingDown = false; #workerUtils?: WorkerUtils; + #verboseLogging?: boolean; + #logger: Logger; constructor(options: ZodWorkerOptions) { this.#name = options.name; this.#schema = options.schema; this.#prisma = options.prisma; + this.#replica = options.replica; this.#runnerOptions = options.runnerOptions; this.#tasks = options.tasks; this.#recurringTasks = options.recurringTasks; @@ -158,6 +167,8 @@ export class ZodWorker { this.#reporter = options.reporter; this.#rateLimiter = options.rateLimiter; this.#shutdownTimeoutInMs = options.shutdownTimeoutInMs ?? 60000; // default to 60 seconds + this.#verboseLogging = options.verboseLogging; + this.#logger = options.logger; } get graphileWorkerSchema() { @@ -179,9 +190,9 @@ export class ZodWorker { const graphileLogger = new GraphileLogger((scope) => { return (level, message, meta) => { - if (env.VERBOSE_GRAPHILE_LOGGING !== "true") return; + if (this.#verboseLogging !== true) return; - logger.debug(`[graphile-worker][${this.#name}][${level}] ${message}`, { + this.#logger.debug(`[graphile-worker][${this.#name}][${level}] ${message}`, { scope, meta, workerName: this.#name, @@ -214,7 +225,7 @@ export class ZodWorker { this.#logDebug("pool:listen:success"); // hijack client instance to listen and react to incoming NOTIFY events - const pgListen = new PgListenService(client, this.#name, logger); + const pgListen = new PgListenService(client, this.#logger, this.#name); await pgListen.on("trigger:graphile:migrate", async ({ latestMigration }) => { this.#logDebug("Detected incoming migration", { latestMigration }); @@ -270,12 +281,12 @@ export class ZodWorker { }); this.#runner?.events.on("worker:getJob:start", ({ worker }) => { - if (env.VERBOSE_GRAPHILE_LOGGING !== "true") return; + if (this.#verboseLogging !== true) return; this.#logDebug("worker:getJob:start", { workerId: worker.workerId }); }); this.#runner?.events.on("job:start", ({ worker, job }) => { - if (env.VERBOSE_GRAPHILE_LOGGING !== "true") return; + if (this.#verboseLogging !== true) return; this.#logDebug("job:start", { workerId: worker.workerId, job }); }); @@ -379,7 +390,7 @@ export class ZodWorker { } ); - logger.debug("Enqueued worker task", { + this.#logger.debug("Enqueued worker task", { identifier, payload, spec, @@ -396,7 +407,7 @@ export class ZodWorker { ): Promise { const results = await this.#removeJob(jobKey, option?.tx ?? this.#prisma); - logger.debug("dequeued worker task", { results, jobKey }); + this.#logger.debug("dequeued worker task", { results, jobKey }); return results; } @@ -435,7 +446,7 @@ export class ZodWorker { const rows = AddJobResultsSchema.safeParse(results); if (!rows.success) { - logger.debug("results returned from add_job could not be parsed", { + this.#logger.debug("results returned from add_job could not be parsed", { identifier, payload, spec, @@ -460,7 +471,7 @@ export class ZodWorker { const job = AddJobResultsSchema.safeParse(result); if (!job.success) { - logger.debug("could not remove job, job_key did not exist", { + this.#logger.debug("could not remove job, job_key did not exist", { jobKey, }); @@ -582,7 +593,7 @@ export class ZodWorker { const payload = messageSchema.parse(rawPayload); const job = helpers.job; - logger.debug("Received worker task, calling handler", { + this.#logger.debug("Received worker task, calling handler", { type: String(typeName), payload, job, @@ -628,7 +639,7 @@ export class ZodWorker { } if (job.attempts >= job.max_attempts) { - logger.debug("Job failed after max attempts", { + this.#logger.debug("Job failed after max attempts", { job, attempts: job.attempts, max_attempts: job.max_attempts, @@ -653,7 +664,7 @@ export class ZodWorker { ): Promise { const job = helpers.job; - logger.debug("Received recurring task, calling handler", { + this.#logger.debug("Received recurring task, calling handler", { type: String(typeName), payload: rawPayload, job, @@ -727,7 +738,7 @@ export class ZodWorker { const job = helpers.job; - logger.debug("Received cleanup task", { + this.#logger.debug("Received cleanup task", { payload: rawPayload, job, }); @@ -745,12 +756,12 @@ export class ZodWorker { // Add the this.#cleanup.ttl to the payload._cron.ts const expirationDate = new Date(payload._cron.ts.getTime() - this.#cleanup.ttl); - logger.debug("Cleaning up old jobs", { + this.#logger.debug("Cleaning up old jobs", { expirationDate, payload, }); - const rawResults = await $replica.$queryRawUnsafe( + const rawResults = await this.#replica.$queryRawUnsafe( `SELECT id FROM ${this.graphileWorkerSchema}.jobs WHERE run_at < $1 @@ -771,7 +782,7 @@ export class ZodWorker { const completedJobs = await this.#workerUtils.completeJobs(results.map((job) => job.id)); - logger.debug("Cleaned up old jobs", { + this.#logger.debug("Cleaned up old jobs", { found: results.length, deleted: completedJobs.length, expirationDate, @@ -793,7 +804,7 @@ export class ZodWorker { return; } - logger.debug("Received reporter task", { + this.#logger.debug("Received reporter task", { payload: rawPayload, }); @@ -813,7 +824,7 @@ export class ZodWorker { const schema = z.array(z.object({ count: z.coerce.number() })); // Count the number of jobs that have been added since the startAt date and before the payload._cron.ts date - const rawAddedResults = await $replica.$queryRawUnsafe( + const rawAddedResults = await this.#replica.$queryRawUnsafe( `SELECT COUNT(*) FROM ${this.graphileWorkerSchema}.jobs WHERE created_at > $1 AND created_at < $2`, startAt, payload._cron.ts @@ -822,13 +833,13 @@ export class ZodWorker { const addedCountResults = schema.parse(rawAddedResults)[0]; // Count the total number of jobs in the jobs table - const rawTotalResults = await $replica.$queryRawUnsafe( + const rawTotalResults = await this.#replica.$queryRawUnsafe( `SELECT COUNT(*) FROM ${this.graphileWorkerSchema}.jobs` ); const totalCountResults = schema.parse(rawTotalResults)[0]; - logger.debug("Calculated metrics about the jobs table", { + this.#logger.debug("Calculated metrics about the jobs table", { rawAddedResults, rawTotalResults, payload, @@ -842,7 +853,7 @@ export class ZodWorker { } #logDebug(message: string, args?: any) { - logger.debug(`[worker][${this.#name}] ${message}`, args); + this.#logger.debug(`[worker][${this.#name}] ${message}`, args); } } diff --git a/apps/webapp/app/services/db/pgListen.server.ts b/internal-packages/zod-worker/src/pgListen.server.ts similarity index 86% rename from apps/webapp/app/services/db/pgListen.server.ts rename to internal-packages/zod-worker/src/pgListen.server.ts index b819ceec6d..5dd61551cf 100644 --- a/apps/webapp/app/services/db/pgListen.server.ts +++ b/internal-packages/zod-worker/src/pgListen.server.ts @@ -1,8 +1,6 @@ import { Logger } from "@trigger.dev/core/logger"; import type { PoolClient } from "pg"; import { z } from "zod"; -import { logger } from "~/services/logger.server"; -import { safeJsonParse } from "~/utils/json"; import { NotificationCatalog, NotificationChannel, notificationCatalog } from "./types"; export class PgListenService { @@ -10,9 +8,9 @@ export class PgListenService { #logger: Logger; #loggerNamespace: string; - constructor(poolClient: PoolClient, loggerNamespace?: string, loggerInstance?: Logger) { + constructor(poolClient: PoolClient, logger: Logger, loggerNamespace?: string) { this.#poolClient = poolClient; - this.#logger = loggerInstance ?? logger; + this.#logger = logger; this.#loggerNamespace = loggerNamespace ?? ""; } @@ -64,3 +62,15 @@ export class PgListenService { this.#logger.debug(`[pgListen]${namespace} ${message}`, args); } } + +export function safeJsonParse(json?: string): unknown { + if (!json) { + return; + } + + try { + return JSON.parse(json); + } catch (e) { + return null; + } +} diff --git a/internal-packages/zod-worker/src/types.ts b/internal-packages/zod-worker/src/types.ts new file mode 100644 index 0000000000..2e5b6d222f --- /dev/null +++ b/internal-packages/zod-worker/src/types.ts @@ -0,0 +1,11 @@ +import { z } from "zod"; + +export const notificationCatalog = { + "trigger:graphile:migrate": z.object({ + latestMigration: z.number(), + }), +}; + +export type NotificationCatalog = typeof notificationCatalog; + +export type NotificationChannel = keyof NotificationCatalog; diff --git a/internal-packages/zod-worker/tsconfig.json b/internal-packages/zod-worker/tsconfig.json new file mode 100644 index 0000000000..bfd15dde0d --- /dev/null +++ b/internal-packages/zod-worker/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "target": "ES2019", + "lib": ["ES2019", "DOM", "DOM.Iterable"], + "module": "CommonJS", + "moduleResolution": "Node10", + "moduleDetection": "force", + "verbatimModuleSyntax": false, + "types": ["vitest/globals"], + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "preserveWatchOutput": true, + "skipLibCheck": true, + "noEmit": true, + "strict": true, + "paths": { + "@trigger.dev/core": ["../../packages/core/src/index"], + "@trigger.dev/core/*": ["../../packages/core/src/*"], + "@trigger.dev/database": ["../database/src/index"], + "@trigger.dev/database/*": ["../database/src/*"] + } + }, + "exclude": ["node_modules"] +} From 99c3a96a0224348c46c7bd784e8c3d9725d40d2b Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 30 Sep 2024 10:45:10 -0700 Subject: [PATCH 059/114] Web app now uses the zod worker package --- apps/webapp/app/services/worker.server.ts | 13 ++++++++++--- apps/webapp/package.json | 1 + apps/webapp/tsconfig.json | 4 +++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/apps/webapp/app/services/worker.server.ts b/apps/webapp/app/services/worker.server.ts index 299b86c775..41aa63fcb6 100644 --- a/apps/webapp/app/services/worker.server.ts +++ b/apps/webapp/app/services/worker.server.ts @@ -1,9 +1,9 @@ -import { DeliverEmailSchema } from "@/../../packages/emails/src"; +import { DeliverEmailSchema } from "emails"; import { ScheduledPayloadSchema, addMissingVersionField } from "@trigger.dev/core"; +import { ZodWorker } from "@internal/zod-worker"; import { z } from "zod"; -import { prisma } from "~/db.server"; +import { $replica, prisma } from "~/db.server"; import { env } from "~/env.server"; -import { ZodWorker } from "~/platform/zodWorker.server"; import { MarqsConcurrencyMonitor } from "~/v3/marqs/concurrencyMonitor.server"; import { RequeueV2Message } from "~/v3/marqs/requeueV2Message.server"; import { RequeueTaskRunService } from "~/v3/requeueTaskRun.server"; @@ -54,6 +54,7 @@ import { CancelDevSessionRunsService, CancelDevSessionRunsServiceOptions, } from "~/v3/services/cancelDevSessionRuns.server"; +import { logger } from "./logger.server"; const workerCatalog = { indexEndpoint: z.object({ @@ -279,6 +280,7 @@ function getWorkerQueue() { return new ZodWorker({ name: "workerQueue", prisma, + replica: $replica, runnerOptions: { connectionString: env.DATABASE_URL, concurrency: env.WORKER_CONCURRENCY, @@ -287,6 +289,7 @@ function getWorkerQueue() { schema: env.WORKER_SCHEMA, maxPoolSize: env.WORKER_CONCURRENCY + 1, }, + logger: logger, shutdownTimeoutInMs: env.GRACEFUL_SHUTDOWN_TIMEOUT, schema: workerCatalog, recurringTasks: { @@ -732,6 +735,8 @@ function getExecutionWorkerQueue() { return new ZodWorker({ name: "executionWorker", prisma, + replica: $replica, + logger: logger, runnerOptions: { connectionString: env.DATABASE_URL, concurrency: env.EXECUTION_WORKER_CONCURRENCY, @@ -786,6 +791,8 @@ function getTaskOperationWorkerQueue() { return new ZodWorker({ name: "taskOperationWorker", prisma, + replica: $replica, + logger: logger, runnerOptions: { connectionString: env.DATABASE_URL, concurrency: env.TASK_OPERATION_WORKER_CONCURRENCY, diff --git a/apps/webapp/package.json b/apps/webapp/package.json index be150ae46e..ffaa426cb0 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -50,6 +50,7 @@ "@headlessui/react": "^1.7.8", "@heroicons/react": "^2.0.12", "@internal/run-engine": "workspace:*", + "@internal/zod-worker": "workspace:*", "@internationalized/date": "^3.5.1", "@lezer/highlight": "^1.1.6", "@opentelemetry/api": "1.9.0", diff --git a/apps/webapp/tsconfig.json b/apps/webapp/tsconfig.json index 6b44d0f7c9..c0832b7413 100644 --- a/apps/webapp/tsconfig.json +++ b/apps/webapp/tsconfig.json @@ -33,7 +33,9 @@ "emails": ["../../internal-packages/emails/src/index"], "emails/*": ["../../internal-packages/emails/src/*"], "@internal/run-engine": ["../../internal-packages/run-engine/src/index"], - "@internal/run-engine/*": ["../../internal-packages/run-engine/src/*"] + "@internal/run-engine/*": ["../../internal-packages/run-engine/src/*"], + "@internal/zod-worker": ["../../internal-packages/zod-worker/src/index"], + "@internal/zod-worker/*": ["../../internal-packages/zod-worker/src/*"] }, "noEmit": true } From c902ce46640a266f67b2483c8c57fd067526fc84 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 30 Sep 2024 10:45:31 -0700 Subject: [PATCH 060/114] Added parseNaturalLanguageDuration to core/apps --- .../v3/services/enqueueDelayedRun.server.ts | 2 +- .../app/v3/services/triggerTask.server.ts | 53 +------------------ packages/core/src/v3/apps/duration.ts | 51 ++++++++++++++++++ packages/core/src/v3/apps/index.ts | 1 + 4 files changed, 54 insertions(+), 53 deletions(-) create mode 100644 packages/core/src/v3/apps/duration.ts diff --git a/apps/webapp/app/v3/services/enqueueDelayedRun.server.ts b/apps/webapp/app/v3/services/enqueueDelayedRun.server.ts index 12beb589f5..297df0ca9b 100644 --- a/apps/webapp/app/v3/services/enqueueDelayedRun.server.ts +++ b/apps/webapp/app/v3/services/enqueueDelayedRun.server.ts @@ -1,9 +1,9 @@ import { logger } from "~/services/logger.server"; import { marqs } from "~/v3/marqs/index.server"; import { BaseService } from "./baseService.server"; -import { parseNaturalLanguageDuration } from "./triggerTask.server"; import { workerQueue } from "~/services/worker.server"; import { $transaction } from "~/db.server"; +import { parseNaturalLanguageDuration } from "@trigger.dev/core/v3/apps"; export class EnqueueDelayedRunService extends BaseService { public async call(runId: string) { diff --git a/apps/webapp/app/v3/services/triggerTask.server.ts b/apps/webapp/app/v3/services/triggerTask.server.ts index b7743e4503..6a40ec70b2 100644 --- a/apps/webapp/app/v3/services/triggerTask.server.ts +++ b/apps/webapp/app/v3/services/triggerTask.server.ts @@ -21,6 +21,7 @@ import { isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus"; import { createTag, MAX_TAGS_PER_RUN } from "~/models/taskRunTag.server"; import { findCurrentWorkerFromEnvironment } from "../models/workerDeployment.server"; import { handleMetadataPacket } from "~/utils/packets"; +import { parseNaturalLanguageDuration } from "@trigger.dev/core/v3/apps"; export type TriggerTaskServiceOptions = { idempotencyKey?: string; @@ -626,58 +627,6 @@ export async function parseDelay(value?: string | Date): Promise= 0) { - result.setDate(result.getDate() + 7 * weeks); - hasMatch = true; - } - } - if (elements[2]) { - const days = Number(elements[2].slice(0, -1)); - if (days >= 0) { - result.setDate(result.getDate() + days); - hasMatch = true; - } - } - if (elements[3]) { - const hours = Number(elements[3].slice(0, -1)); - if (hours >= 0) { - result.setHours(result.getHours() + hours); - hasMatch = true; - } - } - if (elements[4]) { - const minutes = Number(elements[4].slice(0, -1)); - if (minutes >= 0) { - result.setMinutes(result.getMinutes() + minutes); - hasMatch = true; - } - } - if (elements[5]) { - const seconds = Number(elements[5].slice(0, -1)); - if (seconds >= 0) { - result.setSeconds(result.getSeconds() + seconds); - hasMatch = true; - } - } - } - - if (hasMatch) { - return result; - } - - return undefined; -} - function stringifyDuration(seconds: number): string | undefined { if (seconds <= 0) { return; diff --git a/packages/core/src/v3/apps/duration.ts b/packages/core/src/v3/apps/duration.ts new file mode 100644 index 0000000000..85c0dbc88c --- /dev/null +++ b/packages/core/src/v3/apps/duration.ts @@ -0,0 +1,51 @@ +export function parseNaturalLanguageDuration(duration: string): Date | undefined { + const regexPattern = /^(\d+w)?(\d+d)?(\d+h)?(\d+m)?(\d+s)?$/; + + const result: Date = new Date(); + let hasMatch = false; + + const elements = duration.match(regexPattern); + if (elements) { + if (elements[1]) { + const weeks = Number(elements[1].slice(0, -1)); + if (weeks >= 0) { + result.setDate(result.getDate() + 7 * weeks); + hasMatch = true; + } + } + if (elements[2]) { + const days = Number(elements[2].slice(0, -1)); + if (days >= 0) { + result.setDate(result.getDate() + days); + hasMatch = true; + } + } + if (elements[3]) { + const hours = Number(elements[3].slice(0, -1)); + if (hours >= 0) { + result.setHours(result.getHours() + hours); + hasMatch = true; + } + } + if (elements[4]) { + const minutes = Number(elements[4].slice(0, -1)); + if (minutes >= 0) { + result.setMinutes(result.getMinutes() + minutes); + hasMatch = true; + } + } + if (elements[5]) { + const seconds = Number(elements[5].slice(0, -1)); + if (seconds >= 0) { + result.setSeconds(result.getSeconds() + seconds); + hasMatch = true; + } + } + } + + if (hasMatch) { + return result; + } + + return undefined; +} diff --git a/packages/core/src/v3/apps/index.ts b/packages/core/src/v3/apps/index.ts index 73c24ae0e7..97266f729f 100644 --- a/packages/core/src/v3/apps/index.ts +++ b/packages/core/src/v3/apps/index.ts @@ -6,3 +6,4 @@ export * from "./http.js"; export * from "./provider.js"; export * from "./isExecaChildProcess.js"; export * from "./friendlyId.js"; +export * from "./duration.js"; From cd3656425d3f07e20c8953c2a159876aeead6b8e Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 30 Sep 2024 10:45:40 -0700 Subject: [PATCH 061/114] internal-packages/zod-worker in the lockfile --- pnpm-lock.yaml | 114 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 86 insertions(+), 28 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ad27a307c0..ed0de17123 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -935,6 +935,40 @@ importers: specifier: ^1.4.0 version: 1.6.0(@types/node@20.14.14) + internal-packages/zod-worker: + dependencies: + '@opentelemetry/api': + specifier: ^1.9.0 + version: 1.9.0 + '@trigger.dev/core': + specifier: workspace:* + version: link:../../packages/core + '@trigger.dev/database': + specifier: workspace:* + version: link:../database + graphile-worker: + specifier: 0.16.6 + version: 0.16.6(patch_hash=hdpetta7btqcc7xb5wfkcnanoa)(typescript@4.9.5) + lodash.omit: + specifier: ^4.5.0 + version: 4.5.0 + typescript: + specifier: ^4.8.4 + version: 4.9.5 + zod: + specifier: 3.22.3 + version: 3.22.3 + devDependencies: + '@types/lodash.omit': + specifier: ^4.5.7 + version: 4.5.7 + '@types/pg': + specifier: 8.6.6 + version: 8.6.6 + vitest: + specifier: ^1.4.0 + version: 1.6.0(@types/node@20.14.14) + packages/build: dependencies: '@trigger.dev/core': @@ -6942,7 +6976,7 @@ packages: resolution: {integrity: sha512-7kZUAaLscfgbwBQRbvdMYaZOWyMEcPTH/tJjnyAWJ/dvvs9Ef+CERx/qJb9GExJpl1qipaDGn7KqHnFGGixd0w==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} dependencies: - semver: 7.5.4 + semver: 7.6.3 /@npmcli/git@4.1.0: resolution: {integrity: sha512-9hwoB3gStVfa0N31ymBmrX+GuDGdVA/QWShZVqE0HK2Af+7QGGrCTbZia/SW0ImUTjTne7SP91qxDmtXvDHRPQ==} @@ -6954,7 +6988,7 @@ packages: proc-log: 3.0.0 promise-inflight: 1.0.1 promise-retry: 2.0.1 - semver: 7.5.4 + semver: 7.6.3 which: 3.0.1 transitivePeerDependencies: - bluebird @@ -14699,9 +14733,8 @@ packages: resolution: {integrity: sha512-O2xNmXebtwVekJDD+02udOncjVcMZQuTEQEMpKJ0ZRf5E7/9JJX3izhKUcUifBkyKpljyUM6BTgy2trmviKlpw==} dependencies: '@types/node': 18.19.20 - pg-protocol: 1.6.0 + pg-protocol: 1.6.1 pg-types: 2.2.0 - dev: false /@types/prismjs@1.26.0: resolution: {integrity: sha512-ZTaqn/qSqUuAq1YwvOFQfVW1AR/oQJlLSZVustdjwI+GZ8kr0MSHBj0tsXPW1EqHubx50gtBEjbPGsdZwQwCjQ==} @@ -15254,7 +15287,7 @@ packages: /@vitest/snapshot@1.6.0: resolution: {integrity: sha512-+Hx43f8Chus+DCmygqqfetcAZrDJwvTj0ymqjQq4CvmpKFSTVteEOBzCusu1x2tt4OJcvBflyHUE0DZSLgEMtQ==} dependencies: - magic-string: 0.30.8 + magic-string: 0.30.11 pathe: 1.1.2 pretty-format: 29.7.0 dev: true @@ -16590,7 +16623,7 @@ packages: /builtins@5.0.1: resolution: {integrity: sha512-qwVpFEHNfhYJIzNRBvd2C1kyo6jz3ZSMPyyuR47OPdiKWlbYnZNyDWuyR175qDnAJLiCo5fBBqPb3RiXgWlkOQ==} dependencies: - semver: 7.5.4 + semver: 7.6.3 dev: true /bun-types@1.1.17: @@ -17352,6 +17385,22 @@ packages: yaml: 1.10.2 dev: true + /cosmiconfig@8.3.6(typescript@4.9.5): + resolution: {integrity: sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==} + engines: {node: '>=14'} + peerDependencies: + typescript: '>=4.9.5' + peerDependenciesMeta: + typescript: + optional: true + dependencies: + import-fresh: 3.3.0 + js-yaml: 4.1.0 + parse-json: 5.2.0 + path-type: 4.0.0 + typescript: 4.9.5 + dev: false + /cosmiconfig@8.3.6(typescript@5.2.2): resolution: {integrity: sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==} engines: {node: '>=14'} @@ -18197,7 +18246,7 @@ packages: '@one-ini/wasm': 0.1.1 commander: 10.0.1 minimatch: 9.0.1 - semver: 7.5.4 + semver: 7.6.3 dev: false /ee-first@1.1.1: @@ -20480,14 +20529,35 @@ packages: '@types/node': 20.14.14 '@types/semver': 7.5.1 chalk: 4.1.2 - debug: 4.3.6 + debug: 4.3.7 interpret: 3.1.1 - semver: 7.5.4 + semver: 7.6.3 + tslib: 2.6.2 + yargs: 17.7.2 + transitivePeerDependencies: + - supports-color + dev: false + + /graphile-worker@0.16.6(patch_hash=hdpetta7btqcc7xb5wfkcnanoa)(typescript@4.9.5): + resolution: {integrity: sha512-e7gGYDmGqzju2l83MpzX8vNG/lOtVJiSzI3eZpAFubSxh/cxs7sRrRGBGjzBP1kNG0H+c95etPpNRNlH65PYhw==} + engines: {node: '>=14.0.0'} + hasBin: true + dependencies: + '@graphile/logger': 0.2.0 + '@types/debug': 4.1.12 + '@types/pg': 8.11.6 + cosmiconfig: 8.3.6(typescript@4.9.5) + graphile-config: 0.0.1-beta.8 + json5: 2.2.3 + pg: 8.11.5 tslib: 2.6.2 yargs: 17.7.2 transitivePeerDependencies: + - pg-native - supports-color + - typescript dev: false + patched: true /graphile-worker@0.16.6(patch_hash=hdpetta7btqcc7xb5wfkcnanoa)(typescript@5.2.2): resolution: {integrity: sha512-e7gGYDmGqzju2l83MpzX8vNG/lOtVJiSzI3eZpAFubSxh/cxs7sRrRGBGjzBP1kNG0H+c95etPpNRNlH65PYhw==} @@ -21965,7 +22035,7 @@ packages: resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} engines: {node: '>=10'} dependencies: - semver: 7.5.4 + semver: 7.6.3 dev: true /make-error@1.3.6: @@ -23116,7 +23186,7 @@ packages: make-fetch-happen: 13.0.1 nopt: 7.2.0 proc-log: 4.2.0 - semver: 7.5.4 + semver: 7.6.3 tar: 6.2.1 which: 4.0.0 transitivePeerDependencies: @@ -23156,7 +23226,7 @@ packages: dependencies: hosted-git-info: 6.1.1 is-core-module: 2.14.0 - semver: 7.5.4 + semver: 7.6.3 validate-npm-package-license: 3.0.4 dev: true @@ -23192,7 +23262,7 @@ packages: resolution: {integrity: sha512-744wat5wAAHsxa4590mWO0tJ8PKxR8ORZsH9wGpQc3nWTzozMAgBN/XyqYw7mg3yqLM8dLwEnwSfKMmXAjF69g==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} dependencies: - semver: 7.5.4 + semver: 7.6.3 dev: true /npm-normalize-package-bin@2.0.0: @@ -23211,7 +23281,7 @@ packages: dependencies: hosted-git-info: 6.1.1 proc-log: 3.0.0 - semver: 7.5.4 + semver: 7.6.3 validate-npm-package-name: 5.0.0 dev: true @@ -23233,7 +23303,7 @@ packages: npm-install-checks: 6.2.0 npm-normalize-package-bin: 3.0.1 npm-package-arg: 10.1.0 - semver: 7.5.4 + semver: 7.6.3 dev: true /npm-run-all@4.1.5: @@ -24016,7 +24086,6 @@ packages: /pg-int8@1.0.1: resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} engines: {node: '>=4.0.0'} - dev: false /pg-numeric@1.0.2: resolution: {integrity: sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==} @@ -24031,13 +24100,8 @@ packages: pg: 8.11.5 dev: false - /pg-protocol@1.6.0: - resolution: {integrity: sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==} - dev: false - /pg-protocol@1.6.1: resolution: {integrity: sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==} - dev: false /pg-types@2.2.0: resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} @@ -24048,7 +24112,6 @@ packages: postgres-bytea: 1.0.0 postgres-date: 1.0.7 postgres-interval: 1.2.0 - dev: false /pg-types@4.0.2: resolution: {integrity: sha512-cRL3JpS3lKMGsKaWndugWQoLOCoP+Cic8oseVcbr0qhPzYD5DWXK+RZ9LY9wxRf7RQia4SCwQlXk0q6FCPrVng==} @@ -24531,7 +24594,6 @@ packages: /postgres-array@2.0.0: resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} engines: {node: '>=4'} - dev: false /postgres-array@3.0.2: resolution: {integrity: sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==} @@ -24541,7 +24603,6 @@ packages: /postgres-bytea@1.0.0: resolution: {integrity: sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==} engines: {node: '>=0.10.0'} - dev: false /postgres-bytea@3.0.0: resolution: {integrity: sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==} @@ -24553,7 +24614,6 @@ packages: /postgres-date@1.0.7: resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} engines: {node: '>=0.10.0'} - dev: false /postgres-date@2.1.0: resolution: {integrity: sha512-K7Juri8gtgXVcDfZttFKVmhglp7epKb1K4pgrkLxehjqkrgPhfG6OO8LHLkfaqkbpjNRnra018XwAr1yQFWGcA==} @@ -24565,7 +24625,6 @@ packages: engines: {node: '>=0.10.0'} dependencies: xtend: 4.0.2 - dev: false /postgres-interval@3.0.0: resolution: {integrity: sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==} @@ -26437,7 +26496,6 @@ packages: resolution: {integrity: sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==} engines: {node: '>=10'} hasBin: true - dev: false /send@0.18.0: resolution: {integrity: sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==} @@ -29154,7 +29212,7 @@ packages: dependencies: '@types/node': 18.11.18 esbuild: 0.18.11 - postcss: 8.4.29 + postcss: 8.4.44 rollup: 3.29.1 optionalDependencies: fsevents: 2.3.3 From 79733caa620946ab82314e8bccb1e5194fcb6b3e Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 30 Sep 2024 11:37:07 -0700 Subject: [PATCH 062/114] Pass in the master queue, remove old rebalance workers code --- .../run-engine/src/run-queue/index.ts | 191 +++--------------- .../src/run-queue/keyProducer.test.ts | 82 +++----- .../run-engine/src/run-queue/keyProducer.ts | 47 +---- .../run-engine/src/run-queue/types.ts | 10 +- 4 files changed, 71 insertions(+), 259 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 37b4d4f62d..5294537c7d 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -24,7 +24,7 @@ import { RunQueueShortKeyProducer } from "./keyProducer.js"; const SemanticAttributes = { QUEUE: "runqueue.queue", - PARENT_QUEUE: "runqueue.parentQueue", + MASTER_QUEUE: "runqueue.masterQueue", RUN_ID: "runqueue.runId", CONCURRENCY_KEY: "runqueue.concurrencyKey", ORG_ID: "runqueue.orgId", @@ -58,10 +58,9 @@ export class RunQueue { this.redis = new Redis(options.redis); this.logger = options.logger; - this.keys = new RunQueueShortKeyProducer("rq:", options.name); + this.keys = new RunQueueShortKeyProducer("rq:"); this.queuePriorityStrategy = options.queuePriorityStrategy; - this.#startRebalanceWorkers(); this.#registerCommands(); } @@ -158,9 +157,11 @@ export class RunQueue { public async enqueueMessage({ env, message, + masterQueue, }: { env: MinimalAuthenticatedEnvironment; message: InputPayload; + masterQueue: string; }) { return await this.#trace( "enqueueMessage", @@ -169,25 +170,23 @@ export class RunQueue { const queue = this.keys.queueKey(env, message.queue, concurrencyKey); - const parentQueue = this.keys.envSharedQueueKey(env); - propagation.inject(context.active(), message); span.setAttributes({ [SemanticAttributes.QUEUE]: queue, [SemanticAttributes.RUN_ID]: runId, [SemanticAttributes.CONCURRENCY_KEY]: concurrencyKey, - [SemanticAttributes.PARENT_QUEUE]: parentQueue, + [SemanticAttributes.MASTER_QUEUE]: masterQueue, }); const messagePayload: OutputPayload = { ...message, version: "1", queue, - parentQueue, + masterQueue, }; - await this.#callEnqueueMessage(messagePayload, parentQueue); + await this.#callEnqueueMessage(messagePayload, masterQueue); }, { kind: SpanKind.PRODUCER, @@ -201,15 +200,13 @@ export class RunQueue { ); } - public async dequeueMessageInEnv(env: MinimalAuthenticatedEnvironment) { + public async dequeueMessageInEnv(env: MinimalAuthenticatedEnvironment, masterQueue: string) { return this.#trace( "dequeueMessageInEnv", async (span) => { - const parentQueue = this.keys.envSharedQueueKey(env); - // Read the parent queue for matching queues const messageQueue = await this.#getRandomQueueFromParentQueue( - parentQueue, + masterQueue, this.options.envQueuePriorityStrategy, (queue) => this.#calculateMessageQueueCapacities(queue, { checkForDisabled: false }), env.id @@ -217,7 +214,7 @@ export class RunQueue { if (!messageQueue) { this.logger.debug("No message queue found", { - parentQueue, + masterQueue, }); return; @@ -225,7 +222,7 @@ export class RunQueue { const message = await this.#callDequeueMessage({ messageQueue, - parentQueue, + masterQueue, concurrencyLimitKey: this.keys.concurrencyLimitKeyFromQueue(messageQueue), currentConcurrencyKey: this.keys.currentConcurrencyKeyFromQueue(messageQueue), envConcurrencyLimitKey: this.keys.envConcurrencyLimitKeyFromQueue(messageQueue), @@ -246,7 +243,7 @@ export class RunQueue { [SemanticAttributes.QUEUE]: message.message.queue, [SemanticAttributes.RUN_ID]: message.message.runId, [SemanticAttributes.CONCURRENCY_KEY]: message.message.concurrencyKey, - [SemanticAttributes.PARENT_QUEUE]: parentQueue, + [SemanticAttributes.MASTER_QUEUE]: masterQueue, }); return message; @@ -262,14 +259,12 @@ export class RunQueue { ); } - public async getSharedQueueDetails() { - const parentQueue = this.keys.sharedQueueKey(); - + public async getSharedQueueDetails(masterQueue: string) { const { range } = await this.queuePriorityStrategy.nextCandidateSelection( - parentQueue, + masterQueue, "getSharedQueueDetails" ); - const queues = await this.#getChildQueuesWithScores(parentQueue, range); + const queues = await this.#getChildQueuesWithScores(masterQueue, range); const queuesWithScores = await this.#calculateQueueScores(queues, (queue) => this.#calculateMessageQueueCapacities(queue) @@ -278,7 +273,7 @@ export class RunQueue { // We need to priority shuffle here to ensure all workers aren't just working on the highest priority queue const choice = this.queuePriorityStrategy.chooseQueue( queuesWithScores, - parentQueue, + masterQueue, "getSharedQueueDetails", range ); @@ -296,15 +291,13 @@ export class RunQueue { /** * Dequeue a message from the shared queue (this should be used in production environments) */ - public async dequeueMessageInSharedQueue(consumerId: string) { + public async dequeueMessageInSharedQueue(consumerId: string, masterQueue: string) { return this.#trace( "dequeueMessageInSharedQueue", async (span) => { - const parentQueue = this.keys.sharedQueueKey(); - // Read the parent queue for matching queues const messageQueue = await this.#getRandomQueueFromParentQueue( - parentQueue, + masterQueue, this.options.queuePriorityStrategy, (queue) => this.#calculateMessageQueueCapacities(queue, { checkForDisabled: true }), consumerId @@ -317,7 +310,7 @@ export class RunQueue { // If the queue includes a concurrency key, we need to remove the ck:concurrencyKey from the queue name const message = await this.#callDequeueMessage({ messageQueue, - parentQueue, + masterQueue, concurrencyLimitKey: this.keys.concurrencyLimitKeyFromQueue(messageQueue), currentConcurrencyKey: this.keys.currentConcurrencyKeyFromQueue(messageQueue), envConcurrencyLimitKey: this.keys.envConcurrencyLimitKeyFromQueue(messageQueue), @@ -338,7 +331,7 @@ export class RunQueue { [SemanticAttributes.QUEUE]: message.message.queue, [SemanticAttributes.RUN_ID]: message.message.runId, [SemanticAttributes.CONCURRENCY_KEY]: message.message.concurrencyKey, - [SemanticAttributes.PARENT_QUEUE]: parentQueue, + [SemanticAttributes.MASTER_QUEUE]: masterQueue, }); return message; @@ -384,7 +377,7 @@ export class RunQueue { await this.#callAcknowledgeMessage({ messageId, messageQueue: message.queue, - parentQueue: message.parentQueue, + masterQueue: message.masterQueue, messageKey: this.keys.messageKey(orgId, messageId), concurrencyKey: this.keys.currentConcurrencyKeyFromQueue(message.queue), envConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(message.queue), @@ -428,12 +421,12 @@ export class RunQueue { [SemanticAttributes.QUEUE]: message.queue, [SemanticAttributes.RUN_ID]: messageId, [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, - [SemanticAttributes.PARENT_QUEUE]: message.parentQueue, + [SemanticAttributes.MASTER_QUEUE]: message.masterQueue, }); const messageKey = this.keys.messageKey(orgId, messageId); const messageQueue = message.queue; - const parentQueue = message.parentQueue; + const parentQueue = message.masterQueue; const concurrencyKey = this.keys.currentConcurrencyKeyFromQueue(message.queue); const envConcurrencyKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); const taskConcurrencyKey = this.keys.taskIdentifierCurrentConcurrencyKeyFromQueue( @@ -726,7 +719,7 @@ export class RunQueue { attributes: { [SEMATTRS_MESSAGING_OPERATION]: "receive", [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", - [SemanticAttributes.PARENT_QUEUE]: parentQueue, + [SemanticAttributes.MASTER_QUEUE]: parentQueue, }, } ); @@ -797,128 +790,6 @@ export class RunQueue { return result; } - #startRebalanceWorkers() { - if (!this.options.enableRebalancing) { - return; - } - - // Start a new worker to rebalance parent queues periodically - for (let i = 0; i < this.options.workers; i++) { - const worker = new AsyncWorker(this.#rebalanceParentQueues.bind(this), 60_000); - - this.#rebalanceWorkers.push(worker); - - worker.start(); - } - } - - async #rebalanceParentQueues() { - return await new Promise((resolve, reject) => { - // Scan for sorted sets with the parent queue pattern - const pattern = this.keys.sharedQueueScanPattern(); - const redis = this.redis.duplicate(); - const stream = redis.scanStream({ - match: pattern, - type: "zset", - count: 100, - }); - - this.logger.debug("Streaming parent queues based on pattern", { - pattern, - component: "runqueue", - operation: "rebalanceParentQueues", - service: this.name, - }); - - stream.on("data", async (keys) => { - const uniqueKeys = Array.from(new Set(keys)); - - if (uniqueKeys.length === 0) { - return; - } - - stream.pause(); - - this.logger.debug("Rebalancing parent queues", { - component: "runqueue", - operation: "rebalanceParentQueues", - parentQueues: uniqueKeys, - service: this.name, - }); - - Promise.all( - uniqueKeys.map(async (key) => this.#rebalanceParentQueue(this.keys.stripKeyPrefix(key))) - ).finally(() => { - stream.resume(); - }); - }); - - stream.on("end", () => { - redis.quit().finally(() => { - resolve(); - }); - }); - - stream.on("error", (e) => { - redis.quit().finally(() => { - reject(e); - }); - }); - }); - } - - // Parent queue is a sorted set, the values of which are queue keys and the scores are is the oldest message in the queue - // We need to scan the parent queue and rebalance the queues based on the oldest message in the queue - async #rebalanceParentQueue(parentQueue: string) { - return await new Promise((resolve, reject) => { - const redis = this.redis.duplicate(); - - const stream = redis.zscanStream(parentQueue, { - match: "*", - count: 100, - }); - - stream.on("data", async (childQueues) => { - stream.pause(); - - // childQueues is a flat array but of the form [queue1, score1, queue2, score2, ...], we want to group them into pairs - const childQueuesWithScores: Record = {}; - - for (let i = 0; i < childQueues.length; i += 2) { - childQueuesWithScores[childQueues[i]] = childQueues[i + 1]; - } - - this.logger.debug("Rebalancing child queues", { - parentQueue, - childQueuesWithScores, - component: "runqueue", - operation: "rebalanceParentQueues", - service: this.name, - }); - - await Promise.all( - Object.entries(childQueuesWithScores).map(async ([childQueue, currentScore]) => - this.#callRebalanceParentQueueChild({ parentQueue, childQueue, currentScore }) - ) - ).finally(() => { - stream.resume(); - }); - }); - - stream.on("end", () => { - redis.quit().finally(() => { - resolve(); - }); - }); - - stream.on("error", (e) => { - redis.quit().finally(() => { - reject(e); - }); - }); - }); - } - async #callEnqueueMessage(message: OutputPayload, parentQueue: string) { const concurrencyKey = this.keys.currentConcurrencyKeyFromQueue(message.queue); const envConcurrencyKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); @@ -952,7 +823,7 @@ export class RunQueue { async #callDequeueMessage({ messageQueue, - parentQueue, + masterQueue, concurrencyLimitKey, envConcurrencyLimitKey, currentConcurrencyKey, @@ -962,7 +833,7 @@ export class RunQueue { taskCurrentConcurrentKeyPrefix, }: { messageQueue: string; - parentQueue: string; + masterQueue: string; concurrencyLimitKey: string; envConcurrencyLimitKey: string; currentConcurrencyKey: string; @@ -974,7 +845,7 @@ export class RunQueue { const result = await this.redis.dequeueMessage( //keys messageQueue, - parentQueue, + masterQueue, concurrencyLimitKey, envConcurrencyLimitKey, currentConcurrencyKey, @@ -1030,7 +901,7 @@ export class RunQueue { async #callAcknowledgeMessage({ messageId, - parentQueue, + masterQueue, messageKey, messageQueue, concurrencyKey, @@ -1038,7 +909,7 @@ export class RunQueue { taskConcurrencyKey, projectConcurrencyKey, }: { - parentQueue: string; + masterQueue: string; messageKey: string; messageQueue: string; concurrencyKey: string; @@ -1055,12 +926,12 @@ export class RunQueue { projectConcurrencyKey, taskConcurrencyKey, messageId, - parentQueue, + masterQueue, service: this.name, }); return this.redis.acknowledgeMessage( - parentQueue, + masterQueue, messageKey, messageQueue, concurrencyKey, diff --git a/internal-packages/run-engine/src/run-queue/keyProducer.test.ts b/internal-packages/run-engine/src/run-queue/keyProducer.test.ts index 5780e4d91b..2a05fcf5db 100644 --- a/internal-packages/run-engine/src/run-queue/keyProducer.test.ts +++ b/internal-packages/run-engine/src/run-queue/keyProducer.test.ts @@ -6,25 +6,25 @@ import { RunQueueShortKeyProducer } from "./keyProducer.js"; describe("KeyProducer", () => { it("sharedQueueScanPattern", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); - const pattern = keyProducer.sharedQueueScanPattern(); + const keyProducer = new RunQueueShortKeyProducer("test:"); + const pattern = keyProducer.masterQueueScanPattern("main"); expect(pattern).toBe("test:*sharedQueue:main"); }); it("queueCurrentConcurrencyScanPattern", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const pattern = keyProducer.queueCurrentConcurrencyScanPattern(); expect(pattern).toBe("test:{org:*}:proj:*:env:*:queue:*:currentConcurrency"); }); it("stripKeyPrefix", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const key = keyProducer.stripKeyPrefix("test:abc"); expect(key).toBe("abc"); }); it("queueConcurrencyLimitKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const key = keyProducer.queueConcurrencyLimitKey( { id: "e1234", @@ -39,7 +39,7 @@ describe("KeyProducer", () => { }); it("envConcurrencyLimitKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const key = keyProducer.envConcurrencyLimitKey({ id: "e1234", type: "PRODUCTION", @@ -51,7 +51,7 @@ describe("KeyProducer", () => { }); it("queueKey (no concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const key = keyProducer.queueKey( { id: "e1234", @@ -66,7 +66,7 @@ describe("KeyProducer", () => { }); it("queueKey (w concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const key = keyProducer.queueKey( { id: "e1234", @@ -81,38 +81,8 @@ describe("KeyProducer", () => { expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:ck:c1234"); }); - it("envSharedQueueKey dev", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); - const key = keyProducer.envSharedQueueKey({ - id: "e1234", - type: "DEVELOPMENT", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }); - expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:sharedQueue:main"); - }); - - it("envSharedQueueKey deployed", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); - const key = keyProducer.envSharedQueueKey({ - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }); - expect(key).toBe("sharedQueue:main"); - }); - - it("sharedQueueKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); - const key = keyProducer.sharedQueueKey(); - expect(key).toBe("sharedQueue:main"); - }); - it("concurrencyLimitKeyFromQueue (w concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const queueKey = keyProducer.queueKey( { id: "e1234", @@ -129,7 +99,7 @@ describe("KeyProducer", () => { }); it("concurrencyLimitKeyFromQueue (no concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const queueKey = keyProducer.queueKey( { id: "e1234", @@ -145,7 +115,7 @@ describe("KeyProducer", () => { }); it("currentConcurrencyKeyFromQueue (w concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const queueKey = keyProducer.queueKey( { id: "e1234", @@ -164,7 +134,7 @@ describe("KeyProducer", () => { }); it("currentConcurrencyKeyFromQueue (no concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const queueKey = keyProducer.queueKey( { id: "e1234", @@ -180,7 +150,7 @@ describe("KeyProducer", () => { }); it("currentConcurrencyKey (w concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const key = keyProducer.currentConcurrencyKey( { id: "e1234", @@ -198,7 +168,7 @@ describe("KeyProducer", () => { }); it("currentConcurrencyKey (no concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const key = keyProducer.currentConcurrencyKey( { id: "e1234", @@ -214,7 +184,7 @@ describe("KeyProducer", () => { }); it("taskIdentifierCurrentConcurrencyKeyPrefixFromQueue", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const queueKey = keyProducer.queueKey( { id: "e1234", @@ -230,7 +200,7 @@ describe("KeyProducer", () => { }); it("taskIdentifierCurrentConcurrencyKeyFromQueue", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const queueKey = keyProducer.queueKey( { id: "e1234", @@ -246,7 +216,7 @@ describe("KeyProducer", () => { }); it("taskIdentifierCurrentConcurrencyKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const key = keyProducer.taskIdentifierCurrentConcurrencyKey( { id: "e1234", @@ -261,7 +231,7 @@ describe("KeyProducer", () => { }); it("projectCurrentConcurrencyKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const key = keyProducer.projectCurrentConcurrencyKey({ id: "e1234", type: "PRODUCTION", @@ -273,7 +243,7 @@ describe("KeyProducer", () => { }); it("projectCurrentConcurrencyKeyFromQueue", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const key = keyProducer.projectCurrentConcurrencyKeyFromQueue( "{org:o1234}:proj:p1234:currentConcurrency" ); @@ -281,7 +251,7 @@ describe("KeyProducer", () => { }); it("disabledConcurrencyLimitKeyFromQueue", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const queueKey = keyProducer.queueKey( { id: "e1234", @@ -297,7 +267,7 @@ describe("KeyProducer", () => { }); it("envConcurrencyLimitKeyFromQueue", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const queueKey = keyProducer.queueKey( { id: "e1234", @@ -313,7 +283,7 @@ describe("KeyProducer", () => { }); it("envCurrentConcurrencyKeyFromQueue", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const queueKey = keyProducer.queueKey( { id: "e1234", @@ -329,7 +299,7 @@ describe("KeyProducer", () => { }); it("envCurrentConcurrencyKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const key = keyProducer.envCurrentConcurrencyKey({ id: "e1234", type: "PRODUCTION", @@ -341,13 +311,13 @@ describe("KeyProducer", () => { }); it("messageKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const key = keyProducer.messageKey("o1234", "m1234"); expect(key).toBe("{org:o1234}:message:m1234"); }); it("extractComponentsFromQueue (no concurrencyKey)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const queueKey = keyProducer.queueKey( { id: "e1234", @@ -369,7 +339,7 @@ describe("KeyProducer", () => { }); it("extractComponentsFromQueue (w concurrencyKey)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:", "main"); + const keyProducer = new RunQueueShortKeyProducer("test:"); const queueKey = keyProducer.queueKey( { id: "e1234", diff --git a/internal-packages/run-engine/src/run-queue/keyProducer.ts b/internal-packages/run-engine/src/run-queue/keyProducer.ts index ff6a75ee48..8c145ce16e 100644 --- a/internal-packages/run-engine/src/run-queue/keyProducer.ts +++ b/internal-packages/run-engine/src/run-queue/keyProducer.ts @@ -1,5 +1,4 @@ -import { RuntimeEnvironmentType } from "../../../database/src/index.js"; -import { AuthenticatedEnvironment } from "../shared/index.js"; +import { MinimalAuthenticatedEnvironment } from "../shared/index.js"; import { RunQueueKeyProducer } from "./types.js"; const constants = { @@ -16,17 +15,10 @@ const constants = { } as const; export class RunQueueShortKeyProducer implements RunQueueKeyProducer { - sharedQueue: string; + constructor(private _prefix: string) {} - constructor( - private _prefix: string, - sharedQueue: string - ) { - this.sharedQueue = `sharedQueue:${sharedQueue}`; - } - - sharedQueueScanPattern() { - return `${this._prefix}*${this.sharedQueue}`; + masterQueueScanPattern(masterQueue: string) { + return `${this._prefix}*${masterQueue}`; } queueCurrentConcurrencyScanPattern() { @@ -41,11 +33,11 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { return key; } - queueConcurrencyLimitKey(env: AuthenticatedEnvironment, queue: string) { + queueConcurrencyLimitKey(env: MinimalAuthenticatedEnvironment, queue: string) { return [this.queueKey(env, queue), constants.CONCURRENCY_LIMIT_PART].join(":"); } - envConcurrencyLimitKey(env: AuthenticatedEnvironment) { + envConcurrencyLimitKey(env: MinimalAuthenticatedEnvironment) { return [ this.orgKeySection(env.organization.id), this.projKeySection(env.project.id), @@ -54,7 +46,7 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { ].join(":"); } - queueKey(env: AuthenticatedEnvironment, queue: string, concurrencyKey?: string) { + queueKey(env: MinimalAuthenticatedEnvironment, queue: string, concurrencyKey?: string) { return [ this.orgKeySection(env.organization.id), this.projKeySection(env.project.id), @@ -65,23 +57,6 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { .join(":"); } - envSharedQueueKey(env: AuthenticatedEnvironment) { - if (env.type === "DEVELOPMENT") { - return [ - this.orgKeySection(env.organization.id), - this.projKeySection(env.project.id), - this.envKeySection(env.id), - this.sharedQueue, - ].join(":"); - } - - return this.sharedQueueKey(); - } - - sharedQueueKey(): string { - return this.sharedQueue; - } - concurrencyLimitKeyFromQueue(queue: string) { const concurrencyQueueName = queue.replace(/:ck:.+$/, ""); return `${concurrencyQueueName}:${constants.CONCURRENCY_LIMIT_PART}`; @@ -92,7 +67,7 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { } currentConcurrencyKey( - env: AuthenticatedEnvironment, + env: MinimalAuthenticatedEnvironment, queue: string, concurrencyKey?: string ): string { @@ -116,7 +91,7 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { return `{${constants.ORG_PART}:${orgId}}:${constants.ENV_PART}:${envId}:${constants.CURRENT_CONCURRENCY_PART}`; } - envCurrentConcurrencyKey(env: AuthenticatedEnvironment): string { + envCurrentConcurrencyKey(env: MinimalAuthenticatedEnvironment): string { return [ this.orgKeySection(env.organization.id), this.envKeySection(env.id), @@ -137,7 +112,7 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { } taskIdentifierCurrentConcurrencyKey( - env: AuthenticatedEnvironment, + env: MinimalAuthenticatedEnvironment, taskIdentifier: string ): string { return [ @@ -148,7 +123,7 @@ export class RunQueueShortKeyProducer implements RunQueueKeyProducer { ].join(":"); } - projectCurrentConcurrencyKey(env: AuthenticatedEnvironment): string { + projectCurrentConcurrencyKey(env: MinimalAuthenticatedEnvironment): string { return [ this.orgKeySection(env.organization.id), this.projKeySection(env.project.id), diff --git a/internal-packages/run-engine/src/run-queue/types.ts b/internal-packages/run-engine/src/run-queue/types.ts index eaf49e5491..914193fb85 100644 --- a/internal-packages/run-engine/src/run-queue/types.ts +++ b/internal-packages/run-engine/src/run-queue/types.ts @@ -1,8 +1,6 @@ import { z } from "zod"; -import { MinimalAuthenticatedEnvironment } from "../shared/index.js"; import { RuntimeEnvironmentType } from "../../../database/src/index.js"; -import { env } from "process"; -import { version } from "os"; +import { MinimalAuthenticatedEnvironment } from "../shared/index.js"; export const InputPayload = z.object({ runId: z.string(), @@ -19,7 +17,7 @@ export type InputPayload = z.infer; export const OutputPayload = InputPayload.extend({ version: z.literal("1"), - parentQueue: z.string(), + masterQueue: z.string(), }); export type OutputPayload = z.infer; @@ -43,9 +41,7 @@ export type QueueWithScores = { export type QueueRange = { offset: number; count: number }; export interface RunQueueKeyProducer { - envSharedQueueKey(env: MinimalAuthenticatedEnvironment): string; - sharedQueueKey(): string; - sharedQueueScanPattern(): string; + masterQueueScanPattern(masterQueue: string): string; queueCurrentConcurrencyScanPattern(): string; //queue queueKey(env: MinimalAuthenticatedEnvironment, queue: string, concurrencyKey?: string): string; From bb214d28e65b134d0e8da1322d290480710dfcb9 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 30 Sep 2024 11:37:33 -0700 Subject: [PATCH 063/114] Add masterQueue to TaskRun --- internal-packages/database/prisma/schema.prisma | 4 ++++ .../run-engine/src/engine/index.ts | 17 +++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 70f62a8551..6c80ca9f8b 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -1676,8 +1676,12 @@ model TaskRun { project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) projectId String + // The specific queue this run is in queue String + /// The main queue that this run is part of + masterQueue String @default("main") + createdAt DateTime @default(now()) updatedAt DateTime @updatedAt diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 17f5087609..a2f9605318 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -9,7 +9,7 @@ import { createDateTimeWaitpoint, createRunAssociatedWaitpoint, } from "./waitpoint"; -import { generateFriendlyId } from "@trigger.dev/core/v3/apps"; +import { generateFriendlyId, parseNaturalLanguageDuration } from "@trigger.dev/core/v3/apps"; import { run } from "node:test"; type Options = { @@ -33,6 +33,7 @@ type TriggerParams = { parentSpanId?: string; lockedToVersionId?: string; concurrencyKey?: string; + masterQueue: string; queueName: string; queue?: QueueOptions; isTest: boolean; @@ -103,6 +104,7 @@ export class RunEngine { parentSpanId, lockedToVersionId, concurrencyKey, + masterQueue, queueName, queue, isTest, @@ -147,6 +149,7 @@ export class RunEngine { lockedToVersionId, concurrencyKey, queue: queueName, + masterQueue, isTest, delayUntil, queuedAt, @@ -244,13 +247,10 @@ export class RunEngine { completedAfter: taskRun.delayUntil, }); - //waitpoint - - await workerQueue.enqueue( - "v3.enqueueDelayedRun", - { runId: taskRun.id }, - { tx, runAt: delayUntil, jobKey: `v3.enqueueDelayedRun.${taskRun.id}` } - ); + await blockRunWithWaitpoint(prisma, { + runId: taskRun.id, + waitpoint: delayWaitpoint, + }); } if (!taskRun.delayUntil && taskRun.ttl) { @@ -267,6 +267,7 @@ export class RunEngine { await this.runQueue.enqueueMessage({ env: environment, + masterQueue, message: { runId: taskRun.id, taskIdentifier: taskRun.taskIdentifier, From 534b06035420784a35b71fcfb64881f235127542 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 30 Sep 2024 11:45:19 -0700 Subject: [PATCH 064/114] Fixed the tests --- .../run-engine/src/run-queue/index.test.ts | 56 ++++++++++++------- .../src/run-queue/keyProducer.test.ts | 6 +- 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/index.test.ts b/internal-packages/run-engine/src/run-queue/index.test.ts index cfaaedadab..5e96c439be 100644 --- a/internal-packages/run-engine/src/run-queue/index.test.ts +++ b/internal-packages/run-engine/src/run-queue/index.test.ts @@ -148,7 +148,11 @@ describe("RunQueue", () => { expect(oldestScore).toBe(undefined); //enqueue message - await queue.enqueueMessage({ env: authenticatedEnvDev, message: messageDev }); + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: messageDev, + masterQueue: "test_12345", + }); //queue length const result2 = await queue.lengthOfQueue(authenticatedEnvDev, messageDev.queue); @@ -177,13 +181,11 @@ describe("RunQueue", () => { ); expect(taskConcurrency).toBe(0); - const dequeued = await queue.dequeueMessageInEnv(authenticatedEnvDev); + const dequeued = await queue.dequeueMessageInEnv(authenticatedEnvDev, "test_12345"); expect(dequeued?.messageId).toEqual(messageDev.runId); expect(dequeued?.message.orgId).toEqual(messageDev.orgId); expect(dequeued?.message.version).toEqual("1"); - expect(dequeued?.message.parentQueue).toEqual( - "{org:o1234}:proj:p1234:env:e1234:sharedQueue:rq" - ); + expect(dequeued?.message.masterQueue).toEqual("test_12345"); //concurrencies const queueConcurrency2 = await queue.currentConcurrencyOfQueue( @@ -201,7 +203,7 @@ describe("RunQueue", () => { ); expect(taskConcurrency2).toBe(1); - const dequeued2 = await queue.dequeueMessageInEnv(authenticatedEnvDev); + const dequeued2 = await queue.dequeueMessageInEnv(authenticatedEnvDev, "test_12345"); expect(dequeued2).toBe(undefined); } finally { await queue.quit(); @@ -231,7 +233,11 @@ describe("RunQueue", () => { expect(oldestScore).toBe(undefined); //enqueue message - await queue.enqueueMessage({ env: authenticatedEnvProd, message: messageProd }); + await queue.enqueueMessage({ + env: authenticatedEnvProd, + message: messageProd, + masterQueue: "main", + }); //queue length const result2 = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); @@ -261,11 +267,11 @@ describe("RunQueue", () => { expect(taskConcurrency).toBe(0); //dequeue - const dequeued = await queue.dequeueMessageInSharedQueue("test_12345"); + const dequeued = await queue.dequeueMessageInSharedQueue("test_12345", "main"); expect(dequeued?.messageId).toEqual(messageProd.runId); expect(dequeued?.message.orgId).toEqual(messageProd.orgId); expect(dequeued?.message.version).toEqual("1"); - expect(dequeued?.message.parentQueue).toEqual("sharedQueue:rq"); + expect(dequeued?.message.masterQueue).toEqual("main"); //concurrencies const queueConcurrency2 = await queue.currentConcurrencyOfQueue( @@ -287,7 +293,7 @@ describe("RunQueue", () => { const length2 = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); expect(length2).toBe(0); - const dequeued2 = await queue.dequeueMessageInEnv(authenticatedEnvDev); + const dequeued2 = await queue.dequeueMessageInEnv(authenticatedEnvDev, "main"); expect(dequeued2).toBe(undefined); } finally { await queue.quit(); @@ -302,14 +308,18 @@ describe("RunQueue", () => { }); try { - const result = await queue.getSharedQueueDetails(); + const result = await queue.getSharedQueueDetails("main"); expect(result.selectionId).toBe("getSharedQueueDetails"); expect(result.queueCount).toBe(0); expect(result.queueChoice.choice).toStrictEqual({ abort: true }); - await queue.enqueueMessage({ env: authenticatedEnvProd, message: messageProd }); + await queue.enqueueMessage({ + env: authenticatedEnvProd, + message: messageProd, + masterQueue: "main", + }); - const result2 = await queue.getSharedQueueDetails(); + const result2 = await queue.getSharedQueueDetails("main"); expect(result2.selectionId).toBe("getSharedQueueDetails"); expect(result2.queueCount).toBe(1); expect(result2.queues[0].score).toBe(messageProd.timestamp); @@ -328,9 +338,13 @@ describe("RunQueue", () => { }); try { - await queue.enqueueMessage({ env: authenticatedEnvProd, message: messageProd }); + await queue.enqueueMessage({ + env: authenticatedEnvProd, + message: messageProd, + masterQueue: "main", + }); - const message = await queue.dequeueMessageInSharedQueue("test_12345"); + const message = await queue.dequeueMessageInSharedQueue("test_12345", "main"); expect(message).toBeDefined(); //check the message is gone @@ -361,7 +375,7 @@ describe("RunQueue", () => { expect(exists2).toBe(0); //dequeue - const message2 = await queue.dequeueMessageInSharedQueue("test_12345"); + const message2 = await queue.dequeueMessageInSharedQueue("test_12345", "main"); expect(message2).toBeUndefined(); } finally { await queue.quit(); @@ -375,9 +389,13 @@ describe("RunQueue", () => { }); try { - await queue.enqueueMessage({ env: authenticatedEnvProd, message: messageProd }); + await queue.enqueueMessage({ + env: authenticatedEnvProd, + message: messageProd, + masterQueue: "main", + }); - const message = await queue.dequeueMessageInSharedQueue("test_12345"); + const message = await queue.dequeueMessageInSharedQueue("test_12345", "main"); expect(message).toBeDefined(); //check the message is there @@ -424,7 +442,7 @@ describe("RunQueue", () => { expect(exists2).toBe(1); //dequeue - const message2 = await queue.dequeueMessageInSharedQueue("test_12345"); + const message2 = await queue.dequeueMessageInSharedQueue("test_12345", "main"); expect(message2?.messageId).toBe(messageProd.runId); } finally { await queue.quit(); diff --git a/internal-packages/run-engine/src/run-queue/keyProducer.test.ts b/internal-packages/run-engine/src/run-queue/keyProducer.test.ts index 2a05fcf5db..886d695f59 100644 --- a/internal-packages/run-engine/src/run-queue/keyProducer.test.ts +++ b/internal-packages/run-engine/src/run-queue/keyProducer.test.ts @@ -1,14 +1,12 @@ -import { expect, it } from "vitest"; -import { z } from "zod"; -import { redisTest } from "../test/containerTest.js"; import { describe } from "node:test"; +import { expect, it } from "vitest"; import { RunQueueShortKeyProducer } from "./keyProducer.js"; describe("KeyProducer", () => { it("sharedQueueScanPattern", () => { const keyProducer = new RunQueueShortKeyProducer("test:"); const pattern = keyProducer.masterQueueScanPattern("main"); - expect(pattern).toBe("test:*sharedQueue:main"); + expect(pattern).toBe("test:*main"); }); it("queueCurrentConcurrencyScanPattern", () => { From 3295597af533b336ffbfa9561ea1be51d3e51e21 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 30 Sep 2024 17:51:10 -0700 Subject: [PATCH 065/114] Moved waitpoint code into the run engine, also the zod worker --- internal-packages/run-engine/package.json | 1 + .../run-engine/src/engine/index.ts | 185 ++++++++++++++++-- .../run-engine/src/engine/waitpoint.ts | 50 ----- internal-packages/run-engine/tsconfig.json | 2 + internal-packages/zod-worker/src/index.ts | 2 + pnpm-lock.yaml | 6 + 6 files changed, 176 insertions(+), 70 deletions(-) delete mode 100644 internal-packages/run-engine/src/engine/waitpoint.ts diff --git a/internal-packages/run-engine/package.json b/internal-packages/run-engine/package.json index 10e2f95a18..7b9492d053 100644 --- a/internal-packages/run-engine/package.json +++ b/internal-packages/run-engine/package.json @@ -5,6 +5,7 @@ "main": "./src/index.ts", "types": "./src/index.ts", "dependencies": { + "@internal/zod-worker": "workspace:*", "@opentelemetry/api": "^1.9.0", "@opentelemetry/semantic-conventions": "^1.27.0", "@trigger.dev/core": "workspace:*", diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index a2f9605318..7cb36aeaf2 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -1,21 +1,29 @@ +import { RunnerOptions, ZodWorker } from "@internal/zod-worker"; +import { trace } from "@opentelemetry/api"; +import { Logger } from "@trigger.dev/core/logger"; import { QueueOptions } from "@trigger.dev/core/v3"; -import { PrismaClientOrTransaction } from "@trigger.dev/database"; +import { generateFriendlyId, parseNaturalLanguageDuration } from "@trigger.dev/core/v3/apps"; +import { + $transaction, + PrismaClient, + PrismaClientOrTransaction, + Waitpoint, +} from "@trigger.dev/database"; import { Redis, type RedisOptions } from "ioredis"; import Redlock from "redlock"; +import { z } from "zod"; import { RunQueue } from "../run-queue"; +import { SimpleWeightedChoiceStrategy } from "../run-queue/simpleWeightedPriorityStrategy"; import { MinimalAuthenticatedEnvironment } from "../shared"; -import { - blockRunWithWaitpoint, - createDateTimeWaitpoint, - createRunAssociatedWaitpoint, -} from "./waitpoint"; -import { generateFriendlyId, parseNaturalLanguageDuration } from "@trigger.dev/core/v3/apps"; -import { run } from "node:test"; + +import { nanoid } from "nanoid"; type Options = { redis: RedisOptions; - prisma: PrismaClientOrTransaction; - queue: RunQueue; + prisma: PrismaClient; + zodWorker: RunnerOptions & { + shutdownTimeoutInMs: number; + }; }; type TriggerParams = { @@ -55,11 +63,23 @@ type TriggerParams = { isWait: boolean; }; +const schema = { + "runengine.waitpointCompleteDateTime": z.object({ + waitpointId: z.string(), + }), + "runengine.expireRun": z.object({ + runId: z.string(), + }), +}; + +type EngineWorker = ZodWorker; + export class RunEngine { private redis: Redis; - private prisma: PrismaClientOrTransaction; + private prisma: PrismaClient; private redlock: Redlock; private runQueue: RunQueue; + private zodWorker: EngineWorker; constructor(private readonly options: Options) { this.prisma = options.prisma; @@ -72,8 +92,6 @@ export class RunEngine { automaticExtensionThreshold: 500, // time in ms }); - //todo change the way dequeuing works so you have to pass in the name of the shared queue? - this.runQueue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), @@ -83,9 +101,38 @@ export class RunEngine { defaultEnvConcurrency: 10, enableRebalancing: false, logger: new Logger("RunQueue", "warn"), + redis: options.redis, + }); + + this.zodWorker = new ZodWorker({ + name: "runQueueWorker", + prisma: options.prisma, + replica: options.prisma, + logger: new Logger("RunQueueWorker", "debug"), + runnerOptions: options.zodWorker, + shutdownTimeoutInMs: options.zodWorker.shutdownTimeoutInMs, + schema, + tasks: { + "runengine.waitpointCompleteDateTime": { + priority: 0, + maxAttempts: 10, + handler: async (payload, job) => { + await this.#completeWaitpoint(this.prisma, payload.waitpointId); + }, + }, + "runengine.expireRun": { + priority: 0, + maxAttempts: 10, + handler: async (payload, job) => { + await this.expireRun(payload.runId); + }, + }, + }, }); } + //MARK: - Run functions + /** "Triggers" one run, which creates the run */ async trigger( @@ -180,14 +227,14 @@ export class RunEngine { } //create associated waitpoint (this completes when the run completes) - const associatedWaitpoint = await createRunAssociatedWaitpoint(prisma, { + const associatedWaitpoint = await this.#createRunAssociatedWaitpoint(prisma, { projectId: environment.project.id, completedByTaskRunId: taskRun.id, }); if (isWait && parentTaskRunId) { //this will block the parent run from continuing until this waitpoint is completed (and removed) - await blockRunWithWaitpoint(prisma, { + await this.#blockRunWithWaitpoint(prisma, { runId: parentTaskRunId, waitpoint: associatedWaitpoint, }); @@ -242,12 +289,12 @@ export class RunEngine { } if (taskRun.delayUntil) { - const delayWaitpoint = await createDateTimeWaitpoint(prisma, { + const delayWaitpoint = await this.#createDateTimeWaitpoint(prisma, { projectId: environment.project.id, completedAfter: taskRun.delayUntil, }); - await blockRunWithWaitpoint(prisma, { + await this.#blockRunWithWaitpoint(prisma, { runId: taskRun.id, waitpoint: delayWaitpoint, }); @@ -257,10 +304,10 @@ export class RunEngine { const expireAt = parseNaturalLanguageDuration(taskRun.ttl); if (expireAt) { - await workerQueue.enqueue( - "v3.expireRun", + await this.zodWorker.enqueue( + "runengine.expireRun", { runId: taskRun.id }, - { tx, runAt: expireAt, jobKey: `v3.expireRun.${taskRun.id}` } + { tx, runAt: expireAt, jobKey: `runengine.expireRun.${taskRun.id}` } ); } } @@ -306,6 +353,104 @@ export class RunEngine { async prepareForAttempt(runId: string) {} async complete(runId: string, completion: any) {} + + async expireRun(runId: string) {} + + //MARK: - Waitpoints + + async #createRunAssociatedWaitpoint( + tx: PrismaClientOrTransaction, + { projectId, completedByTaskRunId }: { projectId: string; completedByTaskRunId: string } + ) { + return tx.waitpoint.create({ + data: { + type: "RUN", + status: "PENDING", + idempotencyKey: nanoid(24), + userProvidedIdempotencyKey: false, + projectId, + completedByTaskRunId, + }, + }); + } + + async #createDateTimeWaitpoint( + tx: PrismaClientOrTransaction, + { projectId, completedAfter }: { projectId: string; completedAfter: Date } + ) { + const waitpoint = await tx.waitpoint.create({ + data: { + type: "DATETIME", + status: "PENDING", + idempotencyKey: nanoid(24), + userProvidedIdempotencyKey: false, + projectId, + completedAfter, + }, + }); + + await this.zodWorker.enqueue( + "runengine.waitpointCompleteDateTime", + { waitpointId: waitpoint.id }, + { tx, runAt: completedAfter, jobKey: `waitpointCompleteDateTime.${waitpoint.id}` } + ); + + return waitpoint; + } + + async #blockRunWithWaitpoint( + tx: PrismaClientOrTransaction, + { runId, waitpoint }: { runId: string; waitpoint: Waitpoint } + ) { + return tx.taskRunWaitpoint.create({ + data: { + taskRunId: runId, + waitpointId: waitpoint.id, + projectId: waitpoint.projectId, + }, + }); + } + + /** Any runs blocked by this waitpoint will get continued (if no other waitpoints exist) */ + async #completeWaitpoint(prisma: PrismaClientOrTransaction, id: string) { + const waitpoint = await prisma.waitpoint.findUnique({ + where: { id }, + }); + + if (!waitpoint) { + throw new Error(`Waitpoint ${id} not found`); + } + + if (waitpoint.status === "COMPLETED") { + return; + } + + $transaction( + prisma, + async (tx) => { + const blockedRuns = await tx.taskRunWaitpoint.findMany({ + where: { waitpointId: id }, + }); + + for (const blockedRun of blockedRuns) { + const otherWaitpoints = await tx.taskRunWaitpoint.findMany({ + where: { taskRunId: blockedRun.taskRunId }, + }); + + //todo remove the blocker + //todo if there are no other blockers then queue the run to be executed + } + + await tx.waitpoint.update({ + where: { id }, + data: { status: "COMPLETED" }, + }); + }, + (error) => { + throw error; + } + ); + } } /* diff --git a/internal-packages/run-engine/src/engine/waitpoint.ts b/internal-packages/run-engine/src/engine/waitpoint.ts deleted file mode 100644 index 4dae24783a..0000000000 --- a/internal-packages/run-engine/src/engine/waitpoint.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { PrismaClientOrTransaction, Waitpoint } from "@trigger.dev/database"; -import { nanoid } from "nanoid"; - -export async function createRunAssociatedWaitpoint( - prisma: PrismaClientOrTransaction, - { projectId, completedByTaskRunId }: { projectId: string; completedByTaskRunId: string } -) { - return prisma.waitpoint.create({ - data: { - type: "RUN", - status: "PENDING", - idempotencyKey: nanoid(24), - userProvidedIdempotencyKey: false, - projectId, - completedByTaskRunId, - }, - }); -} - -export async function createDateTimeWaitpoint( - prisma: PrismaClientOrTransaction, - { projectId, completedAfter }: { projectId: string; completedAfter: Date } -) { - return prisma.waitpoint.create({ - data: { - type: "DATETIME", - status: "PENDING", - idempotencyKey: nanoid(24), - userProvidedIdempotencyKey: false, - projectId, - completedAfter, - }, - }); -} - -export async function blockRunWithWaitpoint( - prisma: PrismaClientOrTransaction, - { runId, waitpoint }: { runId: string; waitpoint: Waitpoint } -) { - return prisma.taskRunWaitpoint.create({ - data: { - taskRunId: runId, - waitpointId: waitpoint.id, - projectId: waitpoint.projectId, - }, - }); -} - -/** Any runs blocked by this waitpoint will get continued (if no other waitpoints exist) */ -export function completeWaitpoint(id: string) {} diff --git a/internal-packages/run-engine/tsconfig.json b/internal-packages/run-engine/tsconfig.json index bfd15dde0d..fb691b947e 100644 --- a/internal-packages/run-engine/tsconfig.json +++ b/internal-packages/run-engine/tsconfig.json @@ -15,6 +15,8 @@ "noEmit": true, "strict": true, "paths": { + "@internal/zod-worker": ["../../internal-packages/zod-worker/src/index"], + "@internal/zod-worker/*": ["../../internal-packages/zod-worker/src/*"], "@trigger.dev/core": ["../../packages/core/src/index"], "@trigger.dev/core/*": ["../../packages/core/src/*"], "@trigger.dev/database": ["../database/src/index"], diff --git a/internal-packages/zod-worker/src/index.ts b/internal-packages/zod-worker/src/index.ts index ae1bc07229..c597500e54 100644 --- a/internal-packages/zod-worker/src/index.ts +++ b/internal-packages/zod-worker/src/index.ts @@ -28,6 +28,8 @@ import { } from "@trigger.dev/database"; import { PgListenService } from "./pgListen.server"; +export type { RunnerOptions }; + const tracer = trace.getTracer("zodWorker", "3.0.0.dp.1"); export interface MessageCatalogSchema { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ed0de17123..7436ff3f05 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -243,6 +243,9 @@ importers: '@internal/run-engine': specifier: workspace:* version: link:../../internal-packages/run-engine + '@internal/zod-worker': + specifier: workspace:* + version: link:../../internal-packages/zod-worker '@internationalized/date': specifier: ^3.5.1 version: 3.5.1 @@ -894,6 +897,9 @@ importers: internal-packages/run-engine: dependencies: + '@internal/zod-worker': + specifier: workspace:* + version: link:../zod-worker '@opentelemetry/api': specifier: ^1.9.0 version: 1.9.0 From f1232e53d130ca843af597729c97ad3e2f4426c1 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 30 Sep 2024 18:39:56 -0700 Subject: [PATCH 066/114] Completing waitpoints --- .../run-engine/src/engine/index.ts | 104 ++++++++++++------ 1 file changed, 72 insertions(+), 32 deletions(-) diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 7cb36aeaf2..a6c1a148d5 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -7,6 +7,7 @@ import { $transaction, PrismaClient, PrismaClientOrTransaction, + TaskRun, Waitpoint, } from "@trigger.dev/database"; import { Redis, type RedisOptions } from "ioredis"; @@ -222,10 +223,6 @@ export class RunEngine { }); await this.redlock.using([taskRun.id], 5000, async (signal) => { - if (signal.aborted) { - throw signal.error; - } - //create associated waitpoint (this completes when the run completes) const associatedWaitpoint = await this.#createRunAssociatedWaitpoint(prisma, { projectId: environment.project.id, @@ -235,6 +232,7 @@ export class RunEngine { if (isWait && parentTaskRunId) { //this will block the parent run from continuing until this waitpoint is completed (and removed) await this.#blockRunWithWaitpoint(prisma, { + orgId: environment.organization.id, runId: parentTaskRunId, waitpoint: associatedWaitpoint, }); @@ -295,6 +293,7 @@ export class RunEngine { }); await this.#blockRunWithWaitpoint(prisma, { + orgId: environment.organization.id, runId: taskRun.id, waitpoint: delayWaitpoint, }); @@ -312,27 +311,13 @@ export class RunEngine { } } - await this.runQueue.enqueueMessage({ - env: environment, - masterQueue, - message: { - runId: taskRun.id, - taskIdentifier: taskRun.taskIdentifier, - orgId: environment.organization.id, - projectId: environment.project.id, - environmentId: environment.id, - environmentType: environment.type, - queue: taskRun.queue, - concurrencyKey: taskRun.concurrencyKey ?? undefined, - timestamp: Date.now(), - }, - }); + await this.enqueueRun(taskRun, environment, prisma); }); + //todo release parent concurrency (for the project, task, and environment, but not for the queue?) + //todo if this has been triggered with triggerAndWait or batchTriggerAndWait + return taskRun; - //todo waitpoints - //todo enqueue - //todo release concurrency? } /** Triggers multiple runs. @@ -341,7 +326,29 @@ export class RunEngine { async batchTrigger() {} /** The run can be added to the queue. When it's pulled from the queue it will be executed. */ - async prepareForQueue(runId: string) {} + async enqueueRun( + run: TaskRun, + env: MinimalAuthenticatedEnvironment, + tx?: PrismaClientOrTransaction + ) { + await this.runQueue.enqueueMessage({ + env, + masterQueue: run.masterQueue, + message: { + runId: run.id, + taskIdentifier: run.taskIdentifier, + orgId: env.organization.id, + projectId: env.project.id, + environmentId: env.id, + environmentType: env.type, + queue: run.queue, + concurrencyKey: run.concurrencyKey ?? undefined, + timestamp: Date.now(), + }, + }); + + //todo update the TaskRunExecutionSnapshot + } /** We want to actually execute the run, this could be a continuation of a previous execution. * This is called from the queue, when the run has been pulled. */ @@ -357,7 +364,6 @@ export class RunEngine { async expireRun(runId: string) {} //MARK: - Waitpoints - async #createRunAssociatedWaitpoint( tx: PrismaClientOrTransaction, { projectId, completedByTaskRunId }: { projectId: string; completedByTaskRunId: string } @@ -400,8 +406,12 @@ export class RunEngine { async #blockRunWithWaitpoint( tx: PrismaClientOrTransaction, - { runId, waitpoint }: { runId: string; waitpoint: Waitpoint } + { orgId, runId, waitpoint }: { orgId: string; runId: string; waitpoint: Waitpoint } ) { + //todo it would be better if we didn't remove from the queue, because this removes the payload + //todo better would be to have a "block" function which remove it from the queue but doesn't remove the payload + await this.runQueue.acknowledgeMessage(orgId, runId); + return tx.taskRunWaitpoint.create({ data: { taskRunId: runId, @@ -425,20 +435,50 @@ export class RunEngine { return; } - $transaction( + await $transaction( prisma, async (tx) => { - const blockedRuns = await tx.taskRunWaitpoint.findMany({ + const blockers = await tx.taskRunWaitpoint.findMany({ where: { waitpointId: id }, }); - for (const blockedRun of blockedRuns) { - const otherWaitpoints = await tx.taskRunWaitpoint.findMany({ - where: { taskRunId: blockedRun.taskRunId }, + for (const blocker of blockers) { + //remove the waitpoint + await tx.taskRunWaitpoint.delete({ + where: { id: blocker.id }, + }); + + const blockedRun = await tx.taskRun.findFirst({ + where: { id: blocker.taskRunId }, + include: { + _count: { + select: { + blockedByWaitpoints: true, + }, + }, + runtimeEnvironment: { + select: { + id: true, + type: true, + maximumConcurrencyLimit: true, + project: { + select: { + id: true, + }, + }, + organization: { + select: { + id: true, + }, + }, + }, + }, + }, }); - //todo remove the blocker - //todo if there are no other blockers then queue the run to be executed + if (blockedRun && blockedRun._count.blockedByWaitpoints === 0) { + await this.enqueueRun(blockedRun, blockedRun.runtimeEnvironment, tx); + } } await tx.waitpoint.update({ From d958679a1e66829669e20efc244c67f5e0ed9089 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 1 Oct 2024 17:25:47 -0700 Subject: [PATCH 067/114] An experiment to create a new test container with environment --- .../run-engine/src/test/containerTest.ts | 47 ++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/internal-packages/run-engine/src/test/containerTest.ts b/internal-packages/run-engine/src/test/containerTest.ts index 18e5e15d0f..e2d9bd1a66 100644 --- a/internal-packages/run-engine/src/test/containerTest.ts +++ b/internal-packages/run-engine/src/test/containerTest.ts @@ -1,8 +1,8 @@ import { StartedPostgreSqlContainer } from "@testcontainers/postgresql"; import { StartedRedisContainer } from "@testcontainers/redis"; -import { PrismaClient } from "../../../database/src"; import { Redis } from "ioredis"; -import { test, TestAPI } from "vitest"; +import { test } from "vitest"; +import { PrismaClient } from "../../../database/src"; import { createPostgresContainer, createRedisContainer } from "./utils"; type PostgresContext = { @@ -65,3 +65,46 @@ export const containerTest = test.extend({ redisContainer, redis, }); + +export function createContainerWithSetup(setup: (prisma: PrismaClient) => Promise) { + return containerTest.extend<{}>({ + prisma: async ({ prisma }: ContainerContext, use: (prisma: PrismaClient) => Promise) => { + await setup(prisma); + await use(prisma); + }, + }); +} + +async function setupTestDatabase(prisma: PrismaClient): Promise { + // Your database setup logic here + const org = await prisma.organization.create({ + data: { + title: "Test Organization", + slug: "test-organization", + }, + }); + + const project = await prisma.project.create({ + data: { + name: "Test Project", + slug: "test-project", + externalRef: "proj_1234", + organizationId: org.id, + }, + }); + + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "PRODUCTION", + slug: "prod", + projectId: project.id, + organizationId: org.id, + apiKey: "api_key", + pkApiKey: "pk_api_key", + shortcode: "short_code", + }, + }); +} + +export const containerTestWithAuthenticatedEnvironment = + createContainerWithSetup(setupTestDatabase); From cb293493b3425bc8265ae207457e0fa24a9fe4f1 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 1 Oct 2024 17:27:24 -0700 Subject: [PATCH 068/114] More changes to triggering --- .../app/v3/services/triggerTaskV2.server.ts | 13 +- .../run-engine/src/engine/index.ts | 112 ++++++++++-------- 2 files changed, 74 insertions(+), 51 deletions(-) diff --git a/apps/webapp/app/v3/services/triggerTaskV2.server.ts b/apps/webapp/app/v3/services/triggerTaskV2.server.ts index 8207849927..4ea9394d2c 100644 --- a/apps/webapp/app/v3/services/triggerTaskV2.server.ts +++ b/apps/webapp/app/v3/services/triggerTaskV2.server.ts @@ -40,6 +40,7 @@ export class OutOfEntitlementError extends Error { } } +//todo move this to a singleton somewhere const engine = new RunEngine({ prisma, redis: { @@ -50,6 +51,15 @@ const engine = new RunEngine({ enableAutoPipelining: true, ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }), }, + zodWorker: { + connectionString: env.DATABASE_URL, + concurrency: env.WORKER_CONCURRENCY, + pollInterval: env.WORKER_POLL_INTERVAL, + noPreparedStatements: env.DATABASE_URL !== env.DIRECT_URL, + schema: env.WORKER_SCHEMA, + maxPoolSize: env.WORKER_CONCURRENCY + 1, + shutdownTimeoutInMs: env.GRACEFUL_SHUTDOWN_TIMEOUT, + }, }); export class TriggerTaskService extends BaseService { @@ -336,6 +346,8 @@ export class TriggerTaskService extends BaseService { concurrencyKey: body.options?.concurrencyKey, queueName, queue: body.options?.queue, + //todo multiple worker pools support + masterQueue: "main", isTest: body.options?.test ?? false, delayUntil, queuedAt: delayUntil ? undefined : new Date(), @@ -352,7 +364,6 @@ export class TriggerTaskService extends BaseService { metadataType: metadataPacket?.dataType, seedMetadata: metadataPacket?.data, seedMetadataType: metadataPacket?.dataType, - isWait: true, }, this._prisma ); diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index a6c1a148d5..37d8c5e300 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -5,6 +5,7 @@ import { QueueOptions } from "@trigger.dev/core/v3"; import { generateFriendlyId, parseNaturalLanguageDuration } from "@trigger.dev/core/v3/apps"; import { $transaction, + Prisma, PrismaClient, PrismaClientOrTransaction, TaskRun, @@ -55,13 +56,12 @@ type TriggerParams = { parentTaskRunAttemptId?: string; rootTaskRunId?: string; batchId?: string; - resumeParentOnCompletion: boolean; - depth: number; + resumeParentOnCompletion?: boolean; + depth?: number; metadata?: string; metadataType?: string; seedMetadata?: string; seedMetadataType?: string; - isWait: boolean; }; const schema = { @@ -81,6 +81,7 @@ export class RunEngine { private redlock: Redlock; private runQueue: RunQueue; private zodWorker: EngineWorker; + private logger = new Logger("RunEngine", "debug"); constructor(private readonly options: Options) { this.prisma = options.prisma; @@ -118,7 +119,7 @@ export class RunEngine { priority: 0, maxAttempts: 10, handler: async (payload, job) => { - await this.#completeWaitpoint(this.prisma, payload.waitpointId); + await this.#completeWaitpoint(payload.waitpointId); }, }, "runengine.expireRun": { @@ -171,7 +172,7 @@ export class RunEngine { metadataType, seedMetadata, seedMetadataType, - isWait, + }: TriggerParams, tx?: PrismaClientOrTransaction ) { @@ -223,13 +224,19 @@ export class RunEngine { }); await this.redlock.using([taskRun.id], 5000, async (signal) => { + //todo add this in some places throughout this code + if (signal.aborted) { + throw signal.error; + } + //create associated waitpoint (this completes when the run completes) const associatedWaitpoint = await this.#createRunAssociatedWaitpoint(prisma, { projectId: environment.project.id, completedByTaskRunId: taskRun.id, }); - if (isWait && parentTaskRunId) { + //triggerAndWait or batchTriggerAndWait + if (resumeParentOnCompletion && parentTaskRunId) { //this will block the parent run from continuing until this waitpoint is completed (and removed) await this.#blockRunWithWaitpoint(prisma, { orgId: environment.organization.id, @@ -410,7 +417,11 @@ export class RunEngine { ) { //todo it would be better if we didn't remove from the queue, because this removes the payload //todo better would be to have a "block" function which remove it from the queue but doesn't remove the payload - await this.runQueue.acknowledgeMessage(orgId, runId); + //todo + // await this.runQueue.acknowledgeMessage(orgId, runId); + + //todo release concurrency and make sure the run isn't in the queue + // await this.runQueue.blockMessage(orgId, runId); return tx.taskRunWaitpoint.create({ data: { @@ -421,9 +432,10 @@ export class RunEngine { }); } - /** Any runs blocked by this waitpoint will get continued (if no other waitpoints exist) */ - async #completeWaitpoint(prisma: PrismaClientOrTransaction, id: string) { - const waitpoint = await prisma.waitpoint.findUnique({ + /** This completes a waitpoint and then continues any runs blocked by the waitpoint, + * if they're no longer blocked. This doesn't suffer from race conditions. */ + async #completeWaitpoint(id: string) { + const waitpoint = await this.prisma.waitpoint.findUnique({ where: { id }, }); @@ -436,59 +448,59 @@ export class RunEngine { } await $transaction( - prisma, + this.prisma, async (tx) => { - const blockers = await tx.taskRunWaitpoint.findMany({ + // 1. Find the TaskRuns associated with this waitpoint + const affectedTaskRuns = await tx.taskRunWaitpoint.findMany({ where: { waitpointId: id }, + select: { taskRunId: true }, }); - for (const blocker of blockers) { - //remove the waitpoint - await tx.taskRunWaitpoint.delete({ - where: { id: blocker.id }, - }); - - const blockedRun = await tx.taskRun.findFirst({ - where: { id: blocker.taskRunId }, - include: { - _count: { - select: { - blockedByWaitpoints: true, - }, - }, - runtimeEnvironment: { - select: { - id: true, - type: true, - maximumConcurrencyLimit: true, - project: { - select: { - id: true, - }, - }, - organization: { - select: { - id: true, - }, - }, - }, - }, - }, - }); - - if (blockedRun && blockedRun._count.blockedByWaitpoints === 0) { - await this.enqueueRun(blockedRun, blockedRun.runtimeEnvironment, tx); - } + if (affectedTaskRuns.length === 0) { + throw new Error(`No TaskRunWaitpoints found for waitpoint ${id}`); } + // 2. Delete the TaskRunWaitpoint entries for this specific waitpoint + await tx.taskRunWaitpoint.deleteMany({ + where: { waitpointId: id }, + }); + + // 3. Update the waitpoint status await tx.waitpoint.update({ where: { id }, data: { status: "COMPLETED" }, }); + + // 4. Check which of the affected TaskRuns now have no waitpoints + const taskRunsToResume = await tx.taskRun.findMany({ + where: { + id: { in: affectedTaskRuns.map((run) => run.taskRunId) }, + blockedByWaitpoints: { none: {} }, + status: { in: ["PENDING", "WAITING_TO_RESUME"] }, + }, + include: { + runtimeEnvironment: { + select: { + id: true, + type: true, + maximumConcurrencyLimit: true, + project: { select: { id: true } }, + organization: { select: { id: true } }, + }, + }, + }, + }); + + // 5. Continue the runs that have no more waitpoints + for (const run of taskRunsToResume) { + await this.enqueueRun(run, run.runtimeEnvironment, tx); + } }, (error) => { + this.logger.error(`Error completing waitpoint ${id}, retrying`, { error }); throw error; - } + }, + { isolationLevel: Prisma.TransactionIsolationLevel.ReadCommitted } ); } } From f36b4b96586a899fa8ad5b403510de766dae9af3 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 1 Oct 2024 17:32:41 -0700 Subject: [PATCH 069/114] Started testing triggering --- .../run-engine/src/engine/index.test.ts | 100 +++++++++++++++--- .../run-engine/src/test/containerTest.ts | 43 -------- 2 files changed, 86 insertions(+), 57 deletions(-) diff --git a/internal-packages/run-engine/src/engine/index.test.ts b/internal-packages/run-engine/src/engine/index.test.ts index cf139f8c29..987fa5e42a 100644 --- a/internal-packages/run-engine/src/engine/index.test.ts +++ b/internal-packages/run-engine/src/engine/index.test.ts @@ -1,21 +1,93 @@ import { expect } from "vitest"; -import { postgresTest, redisTest } from "../test/containerTest.js"; +import { containerTest } from "../test/containerTest.js"; +import { RunEngine } from "./index.js"; +import { PrismaClient, RuntimeEnvironmentType } from "@trigger.dev/database"; -postgresTest("Prisma create user", { timeout: 15_000 }, async ({ prisma }) => { - await prisma.user.create({ +describe("RunEngine", () => { + containerTest( + "Trigger a run", + { timeout: 15_000 }, + async ({ postgresContainer, prisma, redisContainer }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + redis: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + enableAutoPipelining: true, + }, + zodWorker: { + connectionString: postgresContainer.getConnectionUri(), + shutdownTimeoutInMs: 100, + }, + }); + + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_1234", + environment: authenticatedEnvironment, + taskIdentifier: "test-task", + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + masterQueue: "main", + queueName: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + expect(run).toBeDefined(); + expect(run.friendlyId).toBe("run_1234"); + } + ); +}); + +async function setupAuthenticatedEnvironment(prisma: PrismaClient, type: RuntimeEnvironmentType) { + // Your database setup logic here + const org = await prisma.organization.create({ data: { - authenticationMethod: "MAGIC_LINK", - email: "test@example.com", + title: "Test Organization", + slug: "test-organization", }, }); - const result = await prisma.user.findMany(); - expect(result.length).toEqual(1); - expect(result[0].email).toEqual("test@example.com"); -}); + const project = await prisma.project.create({ + data: { + name: "Test Project", + slug: "test-project", + externalRef: "proj_1234", + organizationId: org.id, + }, + }); -redisTest("Set/get values", async ({ redis }) => { - await redis.set("mykey", "value"); - const value = await redis.get("mykey"); - expect(value).toEqual("value"); -}); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type, + slug: "slug", + projectId: project.id, + organizationId: org.id, + apiKey: "api_key", + pkApiKey: "pk_api_key", + shortcode: "short_code", + }, + }); + + return await prisma.runtimeEnvironment.findUniqueOrThrow({ + where: { + id: environment.id, + }, + include: { + project: true, + organization: true, + orgMember: true, + }, + }); +} diff --git a/internal-packages/run-engine/src/test/containerTest.ts b/internal-packages/run-engine/src/test/containerTest.ts index e2d9bd1a66..bf61865743 100644 --- a/internal-packages/run-engine/src/test/containerTest.ts +++ b/internal-packages/run-engine/src/test/containerTest.ts @@ -65,46 +65,3 @@ export const containerTest = test.extend({ redisContainer, redis, }); - -export function createContainerWithSetup(setup: (prisma: PrismaClient) => Promise) { - return containerTest.extend<{}>({ - prisma: async ({ prisma }: ContainerContext, use: (prisma: PrismaClient) => Promise) => { - await setup(prisma); - await use(prisma); - }, - }); -} - -async function setupTestDatabase(prisma: PrismaClient): Promise { - // Your database setup logic here - const org = await prisma.organization.create({ - data: { - title: "Test Organization", - slug: "test-organization", - }, - }); - - const project = await prisma.project.create({ - data: { - name: "Test Project", - slug: "test-project", - externalRef: "proj_1234", - organizationId: org.id, - }, - }); - - const environment = await prisma.runtimeEnvironment.create({ - data: { - type: "PRODUCTION", - slug: "prod", - projectId: project.id, - organizationId: org.id, - apiKey: "api_key", - pkApiKey: "pk_api_key", - shortcode: "short_code", - }, - }); -} - -export const containerTestWithAuthenticatedEnvironment = - createContainerWithSetup(setupTestDatabase); From 74d9523156a379f250881c93be0de51cc5dd564b Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 1 Oct 2024 17:50:45 -0700 Subject: [PATCH 070/114] Test for a run getting triggered and being enqueued --- .../run-engine/src/engine/index.test.ts | 20 +++++++++++++++++++ .../run-engine/src/engine/index.ts | 3 +-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/internal-packages/run-engine/src/engine/index.test.ts b/internal-packages/run-engine/src/engine/index.test.ts index 987fa5e42a..4e917d7782 100644 --- a/internal-packages/run-engine/src/engine/index.test.ts +++ b/internal-packages/run-engine/src/engine/index.test.ts @@ -46,6 +46,26 @@ describe("RunEngine", () => { expect(run).toBeDefined(); expect(run.friendlyId).toBe("run_1234"); + + //check the waitpoint is created + const runWaitpoint = await prisma.waitpoint.findMany({ + where: { + completedByTaskRunId: run.id, + }, + }); + expect(runWaitpoint.length).toBe(1); + expect(runWaitpoint[0].type).toBe("RUN"); + + //check the queue length + const queueLength = await engine.runQueue.lengthOfQueue(authenticatedEnvironment, run.queue); + expect(queueLength).toBe(1); + + //dequeue the run + const dequeued = await engine.runQueue.dequeueMessageInSharedQueue( + "test_12345", + run.masterQueue + ); + expect(dequeued?.messageId).toBe(run.id); } ); }); diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 37d8c5e300..a1c9340622 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -79,7 +79,7 @@ export class RunEngine { private redis: Redis; private prisma: PrismaClient; private redlock: Redlock; - private runQueue: RunQueue; + runQueue: RunQueue; private zodWorker: EngineWorker; private logger = new Logger("RunEngine", "debug"); @@ -172,7 +172,6 @@ export class RunEngine { metadataType, seedMetadata, seedMetadataType, - }: TriggerParams, tx?: PrismaClientOrTransaction ) { From d632d48028f79040d3aad73e971fb0c09f8380c9 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Wed, 2 Oct 2024 10:09:30 -0700 Subject: [PATCH 071/114] Removed dequeueMessageInEnv --- .../run-engine/src/engine/index.test.ts | 39 +++++++++++- .../run-engine/src/run-queue/index.ts | 59 ------------------- 2 files changed, 38 insertions(+), 60 deletions(-) diff --git a/internal-packages/run-engine/src/engine/index.test.ts b/internal-packages/run-engine/src/engine/index.test.ts index 4e917d7782..85a59541cd 100644 --- a/internal-packages/run-engine/src/engine/index.test.ts +++ b/internal-packages/run-engine/src/engine/index.test.ts @@ -5,7 +5,7 @@ import { PrismaClient, RuntimeEnvironmentType } from "@trigger.dev/database"; describe("RunEngine", () => { containerTest( - "Trigger a run", + "Trigger a simple run", { timeout: 15_000 }, async ({ postgresContainer, prisma, redisContainer }) => { const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); @@ -47,6 +47,15 @@ describe("RunEngine", () => { expect(run).toBeDefined(); expect(run.friendlyId).toBe("run_1234"); + //check it's actually in the db + const runFromDb = await prisma.taskRun.findUnique({ + where: { + friendlyId: "run_1234", + }, + }); + expect(runFromDb).toBeDefined(); + expect(runFromDb?.id).toBe(run.id); + //check the waitpoint is created const runWaitpoint = await prisma.waitpoint.findMany({ where: { @@ -60,14 +69,41 @@ describe("RunEngine", () => { const queueLength = await engine.runQueue.lengthOfQueue(authenticatedEnvironment, run.queue); expect(queueLength).toBe(1); + //concurrency before + const envConcurrencyBefore = await engine.runQueue.currentConcurrencyOfEnvironment( + authenticatedEnvironment + ); + expect(envConcurrencyBefore).toBe(0); + //dequeue the run const dequeued = await engine.runQueue.dequeueMessageInSharedQueue( "test_12345", run.masterQueue ); expect(dequeued?.messageId).toBe(run.id); + + const envConcurrencyAfter = await engine.runQueue.currentConcurrencyOfEnvironment( + authenticatedEnvironment + ); + expect(envConcurrencyAfter).toBe(1); } ); + + //todo triggerAndWait + + //todo batchTriggerAndWait + + //todo checkpoints + + //todo heartbeats + + //todo failing a run + + //todo cancelling a run + + //todo expiring a run + + //todo delaying a run }); async function setupAuthenticatedEnvironment(prisma: PrismaClient, type: RuntimeEnvironmentType) { @@ -97,6 +133,7 @@ async function setupAuthenticatedEnvironment(prisma: PrismaClient, type: Runtime apiKey: "api_key", pkApiKey: "pk_api_key", shortcode: "short_code", + maximumConcurrencyLimit: 10, }, }); diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 5294537c7d..cb07054051 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -200,65 +200,6 @@ export class RunQueue { ); } - public async dequeueMessageInEnv(env: MinimalAuthenticatedEnvironment, masterQueue: string) { - return this.#trace( - "dequeueMessageInEnv", - async (span) => { - // Read the parent queue for matching queues - const messageQueue = await this.#getRandomQueueFromParentQueue( - masterQueue, - this.options.envQueuePriorityStrategy, - (queue) => this.#calculateMessageQueueCapacities(queue, { checkForDisabled: false }), - env.id - ); - - if (!messageQueue) { - this.logger.debug("No message queue found", { - masterQueue, - }); - - return; - } - - const message = await this.#callDequeueMessage({ - messageQueue, - masterQueue, - concurrencyLimitKey: this.keys.concurrencyLimitKeyFromQueue(messageQueue), - currentConcurrencyKey: this.keys.currentConcurrencyKeyFromQueue(messageQueue), - envConcurrencyLimitKey: this.keys.envConcurrencyLimitKeyFromQueue(messageQueue), - envCurrentConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(messageQueue), - projectCurrentConcurrencyKey: - this.keys.projectCurrentConcurrencyKeyFromQueue(messageQueue), - messageKeyPrefix: this.keys.messageKeyPrefixFromQueue(messageQueue), - taskCurrentConcurrentKeyPrefix: - this.keys.taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(messageQueue), - }); - - if (!message) { - return; - } - - span.setAttributes({ - [SEMATTRS_MESSAGE_ID]: message.messageId, - [SemanticAttributes.QUEUE]: message.message.queue, - [SemanticAttributes.RUN_ID]: message.message.runId, - [SemanticAttributes.CONCURRENCY_KEY]: message.message.concurrencyKey, - [SemanticAttributes.MASTER_QUEUE]: masterQueue, - }); - - return message; - }, - { - kind: SpanKind.CONSUMER, - attributes: { - [SEMATTRS_MESSAGING_OPERATION]: "receive", - [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", - ...attributesFromAuthenticatedEnv(env), - }, - } - ); - } - public async getSharedQueueDetails(masterQueue: string) { const { range } = await this.queuePriorityStrategy.nextCandidateSelection( masterQueue, From ad864d0abb216fc203689714e7540cd07e62cb8e Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Wed, 2 Oct 2024 11:25:30 -0700 Subject: [PATCH 072/114] Update dev queue tests to use the shared queue function --- .../run-engine/src/run-queue/index.test.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/index.test.ts b/internal-packages/run-engine/src/run-queue/index.test.ts index 5e96c439be..7685dafec1 100644 --- a/internal-packages/run-engine/src/run-queue/index.test.ts +++ b/internal-packages/run-engine/src/run-queue/index.test.ts @@ -147,11 +147,13 @@ describe("RunQueue", () => { const oldestScore = await queue.oldestMessageInQueue(authenticatedEnvDev, messageDev.queue); expect(oldestScore).toBe(undefined); + const envMasterQueue = `env:${authenticatedEnvDev.id}`; + //enqueue message await queue.enqueueMessage({ env: authenticatedEnvDev, message: messageDev, - masterQueue: "test_12345", + masterQueue: `env:${authenticatedEnvDev.id}`, }); //queue length @@ -181,11 +183,11 @@ describe("RunQueue", () => { ); expect(taskConcurrency).toBe(0); - const dequeued = await queue.dequeueMessageInEnv(authenticatedEnvDev, "test_12345"); + const dequeued = await queue.dequeueMessageInSharedQueue("test_12345", envMasterQueue); expect(dequeued?.messageId).toEqual(messageDev.runId); expect(dequeued?.message.orgId).toEqual(messageDev.orgId); expect(dequeued?.message.version).toEqual("1"); - expect(dequeued?.message.masterQueue).toEqual("test_12345"); + expect(dequeued?.message.masterQueue).toEqual(envMasterQueue); //concurrencies const queueConcurrency2 = await queue.currentConcurrencyOfQueue( @@ -203,7 +205,7 @@ describe("RunQueue", () => { ); expect(taskConcurrency2).toBe(1); - const dequeued2 = await queue.dequeueMessageInEnv(authenticatedEnvDev, "test_12345"); + const dequeued2 = await queue.dequeueMessageInSharedQueue("test_12345", envMasterQueue); expect(dequeued2).toBe(undefined); } finally { await queue.quit(); @@ -293,7 +295,7 @@ describe("RunQueue", () => { const length2 = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); expect(length2).toBe(0); - const dequeued2 = await queue.dequeueMessageInEnv(authenticatedEnvDev, "main"); + const dequeued2 = await queue.dequeueMessageInSharedQueue("test_12345", "main"); expect(dequeued2).toBe(undefined); } finally { await queue.quit(); From e6cb27dad1100b50092eda4f8aaa6bfbf62ddeb3 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Wed, 2 Oct 2024 14:16:52 -0700 Subject: [PATCH 073/114] Schema changes for TaskRunExecutionSnapshot --- .../database/prisma/schema.prisma | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 6c80ca9f8b..ad2fd66be2 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -1857,39 +1857,49 @@ enum RunEngineVersion { /// It has the required information to transactionally progress a run through states, /// and prevent side effects like heartbeats failing a run that has progressed. /// It is optimised for performance and is designed to be cleared at some point, -/// so there are no foreign key relationships to other models. +/// so there are no cascading relationships to other models. model TaskRunExecutionSnapshot { id String @id @default(cuid()) + /// This should never be V1 engine RunEngineVersion @default(V2) - runId String - run TaskRun @relation(fields: [runId], references: [id]) + /// The execution status + executionStatus TaskRunExecutionStatus + /// For debugging + description String + /// Run + runId String + run TaskRun @relation(fields: [runId], references: [id]) runStatus TaskRunStatus - currentAttemptId String? - currentAttempt TaskAttempt? @relation(fields: [currentAttemptId], references: [id]) - + /// Attempt + currentAttemptId String? + currentAttempt TaskAttempt? @relation(fields: [currentAttemptId], references: [id]) currentAttemptStatus TaskAttemptStatus? - /// When set, we can clear the state - completedAt DateTime? + /// todo Checkpoint /// These are only ever appended, so we don't need updatedAt createdAt DateTime @default(now()) ///todo machine spec? - ///worker - - /// For debugging - description String + ///todo worker /// Used to get the latest state quickly @@index([runId, createdAt(sort: Desc)]) } +enum TaskRunExecutionStatus { + RUN_CREATED + DEQUEUED_FOR_EXECUTION + EXECUTING + BLOCKED_BY_WAITPOINTS + FINISHED +} + /// A Waitpoint blocks a run from continuing until it's completed /// If there's a waitpoint blocking a run, it shouldn't be in the queue model Waitpoint { From 8a78b14b55293465e6816883ffd5b50edbf28406 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Wed, 2 Oct 2024 14:17:12 -0700 Subject: [PATCH 074/114] First execution snapshot when the run is created. Dequeue run function added to the engine --- .../run-engine/src/engine/index.ts | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index a1c9340622..f74d11ee91 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -135,7 +135,7 @@ export class RunEngine { //MARK: - Run functions - /** "Triggers" one run, which creates the run + /** "Triggers" one run. */ async trigger( { @@ -177,10 +177,12 @@ export class RunEngine { ) { const prisma = tx ?? this.prisma; + const status = delayUntil ? "DELAYED" : "PENDING"; + //create run const taskRun = await prisma.taskRun.create({ data: { - status: delayUntil ? "DELAYED" : "PENDING", + status, number, friendlyId, runtimeEnvironmentId: environment.id, @@ -219,6 +221,14 @@ export class RunEngine { metadataType, seedMetadata, seedMetadataType, + executionSnapshot: { + create: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: status, + }, + }, }, }); @@ -356,6 +366,13 @@ export class RunEngine { //todo update the TaskRunExecutionSnapshot } + async dequeueRun(consumerId: string, masterQueue: string) { + const message = await this.runQueue.dequeueMessageInSharedQueue(consumerId, masterQueue); + //todo update the TaskRunExecutionSnapshot + //todo update the TaskRun status? + return message; + } + /** We want to actually execute the run, this could be a continuation of a previous execution. * This is called from the queue, when the run has been pulled. */ //todo think more about this, when do we create the attempt? From 9927cdc6b2b0d52fd188b31d5be93bee646711ee Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 3 Oct 2024 15:16:26 -0700 Subject: [PATCH 075/114] Separate internal package for testcontainers so they can be used elsewhere --- internal-packages/run-engine/package.json | 1 + .../run-engine/src/engine/index.test.ts | 2 +- .../run-engine/src/run-queue/index.test.ts | 2 +- internal-packages/run-engine/tsconfig.json | 2 + internal-packages/testcontainers/README.md | 10 +++ internal-packages/testcontainers/package.json | 22 ++++++ internal-packages/testcontainers/src/index.ts | 67 +++++++++++++++++++ internal-packages/testcontainers/src/utils.ts | 35 ++++++++++ .../testcontainers/tsconfig.json | 25 +++++++ .../testcontainers/vitest.config.ts | 8 +++ 10 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 internal-packages/testcontainers/README.md create mode 100644 internal-packages/testcontainers/package.json create mode 100644 internal-packages/testcontainers/src/index.ts create mode 100644 internal-packages/testcontainers/src/utils.ts create mode 100644 internal-packages/testcontainers/tsconfig.json create mode 100644 internal-packages/testcontainers/vitest.config.ts diff --git a/internal-packages/run-engine/package.json b/internal-packages/run-engine/package.json index 7b9492d053..ecf9760883 100644 --- a/internal-packages/run-engine/package.json +++ b/internal-packages/run-engine/package.json @@ -5,6 +5,7 @@ "main": "./src/index.ts", "types": "./src/index.ts", "dependencies": { + "@internal/testcontainers": "workspace:*", "@internal/zod-worker": "workspace:*", "@opentelemetry/api": "^1.9.0", "@opentelemetry/semantic-conventions": "^1.27.0", diff --git a/internal-packages/run-engine/src/engine/index.test.ts b/internal-packages/run-engine/src/engine/index.test.ts index 85a59541cd..64bfe6c06b 100644 --- a/internal-packages/run-engine/src/engine/index.test.ts +++ b/internal-packages/run-engine/src/engine/index.test.ts @@ -1,5 +1,5 @@ import { expect } from "vitest"; -import { containerTest } from "../test/containerTest.js"; +import { containerTest } from "@internal/testcontainers"; import { RunEngine } from "./index.js"; import { PrismaClient, RuntimeEnvironmentType } from "@trigger.dev/database"; diff --git a/internal-packages/run-engine/src/run-queue/index.test.ts b/internal-packages/run-engine/src/run-queue/index.test.ts index 7685dafec1..8a3f40ada9 100644 --- a/internal-packages/run-engine/src/run-queue/index.test.ts +++ b/internal-packages/run-engine/src/run-queue/index.test.ts @@ -1,7 +1,7 @@ import { trace } from "@opentelemetry/api"; import { Logger } from "@trigger.dev/core/logger"; import { describe } from "node:test"; -import { redisTest } from "../test/containerTest.js"; +import { redisTest } from "@internal/testcontainers"; import { RunQueue } from "./index.js"; import { RunQueueShortKeyProducer } from "./keyProducer.js"; import { SimpleWeightedChoiceStrategy } from "./simpleWeightedPriorityStrategy.js"; diff --git a/internal-packages/run-engine/tsconfig.json b/internal-packages/run-engine/tsconfig.json index fb691b947e..096276b211 100644 --- a/internal-packages/run-engine/tsconfig.json +++ b/internal-packages/run-engine/tsconfig.json @@ -15,6 +15,8 @@ "noEmit": true, "strict": true, "paths": { + "@internal/testcontainers": ["../../internal-packages/testcontainers/src/index"], + "@internal/testcontainers/*": ["../../internal-packages/testcontainers/src/*"], "@internal/zod-worker": ["../../internal-packages/zod-worker/src/index"], "@internal/zod-worker/*": ["../../internal-packages/zod-worker/src/*"], "@trigger.dev/core": ["../../packages/core/src/index"], diff --git a/internal-packages/testcontainers/README.md b/internal-packages/testcontainers/README.md new file mode 100644 index 0000000000..c6547d1bed --- /dev/null +++ b/internal-packages/testcontainers/README.md @@ -0,0 +1,10 @@ +# Redis worker + +This is a simple worker that pulls tasks from a Redis queue (also in this package). + +Features + +- Configurable settings for concurrency and pull speed. +- Job payloads. +- A schema so only defined jobs can be added to the queue. +- The ability to have future dates for jobs. diff --git a/internal-packages/testcontainers/package.json b/internal-packages/testcontainers/package.json new file mode 100644 index 0000000000..dc147eec3e --- /dev/null +++ b/internal-packages/testcontainers/package.json @@ -0,0 +1,22 @@ +{ + "name": "@internal/testcontainers", + "private": true, + "version": "0.0.1", + "main": "./src/index.ts", + "types": "./src/index.ts", + "dependencies": { + "@opentelemetry/api": "^1.9.0", + "@trigger.dev/database": "workspace:*", + "ioredis": "^5.3.2", + "typescript": "^4.8.4" + }, + "devDependencies": { + "@testcontainers/postgresql": "^10.13.1", + "@testcontainers/redis": "^10.13.1", + "testcontainers": "^10.13.1", + "vitest": "^1.4.0" + }, + "scripts": { + "typecheck": "tsc --noEmit" + } +} diff --git a/internal-packages/testcontainers/src/index.ts b/internal-packages/testcontainers/src/index.ts new file mode 100644 index 0000000000..24d734c772 --- /dev/null +++ b/internal-packages/testcontainers/src/index.ts @@ -0,0 +1,67 @@ +import { StartedPostgreSqlContainer } from "@testcontainers/postgresql"; +import { StartedRedisContainer } from "@testcontainers/redis"; +import { Redis } from "ioredis"; +import { test } from "vitest"; +import { PrismaClient } from "@trigger.dev/database"; +import { createPostgresContainer, createRedisContainer } from "./utils"; + +type PostgresContext = { + postgresContainer: StartedPostgreSqlContainer; + prisma: PrismaClient; +}; + +type RedisContext = { redisContainer: StartedRedisContainer; redis: Redis }; +type ContainerContext = PostgresContext & RedisContext; + +type Use = (value: T) => Promise; + +const postgresContainer = async ({}, use: Use) => { + const { container } = await createPostgresContainer(); + await use(container); + await container.stop(); +}; + +const prisma = async ( + { postgresContainer }: { postgresContainer: StartedPostgreSqlContainer }, + use: Use +) => { + const prisma = new PrismaClient({ + datasources: { + db: { + url: postgresContainer.getConnectionUri(), + }, + }, + }); + await use(prisma); + await prisma.$disconnect(); +}; + +export const postgresTest = test.extend({ postgresContainer, prisma }); + +const redisContainer = async ({}, use: Use) => { + const { container } = await createRedisContainer(); + await use(container); + await container.stop(); +}; + +const redis = async ( + { redisContainer }: { redisContainer: StartedRedisContainer }, + use: Use +) => { + const redis = new Redis({ + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + }); + await use(redis); + await redis.quit(); +}; + +export const redisTest = test.extend({ redisContainer, redis }); + +export const containerTest = test.extend({ + postgresContainer, + prisma, + redisContainer, + redis, +}); diff --git a/internal-packages/testcontainers/src/utils.ts b/internal-packages/testcontainers/src/utils.ts new file mode 100644 index 0000000000..6140dc2266 --- /dev/null +++ b/internal-packages/testcontainers/src/utils.ts @@ -0,0 +1,35 @@ +import { PostgreSqlContainer } from "@testcontainers/postgresql"; +import { RedisContainer } from "@testcontainers/redis"; +import { execSync } from "child_process"; +import path from "path"; + +export async function createPostgresContainer() { + const container = await new PostgreSqlContainer().start(); + + // Run migrations + const databasePath = path.resolve(__dirname, "../../../database"); + + execSync(`npx prisma db push --schema ${databasePath}/prisma/schema.prisma`, { + env: { + ...process.env, + DATABASE_URL: container.getConnectionUri(), + DIRECT_URL: container.getConnectionUri(), + }, + }); + + // console.log(container.getConnectionUri()); + + return { url: container.getConnectionUri(), container }; +} + +export async function createRedisContainer() { + const container = await new RedisContainer().start(); + try { + return { + container, + }; + } catch (e) { + console.error(e); + throw e; + } +} diff --git a/internal-packages/testcontainers/tsconfig.json b/internal-packages/testcontainers/tsconfig.json new file mode 100644 index 0000000000..ce63734a69 --- /dev/null +++ b/internal-packages/testcontainers/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "target": "ES2019", + "lib": ["ES2019", "DOM", "DOM.Iterable"], + "module": "CommonJS", + "moduleResolution": "Node10", + "moduleDetection": "force", + "verbatimModuleSyntax": false, + "types": ["vitest/globals"], + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "preserveWatchOutput": true, + "skipLibCheck": true, + "noEmit": true, + "strict": true, + "paths": { + "@trigger.dev/core": ["../../packages/core/src/index"], + "@trigger.dev/core/*": ["../../packages/core/src/*"], + "@trigger.dev/database": ["../../internal-packages/database/src/index"], + "@trigger.dev/database/*": ["../../internal-packages/database/src/*"] + } + }, + "exclude": ["node_modules"] +} diff --git a/internal-packages/testcontainers/vitest.config.ts b/internal-packages/testcontainers/vitest.config.ts new file mode 100644 index 0000000000..4afd926425 --- /dev/null +++ b/internal-packages/testcontainers/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["**/*.test.ts"], + globals: true, + }, +}); From ae1c23d700ef1c1094a05ce777566e04417c74cb Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 3 Oct 2024 15:16:57 -0700 Subject: [PATCH 076/114] =?UTF-8?q?Remove=20the=20simple=20queue=20and=20t?= =?UTF-8?q?estcontainers=20from=20the=20run-engine.=20They=E2=80=99re=20go?= =?UTF-8?q?ing=20to=20be=20separate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../run-engine/src/simple-queue/index.test.ts | 90 ----------- .../run-engine/src/simple-queue/index.ts | 149 ------------------ .../src/simple-queue/processor.test.ts | 92 ----------- .../run-engine/src/simple-queue/processor.ts | 144 ----------------- .../run-engine/src/test/containerTest.ts | 67 -------- .../run-engine/src/test/utils.ts | 36 ----- 6 files changed, 578 deletions(-) delete mode 100644 internal-packages/run-engine/src/simple-queue/index.test.ts delete mode 100644 internal-packages/run-engine/src/simple-queue/index.ts delete mode 100644 internal-packages/run-engine/src/simple-queue/processor.test.ts delete mode 100644 internal-packages/run-engine/src/simple-queue/processor.ts delete mode 100644 internal-packages/run-engine/src/test/containerTest.ts delete mode 100644 internal-packages/run-engine/src/test/utils.ts diff --git a/internal-packages/run-engine/src/simple-queue/index.test.ts b/internal-packages/run-engine/src/simple-queue/index.test.ts deleted file mode 100644 index 85918bec81..0000000000 --- a/internal-packages/run-engine/src/simple-queue/index.test.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { describe } from "node:test"; -import { expect } from "vitest"; -import { z } from "zod"; -import { redisTest } from "../test/containerTest.js"; -import { SimpleQueue } from "./index.js"; - -describe("SimpleQueue", () => { - redisTest("enqueue/dequeue", { timeout: 20_000 }, async ({ redisContainer }) => { - const queue = new SimpleQueue( - "test-1", - z.object({ - value: z.number(), - }), - { - host: redisContainer.getHost(), - port: redisContainer.getPort(), - password: redisContainer.getPassword(), - } - ); - - try { - await queue.enqueue("1", { value: 1 }); - await queue.enqueue("2", { value: 2 }); - - const first = await queue.dequeue(); - expect(first).toEqual({ id: "1", item: { value: 1 } }); - - const second = await queue.dequeue(); - expect(second).toEqual({ id: "2", item: { value: 2 } }); - } finally { - await queue.close(); - } - }); - - redisTest("no items", { timeout: 20_000 }, async ({ redisContainer }) => { - const queue = new SimpleQueue( - "test-2", - z.object({ - value: z.number(), - }), - { - host: redisContainer.getHost(), - port: redisContainer.getPort(), - password: redisContainer.getPassword(), - } - ); - - try { - const missOne = await queue.dequeue(); - expect(missOne).toBeNull(); - - await queue.enqueue("1", { value: 1 }); - const hitOne = await queue.dequeue(); - expect(hitOne).toEqual({ id: "1", item: { value: 1 } }); - - const missTwo = await queue.dequeue(); - expect(missTwo).toBeNull(); - } finally { - await queue.close(); - } - }); - - redisTest("future item", { timeout: 20_000 }, async ({ redisContainer }) => { - const queue = new SimpleQueue( - "test-3", - z.object({ - value: z.number(), - }), - { - host: redisContainer.getHost(), - port: redisContainer.getPort(), - password: redisContainer.getPassword(), - } - ); - - try { - await queue.enqueue("1", { value: 1 }, new Date(Date.now() + 50)); - - const miss = await queue.dequeue(); - expect(miss).toBeNull(); - - await new Promise((resolve) => setTimeout(resolve, 50)); - - const first = await queue.dequeue(); - expect(first).toEqual({ id: "1", item: { value: 1 } }); - } finally { - await queue.close(); - } - }); -}); diff --git a/internal-packages/run-engine/src/simple-queue/index.ts b/internal-packages/run-engine/src/simple-queue/index.ts deleted file mode 100644 index 6c1bb8351e..0000000000 --- a/internal-packages/run-engine/src/simple-queue/index.ts +++ /dev/null @@ -1,149 +0,0 @@ -import Redis, { RedisOptions } from "ioredis"; -import { z } from "zod"; -import { Logger } from "@trigger.dev/core/logger"; - -export class SimpleQueue { - name: string; - private redis: Redis; - private schema: T; - private logger: Logger; - - constructor(name: string, schema: T, redisOptions: RedisOptions, logger?: Logger) { - this.name = name; - this.redis = new Redis({ - ...redisOptions, - keyPrefix: `queue:${name}:`, - retryStrategy(times) { - const delay = Math.min(times * 50, 1000); - return delay; - }, - maxRetriesPerRequest: 3, - }); - this.schema = schema; - - this.logger = logger ?? new Logger("SimpleQueue", "debug"); - - this.redis.on("error", (error) => { - this.logger.error(`Redis Error for queue ${this.name}:`, { queue: this.name, error }); - }); - - this.redis.on("connect", () => { - // this.logger.log(`Redis connected for queue ${this.name}`); - }); - } - - async enqueue(id: string, item: z.infer, availableAt?: Date): Promise { - try { - const score = availableAt ? availableAt.getTime() : Date.now(); - const serializedItem = JSON.stringify(item); - - const result = await this.redis - .multi() - .zadd(`queue`, score, id) - .hset(`items`, id, serializedItem) - .exec(); - - if (!result) { - throw new Error("Redis multi command returned null"); - } - - result.forEach((res, index) => { - if (res[0]) { - throw new Error(`Redis operation ${index} failed: ${res[0]}`); - } - }); - } catch (e) { - this.logger.error(`SimpleQueue ${this.name}.enqueue(): error enqueuing`, { - queue: this.name, - error: e, - id, - item, - }); - throw e; - } - } - - async dequeue(): Promise<{ id: string; item: z.infer } | null> { - const now = Date.now(); - - try { - const result = await this.redis - .multi() - .zrangebyscore(`queue`, "-inf", now, "WITHSCORES", "LIMIT", 0, 1) - .exec(); - - if (!result) { - throw new Error("Redis multi command returned null"); - } - - result.forEach((res, index) => { - if (res[0]) { - throw new Error(`Redis operation ${index} failed: ${res[0]}`); - } - }); - - if (!result[0][1] || (result[0][1] as string[]).length === 0) { - return null; - } - - const [id, score] = result[0][1] as string[]; - - // Check if the item is available now - if (parseInt(score) > now) { - return null; - } - - // Remove the item from the sorted set - await this.redis.zrem(`queue`, id); - - const serializedItem = await this.redis.hget(`items`, id); - - if (!serializedItem) { - this.logger.warn(`Item ${id} not found in hash, might have been deleted`, { - queue: this.name, - id, - }); - return null; - } - - await this.redis.hdel(`items`, id); - const parsedItem = JSON.parse(serializedItem); - const validatedItem = this.schema.safeParse(parsedItem); - - if (!validatedItem.success) { - this.logger.error("Invalid item in queue", { - queue: this.name, - id, - item: parsedItem, - errors: validatedItem.error, - }); - return null; - } - - return { id, item: validatedItem.data }; - } catch (e) { - this.logger.error(`SimpleQueue ${this.name}.dequeue(): error dequeuing`, { - queue: this.name, - error: e, - }); - throw e; - } - } - - async size(): Promise { - try { - const result = await this.redis.zcard(`queue`); - return result; - } catch (e) { - this.logger.error(`SimpleQueue ${this.name}.size(): error getting queue size`, { - queue: this.name, - error: e, - }); - throw e; - } - } - - async close(): Promise { - await this.redis.quit(); - } -} diff --git a/internal-packages/run-engine/src/simple-queue/processor.test.ts b/internal-packages/run-engine/src/simple-queue/processor.test.ts deleted file mode 100644 index 2904f8a5d5..0000000000 --- a/internal-packages/run-engine/src/simple-queue/processor.test.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { expect, it } from "vitest"; -import { z } from "zod"; -import { redisTest } from "../test/containerTest.js"; -import { SimpleQueue } from "./index.js"; -import { describe } from "node:test"; -import { createQueueProcessor } from "./processor.js"; -import { Logger } from "@trigger.dev/core/logger"; - -describe("SimpleQueue processor", () => { - redisTest("Read items", { timeout: 20_000 }, async ({ redisContainer }) => { - const queue = new SimpleQueue( - "processor-1", - z.object({ - value: z.number(), - }), - { - host: redisContainer.getHost(), - port: redisContainer.getPort(), - password: redisContainer.getPassword(), - } - ); - - try { - let itemCount = 10; - for (let i = 0; i < itemCount; i++) { - await queue.enqueue(i.toString(), { value: i }); - } - - let itemsProcessed = 0; - - const processor = createQueueProcessor(queue, { - onItem: async (id, item) => { - expect(item).toEqual({ value: parseInt(id) }); - itemsProcessed++; - }, - }); - - processor.start(); - await new Promise((resolve) => setTimeout(resolve, 200)); - expect(itemsProcessed).toEqual(itemCount); - processor.stop(); - } finally { - await queue.close(); - } - }); - - redisTest("Retrying", { timeout: 20_000 }, async ({ redisContainer }) => { - const queue = new SimpleQueue( - "processor-2", - z.object({ - value: z.number(), - }), - { - host: redisContainer.getHost(), - port: redisContainer.getPort(), - password: redisContainer.getPassword(), - } - ); - - try { - await queue.enqueue("1", { value: 1 }); - - let attempts = 0; - let itemsProcessed = 0; - - const processor = createQueueProcessor(queue, { - retry: { - delay: { - initial: 10, - factor: 1, - }, - maxAttempts: 2, - }, - onItem: async (id, item) => { - attempts++; - if (attempts === 1) { - throw new Error("Test retry"); - } - expect(item).toEqual({ value: parseInt(id) }); - itemsProcessed++; - }, - }); - - processor.start(); - await new Promise((resolve) => setTimeout(resolve, 2_000)); - expect(itemsProcessed).toEqual(1); - processor.stop(); - } finally { - await queue.close(); - } - }); -}); diff --git a/internal-packages/run-engine/src/simple-queue/processor.ts b/internal-packages/run-engine/src/simple-queue/processor.ts deleted file mode 100644 index a0c5aa0862..0000000000 --- a/internal-packages/run-engine/src/simple-queue/processor.ts +++ /dev/null @@ -1,144 +0,0 @@ -import { z } from "zod"; -import { SimpleQueue } from "./index"; -import { Logger } from "@trigger.dev/core/logger"; - -type QueueProcessorOptions = { - timeout?: number; - retry?: { - delay?: { - initial?: number; - max?: number; - factor?: number; - }; - maxAttempts?: number; - }; - logger?: Logger; - onItem: (id: string, item: z.infer) => Promise | void; -}; - -const defaultRetryOptions = { - delay: { - initial: 1000, - max: 10000, - factor: 2, - }, - maxAttempts: 10, -}; - -export function createQueueProcessor( - queue: SimpleQueue, - options: QueueProcessorOptions -) { - let { timeout = 1000, onItem, logger = new Logger("QueueProcessor", "debug") } = options; - - const retry = deepMerge(defaultRetryOptions, options.retry ?? {}); - - const failures = new Map(); - let isRunning = false; - - async function processQueue() { - if (!isRunning) return; - - try { - const result = await queue.dequeue(); - if (result) { - const { id, item } = result; - try { - await onItem(id, item); - } catch (error) { - logger.warn("Error processing item:", { error, id, item, queue: queue.name }); - - const retryCount = failures.get(id) || 0; - if (retryCount >= retry.maxAttempts) { - logger.error(`QueueProcessor: max attempts reached for item ${id}`, { - queue: queue.name, - id, - item, - }); - return; - } - - //requeue with delay - const delay = Math.min( - retry.delay.initial * Math.pow(retry.delay.factor, retryCount), - retry.delay.max - ); - logger.log(`QueueProcessor: requeueing item`, { item, id, delay, queue: queue.name }); - await queue.enqueue(id, item, new Date(Date.now() + delay)); - - failures.set(id, retryCount + 1); - } - // Continue processing immediately if still running - if (isRunning) { - setImmediate(processQueue); - } - } else { - // No item found, wait before checking again if still running - if (isRunning) { - setTimeout(processQueue, timeout); - } - } - } catch (error) { - logger.error("Error processing queue:", { error }); - setTimeout(processQueue, timeout); - } - } - - return { - start: () => { - if (!isRunning) { - logger.log("Starting queue processor..."); - isRunning = true; - processQueue(); - } else { - logger.log("Queue processor is already running."); - } - }, - stop: () => { - if (isRunning) { - logger.log("Stopping queue processor..."); - isRunning = false; - } else { - logger.log("Queue processor is already stopped."); - } - }, - }; -} - -type DeepPartial = T extends object - ? { - [P in keyof T]?: DeepPartial; - } - : T; - -function isObject(item: unknown): item is Record { - return typeof item === "object" && item !== null && !Array.isArray(item); -} - -function deepMerge(target: T, source: DeepPartial): T { - if (!isObject(target) || !isObject(source)) { - return source as T; - } - - const output = { ...target } as T; - - (Object.keys(source) as Array).forEach((key) => { - if (key in target) { - const targetValue = target[key]; - const sourceValue = source[key]; - - if (isObject(targetValue) && isObject(sourceValue)) { - (output as any)[key] = deepMerge( - targetValue, - sourceValue as DeepPartial - ); - } else if (sourceValue !== undefined) { - (output as any)[key] = sourceValue; - } - } else if (source[key as keyof DeepPartial] !== undefined) { - (output as any)[key] = source[key as keyof DeepPartial]; - } - }); - - return output; -} diff --git a/internal-packages/run-engine/src/test/containerTest.ts b/internal-packages/run-engine/src/test/containerTest.ts deleted file mode 100644 index bf61865743..0000000000 --- a/internal-packages/run-engine/src/test/containerTest.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { StartedPostgreSqlContainer } from "@testcontainers/postgresql"; -import { StartedRedisContainer } from "@testcontainers/redis"; -import { Redis } from "ioredis"; -import { test } from "vitest"; -import { PrismaClient } from "../../../database/src"; -import { createPostgresContainer, createRedisContainer } from "./utils"; - -type PostgresContext = { - postgresContainer: StartedPostgreSqlContainer; - prisma: PrismaClient; -}; - -type RedisContext = { redisContainer: StartedRedisContainer; redis: Redis }; -type ContainerContext = PostgresContext & RedisContext; - -type Use = (value: T) => Promise; - -const postgresContainer = async ({}, use: Use) => { - const { container } = await createPostgresContainer(); - await use(container); - await container.stop(); -}; - -const prisma = async ( - { postgresContainer }: { postgresContainer: StartedPostgreSqlContainer }, - use: Use -) => { - const prisma = new PrismaClient({ - datasources: { - db: { - url: postgresContainer.getConnectionUri(), - }, - }, - }); - await use(prisma); - await prisma.$disconnect(); -}; - -export const postgresTest = test.extend({ postgresContainer, prisma }); - -const redisContainer = async ({}, use: Use) => { - const { container } = await createRedisContainer(); - await use(container); - await container.stop(); -}; - -const redis = async ( - { redisContainer }: { redisContainer: StartedRedisContainer }, - use: Use -) => { - const redis = new Redis({ - host: redisContainer.getHost(), - port: redisContainer.getPort(), - password: redisContainer.getPassword(), - }); - await use(redis); - await redis.quit(); -}; - -export const redisTest = test.extend({ redisContainer, redis }); - -export const containerTest = test.extend({ - postgresContainer, - prisma, - redisContainer, - redis, -}); diff --git a/internal-packages/run-engine/src/test/utils.ts b/internal-packages/run-engine/src/test/utils.ts deleted file mode 100644 index e998f7afa9..0000000000 --- a/internal-packages/run-engine/src/test/utils.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { PostgreSqlContainer } from "@testcontainers/postgresql"; -import { RedisContainer } from "@testcontainers/redis"; -import { PrismaClient } from "../../../database/src"; -import { execSync } from "child_process"; -import path from "path"; - -export async function createPostgresContainer() { - const container = await new PostgreSqlContainer().start(); - - // Run migrations - const databasePath = path.resolve(__dirname, "../../../database"); - - execSync(`npx prisma db push --schema ${databasePath}/prisma/schema.prisma`, { - env: { - ...process.env, - DATABASE_URL: container.getConnectionUri(), - DIRECT_URL: container.getConnectionUri(), - }, - }); - - // console.log(container.getConnectionUri()); - - return { url: container.getConnectionUri(), container }; -} - -export async function createRedisContainer() { - const container = await new RedisContainer().start(); - try { - return { - container, - }; - } catch (e) { - console.error(e); - throw e; - } -} From 9f96eb3afdaf7fba7b25a599f9170b906d2dbd21 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 3 Oct 2024 15:21:21 -0700 Subject: [PATCH 077/114] Fix for the wrong path to the Prisma schem,a --- internal-packages/testcontainers/src/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-packages/testcontainers/src/utils.ts b/internal-packages/testcontainers/src/utils.ts index 6140dc2266..73c46ab4ff 100644 --- a/internal-packages/testcontainers/src/utils.ts +++ b/internal-packages/testcontainers/src/utils.ts @@ -7,7 +7,7 @@ export async function createPostgresContainer() { const container = await new PostgreSqlContainer().start(); // Run migrations - const databasePath = path.resolve(__dirname, "../../../database"); + const databasePath = path.resolve(__dirname, "../../database"); execSync(`npx prisma db push --schema ${databasePath}/prisma/schema.prisma`, { env: { From 28a922ce219edcdd925ac7476d9fe69573ee4113 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 3 Oct 2024 15:21:35 -0700 Subject: [PATCH 078/114] Added the testcontainers package to the run-engine --- internal-packages/run-engine/package.json | 5 +- pnpm-lock.yaml | 76 +++++++++++++++++++---- 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/internal-packages/run-engine/package.json b/internal-packages/run-engine/package.json index ecf9760883..9c4af18104 100644 --- a/internal-packages/run-engine/package.json +++ b/internal-packages/run-engine/package.json @@ -5,7 +5,6 @@ "main": "./src/index.ts", "types": "./src/index.ts", "dependencies": { - "@internal/testcontainers": "workspace:*", "@internal/zod-worker": "workspace:*", "@opentelemetry/api": "^1.9.0", "@opentelemetry/semantic-conventions": "^1.27.0", @@ -18,9 +17,7 @@ "zod": "3.22.3" }, "devDependencies": { - "@testcontainers/postgresql": "^10.13.1", - "@testcontainers/redis": "^10.13.1", - "testcontainers": "^10.13.1", + "@internal/testcontainers": "workspace:*", "vitest": "^1.4.0" }, "scripts": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7de3041398..6e20ce17ce 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -895,6 +895,37 @@ importers: specifier: ^5.5.0 version: 5.5.4 + internal-packages/redis-worker: + dependencies: + '@opentelemetry/api': + specifier: ^1.9.0 + version: 1.9.0 + '@trigger.dev/core': + specifier: workspace:* + version: link:../../packages/core + ioredis: + specifier: ^5.3.2 + version: 5.3.2 + lodash.omit: + specifier: ^4.5.0 + version: 4.5.0 + typescript: + specifier: ^4.8.4 + version: 4.9.5 + zod: + specifier: 3.22.3 + version: 3.22.3 + devDependencies: + '@internal/testcontainers': + specifier: workspace:* + version: link:../testcontainers + '@types/lodash.omit': + specifier: ^4.5.7 + version: 4.5.7 + vitest: + specifier: ^1.4.0 + version: 1.6.0(@types/node@20.14.14) + internal-packages/run-engine: dependencies: '@internal/zod-worker': @@ -927,6 +958,28 @@ importers: zod: specifier: 3.22.3 version: 3.22.3 + devDependencies: + '@internal/testcontainers': + specifier: workspace:* + version: link:../testcontainers + vitest: + specifier: ^1.4.0 + version: 1.6.0(@types/node@20.14.14) + + internal-packages/testcontainers: + dependencies: + '@opentelemetry/api': + specifier: ^1.9.0 + version: 1.9.0 + '@trigger.dev/database': + specifier: workspace:* + version: link:../database + ioredis: + specifier: ^5.3.2 + version: 5.3.2 + typescript: + specifier: ^4.8.4 + version: 4.9.5 devDependencies: '@testcontainers/postgresql': specifier: ^10.13.1 @@ -2347,7 +2400,7 @@ packages: '@babel/traverse': 7.24.7 '@babel/types': 7.24.0 convert-source-map: 1.9.0 - debug: 4.3.4 + debug: 4.3.7 gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -3903,7 +3956,7 @@ packages: '@babel/helper-split-export-declaration': 7.22.6 '@babel/parser': 7.24.7 '@babel/types': 7.24.0 - debug: 4.3.4 + debug: 4.3.7 globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -6075,7 +6128,7 @@ packages: engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} dependencies: ajv: 6.12.6 - debug: 4.3.4 + debug: 4.3.7 espree: 9.6.0 globals: 13.19.0 ignore: 5.2.4 @@ -6333,7 +6386,7 @@ packages: engines: {node: '>=10.10.0'} dependencies: '@humanwhocodes/object-schema': 1.2.1 - debug: 4.3.4 + debug: 4.3.7 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -15029,7 +15082,7 @@ packages: dependencies: '@typescript-eslint/typescript-estree': 5.59.6(typescript@5.2.2) '@typescript-eslint/utils': 5.59.6(eslint@8.31.0)(typescript@5.2.2) - debug: 4.3.4 + debug: 4.3.7 eslint: 8.31.0 tsutils: 3.21.0(typescript@5.2.2) typescript: 5.2.2 @@ -15053,7 +15106,7 @@ packages: dependencies: '@typescript-eslint/types': 5.59.6 '@typescript-eslint/visitor-keys': 5.59.6 - debug: 4.3.4 + debug: 4.3.7 globby: 11.1.0 is-glob: 4.0.3 semver: 7.5.4 @@ -17865,6 +17918,7 @@ packages: optional: true dependencies: ms: 2.1.2 + dev: true /debug@4.3.7: resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==} @@ -19045,7 +19099,7 @@ packages: eslint: '*' eslint-plugin-import: '*' dependencies: - debug: 4.3.4 + debug: 4.3.7 enhanced-resolve: 5.15.0 eslint: 8.31.0 eslint-module-utils: 2.7.4(@typescript-eslint/parser@5.59.6)(eslint-import-resolver-node@0.3.7)(eslint-import-resolver-typescript@3.5.5)(eslint@8.31.0) @@ -21080,7 +21134,7 @@ packages: dependencies: '@ioredis/commands': 1.2.0 cluster-key-slot: 1.1.2 - debug: 4.3.6 + debug: 4.3.7 denque: 2.1.0 lodash.defaults: 4.2.0 lodash.isarguments: 3.1.0 @@ -24947,7 +25001,7 @@ packages: engines: {node: '>= 14'} dependencies: agent-base: 7.1.0 - debug: 4.3.6 + debug: 4.3.7 http-proxy-agent: 7.0.0 https-proxy-agent: 7.0.1 lru-cache: 7.18.3 @@ -26141,7 +26195,7 @@ packages: resolution: {integrity: sha512-OScOjQjrrjhAdFpQmnkE/qbIBGCRFhQB/YaJhcC3CPOlmhe7llnW46Ac1J5+EjcNXOTnDdpF96Erw/yedsGksQ==} engines: {node: '>=8.6.0'} dependencies: - debug: 4.3.6 + debug: 4.3.7 module-details-from-path: 1.0.3 resolve: 1.22.8 transitivePeerDependencies: @@ -29056,7 +29110,7 @@ packages: hasBin: true dependencies: cac: 6.7.14 - debug: 4.3.6 + debug: 4.3.7 pathe: 1.1.2 picocolors: 1.0.0 vite: 5.2.7(@types/node@18.11.18) From c7799919dd23c9ca7c93987d48fbd000d0f47863 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 3 Oct 2024 15:23:21 -0700 Subject: [PATCH 079/114] redis-worker package, just a copy of the simple queue for now --- internal-packages/redis-worker/README.md | 10 ++ internal-packages/redis-worker/package.json | 24 +++ .../redis-worker/src/processor.test.ts | 91 +++++++++++ .../redis-worker/src/processor.ts | 144 +++++++++++++++++ .../redis-worker/src/queue.test.ts | 90 +++++++++++ internal-packages/redis-worker/src/queue.ts | 149 ++++++++++++++++++ internal-packages/redis-worker/tsconfig.json | 25 +++ .../redis-worker/vitest.config.ts | 8 + 8 files changed, 541 insertions(+) create mode 100644 internal-packages/redis-worker/README.md create mode 100644 internal-packages/redis-worker/package.json create mode 100644 internal-packages/redis-worker/src/processor.test.ts create mode 100644 internal-packages/redis-worker/src/processor.ts create mode 100644 internal-packages/redis-worker/src/queue.test.ts create mode 100644 internal-packages/redis-worker/src/queue.ts create mode 100644 internal-packages/redis-worker/tsconfig.json create mode 100644 internal-packages/redis-worker/vitest.config.ts diff --git a/internal-packages/redis-worker/README.md b/internal-packages/redis-worker/README.md new file mode 100644 index 0000000000..c6547d1bed --- /dev/null +++ b/internal-packages/redis-worker/README.md @@ -0,0 +1,10 @@ +# Redis worker + +This is a simple worker that pulls tasks from a Redis queue (also in this package). + +Features + +- Configurable settings for concurrency and pull speed. +- Job payloads. +- A schema so only defined jobs can be added to the queue. +- The ability to have future dates for jobs. diff --git a/internal-packages/redis-worker/package.json b/internal-packages/redis-worker/package.json new file mode 100644 index 0000000000..5c21154b55 --- /dev/null +++ b/internal-packages/redis-worker/package.json @@ -0,0 +1,24 @@ +{ + "name": "@internal/redis-worker", + "private": true, + "version": "0.0.1", + "main": "./src/index.ts", + "types": "./src/index.ts", + "dependencies": { + "@opentelemetry/api": "^1.9.0", + "@trigger.dev/core": "workspace:*", + "ioredis": "^5.3.2", + "lodash.omit": "^4.5.0", + "typescript": "^4.8.4", + "zod": "3.22.3" + }, + "devDependencies": { + "@internal/testcontainers": "workspace:*", + "@types/lodash.omit": "^4.5.7", + "vitest": "^1.4.0" + }, + "scripts": { + "typecheck": "tsc --noEmit", + "test": "vitest" + } +} diff --git a/internal-packages/redis-worker/src/processor.test.ts b/internal-packages/redis-worker/src/processor.test.ts new file mode 100644 index 0000000000..bbfab8dca6 --- /dev/null +++ b/internal-packages/redis-worker/src/processor.test.ts @@ -0,0 +1,91 @@ +import { expect, it } from "vitest"; +import { z } from "zod"; +import { redisTest } from "@internal/testcontainers"; +import { SimpleQueue } from "./queue.js"; +import { describe } from "node:test"; +import { createQueueProcessor } from "./processor.js"; + +describe("SimpleQueue processor", () => { + redisTest("Read items", { timeout: 20_000 }, async ({ redisContainer }) => { + const queue = new SimpleQueue( + "processor-1", + z.object({ + value: z.number(), + }), + { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + } + ); + + try { + let itemCount = 10; + for (let i = 0; i < itemCount; i++) { + await queue.enqueue(i.toString(), { value: i }); + } + + let itemsProcessed = 0; + + const processor = createQueueProcessor(queue, { + onItem: async (id, item) => { + expect(item).toEqual({ value: parseInt(id) }); + itemsProcessed++; + }, + }); + + processor.start(); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(itemsProcessed).toEqual(itemCount); + processor.stop(); + } finally { + await queue.close(); + } + }); + + redisTest("Retrying", { timeout: 20_000 }, async ({ redisContainer }) => { + const queue = new SimpleQueue( + "processor-2", + z.object({ + value: z.number(), + }), + { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + } + ); + + try { + await queue.enqueue("1", { value: 1 }); + + let attempts = 0; + let itemsProcessed = 0; + + const processor = createQueueProcessor(queue, { + retry: { + delay: { + initial: 10, + factor: 1, + }, + maxAttempts: 2, + }, + onItem: async (id, item) => { + attempts++; + if (attempts === 1) { + throw new Error("Test retry"); + } + expect(item).toEqual({ value: parseInt(id) }); + itemsProcessed++; + }, + }); + + processor.start(); + await new Promise((resolve) => setTimeout(resolve, 2_000)); + expect(itemsProcessed).toEqual(1); + processor.stop(); + } finally { + await queue.close(); + } + }); +}); diff --git a/internal-packages/redis-worker/src/processor.ts b/internal-packages/redis-worker/src/processor.ts new file mode 100644 index 0000000000..041b077c24 --- /dev/null +++ b/internal-packages/redis-worker/src/processor.ts @@ -0,0 +1,144 @@ +import { z } from "zod"; +import { SimpleQueue } from "./queue.js"; +import { Logger } from "@trigger.dev/core/logger"; + +type QueueProcessorOptions = { + timeout?: number; + retry?: { + delay?: { + initial?: number; + max?: number; + factor?: number; + }; + maxAttempts?: number; + }; + logger?: Logger; + onItem: (id: string, item: z.infer) => Promise | void; +}; + +const defaultRetryOptions = { + delay: { + initial: 1000, + max: 10000, + factor: 2, + }, + maxAttempts: 10, +}; + +export function createQueueProcessor( + queue: SimpleQueue, + options: QueueProcessorOptions +) { + let { timeout = 1000, onItem, logger = new Logger("QueueProcessor", "debug") } = options; + + const retry = deepMerge(defaultRetryOptions, options.retry ?? {}); + + const failures = new Map(); + let isRunning = false; + + async function processQueue() { + if (!isRunning) return; + + try { + const result = await queue.dequeue(); + if (result) { + const { id, item } = result; + try { + await onItem(id, item); + } catch (error) { + logger.warn("Error processing item:", { error, id, item, queue: queue.name }); + + const retryCount = failures.get(id) || 0; + if (retryCount >= retry.maxAttempts) { + logger.error(`QueueProcessor: max attempts reached for item ${id}`, { + queue: queue.name, + id, + item, + }); + return; + } + + //requeue with delay + const delay = Math.min( + retry.delay.initial * Math.pow(retry.delay.factor, retryCount), + retry.delay.max + ); + logger.log(`QueueProcessor: requeueing item`, { item, id, delay, queue: queue.name }); + await queue.enqueue(id, item, new Date(Date.now() + delay)); + + failures.set(id, retryCount + 1); + } + // Continue processing immediately if still running + if (isRunning) { + setImmediate(processQueue); + } + } else { + // No item found, wait before checking again if still running + if (isRunning) { + setTimeout(processQueue, timeout); + } + } + } catch (error) { + logger.error("Error processing queue:", { error }); + setTimeout(processQueue, timeout); + } + } + + return { + start: () => { + if (!isRunning) { + logger.log("Starting queue processor..."); + isRunning = true; + processQueue(); + } else { + logger.log("Queue processor is already running."); + } + }, + stop: () => { + if (isRunning) { + logger.log("Stopping queue processor..."); + isRunning = false; + } else { + logger.log("Queue processor is already stopped."); + } + }, + }; +} + +type DeepPartial = T extends object + ? { + [P in keyof T]?: DeepPartial; + } + : T; + +function isObject(item: unknown): item is Record { + return typeof item === "object" && item !== null && !Array.isArray(item); +} + +function deepMerge(target: T, source: DeepPartial): T { + if (!isObject(target) || !isObject(source)) { + return source as T; + } + + const output = { ...target } as T; + + (Object.keys(source) as Array).forEach((key) => { + if (key in target) { + const targetValue = target[key]; + const sourceValue = source[key]; + + if (isObject(targetValue) && isObject(sourceValue)) { + (output as any)[key] = deepMerge( + targetValue, + sourceValue as DeepPartial + ); + } else if (sourceValue !== undefined) { + (output as any)[key] = sourceValue; + } + } else if (source[key as keyof DeepPartial] !== undefined) { + (output as any)[key] = source[key as keyof DeepPartial]; + } + }); + + return output; +} diff --git a/internal-packages/redis-worker/src/queue.test.ts b/internal-packages/redis-worker/src/queue.test.ts new file mode 100644 index 0000000000..d84d59a670 --- /dev/null +++ b/internal-packages/redis-worker/src/queue.test.ts @@ -0,0 +1,90 @@ +import { describe } from "node:test"; +import { expect } from "vitest"; +import { z } from "zod"; +import { redisTest } from "@internal/testcontainers"; +import { SimpleQueue } from "./queue.js"; + +describe("SimpleQueue", () => { + redisTest("enqueue/dequeue", { timeout: 20_000 }, async ({ redisContainer }) => { + const queue = new SimpleQueue( + "test-1", + z.object({ + value: z.number(), + }), + { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + } + ); + + try { + await queue.enqueue("1", { value: 1 }); + await queue.enqueue("2", { value: 2 }); + + const first = await queue.dequeue(); + expect(first).toEqual({ id: "1", item: { value: 1 } }); + + const second = await queue.dequeue(); + expect(second).toEqual({ id: "2", item: { value: 2 } }); + } finally { + await queue.close(); + } + }); + + redisTest("no items", { timeout: 20_000 }, async ({ redisContainer }) => { + const queue = new SimpleQueue( + "test-2", + z.object({ + value: z.number(), + }), + { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + } + ); + + try { + const missOne = await queue.dequeue(); + expect(missOne).toBeNull(); + + await queue.enqueue("1", { value: 1 }); + const hitOne = await queue.dequeue(); + expect(hitOne).toEqual({ id: "1", item: { value: 1 } }); + + const missTwo = await queue.dequeue(); + expect(missTwo).toBeNull(); + } finally { + await queue.close(); + } + }); + + redisTest("future item", { timeout: 20_000 }, async ({ redisContainer }) => { + const queue = new SimpleQueue( + "test-3", + z.object({ + value: z.number(), + }), + { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + } + ); + + try { + await queue.enqueue("1", { value: 1 }, new Date(Date.now() + 50)); + + const miss = await queue.dequeue(); + expect(miss).toBeNull(); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + const first = await queue.dequeue(); + expect(first).toEqual({ id: "1", item: { value: 1 } }); + } finally { + await queue.close(); + } + }); +}); diff --git a/internal-packages/redis-worker/src/queue.ts b/internal-packages/redis-worker/src/queue.ts new file mode 100644 index 0000000000..6c1bb8351e --- /dev/null +++ b/internal-packages/redis-worker/src/queue.ts @@ -0,0 +1,149 @@ +import Redis, { RedisOptions } from "ioredis"; +import { z } from "zod"; +import { Logger } from "@trigger.dev/core/logger"; + +export class SimpleQueue { + name: string; + private redis: Redis; + private schema: T; + private logger: Logger; + + constructor(name: string, schema: T, redisOptions: RedisOptions, logger?: Logger) { + this.name = name; + this.redis = new Redis({ + ...redisOptions, + keyPrefix: `queue:${name}:`, + retryStrategy(times) { + const delay = Math.min(times * 50, 1000); + return delay; + }, + maxRetriesPerRequest: 3, + }); + this.schema = schema; + + this.logger = logger ?? new Logger("SimpleQueue", "debug"); + + this.redis.on("error", (error) => { + this.logger.error(`Redis Error for queue ${this.name}:`, { queue: this.name, error }); + }); + + this.redis.on("connect", () => { + // this.logger.log(`Redis connected for queue ${this.name}`); + }); + } + + async enqueue(id: string, item: z.infer, availableAt?: Date): Promise { + try { + const score = availableAt ? availableAt.getTime() : Date.now(); + const serializedItem = JSON.stringify(item); + + const result = await this.redis + .multi() + .zadd(`queue`, score, id) + .hset(`items`, id, serializedItem) + .exec(); + + if (!result) { + throw new Error("Redis multi command returned null"); + } + + result.forEach((res, index) => { + if (res[0]) { + throw new Error(`Redis operation ${index} failed: ${res[0]}`); + } + }); + } catch (e) { + this.logger.error(`SimpleQueue ${this.name}.enqueue(): error enqueuing`, { + queue: this.name, + error: e, + id, + item, + }); + throw e; + } + } + + async dequeue(): Promise<{ id: string; item: z.infer } | null> { + const now = Date.now(); + + try { + const result = await this.redis + .multi() + .zrangebyscore(`queue`, "-inf", now, "WITHSCORES", "LIMIT", 0, 1) + .exec(); + + if (!result) { + throw new Error("Redis multi command returned null"); + } + + result.forEach((res, index) => { + if (res[0]) { + throw new Error(`Redis operation ${index} failed: ${res[0]}`); + } + }); + + if (!result[0][1] || (result[0][1] as string[]).length === 0) { + return null; + } + + const [id, score] = result[0][1] as string[]; + + // Check if the item is available now + if (parseInt(score) > now) { + return null; + } + + // Remove the item from the sorted set + await this.redis.zrem(`queue`, id); + + const serializedItem = await this.redis.hget(`items`, id); + + if (!serializedItem) { + this.logger.warn(`Item ${id} not found in hash, might have been deleted`, { + queue: this.name, + id, + }); + return null; + } + + await this.redis.hdel(`items`, id); + const parsedItem = JSON.parse(serializedItem); + const validatedItem = this.schema.safeParse(parsedItem); + + if (!validatedItem.success) { + this.logger.error("Invalid item in queue", { + queue: this.name, + id, + item: parsedItem, + errors: validatedItem.error, + }); + return null; + } + + return { id, item: validatedItem.data }; + } catch (e) { + this.logger.error(`SimpleQueue ${this.name}.dequeue(): error dequeuing`, { + queue: this.name, + error: e, + }); + throw e; + } + } + + async size(): Promise { + try { + const result = await this.redis.zcard(`queue`); + return result; + } catch (e) { + this.logger.error(`SimpleQueue ${this.name}.size(): error getting queue size`, { + queue: this.name, + error: e, + }); + throw e; + } + } + + async close(): Promise { + await this.redis.quit(); + } +} diff --git a/internal-packages/redis-worker/tsconfig.json b/internal-packages/redis-worker/tsconfig.json new file mode 100644 index 0000000000..53593e2d26 --- /dev/null +++ b/internal-packages/redis-worker/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "target": "ES2019", + "lib": ["ES2019", "DOM", "DOM.Iterable"], + "module": "CommonJS", + "moduleResolution": "Node10", + "moduleDetection": "force", + "verbatimModuleSyntax": false, + "types": ["vitest/globals"], + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "preserveWatchOutput": true, + "skipLibCheck": true, + "noEmit": true, + "strict": true, + "paths": { + "@internal/testcontainers": ["../../internal-packages/testcontainers/src/index"], + "@internal/testcontainers/*": ["../../internal-packages/testcontainers/src/*"], + "@trigger.dev/core": ["../../packages/core/src/index"], + "@trigger.dev/core/*": ["../../packages/core/src/*"] + } + }, + "exclude": ["node_modules"] +} diff --git a/internal-packages/redis-worker/vitest.config.ts b/internal-packages/redis-worker/vitest.config.ts new file mode 100644 index 0000000000..4afd926425 --- /dev/null +++ b/internal-packages/redis-worker/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["**/*.test.ts"], + globals: true, + }, +}); From be98dbf21b192920abe3cb8be27c4a4b2ae8e523 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 3 Oct 2024 16:01:56 -0700 Subject: [PATCH 080/114] The queue now uses Lua to enqueue dequeue --- .../redis-worker/src/processor.test.ts | 24 +-- .../redis-worker/src/queue.test.ts | 36 ++--- internal-packages/redis-worker/src/queue.ts | 145 ++++++++++++------ 3 files changed, 125 insertions(+), 80 deletions(-) diff --git a/internal-packages/redis-worker/src/processor.test.ts b/internal-packages/redis-worker/src/processor.test.ts index bbfab8dca6..14520abc4a 100644 --- a/internal-packages/redis-worker/src/processor.test.ts +++ b/internal-packages/redis-worker/src/processor.test.ts @@ -7,17 +7,17 @@ import { createQueueProcessor } from "./processor.js"; describe("SimpleQueue processor", () => { redisTest("Read items", { timeout: 20_000 }, async ({ redisContainer }) => { - const queue = new SimpleQueue( - "processor-1", - z.object({ + const queue = new SimpleQueue({ + name: "processor-1", + schema: z.object({ value: z.number(), }), - { + redisOptions: { host: redisContainer.getHost(), port: redisContainer.getPort(), password: redisContainer.getPassword(), - } - ); + }, + }); try { let itemCount = 10; @@ -44,17 +44,17 @@ describe("SimpleQueue processor", () => { }); redisTest("Retrying", { timeout: 20_000 }, async ({ redisContainer }) => { - const queue = new SimpleQueue( - "processor-2", - z.object({ + const queue = new SimpleQueue({ + name: "processor-2", + schema: z.object({ value: z.number(), }), - { + redisOptions: { host: redisContainer.getHost(), port: redisContainer.getPort(), password: redisContainer.getPassword(), - } - ); + }, + }); try { await queue.enqueue("1", { value: 1 }); diff --git a/internal-packages/redis-worker/src/queue.test.ts b/internal-packages/redis-worker/src/queue.test.ts index d84d59a670..593b873af9 100644 --- a/internal-packages/redis-worker/src/queue.test.ts +++ b/internal-packages/redis-worker/src/queue.test.ts @@ -6,17 +6,17 @@ import { SimpleQueue } from "./queue.js"; describe("SimpleQueue", () => { redisTest("enqueue/dequeue", { timeout: 20_000 }, async ({ redisContainer }) => { - const queue = new SimpleQueue( - "test-1", - z.object({ + const queue = new SimpleQueue({ + name: "test-1", + schema: z.object({ value: z.number(), }), - { + redisOptions: { host: redisContainer.getHost(), port: redisContainer.getPort(), password: redisContainer.getPassword(), - } - ); + }, + }); try { await queue.enqueue("1", { value: 1 }); @@ -33,17 +33,17 @@ describe("SimpleQueue", () => { }); redisTest("no items", { timeout: 20_000 }, async ({ redisContainer }) => { - const queue = new SimpleQueue( - "test-2", - z.object({ + const queue = new SimpleQueue({ + name: "test-2", + schema: z.object({ value: z.number(), }), - { + redisOptions: { host: redisContainer.getHost(), port: redisContainer.getPort(), password: redisContainer.getPassword(), - } - ); + }, + }); try { const missOne = await queue.dequeue(); @@ -61,17 +61,17 @@ describe("SimpleQueue", () => { }); redisTest("future item", { timeout: 20_000 }, async ({ redisContainer }) => { - const queue = new SimpleQueue( - "test-3", - z.object({ + const queue = new SimpleQueue({ + name: "test-3", + schema: z.object({ value: z.number(), }), - { + redisOptions: { host: redisContainer.getHost(), port: redisContainer.getPort(), password: redisContainer.getPassword(), - } - ); + }, + }); try { await queue.enqueue("1", { value: 1 }, new Date(Date.now() + 50)); diff --git a/internal-packages/redis-worker/src/queue.ts b/internal-packages/redis-worker/src/queue.ts index 6c1bb8351e..9ba096848e 100644 --- a/internal-packages/redis-worker/src/queue.ts +++ b/internal-packages/redis-worker/src/queue.ts @@ -1,6 +1,6 @@ -import Redis, { RedisOptions } from "ioredis"; -import { z } from "zod"; import { Logger } from "@trigger.dev/core/logger"; +import Redis, { type Callback, type RedisOptions, type Result } from "ioredis"; +import { z } from "zod"; export class SimpleQueue { name: string; @@ -8,7 +8,17 @@ export class SimpleQueue { private schema: T; private logger: Logger; - constructor(name: string, schema: T, redisOptions: RedisOptions, logger?: Logger) { + constructor({ + name, + schema, + redisOptions, + logger, + }: { + name: string; + schema: T; + redisOptions: RedisOptions; + logger?: Logger; + }) { this.name = name; this.redis = new Redis({ ...redisOptions, @@ -19,6 +29,7 @@ export class SimpleQueue { }, maxRetriesPerRequest: 3, }); + this.#registerCommands(); this.schema = schema; this.logger = logger ?? new Logger("SimpleQueue", "debug"); @@ -37,21 +48,11 @@ export class SimpleQueue { const score = availableAt ? availableAt.getTime() : Date.now(); const serializedItem = JSON.stringify(item); - const result = await this.redis - .multi() - .zadd(`queue`, score, id) - .hset(`items`, id, serializedItem) - .exec(); + const result = await this.redis.enqueueItem(`queue`, `items`, id, score, serializedItem); - if (!result) { - throw new Error("Redis multi command returned null"); + if (result !== 1) { + throw new Error("Enqueue operation failed"); } - - result.forEach((res, index) => { - if (res[0]) { - throw new Error(`Redis operation ${index} failed: ${res[0]}`); - } - }); } catch (e) { this.logger.error(`SimpleQueue ${this.name}.enqueue(): error enqueuing`, { queue: this.name, @@ -67,46 +68,14 @@ export class SimpleQueue { const now = Date.now(); try { - const result = await this.redis - .multi() - .zrangebyscore(`queue`, "-inf", now, "WITHSCORES", "LIMIT", 0, 1) - .exec(); + const result = await this.redis.dequeueItem(`queue`, `items`, now); if (!result) { - throw new Error("Redis multi command returned null"); - } - - result.forEach((res, index) => { - if (res[0]) { - throw new Error(`Redis operation ${index} failed: ${res[0]}`); - } - }); - - if (!result[0][1] || (result[0][1] as string[]).length === 0) { return null; } - const [id, score] = result[0][1] as string[]; + const [id, serializedItem] = result; - // Check if the item is available now - if (parseInt(score) > now) { - return null; - } - - // Remove the item from the sorted set - await this.redis.zrem(`queue`, id); - - const serializedItem = await this.redis.hget(`items`, id); - - if (!serializedItem) { - this.logger.warn(`Item ${id} not found in hash, might have been deleted`, { - queue: this.name, - id, - }); - return null; - } - - await this.redis.hdel(`items`, id); const parsedItem = JSON.parse(serializedItem); const validatedItem = this.schema.safeParse(parsedItem); @@ -146,4 +115,80 @@ export class SimpleQueue { async close(): Promise { await this.redis.quit(); } + + #registerCommands() { + this.redis.defineCommand("enqueueItem", { + numberOfKeys: 2, + lua: ` + local queue = KEYS[1] + local items = KEYS[2] + local id = ARGV[1] + local score = ARGV[2] + local serializedItem = ARGV[3] + + redis.call('ZADD', queue, score, id) + redis.call('HSET', items, id, serializedItem) + + return 1 + `, + }); + + this.redis.defineCommand("dequeueItem", { + numberOfKeys: 2, + lua: ` + local queue = KEYS[1] + local items = KEYS[2] + local now = tonumber(ARGV[1]) + + local result = redis.call('ZRANGEBYSCORE', queue, '-inf', now, 'WITHSCORES', 'LIMIT', 0, 1) + + if #result == 0 then + return nil + end + + local id = result[1] + local score = tonumber(result[2]) + + if score > now then + return nil + end + + redis.call('ZREM', queue, id) + + local serializedItem = redis.call('HGET', items, id) + + if not serializedItem then + return nil + end + + redis.call('HDEL', items, id) + + return {id, serializedItem} + `, + }); + } +} + +declare module "ioredis" { + interface RedisCommander { + enqueueItem( + //keys + queue: string, + items: string, + //args + id: string, + score: number, + serializedItem: string, + callback?: Callback + ): Result; + + dequeueItem( + //keys + queue: string, + items: string, + //args + now: number, + callback?: Callback<[string, string] | null> + ): Result<[string, string] | null, Context>; + } } From 3d0bbaca398df0814119193aa1278f93243696e4 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 4 Oct 2024 13:40:10 -0700 Subject: [PATCH 081/114] The queue now has a catalog and an invisible period after dequeuing --- internal-packages/redis-worker/package.json | 1 + .../redis-worker/src/queue.test.ts | 45 ++++--- internal-packages/redis-worker/src/queue.ts | 116 +++++++++++++----- pnpm-lock.yaml | 9 ++ 4 files changed, 124 insertions(+), 47 deletions(-) diff --git a/internal-packages/redis-worker/package.json b/internal-packages/redis-worker/package.json index 5c21154b55..83df6f862c 100644 --- a/internal-packages/redis-worker/package.json +++ b/internal-packages/redis-worker/package.json @@ -9,6 +9,7 @@ "@trigger.dev/core": "workspace:*", "ioredis": "^5.3.2", "lodash.omit": "^4.5.0", + "nanoid": "^5.0.7", "typescript": "^4.8.4", "zod": "3.22.3" }, diff --git a/internal-packages/redis-worker/src/queue.test.ts b/internal-packages/redis-worker/src/queue.test.ts index 593b873af9..3a84b6fd3f 100644 --- a/internal-packages/redis-worker/src/queue.test.ts +++ b/internal-packages/redis-worker/src/queue.test.ts @@ -8,9 +8,11 @@ describe("SimpleQueue", () => { redisTest("enqueue/dequeue", { timeout: 20_000 }, async ({ redisContainer }) => { const queue = new SimpleQueue({ name: "test-1", - schema: z.object({ - value: z.number(), - }), + schema: { + test: z.object({ + value: z.number(), + }), + }, redisOptions: { host: redisContainer.getHost(), port: redisContainer.getPort(), @@ -19,14 +21,14 @@ describe("SimpleQueue", () => { }); try { - await queue.enqueue("1", { value: 1 }); - await queue.enqueue("2", { value: 2 }); + await queue.enqueue({ id: "1", job: "test", item: { value: 1 } }); + await queue.enqueue({ id: "2", job: "test", item: { value: 2 } }); const first = await queue.dequeue(); - expect(first).toEqual({ id: "1", item: { value: 1 } }); + expect(first).toEqual({ id: "1", job: "test", item: { value: 1 } }); const second = await queue.dequeue(); - expect(second).toEqual({ id: "2", item: { value: 2 } }); + expect(second).toEqual({ id: "2", job: "test", item: { value: 2 } }); } finally { await queue.close(); } @@ -35,9 +37,11 @@ describe("SimpleQueue", () => { redisTest("no items", { timeout: 20_000 }, async ({ redisContainer }) => { const queue = new SimpleQueue({ name: "test-2", - schema: z.object({ - value: z.number(), - }), + schema: { + test: z.object({ + value: z.number(), + }), + }, redisOptions: { host: redisContainer.getHost(), port: redisContainer.getPort(), @@ -49,9 +53,9 @@ describe("SimpleQueue", () => { const missOne = await queue.dequeue(); expect(missOne).toBeNull(); - await queue.enqueue("1", { value: 1 }); + await queue.enqueue({ id: "1", job: "test", item: { value: 1 } }); const hitOne = await queue.dequeue(); - expect(hitOne).toEqual({ id: "1", item: { value: 1 } }); + expect(hitOne).toEqual({ id: "1", job: "test", item: { value: 1 } }); const missTwo = await queue.dequeue(); expect(missTwo).toBeNull(); @@ -63,9 +67,11 @@ describe("SimpleQueue", () => { redisTest("future item", { timeout: 20_000 }, async ({ redisContainer }) => { const queue = new SimpleQueue({ name: "test-3", - schema: z.object({ - value: z.number(), - }), + schema: { + test: z.object({ + value: z.number(), + }), + }, redisOptions: { host: redisContainer.getHost(), port: redisContainer.getPort(), @@ -74,7 +80,12 @@ describe("SimpleQueue", () => { }); try { - await queue.enqueue("1", { value: 1 }, new Date(Date.now() + 50)); + await queue.enqueue({ + id: "1", + job: "test", + item: { value: 1 }, + availableAt: new Date(Date.now() + 50), + }); const miss = await queue.dequeue(); expect(miss).toBeNull(); @@ -82,7 +93,7 @@ describe("SimpleQueue", () => { await new Promise((resolve) => setTimeout(resolve, 50)); const first = await queue.dequeue(); - expect(first).toEqual({ id: "1", item: { value: 1 } }); + expect(first).toEqual({ id: "1", job: "test", item: { value: 1 } }); } finally { await queue.close(); } diff --git a/internal-packages/redis-worker/src/queue.ts b/internal-packages/redis-worker/src/queue.ts index 9ba096848e..45b15b8b7a 100644 --- a/internal-packages/redis-worker/src/queue.ts +++ b/internal-packages/redis-worker/src/queue.ts @@ -1,11 +1,28 @@ import { Logger } from "@trigger.dev/core/logger"; import Redis, { type Callback, type RedisOptions, type Result } from "ioredis"; +import { nanoid } from "nanoid"; import { z } from "zod"; -export class SimpleQueue { +//todo maybe move the shutdown to the consumer. +//todo when we dequeue we need to keep it in the queue with a future date. +//todo add an ack method so when an item has been successfully processed it is removed. +//todo can we dequeue multiple items at once, pass in the number of items to dequeue. +//todo change the queue so it has a catalog instead of a schema. + +export interface MessageCatalogSchema { + [key: string]: z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion; +} + +type MessageCatalogKey = keyof TMessageCatalog; +type MessageCatalogValue< + TMessageCatalog extends MessageCatalogSchema, + TKey extends MessageCatalogKey, +> = z.infer; + +export class SimpleQueue { name: string; private redis: Redis; - private schema: T; + private schema: TMessageCatalog; private logger: Logger; constructor({ @@ -15,9 +32,10 @@ export class SimpleQueue { logger, }: { name: string; - schema: T; + schema: TMessageCatalog; redisOptions: RedisOptions; logger?: Logger; + shutdownTimeMs?: number; }) { this.name = name; this.redis = new Redis({ @@ -43,12 +61,28 @@ export class SimpleQueue { }); } - async enqueue(id: string, item: z.infer, availableAt?: Date): Promise { + async enqueue({ + id, + job, + item, + availableAt, + }: { + id?: string; + job: MessageCatalogKey; + item: MessageCatalogValue>; + availableAt?: Date; + }): Promise { try { const score = availableAt ? availableAt.getTime() : Date.now(); - const serializedItem = JSON.stringify(item); + const serializedItem = JSON.stringify({ job, item }); - const result = await this.redis.enqueueItem(`queue`, `items`, id, score, serializedItem); + const result = await this.redis.enqueueItem( + `queue`, + `items`, + id ?? nanoid(), + score, + serializedItem + ); if (result !== 1) { throw new Error("Enqueue operation failed"); @@ -64,11 +98,16 @@ export class SimpleQueue { } } - async dequeue(): Promise<{ id: string; item: z.infer } | null> { + async dequeue(visibilityTimeoutSeconds: number = 120): Promise<{ + id: string; + job: MessageCatalogKey; + item: MessageCatalogValue>; + } | null> { const now = Date.now(); + const invisibleUntil = now + visibilityTimeoutSeconds * 1000; try { - const result = await this.redis.dequeueItem(`queue`, `items`, now); + const result = await this.redis.dequeueItem(`queue`, `items`, now, invisibleUntil); if (!result) { return null; @@ -77,7 +116,24 @@ export class SimpleQueue { const [id, serializedItem] = result; const parsedItem = JSON.parse(serializedItem); - const validatedItem = this.schema.safeParse(parsedItem); + if (typeof parsedItem.job !== "string") { + this.logger.error(`Invalid item in queue`, { queue: this.name, id, item: parsedItem }); + return null; + } + + const schema = this.schema[parsedItem.job]; + + if (!schema) { + this.logger.error(`Invalid item in queue, schema not found`, { + queue: this.name, + id, + item: parsedItem, + job: parsedItem.job, + }); + return null; + } + + const validatedItem = schema.safeParse(parsedItem.item); if (!validatedItem.success) { this.logger.error("Invalid item in queue", { @@ -89,7 +145,7 @@ export class SimpleQueue { return null; } - return { id, item: validatedItem.data }; + return { id, job: parsedItem.job, item: validatedItem.data }; } catch (e) { this.logger.error(`SimpleQueue ${this.name}.dequeue(): error dequeuing`, { queue: this.name, @@ -136,34 +192,33 @@ export class SimpleQueue { this.redis.defineCommand("dequeueItem", { numberOfKeys: 2, lua: ` - local queue = KEYS[1] - local items = KEYS[2] - local now = tonumber(ARGV[1]) - - local result = redis.call('ZRANGEBYSCORE', queue, '-inf', now, 'WITHSCORES', 'LIMIT', 0, 1) + local queue = KEYS[1] + local items = KEYS[2] + local now = tonumber(ARGV[1]) + local invisibleUntil = tonumber(ARGV[2]) - if #result == 0 then - return nil - end + local result = redis.call('ZRANGEBYSCORE', queue, '-inf', now, 'WITHSCORES', 'LIMIT', 0, 1) - local id = result[1] - local score = tonumber(result[2]) + if #result == 0 then + return nil + end - if score > now then - return nil - end + local id = result[1] + local score = tonumber(result[2]) - redis.call('ZREM', queue, id) + if score > now then + return nil + end - local serializedItem = redis.call('HGET', items, id) + redis.call('ZADD', queue, invisibleUntil, id) - if not serializedItem then - return nil - end + local serializedItem = redis.call('HGET', items, id) - redis.call('HDEL', items, id) + if not serializedItem then + return nil + end - return {id, serializedItem} + return {id, serializedItem} `, }); } @@ -188,6 +243,7 @@ declare module "ioredis" { items: string, //args now: number, + invisibleUntil: number, callback?: Callback<[string, string] | null> ): Result<[string, string] | null, Context>; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6e20ce17ce..57ddf80d5b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -909,6 +909,9 @@ importers: lodash.omit: specifier: ^4.5.0 version: 4.5.0 + nanoid: + specifier: ^5.0.7 + version: 5.0.7 typescript: specifier: ^4.8.4 version: 4.9.5 @@ -23042,6 +23045,12 @@ packages: hasBin: true dev: false + /nanoid@5.0.7: + resolution: {integrity: sha512-oLxFY2gd2IqnjcYyOXD8XGCftpGtZP2AbHbOkthDkvRywH5ayNtPVy9YlOPcHckXzbLTCHpkb7FB+yuxKV13pQ==} + engines: {node: ^18 || >=20} + hasBin: true + dev: false + /natural-compare-lite@1.4.0: resolution: {integrity: sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==} dev: true From a7e114c91c839492f279c1e7e0a3d9ab540bc487 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 4 Oct 2024 13:55:44 -0700 Subject: [PATCH 082/114] Added a visibility timeout and acking, with tests --- .../redis-worker/src/queue.test.ts | 46 +++++++++- internal-packages/redis-worker/src/queue.ts | 89 ++++++++++++++----- 2 files changed, 110 insertions(+), 25 deletions(-) diff --git a/internal-packages/redis-worker/src/queue.test.ts b/internal-packages/redis-worker/src/queue.test.ts index 3a84b6fd3f..aa21ad6e4b 100644 --- a/internal-packages/redis-worker/src/queue.test.ts +++ b/internal-packages/redis-worker/src/queue.test.ts @@ -1,7 +1,7 @@ +import { redisTest } from "@internal/testcontainers"; import { describe } from "node:test"; import { expect } from "vitest"; import { z } from "zod"; -import { redisTest } from "@internal/testcontainers"; import { SimpleQueue } from "./queue.js"; describe("SimpleQueue", () => { @@ -22,13 +22,24 @@ describe("SimpleQueue", () => { try { await queue.enqueue({ id: "1", job: "test", item: { value: 1 } }); + expect(await queue.size()).toBe(1); + await queue.enqueue({ id: "2", job: "test", item: { value: 2 } }); + expect(await queue.size()).toBe(2); const first = await queue.dequeue(); expect(first).toEqual({ id: "1", job: "test", item: { value: 1 } }); + expect(await queue.size()).toBe(1); + expect(await queue.size({ includeFuture: true })).toBe(2); + + await queue.ack(first!.id); + expect(await queue.size({ includeFuture: true })).toBe(1); const second = await queue.dequeue(); expect(second).toEqual({ id: "2", job: "test", item: { value: 2 } }); + + await queue.ack(second!.id); + expect(await queue.size({ includeFuture: true })).toBe(0); } finally { await queue.close(); } @@ -98,4 +109,37 @@ describe("SimpleQueue", () => { await queue.close(); } }); + + redisTest("invisibility timeout", { timeout: 20_000 }, async ({ redisContainer }) => { + const queue = new SimpleQueue({ + name: "test-4", + schema: { + test: z.object({ + value: z.number(), + }), + }, + redisOptions: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + }, + }); + + try { + await queue.enqueue({ id: "1", job: "test", item: { value: 1 } }); + + const first = await queue.dequeue(2_000); + expect(first).toEqual({ id: "1", job: "test", item: { value: 1 } }); + + const missImmediate = await queue.dequeue(); + expect(missImmediate).toBeNull(); + + await new Promise((resolve) => setTimeout(resolve, 2_000)); + + const second = await queue.dequeue(); + expect(second).toEqual({ id: "1", job: "test", item: { value: 1 } }); + } finally { + await queue.close(); + } + }); }); diff --git a/internal-packages/redis-worker/src/queue.ts b/internal-packages/redis-worker/src/queue.ts index 45b15b8b7a..a3eba0ca86 100644 --- a/internal-packages/redis-worker/src/queue.ts +++ b/internal-packages/redis-worker/src/queue.ts @@ -98,13 +98,13 @@ export class SimpleQueue { } } - async dequeue(visibilityTimeoutSeconds: number = 120): Promise<{ + async dequeue(visibilityTimeoutMs: number = 120_000): Promise<{ id: string; job: MessageCatalogKey; item: MessageCatalogValue>; } | null> { const now = Date.now(); - const invisibleUntil = now + visibilityTimeoutSeconds * 1000; + const invisibleUntil = now + visibilityTimeoutMs; try { const result = await this.redis.dequeueItem(`queue`, `items`, now, invisibleUntil); @@ -155,14 +155,34 @@ export class SimpleQueue { } } - async size(): Promise { + async ack(id: string): Promise { try { - const result = await this.redis.zcard(`queue`); - return result; + await this.redis.ackItem(`queue`, `items`, id); + } catch (e) { + this.logger.error(`SimpleQueue ${this.name}.ack(): error acknowledging item`, { + queue: this.name, + error: e, + id, + }); + throw e; + } + } + + async size({ includeFuture = false }: { includeFuture?: boolean } = {}): Promise { + try { + if (includeFuture) { + // If includeFuture is true, return the total count of all items + return await this.redis.zcard(`queue`); + } else { + // If includeFuture is false, return the count of items available now + const now = Date.now(); + return await this.redis.zcount(`queue`, "-inf", now); + } } catch (e) { this.logger.error(`SimpleQueue ${this.name}.size(): error getting queue size`, { queue: this.name, error: e, + includeFuture, }); throw e; } @@ -192,33 +212,47 @@ export class SimpleQueue { this.redis.defineCommand("dequeueItem", { numberOfKeys: 2, lua: ` - local queue = KEYS[1] - local items = KEYS[2] - local now = tonumber(ARGV[1]) - local invisibleUntil = tonumber(ARGV[2]) + local queue = KEYS[1] + local items = KEYS[2] + local now = tonumber(ARGV[1]) + local invisibleUntil = tonumber(ARGV[2]) + + local result = redis.call('ZRANGEBYSCORE', queue, '-inf', now, 'WITHSCORES', 'LIMIT', 0, 1) + + if #result == 0 then + return nil + end - local result = redis.call('ZRANGEBYSCORE', queue, '-inf', now, 'WITHSCORES', 'LIMIT', 0, 1) + local id = result[1] + local score = tonumber(result[2]) - if #result == 0 then - return nil - end + if score > now then + return nil + end - local id = result[1] - local score = tonumber(result[2]) + redis.call('ZADD', queue, invisibleUntil, id) - if score > now then - return nil - end + local serializedItem = redis.call('HGET', items, id) - redis.call('ZADD', queue, invisibleUntil, id) + if not serializedItem then + return nil + end - local serializedItem = redis.call('HGET', items, id) + return {id, serializedItem} + `, + }); + + this.redis.defineCommand("ackItem", { + numberOfKeys: 2, + lua: ` + local queue = KEYS[1] + local items = KEYS[2] + local id = ARGV[1] - if not serializedItem then - return nil - end + redis.call('ZREM', queue, id) + redis.call('HDEL', items, id) - return {id, serializedItem} + return 1 `, }); } @@ -246,5 +280,12 @@ declare module "ioredis" { invisibleUntil: number, callback?: Callback<[string, string] | null> ): Result<[string, string] | null, Context>; + + ackItem( + queue: string, + items: string, + id: string, + callback?: Callback + ): Result; } } From 75c2ad30d6a22ef6e0868ba6386d52fc65d5e2b9 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 4 Oct 2024 14:03:56 -0700 Subject: [PATCH 083/114] Added more Redis connection logging, deleted todos --- internal-packages/redis-worker/src/queue.test.ts | 5 +++++ internal-packages/redis-worker/src/queue.ts | 16 +++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/internal-packages/redis-worker/src/queue.test.ts b/internal-packages/redis-worker/src/queue.test.ts index aa21ad6e4b..adbfcf13f7 100644 --- a/internal-packages/redis-worker/src/queue.test.ts +++ b/internal-packages/redis-worker/src/queue.test.ts @@ -3,6 +3,7 @@ import { describe } from "node:test"; import { expect } from "vitest"; import { z } from "zod"; import { SimpleQueue } from "./queue.js"; +import { Logger } from "@trigger.dev/core/logger"; describe("SimpleQueue", () => { redisTest("enqueue/dequeue", { timeout: 20_000 }, async ({ redisContainer }) => { @@ -18,6 +19,7 @@ describe("SimpleQueue", () => { port: redisContainer.getPort(), password: redisContainer.getPassword(), }, + logger: new Logger("test", "log"), }); try { @@ -58,6 +60,7 @@ describe("SimpleQueue", () => { port: redisContainer.getPort(), password: redisContainer.getPassword(), }, + logger: new Logger("test", "error"), }); try { @@ -88,6 +91,7 @@ describe("SimpleQueue", () => { port: redisContainer.getPort(), password: redisContainer.getPassword(), }, + logger: new Logger("test", "error"), }); try { @@ -123,6 +127,7 @@ describe("SimpleQueue", () => { port: redisContainer.getPort(), password: redisContainer.getPassword(), }, + logger: new Logger("test", "error"), }); try { diff --git a/internal-packages/redis-worker/src/queue.ts b/internal-packages/redis-worker/src/queue.ts index a3eba0ca86..f18bd9f8e8 100644 --- a/internal-packages/redis-worker/src/queue.ts +++ b/internal-packages/redis-worker/src/queue.ts @@ -3,12 +3,6 @@ import Redis, { type Callback, type RedisOptions, type Result } from "ioredis"; import { nanoid } from "nanoid"; import { z } from "zod"; -//todo maybe move the shutdown to the consumer. -//todo when we dequeue we need to keep it in the queue with a future date. -//todo add an ack method so when an item has been successfully processed it is removed. -//todo can we dequeue multiple items at once, pass in the number of items to dequeue. -//todo change the queue so it has a catalog instead of a schema. - export interface MessageCatalogSchema { [key: string]: z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion; } @@ -57,7 +51,15 @@ export class SimpleQueue { }); this.redis.on("connect", () => { - // this.logger.log(`Redis connected for queue ${this.name}`); + this.logger.log(`Redis connected for queue ${this.name}`); + }); + + this.redis.on("reconnecting", () => { + this.logger.warn(`Redis reconnecting for queue ${this.name}`); + }); + + this.redis.on("close", () => { + this.logger.warn(`Redis connection closed for queue ${this.name}`); }); } From d8065167ecf2182c9cf6d63857ca89f15463faa5 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 4 Oct 2024 14:40:39 -0700 Subject: [PATCH 084/114] Visibility timeouts are now defined on the catalog and can be overridden when enqueuing --- .../redis-worker/src/queue.test.ts | 98 +++++++++++++------ internal-packages/redis-worker/src/queue.ts | 45 ++++++--- 2 files changed, 102 insertions(+), 41 deletions(-) diff --git a/internal-packages/redis-worker/src/queue.test.ts b/internal-packages/redis-worker/src/queue.test.ts index adbfcf13f7..59f4a99963 100644 --- a/internal-packages/redis-worker/src/queue.test.ts +++ b/internal-packages/redis-worker/src/queue.test.ts @@ -10,9 +10,12 @@ describe("SimpleQueue", () => { const queue = new SimpleQueue({ name: "test-1", schema: { - test: z.object({ - value: z.number(), - }), + test: { + schema: z.object({ + value: z.number(), + }), + defaultVisibilityTimeoutMs: 2000, + }, }, redisOptions: { host: redisContainer.getHost(), @@ -30,7 +33,12 @@ describe("SimpleQueue", () => { expect(await queue.size()).toBe(2); const first = await queue.dequeue(); - expect(first).toEqual({ id: "1", job: "test", item: { value: 1 } }); + expect(first).toEqual({ + id: "1", + job: "test", + item: { value: 1 }, + visibilityTimeoutMs: 2000, + }); expect(await queue.size()).toBe(1); expect(await queue.size({ includeFuture: true })).toBe(2); @@ -38,7 +46,12 @@ describe("SimpleQueue", () => { expect(await queue.size({ includeFuture: true })).toBe(1); const second = await queue.dequeue(); - expect(second).toEqual({ id: "2", job: "test", item: { value: 2 } }); + expect(second).toEqual({ + id: "2", + job: "test", + item: { value: 2 }, + visibilityTimeoutMs: 2000, + }); await queue.ack(second!.id); expect(await queue.size({ includeFuture: true })).toBe(0); @@ -49,27 +62,35 @@ describe("SimpleQueue", () => { redisTest("no items", { timeout: 20_000 }, async ({ redisContainer }) => { const queue = new SimpleQueue({ - name: "test-2", + name: "test-1", schema: { - test: z.object({ - value: z.number(), - }), + test: { + schema: z.object({ + value: z.number(), + }), + defaultVisibilityTimeoutMs: 2000, + }, }, redisOptions: { host: redisContainer.getHost(), port: redisContainer.getPort(), password: redisContainer.getPassword(), }, - logger: new Logger("test", "error"), + logger: new Logger("test", "log"), }); try { const missOne = await queue.dequeue(); expect(missOne).toBeNull(); - await queue.enqueue({ id: "1", job: "test", item: { value: 1 } }); + await queue.enqueue({ id: "1", job: "test", item: { value: 1 }, visibilityTimeoutMs: 2000 }); const hitOne = await queue.dequeue(); - expect(hitOne).toEqual({ id: "1", job: "test", item: { value: 1 } }); + expect(hitOne).toEqual({ + id: "1", + job: "test", + item: { value: 1 }, + visibilityTimeoutMs: 2000, + }); const missTwo = await queue.dequeue(); expect(missTwo).toBeNull(); @@ -80,18 +101,21 @@ describe("SimpleQueue", () => { redisTest("future item", { timeout: 20_000 }, async ({ redisContainer }) => { const queue = new SimpleQueue({ - name: "test-3", + name: "test-1", schema: { - test: z.object({ - value: z.number(), - }), + test: { + schema: z.object({ + value: z.number(), + }), + defaultVisibilityTimeoutMs: 2000, + }, }, redisOptions: { host: redisContainer.getHost(), port: redisContainer.getPort(), password: redisContainer.getPassword(), }, - logger: new Logger("test", "error"), + logger: new Logger("test", "log"), }); try { @@ -108,7 +132,12 @@ describe("SimpleQueue", () => { await new Promise((resolve) => setTimeout(resolve, 50)); const first = await queue.dequeue(); - expect(first).toEqual({ id: "1", job: "test", item: { value: 1 } }); + expect(first).toEqual({ + id: "1", + job: "test", + item: { value: 1 }, + visibilityTimeoutMs: 2000, + }); } finally { await queue.close(); } @@ -116,33 +145,46 @@ describe("SimpleQueue", () => { redisTest("invisibility timeout", { timeout: 20_000 }, async ({ redisContainer }) => { const queue = new SimpleQueue({ - name: "test-4", + name: "test-1", schema: { - test: z.object({ - value: z.number(), - }), + test: { + schema: z.object({ + value: z.number(), + }), + defaultVisibilityTimeoutMs: 2000, + }, }, redisOptions: { host: redisContainer.getHost(), port: redisContainer.getPort(), password: redisContainer.getPassword(), }, - logger: new Logger("test", "error"), + logger: new Logger("test", "log"), }); try { - await queue.enqueue({ id: "1", job: "test", item: { value: 1 } }); + await queue.enqueue({ id: "1", job: "test", item: { value: 1 }, visibilityTimeoutMs: 1_000 }); - const first = await queue.dequeue(2_000); - expect(first).toEqual({ id: "1", job: "test", item: { value: 1 } }); + const first = await queue.dequeue(); + expect(first).toEqual({ + id: "1", + job: "test", + item: { value: 1 }, + visibilityTimeoutMs: 1_000, + }); const missImmediate = await queue.dequeue(); expect(missImmediate).toBeNull(); - await new Promise((resolve) => setTimeout(resolve, 2_000)); + await new Promise((resolve) => setTimeout(resolve, 1_000)); const second = await queue.dequeue(); - expect(second).toEqual({ id: "1", job: "test", item: { value: 1 } }); + expect(second).toEqual({ + id: "1", + job: "test", + item: { value: 1 }, + visibilityTimeoutMs: 1_000, + }); } finally { await queue.close(); } diff --git a/internal-packages/redis-worker/src/queue.ts b/internal-packages/redis-worker/src/queue.ts index f18bd9f8e8..530b221ea3 100644 --- a/internal-packages/redis-worker/src/queue.ts +++ b/internal-packages/redis-worker/src/queue.ts @@ -4,14 +4,17 @@ import { nanoid } from "nanoid"; import { z } from "zod"; export interface MessageCatalogSchema { - [key: string]: z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion; + [key: string]: { + schema: z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion; + defaultVisibilityTimeoutMs: number; + }; } type MessageCatalogKey = keyof TMessageCatalog; type MessageCatalogValue< TMessageCatalog extends MessageCatalogSchema, TKey extends MessageCatalogKey, -> = z.infer; +> = z.infer; export class SimpleQueue { name: string; @@ -68,15 +71,23 @@ export class SimpleQueue { job, item, availableAt, + visibilityTimeoutMs, }: { id?: string; job: MessageCatalogKey; item: MessageCatalogValue>; availableAt?: Date; + visibilityTimeoutMs?: number; }): Promise { try { const score = availableAt ? availableAt.getTime() : Date.now(); - const serializedItem = JSON.stringify({ job, item }); + const jobSchema = this.schema[job]; + const actualVisibilityTimeoutMs = visibilityTimeoutMs ?? jobSchema.defaultVisibilityTimeoutMs; + const serializedItem = JSON.stringify({ + job, + item, + visibilityTimeoutMs: actualVisibilityTimeoutMs, + }); const result = await this.redis.enqueueItem( `queue`, @@ -100,16 +111,16 @@ export class SimpleQueue { } } - async dequeue(visibilityTimeoutMs: number = 120_000): Promise<{ + async dequeue(): Promise<{ id: string; job: MessageCatalogKey; item: MessageCatalogValue>; + visibilityTimeoutMs: number; } | null> { const now = Date.now(); - const invisibleUntil = now + visibilityTimeoutMs; try { - const result = await this.redis.dequeueItem(`queue`, `items`, now, invisibleUntil); + const result = await this.redis.dequeueItem(`queue`, `items`, now); if (!result) { return null; @@ -135,7 +146,7 @@ export class SimpleQueue { return null; } - const validatedItem = schema.safeParse(parsedItem.item); + const validatedItem = schema.schema.safeParse(parsedItem.item); if (!validatedItem.success) { this.logger.error("Invalid item in queue", { @@ -147,7 +158,13 @@ export class SimpleQueue { return null; } - return { id, job: parsedItem.job, item: validatedItem.data }; + const visibilityTimeoutMs = + (parsedItem.visibilityTimeoutMs as number) || schema.defaultVisibilityTimeoutMs; + const invisibleUntil = now + visibilityTimeoutMs; + + await this.redis.zadd(`queue`, invisibleUntil, id); + + return { id, job: parsedItem.job, item: validatedItem.data, visibilityTimeoutMs }; } catch (e) { this.logger.error(`SimpleQueue ${this.name}.dequeue(): error dequeuing`, { queue: this.name, @@ -217,7 +234,6 @@ export class SimpleQueue { local queue = KEYS[1] local items = KEYS[2] local now = tonumber(ARGV[1]) - local invisibleUntil = tonumber(ARGV[2]) local result = redis.call('ZRANGEBYSCORE', queue, '-inf', now, 'WITHSCORES', 'LIMIT', 0, 1) @@ -232,16 +248,20 @@ export class SimpleQueue { return nil end - redis.call('ZADD', queue, invisibleUntil, id) - local serializedItem = redis.call('HGET', items, id) if not serializedItem then return nil end + local item = cjson.decode(serializedItem) + local visibilityTimeoutMs = tonumber(item.visibilityTimeoutMs) + local invisibleUntil = now + visibilityTimeoutMs + + redis.call('ZADD', queue, invisibleUntil, id) + return {id, serializedItem} - `, + `, }); this.redis.defineCommand("ackItem", { @@ -279,7 +299,6 @@ declare module "ioredis" { items: string, //args now: number, - invisibleUntil: number, callback?: Callback<[string, string] | null> ): Result<[string, string] | null, Context>; From 4a1fd0dc17f85fd3934f51dde6722d7369be6731 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 4 Oct 2024 14:56:07 -0700 Subject: [PATCH 085/114] Dequeue multiple items at once --- .../redis-worker/src/queue.test.ts | 32 ++-- internal-packages/redis-worker/src/queue.ts | 145 ++++++++++-------- 2 files changed, 96 insertions(+), 81 deletions(-) diff --git a/internal-packages/redis-worker/src/queue.test.ts b/internal-packages/redis-worker/src/queue.test.ts index 59f4a99963..f224343724 100644 --- a/internal-packages/redis-worker/src/queue.test.ts +++ b/internal-packages/redis-worker/src/queue.test.ts @@ -32,7 +32,7 @@ describe("SimpleQueue", () => { await queue.enqueue({ id: "2", job: "test", item: { value: 2 } }); expect(await queue.size()).toBe(2); - const first = await queue.dequeue(); + const [first] = await queue.dequeue(1); expect(first).toEqual({ id: "1", job: "test", @@ -42,10 +42,10 @@ describe("SimpleQueue", () => { expect(await queue.size()).toBe(1); expect(await queue.size({ includeFuture: true })).toBe(2); - await queue.ack(first!.id); + await queue.ack(first.id); expect(await queue.size({ includeFuture: true })).toBe(1); - const second = await queue.dequeue(); + const [second] = await queue.dequeue(1); expect(second).toEqual({ id: "2", job: "test", @@ -53,7 +53,7 @@ describe("SimpleQueue", () => { visibilityTimeoutMs: 2000, }); - await queue.ack(second!.id); + await queue.ack(second.id); expect(await queue.size({ includeFuture: true })).toBe(0); } finally { await queue.close(); @@ -80,11 +80,11 @@ describe("SimpleQueue", () => { }); try { - const missOne = await queue.dequeue(); - expect(missOne).toBeNull(); + const missOne = await queue.dequeue(1); + expect(missOne).toEqual([]); await queue.enqueue({ id: "1", job: "test", item: { value: 1 }, visibilityTimeoutMs: 2000 }); - const hitOne = await queue.dequeue(); + const [hitOne] = await queue.dequeue(1); expect(hitOne).toEqual({ id: "1", job: "test", @@ -92,8 +92,8 @@ describe("SimpleQueue", () => { visibilityTimeoutMs: 2000, }); - const missTwo = await queue.dequeue(); - expect(missTwo).toBeNull(); + const missTwo = await queue.dequeue(1); + expect(missTwo).toEqual([]); } finally { await queue.close(); } @@ -126,12 +126,12 @@ describe("SimpleQueue", () => { availableAt: new Date(Date.now() + 50), }); - const miss = await queue.dequeue(); - expect(miss).toBeNull(); + const miss = await queue.dequeue(1); + expect(miss).toEqual([]); await new Promise((resolve) => setTimeout(resolve, 50)); - const first = await queue.dequeue(); + const [first] = await queue.dequeue(); expect(first).toEqual({ id: "1", job: "test", @@ -165,7 +165,7 @@ describe("SimpleQueue", () => { try { await queue.enqueue({ id: "1", job: "test", item: { value: 1 }, visibilityTimeoutMs: 1_000 }); - const first = await queue.dequeue(); + const [first] = await queue.dequeue(); expect(first).toEqual({ id: "1", job: "test", @@ -173,12 +173,12 @@ describe("SimpleQueue", () => { visibilityTimeoutMs: 1_000, }); - const missImmediate = await queue.dequeue(); - expect(missImmediate).toBeNull(); + const missImmediate = await queue.dequeue(1); + expect(missImmediate).toEqual([]); await new Promise((resolve) => setTimeout(resolve, 1_000)); - const second = await queue.dequeue(); + const [second] = await queue.dequeue(); expect(second).toEqual({ id: "1", job: "test", diff --git a/internal-packages/redis-worker/src/queue.ts b/internal-packages/redis-worker/src/queue.ts index 530b221ea3..2c62410853 100644 --- a/internal-packages/redis-worker/src/queue.ts +++ b/internal-packages/redis-worker/src/queue.ts @@ -110,65 +110,76 @@ export class SimpleQueue { throw e; } } - - async dequeue(): Promise<{ - id: string; - job: MessageCatalogKey; - item: MessageCatalogValue>; - visibilityTimeoutMs: number; - } | null> { + async dequeue(count: number = 1): Promise< + Array<{ + id: string; + job: MessageCatalogKey; + item: MessageCatalogValue>; + visibilityTimeoutMs: number; + }> + > { const now = Date.now(); try { - const result = await this.redis.dequeueItem(`queue`, `items`, now); + const results = await this.redis.dequeueItems(`queue`, `items`, now, count); - if (!result) { - return null; + if (!results || results.length === 0) { + return []; } - const [id, serializedItem] = result; - - const parsedItem = JSON.parse(serializedItem); - if (typeof parsedItem.job !== "string") { - this.logger.error(`Invalid item in queue`, { queue: this.name, id, item: parsedItem }); - return null; - } - - const schema = this.schema[parsedItem.job]; - - if (!schema) { - this.logger.error(`Invalid item in queue, schema not found`, { - queue: this.name, + const dequeuedItems = []; + + for (const [id, serializedItem] of results) { + const parsedItem = JSON.parse(serializedItem); + if (typeof parsedItem.job !== "string") { + this.logger.error(`Invalid item in queue`, { queue: this.name, id, item: parsedItem }); + continue; + } + + const schema = this.schema[parsedItem.job]; + + if (!schema) { + this.logger.error(`Invalid item in queue, schema not found`, { + queue: this.name, + id, + item: parsedItem, + job: parsedItem.job, + }); + continue; + } + + const validatedItem = schema.schema.safeParse(parsedItem.item); + + if (!validatedItem.success) { + this.logger.error("Invalid item in queue", { + queue: this.name, + id, + item: parsedItem, + errors: validatedItem.error, + }); + continue; + } + + const visibilityTimeoutMs = + (parsedItem.visibilityTimeoutMs as number) || schema.defaultVisibilityTimeoutMs; + const invisibleUntil = now + visibilityTimeoutMs; + + await this.redis.zadd(`queue`, invisibleUntil, id); + + dequeuedItems.push({ id, - item: parsedItem, job: parsedItem.job, + item: validatedItem.data, + visibilityTimeoutMs, }); - return null; - } - - const validatedItem = schema.schema.safeParse(parsedItem.item); - - if (!validatedItem.success) { - this.logger.error("Invalid item in queue", { - queue: this.name, - id, - item: parsedItem, - errors: validatedItem.error, - }); - return null; } - const visibilityTimeoutMs = - (parsedItem.visibilityTimeoutMs as number) || schema.defaultVisibilityTimeoutMs; - const invisibleUntil = now + visibilityTimeoutMs; - - await this.redis.zadd(`queue`, invisibleUntil, id); - - return { id, job: parsedItem.job, item: validatedItem.data, visibilityTimeoutMs }; + return dequeuedItems; } catch (e) { this.logger.error(`SimpleQueue ${this.name}.dequeue(): error dequeuing`, { queue: this.name, error: e, + count, }); throw e; } @@ -228,39 +239,43 @@ export class SimpleQueue { `, }); - this.redis.defineCommand("dequeueItem", { + this.redis.defineCommand("dequeueItems", { numberOfKeys: 2, lua: ` local queue = KEYS[1] local items = KEYS[2] local now = tonumber(ARGV[1]) + local count = tonumber(ARGV[2]) - local result = redis.call('ZRANGEBYSCORE', queue, '-inf', now, 'WITHSCORES', 'LIMIT', 0, 1) + local result = redis.call('ZRANGEBYSCORE', queue, '-inf', now, 'WITHSCORES', 'LIMIT', 0, count) if #result == 0 then - return nil + return {} end - local id = result[1] - local score = tonumber(result[2]) + local dequeued = {} - if score > now then - return nil - end + for i = 1, #result, 2 do + local id = result[i] + local score = tonumber(result[i + 1]) - local serializedItem = redis.call('HGET', items, id) + if score > now then + break + end - if not serializedItem then - return nil - end + local serializedItem = redis.call('HGET', items, id) - local item = cjson.decode(serializedItem) - local visibilityTimeoutMs = tonumber(item.visibilityTimeoutMs) - local invisibleUntil = now + visibilityTimeoutMs + if serializedItem then + local item = cjson.decode(serializedItem) + local visibilityTimeoutMs = tonumber(item.visibilityTimeoutMs) + local invisibleUntil = now + visibilityTimeoutMs - redis.call('ZADD', queue, invisibleUntil, id) + redis.call('ZADD', queue, invisibleUntil, id) + table.insert(dequeued, {id, serializedItem}) + end + end - return {id, serializedItem} + return dequeued `, }); @@ -292,15 +307,15 @@ declare module "ioredis" { serializedItem: string, callback?: Callback ): Result; - - dequeueItem( + dequeueItems( //keys queue: string, items: string, //args now: number, - callback?: Callback<[string, string] | null> - ): Result<[string, string] | null, Context>; + count: number, + callback?: Callback> + ): Result, Context>; ackItem( queue: string, From c8ce87870e0fcfb7a80b66ba3c33acb37d9e2492 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 4 Oct 2024 15:00:00 -0700 Subject: [PATCH 086/114] Test for dequeuing multiple items --- .../redis-worker/src/queue.test.ts | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/internal-packages/redis-worker/src/queue.test.ts b/internal-packages/redis-worker/src/queue.test.ts index f224343724..16c509e0c4 100644 --- a/internal-packages/redis-worker/src/queue.test.ts +++ b/internal-packages/redis-worker/src/queue.test.ts @@ -190,3 +190,67 @@ describe("SimpleQueue", () => { } }); }); + +redisTest("dequeue multiple items", { timeout: 20_000 }, async ({ redisContainer }) => { + const queue = new SimpleQueue({ + name: "test-multi", + schema: { + test: { + schema: z.object({ + value: z.number(), + }), + defaultVisibilityTimeoutMs: 2000, + }, + }, + redisOptions: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + }, + logger: new Logger("test", "log"), + }); + + try { + await queue.enqueue({ id: "1", job: "test", item: { value: 1 } }); + await queue.enqueue({ id: "2", job: "test", item: { value: 2 } }); + await queue.enqueue({ id: "3", job: "test", item: { value: 3 } }); + + expect(await queue.size()).toBe(3); + + const dequeued = await queue.dequeue(2); + expect(dequeued).toHaveLength(2); + expect(dequeued[0]).toEqual({ + id: "1", + job: "test", + item: { value: 1 }, + visibilityTimeoutMs: 2000, + }); + expect(dequeued[1]).toEqual({ + id: "2", + job: "test", + item: { value: 2 }, + visibilityTimeoutMs: 2000, + }); + + expect(await queue.size()).toBe(1); + expect(await queue.size({ includeFuture: true })).toBe(3); + + await queue.ack(dequeued[0].id); + await queue.ack(dequeued[1].id); + + expect(await queue.size({ includeFuture: true })).toBe(1); + + const [last] = await queue.dequeue(1); + expect(last).toEqual({ + id: "3", + job: "test", + item: { value: 3 }, + visibilityTimeoutMs: 2000, + }); + + await queue.ack(last.id); + expect(await queue.size({ includeFuture: true })).toBe(0); + } finally { + await queue.close(); + } +}); From cd2cfb948d3444f3b714d6b9c56023d57c6215ca Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 4 Oct 2024 15:30:58 -0700 Subject: [PATCH 087/114] Export some types to be used elsewhere --- internal-packages/redis-worker/src/queue.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal-packages/redis-worker/src/queue.ts b/internal-packages/redis-worker/src/queue.ts index 2c62410853..6f428340ca 100644 --- a/internal-packages/redis-worker/src/queue.ts +++ b/internal-packages/redis-worker/src/queue.ts @@ -10,8 +10,8 @@ export interface MessageCatalogSchema { }; } -type MessageCatalogKey = keyof TMessageCatalog; -type MessageCatalogValue< +export type MessageCatalogKey = keyof TMessageCatalog; +export type MessageCatalogValue< TMessageCatalog extends MessageCatalogSchema, TKey extends MessageCatalogKey, > = z.infer; From 7d8db07227c2b0fd1bc2d06a0fb3ddfe26a48a41 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 4 Oct 2024 15:31:07 -0700 Subject: [PATCH 088/114] Partial refactor of the processor --- .../redis-worker/src/processor.test.ts | 6 +- .../redis-worker/src/processor.ts | 81 ++++++++++++++----- 2 files changed, 64 insertions(+), 23 deletions(-) diff --git a/internal-packages/redis-worker/src/processor.test.ts b/internal-packages/redis-worker/src/processor.test.ts index 14520abc4a..c6616c61a6 100644 --- a/internal-packages/redis-worker/src/processor.test.ts +++ b/internal-packages/redis-worker/src/processor.test.ts @@ -22,13 +22,13 @@ describe("SimpleQueue processor", () => { try { let itemCount = 10; for (let i = 0; i < itemCount; i++) { - await queue.enqueue(i.toString(), { value: i }); + await queue.enqueue({ id: i.toString(), item: { value: i } }); } let itemsProcessed = 0; const processor = createQueueProcessor(queue, { - onItem: async (id, item) => { + onItem: async (id, job, item) => { expect(item).toEqual({ value: parseInt(id) }); itemsProcessed++; }, @@ -57,7 +57,7 @@ describe("SimpleQueue processor", () => { }); try { - await queue.enqueue("1", { value: 1 }); + await queue.enqueue({ id: "1", item: { value: 1 } }); let attempts = 0; let itemsProcessed = 0; diff --git a/internal-packages/redis-worker/src/processor.ts b/internal-packages/redis-worker/src/processor.ts index 041b077c24..bb146f95f4 100644 --- a/internal-packages/redis-worker/src/processor.ts +++ b/internal-packages/redis-worker/src/processor.ts @@ -1,8 +1,12 @@ -import { z } from "zod"; -import { SimpleQueue } from "./queue.js"; import { Logger } from "@trigger.dev/core/logger"; +import { SimpleQueue } from "./queue.js"; + +//todo can we dequeue multiple items at once, pass in the number of items to dequeue. +//todo use Node workers so we make the most of the cores? + +import { MessageCatalogKey, MessageCatalogSchema, MessageCatalogValue } from "./queue.js"; -type QueueProcessorOptions = { +type QueueProcessorOptions = { timeout?: number; retry?: { delay?: { @@ -13,7 +17,12 @@ type QueueProcessorOptions = { maxAttempts?: number; }; logger?: Logger; - onItem: (id: string, item: z.infer) => Promise | void; + onItem: ( + id: string, + job: MessageCatalogKey, + item: MessageCatalogValue> + ) => Promise; + shutdownTimeMs?: number; }; const defaultRetryOptions = { @@ -25,9 +34,9 @@ const defaultRetryOptions = { maxAttempts: 10, }; -export function createQueueProcessor( - queue: SimpleQueue, - options: QueueProcessorOptions +export function createQueueProcessor( + queue: SimpleQueue, + options: QueueProcessorOptions ) { let { timeout = 1000, onItem, logger = new Logger("QueueProcessor", "debug") } = options; @@ -36,23 +45,27 @@ export function createQueueProcessor( const failures = new Map(); let isRunning = false; + let shutdown = false; + const shutdownTimeMs = options.shutdownTimeMs ?? 5000; // Default to 5 seconds + async function processQueue() { - if (!isRunning) return; + if (!isRunning || shutdown) return; try { - const result = await queue.dequeue(); - if (result) { - const { id, item } = result; + const result = await queue.dequeue(1); + if (result && result.length > 0) { + const { id, job, item } = result[0]; try { - await onItem(id, item); + await onItem(id, job, item); } catch (error) { - logger.warn("Error processing item:", { error, id, item, queue: queue.name }); + logger.warn("Error processing item:", { error, id, job, item, queue: queue.name }); const retryCount = failures.get(id) || 0; if (retryCount >= retry.maxAttempts) { logger.error(`QueueProcessor: max attempts reached for item ${id}`, { queue: queue.name, id, + job, item, }); return; @@ -63,27 +76,55 @@ export function createQueueProcessor( retry.delay.initial * Math.pow(retry.delay.factor, retryCount), retry.delay.max ); - logger.log(`QueueProcessor: requeueing item`, { item, id, delay, queue: queue.name }); - await queue.enqueue(id, item, new Date(Date.now() + delay)); + logger.log(`QueueProcessor: requeueing item`, { + item, + id, + job, + delay, + queue: queue.name, + }); + await queue.enqueue({ id, job, item, availableAt: new Date(Date.now() + delay) }); failures.set(id, retryCount + 1); } - // Continue processing immediately if still running - if (isRunning) { + // Continue processing immediately if still running and not shutting down + if (isRunning && !shutdown) { setImmediate(processQueue); } } else { - // No item found, wait before checking again if still running - if (isRunning) { + // No item found, wait before checking again if still running and not shutting down + if (isRunning && !shutdown) { setTimeout(processQueue, timeout); } } } catch (error) { logger.error("Error processing queue:", { error }); - setTimeout(processQueue, timeout); + if (!shutdown) { + setTimeout(processQueue, timeout); + } } } + function handleProcessSignal(signal: string) { + if (shutdown) { + return; + } + + shutdown = true; + + logger.debug( + `Received ${signal}, shutting down QueueProcessor for ${queue.name} with shutdown time ${shutdownTimeMs}ms` + ); + + setTimeout(() => { + logger.debug(`Shutdown timeout of ${shutdownTimeMs}ms reached, exiting process`); + process.exit(0); + }, shutdownTimeMs); + } + + process.on("SIGTERM", handleProcessSignal); + process.on("SIGINT", handleProcessSignal); + return { start: () => { if (!isRunning) { From 1c31aa0acc44174095e7be4d5943d242c9f96049 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 4 Oct 2024 16:46:10 -0700 Subject: [PATCH 089/114] First stab at a worker with concurrency and NodeWorkers --- internal-packages/redis-worker/src/worker.ts | 155 +++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 internal-packages/redis-worker/src/worker.ts diff --git a/internal-packages/redis-worker/src/worker.ts b/internal-packages/redis-worker/src/worker.ts new file mode 100644 index 0000000000..556d62875f --- /dev/null +++ b/internal-packages/redis-worker/src/worker.ts @@ -0,0 +1,155 @@ +import { + MessageCatalogSchema, + SimpleQueue, + MessageCatalogKey, + MessageCatalogValue, +} from "./queue.js"; +import { Logger } from "@trigger.dev/core/logger"; +import { Worker as NodeWorker } from "worker_threads"; +import os from "os"; + +type JobHandler = (params: { + id: string; + payload: MessageCatalogValue>; + visibilityTimeoutMs: number; +}) => Promise; + +type WorkerOptions = { + queue: SimpleQueue; + jobs: { + [K in MessageCatalogKey]: JobHandler; + }; + concurrency?: { + workers?: number; + tasksPerWorker?: number; + }; + logger?: Logger; +}; + +class Worker { + private queue: SimpleQueue; + private jobs: WorkerOptions["jobs"]; + private logger: Logger; + private workers: NodeWorker[] = []; + private isShuttingDown = false; + private concurrency: Required["concurrency"]>>; + + constructor(options: WorkerOptions) { + this.queue = options.queue; + this.jobs = options.jobs; + this.logger = options.logger ?? new Logger("Worker", "debug"); + + const { workers = os.cpus().length, tasksPerWorker = 1 } = options.concurrency ?? {}; + this.concurrency = { workers, tasksPerWorker }; + + // Initialize worker threads + for (let i = 0; i < workers; i++) { + this.createWorker(tasksPerWorker); + } + + this.setupShutdownHandlers(); + } + + private createWorker(tasksPerWorker: number) { + const worker = new NodeWorker( + ` + const { parentPort } = require('worker_threads'); + + parentPort.on('message', async (message) => { + if (message.type === 'process') { + // Process items here + parentPort.postMessage({ type: 'done' }); + } + }); + `, + { eval: true } + ); + + worker.on("message", (message) => { + if (message.type === "done") { + this.processItems(worker, tasksPerWorker); + } + }); + + worker.on("error", (error) => { + this.logger.error("Worker error:", { error }); + }); + + worker.on("exit", (code) => { + if (code !== 0) { + this.logger.warn(`Worker stopped with exit code ${code}`); + } + if (!this.isShuttingDown) { + this.createWorker(tasksPerWorker); + } + }); + + this.workers.push(worker); + this.processItems(worker, tasksPerWorker); + } + + private async processItems(worker: NodeWorker, count: number) { + if (this.isShuttingDown) return; + + try { + const items = await this.queue.dequeue(count); + if (items.length === 0) { + setTimeout(() => this.processItems(worker, count), 1000); // Wait before trying again + return; + } + + worker.postMessage({ type: "process", items }); + + for (const { id, job, item, visibilityTimeoutMs } of items) { + const handler = this.jobs[job]; + if (!handler) { + this.logger.error(`No handler found for job type: ${job as string}`); + continue; + } + + try { + await handler({ id, payload: item, visibilityTimeoutMs }); + await this.queue.ack(id); + } catch (error) { + this.logger.error(`Error processing item ${id}:`, { error }); + // Here you might want to implement a retry mechanism or dead-letter queue + } + } + } catch (error) { + this.logger.error("Error dequeuing items:", { error }); + setTimeout(() => this.processItems(worker, count), 1000); // Wait before trying again + } + } + + private setupShutdownHandlers() { + process.on("SIGTERM", this.shutdown.bind(this)); + process.on("SIGINT", this.shutdown.bind(this)); + } + + private async shutdown() { + if (this.isShuttingDown) return; + this.isShuttingDown = true; + this.logger.log("Shutting down workers..."); + + for (const worker of this.workers) { + worker.terminate(); + } + + await this.queue.close(); + this.logger.log("All workers shut down."); + } + + public start() { + this.logger.log("Starting workers..."); + this.isShuttingDown = false; + for (const worker of this.workers) { + this.processItems(worker, this.concurrency.tasksPerWorker); + } + } + + public stop() { + this.shutdown(); + } +} + +export { Worker }; From fff3a3514e9d7afd472088ba37070604b82fad25 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 4 Oct 2024 16:49:34 -0700 Subject: [PATCH 090/114] =?UTF-8?q?Don=E2=80=99t=20have=20a=20default=20vi?= =?UTF-8?q?sibility=20timeout=20in=20the=20queue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../redis-worker/src/queue.test.ts | 58 +++++++------------ internal-packages/redis-worker/src/queue.ts | 18 ++---- 2 files changed, 28 insertions(+), 48 deletions(-) diff --git a/internal-packages/redis-worker/src/queue.test.ts b/internal-packages/redis-worker/src/queue.test.ts index 16c509e0c4..b89b443345 100644 --- a/internal-packages/redis-worker/src/queue.test.ts +++ b/internal-packages/redis-worker/src/queue.test.ts @@ -10,12 +10,9 @@ describe("SimpleQueue", () => { const queue = new SimpleQueue({ name: "test-1", schema: { - test: { - schema: z.object({ - value: z.number(), - }), - defaultVisibilityTimeoutMs: 2000, - }, + test: z.object({ + value: z.number(), + }), }, redisOptions: { host: redisContainer.getHost(), @@ -26,10 +23,10 @@ describe("SimpleQueue", () => { }); try { - await queue.enqueue({ id: "1", job: "test", item: { value: 1 } }); + await queue.enqueue({ id: "1", job: "test", item: { value: 1 }, visibilityTimeoutMs: 2000 }); expect(await queue.size()).toBe(1); - await queue.enqueue({ id: "2", job: "test", item: { value: 2 } }); + await queue.enqueue({ id: "2", job: "test", item: { value: 2 }, visibilityTimeoutMs: 2000 }); expect(await queue.size()).toBe(2); const [first] = await queue.dequeue(1); @@ -64,12 +61,9 @@ describe("SimpleQueue", () => { const queue = new SimpleQueue({ name: "test-1", schema: { - test: { - schema: z.object({ - value: z.number(), - }), - defaultVisibilityTimeoutMs: 2000, - }, + test: z.object({ + value: z.number(), + }), }, redisOptions: { host: redisContainer.getHost(), @@ -103,12 +97,9 @@ describe("SimpleQueue", () => { const queue = new SimpleQueue({ name: "test-1", schema: { - test: { - schema: z.object({ - value: z.number(), - }), - defaultVisibilityTimeoutMs: 2000, - }, + test: z.object({ + value: z.number(), + }), }, redisOptions: { host: redisContainer.getHost(), @@ -124,6 +115,7 @@ describe("SimpleQueue", () => { job: "test", item: { value: 1 }, availableAt: new Date(Date.now() + 50), + visibilityTimeoutMs: 2000, }); const miss = await queue.dequeue(1); @@ -147,12 +139,9 @@ describe("SimpleQueue", () => { const queue = new SimpleQueue({ name: "test-1", schema: { - test: { - schema: z.object({ - value: z.number(), - }), - defaultVisibilityTimeoutMs: 2000, - }, + test: z.object({ + value: z.number(), + }), }, redisOptions: { host: redisContainer.getHost(), @@ -193,14 +182,11 @@ describe("SimpleQueue", () => { redisTest("dequeue multiple items", { timeout: 20_000 }, async ({ redisContainer }) => { const queue = new SimpleQueue({ - name: "test-multi", + name: "test-1", schema: { - test: { - schema: z.object({ - value: z.number(), - }), - defaultVisibilityTimeoutMs: 2000, - }, + test: z.object({ + value: z.number(), + }), }, redisOptions: { host: redisContainer.getHost(), @@ -211,9 +197,9 @@ redisTest("dequeue multiple items", { timeout: 20_000 }, async ({ redisContainer }); try { - await queue.enqueue({ id: "1", job: "test", item: { value: 1 } }); - await queue.enqueue({ id: "2", job: "test", item: { value: 2 } }); - await queue.enqueue({ id: "3", job: "test", item: { value: 3 } }); + await queue.enqueue({ id: "1", job: "test", item: { value: 1 }, visibilityTimeoutMs: 2000 }); + await queue.enqueue({ id: "2", job: "test", item: { value: 2 }, visibilityTimeoutMs: 2000 }); + await queue.enqueue({ id: "3", job: "test", item: { value: 3 }, visibilityTimeoutMs: 2000 }); expect(await queue.size()).toBe(3); diff --git a/internal-packages/redis-worker/src/queue.ts b/internal-packages/redis-worker/src/queue.ts index 6f428340ca..c09ea324c2 100644 --- a/internal-packages/redis-worker/src/queue.ts +++ b/internal-packages/redis-worker/src/queue.ts @@ -4,17 +4,14 @@ import { nanoid } from "nanoid"; import { z } from "zod"; export interface MessageCatalogSchema { - [key: string]: { - schema: z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion; - defaultVisibilityTimeoutMs: number; - }; + [key: string]: z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion; } export type MessageCatalogKey = keyof TMessageCatalog; export type MessageCatalogValue< TMessageCatalog extends MessageCatalogSchema, TKey extends MessageCatalogKey, -> = z.infer; +> = z.infer; export class SimpleQueue { name: string; @@ -77,16 +74,14 @@ export class SimpleQueue { job: MessageCatalogKey; item: MessageCatalogValue>; availableAt?: Date; - visibilityTimeoutMs?: number; + visibilityTimeoutMs: number; }): Promise { try { const score = availableAt ? availableAt.getTime() : Date.now(); - const jobSchema = this.schema[job]; - const actualVisibilityTimeoutMs = visibilityTimeoutMs ?? jobSchema.defaultVisibilityTimeoutMs; const serializedItem = JSON.stringify({ job, item, - visibilityTimeoutMs: actualVisibilityTimeoutMs, + visibilityTimeoutMs, }); const result = await this.redis.enqueueItem( @@ -148,7 +143,7 @@ export class SimpleQueue { continue; } - const validatedItem = schema.schema.safeParse(parsedItem.item); + const validatedItem = schema.safeParse(parsedItem.item); if (!validatedItem.success) { this.logger.error("Invalid item in queue", { @@ -160,8 +155,7 @@ export class SimpleQueue { continue; } - const visibilityTimeoutMs = - (parsedItem.visibilityTimeoutMs as number) || schema.defaultVisibilityTimeoutMs; + const visibilityTimeoutMs = parsedItem.visibilityTimeoutMs as number; const invisibleUntil = now + visibilityTimeoutMs; await this.redis.zadd(`queue`, invisibleUntil, id); From 9e2b9e0f37535a1a8a503ef73560f0723ffcd404 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 4 Oct 2024 17:53:06 -0700 Subject: [PATCH 091/114] Worker setup and processing items in a simple test --- .../redis-worker/src/processor.test.ts | 91 --------- .../redis-worker/src/processor.ts | 185 ------------------ internal-packages/redis-worker/src/queue.ts | 1 - .../redis-worker/src/worker.test.ts | 61 ++++++ internal-packages/redis-worker/src/worker.ts | 88 +++++++-- 5 files changed, 129 insertions(+), 297 deletions(-) delete mode 100644 internal-packages/redis-worker/src/processor.test.ts delete mode 100644 internal-packages/redis-worker/src/processor.ts create mode 100644 internal-packages/redis-worker/src/worker.test.ts diff --git a/internal-packages/redis-worker/src/processor.test.ts b/internal-packages/redis-worker/src/processor.test.ts deleted file mode 100644 index c6616c61a6..0000000000 --- a/internal-packages/redis-worker/src/processor.test.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { expect, it } from "vitest"; -import { z } from "zod"; -import { redisTest } from "@internal/testcontainers"; -import { SimpleQueue } from "./queue.js"; -import { describe } from "node:test"; -import { createQueueProcessor } from "./processor.js"; - -describe("SimpleQueue processor", () => { - redisTest("Read items", { timeout: 20_000 }, async ({ redisContainer }) => { - const queue = new SimpleQueue({ - name: "processor-1", - schema: z.object({ - value: z.number(), - }), - redisOptions: { - host: redisContainer.getHost(), - port: redisContainer.getPort(), - password: redisContainer.getPassword(), - }, - }); - - try { - let itemCount = 10; - for (let i = 0; i < itemCount; i++) { - await queue.enqueue({ id: i.toString(), item: { value: i } }); - } - - let itemsProcessed = 0; - - const processor = createQueueProcessor(queue, { - onItem: async (id, job, item) => { - expect(item).toEqual({ value: parseInt(id) }); - itemsProcessed++; - }, - }); - - processor.start(); - await new Promise((resolve) => setTimeout(resolve, 200)); - expect(itemsProcessed).toEqual(itemCount); - processor.stop(); - } finally { - await queue.close(); - } - }); - - redisTest("Retrying", { timeout: 20_000 }, async ({ redisContainer }) => { - const queue = new SimpleQueue({ - name: "processor-2", - schema: z.object({ - value: z.number(), - }), - redisOptions: { - host: redisContainer.getHost(), - port: redisContainer.getPort(), - password: redisContainer.getPassword(), - }, - }); - - try { - await queue.enqueue({ id: "1", item: { value: 1 } }); - - let attempts = 0; - let itemsProcessed = 0; - - const processor = createQueueProcessor(queue, { - retry: { - delay: { - initial: 10, - factor: 1, - }, - maxAttempts: 2, - }, - onItem: async (id, item) => { - attempts++; - if (attempts === 1) { - throw new Error("Test retry"); - } - expect(item).toEqual({ value: parseInt(id) }); - itemsProcessed++; - }, - }); - - processor.start(); - await new Promise((resolve) => setTimeout(resolve, 2_000)); - expect(itemsProcessed).toEqual(1); - processor.stop(); - } finally { - await queue.close(); - } - }); -}); diff --git a/internal-packages/redis-worker/src/processor.ts b/internal-packages/redis-worker/src/processor.ts deleted file mode 100644 index bb146f95f4..0000000000 --- a/internal-packages/redis-worker/src/processor.ts +++ /dev/null @@ -1,185 +0,0 @@ -import { Logger } from "@trigger.dev/core/logger"; -import { SimpleQueue } from "./queue.js"; - -//todo can we dequeue multiple items at once, pass in the number of items to dequeue. -//todo use Node workers so we make the most of the cores? - -import { MessageCatalogKey, MessageCatalogSchema, MessageCatalogValue } from "./queue.js"; - -type QueueProcessorOptions = { - timeout?: number; - retry?: { - delay?: { - initial?: number; - max?: number; - factor?: number; - }; - maxAttempts?: number; - }; - logger?: Logger; - onItem: ( - id: string, - job: MessageCatalogKey, - item: MessageCatalogValue> - ) => Promise; - shutdownTimeMs?: number; -}; - -const defaultRetryOptions = { - delay: { - initial: 1000, - max: 10000, - factor: 2, - }, - maxAttempts: 10, -}; - -export function createQueueProcessor( - queue: SimpleQueue, - options: QueueProcessorOptions -) { - let { timeout = 1000, onItem, logger = new Logger("QueueProcessor", "debug") } = options; - - const retry = deepMerge(defaultRetryOptions, options.retry ?? {}); - - const failures = new Map(); - let isRunning = false; - - let shutdown = false; - const shutdownTimeMs = options.shutdownTimeMs ?? 5000; // Default to 5 seconds - - async function processQueue() { - if (!isRunning || shutdown) return; - - try { - const result = await queue.dequeue(1); - if (result && result.length > 0) { - const { id, job, item } = result[0]; - try { - await onItem(id, job, item); - } catch (error) { - logger.warn("Error processing item:", { error, id, job, item, queue: queue.name }); - - const retryCount = failures.get(id) || 0; - if (retryCount >= retry.maxAttempts) { - logger.error(`QueueProcessor: max attempts reached for item ${id}`, { - queue: queue.name, - id, - job, - item, - }); - return; - } - - //requeue with delay - const delay = Math.min( - retry.delay.initial * Math.pow(retry.delay.factor, retryCount), - retry.delay.max - ); - logger.log(`QueueProcessor: requeueing item`, { - item, - id, - job, - delay, - queue: queue.name, - }); - await queue.enqueue({ id, job, item, availableAt: new Date(Date.now() + delay) }); - - failures.set(id, retryCount + 1); - } - // Continue processing immediately if still running and not shutting down - if (isRunning && !shutdown) { - setImmediate(processQueue); - } - } else { - // No item found, wait before checking again if still running and not shutting down - if (isRunning && !shutdown) { - setTimeout(processQueue, timeout); - } - } - } catch (error) { - logger.error("Error processing queue:", { error }); - if (!shutdown) { - setTimeout(processQueue, timeout); - } - } - } - - function handleProcessSignal(signal: string) { - if (shutdown) { - return; - } - - shutdown = true; - - logger.debug( - `Received ${signal}, shutting down QueueProcessor for ${queue.name} with shutdown time ${shutdownTimeMs}ms` - ); - - setTimeout(() => { - logger.debug(`Shutdown timeout of ${shutdownTimeMs}ms reached, exiting process`); - process.exit(0); - }, shutdownTimeMs); - } - - process.on("SIGTERM", handleProcessSignal); - process.on("SIGINT", handleProcessSignal); - - return { - start: () => { - if (!isRunning) { - logger.log("Starting queue processor..."); - isRunning = true; - processQueue(); - } else { - logger.log("Queue processor is already running."); - } - }, - stop: () => { - if (isRunning) { - logger.log("Stopping queue processor..."); - isRunning = false; - } else { - logger.log("Queue processor is already stopped."); - } - }, - }; -} - -type DeepPartial = T extends object - ? { - [P in keyof T]?: DeepPartial; - } - : T; - -function isObject(item: unknown): item is Record { - return typeof item === "object" && item !== null && !Array.isArray(item); -} - -function deepMerge(target: T, source: DeepPartial): T { - if (!isObject(target) || !isObject(source)) { - return source as T; - } - - const output = { ...target } as T; - - (Object.keys(source) as Array).forEach((key) => { - if (key in target) { - const targetValue = target[key]; - const sourceValue = source[key]; - - if (isObject(targetValue) && isObject(sourceValue)) { - (output as any)[key] = deepMerge( - targetValue, - sourceValue as DeepPartial - ); - } else if (sourceValue !== undefined) { - (output as any)[key] = sourceValue; - } - } else if (source[key as keyof DeepPartial] !== undefined) { - (output as any)[key] = source[key as keyof DeepPartial]; - } - }); - - return output; -} diff --git a/internal-packages/redis-worker/src/queue.ts b/internal-packages/redis-worker/src/queue.ts index c09ea324c2..7e4e001e62 100644 --- a/internal-packages/redis-worker/src/queue.ts +++ b/internal-packages/redis-worker/src/queue.ts @@ -29,7 +29,6 @@ export class SimpleQueue { schema: TMessageCatalog; redisOptions: RedisOptions; logger?: Logger; - shutdownTimeMs?: number; }) { this.name = name; this.redis = new Redis({ diff --git a/internal-packages/redis-worker/src/worker.test.ts b/internal-packages/redis-worker/src/worker.test.ts new file mode 100644 index 0000000000..5848b6c0bc --- /dev/null +++ b/internal-packages/redis-worker/src/worker.test.ts @@ -0,0 +1,61 @@ +import { redisTest } from "@internal/testcontainers"; +import { describe, it } from "node:test"; +import { expect } from "vitest"; +import { z } from "zod"; +import { Worker } from "./worker.js"; +import { Logger } from "@trigger.dev/core/logger"; +import { SimpleQueue } from "./queue.js"; + +describe("Worker", () => { + // Tests will be added here +}); + +redisTest("concurrency settings", { timeout: 30_000 }, async ({ redisContainer }) => { + const processedItems: number[] = []; + const worker = new Worker({ + name: "test-worker", + redisOptions: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + }, + catalog: { + testJob: { + schema: z.object({ value: z.number() }), + visibilityTimeoutMs: 5000, + retry: { maxAttempts: 3 }, + }, + }, + jobs: { + testJob: async ({ payload }) => { + await new Promise((resolve) => setTimeout(resolve, 30)); // Simulate work + processedItems.push(payload.value); + }, + }, + concurrency: { + workers: 2, + tasksPerWorker: 3, + }, + logger: new Logger("test", "log"), + }); + + // Enqueue 10 items + for (let i = 0; i < 10; i++) { + await worker.enqueue({ + id: `item-${i}`, + job: "testJob", + payload: { value: i }, + visibilityTimeoutMs: 5000, + }); + } + + worker.start(); + + // Wait for items to be processed + await new Promise((resolve) => setTimeout(resolve, 600)); + + worker.stop(); + + expect(processedItems.length).toBe(10); + expect(new Set(processedItems).size).toBe(10); // Ensure all items were processed uniquely +}); diff --git a/internal-packages/redis-worker/src/worker.ts b/internal-packages/redis-worker/src/worker.ts index 556d62875f..a0a6aa78f4 100644 --- a/internal-packages/redis-worker/src/worker.ts +++ b/internal-packages/redis-worker/src/worker.ts @@ -1,23 +1,38 @@ -import { - MessageCatalogSchema, - SimpleQueue, - MessageCatalogKey, - MessageCatalogValue, -} from "./queue.js"; import { Logger } from "@trigger.dev/core/logger"; -import { Worker as NodeWorker } from "worker_threads"; +import { type RedisOptions } from "ioredis"; import os from "os"; +import { Worker as NodeWorker } from "worker_threads"; +import { z } from "zod"; +import { SimpleQueue } from "./queue.js"; + +type WorkerCatalog = { + [key: string]: { + schema: z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion; + visibilityTimeoutMs: number; + retry: { + maxAttempts: number; + minDelayMs?: number; + scaleFactor?: number; + }; + }; +}; -type JobHandler = (params: { +type QueueCatalogFromWorkerCatalog = { + [K in keyof Catalog]: Catalog[K]["schema"]; +}; + +type JobHandler = (params: { id: string; - payload: MessageCatalogValue>; + payload: z.infer; visibilityTimeoutMs: number; }) => Promise; -type WorkerOptions = { - queue: SimpleQueue; +type WorkerOptions = { + name: string; + redisOptions: RedisOptions; + catalog: TCatalog; jobs: { - [K in MessageCatalogKey]: JobHandler; + [K in keyof TCatalog]: JobHandler; }; concurrency?: { workers?: number; @@ -26,19 +41,32 @@ type WorkerOptions = { logger?: Logger; }; -class Worker { - private queue: SimpleQueue; - private jobs: WorkerOptions["jobs"]; +class Worker { + private queue: SimpleQueue>; + private jobs: WorkerOptions["jobs"]; private logger: Logger; private workers: NodeWorker[] = []; private isShuttingDown = false; - private concurrency: Required["concurrency"]>>; + private concurrency: Required["concurrency"]>>; + private catalog: TCatalog; - constructor(options: WorkerOptions) { - this.queue = options.queue; - this.jobs = options.jobs; + constructor(options: WorkerOptions) { + this.catalog = options.catalog; this.logger = options.logger ?? new Logger("Worker", "debug"); + const schema: QueueCatalogFromWorkerCatalog = Object.fromEntries( + Object.entries(options.catalog).map(([key, value]) => [key, value.schema]) + ) as QueueCatalogFromWorkerCatalog; + + this.queue = new SimpleQueue({ + name: options.name, + redisOptions: options.redisOptions, + logger: this.logger, + schema, + }); + + this.jobs = options.jobs; + const { workers = os.cpus().length, tasksPerWorker = 1 } = options.concurrency ?? {}; this.concurrency = { workers, tasksPerWorker }; @@ -50,6 +78,26 @@ class Worker { this.setupShutdownHandlers(); } + enqueue({ + id, + job, + payload, + visibilityTimeoutMs, + }: { + id?: string; + job: K; + payload: z.infer; + visibilityTimeoutMs?: number; + }) { + const timeout = visibilityTimeoutMs ?? this.catalog[job].visibilityTimeoutMs; + return this.queue.enqueue({ + id, + job, + item: payload, + visibilityTimeoutMs: timeout, + }); + } + private createWorker(tasksPerWorker: number) { const worker = new NodeWorker( ` @@ -101,7 +149,7 @@ class Worker { worker.postMessage({ type: "process", items }); for (const { id, job, item, visibilityTimeoutMs } of items) { - const handler = this.jobs[job]; + const handler = this.jobs[job as any]; if (!handler) { this.logger.error(`No handler found for job type: ${job as string}`); continue; From b8d73803de79bc1319ef5b362479f7f76a2c8096 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 4 Oct 2024 18:28:39 -0700 Subject: [PATCH 092/114] Process jobs in parallel with retrying --- .../redis-worker/src/worker.test.ts | 151 +++++++++++++----- internal-packages/redis-worker/src/worker.ts | 92 ++++++++--- 2 files changed, 175 insertions(+), 68 deletions(-) diff --git a/internal-packages/redis-worker/src/worker.test.ts b/internal-packages/redis-worker/src/worker.test.ts index 5848b6c0bc..c74483a780 100644 --- a/internal-packages/redis-worker/src/worker.test.ts +++ b/internal-packages/redis-worker/src/worker.test.ts @@ -7,55 +7,120 @@ import { Logger } from "@trigger.dev/core/logger"; import { SimpleQueue } from "./queue.js"; describe("Worker", () => { - // Tests will be added here -}); - -redisTest("concurrency settings", { timeout: 30_000 }, async ({ redisContainer }) => { - const processedItems: number[] = []; - const worker = new Worker({ - name: "test-worker", - redisOptions: { - host: redisContainer.getHost(), - port: redisContainer.getPort(), - password: redisContainer.getPassword(), - }, - catalog: { - testJob: { - schema: z.object({ value: z.number() }), - visibilityTimeoutMs: 5000, - retry: { maxAttempts: 3 }, + redisTest("Process items that don't throw", { timeout: 30_000 }, async ({ redisContainer }) => { + const processedItems: number[] = []; + const worker = new Worker({ + name: "test-worker", + redisOptions: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + }, + catalog: { + testJob: { + schema: z.object({ value: z.number() }), + visibilityTimeoutMs: 5000, + retry: { maxAttempts: 3 }, + }, + }, + jobs: { + testJob: async ({ payload }) => { + await new Promise((resolve) => setTimeout(resolve, 30)); // Simulate work + processedItems.push(payload.value); + }, }, - }, - jobs: { - testJob: async ({ payload }) => { - await new Promise((resolve) => setTimeout(resolve, 30)); // Simulate work - processedItems.push(payload.value); + concurrency: { + workers: 2, + tasksPerWorker: 3, }, - }, - concurrency: { - workers: 2, - tasksPerWorker: 3, - }, - logger: new Logger("test", "log"), + logger: new Logger("test", "log"), + }); + + // Enqueue 10 items + for (let i = 0; i < 10; i++) { + await worker.enqueue({ + id: `item-${i}`, + job: "testJob", + payload: { value: i }, + visibilityTimeoutMs: 5000, + }); + } + + worker.start(); + + // Wait for items to be processed + await new Promise((resolve) => setTimeout(resolve, 600)); + + worker.stop(); + + expect(processedItems.length).toBe(10); + expect(new Set(processedItems).size).toBe(10); // Ensure all items were processed uniquely }); - // Enqueue 10 items - for (let i = 0; i < 10; i++) { - await worker.enqueue({ - id: `item-${i}`, - job: "testJob", - payload: { value: i }, - visibilityTimeoutMs: 5000, - }); - } + redisTest( + "Process items that throw an error", + { timeout: 30_000 }, + async ({ redisContainer }) => { + const processedItems: number[] = []; + const hadAttempt = new Set(); - worker.start(); + const worker = new Worker({ + name: "test-worker", + redisOptions: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + }, + catalog: { + testJob: { + schema: z.object({ value: z.number() }), + visibilityTimeoutMs: 5000, + retry: { maxAttempts: 3, minDelayMs: 10 }, + }, + }, + jobs: { + testJob: async ({ id, payload }) => { + if (!hadAttempt.has(id)) { + hadAttempt.add(id); + throw new Error("Test error"); + } - // Wait for items to be processed - await new Promise((resolve) => setTimeout(resolve, 600)); + await new Promise((resolve) => setTimeout(resolve, 30)); // Simulate work + processedItems.push(payload.value); + }, + }, + concurrency: { + workers: 2, + tasksPerWorker: 3, + }, + pollIntervalMs: 50, + logger: new Logger("test", "error"), + }); - worker.stop(); + // Enqueue 10 items + for (let i = 0; i < 10; i++) { + await worker.enqueue({ + id: `item-${i}`, + job: "testJob", + payload: { value: i }, + visibilityTimeoutMs: 5000, + }); + } - expect(processedItems.length).toBe(10); - expect(new Set(processedItems).size).toBe(10); // Ensure all items were processed uniquely + worker.start(); + + // Wait for items to be processed + await new Promise((resolve) => setTimeout(resolve, 500)); + + worker.stop(); + + expect(processedItems.length).toBe(10); + expect(new Set(processedItems).size).toBe(10); // Ensure all items were processed uniquely + } + ); }); + +//todo test throwing an error and that retrying works +//todo test that throwing an error doesn't screw up the other items +//todo change the processItems to be in parallel using Promise.allResolved +//process more items when finished diff --git a/internal-packages/redis-worker/src/worker.ts b/internal-packages/redis-worker/src/worker.ts index a0a6aa78f4..66e89b0fd2 100644 --- a/internal-packages/redis-worker/src/worker.ts +++ b/internal-packages/redis-worker/src/worker.ts @@ -38,6 +38,7 @@ type WorkerOptions = { workers?: number; tasksPerWorker?: number; }; + pollIntervalMs?: number; logger?: Logger; }; @@ -48,14 +49,12 @@ class Worker { private workers: NodeWorker[] = []; private isShuttingDown = false; private concurrency: Required["concurrency"]>>; - private catalog: TCatalog; - constructor(options: WorkerOptions) { - this.catalog = options.catalog; + constructor(private options: WorkerOptions) { this.logger = options.logger ?? new Logger("Worker", "debug"); const schema: QueueCatalogFromWorkerCatalog = Object.fromEntries( - Object.entries(options.catalog).map(([key, value]) => [key, value.schema]) + Object.entries(this.options.catalog).map(([key, value]) => [key, value.schema]) ) as QueueCatalogFromWorkerCatalog; this.queue = new SimpleQueue({ @@ -89,7 +88,7 @@ class Worker { payload: z.infer; visibilityTimeoutMs?: number; }) { - const timeout = visibilityTimeoutMs ?? this.catalog[job].visibilityTimeoutMs; + const timeout = visibilityTimeoutMs ?? this.options.catalog[job].visibilityTimeoutMs; return this.queue.enqueue({ id, job, @@ -139,34 +138,77 @@ class Worker { private async processItems(worker: NodeWorker, count: number) { if (this.isShuttingDown) return; + const pollIntervalMs = this.options.pollIntervalMs ?? 1000; + try { const items = await this.queue.dequeue(count); if (items.length === 0) { - setTimeout(() => this.processItems(worker, count), 1000); // Wait before trying again + setTimeout(() => this.processItems(worker, count), pollIntervalMs); return; } - worker.postMessage({ type: "process", items }); - - for (const { id, job, item, visibilityTimeoutMs } of items) { - const handler = this.jobs[job as any]; - if (!handler) { - this.logger.error(`No handler found for job type: ${job as string}`); - continue; - } - - try { - await handler({ id, payload: item, visibilityTimeoutMs }); - await this.queue.ack(id); - } catch (error) { - this.logger.error(`Error processing item ${id}:`, { error }); - // Here you might want to implement a retry mechanism or dead-letter queue - } - } + await Promise.all( + items.map(async ({ id, job, item, visibilityTimeoutMs }) => { + const catalogItem = this.options.catalog[job as any]; + const handler = this.jobs[job as any]; + if (!handler) { + this.logger.error(`No handler found for job type: ${job as string}`); + return; + } + + try { + await handler({ id, payload: item, visibilityTimeoutMs }); + await this.queue.ack(id); + } catch (error) { + this.logger.error(`Error processing item, it threw an error:`, { + name: this.options.name, + id, + job, + item, + visibilityTimeoutMs, + error, + }); + // Requeue the failed item with a delay + try { + const retryDelay = catalogItem.retry.minDelayMs ?? 1_000; + const retryDate = new Date(Date.now() + retryDelay); + this.logger.info(`Requeued failed item ${id} with delay`, { + name: this.options.name, + id, + job, + item, + retryDate, + retryDelay, + visibilityTimeoutMs, + }); + await this.queue.enqueue({ + id, + job, + item, + availableAt: retryDate, + visibilityTimeoutMs, + }); + } catch (requeueError) { + this.logger.error(`Failed to requeue item, threw error:`, { + name: this.options.name, + id, + job, + item, + visibilityTimeoutMs, + error: requeueError, + }); + } + } + }) + ); } catch (error) { - this.logger.error("Error dequeuing items:", { error }); - setTimeout(() => this.processItems(worker, count), 1000); // Wait before trying again + this.logger.error("Error dequeuing items:", { name: this.options.name, error }); + setTimeout(() => this.processItems(worker, count), pollIntervalMs); + return; } + + // Immediately process next batch because there were items in the queue + this.processItems(worker, count); } private setupShutdownHandlers() { From 461ec42dfce656e339d49cc59db3f9bc679c2b5f Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Oct 2024 16:02:46 -0700 Subject: [PATCH 093/114] Get the attempt when dequeuing --- internal-packages/redis-worker/src/queue.test.ts | 10 ++++++++++ internal-packages/redis-worker/src/queue.ts | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/internal-packages/redis-worker/src/queue.test.ts b/internal-packages/redis-worker/src/queue.test.ts index b89b443345..b254ff14ee 100644 --- a/internal-packages/redis-worker/src/queue.test.ts +++ b/internal-packages/redis-worker/src/queue.test.ts @@ -35,6 +35,7 @@ describe("SimpleQueue", () => { job: "test", item: { value: 1 }, visibilityTimeoutMs: 2000, + attempt: 0, }); expect(await queue.size()).toBe(1); expect(await queue.size({ includeFuture: true })).toBe(2); @@ -48,6 +49,7 @@ describe("SimpleQueue", () => { job: "test", item: { value: 2 }, visibilityTimeoutMs: 2000, + attempt: 0, }); await queue.ack(second.id); @@ -84,6 +86,7 @@ describe("SimpleQueue", () => { job: "test", item: { value: 1 }, visibilityTimeoutMs: 2000, + attempt: 0, }); const missTwo = await queue.dequeue(1); @@ -116,6 +119,7 @@ describe("SimpleQueue", () => { item: { value: 1 }, availableAt: new Date(Date.now() + 50), visibilityTimeoutMs: 2000, + attempt: 0, }); const miss = await queue.dequeue(1); @@ -129,6 +133,7 @@ describe("SimpleQueue", () => { job: "test", item: { value: 1 }, visibilityTimeoutMs: 2000, + attempt: 0, }); } finally { await queue.close(); @@ -160,6 +165,7 @@ describe("SimpleQueue", () => { job: "test", item: { value: 1 }, visibilityTimeoutMs: 1_000, + attempt: 0, }); const missImmediate = await queue.dequeue(1); @@ -173,6 +179,7 @@ describe("SimpleQueue", () => { job: "test", item: { value: 1 }, visibilityTimeoutMs: 1_000, + attempt: 0, }); } finally { await queue.close(); @@ -210,12 +217,14 @@ redisTest("dequeue multiple items", { timeout: 20_000 }, async ({ redisContainer job: "test", item: { value: 1 }, visibilityTimeoutMs: 2000, + attempt: 0, }); expect(dequeued[1]).toEqual({ id: "2", job: "test", item: { value: 2 }, visibilityTimeoutMs: 2000, + attempt: 0, }); expect(await queue.size()).toBe(1); @@ -232,6 +241,7 @@ redisTest("dequeue multiple items", { timeout: 20_000 }, async ({ redisContainer job: "test", item: { value: 3 }, visibilityTimeoutMs: 2000, + attempt: 0, }); await queue.ack(last.id); diff --git a/internal-packages/redis-worker/src/queue.ts b/internal-packages/redis-worker/src/queue.ts index 7e4e001e62..e142560f5a 100644 --- a/internal-packages/redis-worker/src/queue.ts +++ b/internal-packages/redis-worker/src/queue.ts @@ -66,12 +66,14 @@ export class SimpleQueue { id, job, item, + attempt, availableAt, visibilityTimeoutMs, }: { id?: string; job: MessageCatalogKey; item: MessageCatalogValue>; + attempt?: number; availableAt?: Date; visibilityTimeoutMs: number; }): Promise { @@ -81,6 +83,7 @@ export class SimpleQueue { job, item, visibilityTimeoutMs, + attempt, }); const result = await this.redis.enqueueItem( @@ -110,6 +113,7 @@ export class SimpleQueue { job: MessageCatalogKey; item: MessageCatalogValue>; visibilityTimeoutMs: number; + attempt: number; }> > { const now = Date.now(); @@ -150,6 +154,7 @@ export class SimpleQueue { id, item: parsedItem, errors: validatedItem.error, + attempt: parsedItem.attempt, }); continue; } @@ -164,6 +169,7 @@ export class SimpleQueue { job: parsedItem.job, item: validatedItem.data, visibilityTimeoutMs, + attempt: parsedItem.attempt ?? 0, }); } From 1a713b02cd34b9c22e45b1c9058501ad7e333f9d Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Oct 2024 16:19:34 -0700 Subject: [PATCH 094/114] Workers do exponential backoff --- .../redis-worker/src/worker.test.ts | 2 +- internal-packages/redis-worker/src/worker.ts | 52 +++++++++++++------ 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/internal-packages/redis-worker/src/worker.test.ts b/internal-packages/redis-worker/src/worker.test.ts index c74483a780..e5b7516182 100644 --- a/internal-packages/redis-worker/src/worker.test.ts +++ b/internal-packages/redis-worker/src/worker.test.ts @@ -75,7 +75,7 @@ describe("Worker", () => { testJob: { schema: z.object({ value: z.number() }), visibilityTimeoutMs: 5000, - retry: { maxAttempts: 3, minDelayMs: 10 }, + retry: { maxAttempts: 3, minTimeoutInMs: 10, maxTimeoutInMs: 10 }, }, }, jobs: { diff --git a/internal-packages/redis-worker/src/worker.ts b/internal-packages/redis-worker/src/worker.ts index 66e89b0fd2..09aac1325a 100644 --- a/internal-packages/redis-worker/src/worker.ts +++ b/internal-packages/redis-worker/src/worker.ts @@ -1,4 +1,6 @@ import { Logger } from "@trigger.dev/core/logger"; +import { type RetryOptions } from "@trigger.dev/core/v3/schemas"; +import { calculateNextRetryDelay } from "@trigger.dev/core/v3"; import { type RedisOptions } from "ioredis"; import os from "os"; import { Worker as NodeWorker } from "worker_threads"; @@ -9,11 +11,7 @@ type WorkerCatalog = { [key: string]: { schema: z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion; visibilityTimeoutMs: number; - retry: { - maxAttempts: number; - minDelayMs?: number; - scaleFactor?: number; - }; + retry: RetryOptions; }; }; @@ -25,6 +23,7 @@ type JobHandler = (param id: string; payload: z.infer; visibilityTimeoutMs: number; + attempt: number; }) => Promise; type WorkerOptions = { @@ -148,7 +147,7 @@ class Worker { } await Promise.all( - items.map(async ({ id, job, item, visibilityTimeoutMs }) => { + items.map(async ({ id, job, item, visibilityTimeoutMs, attempt }) => { const catalogItem = this.options.catalog[job as any]; const handler = this.jobs[job as any]; if (!handler) { @@ -157,7 +156,7 @@ class Worker { } try { - await handler({ id, payload: item, visibilityTimeoutMs }); + await handler({ id, payload: item, visibilityTimeoutMs, attempt }); await this.queue.ack(id); } catch (error) { this.logger.error(`Error processing item, it threw an error:`, { @@ -170,7 +169,23 @@ class Worker { }); // Requeue the failed item with a delay try { - const retryDelay = catalogItem.retry.minDelayMs ?? 1_000; + attempt = attempt + 1; + + const retryDelay = calculateNextRetryDelay(catalogItem.retry, attempt); + + if (!retryDelay) { + this.logger.error(`Failed item ${id} has reached max attempts, acking.`, { + name: this.options.name, + id, + job, + item, + visibilityTimeoutMs, + attempt, + }); + await this.queue.ack(id); + return; + } + const retryDate = new Date(Date.now() + retryDelay); this.logger.info(`Requeued failed item ${id} with delay`, { name: this.options.name, @@ -180,23 +195,28 @@ class Worker { retryDate, retryDelay, visibilityTimeoutMs, + attempt, }); await this.queue.enqueue({ id, job, item, availableAt: retryDate, + attempt, visibilityTimeoutMs, }); } catch (requeueError) { - this.logger.error(`Failed to requeue item, threw error:`, { - name: this.options.name, - id, - job, - item, - visibilityTimeoutMs, - error: requeueError, - }); + this.logger.error( + `Failed to requeue item, threw error. Will automatically get rescheduled after the visilibity timeout.`, + { + name: this.options.name, + id, + job, + item, + visibilityTimeoutMs, + error: requeueError, + } + ); } } }) From eee5649a2199024e0b790b4abdeca2d9b8e3a457 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Oct 2024 16:20:33 -0700 Subject: [PATCH 095/114] Moved todos --- internal-packages/redis-worker/src/worker.test.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/internal-packages/redis-worker/src/worker.test.ts b/internal-packages/redis-worker/src/worker.test.ts index e5b7516182..5aafb4345a 100644 --- a/internal-packages/redis-worker/src/worker.test.ts +++ b/internal-packages/redis-worker/src/worker.test.ts @@ -118,9 +118,7 @@ describe("Worker", () => { expect(new Set(processedItems).size).toBe(10); // Ensure all items were processed uniquely } ); -}); -//todo test throwing an error and that retrying works -//todo test that throwing an error doesn't screw up the other items -//todo change the processItems to be in parallel using Promise.allResolved -//process more items when finished + //todo test that throwing an error doesn't screw up the other items + //todo process more items when finished +}); From 0c25754be471b68ef4db890c724f1ad5051479da Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 8 Oct 2024 13:18:26 +0100 Subject: [PATCH 096/114] DLQ functionality --- internal-packages/redis-worker/src/queue.ts | 141 ++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/internal-packages/redis-worker/src/queue.ts b/internal-packages/redis-worker/src/queue.ts index e142560f5a..8f0065da76 100644 --- a/internal-packages/redis-worker/src/queue.ts +++ b/internal-packages/redis-worker/src/queue.ts @@ -217,6 +217,72 @@ export class SimpleQueue { } } + async moveToDeadLetterQueue(id: string, errorMessage: string): Promise { + try { + const result = await this.redis.moveToDeadLetterQueue( + `queue`, + `items`, + `dlq`, + `dlq:items`, + id, + errorMessage + ); + + if (result !== 1) { + throw new Error("Move to Dead Letter Queue operation failed"); + } + } catch (e) { + this.logger.error( + `SimpleQueue ${this.name}.moveToDeadLetterQueue(): error moving item to DLQ`, + { + queue: this.name, + error: e, + id, + errorMessage, + } + ); + throw e; + } + } + + async sizeOfDeadLetterQueue(): Promise { + try { + return await this.redis.zcard(`dlq`); + } catch (e) { + this.logger.error(`SimpleQueue ${this.name}.dlqSize(): error getting DLQ size`, { + queue: this.name, + error: e, + }); + throw e; + } + } + + async redriveFromDeadLetterQueue(id: string): Promise { + try { + const result = await this.redis.redriveFromDeadLetterQueue( + `queue`, + `items`, + `dlq`, + `dlq:items`, + id + ); + + if (result !== 1) { + throw new Error("Redrive from Dead Letter Queue operation failed"); + } + } catch (e) { + this.logger.error( + `SimpleQueue ${this.name}.redriveFromDeadLetterQueue(): error redriving item from DLQ`, + { + queue: this.name, + error: e, + id, + } + ); + throw e; + } + } + async close(): Promise { await this.redis.quit(); } @@ -291,6 +357,61 @@ export class SimpleQueue { return 1 `, }); + + this.redis.defineCommand("moveToDeadLetterQueue", { + numberOfKeys: 4, + lua: ` + local queue = KEYS[1] + local items = KEYS[2] + local dlq = KEYS[3] + local dlqItems = KEYS[4] + local id = ARGV[1] + local errorMessage = ARGV[2] + + local item = redis.call('HGET', items, id) + if not item then + return 0 + end + + local parsedItem = cjson.decode(item) + parsedItem.errorMessage = errorMessage + + redis.call('ZREM', queue, id) + redis.call('HDEL', items, id) + + redis.call('ZADD', dlq, redis.call('TIME')[1], id) + redis.call('HSET', dlqItems, id, cjson.encode(parsedItem)) + + return 1 + `, + }); + + this.redis.defineCommand("redriveFromDeadLetterQueue", { + numberOfKeys: 4, + lua: ` + local queue = KEYS[1] + local items = KEYS[2] + local dlq = KEYS[3] + local dlqItems = KEYS[4] + local id = ARGV[1] + + local item = redis.call('HGET', dlqItems, id) + if not item then + return 0 + end + + local parsedItem = cjson.decode(item) + parsedItem.errorMessage = nil + + redis.call('ZREM', dlq, id) + redis.call('HDEL', dlqItems, id) + + redis.call('ZADD', queue, redis.call('TIME')[1], id) + redis.call('HSET', items, id, cjson.encode(parsedItem)) + + return 1 + `, + }); } } @@ -306,6 +427,7 @@ declare module "ioredis" { serializedItem: string, callback?: Callback ): Result; + dequeueItems( //keys queue: string, @@ -322,5 +444,24 @@ declare module "ioredis" { id: string, callback?: Callback ): Result; + + redriveFromDeadLetterQueue( + queue: string, + items: string, + dlq: string, + dlqItems: string, + id: string, + callback?: Callback + ): Result; + + moveToDeadLetterQueue( + queue: string, + items: string, + dlq: string, + dlqItems: string, + id: string, + errorMessage: string, + callback?: Callback + ): Result; } } From aa84a696722b32c752744fb6af53837d88400904 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 8 Oct 2024 13:24:41 +0100 Subject: [PATCH 097/114] DLQ tests --- .../redis-worker/src/queue.test.ts | 165 ++++++++++++------ 1 file changed, 110 insertions(+), 55 deletions(-) diff --git a/internal-packages/redis-worker/src/queue.test.ts b/internal-packages/redis-worker/src/queue.test.ts index b254ff14ee..075961eba5 100644 --- a/internal-packages/redis-worker/src/queue.test.ts +++ b/internal-packages/redis-worker/src/queue.test.ts @@ -185,68 +185,123 @@ describe("SimpleQueue", () => { await queue.close(); } }); -}); -redisTest("dequeue multiple items", { timeout: 20_000 }, async ({ redisContainer }) => { - const queue = new SimpleQueue({ - name: "test-1", - schema: { - test: z.object({ - value: z.number(), - }), - }, - redisOptions: { - host: redisContainer.getHost(), - port: redisContainer.getPort(), - password: redisContainer.getPassword(), - }, - logger: new Logger("test", "log"), + redisTest("dequeue multiple items", { timeout: 20_000 }, async ({ redisContainer }) => { + const queue = new SimpleQueue({ + name: "test-1", + schema: { + test: z.object({ + value: z.number(), + }), + }, + redisOptions: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + }, + logger: new Logger("test", "log"), + }); + + try { + await queue.enqueue({ id: "1", job: "test", item: { value: 1 }, visibilityTimeoutMs: 2000 }); + await queue.enqueue({ id: "2", job: "test", item: { value: 2 }, visibilityTimeoutMs: 2000 }); + await queue.enqueue({ id: "3", job: "test", item: { value: 3 }, visibilityTimeoutMs: 2000 }); + + expect(await queue.size()).toBe(3); + + const dequeued = await queue.dequeue(2); + expect(dequeued).toHaveLength(2); + expect(dequeued[0]).toEqual({ + id: "1", + job: "test", + item: { value: 1 }, + visibilityTimeoutMs: 2000, + attempt: 0, + }); + expect(dequeued[1]).toEqual({ + id: "2", + job: "test", + item: { value: 2 }, + visibilityTimeoutMs: 2000, + attempt: 0, + }); + + expect(await queue.size()).toBe(1); + expect(await queue.size({ includeFuture: true })).toBe(3); + + await queue.ack(dequeued[0].id); + await queue.ack(dequeued[1].id); + + expect(await queue.size({ includeFuture: true })).toBe(1); + + const [last] = await queue.dequeue(1); + expect(last).toEqual({ + id: "3", + job: "test", + item: { value: 3 }, + visibilityTimeoutMs: 2000, + attempt: 0, + }); + + await queue.ack(last.id); + expect(await queue.size({ includeFuture: true })).toBe(0); + } finally { + await queue.close(); + } }); - try { - await queue.enqueue({ id: "1", job: "test", item: { value: 1 }, visibilityTimeoutMs: 2000 }); - await queue.enqueue({ id: "2", job: "test", item: { value: 2 }, visibilityTimeoutMs: 2000 }); - await queue.enqueue({ id: "3", job: "test", item: { value: 3 }, visibilityTimeoutMs: 2000 }); - - expect(await queue.size()).toBe(3); - - const dequeued = await queue.dequeue(2); - expect(dequeued).toHaveLength(2); - expect(dequeued[0]).toEqual({ - id: "1", - job: "test", - item: { value: 1 }, - visibilityTimeoutMs: 2000, - attempt: 0, - }); - expect(dequeued[1]).toEqual({ - id: "2", - job: "test", - item: { value: 2 }, - visibilityTimeoutMs: 2000, - attempt: 0, + redisTest("Dead Letter Queue", { timeout: 20_000 }, async ({ redisContainer }) => { + const queue = new SimpleQueue({ + name: "test-dlq", + schema: { + test: z.object({ + value: z.number(), + }), + }, + redisOptions: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + }, + logger: new Logger("test", "log"), }); - expect(await queue.size()).toBe(1); - expect(await queue.size({ includeFuture: true })).toBe(3); + try { + // Enqueue an item + await queue.enqueue({ id: "1", job: "test", item: { value: 1 }, visibilityTimeoutMs: 2000 }); + expect(await queue.size()).toBe(1); + expect(await queue.sizeOfDeadLetterQueue()).toBe(0); - await queue.ack(dequeued[0].id); - await queue.ack(dequeued[1].id); + // Move item to DLQ + await queue.moveToDeadLetterQueue("1", "Test error message"); + expect(await queue.size()).toBe(0); + expect(await queue.sizeOfDeadLetterQueue()).toBe(1); - expect(await queue.size({ includeFuture: true })).toBe(1); + // Attempt to dequeue from the main queue should return empty + const dequeued = await queue.dequeue(1); + expect(dequeued).toEqual([]); - const [last] = await queue.dequeue(1); - expect(last).toEqual({ - id: "3", - job: "test", - item: { value: 3 }, - visibilityTimeoutMs: 2000, - attempt: 0, - }); + // Redrive item from DLQ + await queue.redriveFromDeadLetterQueue("1"); + expect(await queue.size()).toBe(1); + expect(await queue.sizeOfDeadLetterQueue()).toBe(0); - await queue.ack(last.id); - expect(await queue.size({ includeFuture: true })).toBe(0); - } finally { - await queue.close(); - } + // Dequeue the redriven item + const [redrivenItem] = await queue.dequeue(1); + expect(redrivenItem).toEqual({ + id: "1", + job: "test", + item: { value: 1 }, + visibilityTimeoutMs: 2000, + attempt: 0, + }); + + // Acknowledge the item + await queue.ack(redrivenItem.id); + expect(await queue.size()).toBe(0); + expect(await queue.sizeOfDeadLetterQueue()).toBe(0); + } finally { + await queue.close(); + } + }); }); From 62c8b2c192eee855225219a1431576ca16cf8e87 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 8 Oct 2024 13:26:18 +0100 Subject: [PATCH 098/114] Same cluster for all keys in the same queue --- internal-packages/redis-worker/src/queue.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-packages/redis-worker/src/queue.ts b/internal-packages/redis-worker/src/queue.ts index 8f0065da76..04c08a30d2 100644 --- a/internal-packages/redis-worker/src/queue.ts +++ b/internal-packages/redis-worker/src/queue.ts @@ -33,7 +33,7 @@ export class SimpleQueue { this.name = name; this.redis = new Redis({ ...redisOptions, - keyPrefix: `queue:${name}:`, + keyPrefix: `{queue:${name}:}`, retryStrategy(times) { const delay = Math.min(times * 50, 1000); return delay; From da02da9af5d7ee57ec5e89dff83ef6f757e7ea8d Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 8 Oct 2024 13:38:16 +0100 Subject: [PATCH 099/114] Added DLQ tests --- .../redis-worker/src/worker.test.ts | 139 ++++++++++++++---- internal-packages/redis-worker/src/worker.ts | 29 ++-- 2 files changed, 126 insertions(+), 42 deletions(-) diff --git a/internal-packages/redis-worker/src/worker.test.ts b/internal-packages/redis-worker/src/worker.test.ts index 5aafb4345a..108fa03d37 100644 --- a/internal-packages/redis-worker/src/worker.test.ts +++ b/internal-packages/redis-worker/src/worker.test.ts @@ -1,10 +1,9 @@ import { redisTest } from "@internal/testcontainers"; -import { describe, it } from "node:test"; +import { Logger } from "@trigger.dev/core/logger"; +import { describe } from "node:test"; import { expect } from "vitest"; import { z } from "zod"; import { Worker } from "./worker.js"; -import { Logger } from "@trigger.dev/core/logger"; -import { SimpleQueue } from "./queue.js"; describe("Worker", () => { redisTest("Process items that don't throw", { timeout: 30_000 }, async ({ redisContainer }) => { @@ -35,26 +34,27 @@ describe("Worker", () => { }, logger: new Logger("test", "log"), }); + try { + // Enqueue 10 items + for (let i = 0; i < 10; i++) { + await worker.enqueue({ + id: `item-${i}`, + job: "testJob", + payload: { value: i }, + visibilityTimeoutMs: 5000, + }); + } - // Enqueue 10 items - for (let i = 0; i < 10; i++) { - await worker.enqueue({ - id: `item-${i}`, - job: "testJob", - payload: { value: i }, - visibilityTimeoutMs: 5000, - }); - } - - worker.start(); - - // Wait for items to be processed - await new Promise((resolve) => setTimeout(resolve, 600)); + worker.start(); - worker.stop(); + // Wait for items to be processed + await new Promise((resolve) => setTimeout(resolve, 600)); - expect(processedItems.length).toBe(10); - expect(new Set(processedItems).size).toBe(10); // Ensure all items were processed uniquely + expect(processedItems.length).toBe(10); + expect(new Set(processedItems).size).toBe(10); // Ensure all items were processed uniquely + } finally { + worker.stop(); + } }); redisTest( @@ -97,28 +97,103 @@ describe("Worker", () => { logger: new Logger("test", "error"), }); - // Enqueue 10 items - for (let i = 0; i < 10; i++) { + try { + // Enqueue 10 items + for (let i = 0; i < 10; i++) { + await worker.enqueue({ + id: `item-${i}`, + job: "testJob", + payload: { value: i }, + visibilityTimeoutMs: 5000, + }); + } + + worker.start(); + + // Wait for items to be processed + await new Promise((resolve) => setTimeout(resolve, 500)); + + expect(processedItems.length).toBe(10); + expect(new Set(processedItems).size).toBe(10); // Ensure all items were processed uniquely + } finally { + worker.stop(); + } + } + ); + + redisTest( + "Process an item that permanently fails and ends up in DLQ", + { timeout: 30_000 }, + async ({ redisContainer }) => { + const processedItems: number[] = []; + const failedItemId = "permanent-fail-item"; + + const worker = new Worker({ + name: "test-worker", + redisOptions: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + }, + catalog: { + testJob: { + schema: z.object({ value: z.number() }), + visibilityTimeoutMs: 1000, + retry: { maxAttempts: 3, minTimeoutInMs: 10, maxTimeoutInMs: 50 }, + }, + }, + jobs: { + testJob: async ({ id, payload }) => { + if (id === failedItemId) { + throw new Error("Permanent failure"); + } + processedItems.push(payload.value); + }, + }, + concurrency: { + workers: 1, + tasksPerWorker: 1, + }, + pollIntervalMs: 50, + logger: new Logger("test", "error"), + }); + + try { + // Enqueue the item that will permanently fail await worker.enqueue({ - id: `item-${i}`, + id: failedItemId, job: "testJob", - payload: { value: i }, - visibilityTimeoutMs: 5000, + payload: { value: 999 }, }); - } - worker.start(); + // Enqueue a normal item + await worker.enqueue({ + id: "normal-item", + job: "testJob", + payload: { value: 1 }, + }); - // Wait for items to be processed - await new Promise((resolve) => setTimeout(resolve, 500)); + worker.start(); - worker.stop(); + // Wait for items to be processed and retried + await new Promise((resolve) => setTimeout(resolve, 1000)); - expect(processedItems.length).toBe(10); - expect(new Set(processedItems).size).toBe(10); // Ensure all items were processed uniquely + // Check that the normal item was processed + expect(processedItems).toEqual([1]); + + // Check that the failed item is in the DLQ + const dlqSize = await worker.queue.sizeOfDeadLetterQueue(); + expect(dlqSize).toBe(1); + } finally { + worker.stop(); + } } ); //todo test that throwing an error doesn't screw up the other items //todo process more items when finished + + //todo add a Dead Letter Queue when items are failed, with the error + //todo add a function on the worker to redrive them + //todo add an API endpoint to redrive with an ID }); diff --git a/internal-packages/redis-worker/src/worker.ts b/internal-packages/redis-worker/src/worker.ts index 09aac1325a..9a09b82b9a 100644 --- a/internal-packages/redis-worker/src/worker.ts +++ b/internal-packages/redis-worker/src/worker.ts @@ -42,7 +42,7 @@ type WorkerOptions = { }; class Worker { - private queue: SimpleQueue>; + queue: SimpleQueue>; private jobs: WorkerOptions["jobs"]; private logger: Logger; private workers: NodeWorker[] = []; @@ -157,8 +157,11 @@ class Worker { try { await handler({ id, payload: item, visibilityTimeoutMs, attempt }); + + //succeeded, acking the item await this.queue.ack(id); } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); this.logger.error(`Error processing item, it threw an error:`, { name: this.options.name, id, @@ -166,6 +169,7 @@ class Worker { item, visibilityTimeoutMs, error, + errorMessage, }); // Requeue the failed item with a delay try { @@ -174,15 +178,20 @@ class Worker { const retryDelay = calculateNextRetryDelay(catalogItem.retry, attempt); if (!retryDelay) { - this.logger.error(`Failed item ${id} has reached max attempts, acking.`, { - name: this.options.name, - id, - job, - item, - visibilityTimeoutMs, - attempt, - }); - await this.queue.ack(id); + this.logger.error( + `Failed item ${id} has reached max attempts, moving to the DLQ.`, + { + name: this.options.name, + id, + job, + item, + visibilityTimeoutMs, + attempt, + errorMessage, + } + ); + + await this.queue.moveToDeadLetterQueue(id, errorMessage); return; } From decf82605b38a83a5de33108c6bbd160c57342e0 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 8 Oct 2024 13:48:44 +0100 Subject: [PATCH 100/114] Whitespace --- internal-packages/redis-worker/src/worker.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/internal-packages/redis-worker/src/worker.test.ts b/internal-packages/redis-worker/src/worker.test.ts index 108fa03d37..88583b18fc 100644 --- a/internal-packages/redis-worker/src/worker.test.ts +++ b/internal-packages/redis-worker/src/worker.test.ts @@ -34,6 +34,7 @@ describe("Worker", () => { }, logger: new Logger("test", "log"), }); + try { // Enqueue 10 items for (let i = 0; i < 10; i++) { From d5fdf7fa8a61fe995b81b5991bc29744580a6a21 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 8 Oct 2024 14:08:47 +0100 Subject: [PATCH 101/114] Redis pubsub to redrive from the worker --- .../redis-worker/src/worker.test.ts | 84 +++++++++++++++++++ internal-packages/redis-worker/src/worker.ts | 39 ++++++++- 2 files changed, 121 insertions(+), 2 deletions(-) diff --git a/internal-packages/redis-worker/src/worker.test.ts b/internal-packages/redis-worker/src/worker.test.ts index 88583b18fc..a55a653887 100644 --- a/internal-packages/redis-worker/src/worker.test.ts +++ b/internal-packages/redis-worker/src/worker.test.ts @@ -4,6 +4,7 @@ import { describe } from "node:test"; import { expect } from "vitest"; import { z } from "zod"; import { Worker } from "./worker.js"; +import Redis from "ioredis"; describe("Worker", () => { redisTest("Process items that don't throw", { timeout: 30_000 }, async ({ redisContainer }) => { @@ -191,6 +192,89 @@ describe("Worker", () => { } ); + redisTest( + "Redrive an item from DLQ and process it successfully", + { timeout: 30_000 }, + async ({ redisContainer }) => { + const processedItems: number[] = []; + const failedItemId = "fail-then-redrive-item"; + let attemptCount = 0; + + const worker = new Worker({ + name: "test-worker", + redisOptions: { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + }, + catalog: { + testJob: { + schema: z.object({ value: z.number() }), + visibilityTimeoutMs: 1000, + retry: { maxAttempts: 3, minTimeoutInMs: 10, maxTimeoutInMs: 50 }, + }, + }, + jobs: { + testJob: async ({ id, payload }) => { + if (id === failedItemId && attemptCount < 3) { + attemptCount++; + throw new Error("Temporary failure"); + } + processedItems.push(payload.value); + }, + }, + concurrency: { + workers: 1, + tasksPerWorker: 1, + }, + pollIntervalMs: 50, + logger: new Logger("test", "error"), + }); + + try { + // Enqueue the item that will fail 3 times + await worker.enqueue({ + id: failedItemId, + job: "testJob", + payload: { value: 999 }, + }); + + worker.start(); + + // Wait for the item to be processed and moved to DLQ + await new Promise((resolve) => setTimeout(resolve, 1000)); + + // Check that the item is in the DLQ + let dlqSize = await worker.queue.sizeOfDeadLetterQueue(); + expect(dlqSize).toBe(1); + + // Create a Redis client to publish the redrive message + const redisClient = new Redis({ + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + }); + + // Publish redrive message + await redisClient.publish("test-worker:redrive", JSON.stringify({ id: failedItemId })); + + // Wait for the item to be redrived and processed + await new Promise((resolve) => setTimeout(resolve, 1000)); + + // Check that the item was processed successfully + expect(processedItems).toEqual([999]); + + // Check that the DLQ is now empty + dlqSize = await worker.queue.sizeOfDeadLetterQueue(); + expect(dlqSize).toBe(0); + + await redisClient.quit(); + } finally { + worker.stop(); + } + } + ); + //todo test that throwing an error doesn't screw up the other items //todo process more items when finished diff --git a/internal-packages/redis-worker/src/worker.ts b/internal-packages/redis-worker/src/worker.ts index 9a09b82b9a..601f5e1708 100644 --- a/internal-packages/redis-worker/src/worker.ts +++ b/internal-packages/redis-worker/src/worker.ts @@ -7,6 +7,8 @@ import { Worker as NodeWorker } from "worker_threads"; import { z } from "zod"; import { SimpleQueue } from "./queue.js"; +import Redis from "ioredis"; + type WorkerCatalog = { [key: string]: { schema: z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion; @@ -42,6 +44,8 @@ type WorkerOptions = { }; class Worker { + private subscriber: Redis; + queue: SimpleQueue>; private jobs: WorkerOptions["jobs"]; private logger: Logger; @@ -55,7 +59,7 @@ class Worker { const schema: QueueCatalogFromWorkerCatalog = Object.fromEntries( Object.entries(this.options.catalog).map(([key, value]) => [key, value.schema]) ) as QueueCatalogFromWorkerCatalog; - + // this.queue = new SimpleQueue({ name: options.name, redisOptions: options.redisOptions, @@ -74,6 +78,9 @@ class Worker { } this.setupShutdownHandlers(); + + this.subscriber = new Redis(options.redisOptions); + this.setupSubscriber(); } enqueue({ @@ -240,6 +247,32 @@ class Worker { this.processItems(worker, count); } + private setupSubscriber() { + const channel = `${this.options.name}:redrive`; + this.subscriber.subscribe(channel, (err) => { + if (err) { + this.logger.error(`Failed to subscribe to ${channel}`, { error: err }); + } else { + this.logger.log(`Subscribed to ${channel}`); + } + }); + + this.subscriber.on("message", this.handleRedriveMessage.bind(this)); + } + + private async handleRedriveMessage(channel: string, message: string) { + try { + const { id } = JSON.parse(message); + if (typeof id !== "string") { + throw new Error("Invalid message format: id must be a string"); + } + await this.queue.redriveFromDeadLetterQueue(id); + this.logger.log(`Redrived item ${id} from Dead Letter Queue`); + } catch (error) { + this.logger.error("Error processing redrive message", { error, message }); + } + } + private setupShutdownHandlers() { process.on("SIGTERM", this.shutdown.bind(this)); process.on("SIGINT", this.shutdown.bind(this)); @@ -254,8 +287,10 @@ class Worker { worker.terminate(); } + await this.subscriber.unsubscribe(); + await this.subscriber.quit(); await this.queue.close(); - this.logger.log("All workers shut down."); + this.logger.log("All workers and subscribers shut down."); } public start() { From 844cb77ab11ecea3271fa2ca1566273044d88cdc Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 8 Oct 2024 15:37:52 +0100 Subject: [PATCH 102/114] Fixed database paths --- .../ConfigureEndpointSheet.tsx | 2 +- .../route.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/ConfigureEndpointSheet.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/ConfigureEndpointSheet.tsx index ed824a9095..cc241ea180 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/ConfigureEndpointSheet.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/ConfigureEndpointSheet.tsx @@ -22,7 +22,7 @@ import { Paragraph } from "~/components/primitives/Paragraph"; import { Sheet, SheetBody, SheetContent, SheetHeader } from "~/components/primitives/Sheet"; import { ClientEndpoint } from "~/presenters/EnvironmentsPresenter.server"; import { endpointStreamingPath } from "~/utils/pathBuilder"; -import { EndpointIndexStatus, RuntimeEnvironmentType } from "../../../../../packages/database/src"; +import { EndpointIndexStatus, RuntimeEnvironmentType } from "@trigger.dev/database"; import { bodySchema } from "../resources.environments.$environmentParam.endpoint"; type ConfigureEndpointSheetProps = { diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/route.tsx index fd28fced20..94f97f085e 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/route.tsx @@ -39,7 +39,7 @@ import { projectEnvironmentsStreamingPath, } from "~/utils/pathBuilder"; import { requestUrl } from "~/utils/requestUrl.server"; -import { RuntimeEnvironmentType } from "../../../../../packages/database/src"; +import { RuntimeEnvironmentType } from "@trigger.dev/database"; import { ConfigureEndpointSheet } from "./ConfigureEndpointSheet"; import { FirstEndpointSheet } from "./FirstEndpointSheet"; import { BookOpenIcon } from "@heroicons/react/20/solid"; From 25ee43bf12270f95818ef6fd6fd4d28fbcfa7f30 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 8 Oct 2024 15:37:59 +0100 Subject: [PATCH 103/114] Fix for path to zod-worker --- apps/webapp/app/services/runExecutionRateLimiter.server.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/webapp/app/services/runExecutionRateLimiter.server.ts b/apps/webapp/app/services/runExecutionRateLimiter.server.ts index df82574dcb..ff327c201c 100644 --- a/apps/webapp/app/services/runExecutionRateLimiter.server.ts +++ b/apps/webapp/app/services/runExecutionRateLimiter.server.ts @@ -11,7 +11,7 @@ import { import { JobHelpers, Task } from "graphile-worker"; import { singleton } from "~/utils/singleton"; import { logger } from "./logger.server"; -import { ZodWorkerRateLimiter } from "~/platform/zodWorker.server"; +import { ZodWorkerRateLimiter } from "@internal/zod-worker"; import { ConcurrencyLimitGroup, JobRun, @@ -117,7 +117,7 @@ if currentSize < maxSize then return true else redis.call('SADD', forbiddenFlagsKey, forbiddenFlag) - + return false end `, From 932dd5e3089ceaf519817a515a302540e20dae36 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 8 Oct 2024 15:38:26 +0100 Subject: [PATCH 104/114] Fixes for typecheck errors, mostly with TS versions and module resolution --- internal-packages/redis-worker/package.json | 3 +- internal-packages/redis-worker/tsconfig.json | 2 +- internal-packages/run-engine/package.json | 3 +- internal-packages/run-engine/tsconfig.json | 2 +- .../testcontainers/tsconfig.json | 7 ++--- internal-packages/zod-worker/package.json | 2 +- internal-packages/zod-worker/tsconfig.json | 2 +- pnpm-lock.yaml | 31 ++++++++++--------- 8 files changed, 27 insertions(+), 25 deletions(-) diff --git a/internal-packages/redis-worker/package.json b/internal-packages/redis-worker/package.json index 83df6f862c..29e7519ad2 100644 --- a/internal-packages/redis-worker/package.json +++ b/internal-packages/redis-worker/package.json @@ -4,13 +4,14 @@ "version": "0.0.1", "main": "./src/index.ts", "types": "./src/index.ts", + "type": "module", "dependencies": { "@opentelemetry/api": "^1.9.0", "@trigger.dev/core": "workspace:*", "ioredis": "^5.3.2", "lodash.omit": "^4.5.0", "nanoid": "^5.0.7", - "typescript": "^4.8.4", + "typescript": "^5.5.4", "zod": "3.22.3" }, "devDependencies": { diff --git a/internal-packages/redis-worker/tsconfig.json b/internal-packages/redis-worker/tsconfig.json index 53593e2d26..766df37eae 100644 --- a/internal-packages/redis-worker/tsconfig.json +++ b/internal-packages/redis-worker/tsconfig.json @@ -3,7 +3,7 @@ "target": "ES2019", "lib": ["ES2019", "DOM", "DOM.Iterable"], "module": "CommonJS", - "moduleResolution": "Node10", + "moduleResolution": "Node", "moduleDetection": "force", "verbatimModuleSyntax": false, "types": ["vitest/globals"], diff --git a/internal-packages/run-engine/package.json b/internal-packages/run-engine/package.json index 9c4af18104..d0012b035e 100644 --- a/internal-packages/run-engine/package.json +++ b/internal-packages/run-engine/package.json @@ -4,6 +4,7 @@ "version": "0.0.1", "main": "./src/index.ts", "types": "./src/index.ts", + "type": "module", "dependencies": { "@internal/zod-worker": "workspace:*", "@opentelemetry/api": "^1.9.0", @@ -13,7 +14,7 @@ "ioredis": "^5.3.2", "nanoid": "^3.3.4", "redlock": "5.0.0-beta.2", - "typescript": "^4.8.4", + "typescript": "^5.5.4", "zod": "3.22.3" }, "devDependencies": { diff --git a/internal-packages/run-engine/tsconfig.json b/internal-packages/run-engine/tsconfig.json index 096276b211..515b521967 100644 --- a/internal-packages/run-engine/tsconfig.json +++ b/internal-packages/run-engine/tsconfig.json @@ -3,7 +3,7 @@ "target": "ES2019", "lib": ["ES2019", "DOM", "DOM.Iterable"], "module": "CommonJS", - "moduleResolution": "Node10", + "moduleResolution": "Node", "moduleDetection": "force", "verbatimModuleSyntax": false, "types": ["vitest/globals"], diff --git a/internal-packages/testcontainers/tsconfig.json b/internal-packages/testcontainers/tsconfig.json index ce63734a69..4a36f08ffc 100644 --- a/internal-packages/testcontainers/tsconfig.json +++ b/internal-packages/testcontainers/tsconfig.json @@ -1,11 +1,10 @@ { "compilerOptions": { - "target": "ES2019", - "lib": ["ES2019", "DOM", "DOM.Iterable"], + "target": "es2022", + "lib": ["ES2022", "DOM", "DOM.Iterable"], "module": "CommonJS", - "moduleResolution": "Node10", + "moduleResolution": "Node", "moduleDetection": "force", - "verbatimModuleSyntax": false, "types": ["vitest/globals"], "esModuleInterop": true, "forceConsistentCasingInFileNames": true, diff --git a/internal-packages/zod-worker/package.json b/internal-packages/zod-worker/package.json index 3c6d0c1f9b..a10886d659 100644 --- a/internal-packages/zod-worker/package.json +++ b/internal-packages/zod-worker/package.json @@ -10,7 +10,7 @@ "@trigger.dev/database": "workspace:*", "graphile-worker": "0.16.6", "lodash.omit": "^4.5.0", - "typescript": "^4.8.4", + "typescript": "^5.5.4", "zod": "3.22.3" }, "devDependencies": { diff --git a/internal-packages/zod-worker/tsconfig.json b/internal-packages/zod-worker/tsconfig.json index bfd15dde0d..66ecfc9677 100644 --- a/internal-packages/zod-worker/tsconfig.json +++ b/internal-packages/zod-worker/tsconfig.json @@ -3,7 +3,7 @@ "target": "ES2019", "lib": ["ES2019", "DOM", "DOM.Iterable"], "module": "CommonJS", - "moduleResolution": "Node10", + "moduleResolution": "Node", "moduleDetection": "force", "verbatimModuleSyntax": false, "types": ["vitest/globals"], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 57ddf80d5b..2bdb89cc0e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -913,8 +913,8 @@ importers: specifier: ^5.0.7 version: 5.0.7 typescript: - specifier: ^4.8.4 - version: 4.9.5 + specifier: ^5.5.4 + version: 5.5.4 zod: specifier: 3.22.3 version: 3.22.3 @@ -956,8 +956,8 @@ importers: specifier: 5.0.0-beta.2 version: 5.0.0-beta.2 typescript: - specifier: ^4.8.4 - version: 4.9.5 + specifier: ^5.5.4 + version: 5.5.4 zod: specifier: 3.22.3 version: 3.22.3 @@ -1010,13 +1010,13 @@ importers: version: link:../database graphile-worker: specifier: 0.16.6 - version: 0.16.6(patch_hash=hdpetta7btqcc7xb5wfkcnanoa)(typescript@4.9.5) + version: 0.16.6(patch_hash=hdpetta7btqcc7xb5wfkcnanoa)(typescript@5.5.4) lodash.omit: specifier: ^4.5.0 version: 4.5.0 typescript: - specifier: ^4.8.4 - version: 4.9.5 + specifier: ^5.5.4 + version: 5.5.4 zod: specifier: 3.22.3 version: 3.22.3 @@ -17447,7 +17447,7 @@ packages: yaml: 1.10.2 dev: true - /cosmiconfig@8.3.6(typescript@4.9.5): + /cosmiconfig@8.3.6(typescript@5.2.2): resolution: {integrity: sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==} engines: {node: '>=14'} peerDependencies: @@ -17460,10 +17460,10 @@ packages: js-yaml: 4.1.0 parse-json: 5.2.0 path-type: 4.0.0 - typescript: 4.9.5 + typescript: 5.2.2 dev: false - /cosmiconfig@8.3.6(typescript@5.2.2): + /cosmiconfig@8.3.6(typescript@5.5.4): resolution: {integrity: sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==} engines: {node: '>=14'} peerDependencies: @@ -17476,7 +17476,7 @@ packages: js-yaml: 4.1.0 parse-json: 5.2.0 path-type: 4.0.0 - typescript: 5.2.2 + typescript: 5.5.4 dev: false /cosmiconfig@9.0.0(typescript@5.2.2): @@ -19480,6 +19480,7 @@ packages: /eslint@8.45.0: resolution: {integrity: sha512-pd8KSxiQpdYRfYa9Wufvdoct3ZPQQuVuU5O6scNgMuOMYuxvH0IGaYK0wUFjo4UYYQQCUndlXiMbnxopwvvTiw==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options. hasBin: true dependencies: '@eslint-community/eslint-utils': 4.4.0(eslint@8.45.0) @@ -20601,7 +20602,7 @@ packages: - supports-color dev: false - /graphile-worker@0.16.6(patch_hash=hdpetta7btqcc7xb5wfkcnanoa)(typescript@4.9.5): + /graphile-worker@0.16.6(patch_hash=hdpetta7btqcc7xb5wfkcnanoa)(typescript@5.2.2): resolution: {integrity: sha512-e7gGYDmGqzju2l83MpzX8vNG/lOtVJiSzI3eZpAFubSxh/cxs7sRrRGBGjzBP1kNG0H+c95etPpNRNlH65PYhw==} engines: {node: '>=14.0.0'} hasBin: true @@ -20609,7 +20610,7 @@ packages: '@graphile/logger': 0.2.0 '@types/debug': 4.1.12 '@types/pg': 8.11.6 - cosmiconfig: 8.3.6(typescript@4.9.5) + cosmiconfig: 8.3.6(typescript@5.2.2) graphile-config: 0.0.1-beta.8 json5: 2.2.3 pg: 8.11.5 @@ -20622,7 +20623,7 @@ packages: dev: false patched: true - /graphile-worker@0.16.6(patch_hash=hdpetta7btqcc7xb5wfkcnanoa)(typescript@5.2.2): + /graphile-worker@0.16.6(patch_hash=hdpetta7btqcc7xb5wfkcnanoa)(typescript@5.5.4): resolution: {integrity: sha512-e7gGYDmGqzju2l83MpzX8vNG/lOtVJiSzI3eZpAFubSxh/cxs7sRrRGBGjzBP1kNG0H+c95etPpNRNlH65PYhw==} engines: {node: '>=14.0.0'} hasBin: true @@ -20630,7 +20631,7 @@ packages: '@graphile/logger': 0.2.0 '@types/debug': 4.1.12 '@types/pg': 8.11.6 - cosmiconfig: 8.3.6(typescript@5.2.2) + cosmiconfig: 8.3.6(typescript@5.5.4) graphile-config: 0.0.1-beta.8 json5: 2.2.3 pg: 8.11.5 From c0ad06548825dc5aeb4fb0ff4bc86b37838ae66d Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 8 Oct 2024 16:16:44 +0100 Subject: [PATCH 105/114] Redlock required a patch --- internal-packages/run-engine/package.json | 1 - package.json | 9 +- patches/redlock@5.0.0-beta.2.patch | 15 +++ pnpm-lock.yaml | 132 +++++++++------------- 4 files changed, 75 insertions(+), 82 deletions(-) create mode 100644 patches/redlock@5.0.0-beta.2.patch diff --git a/internal-packages/run-engine/package.json b/internal-packages/run-engine/package.json index d0012b035e..a2cf01cbdc 100644 --- a/internal-packages/run-engine/package.json +++ b/internal-packages/run-engine/package.json @@ -4,7 +4,6 @@ "version": "0.0.1", "main": "./src/index.ts", "types": "./src/index.ts", - "type": "module", "dependencies": { "@internal/zod-worker": "workspace:*", "@opentelemetry/api": "^1.9.0", diff --git a/package.json b/package.json index 2021b00840..a7e006755d 100644 --- a/package.json +++ b/package.json @@ -51,16 +51,16 @@ "@playwright/test": "^1.36.2", "@trigger.dev/database": "workspace:*", "@types/node": "20.14.14", - "typescript": "^5.5.4", "autoprefixer": "^10.4.12", "eslint-plugin-turbo": "^2.0.4", + "pkg-types": "1.1.3", "prettier": "^3.0.0", "tsx": "^3.7.1", "turbo": "^1.10.3", + "typescript": "^5.5.4", "vite": "^4.1.1", "vite-tsconfig-paths": "^4.0.5", - "vitest": "^0.28.4", - "pkg-types": "1.1.3" + "vitest": "^0.28.4" }, "packageManager": "pnpm@8.15.5", "dependencies": { @@ -72,7 +72,8 @@ "patchedDependencies": { "@changesets/assemble-release-plan@5.2.4": "patches/@changesets__assemble-release-plan@5.2.4.patch", "engine.io-parser@5.2.2": "patches/engine.io-parser@5.2.2.patch", - "graphile-worker@0.16.6": "patches/graphile-worker@0.16.6.patch" + "graphile-worker@0.16.6": "patches/graphile-worker@0.16.6.patch", + "redlock@5.0.0-beta.2": "patches/redlock@5.0.0-beta.2.patch" } } } \ No newline at end of file diff --git a/patches/redlock@5.0.0-beta.2.patch b/patches/redlock@5.0.0-beta.2.patch new file mode 100644 index 0000000000..a13e6fcfe6 --- /dev/null +++ b/patches/redlock@5.0.0-beta.2.patch @@ -0,0 +1,15 @@ +diff --git a/CHANGELOG.md b/CHANGELOG.md +deleted file mode 100644 +index 8700963f5c9ec9fc1fdc53846846130bfc2f2108..0000000000000000000000000000000000000000 +diff --git a/package.json b/package.json +index 606d1775fb0c11aab4918c50d20d72bce1684158..45e6d46410792ef273d7d2b2bc92cd5964a3eef5 100644 +--- a/package.json ++++ b/package.json +@@ -15,6 +15,7 @@ + "types": "./dist/index.d.ts", + "exports": { + ".": { ++ "types": "./dist/index.d.ts", + "import": "./dist/esm/index.js", + "require": "./dist/cjs/index.js" + } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2bdb89cc0e..621a077da4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,9 @@ patchedDependencies: graphile-worker@0.16.6: hash: hdpetta7btqcc7xb5wfkcnanoa path: patches/graphile-worker@0.16.6.patch + redlock@5.0.0-beta.2: + hash: rwyegdki7iserrd7fgjwxkhnlu + path: patches/redlock@5.0.0-beta.2.patch importers: @@ -954,7 +957,7 @@ importers: version: 3.3.7 redlock: specifier: 5.0.0-beta.2 - version: 5.0.0-beta.2 + version: 5.0.0-beta.2(patch_hash=rwyegdki7iserrd7fgjwxkhnlu) typescript: specifier: ^5.5.4 version: 5.5.4 @@ -1867,7 +1870,7 @@ packages: dependencies: '@andrewbranch/untar.js': 1.0.3 fflate: 0.8.2 - semver: 7.5.4 + semver: 7.6.3 ts-expose-internals-conditionally: 1.0.0-empty.0 typescript: 5.3.3 validate-npm-package-name: 5.0.0 @@ -7498,7 +7501,7 @@ packages: '@types/shimmer': 1.0.2 import-in-the-middle: 1.7.4 require-in-the-middle: 7.1.1 - semver: 7.5.4 + semver: 7.6.3 shimmer: 1.2.1 transitivePeerDependencies: - supports-color @@ -7902,7 +7905,7 @@ packages: '@opentelemetry/propagator-b3': 1.22.0(@opentelemetry/api@1.4.1) '@opentelemetry/propagator-jaeger': 1.22.0(@opentelemetry/api@1.4.1) '@opentelemetry/sdk-trace-base': 1.22.0(@opentelemetry/api@1.4.1) - semver: 7.5.4 + semver: 7.6.3 dev: true /@opentelemetry/sdk-trace-node@1.25.1(@opentelemetry/api@1.4.1): @@ -16074,10 +16077,10 @@ packages: resolution: {integrity: sha512-sgTbLvL6cNnw24FnbaDyjmvddQ2ML8arZsgaJhoABMoplz/4QRhtrYS+alr1BUM1Bwp6dhx8vVCBSLG+StwOFw==} engines: {node: '>= 0.4'} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 define-properties: 1.1.4 es-abstract: 1.21.1 - get-intrinsic: 1.1.3 + get-intrinsic: 1.2.4 is-string: 1.0.7 dev: true @@ -16113,7 +16116,7 @@ packages: resolution: {integrity: sha512-roTU0KWIOmJ4DRLmwKd19Otg0/mT3qPNt0Qb3GWW8iObuZXxrjB/pzn0R3hqpRSWg4HCwqx+0vwOnWnvlOyeIA==} engines: {node: '>= 0.4'} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 define-properties: 1.1.4 es-abstract: 1.21.1 es-shim-unscopables: 1.0.0 @@ -16133,7 +16136,7 @@ packages: resolution: {integrity: sha512-8UGn9O1FDVvMNB0UlLv4voxRMze7+FpHyF5mSMRjWHUMlpoDViniy05870VlxhfgTnLbpuwTzvD76MTtWxB/mQ==} engines: {node: '>= 0.4'} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 define-properties: 1.1.4 es-abstract: 1.21.1 es-shim-unscopables: 1.0.0 @@ -16152,11 +16155,11 @@ packages: /array.prototype.tosorted@1.1.1: resolution: {integrity: sha512-pZYPXPRl2PqWcsUs6LOMn+1f1532nEoPTYowBtqLwAW+W8vSVhkIGnmOX1t/UQjD6YGI0vcD2B1U7ZFGQH9jnQ==} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 define-properties: 1.1.4 es-abstract: 1.21.1 es-shim-unscopables: 1.0.0 - get-intrinsic: 1.1.3 + get-intrinsic: 1.2.4 dev: true /arraybuffer.prototype.slice@1.0.3: @@ -16802,7 +16805,8 @@ packages: resolution: {integrity: sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==} dependencies: function-bind: 1.1.2 - get-intrinsic: 1.1.3 + get-intrinsic: 1.2.4 + dev: true /call-bind@1.0.7: resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==} @@ -16813,7 +16817,6 @@ packages: function-bind: 1.1.2 get-intrinsic: 1.2.4 set-function-length: 1.2.2 - dev: true /callsites@3.1.0: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} @@ -18021,7 +18024,6 @@ packages: es-define-property: 1.0.0 es-errors: 1.3.0 gopd: 1.0.1 - dev: true /define-lazy-prop@2.0.0: resolution: {integrity: sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==} @@ -18037,7 +18039,7 @@ packages: resolution: {integrity: sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==} engines: {node: '>= 0.4'} dependencies: - has-property-descriptors: 1.0.0 + has-property-descriptors: 1.0.2 object-keys: 1.1.1 /define-properties@1.2.1: @@ -18045,7 +18047,7 @@ packages: engines: {node: '>= 0.4'} dependencies: define-data-property: 1.1.4 - has-property-descriptors: 1.0.0 + has-property-descriptors: 1.0.2 object-keys: 1.1.1 dev: true @@ -18441,18 +18443,18 @@ packages: engines: {node: '>= 0.4'} dependencies: available-typed-arrays: 1.0.5 - call-bind: 1.0.2 + call-bind: 1.0.7 es-set-tostringtag: 2.0.1 es-to-primitive: 1.2.1 function-bind: 1.1.2 function.prototype.name: 1.1.5 - get-intrinsic: 1.1.3 + get-intrinsic: 1.2.4 get-symbol-description: 1.0.0 globalthis: 1.0.3 gopd: 1.0.1 has: 1.0.3 - has-property-descriptors: 1.0.0 - has-proto: 1.0.1 + has-property-descriptors: 1.0.2 + has-proto: 1.0.3 has-symbols: 1.0.3 internal-slot: 1.0.4 is-array-buffer: 3.0.1 @@ -18531,12 +18533,10 @@ packages: engines: {node: '>= 0.4'} dependencies: get-intrinsic: 1.2.4 - dev: true /es-errors@1.3.0: resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} engines: {node: '>= 0.4'} - dev: true /es-module-lexer@1.3.1: resolution: {integrity: sha512-JUFAyicQV9mXc3YRxPnDlrfBKpqt6hUYzz9/boprUJHs4e4KVr3XwOF70doO6gwXUor6EWZJAyWAfKki84t20Q==} @@ -18552,7 +18552,7 @@ packages: resolution: {integrity: sha512-g3OMbtlwY3QewlqAiMLI47KywjWZoEytKr8pf6iTC8uJq5bIAH52Z9pnQ8pVL6whrCto53JZDuUIsifGeLorTg==} engines: {node: '>= 0.4'} dependencies: - get-intrinsic: 1.1.3 + get-intrinsic: 1.2.4 has: 1.0.3 has-tostringtag: 1.0.0 @@ -20221,9 +20221,6 @@ packages: requiresBuild: true optional: true - /function-bind@1.1.1: - resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==} - /function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} @@ -20231,7 +20228,7 @@ packages: resolution: {integrity: sha512-uN7m/BzVKQnCUF/iW8jYea67v++2u7m5UgENbHRtdDVclOUP+FMPlCNdmk0h/ysGyo2tavMJEDqJAkJdRa1vMA==} engines: {node: '>= 0.4'} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 define-properties: 1.1.4 es-abstract: 1.21.1 functions-have-names: 1.2.3 @@ -20278,23 +20275,15 @@ packages: resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==} dev: true - /get-intrinsic@1.1.3: - resolution: {integrity: sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==} - dependencies: - function-bind: 1.1.2 - has: 1.0.3 - has-symbols: 1.0.3 - /get-intrinsic@1.2.4: resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==} engines: {node: '>= 0.4'} dependencies: es-errors: 1.3.0 function-bind: 1.1.2 - has-proto: 1.0.1 + has-proto: 1.0.3 has-symbols: 1.0.3 hasown: 2.0.2 - dev: true /get-nonce@1.0.1: resolution: {integrity: sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==} @@ -20337,8 +20326,8 @@ packages: resolution: {integrity: sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==} engines: {node: '>= 0.4'} dependencies: - call-bind: 1.0.2 - get-intrinsic: 1.1.3 + call-bind: 1.0.7 + get-intrinsic: 1.2.4 /get-symbol-description@1.0.2: resolution: {integrity: sha512-g0QYk1dZBxGwk+Ngc+ltRH2IBp2f7zBkBMBJZCDerh6EhlhSR6+9irMCuT/09zD6qkarHUSn529sK/yL4S27mg==} @@ -20544,7 +20533,7 @@ packages: /gopd@1.0.1: resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} dependencies: - get-intrinsic: 1.1.3 + get-intrinsic: 1.2.4 /got@9.6.0: resolution: {integrity: sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q==} @@ -20696,25 +20685,14 @@ packages: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} - /has-property-descriptors@1.0.0: - resolution: {integrity: sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==} - dependencies: - get-intrinsic: 1.1.3 - /has-property-descriptors@1.0.2: resolution: {integrity: sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==} dependencies: es-define-property: 1.0.0 - dev: true - - /has-proto@1.0.1: - resolution: {integrity: sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==} - engines: {node: '>= 0.4'} /has-proto@1.0.3: resolution: {integrity: sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==} engines: {node: '>= 0.4'} - dev: true /has-symbols@1.0.3: resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==} @@ -20737,7 +20715,7 @@ packages: resolution: {integrity: sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==} engines: {node: '>= 0.4.0'} dependencies: - function-bind: 1.1.1 + function-bind: 1.1.2 /hasha@5.2.2: resolution: {integrity: sha512-Hrp5vIK/xr5SkeN2onO32H0MgNZ0f17HRNH39WfL0SYUNOTZ5Lz1TJ8Pajo/87dYGEFlLMm7mIc/k/s6Bvz9HQ==} @@ -21084,7 +21062,7 @@ packages: resolution: {integrity: sha512-tA8URYccNzMo94s5MQZgH8NB/XTa6HsOo0MLfXTKKEnHVVdegzaQoFZ7Jp44bdvLvY2waT5dc+j5ICEswhi7UQ==} engines: {node: '>= 0.4'} dependencies: - get-intrinsic: 1.1.3 + get-intrinsic: 1.2.4 has: 1.0.3 side-channel: 1.0.4 @@ -21184,14 +21162,14 @@ packages: resolution: {integrity: sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==} engines: {node: '>= 0.4'} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 has-tostringtag: 1.0.0 /is-array-buffer@3.0.1: resolution: {integrity: sha512-ASfLknmY8Xa2XtB4wmbz13Wu202baeA18cJBCeCy0wXUHZF0IPyVEXqKEcd+t2fNSLLL1vC6k7lxZEojNbISXQ==} dependencies: - call-bind: 1.0.2 - get-intrinsic: 1.1.3 + call-bind: 1.0.7 + get-intrinsic: 1.2.4 is-typed-array: 1.1.10 /is-array-buffer@3.0.4: @@ -21224,7 +21202,7 @@ packages: resolution: {integrity: sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==} engines: {node: '>= 0.4'} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 has-tostringtag: 1.0.0 /is-buffer@2.0.5: @@ -21407,13 +21385,13 @@ packages: resolution: {integrity: sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==} engines: {node: '>= 0.4'} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 has-tostringtag: 1.0.0 /is-shared-array-buffer@1.0.2: resolution: {integrity: sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 /is-shared-array-buffer@1.0.3: resolution: {integrity: sha512-nA2hv5XIhLR3uVzDDfCIknerhx8XUKnstuOERPNNIinXG7v9u+ohXF67vxm4TPTEPU6lm61ZkwP3c9PCB97rhg==} @@ -21460,7 +21438,7 @@ packages: engines: {node: '>= 0.4'} dependencies: available-typed-arrays: 1.0.5 - call-bind: 1.0.2 + call-bind: 1.0.7 for-each: 0.3.3 gopd: 1.0.1 has-tostringtag: 1.0.0 @@ -21483,7 +21461,7 @@ packages: /is-weakref@1.0.2: resolution: {integrity: sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 /is-what@4.1.16: resolution: {integrity: sha512-ZhMwEosbFJkA0YhFnNDgTM4ZxDRsS6HqTo7qsZM08fehyRYIYa0yHu5R6mgo1n/8MgaPBXiPimPD77baVFYg+A==} @@ -23455,7 +23433,7 @@ packages: resolution: {integrity: sha512-1mxKf0e58bvyjSCtKYY4sRe9itRk3PJpquJOjeIkz885CczcI4IvJJDLPS72oowuSh+pBxUFROpX+TU++hxhZQ==} engines: {node: '>= 0.4'} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 define-properties: 1.1.4 has-symbols: 1.0.3 object-keys: 1.1.1 @@ -23474,7 +23452,7 @@ packages: resolution: {integrity: sha512-leTPzo4Zvg3pmbQ3rDK69Rl8GQvIqMWubrkxONG9/ojtFE2rD9fjMKfSI5BxW3osRH1m6VdzmqK8oAY9aT4x5w==} engines: {node: '>= 0.4'} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 define-properties: 1.1.4 es-abstract: 1.21.1 dev: true @@ -23483,7 +23461,7 @@ packages: resolution: {integrity: sha512-VciD13dswC4j1Xt5394WR4MzmAQmlgN72phd/riNp9vtD7tp4QQWJ0R4wvclXcafgcYK8veHRed2W6XeGBvcfg==} engines: {node: '>= 0.4'} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 define-properties: 1.1.4 es-abstract: 1.21.1 dev: true @@ -23518,7 +23496,7 @@ packages: resolution: {integrity: sha512-FVVTkD1vENCsAcwNs9k6jea2uHC/X0+JcjG8YA60FN5CMaJmG95wT9jek/xX9nornqGRrBkKtzuAu2wuHpKqvw==} engines: {node: '>= 0.4'} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 define-properties: 1.1.4 es-abstract: 1.21.1 dev: true @@ -25878,12 +25856,13 @@ packages: redis-errors: 1.2.0 dev: false - /redlock@5.0.0-beta.2: + /redlock@5.0.0-beta.2(patch_hash=rwyegdki7iserrd7fgjwxkhnlu): resolution: {integrity: sha512-2RDWXg5jgRptDrB1w9O/JgSZC0j7y4SlaXnor93H/UJm/QyDiFgBKNtrh0TI6oCXqYSaSoXxFh6Sd3VtYfhRXw==} engines: {node: '>=12'} dependencies: node-abort-controller: 3.1.1 dev: false + patched: true /reduce-css-calc@2.1.8: resolution: {integrity: sha512-8liAVezDmUcH+tdzoEGrhfbGcP7nOV4NkGE3a74+qqvE7nt9i4sKLGBuZNOnpI4WiGksiNPklZxva80061QiPg==} @@ -25944,7 +25923,7 @@ packages: resolution: {integrity: sha512-fjggEOO3slI6Wvgjwflkc4NFRCTZAu5CnNfBd5qOMYhWdn67nJBBu34/TkD++eeFmd8C9r9jfXJ27+nSiRkSUA==} engines: {node: '>= 0.4'} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 define-properties: 1.1.4 functions-have-names: 1.2.3 @@ -26466,8 +26445,8 @@ packages: /safe-regex-test@1.0.0: resolution: {integrity: sha512-JBUUzyOgEwXQY1NuPtvcj/qcBDbDmEvWufhlnXZIm75DEHp+afM1r1ujJpJsV/gSM4t59tpDyPi1sd6ZaPFfsA==} dependencies: - call-bind: 1.0.2 - get-intrinsic: 1.1.3 + call-bind: 1.0.7 + get-intrinsic: 1.2.4 is-regex: 1.1.4 /safe-regex-test@1.0.3: @@ -26624,7 +26603,6 @@ packages: get-intrinsic: 1.2.4 gopd: 1.0.1 has-property-descriptors: 1.0.2 - dev: true /set-function-name@2.0.2: resolution: {integrity: sha512-7PGFlmtwsEADb0WYyvCMa1t+yke6daIG4Wirafur5kcf+MhUnPms1UeR0CKQdTZD81yESwMHbtn+TR+dMviakQ==} @@ -26692,8 +26670,8 @@ packages: /side-channel@1.0.4: resolution: {integrity: sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==} dependencies: - call-bind: 1.0.2 - get-intrinsic: 1.1.3 + call-bind: 1.0.7 + get-intrinsic: 1.2.4 object-inspect: 1.12.2 /siginfo@2.0.0: @@ -27194,10 +27172,10 @@ packages: /string.prototype.matchall@4.0.8: resolution: {integrity: sha512-6zOCOcJ+RJAQshcTvXPHoxoQGONa3e/Lqx90wUA+wEzX78sg5Bo+1tQo4N0pohS0erG9qtCqJDjNCQBjeWVxyg==} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 define-properties: 1.1.4 es-abstract: 1.21.1 - get-intrinsic: 1.1.3 + get-intrinsic: 1.2.4 has-symbols: 1.0.3 internal-slot: 1.0.4 regexp.prototype.flags: 1.4.3 @@ -27226,7 +27204,7 @@ packages: /string.prototype.trimend@1.0.6: resolution: {integrity: sha512-JySq+4mrPf9EsDBEDYMOb/lM7XQLulwg5R/m1r0PXEFqrV0qHvl58sdTilSXtKOflCsK2E8jxf+GKC0T07RWwQ==} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 define-properties: 1.1.4 es-abstract: 1.21.1 @@ -27241,7 +27219,7 @@ packages: /string.prototype.trimstart@1.0.6: resolution: {integrity: sha512-omqjMDaY92pbn5HOX7f9IccLA+U1tA9GvtU4JrodiXFfYB7jPzzHpRzpglLAjtUV6bB557zwClJezTqnAiYnQA==} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 define-properties: 1.1.4 es-abstract: 1.21.1 @@ -28470,7 +28448,7 @@ packages: /typed-array-length@1.0.4: resolution: {integrity: sha512-KjZypGq+I/H7HI5HlOoGHkWUUGq+Q0TPhQurLbyrVrvnKTBgzLhIJ7j6J/XTQOi0d1RjyZ0wdas8bKs2p0x3Ng==} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 for-each: 0.3.3 is-typed-array: 1.1.10 @@ -28625,7 +28603,7 @@ packages: /unbox-primitive@1.0.2: resolution: {integrity: sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==} dependencies: - call-bind: 1.0.2 + call-bind: 1.0.7 has-bigints: 1.0.2 has-symbols: 1.0.3 which-boxed-primitive: 1.0.2 @@ -29816,7 +29794,7 @@ packages: engines: {node: '>= 0.4'} dependencies: available-typed-arrays: 1.0.5 - call-bind: 1.0.2 + call-bind: 1.0.7 for-each: 0.3.3 gopd: 1.0.1 has-tostringtag: 1.0.0 From 2664c1d813dd1cf331db6c2d4efacb717694e32b Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 8 Oct 2024 16:27:31 +0100 Subject: [PATCH 106/114] Moved the new DB migrations to the new database package folder --- .../migration.sql | 0 .../migration.sql | 0 .../migration.sql | 0 .../20241003002757_add_max_queue_sizes_to_org/migration.sql | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename {packages => internal-packages}/database/prisma/migrations/20241002155751_add_timed_out_status_to_task_run_status/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20241002163757_add_max_duration_to_background_worker_task/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20241002164627_add_max_duration_in_seconds_to_task_run/migration.sql (100%) rename {packages => internal-packages}/database/prisma/migrations/20241003002757_add_max_queue_sizes_to_org/migration.sql (100%) diff --git a/packages/database/prisma/migrations/20241002155751_add_timed_out_status_to_task_run_status/migration.sql b/internal-packages/database/prisma/migrations/20241002155751_add_timed_out_status_to_task_run_status/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20241002155751_add_timed_out_status_to_task_run_status/migration.sql rename to internal-packages/database/prisma/migrations/20241002155751_add_timed_out_status_to_task_run_status/migration.sql diff --git a/packages/database/prisma/migrations/20241002163757_add_max_duration_to_background_worker_task/migration.sql b/internal-packages/database/prisma/migrations/20241002163757_add_max_duration_to_background_worker_task/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20241002163757_add_max_duration_to_background_worker_task/migration.sql rename to internal-packages/database/prisma/migrations/20241002163757_add_max_duration_to_background_worker_task/migration.sql diff --git a/packages/database/prisma/migrations/20241002164627_add_max_duration_in_seconds_to_task_run/migration.sql b/internal-packages/database/prisma/migrations/20241002164627_add_max_duration_in_seconds_to_task_run/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20241002164627_add_max_duration_in_seconds_to_task_run/migration.sql rename to internal-packages/database/prisma/migrations/20241002164627_add_max_duration_in_seconds_to_task_run/migration.sql diff --git a/packages/database/prisma/migrations/20241003002757_add_max_queue_sizes_to_org/migration.sql b/internal-packages/database/prisma/migrations/20241003002757_add_max_queue_sizes_to_org/migration.sql similarity index 100% rename from packages/database/prisma/migrations/20241003002757_add_max_queue_sizes_to_org/migration.sql rename to internal-packages/database/prisma/migrations/20241003002757_add_max_queue_sizes_to_org/migration.sql From 386ee00e852d906159f4e7d6801d85fcc0ea0290 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 8 Oct 2024 16:50:47 +0100 Subject: [PATCH 107/114] Remove the run-engine package --- internal-packages/run-engine/README.md | 324 ---- internal-packages/run-engine/package.json | 27 - .../run-engine/src/engine/index.test.ts | 150 -- .../run-engine/src/engine/index.ts | 541 ------- internal-packages/run-engine/src/index.ts | 1 - .../run-engine/src/run-queue/index.test.ts | 453 ------ .../run-engine/src/run-queue/index.ts | 1384 ----------------- .../src/run-queue/keyProducer.test.ts | 361 ----- .../run-engine/src/run-queue/keyProducer.ts | 195 --- .../simpleWeightedPriorityStrategy.ts | 119 -- .../run-engine/src/run-queue/types.ts | 116 -- .../run-engine/src/shared/asyncWorker.ts | 34 - .../run-engine/src/shared/index.ts | 39 - internal-packages/run-engine/tsconfig.json | 29 - internal-packages/run-engine/vitest.config.ts | 8 - 15 files changed, 3781 deletions(-) delete mode 100644 internal-packages/run-engine/README.md delete mode 100644 internal-packages/run-engine/package.json delete mode 100644 internal-packages/run-engine/src/engine/index.test.ts delete mode 100644 internal-packages/run-engine/src/engine/index.ts delete mode 100644 internal-packages/run-engine/src/index.ts delete mode 100644 internal-packages/run-engine/src/run-queue/index.test.ts delete mode 100644 internal-packages/run-engine/src/run-queue/index.ts delete mode 100644 internal-packages/run-engine/src/run-queue/keyProducer.test.ts delete mode 100644 internal-packages/run-engine/src/run-queue/keyProducer.ts delete mode 100644 internal-packages/run-engine/src/run-queue/simpleWeightedPriorityStrategy.ts delete mode 100644 internal-packages/run-engine/src/run-queue/types.ts delete mode 100644 internal-packages/run-engine/src/shared/asyncWorker.ts delete mode 100644 internal-packages/run-engine/src/shared/index.ts delete mode 100644 internal-packages/run-engine/tsconfig.json delete mode 100644 internal-packages/run-engine/vitest.config.ts diff --git a/internal-packages/run-engine/README.md b/internal-packages/run-engine/README.md deleted file mode 100644 index 2960756f71..0000000000 --- a/internal-packages/run-engine/README.md +++ /dev/null @@ -1,324 +0,0 @@ -# Trigger.dev Run Engine - -The Run Engine process runs from triggering, to executing, and completing them. - -It is responsible for: - -- Creating and updating runs as they progress. -- Operating the run queue, including handling concurrency. - -## Components - -### Run Engine - -This is used to actually process a run and store the state at each step. It coordinates with the other components. - -#### Atomicity - -Operations on the run are "atomic" in the sense that only a single operation can mutate them at a time. We use RedLock to ensure this. - -#### Valid state transitions - -The run engine ensures that the run can only transition to valid states. - -#### State history - -When a run is mutated in any way, we store the state. This data is used for the next step for the run, and also for debugging. - -`TaskRunState` is a decent table name. We should have a "description" column which describes the change, this would be purely for internal use but would be very useful for debugging. - -### Run Queue - -This is used to queue, dequeue, and manage concurrency. It also provides visibility into the concurrency for the env, org, etc. - -Run IDs are enqueued. They're pulled from the queue in a fair way with advanced options for debouncing and visibility. - -### Heartbeats - -Heartbeats are used to determine if a run has stopped responding. If a heartbeat isn't received within a defined period then the run is judged to have become stuck and the attempt is failed. - -### Checkpoints - -Checkpoints allow pausing an executing run and then resuming it later. - -## Waitpoints - -A "Waitpoint" is something that prevents a run from continuing: - -- `wait.for()` a future time. -- `triggerAndWait()` until the run is finished. -- `batchTriggerAndWait()` until all runs are finished. -- `wait.forRequest()` wait until a request has been received (not implemented yet). - -They block run execution from continuing until all of them are completed/removed. - -Some of them have data associated with them, e.g. the finished run payload. - -Could a run have multiple at once? That might allow us to support Promise.all wrapped. It would also allow more advanced use cases. - -Could this be how we implement other features like `delay`, `rate limit`, and retries waiting before the next try? - -Could we even open up a direct API/SDK for creating one inside a run (would pause execution)? And then completing one (would continue execution)? It could also be "failed" which the run could act upon differently. - -## Notes from call with Eric - -We could expose the API/SDK for creating/completing Waitpoints. - -> They need to be associated with attempts, because that's what gets continued. And if an attempts fails, we don't want to keep the waitpoints. - -> We should have idempotency keys for `wait.for()` and `wait.until()`, so they wouldn't wait on a second attempt. "Waitpoints" have idempotency keys, and these are used for a `wait.forEvent()` (or whatever we call it). - -> How would debounce use this? When the waitpoint is completed, we would "clear" the "idempotencyKey" which would be the user-provided "debounceKey". It wouldn't literally clear it necessarily. Maybe another column `idempotencyKeyActive` would be set to `false`. Or move the key to another column, which is just for reference. - -> `triggerAndWait`, cancelling a child task run. It would clear the waitpoint `idempotencyKey`, same as above. - -> Copying the output from the run into the waitpoint actually does make sense. It simplifies the API for continuing runs. - -> Inside a run you could wait for another run or runs using the run ID. `const output = await wait.forRunToComplete(runId)`. This would basically just get a run by ID, then wait for it's waitpoint to be completed. This means every run would have a waitpoint associated with it. - -```ts -//inside a run function -import { runs } from "@trigger.dev/sdk/v3"; - -// Loop through all runs with the tag "user_123456" that have completed - -for await (const run of runs.list({ tag: "user_123456" })) { - await wait.forRunToComplete(run.id); -} - -//wait for many runs to complete -await wait.forRunToComplete(runId); -await wait.forRunsToComplete({ tag: "user_123456" }); -``` - -Rate limit inside a task. This is much trickier. - -```ts -//simple time-based rate limit -await wait.forRateLimit(`timed-${payload.user.id}`, { per: { minute: 10 } }); - -const openAiResult = await wait.forRateLimit( - `openai-${payload.user.id}`, - { limit: 100, recharge: { seconds: 2 } }, - (rateLimit, refreshes) => { - const result = await openai.createCompletion({ - model: "gpt-3.5-turbo", - prompt: "What is the meaning of life?", - }); - const tokensUsed = result.tokensUsed; - - await rateLimit.used(tokensUsed); - - return result; - } -); - -//do stuff with openAiResult -``` - -#### `triggerAndWait()` implementation - -Inside the SDK - -```ts -function triggerAndWait_internal(data) { - //if you don't pass in a string, it won't have a "key" - const waitpoint = await createWaitpoint(); - const response = await apiClient.triggerTask({ ...data, waitpointId: waitpoint.id }); - - //...do normal stuff - - // wait for the waitpoint to be completed - // in reality this probably needs to happen inside the runtime - const result = await waitpointCompletion(waitpoint.id); -} -``` - -Pseudo-code for completing a run and completing the waitpoint: - -```ts -function completeRun(tx, data) { - //complete the child run - const run = await tx.taskRun.update({ where: { id: runId }, data, include: { waitpoint } }); - if (run.waitpoint) { - await completeWaitpoint(tx, { id: run.waitpoint.id }); - - //todo in completeWaitpoint it would check if the blocked runs can now continue - //if they have no more blockers then they can continue - - //batchTriggerAndWait with two items - //blocked_by: ["w_1", "w_2"] - //blocked_by: ["w_2"] - //blocked_by: [] then you can continue - } - - const state = await tx.taskRunState.create({ - where: { runId: id }, - data: { runId, status: run.status }, - }); - - const previousState = await tx.taskRunState.findFirst({ where: { runId: runId, latest: true } }); - const waitingOn = previousState.waitingOn?.filter((w) => w !== waitpoint?.id) ?? []; - - if (waitingOn.length === 0) { - } -} -``` - -#### `batchTriggerAndWait()` implementation - -```ts -//todo -``` - -### Example: User-defined waitpoint - -A user's backend code: - -```ts -import { waitpoint } from "@trigger.dev/sdk/v3"; -import type { NextApiRequest, NextApiResponse } from "next"; - -export default async function handler(req: NextApiRequest, res: NextApiResponse<{ id: string }>) { - const userId = req.query.userId; - const isPaying = req.query.isPaying; - - //internal SDK calls, this would be nicer for users to use - const waitpoint = waitpoint(`${userId}/onboarding-completed`); - await waitpoint.complete({ data: { isPaying } }); - - //todo instead this would be a single call - - res.status(200).json(handle); -} -``` - -Inside a user's run - -```ts -export const myTask = task({ - id: "my-task", - run: async (payload) => { - //it doesn't matter if this was completed before the run started - const result = await wait.forPoint<{ isPaying: boolean }>( - `${payload.userId}/onboarding-completed` - ); - }, -}); -``` - -### How would we implement `batchTriggerAndWait`? - -```ts - -``` - -## How does it work? - -It's very important that a run can only be acted on by one process at a time. We lock runs using RedLock while they're being mutated. This prevents some network-related race conditions like the timing of checkpoints and heartbeats permanently hanging runs. - -# Legacy system - -These are all the TaskRun mutations happening right now: - -## 1. TriggerTaskService - -This is called from: - -- trigger task API -- `BatchTriggerTaskService` for each item -- `ReplayTaskRunService` -- `TestTaskService` -- `TriggerScheduledTaskService` when the CRON fires - -Directly creates a run if it doesn't exist, either in the `PENDING` or `DELAYED` states. -Enqueues the run. - -[TriggerTaskService.call()](/apps//webapp/app/v3/services/triggerTask.server.ts#246) - -## 2. Batch trigger - -## 3. DevQueueConsumer executing a run - -### a. Lock run and set status to `EXECUTING` - -[DevQueueConsumer.#doWorkInternal()](/apps/webapp/app/v3/marqs/devQueueConsumer.server.ts#371) - -### b. If an error is thrown, unlock the run and set status to `PENDING` - -[DevQueueConsumer.#doWorkInternal()](/apps/webapp/app/v3/marqs/devQueueConsumer.server.ts#477) - -## 4. SharedQueueConsumer executing a run - -### a. `EXECUTE`, lock the run - -We lock the run and update some basic metadata (but not status). -[SharedQueueConsumer.#doWorkInternal()](/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts#394) - -### b. `EXECUTE`, if an error is thrown, unlock the run - -We unlock the run, but don't change the status. -[SharedQueueConsumer.#doWorkInternal()](/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts#552) - -### c. `EXECUTE`, if the run has no deployment set the status to `WAITING_FOR_DEPLOY` - -[SharedQueueConsumer.#doWorkInternal()](/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts#876) - -## 5. CompleteAttemptService retrying a run - -### a. When an attempt has failed, we set the status to `RETRYING_AFTER_FAILURE` - -[CompleteAttemptService.#completeAttemptFailed()](/apps/webapp/app/v3/services/completeAttempt.server.ts#239) - -## 6. CreateTaskRunAttemptService creating a new attempt, setting the run to `EXECUTING` - -We call this when: - -- [Executing a DEV run from the CLI.](/packages/cli-v3//src/dev/workerRuntime.ts#305) -- [Deprecated: directly from the SharedQueueCOnsumer when we don't support lazy attempts](/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts#501) -- [When we receive a `CREATE_TASK_RUN_ATTEMPT` message from the coordinator](/apps/webapp//app/v3//handleSocketIo.server.ts#187) - -This is the actual very simple TaskRun update: -[CreateTaskRunAttemptService.call()](/apps/webapp/app/v3/services/createTaskRunAttempt.server.ts#134) - -## 7. EnqueueDelayedRunService set a run to `PENDING` when the `delay` has elapsed - -When the run attempt gets created it will be marked as `EXECUTING`. - -[EnqueueDelayedRunService.#call()](/apps/webapp/app/v3/services/enqueueDelayedRun.server.ts#41) - -## 8. FinalizeTaskRunService finalizing a run - -This service is called from many places, when a run is in a "final" state. This means the run can't be acted on anymore. - -We set the status, expiredAt and completedAt fields. - -[FinalizeTaskRunService.#call()](/apps/webapp/app/v3/services/finalizeTaskRun.server.ts#63) - -This function is called from: - -- [`FailedTaskRunService` when a run has SYSTEM_FAILURE](/apps/webapp/app/v3/failedTaskRun.server.ts#41) -- [`CancelAttemptService` when an attempt is canceled](/apps/webapp/app/v3/services/cancelAttempt.server.ts#66) -- [`CancelTaskRunService` when a run is canceled](/apps/webapp/app/v3/services/cancelTaskRun.server.ts#51) -- `CompleteAttemptService` when a SYSTEM_FAILURE happens - - [No attempt](/apps/webapp/app/v3/services/completeAttempt.server.ts#74) - - [`completeAttemptFailed` and there's no checkpoint](/apps/webapp/app/v3/services/completeAttempt.server.ts#280) - - [`completeAttemptFailed` and the error is internal and a graceful exit timeout](/apps/webapp/app/v3/services/completeAttempt.server.ts#321) -- `CompleteTaskRunService` when a run has failed (this isn't a bug) - - [`completeAttemptFailed`](/apps/webapp/app/v3/services/completeAttempt.server.ts#352) -- `CompleteTaskRunService` when a run is completed successfully - - [`completeAttemptSuccessfully`](/apps/webapp/app/v3/services/completeAttempt.server.ts#135) -- `CrashTaskRunService` when a run has crashed - - [`call`](/apps/webapp/app/v3/services/crashTaskRun.server.ts#47) -- `ExpireEnqueuedRunService` when a run has expired - - [`call`](/apps/webapp/app/v3/services/expireEnqueuedRun.server.ts#42) - -## 9. RescheduleTaskRunService (when further delaying a delayed run) - -[RescheduleTaskRunService.#call()](/apps/webapp/app/v3/services/rescheduleTaskRun.server.ts#21) - -## 10. Triggering a scheduled run - -Graphile Worker calls this function based on the schedule. We add the schedule data onto the run, and call `TriggerTaskService.call()`. - -[TriggerScheduledRunService.#call()](/apps/webapp/app/v3/services/triggerScheduledTask.server.ts#131) diff --git a/internal-packages/run-engine/package.json b/internal-packages/run-engine/package.json deleted file mode 100644 index a2cf01cbdc..0000000000 --- a/internal-packages/run-engine/package.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "@internal/run-engine", - "private": true, - "version": "0.0.1", - "main": "./src/index.ts", - "types": "./src/index.ts", - "dependencies": { - "@internal/zod-worker": "workspace:*", - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/semantic-conventions": "^1.27.0", - "@trigger.dev/core": "workspace:*", - "@trigger.dev/database": "workspace:*", - "ioredis": "^5.3.2", - "nanoid": "^3.3.4", - "redlock": "5.0.0-beta.2", - "typescript": "^5.5.4", - "zod": "3.22.3" - }, - "devDependencies": { - "@internal/testcontainers": "workspace:*", - "vitest": "^1.4.0" - }, - "scripts": { - "typecheck": "tsc --noEmit", - "test": "vitest" - } -} diff --git a/internal-packages/run-engine/src/engine/index.test.ts b/internal-packages/run-engine/src/engine/index.test.ts deleted file mode 100644 index 64bfe6c06b..0000000000 --- a/internal-packages/run-engine/src/engine/index.test.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { expect } from "vitest"; -import { containerTest } from "@internal/testcontainers"; -import { RunEngine } from "./index.js"; -import { PrismaClient, RuntimeEnvironmentType } from "@trigger.dev/database"; - -describe("RunEngine", () => { - containerTest( - "Trigger a simple run", - { timeout: 15_000 }, - async ({ postgresContainer, prisma, redisContainer }) => { - const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - - const engine = new RunEngine({ - prisma, - redis: { - host: redisContainer.getHost(), - port: redisContainer.getPort(), - password: redisContainer.getPassword(), - enableAutoPipelining: true, - }, - zodWorker: { - connectionString: postgresContainer.getConnectionUri(), - shutdownTimeoutInMs: 100, - }, - }); - - const run = await engine.trigger( - { - number: 1, - friendlyId: "run_1234", - environment: authenticatedEnvironment, - taskIdentifier: "test-task", - payload: "{}", - payloadType: "application/json", - context: {}, - traceContext: {}, - traceId: "t12345", - spanId: "s12345", - masterQueue: "main", - queueName: "task/test-task", - isTest: false, - tags: [], - }, - prisma - ); - - expect(run).toBeDefined(); - expect(run.friendlyId).toBe("run_1234"); - - //check it's actually in the db - const runFromDb = await prisma.taskRun.findUnique({ - where: { - friendlyId: "run_1234", - }, - }); - expect(runFromDb).toBeDefined(); - expect(runFromDb?.id).toBe(run.id); - - //check the waitpoint is created - const runWaitpoint = await prisma.waitpoint.findMany({ - where: { - completedByTaskRunId: run.id, - }, - }); - expect(runWaitpoint.length).toBe(1); - expect(runWaitpoint[0].type).toBe("RUN"); - - //check the queue length - const queueLength = await engine.runQueue.lengthOfQueue(authenticatedEnvironment, run.queue); - expect(queueLength).toBe(1); - - //concurrency before - const envConcurrencyBefore = await engine.runQueue.currentConcurrencyOfEnvironment( - authenticatedEnvironment - ); - expect(envConcurrencyBefore).toBe(0); - - //dequeue the run - const dequeued = await engine.runQueue.dequeueMessageInSharedQueue( - "test_12345", - run.masterQueue - ); - expect(dequeued?.messageId).toBe(run.id); - - const envConcurrencyAfter = await engine.runQueue.currentConcurrencyOfEnvironment( - authenticatedEnvironment - ); - expect(envConcurrencyAfter).toBe(1); - } - ); - - //todo triggerAndWait - - //todo batchTriggerAndWait - - //todo checkpoints - - //todo heartbeats - - //todo failing a run - - //todo cancelling a run - - //todo expiring a run - - //todo delaying a run -}); - -async function setupAuthenticatedEnvironment(prisma: PrismaClient, type: RuntimeEnvironmentType) { - // Your database setup logic here - const org = await prisma.organization.create({ - data: { - title: "Test Organization", - slug: "test-organization", - }, - }); - - const project = await prisma.project.create({ - data: { - name: "Test Project", - slug: "test-project", - externalRef: "proj_1234", - organizationId: org.id, - }, - }); - - const environment = await prisma.runtimeEnvironment.create({ - data: { - type, - slug: "slug", - projectId: project.id, - organizationId: org.id, - apiKey: "api_key", - pkApiKey: "pk_api_key", - shortcode: "short_code", - maximumConcurrencyLimit: 10, - }, - }); - - return await prisma.runtimeEnvironment.findUniqueOrThrow({ - where: { - id: environment.id, - }, - include: { - project: true, - organization: true, - orgMember: true, - }, - }); -} diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts deleted file mode 100644 index f74d11ee91..0000000000 --- a/internal-packages/run-engine/src/engine/index.ts +++ /dev/null @@ -1,541 +0,0 @@ -import { RunnerOptions, ZodWorker } from "@internal/zod-worker"; -import { trace } from "@opentelemetry/api"; -import { Logger } from "@trigger.dev/core/logger"; -import { QueueOptions } from "@trigger.dev/core/v3"; -import { generateFriendlyId, parseNaturalLanguageDuration } from "@trigger.dev/core/v3/apps"; -import { - $transaction, - Prisma, - PrismaClient, - PrismaClientOrTransaction, - TaskRun, - Waitpoint, -} from "@trigger.dev/database"; -import { Redis, type RedisOptions } from "ioredis"; -import Redlock from "redlock"; -import { z } from "zod"; -import { RunQueue } from "../run-queue"; -import { SimpleWeightedChoiceStrategy } from "../run-queue/simpleWeightedPriorityStrategy"; -import { MinimalAuthenticatedEnvironment } from "../shared"; - -import { nanoid } from "nanoid"; - -type Options = { - redis: RedisOptions; - prisma: PrismaClient; - zodWorker: RunnerOptions & { - shutdownTimeoutInMs: number; - }; -}; - -type TriggerParams = { - friendlyId: string; - number: number; - environment: MinimalAuthenticatedEnvironment; - idempotencyKey?: string; - taskIdentifier: string; - payload: string; - payloadType: string; - context: any; - traceContext: Record; - traceId: string; - spanId: string; - parentSpanId?: string; - lockedToVersionId?: string; - concurrencyKey?: string; - masterQueue: string; - queueName: string; - queue?: QueueOptions; - isTest: boolean; - delayUntil?: Date; - queuedAt?: Date; - maxAttempts?: number; - ttl?: string; - tags: string[]; - parentTaskRunId?: string; - parentTaskRunAttemptId?: string; - rootTaskRunId?: string; - batchId?: string; - resumeParentOnCompletion?: boolean; - depth?: number; - metadata?: string; - metadataType?: string; - seedMetadata?: string; - seedMetadataType?: string; -}; - -const schema = { - "runengine.waitpointCompleteDateTime": z.object({ - waitpointId: z.string(), - }), - "runengine.expireRun": z.object({ - runId: z.string(), - }), -}; - -type EngineWorker = ZodWorker; - -export class RunEngine { - private redis: Redis; - private prisma: PrismaClient; - private redlock: Redlock; - runQueue: RunQueue; - private zodWorker: EngineWorker; - private logger = new Logger("RunEngine", "debug"); - - constructor(private readonly options: Options) { - this.prisma = options.prisma; - this.redis = new Redis(options.redis); - this.redlock = new Redlock([this.redis], { - driftFactor: 0.01, - retryCount: 10, - retryDelay: 200, // time in ms - retryJitter: 200, // time in ms - automaticExtensionThreshold: 500, // time in ms - }); - - this.runQueue = new RunQueue({ - name: "rq", - tracer: trace.getTracer("rq"), - queuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 36 }), - envQueuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 12 }), - workers: 1, - defaultEnvConcurrency: 10, - enableRebalancing: false, - logger: new Logger("RunQueue", "warn"), - redis: options.redis, - }); - - this.zodWorker = new ZodWorker({ - name: "runQueueWorker", - prisma: options.prisma, - replica: options.prisma, - logger: new Logger("RunQueueWorker", "debug"), - runnerOptions: options.zodWorker, - shutdownTimeoutInMs: options.zodWorker.shutdownTimeoutInMs, - schema, - tasks: { - "runengine.waitpointCompleteDateTime": { - priority: 0, - maxAttempts: 10, - handler: async (payload, job) => { - await this.#completeWaitpoint(payload.waitpointId); - }, - }, - "runengine.expireRun": { - priority: 0, - maxAttempts: 10, - handler: async (payload, job) => { - await this.expireRun(payload.runId); - }, - }, - }, - }); - } - - //MARK: - Run functions - - /** "Triggers" one run. - */ - async trigger( - { - friendlyId, - number, - environment, - idempotencyKey, - taskIdentifier, - payload, - payloadType, - context, - traceContext, - traceId, - spanId, - parentSpanId, - lockedToVersionId, - concurrencyKey, - masterQueue, - queueName, - queue, - isTest, - delayUntil, - queuedAt, - maxAttempts, - ttl, - tags, - parentTaskRunId, - parentTaskRunAttemptId, - rootTaskRunId, - batchId, - resumeParentOnCompletion, - depth, - metadata, - metadataType, - seedMetadata, - seedMetadataType, - }: TriggerParams, - tx?: PrismaClientOrTransaction - ) { - const prisma = tx ?? this.prisma; - - const status = delayUntil ? "DELAYED" : "PENDING"; - - //create run - const taskRun = await prisma.taskRun.create({ - data: { - status, - number, - friendlyId, - runtimeEnvironmentId: environment.id, - projectId: environment.project.id, - idempotencyKey, - taskIdentifier, - payload, - payloadType, - context, - traceContext, - traceId, - spanId, - parentSpanId, - lockedToVersionId, - concurrencyKey, - queue: queueName, - masterQueue, - isTest, - delayUntil, - queuedAt, - maxAttempts, - ttl, - tags: - tags.length === 0 - ? undefined - : { - connect: tags.map((id) => ({ id })), - }, - parentTaskRunId, - parentTaskRunAttemptId, - rootTaskRunId, - batchId, - resumeParentOnCompletion, - depth, - metadata, - metadataType, - seedMetadata, - seedMetadataType, - executionSnapshot: { - create: { - engine: "V2", - executionStatus: "RUN_CREATED", - description: "Run was created", - runStatus: status, - }, - }, - }, - }); - - await this.redlock.using([taskRun.id], 5000, async (signal) => { - //todo add this in some places throughout this code - if (signal.aborted) { - throw signal.error; - } - - //create associated waitpoint (this completes when the run completes) - const associatedWaitpoint = await this.#createRunAssociatedWaitpoint(prisma, { - projectId: environment.project.id, - completedByTaskRunId: taskRun.id, - }); - - //triggerAndWait or batchTriggerAndWait - if (resumeParentOnCompletion && parentTaskRunId) { - //this will block the parent run from continuing until this waitpoint is completed (and removed) - await this.#blockRunWithWaitpoint(prisma, { - orgId: environment.organization.id, - runId: parentTaskRunId, - waitpoint: associatedWaitpoint, - }); - } - - if (queue) { - const concurrencyLimit = - typeof queue.concurrencyLimit === "number" - ? Math.max(0, queue.concurrencyLimit) - : undefined; - - let taskQueue = await prisma.taskQueue.findFirst({ - where: { - runtimeEnvironmentId: environment.id, - name: queueName, - }, - }); - - if (taskQueue) { - taskQueue = await prisma.taskQueue.update({ - where: { - id: taskQueue.id, - }, - data: { - concurrencyLimit, - rateLimit: queue.rateLimit, - }, - }); - } else { - taskQueue = await prisma.taskQueue.create({ - data: { - friendlyId: generateFriendlyId("queue"), - name: queueName, - concurrencyLimit, - runtimeEnvironmentId: environment.id, - projectId: environment.project.id, - rateLimit: queue.rateLimit, - type: "NAMED", - }, - }); - } - - if (typeof taskQueue.concurrencyLimit === "number") { - await this.runQueue.updateQueueConcurrencyLimits( - environment, - taskQueue.name, - taskQueue.concurrencyLimit - ); - } else { - await this.runQueue.removeQueueConcurrencyLimits(environment, taskQueue.name); - } - } - - if (taskRun.delayUntil) { - const delayWaitpoint = await this.#createDateTimeWaitpoint(prisma, { - projectId: environment.project.id, - completedAfter: taskRun.delayUntil, - }); - - await this.#blockRunWithWaitpoint(prisma, { - orgId: environment.organization.id, - runId: taskRun.id, - waitpoint: delayWaitpoint, - }); - } - - if (!taskRun.delayUntil && taskRun.ttl) { - const expireAt = parseNaturalLanguageDuration(taskRun.ttl); - - if (expireAt) { - await this.zodWorker.enqueue( - "runengine.expireRun", - { runId: taskRun.id }, - { tx, runAt: expireAt, jobKey: `runengine.expireRun.${taskRun.id}` } - ); - } - } - - await this.enqueueRun(taskRun, environment, prisma); - }); - - //todo release parent concurrency (for the project, task, and environment, but not for the queue?) - //todo if this has been triggered with triggerAndWait or batchTriggerAndWait - - return taskRun; - } - - /** Triggers multiple runs. - * This doesn't start execution, but it will create a batch and schedule them for execution. - */ - async batchTrigger() {} - - /** The run can be added to the queue. When it's pulled from the queue it will be executed. */ - async enqueueRun( - run: TaskRun, - env: MinimalAuthenticatedEnvironment, - tx?: PrismaClientOrTransaction - ) { - await this.runQueue.enqueueMessage({ - env, - masterQueue: run.masterQueue, - message: { - runId: run.id, - taskIdentifier: run.taskIdentifier, - orgId: env.organization.id, - projectId: env.project.id, - environmentId: env.id, - environmentType: env.type, - queue: run.queue, - concurrencyKey: run.concurrencyKey ?? undefined, - timestamp: Date.now(), - }, - }); - - //todo update the TaskRunExecutionSnapshot - } - - async dequeueRun(consumerId: string, masterQueue: string) { - const message = await this.runQueue.dequeueMessageInSharedQueue(consumerId, masterQueue); - //todo update the TaskRunExecutionSnapshot - //todo update the TaskRun status? - return message; - } - - /** We want to actually execute the run, this could be a continuation of a previous execution. - * This is called from the queue, when the run has been pulled. */ - //todo think more about this, when do we create the attempt? - //todo what does this actually do? - //todo how does it get sent to the worker? DEV and PROD - async prepareForExecution(runId: string) {} - - async prepareForAttempt(runId: string) {} - - async complete(runId: string, completion: any) {} - - async expireRun(runId: string) {} - - //MARK: - Waitpoints - async #createRunAssociatedWaitpoint( - tx: PrismaClientOrTransaction, - { projectId, completedByTaskRunId }: { projectId: string; completedByTaskRunId: string } - ) { - return tx.waitpoint.create({ - data: { - type: "RUN", - status: "PENDING", - idempotencyKey: nanoid(24), - userProvidedIdempotencyKey: false, - projectId, - completedByTaskRunId, - }, - }); - } - - async #createDateTimeWaitpoint( - tx: PrismaClientOrTransaction, - { projectId, completedAfter }: { projectId: string; completedAfter: Date } - ) { - const waitpoint = await tx.waitpoint.create({ - data: { - type: "DATETIME", - status: "PENDING", - idempotencyKey: nanoid(24), - userProvidedIdempotencyKey: false, - projectId, - completedAfter, - }, - }); - - await this.zodWorker.enqueue( - "runengine.waitpointCompleteDateTime", - { waitpointId: waitpoint.id }, - { tx, runAt: completedAfter, jobKey: `waitpointCompleteDateTime.${waitpoint.id}` } - ); - - return waitpoint; - } - - async #blockRunWithWaitpoint( - tx: PrismaClientOrTransaction, - { orgId, runId, waitpoint }: { orgId: string; runId: string; waitpoint: Waitpoint } - ) { - //todo it would be better if we didn't remove from the queue, because this removes the payload - //todo better would be to have a "block" function which remove it from the queue but doesn't remove the payload - //todo - // await this.runQueue.acknowledgeMessage(orgId, runId); - - //todo release concurrency and make sure the run isn't in the queue - // await this.runQueue.blockMessage(orgId, runId); - - return tx.taskRunWaitpoint.create({ - data: { - taskRunId: runId, - waitpointId: waitpoint.id, - projectId: waitpoint.projectId, - }, - }); - } - - /** This completes a waitpoint and then continues any runs blocked by the waitpoint, - * if they're no longer blocked. This doesn't suffer from race conditions. */ - async #completeWaitpoint(id: string) { - const waitpoint = await this.prisma.waitpoint.findUnique({ - where: { id }, - }); - - if (!waitpoint) { - throw new Error(`Waitpoint ${id} not found`); - } - - if (waitpoint.status === "COMPLETED") { - return; - } - - await $transaction( - this.prisma, - async (tx) => { - // 1. Find the TaskRuns associated with this waitpoint - const affectedTaskRuns = await tx.taskRunWaitpoint.findMany({ - where: { waitpointId: id }, - select: { taskRunId: true }, - }); - - if (affectedTaskRuns.length === 0) { - throw new Error(`No TaskRunWaitpoints found for waitpoint ${id}`); - } - - // 2. Delete the TaskRunWaitpoint entries for this specific waitpoint - await tx.taskRunWaitpoint.deleteMany({ - where: { waitpointId: id }, - }); - - // 3. Update the waitpoint status - await tx.waitpoint.update({ - where: { id }, - data: { status: "COMPLETED" }, - }); - - // 4. Check which of the affected TaskRuns now have no waitpoints - const taskRunsToResume = await tx.taskRun.findMany({ - where: { - id: { in: affectedTaskRuns.map((run) => run.taskRunId) }, - blockedByWaitpoints: { none: {} }, - status: { in: ["PENDING", "WAITING_TO_RESUME"] }, - }, - include: { - runtimeEnvironment: { - select: { - id: true, - type: true, - maximumConcurrencyLimit: true, - project: { select: { id: true } }, - organization: { select: { id: true } }, - }, - }, - }, - }); - - // 5. Continue the runs that have no more waitpoints - for (const run of taskRunsToResume) { - await this.enqueueRun(run, run.runtimeEnvironment, tx); - } - }, - (error) => { - this.logger.error(`Error completing waitpoint ${id}, retrying`, { error }); - throw error; - }, - { isolationLevel: Prisma.TransactionIsolationLevel.ReadCommitted } - ); - } -} - -/* -Starting execution flow: - -1. Run id is pulled from a queue -2. Prepare the run for an attempt (returns data to send to the worker) - a. The run is marked as "waiting to start"? - b. Create a TaskRunState with the run id, and the state "waiting to start". - c. Start a heartbeat with the TaskRunState id, in case it never starts. -3. The run is sent to the worker -4. When the worker has received the run, it ask the platform for an attempt -5. The attempt is created - a. The attempt is created - b. The TaskRunState is updated to "EXECUTING" - c. Start a heartbeat with the TaskRunState id. - c. The TaskRun is updated to "EXECUTING" -6. A response is sent back to the worker with the attempt data -7. The code executes... -*/ diff --git a/internal-packages/run-engine/src/index.ts b/internal-packages/run-engine/src/index.ts deleted file mode 100644 index b71175be2a..0000000000 --- a/internal-packages/run-engine/src/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { RunEngine } from "./engine/index"; diff --git a/internal-packages/run-engine/src/run-queue/index.test.ts b/internal-packages/run-engine/src/run-queue/index.test.ts deleted file mode 100644 index 8a3f40ada9..0000000000 --- a/internal-packages/run-engine/src/run-queue/index.test.ts +++ /dev/null @@ -1,453 +0,0 @@ -import { trace } from "@opentelemetry/api"; -import { Logger } from "@trigger.dev/core/logger"; -import { describe } from "node:test"; -import { redisTest } from "@internal/testcontainers"; -import { RunQueue } from "./index.js"; -import { RunQueueShortKeyProducer } from "./keyProducer.js"; -import { SimpleWeightedChoiceStrategy } from "./simpleWeightedPriorityStrategy.js"; -import { InputPayload } from "./types.js"; -import { abort } from "node:process"; - -const testOptions = { - name: "rq", - tracer: trace.getTracer("rq"), - queuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 36 }), - envQueuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 12 }), - workers: 1, - defaultEnvConcurrency: 10, - enableRebalancing: false, - logger: new Logger("RunQueue", "warn"), -}; - -const authenticatedEnvProd = { - id: "e1234", - type: "PRODUCTION" as const, - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, -}; - -const authenticatedEnvDev = { - id: "e1234", - type: "DEVELOPMENT" as const, - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, -}; - -const messageProd: InputPayload = { - runId: "r1234", - taskIdentifier: "task/my-task", - orgId: "o1234", - projectId: "p1234", - environmentId: "e1234", - environmentType: "PRODUCTION", - queue: "task/my-task", - timestamp: Date.now(), -}; - -const messageDev: InputPayload = { - runId: "r4321", - taskIdentifier: "task/my-task", - orgId: "o1234", - projectId: "p1234", - environmentId: "e4321", - environmentType: "DEVELOPMENT", - queue: "task/my-task", - timestamp: Date.now(), -}; - -describe("RunQueue", () => { - redisTest( - "Get/set Queue concurrency limit", - { timeout: 5_000 }, - async ({ redisContainer, redis }) => { - const queue = new RunQueue({ - ...testOptions, - redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, - }); - - try { - //initial value - const initial = await queue.getQueueConcurrencyLimit(authenticatedEnvProd, "task/my-task"); - expect(initial).toBe(undefined); - - //set 20 - const result = await queue.updateQueueConcurrencyLimits( - authenticatedEnvProd, - "task/my-task", - 20 - ); - expect(result).toBe("OK"); - - //get 20 - const updated = await queue.getQueueConcurrencyLimit(authenticatedEnvProd, "task/my-task"); - expect(updated).toBe(20); - - //remove - const result2 = await queue.removeQueueConcurrencyLimits( - authenticatedEnvProd, - "task/my-task" - ); - expect(result2).toBe(1); - - //get undefined - const removed = await queue.getQueueConcurrencyLimit(authenticatedEnvProd, "task/my-task"); - expect(removed).toBe(undefined); - } finally { - await queue.quit(); - } - } - ); - - redisTest( - "Update env concurrency limits", - { timeout: 5_000 }, - async ({ redisContainer, redis }) => { - const queue = new RunQueue({ - ...testOptions, - redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, - }); - - try { - //initial value - const initial = await queue.getEnvConcurrencyLimit(authenticatedEnvProd); - expect(initial).toBe(10); - - //set 20 - await queue.updateEnvConcurrencyLimits({ - ...authenticatedEnvProd, - maximumConcurrencyLimit: 20, - }); - - //get 20 - const updated = await queue.getEnvConcurrencyLimit(authenticatedEnvProd); - expect(updated).toBe(20); - } finally { - await queue.quit(); - } - } - ); - - redisTest( - "Enqueue/Dequeue a message in env (DEV run, no concurrency key)", - { timeout: 5_000 }, - async ({ redisContainer, redis }) => { - const queue = new RunQueue({ - ...testOptions, - redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, - }); - - try { - //initial queue length - const result = await queue.lengthOfQueue(authenticatedEnvDev, messageDev.queue); - expect(result).toBe(0); - - //initial oldest message - const oldestScore = await queue.oldestMessageInQueue(authenticatedEnvDev, messageDev.queue); - expect(oldestScore).toBe(undefined); - - const envMasterQueue = `env:${authenticatedEnvDev.id}`; - - //enqueue message - await queue.enqueueMessage({ - env: authenticatedEnvDev, - message: messageDev, - masterQueue: `env:${authenticatedEnvDev.id}`, - }); - - //queue length - const result2 = await queue.lengthOfQueue(authenticatedEnvDev, messageDev.queue); - expect(result2).toBe(1); - - //oldest message - const oldestScore2 = await queue.oldestMessageInQueue( - authenticatedEnvDev, - messageDev.queue - ); - expect(oldestScore2).toBe(messageDev.timestamp); - - //concurrencies - const queueConcurrency = await queue.currentConcurrencyOfQueue( - authenticatedEnvDev, - messageDev.queue - ); - expect(queueConcurrency).toBe(0); - const envConcurrency = await queue.currentConcurrencyOfEnvironment(authenticatedEnvDev); - expect(envConcurrency).toBe(0); - const projectConcurrency = await queue.currentConcurrencyOfProject(authenticatedEnvDev); - expect(projectConcurrency).toBe(0); - const taskConcurrency = await queue.currentConcurrencyOfTask( - authenticatedEnvDev, - messageDev.taskIdentifier - ); - expect(taskConcurrency).toBe(0); - - const dequeued = await queue.dequeueMessageInSharedQueue("test_12345", envMasterQueue); - expect(dequeued?.messageId).toEqual(messageDev.runId); - expect(dequeued?.message.orgId).toEqual(messageDev.orgId); - expect(dequeued?.message.version).toEqual("1"); - expect(dequeued?.message.masterQueue).toEqual(envMasterQueue); - - //concurrencies - const queueConcurrency2 = await queue.currentConcurrencyOfQueue( - authenticatedEnvDev, - messageDev.queue - ); - expect(queueConcurrency2).toBe(1); - const envConcurrency2 = await queue.currentConcurrencyOfEnvironment(authenticatedEnvDev); - expect(envConcurrency2).toBe(1); - const projectConcurrency2 = await queue.currentConcurrencyOfProject(authenticatedEnvDev); - expect(projectConcurrency2).toBe(1); - const taskConcurrency2 = await queue.currentConcurrencyOfTask( - authenticatedEnvDev, - messageDev.taskIdentifier - ); - expect(taskConcurrency2).toBe(1); - - const dequeued2 = await queue.dequeueMessageInSharedQueue("test_12345", envMasterQueue); - expect(dequeued2).toBe(undefined); - } finally { - await queue.quit(); - } - } - ); - - redisTest( - "Enqueue/Dequeue a message from the shared queue (PROD run, no concurrency key)", - { timeout: 5_000 }, - async ({ redisContainer, redis }) => { - const queue = new RunQueue({ - ...testOptions, - redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, - }); - - try { - //initial queue length - const result = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); - expect(result).toBe(0); - - //initial oldest message - const oldestScore = await queue.oldestMessageInQueue( - authenticatedEnvProd, - messageProd.queue - ); - expect(oldestScore).toBe(undefined); - - //enqueue message - await queue.enqueueMessage({ - env: authenticatedEnvProd, - message: messageProd, - masterQueue: "main", - }); - - //queue length - const result2 = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); - expect(result2).toBe(1); - - //oldest message - const oldestScore2 = await queue.oldestMessageInQueue( - authenticatedEnvProd, - messageProd.queue - ); - expect(oldestScore2).toBe(messageProd.timestamp); - - //concurrencies - const queueConcurrency = await queue.currentConcurrencyOfQueue( - authenticatedEnvProd, - messageProd.queue - ); - expect(queueConcurrency).toBe(0); - const envConcurrency = await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd); - expect(envConcurrency).toBe(0); - const projectConcurrency = await queue.currentConcurrencyOfProject(authenticatedEnvProd); - expect(projectConcurrency).toBe(0); - const taskConcurrency = await queue.currentConcurrencyOfTask( - authenticatedEnvProd, - messageProd.taskIdentifier - ); - expect(taskConcurrency).toBe(0); - - //dequeue - const dequeued = await queue.dequeueMessageInSharedQueue("test_12345", "main"); - expect(dequeued?.messageId).toEqual(messageProd.runId); - expect(dequeued?.message.orgId).toEqual(messageProd.orgId); - expect(dequeued?.message.version).toEqual("1"); - expect(dequeued?.message.masterQueue).toEqual("main"); - - //concurrencies - const queueConcurrency2 = await queue.currentConcurrencyOfQueue( - authenticatedEnvProd, - messageProd.queue - ); - expect(queueConcurrency2).toBe(1); - const envConcurrency2 = await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd); - expect(envConcurrency2).toBe(1); - const projectConcurrency2 = await queue.currentConcurrencyOfProject(authenticatedEnvProd); - expect(projectConcurrency2).toBe(1); - const taskConcurrency2 = await queue.currentConcurrencyOfTask( - authenticatedEnvProd, - messageProd.taskIdentifier - ); - expect(taskConcurrency2).toBe(1); - - //queue length - const length2 = await queue.lengthOfQueue(authenticatedEnvProd, messageProd.queue); - expect(length2).toBe(0); - - const dequeued2 = await queue.dequeueMessageInSharedQueue("test_12345", "main"); - expect(dequeued2).toBe(undefined); - } finally { - await queue.quit(); - } - } - ); - - redisTest("Get shared queue details", { timeout: 5_000 }, async ({ redisContainer, redis }) => { - const queue = new RunQueue({ - ...testOptions, - redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, - }); - - try { - const result = await queue.getSharedQueueDetails("main"); - expect(result.selectionId).toBe("getSharedQueueDetails"); - expect(result.queueCount).toBe(0); - expect(result.queueChoice.choice).toStrictEqual({ abort: true }); - - await queue.enqueueMessage({ - env: authenticatedEnvProd, - message: messageProd, - masterQueue: "main", - }); - - const result2 = await queue.getSharedQueueDetails("main"); - expect(result2.selectionId).toBe("getSharedQueueDetails"); - expect(result2.queueCount).toBe(1); - expect(result2.queues[0].score).toBe(messageProd.timestamp); - expect(result2.queueChoice.choice).toBe( - "{org:o1234}:proj:p1234:env:e1234:queue:task/my-task" - ); - } finally { - await queue.quit(); - } - }); - - redisTest("Acking", { timeout: 5_000 }, async ({ redisContainer, redis }) => { - const queue = new RunQueue({ - ...testOptions, - redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, - }); - - try { - await queue.enqueueMessage({ - env: authenticatedEnvProd, - message: messageProd, - masterQueue: "main", - }); - - const message = await queue.dequeueMessageInSharedQueue("test_12345", "main"); - expect(message).toBeDefined(); - - //check the message is gone - const key = queue.keys.messageKey(message!.message.orgId, message!.messageId); - const exists = await redis.exists(key); - expect(exists).toBe(1); - - await queue.acknowledgeMessage(message!.message.orgId, message!.messageId); - - //concurrencies - const queueConcurrency = await queue.currentConcurrencyOfQueue( - authenticatedEnvProd, - messageProd.queue - ); - expect(queueConcurrency).toBe(0); - const envConcurrency = await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd); - expect(envConcurrency).toBe(0); - const projectConcurrency = await queue.currentConcurrencyOfProject(authenticatedEnvProd); - expect(projectConcurrency).toBe(0); - const taskConcurrency = await queue.currentConcurrencyOfTask( - authenticatedEnvProd, - messageProd.taskIdentifier - ); - expect(taskConcurrency).toBe(0); - - //check the message is gone - const exists2 = await redis.exists(key); - expect(exists2).toBe(0); - - //dequeue - const message2 = await queue.dequeueMessageInSharedQueue("test_12345", "main"); - expect(message2).toBeUndefined(); - } finally { - await queue.quit(); - } - }); - - redisTest("Nacking", { timeout: 5_000 }, async ({ redisContainer, redis }) => { - const queue = new RunQueue({ - ...testOptions, - redis: { host: redisContainer.getHost(), port: redisContainer.getPort() }, - }); - - try { - await queue.enqueueMessage({ - env: authenticatedEnvProd, - message: messageProd, - masterQueue: "main", - }); - - const message = await queue.dequeueMessageInSharedQueue("test_12345", "main"); - expect(message).toBeDefined(); - - //check the message is there - const key = queue.keys.messageKey(message!.message.orgId, message!.messageId); - const exists = await redis.exists(key); - expect(exists).toBe(1); - - //concurrencies - const queueConcurrency = await queue.currentConcurrencyOfQueue( - authenticatedEnvProd, - messageProd.queue - ); - expect(queueConcurrency).toBe(1); - const envConcurrency = await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd); - expect(envConcurrency).toBe(1); - const projectConcurrency = await queue.currentConcurrencyOfProject(authenticatedEnvProd); - expect(projectConcurrency).toBe(1); - const taskConcurrency = await queue.currentConcurrencyOfTask( - authenticatedEnvProd, - messageProd.taskIdentifier - ); - expect(taskConcurrency).toBe(1); - - await queue.nackMessage(message!.message.orgId, message!.messageId); - - //concurrencies - const queueConcurrency2 = await queue.currentConcurrencyOfQueue( - authenticatedEnvProd, - messageProd.queue - ); - expect(queueConcurrency2).toBe(0); - const envConcurrency2 = await queue.currentConcurrencyOfEnvironment(authenticatedEnvProd); - expect(envConcurrency2).toBe(0); - const projectConcurrency2 = await queue.currentConcurrencyOfProject(authenticatedEnvProd); - expect(projectConcurrency2).toBe(0); - const taskConcurrency2 = await queue.currentConcurrencyOfTask( - authenticatedEnvProd, - messageProd.taskIdentifier - ); - expect(taskConcurrency2).toBe(0); - - //check the message is there - const exists2 = await redis.exists(key); - expect(exists2).toBe(1); - - //dequeue - const message2 = await queue.dequeueMessageInSharedQueue("test_12345", "main"); - expect(message2?.messageId).toBe(messageProd.runId); - } finally { - await queue.quit(); - } - }); -}); diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts deleted file mode 100644 index cb07054051..0000000000 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ /dev/null @@ -1,1384 +0,0 @@ -import { context, propagation, Span, SpanKind, SpanOptions, Tracer } from "@opentelemetry/api"; -import { - SEMATTRS_MESSAGE_ID, - SEMATTRS_MESSAGING_OPERATION, - SEMATTRS_MESSAGING_SYSTEM, -} from "@opentelemetry/semantic-conventions"; -import { Logger } from "@trigger.dev/core/logger"; -import { flattenAttributes } from "@trigger.dev/core/v3"; -import { Redis, type Callback, type RedisOptions, type Result } from "ioredis"; -import { AsyncWorker } from "../shared/asyncWorker.js"; -import { - attributesFromAuthenticatedEnv, - MinimalAuthenticatedEnvironment, -} from "../shared/index.js"; -import { - InputPayload, - OutputPayload, - QueueCapacities, - QueueRange, - RunQueueKeyProducer, - RunQueuePriorityStrategy, -} from "./types.js"; -import { RunQueueShortKeyProducer } from "./keyProducer.js"; - -const SemanticAttributes = { - QUEUE: "runqueue.queue", - MASTER_QUEUE: "runqueue.masterQueue", - RUN_ID: "runqueue.runId", - CONCURRENCY_KEY: "runqueue.concurrencyKey", - ORG_ID: "runqueue.orgId", -}; - -export type RunQueueOptions = { - name: string; - tracer: Tracer; - redis: RedisOptions; - defaultEnvConcurrency: number; - windowSize?: number; - workers: number; - queuePriorityStrategy: RunQueuePriorityStrategy; - envQueuePriorityStrategy: RunQueuePriorityStrategy; - enableRebalancing?: boolean; - verbose?: boolean; - logger: Logger; -}; - -/** - * RunQueue โ€“ the queue that's used to process runs - */ -export class RunQueue { - private logger: Logger; - private redis: Redis; - public keys: RunQueueKeyProducer; - private queuePriorityStrategy: RunQueuePriorityStrategy; - #rebalanceWorkers: Array = []; - - constructor(private readonly options: RunQueueOptions) { - this.redis = new Redis(options.redis); - this.logger = options.logger; - - this.keys = new RunQueueShortKeyProducer("rq:"); - this.queuePriorityStrategy = options.queuePriorityStrategy; - - this.#registerCommands(); - } - - get name() { - return this.options.name; - } - - get tracer() { - return this.options.tracer; - } - - public async updateQueueConcurrencyLimits( - env: MinimalAuthenticatedEnvironment, - queue: string, - concurrency: number - ) { - return this.redis.set(this.keys.queueConcurrencyLimitKey(env, queue), concurrency); - } - - public async removeQueueConcurrencyLimits(env: MinimalAuthenticatedEnvironment, queue: string) { - return this.redis.del(this.keys.queueConcurrencyLimitKey(env, queue)); - } - - public async getQueueConcurrencyLimit(env: MinimalAuthenticatedEnvironment, queue: string) { - const result = await this.redis.get(this.keys.queueConcurrencyLimitKey(env, queue)); - - return result ? Number(result) : undefined; - } - - public async updateEnvConcurrencyLimits(env: MinimalAuthenticatedEnvironment) { - await this.#callUpdateGlobalConcurrencyLimits({ - envConcurrencyLimitKey: this.keys.envConcurrencyLimitKey(env), - envConcurrencyLimit: env.maximumConcurrencyLimit, - }); - } - - public async getEnvConcurrencyLimit(env: MinimalAuthenticatedEnvironment) { - const result = await this.redis.get(this.keys.envConcurrencyLimitKey(env)); - - return result ? Number(result) : this.options.defaultEnvConcurrency; - } - - public async lengthOfQueue( - env: MinimalAuthenticatedEnvironment, - queue: string, - concurrencyKey?: string - ) { - return this.redis.zcard(this.keys.queueKey(env, queue, concurrencyKey)); - } - - public async oldestMessageInQueue( - env: MinimalAuthenticatedEnvironment, - queue: string, - concurrencyKey?: string - ) { - // Get the "score" of the sorted set to get the oldest message score - const result = await this.redis.zrange( - this.keys.queueKey(env, queue, concurrencyKey), - 0, - 0, - "WITHSCORES" - ); - - if (result.length === 0) { - return; - } - - return Number(result[1]); - } - - public async currentConcurrencyOfQueue( - env: MinimalAuthenticatedEnvironment, - queue: string, - concurrencyKey?: string - ) { - return this.redis.scard(this.keys.currentConcurrencyKey(env, queue, concurrencyKey)); - } - - public async currentConcurrencyOfEnvironment(env: MinimalAuthenticatedEnvironment) { - return this.redis.scard(this.keys.envCurrentConcurrencyKey(env)); - } - - public async currentConcurrencyOfProject(env: MinimalAuthenticatedEnvironment) { - return this.redis.scard(this.keys.projectCurrentConcurrencyKey(env)); - } - - public async currentConcurrencyOfTask( - env: MinimalAuthenticatedEnvironment, - taskIdentifier: string - ) { - return this.redis.scard(this.keys.taskIdentifierCurrentConcurrencyKey(env, taskIdentifier)); - } - - public async enqueueMessage({ - env, - message, - masterQueue, - }: { - env: MinimalAuthenticatedEnvironment; - message: InputPayload; - masterQueue: string; - }) { - return await this.#trace( - "enqueueMessage", - async (span) => { - const { runId, concurrencyKey } = message; - - const queue = this.keys.queueKey(env, message.queue, concurrencyKey); - - propagation.inject(context.active(), message); - - span.setAttributes({ - [SemanticAttributes.QUEUE]: queue, - [SemanticAttributes.RUN_ID]: runId, - [SemanticAttributes.CONCURRENCY_KEY]: concurrencyKey, - [SemanticAttributes.MASTER_QUEUE]: masterQueue, - }); - - const messagePayload: OutputPayload = { - ...message, - version: "1", - queue, - masterQueue, - }; - - await this.#callEnqueueMessage(messagePayload, masterQueue); - }, - { - kind: SpanKind.PRODUCER, - attributes: { - [SEMATTRS_MESSAGING_OPERATION]: "publish", - [SEMATTRS_MESSAGE_ID]: message.runId, - [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", - ...attributesFromAuthenticatedEnv(env), - }, - } - ); - } - - public async getSharedQueueDetails(masterQueue: string) { - const { range } = await this.queuePriorityStrategy.nextCandidateSelection( - masterQueue, - "getSharedQueueDetails" - ); - const queues = await this.#getChildQueuesWithScores(masterQueue, range); - - const queuesWithScores = await this.#calculateQueueScores(queues, (queue) => - this.#calculateMessageQueueCapacities(queue) - ); - - // We need to priority shuffle here to ensure all workers aren't just working on the highest priority queue - const choice = this.queuePriorityStrategy.chooseQueue( - queuesWithScores, - masterQueue, - "getSharedQueueDetails", - range - ); - - return { - selectionId: "getSharedQueueDetails", - queues, - queuesWithScores, - nextRange: range, - queueCount: queues.length, - queueChoice: choice, - }; - } - - /** - * Dequeue a message from the shared queue (this should be used in production environments) - */ - public async dequeueMessageInSharedQueue(consumerId: string, masterQueue: string) { - return this.#trace( - "dequeueMessageInSharedQueue", - async (span) => { - // Read the parent queue for matching queues - const messageQueue = await this.#getRandomQueueFromParentQueue( - masterQueue, - this.options.queuePriorityStrategy, - (queue) => this.#calculateMessageQueueCapacities(queue, { checkForDisabled: true }), - consumerId - ); - - if (!messageQueue) { - return; - } - - // If the queue includes a concurrency key, we need to remove the ck:concurrencyKey from the queue name - const message = await this.#callDequeueMessage({ - messageQueue, - masterQueue, - concurrencyLimitKey: this.keys.concurrencyLimitKeyFromQueue(messageQueue), - currentConcurrencyKey: this.keys.currentConcurrencyKeyFromQueue(messageQueue), - envConcurrencyLimitKey: this.keys.envConcurrencyLimitKeyFromQueue(messageQueue), - envCurrentConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(messageQueue), - projectCurrentConcurrencyKey: - this.keys.projectCurrentConcurrencyKeyFromQueue(messageQueue), - messageKeyPrefix: this.keys.messageKeyPrefixFromQueue(messageQueue), - taskCurrentConcurrentKeyPrefix: - this.keys.taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(messageQueue), - }); - - if (!message) { - return; - } - - span.setAttributes({ - [SEMATTRS_MESSAGE_ID]: message.messageId, - [SemanticAttributes.QUEUE]: message.message.queue, - [SemanticAttributes.RUN_ID]: message.message.runId, - [SemanticAttributes.CONCURRENCY_KEY]: message.message.concurrencyKey, - [SemanticAttributes.MASTER_QUEUE]: masterQueue, - }); - - return message; - }, - { - kind: SpanKind.CONSUMER, - attributes: { - [SEMATTRS_MESSAGING_OPERATION]: "receive", - [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", - }, - } - ); - } - - /** - * Acknowledge a message, which will: - * - remove all data from the queue - * - release all concurrency - * This is done when the run is in a final state. - * @param messageId - */ - public async acknowledgeMessage(orgId: string, messageId: string) { - return this.#trace( - "acknowledgeMessage", - async (span) => { - const message = await this.#readMessage(orgId, messageId); - - if (!message) { - this.logger.log(`[${this.name}].acknowledgeMessage() message not found`, { - messageId, - service: this.name, - }); - return; - } - - span.setAttributes({ - [SemanticAttributes.QUEUE]: message.queue, - [SemanticAttributes.ORG_ID]: message.orgId, - [SemanticAttributes.RUN_ID]: messageId, - [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, - }); - - await this.#callAcknowledgeMessage({ - messageId, - messageQueue: message.queue, - masterQueue: message.masterQueue, - messageKey: this.keys.messageKey(orgId, messageId), - concurrencyKey: this.keys.currentConcurrencyKeyFromQueue(message.queue), - envConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(message.queue), - taskConcurrencyKey: this.keys.taskIdentifierCurrentConcurrencyKeyFromQueue( - message.queue, - message.taskIdentifier - ), - projectConcurrencyKey: this.keys.projectCurrentConcurrencyKeyFromQueue(message.queue), - }); - }, - { - kind: SpanKind.CONSUMER, - attributes: { - [SEMATTRS_MESSAGING_OPERATION]: "ack", - [SEMATTRS_MESSAGE_ID]: messageId, - [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", - }, - } - ); - } - - /** - * Negative acknowledge a message, which will requeue the message (with an optional future date) - */ - public async nackMessage(orgId: string, messageId: string, retryAt: number = Date.now()) { - return this.#trace( - "nackMessage", - async (span) => { - const message = await this.#readMessage(orgId, messageId); - if (!message) { - this.logger.log(`[${this.name}].nackMessage() message not found`, { - orgId, - messageId, - retryAt, - service: this.name, - }); - return; - } - - span.setAttributes({ - [SemanticAttributes.QUEUE]: message.queue, - [SemanticAttributes.RUN_ID]: messageId, - [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, - [SemanticAttributes.MASTER_QUEUE]: message.masterQueue, - }); - - const messageKey = this.keys.messageKey(orgId, messageId); - const messageQueue = message.queue; - const parentQueue = message.masterQueue; - const concurrencyKey = this.keys.currentConcurrencyKeyFromQueue(message.queue); - const envConcurrencyKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); - const taskConcurrencyKey = this.keys.taskIdentifierCurrentConcurrencyKeyFromQueue( - message.queue, - message.taskIdentifier - ); - const projectConcurrencyKey = this.keys.projectCurrentConcurrencyKeyFromQueue( - message.queue - ); - - const messageScore = retryAt; - - this.logger.debug("Calling nackMessage", { - messageKey, - messageQueue, - parentQueue, - concurrencyKey, - envConcurrencyKey, - projectConcurrencyKey, - taskConcurrencyKey, - messageId, - messageScore, - service: this.name, - }); - - await this.redis.nackMessage( - //keys - messageKey, - messageQueue, - parentQueue, - concurrencyKey, - envConcurrencyKey, - projectConcurrencyKey, - taskConcurrencyKey, - //args - messageId, - String(messageScore) - ); - }, - { - kind: SpanKind.CONSUMER, - attributes: { - [SEMATTRS_MESSAGING_OPERATION]: "nack", - [SEMATTRS_MESSAGE_ID]: messageId, - [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", - }, - } - ); - } - - public async releaseConcurrency(messageId: string, releaseForRun: boolean = false) { - return this.#trace( - "releaseConcurrency", - async (span) => { - // span.setAttributes({ - // [SemanticAttributes.MESSAGE_ID]: messageId, - // }); - // const message = await this.readMessage(messageId); - // if (!message) { - // logger.log(`[${this.name}].releaseConcurrency() message not found`, { - // messageId, - // releaseForRun, - // service: this.name, - // }); - // return; - // } - // span.setAttributes({ - // [SemanticAttributes.QUEUE]: message.queue, - // [SemanticAttributes.MESSAGE_ID]: message.messageId, - // [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey, - // [SemanticAttributes.PARENT_QUEUE]: message.parentQueue, - // }); - // const concurrencyKey = this.keys.currentConcurrencyKeyFromQueue(message.queue); - // const envConcurrencyKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); - // const orgConcurrencyKey = this.keys.orgCurrentConcurrencyKeyFromQueue(message.queue); - // logger.debug("Calling releaseConcurrency", { - // messageId, - // queue: message.queue, - // concurrencyKey, - // envConcurrencyKey, - // orgConcurrencyKey, - // service: this.name, - // releaseForRun, - // }); - // return this.redis.releaseConcurrency( - // //don't release the for the run, it breaks concurrencyLimits - // releaseForRun ? concurrencyKey : "", - // envConcurrencyKey, - // orgConcurrencyKey, - // message.messageId - // ); - }, - { - kind: SpanKind.CONSUMER, - attributes: { - [SEMATTRS_MESSAGING_OPERATION]: "releaseConcurrency", - [SEMATTRS_MESSAGE_ID]: messageId, - [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", - }, - } - ); - } - - queueConcurrencyScanStream( - count: number = 100, - onEndCallback?: () => void, - onErrorCallback?: (error: Error) => void - ) { - const pattern = this.keys.queueCurrentConcurrencyScanPattern(); - - this.logger.debug("Starting queue concurrency scan stream", { - pattern, - component: "runqueue", - operation: "queueConcurrencyScanStream", - service: this.name, - count, - }); - - const redis = this.redis.duplicate(); - - const stream = redis.scanStream({ - match: pattern, - type: "set", - count, - }); - - stream.on("end", () => { - onEndCallback?.(); - redis.quit(); - }); - - stream.on("error", (error) => { - onErrorCallback?.(error); - redis.quit(); - }); - - return { stream, redis }; - } - - async quit() { - await Promise.all(this.#rebalanceWorkers.map((worker) => worker.stop())); - await this.redis.quit(); - } - - async #trace( - name: string, - fn: (span: Span) => Promise, - options?: SpanOptions & { sampleRate?: number } - ): Promise { - return this.tracer.startActiveSpan( - name, - { - ...options, - attributes: { - ...options?.attributes, - }, - }, - async (span) => { - try { - return await fn(span); - } catch (e) { - if (e instanceof Error) { - span.recordException(e); - } else { - span.recordException(new Error(String(e))); - } - - throw e; - } finally { - span.end(); - } - } - ); - } - - async #readMessage(orgId: string, messageId: string) { - return this.#trace( - "readMessage", - async (span) => { - const rawMessage = await this.redis.get(this.keys.messageKey(orgId, messageId)); - - if (!rawMessage) { - return; - } - - const message = OutputPayload.safeParse(JSON.parse(rawMessage)); - - if (!message.success) { - this.logger.error(`[${this.name}] Failed to parse message`, { - messageId, - error: message.error, - service: this.name, - }); - - return; - } - - return message.data; - }, - { - attributes: { - [SEMATTRS_MESSAGING_OPERATION]: "receive", - [SEMATTRS_MESSAGE_ID]: messageId, - [SEMATTRS_MESSAGING_SYSTEM]: "marqs", - [SemanticAttributes.RUN_ID]: messageId, - }, - } - ); - } - - async #getRandomQueueFromParentQueue( - parentQueue: string, - queuePriorityStrategy: RunQueuePriorityStrategy, - calculateCapacities: (queue: string) => Promise, - consumerId: string - ) { - return this.#trace( - "getRandomQueueFromParentQueue", - async (span) => { - span.setAttribute("consumerId", consumerId); - - const { range } = await queuePriorityStrategy.nextCandidateSelection( - parentQueue, - consumerId - ); - - const queues = await this.#getChildQueuesWithScores(parentQueue, range, span); - span.setAttribute("queueCount", queues.length); - - const queuesWithScores = await this.#calculateQueueScores(queues, calculateCapacities); - span.setAttribute("queuesWithScoresCount", queuesWithScores.length); - - // We need to priority shuffle here to ensure all workers aren't just working on the highest priority queue - const { choice, nextRange } = this.queuePriorityStrategy.chooseQueue( - queuesWithScores, - parentQueue, - consumerId, - range - ); - - span.setAttributes({ - ...flattenAttributes(queues, "runqueue.queues"), - }); - span.setAttributes({ - ...flattenAttributes(queuesWithScores, "runqueue.queuesWithScores"), - }); - span.setAttribute("range.offset", range.offset); - span.setAttribute("range.count", range.count); - span.setAttribute("nextRange.offset", nextRange.offset); - span.setAttribute("nextRange.count", nextRange.count); - - if (this.options.verbose || nextRange.offset > 0) { - if (typeof choice === "string") { - this.logger.debug(`[${this.name}] getRandomQueueFromParentQueue`, { - queues, - queuesWithScores, - range, - nextRange, - queueCount: queues.length, - queuesWithScoresCount: queuesWithScores.length, - queueChoice: choice, - consumerId, - }); - } else { - this.logger.debug(`[${this.name}] getRandomQueueFromParentQueue`, { - queues, - queuesWithScores, - range, - nextRange, - queueCount: queues.length, - queuesWithScoresCount: queuesWithScores.length, - noQueueChoice: true, - consumerId, - }); - } - } - - if (typeof choice !== "string") { - span.setAttribute("noQueueChoice", true); - - return; - } else { - span.setAttribute("queueChoice", choice); - - return choice; - } - }, - { - kind: SpanKind.CONSUMER, - attributes: { - [SEMATTRS_MESSAGING_OPERATION]: "receive", - [SEMATTRS_MESSAGING_SYSTEM]: "runqueue", - [SemanticAttributes.MASTER_QUEUE]: parentQueue, - }, - } - ); - } - - // Calculate the weights of the queues based on the age and the capacity - async #calculateQueueScores( - queues: Array<{ value: string; score: number }>, - calculateCapacities: (queue: string) => Promise - ) { - const now = Date.now(); - - const queueScores = await Promise.all( - queues.map(async (queue) => { - return { - queue: queue.value, - capacities: await calculateCapacities(queue.value), - age: now - queue.score, - size: await this.redis.zcard(queue.value), - }; - }) - ); - - return queueScores; - } - - async #calculateMessageQueueCapacities(queue: string, options?: { checkForDisabled?: boolean }) { - return await this.#callCalculateMessageCapacities({ - currentConcurrencyKey: this.keys.currentConcurrencyKeyFromQueue(queue), - currentEnvConcurrencyKey: this.keys.envCurrentConcurrencyKeyFromQueue(queue), - concurrencyLimitKey: this.keys.concurrencyLimitKeyFromQueue(queue), - envConcurrencyLimitKey: this.keys.envConcurrencyLimitKeyFromQueue(queue), - disabledConcurrencyLimitKey: options?.checkForDisabled - ? this.keys.disabledConcurrencyLimitKeyFromQueue(queue) - : undefined, - }); - } - - async #getChildQueuesWithScores( - key: string, - range: QueueRange, - span?: Span - ): Promise> { - const valuesWithScores = await this.redis.zrangebyscore( - key, - "-inf", - Date.now(), - "WITHSCORES", - "LIMIT", - range.offset, - range.count - ); - - span?.setAttribute("zrangebyscore.valuesWithScores.rawLength", valuesWithScores.length); - span?.setAttributes({ - ...flattenAttributes(valuesWithScores, "zrangebyscore.valuesWithScores.rawValues"), - }); - - const result: Array<{ value: string; score: number }> = []; - - for (let i = 0; i < valuesWithScores.length; i += 2) { - result.push({ - value: valuesWithScores[i], - score: Number(valuesWithScores[i + 1]), - }); - } - - return result; - } - - async #callEnqueueMessage(message: OutputPayload, parentQueue: string) { - const concurrencyKey = this.keys.currentConcurrencyKeyFromQueue(message.queue); - const envConcurrencyKey = this.keys.envCurrentConcurrencyKeyFromQueue(message.queue); - const taskConcurrencyKey = this.keys.taskIdentifierCurrentConcurrencyKeyFromQueue( - message.queue, - message.taskIdentifier - ); - const projectConcurrencyKey = this.keys.projectCurrentConcurrencyKeyFromQueue(message.queue); - - this.logger.debug("Calling enqueueMessage", { - messagePayload: message, - concurrencyKey, - envConcurrencyKey, - service: this.name, - }); - - return this.redis.enqueueMessage( - message.queue, - parentQueue, - this.keys.messageKey(message.orgId, message.runId), - concurrencyKey, - envConcurrencyKey, - taskConcurrencyKey, - projectConcurrencyKey, - message.queue, - message.runId, - JSON.stringify(message), - String(message.timestamp) - ); - } - - async #callDequeueMessage({ - messageQueue, - masterQueue, - concurrencyLimitKey, - envConcurrencyLimitKey, - currentConcurrencyKey, - envCurrentConcurrencyKey, - projectCurrentConcurrencyKey, - messageKeyPrefix, - taskCurrentConcurrentKeyPrefix, - }: { - messageQueue: string; - masterQueue: string; - concurrencyLimitKey: string; - envConcurrencyLimitKey: string; - currentConcurrencyKey: string; - envCurrentConcurrencyKey: string; - projectCurrentConcurrencyKey: string; - messageKeyPrefix: string; - taskCurrentConcurrentKeyPrefix: string; - }) { - const result = await this.redis.dequeueMessage( - //keys - messageQueue, - masterQueue, - concurrencyLimitKey, - envConcurrencyLimitKey, - currentConcurrencyKey, - envCurrentConcurrencyKey, - projectCurrentConcurrencyKey, - messageKeyPrefix, - taskCurrentConcurrentKeyPrefix, - //args - messageQueue, - String(Date.now()), - String(this.options.defaultEnvConcurrency) - ); - - if (!result) { - return; - } - - this.logger.debug("Dequeue message result", { - result, - service: this.name, - }); - - if (result.length !== 3) { - this.logger.error("Invalid dequeue message result", { - result, - service: this.name, - }); - return; - } - - const [messageId, messageScore, rawMessage] = result; - - //read message - const parsedMessage = OutputPayload.safeParse(JSON.parse(rawMessage)); - if (!parsedMessage.success) { - this.logger.error(`[${this.name}] Failed to parse message`, { - messageId, - error: parsedMessage.error, - service: this.name, - }); - - return; - } - - const message = parsedMessage.data; - - return { - messageId, - messageScore, - message, - }; - } - - async #callAcknowledgeMessage({ - messageId, - masterQueue, - messageKey, - messageQueue, - concurrencyKey, - envConcurrencyKey, - taskConcurrencyKey, - projectConcurrencyKey, - }: { - masterQueue: string; - messageKey: string; - messageQueue: string; - concurrencyKey: string; - envConcurrencyKey: string; - taskConcurrencyKey: string; - projectConcurrencyKey: string; - messageId: string; - }) { - this.logger.debug("Calling acknowledgeMessage", { - messageKey, - messageQueue, - concurrencyKey, - envConcurrencyKey, - projectConcurrencyKey, - taskConcurrencyKey, - messageId, - masterQueue, - service: this.name, - }); - - return this.redis.acknowledgeMessage( - masterQueue, - messageKey, - messageQueue, - concurrencyKey, - envConcurrencyKey, - projectConcurrencyKey, - taskConcurrencyKey, - messageId - ); - } - - async #callCalculateMessageCapacities({ - currentConcurrencyKey, - currentEnvConcurrencyKey, - concurrencyLimitKey, - envConcurrencyLimitKey, - disabledConcurrencyLimitKey, - }: { - currentConcurrencyKey: string; - currentEnvConcurrencyKey: string; - concurrencyLimitKey: string; - envConcurrencyLimitKey: string; - disabledConcurrencyLimitKey: string | undefined; - }): Promise { - const capacities = disabledConcurrencyLimitKey - ? await this.redis.calculateMessageQueueCapacitiesWithDisabling( - currentConcurrencyKey, - currentEnvConcurrencyKey, - concurrencyLimitKey, - envConcurrencyLimitKey, - disabledConcurrencyLimitKey, - String(this.options.defaultEnvConcurrency) - ) - : await this.redis.calculateMessageQueueCapacities( - currentConcurrencyKey, - currentEnvConcurrencyKey, - concurrencyLimitKey, - envConcurrencyLimitKey, - String(this.options.defaultEnvConcurrency) - ); - - const queueCurrent = Number(capacities[0]); - const envLimit = Number(capacities[3]); - const isOrgEnabled = Boolean(capacities[4]); - const queueLimit = capacities[1] - ? Number(capacities[1]) - : Math.min(envLimit, isOrgEnabled ? Infinity : 0); - const envCurrent = Number(capacities[2]); - - return { - queue: { current: queueCurrent, limit: queueLimit }, - env: { current: envCurrent, limit: envLimit }, - }; - } - - #callUpdateGlobalConcurrencyLimits({ - envConcurrencyLimitKey, - envConcurrencyLimit, - }: { - envConcurrencyLimitKey: string; - envConcurrencyLimit: number; - }) { - return this.redis.updateGlobalConcurrencyLimits( - envConcurrencyLimitKey, - String(envConcurrencyLimit) - ); - } - - async #callRebalanceParentQueueChild({ - parentQueue, - childQueue, - currentScore, - }: { - parentQueue: string; - childQueue: string; - currentScore: string; - }) { - const rebalanceResult = await this.redis.rebalanceParentQueueChild( - childQueue, - parentQueue, - childQueue, - currentScore - ); - - if (rebalanceResult) { - this.logger.debug("Rebalanced parent queue child", { - parentQueue, - childQueue, - currentScore, - rebalanceResult, - operation: "rebalanceParentQueueChild", - service: this.name, - }); - } - - return rebalanceResult; - } - - #registerCommands() { - this.redis.defineCommand("enqueueMessage", { - numberOfKeys: 7, - lua: ` -local queue = KEYS[1] -local parentQueue = KEYS[2] -local messageKey = KEYS[3] -local concurrencyKey = KEYS[4] -local envConcurrencyKey = KEYS[5] -local taskConcurrencyKey = KEYS[6] -local projectConcurrencyKey = KEYS[7] - -local queueName = ARGV[1] -local messageId = ARGV[2] -local messageData = ARGV[3] -local messageScore = ARGV[4] - --- Write the message to the message key -redis.call('SET', messageKey, messageData) - --- Add the message to the queue -redis.call('ZADD', queue, messageScore, messageId) - --- Rebalance the parent queue -local earliestMessage = redis.call('ZRANGE', queue, 0, 0, 'WITHSCORES') -if #earliestMessage == 0 then - redis.call('ZREM', parentQueue, queueName) -else - redis.call('ZADD', parentQueue, earliestMessage[2], queueName) -end - --- Update the concurrency keys -redis.call('SREM', concurrencyKey, messageId) -redis.call('SREM', envConcurrencyKey, messageId) -redis.call('SREM', taskConcurrencyKey, messageId) -redis.call('SREM', projectConcurrencyKey, messageId) - `, - }); - - this.redis.defineCommand("dequeueMessage", { - numberOfKeys: 9, - lua: ` -local childQueue = KEYS[1] -local parentQueue = KEYS[2] -local concurrencyLimitKey = KEYS[3] -local envConcurrencyLimitKey = KEYS[4] -local currentConcurrencyKey = KEYS[5] -local envCurrentConcurrencyKey = KEYS[6] -local projectConcurrencyKey = KEYS[7] -local messageKeyPrefix = KEYS[8] -local taskCurrentConcurrentKeyPrefix = KEYS[9] - -local childQueueName = ARGV[1] -local currentTime = tonumber(ARGV[2]) -local defaultEnvConcurrencyLimit = ARGV[3] - --- Check current env concurrency against the limit -local envCurrentConcurrency = tonumber(redis.call('SCARD', envCurrentConcurrencyKey) or '0') -local envConcurrencyLimit = tonumber(redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit) - -if envCurrentConcurrency >= envConcurrencyLimit then - return nil -end - --- Check current queue concurrency against the limit -local currentConcurrency = tonumber(redis.call('SCARD', currentConcurrencyKey) or '0') -local concurrencyLimit = tonumber(redis.call('GET', concurrencyLimitKey) or '1000000') - --- Check condition only if concurrencyLimit exists -if currentConcurrency >= concurrencyLimit then - return nil -end - --- Attempt to dequeue the next message -local messages = redis.call('ZRANGEBYSCORE', childQueue, '-inf', currentTime, 'WITHSCORES', 'LIMIT', 0, 1) - -if #messages == 0 then - return nil -end - -local messageId = messages[1] -local messageScore = tonumber(messages[2]) - --- Get the message payload -local messageKey = messageKeyPrefix .. messageId -local messagePayload = redis.call('GET', messageKey) - --- Parse JSON payload and extract taskIdentifier -local taskIdentifier = cjson.decode(messagePayload).taskIdentifier - --- Perform SADD with taskIdentifier and messageId -local taskConcurrencyKey = taskCurrentConcurrentKeyPrefix .. taskIdentifier - --- Update concurrency -redis.call('ZREM', childQueue, messageId) -redis.call('SADD', currentConcurrencyKey, messageId) -redis.call('SADD', envCurrentConcurrencyKey, messageId) -redis.call('SADD', projectConcurrencyKey, messageId) -redis.call('SADD', taskConcurrencyKey, messageId) - --- Rebalance the parent queue -local earliestMessage = redis.call('ZRANGE', childQueue, 0, 0, 'WITHSCORES') -if #earliestMessage == 0 then - redis.call('ZREM', parentQueue, childQueueName) -else - redis.call('ZADD', parentQueue, earliestMessage[2], childQueueName) -end - -return {messageId, messageScore, messagePayload} -- Return message details - `, - }); - - this.redis.defineCommand("acknowledgeMessage", { - numberOfKeys: 7, - lua: ` --- Keys: -local parentQueue = KEYS[1] -local messageKey = KEYS[2] -local messageQueue = KEYS[3] -local concurrencyKey = KEYS[4] -local envCurrentConcurrencyKey = KEYS[5] -local projectCurrentConcurrencyKey = KEYS[6] -local taskCurrentConcurrencyKey = KEYS[7] - --- Args: -local messageId = ARGV[1] - --- Remove the message from the message key -redis.call('DEL', messageKey) - --- Remove the message from the queue -redis.call('ZREM', messageQueue, messageId) - --- Rebalance the parent queue -local earliestMessage = redis.call('ZRANGE', messageQueue, 0, 0, 'WITHSCORES') -if #earliestMessage == 0 then - redis.call('ZREM', parentQueue, messageQueue) -else - redis.call('ZADD', parentQueue, earliestMessage[2], messageQueue) -end - --- Update the concurrency keys -redis.call('SREM', concurrencyKey, messageId) -redis.call('SREM', envCurrentConcurrencyKey, messageId) -redis.call('SREM', projectCurrentConcurrencyKey, messageId) -redis.call('SREM', taskCurrentConcurrencyKey, messageId) -`, - }); - - this.redis.defineCommand("nackMessage", { - numberOfKeys: 7, - lua: ` --- Keys: -local messageKey = KEYS[1] -local messageQueueKey = KEYS[2] -local parentQueueKey = KEYS[3] -local concurrencyKey = KEYS[4] -local envConcurrencyKey = KEYS[5] -local projectConcurrencyKey = KEYS[6] -local taskConcurrencyKey = KEYS[7] - --- Args: -local messageId = ARGV[1] -local messageScore = tonumber(ARGV[2]) - --- Update the concurrency keys -redis.call('SREM', concurrencyKey, messageId) -redis.call('SREM', envConcurrencyKey, messageId) -redis.call('SREM', projectConcurrencyKey, messageId) -redis.call('SREM', taskConcurrencyKey, messageId) - --- Enqueue the message into the queue -redis.call('ZADD', messageQueueKey, messageScore, messageId) - --- Rebalance the parent queue -local earliestMessage = redis.call('ZRANGE', messageQueueKey, 0, 0, 'WITHSCORES') -if #earliestMessage == 0 then - redis.call('ZREM', parentQueueKey, messageQueueKey) -else - redis.call('ZADD', parentQueueKey, earliestMessage[2], messageQueueKey) -end -`, - }); - - this.redis.defineCommand("releaseConcurrency", { - numberOfKeys: 3, - lua: ` -local concurrencyKey = KEYS[1] -local envCurrentConcurrencyKey = KEYS[2] -local orgCurrentConcurrencyKey = KEYS[3] - -local messageId = ARGV[1] - --- Update the concurrency keys -if concurrencyKey ~= "" then - redis.call('SREM', concurrencyKey, messageId) -end -redis.call('SREM', envCurrentConcurrencyKey, messageId) -redis.call('SREM', orgCurrentConcurrencyKey, messageId) -`, - }); - - this.redis.defineCommand("calculateMessageQueueCapacitiesWithDisabling", { - numberOfKeys: 5, - lua: ` --- Keys -local currentConcurrencyKey = KEYS[1] -local currentEnvConcurrencyKey = KEYS[2] -local concurrencyLimitKey = KEYS[3] -local envConcurrencyLimitKey = KEYS[4] -local disabledConcurrencyLimitKey = KEYS[5] - --- Args -local defaultEnvConcurrencyLimit = tonumber(ARGV[1]) - --- Check if disabledConcurrencyLimitKey exists -local orgIsEnabled -if redis.call('EXISTS', disabledConcurrencyLimitKey) == 1 then - orgIsEnabled = false -else - orgIsEnabled = true -end - -local currentEnvConcurrency = tonumber(redis.call('SCARD', currentEnvConcurrencyKey) or '0') -local envConcurrencyLimit = tonumber(redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit) - -local currentConcurrency = tonumber(redis.call('SCARD', currentConcurrencyKey) or '0') -local concurrencyLimit = redis.call('GET', concurrencyLimitKey) - --- Return current capacity and concurrency limits for the queue, env, org -return { currentConcurrency, concurrencyLimit, currentEnvConcurrency, envConcurrencyLimit, orgIsEnabled } - `, - }); - - this.redis.defineCommand("calculateMessageQueueCapacities", { - numberOfKeys: 4, - lua: ` --- Keys: -local currentConcurrencyKey = KEYS[1] -local currentEnvConcurrencyKey = KEYS[2] -local concurrencyLimitKey = KEYS[3] -local envConcurrencyLimitKey = KEYS[4] - --- Args -local defaultEnvConcurrencyLimit = tonumber(ARGV[1]) - -local currentEnvConcurrency = tonumber(redis.call('SCARD', currentEnvConcurrencyKey) or '0') -local envConcurrencyLimit = tonumber(redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit) - -local currentConcurrency = tonumber(redis.call('SCARD', currentConcurrencyKey) or '0') -local concurrencyLimit = redis.call('GET', concurrencyLimitKey) - --- Return current capacity and concurrency limits for the queue, env, org -return { currentConcurrency, concurrencyLimit, currentEnvConcurrency, envConcurrencyLimit, true } - `, - }); - - this.redis.defineCommand("updateGlobalConcurrencyLimits", { - numberOfKeys: 1, - lua: ` --- Keys: envConcurrencyLimitKey, orgConcurrencyLimitKey -local envConcurrencyLimitKey = KEYS[1] - --- Args: envConcurrencyLimit, orgConcurrencyLimit -local envConcurrencyLimit = ARGV[1] - -redis.call('SET', envConcurrencyLimitKey, envConcurrencyLimit) - `, - }); - - this.redis.defineCommand("rebalanceParentQueueChild", { - numberOfKeys: 2, - lua: ` --- Keys: childQueueKey, parentQueueKey -local childQueueKey = KEYS[1] -local parentQueueKey = KEYS[2] - --- Args: childQueueName, currentScore -local childQueueName = ARGV[1] -local currentScore = ARGV[2] - --- Rebalance the parent queue -local earliestMessage = redis.call('ZRANGE', childQueueKey, 0, 0, 'WITHSCORES') -if #earliestMessage == 0 then - redis.call('ZREM', parentQueueKey, childQueueName) - - -- Return true because the parent queue was rebalanced - return true -else - -- If the earliest message is different, update the parent queue and return true, else return false - if earliestMessage[2] == currentScore then - return false - end - - redis.call('ZADD', parentQueueKey, earliestMessage[2], childQueueName) - - return earliestMessage[2] -end -`, - }); - } -} - -declare module "ioredis" { - interface RedisCommander { - enqueueMessage( - //keys - queue: string, - parentQueue: string, - messageKey: string, - concurrencyKey: string, - envConcurrencyKey: string, - taskConcurrencyKey: string, - projectConcurrencyKey: string, - //args - queueName: string, - messageId: string, - messageData: string, - messageScore: string, - callback?: Callback - ): Result; - - dequeueMessage( - //keys - childQueue: string, - parentQueue: string, - concurrencyLimitKey: string, - envConcurrencyLimitKey: string, - currentConcurrencyKey: string, - envConcurrencyKey: string, - projectConcurrencyKey: string, - messageKeyPrefix: string, - taskCurrentConcurrentKeyPrefix: string, - //args - childQueueName: string, - currentTime: string, - defaultEnvConcurrencyLimit: string, - callback?: Callback<[string, string]> - ): Result<[string, string, string] | null, Context>; - - acknowledgeMessage( - parentQueue: string, - messageKey: string, - messageQueue: string, - concurrencyKey: string, - envConcurrencyKey: string, - projectConcurrencyKey: string, - taskConcurrencyKey: string, - messageId: string, - callback?: Callback - ): Result; - - nackMessage( - messageKey: string, - messageQueue: string, - parentQueueKey: string, - concurrencyKey: string, - envConcurrencyKey: string, - projectConcurrencyKey: string, - taskConcurrencyKey: string, - messageId: string, - messageScore: string, - callback?: Callback - ): Result; - - releaseConcurrency( - concurrencyKey: string, - envConcurrencyKey: string, - orgConcurrencyKey: string, - messageId: string, - callback?: Callback - ): Result; - - calculateMessageQueueCapacities( - currentConcurrencyKey: string, - currentEnvConcurrencyKey: string, - concurrencyLimitKey: string, - envConcurrencyLimitKey: string, - defaultEnvConcurrencyLimit: string, - callback?: Callback - ): Result<[number, number, number, number, boolean], Context>; - - calculateMessageQueueCapacitiesWithDisabling( - currentConcurrencyKey: string, - currentEnvConcurrencyKey: string, - concurrencyLimitKey: string, - envConcurrencyLimitKey: string, - disabledConcurrencyLimitKey: string, - defaultEnvConcurrencyLimit: string, - callback?: Callback - ): Result<[number, number, number, number, boolean], Context>; - - updateGlobalConcurrencyLimits( - envConcurrencyLimitKey: string, - envConcurrencyLimit: string, - callback?: Callback - ): Result; - - rebalanceParentQueueChild( - childQueueKey: string, - parentQueueKey: string, - childQueueName: string, - currentScore: string, - callback?: Callback - ): Result; - } -} - -// Only allow alphanumeric characters, underscores, hyphens, and slashes (and only the first 128 characters) -export function sanitizeQueueName(queueName: string) { - return queueName.replace(/[^a-zA-Z0-9_\-\/]/g, "").substring(0, 128); -} diff --git a/internal-packages/run-engine/src/run-queue/keyProducer.test.ts b/internal-packages/run-engine/src/run-queue/keyProducer.test.ts deleted file mode 100644 index 886d695f59..0000000000 --- a/internal-packages/run-engine/src/run-queue/keyProducer.test.ts +++ /dev/null @@ -1,361 +0,0 @@ -import { describe } from "node:test"; -import { expect, it } from "vitest"; -import { RunQueueShortKeyProducer } from "./keyProducer.js"; - -describe("KeyProducer", () => { - it("sharedQueueScanPattern", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const pattern = keyProducer.masterQueueScanPattern("main"); - expect(pattern).toBe("test:*main"); - }); - - it("queueCurrentConcurrencyScanPattern", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const pattern = keyProducer.queueCurrentConcurrencyScanPattern(); - expect(pattern).toBe("test:{org:*}:proj:*:env:*:queue:*:currentConcurrency"); - }); - - it("stripKeyPrefix", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const key = keyProducer.stripKeyPrefix("test:abc"); - expect(key).toBe("abc"); - }); - - it("queueConcurrencyLimitKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const key = keyProducer.queueConcurrencyLimitKey( - { - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }, - "task/task-name" - ); - expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:concurrency"); - }); - - it("envConcurrencyLimitKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const key = keyProducer.envConcurrencyLimitKey({ - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }); - expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:concurrency"); - }); - - it("queueKey (no concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const key = keyProducer.queueKey( - { - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }, - "task/task-name" - ); - expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name"); - }); - - it("queueKey (w concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const key = keyProducer.queueKey( - { - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }, - "task/task-name", - "c1234" - ); - expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:ck:c1234"); - }); - - it("concurrencyLimitKeyFromQueue (w concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const queueKey = keyProducer.queueKey( - { - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }, - "task/task-name", - "c1234" - ); - const key = keyProducer.concurrencyLimitKeyFromQueue(queueKey); - expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:concurrency"); - }); - - it("concurrencyLimitKeyFromQueue (no concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const queueKey = keyProducer.queueKey( - { - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }, - "task/task-name" - ); - const key = keyProducer.concurrencyLimitKeyFromQueue(queueKey); - expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:concurrency"); - }); - - it("currentConcurrencyKeyFromQueue (w concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const queueKey = keyProducer.queueKey( - { - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }, - "task/task-name", - "c1234" - ); - const key = keyProducer.currentConcurrencyKeyFromQueue(queueKey); - expect(key).toBe( - "{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:ck:c1234:currentConcurrency" - ); - }); - - it("currentConcurrencyKeyFromQueue (no concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const queueKey = keyProducer.queueKey( - { - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }, - "task/task-name" - ); - const key = keyProducer.currentConcurrencyKeyFromQueue(queueKey); - expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:currentConcurrency"); - }); - - it("currentConcurrencyKey (w concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const key = keyProducer.currentConcurrencyKey( - { - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }, - "task/task-name", - "c1234" - ); - expect(key).toBe( - "{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:ck:c1234:currentConcurrency" - ); - }); - - it("currentConcurrencyKey (no concurrency)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const key = keyProducer.currentConcurrencyKey( - { - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }, - "task/task-name" - ); - - expect(key).toBe("{org:o1234}:proj:p1234:env:e1234:queue:task/task-name:currentConcurrency"); - }); - - it("taskIdentifierCurrentConcurrencyKeyPrefixFromQueue", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const queueKey = keyProducer.queueKey( - { - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }, - "task/task-name" - ); - const key = keyProducer.taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(queueKey); - expect(key).toBe("{org:o1234}:proj:p1234:task:"); - }); - - it("taskIdentifierCurrentConcurrencyKeyFromQueue", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const queueKey = keyProducer.queueKey( - { - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }, - "task/task-name" - ); - const key = keyProducer.taskIdentifierCurrentConcurrencyKeyFromQueue(queueKey, "task-name"); - expect(key).toBe("{org:o1234}:proj:p1234:task:task-name"); - }); - - it("taskIdentifierCurrentConcurrencyKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const key = keyProducer.taskIdentifierCurrentConcurrencyKey( - { - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }, - "task-name" - ); - expect(key).toBe("{org:o1234}:proj:p1234:task:task-name"); - }); - - it("projectCurrentConcurrencyKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const key = keyProducer.projectCurrentConcurrencyKey({ - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }); - expect(key).toBe("{org:o1234}:proj:p1234:currentConcurrency"); - }); - - it("projectCurrentConcurrencyKeyFromQueue", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const key = keyProducer.projectCurrentConcurrencyKeyFromQueue( - "{org:o1234}:proj:p1234:currentConcurrency" - ); - expect(key).toBe("{org:o1234}:proj:p1234:currentConcurrency"); - }); - - it("disabledConcurrencyLimitKeyFromQueue", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const queueKey = keyProducer.queueKey( - { - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }, - "task/task-name" - ); - const key = keyProducer.disabledConcurrencyLimitKeyFromQueue(queueKey); - expect(key).toBe("{org:o1234}:disabledConcurrency"); - }); - - it("envConcurrencyLimitKeyFromQueue", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const queueKey = keyProducer.queueKey( - { - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }, - "task/task-name" - ); - const key = keyProducer.envConcurrencyLimitKeyFromQueue(queueKey); - expect(key).toBe("{org:o1234}:env:e1234:concurrency"); - }); - - it("envCurrentConcurrencyKeyFromQueue", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const queueKey = keyProducer.queueKey( - { - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }, - "task/task-name" - ); - const key = keyProducer.envCurrentConcurrencyKeyFromQueue(queueKey); - expect(key).toBe("{org:o1234}:env:e1234:currentConcurrency"); - }); - - it("envCurrentConcurrencyKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const key = keyProducer.envCurrentConcurrencyKey({ - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }); - expect(key).toBe("{org:o1234}:env:e1234:currentConcurrency"); - }); - - it("messageKey", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const key = keyProducer.messageKey("o1234", "m1234"); - expect(key).toBe("{org:o1234}:message:m1234"); - }); - - it("extractComponentsFromQueue (no concurrencyKey)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const queueKey = keyProducer.queueKey( - { - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }, - "task/task-name" - ); - const components = keyProducer.extractComponentsFromQueue(queueKey); - expect(components).toEqual({ - orgId: "o1234", - projectId: "p1234", - envId: "e1234", - queue: "task/task-name", - concurrencyKey: undefined, - }); - }); - - it("extractComponentsFromQueue (w concurrencyKey)", () => { - const keyProducer = new RunQueueShortKeyProducer("test:"); - const queueKey = keyProducer.queueKey( - { - id: "e1234", - type: "PRODUCTION", - maximumConcurrencyLimit: 10, - project: { id: "p1234" }, - organization: { id: "o1234" }, - }, - "task/task-name", - "c1234" - ); - const components = keyProducer.extractComponentsFromQueue(queueKey); - expect(components).toEqual({ - orgId: "o1234", - projectId: "p1234", - envId: "e1234", - queue: "task/task-name", - concurrencyKey: "c1234", - }); - }); -}); diff --git a/internal-packages/run-engine/src/run-queue/keyProducer.ts b/internal-packages/run-engine/src/run-queue/keyProducer.ts deleted file mode 100644 index 8c145ce16e..0000000000 --- a/internal-packages/run-engine/src/run-queue/keyProducer.ts +++ /dev/null @@ -1,195 +0,0 @@ -import { MinimalAuthenticatedEnvironment } from "../shared/index.js"; -import { RunQueueKeyProducer } from "./types.js"; - -const constants = { - CURRENT_CONCURRENCY_PART: "currentConcurrency", - CONCURRENCY_LIMIT_PART: "concurrency", - DISABLED_CONCURRENCY_LIMIT_PART: "disabledConcurrency", - ENV_PART: "env", - ORG_PART: "org", - PROJECT_PART: "proj", - QUEUE_PART: "queue", - CONCURRENCY_KEY_PART: "ck", - TASK_PART: "task", - MESSAGE_PART: "message", -} as const; - -export class RunQueueShortKeyProducer implements RunQueueKeyProducer { - constructor(private _prefix: string) {} - - masterQueueScanPattern(masterQueue: string) { - return `${this._prefix}*${masterQueue}`; - } - - queueCurrentConcurrencyScanPattern() { - return `${this._prefix}{${constants.ORG_PART}:*}:${constants.PROJECT_PART}:*:${constants.ENV_PART}:*:${constants.QUEUE_PART}:*:${constants.CURRENT_CONCURRENCY_PART}`; - } - - stripKeyPrefix(key: string): string { - if (key.startsWith(this._prefix)) { - return key.slice(this._prefix.length); - } - - return key; - } - - queueConcurrencyLimitKey(env: MinimalAuthenticatedEnvironment, queue: string) { - return [this.queueKey(env, queue), constants.CONCURRENCY_LIMIT_PART].join(":"); - } - - envConcurrencyLimitKey(env: MinimalAuthenticatedEnvironment) { - return [ - this.orgKeySection(env.organization.id), - this.projKeySection(env.project.id), - this.envKeySection(env.id), - constants.CONCURRENCY_LIMIT_PART, - ].join(":"); - } - - queueKey(env: MinimalAuthenticatedEnvironment, queue: string, concurrencyKey?: string) { - return [ - this.orgKeySection(env.organization.id), - this.projKeySection(env.project.id), - this.envKeySection(env.id), - this.queueSection(queue), - ] - .concat(concurrencyKey ? this.concurrencyKeySection(concurrencyKey) : []) - .join(":"); - } - - concurrencyLimitKeyFromQueue(queue: string) { - const concurrencyQueueName = queue.replace(/:ck:.+$/, ""); - return `${concurrencyQueueName}:${constants.CONCURRENCY_LIMIT_PART}`; - } - - currentConcurrencyKeyFromQueue(queue: string) { - return `${queue}:${constants.CURRENT_CONCURRENCY_PART}`; - } - - currentConcurrencyKey( - env: MinimalAuthenticatedEnvironment, - queue: string, - concurrencyKey?: string - ): string { - return [this.queueKey(env, queue, concurrencyKey), constants.CURRENT_CONCURRENCY_PART].join( - ":" - ); - } - - disabledConcurrencyLimitKeyFromQueue(queue: string) { - const { orgId } = this.extractComponentsFromQueue(queue); - return `{${constants.ORG_PART}:${orgId}}:${constants.DISABLED_CONCURRENCY_LIMIT_PART}`; - } - - envConcurrencyLimitKeyFromQueue(queue: string) { - const { orgId, envId } = this.extractComponentsFromQueue(queue); - return `{${constants.ORG_PART}:${orgId}}:${constants.ENV_PART}:${envId}:${constants.CONCURRENCY_LIMIT_PART}`; - } - - envCurrentConcurrencyKeyFromQueue(queue: string) { - const { orgId, envId } = this.extractComponentsFromQueue(queue); - return `{${constants.ORG_PART}:${orgId}}:${constants.ENV_PART}:${envId}:${constants.CURRENT_CONCURRENCY_PART}`; - } - - envCurrentConcurrencyKey(env: MinimalAuthenticatedEnvironment): string { - return [ - this.orgKeySection(env.organization.id), - this.envKeySection(env.id), - constants.CURRENT_CONCURRENCY_PART, - ].join(":"); - } - - taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(queue: string) { - const { orgId, projectId } = this.extractComponentsFromQueue(queue); - - return `${[this.orgKeySection(orgId), this.projKeySection(projectId), constants.TASK_PART] - .filter(Boolean) - .join(":")}:`; - } - - taskIdentifierCurrentConcurrencyKeyFromQueue(queue: string, taskIdentifier: string) { - return `${this.taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(queue)}${taskIdentifier}`; - } - - taskIdentifierCurrentConcurrencyKey( - env: MinimalAuthenticatedEnvironment, - taskIdentifier: string - ): string { - return [ - this.orgKeySection(env.organization.id), - this.projKeySection(env.project.id), - constants.TASK_PART, - taskIdentifier, - ].join(":"); - } - - projectCurrentConcurrencyKey(env: MinimalAuthenticatedEnvironment): string { - return [ - this.orgKeySection(env.organization.id), - this.projKeySection(env.project.id), - constants.CURRENT_CONCURRENCY_PART, - ].join(":"); - } - - projectCurrentConcurrencyKeyFromQueue(queue: string): string { - const { orgId, projectId } = this.extractComponentsFromQueue(queue); - return `${this.orgKeySection(orgId)}:${this.projKeySection(projectId)}:${ - constants.CURRENT_CONCURRENCY_PART - }`; - } - - messageKeyPrefixFromQueue(queue: string) { - const { orgId } = this.extractComponentsFromQueue(queue); - return `${this.orgKeySection(orgId)}:${constants.MESSAGE_PART}:`; - } - - messageKey(orgId: string, messageId: string) { - return [this.orgKeySection(orgId), `${constants.MESSAGE_PART}:${messageId}`] - .filter(Boolean) - .join(":"); - } - - extractComponentsFromQueue(queue: string) { - const parts = this.normalizeQueue(queue).split(":"); - return { - orgId: parts[1].replace("{", "").replace("}", ""), - projectId: parts[3], - envId: parts[5], - queue: parts[7], - concurrencyKey: parts.at(9), - }; - } - - private envKeySection(envId: string) { - return `${constants.ENV_PART}:${envId}`; - } - - private projKeySection(projId: string) { - return `${constants.PROJECT_PART}:${projId}`; - } - - private orgKeySection(orgId: string) { - return `{${constants.ORG_PART}:${orgId}}`; - } - - private queueSection(queue: string) { - return `${constants.QUEUE_PART}:${queue}`; - } - - private concurrencyKeySection(concurrencyKey: string) { - return `${constants.CONCURRENCY_KEY_PART}:${concurrencyKey}`; - } - - private taskIdentifierSection(taskIdentifier: string) { - return `${constants.TASK_PART}:${taskIdentifier}`; - } - - // This removes the leading prefix from the queue name if it exists - private normalizeQueue(queue: string) { - if (queue.startsWith(this._prefix)) { - return queue.slice(this._prefix.length); - } - - return queue; - } -} diff --git a/internal-packages/run-engine/src/run-queue/simpleWeightedPriorityStrategy.ts b/internal-packages/run-engine/src/run-queue/simpleWeightedPriorityStrategy.ts deleted file mode 100644 index b85019e449..0000000000 --- a/internal-packages/run-engine/src/run-queue/simpleWeightedPriorityStrategy.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { - RunQueuePriorityStrategy, - PriorityStrategyChoice, - QueueRange, - QueueWithScores, -} from "./types.js"; - -export type SimpleWeightedChoiceStrategyOptions = { - queueSelectionCount: number; - randomSeed?: string; - excludeEnvCapacity?: boolean; -}; - -export class SimpleWeightedChoiceStrategy implements RunQueuePriorityStrategy { - private _nextRangesByParentQueue: Map = new Map(); - - constructor(private options: SimpleWeightedChoiceStrategyOptions) {} - - private nextRangeForParentQueue(parentQueue: string, consumerId: string): QueueRange { - return ( - this._nextRangesByParentQueue.get(`${consumerId}:${parentQueue}`) ?? { - offset: 0, - count: this.options.queueSelectionCount, - } - ); - } - - chooseQueue( - queues: QueueWithScores[], - parentQueue: string, - consumerId: string, - previousRange: QueueRange - ): { choice: PriorityStrategyChoice; nextRange: QueueRange } { - const filteredQueues = filterQueuesAtCapacity(queues); - - if (queues.length === this.options.queueSelectionCount) { - const nextRange: QueueRange = { - offset: previousRange.offset + this.options.queueSelectionCount, - count: this.options.queueSelectionCount, - }; - - // If all queues are at capacity, and we were passed the max number of queues, then we will slide the window "to the right" - this._nextRangesByParentQueue.set(`${consumerId}:${parentQueue}`, nextRange); - } else { - this._nextRangesByParentQueue.delete(`${consumerId}:${parentQueue}`); - } - - if (filteredQueues.length === 0) { - return { - choice: { abort: true }, - nextRange: this.nextRangeForParentQueue(parentQueue, consumerId), - }; - } - - const queueWeights = this.#calculateQueueWeights(filteredQueues); - - const choice = weightedRandomChoice(queueWeights); - - return { - choice, - nextRange: this.nextRangeForParentQueue(parentQueue, consumerId), - }; - } - - async nextCandidateSelection( - parentQueue: string, - consumerId: string - ): Promise<{ range: QueueRange }> { - return { - range: this.nextRangeForParentQueue(parentQueue, consumerId), - }; - } - - #calculateQueueWeights(queues: QueueWithScores[]) { - const avgQueueSize = queues.reduce((acc, { size }) => acc + size, 0) / queues.length; - const avgMessageAge = queues.reduce((acc, { age }) => acc + age, 0) / queues.length; - - return queues.map(({ capacities, age, queue, size }) => { - let totalWeight = 1; - - if (size > avgQueueSize) { - totalWeight += Math.min(size / avgQueueSize, 4); - } - - if (age > avgMessageAge) { - totalWeight += Math.min(age / avgMessageAge, 4); - } - - return { - queue, - totalWeight: age, - }; - }); - } -} - -function filterQueuesAtCapacity(queues: QueueWithScores[]) { - return queues.filter( - (queue) => - queue.capacities.queue.current < queue.capacities.queue.limit && - queue.capacities.env.current < queue.capacities.env.limit - ); -} - -function weightedRandomChoice(queues: Array<{ queue: string; totalWeight: number }>) { - const totalWeight = queues.reduce((acc, queue) => acc + queue.totalWeight, 0); - let randomNum = Math.random() * totalWeight; - - for (const queue of queues) { - if (randomNum < queue.totalWeight) { - return queue.queue; - } - - randomNum -= queue.totalWeight; - } - - // If we get here, we should just return a random queue - return queues[Math.floor(Math.random() * queues.length)].queue; -} diff --git a/internal-packages/run-engine/src/run-queue/types.ts b/internal-packages/run-engine/src/run-queue/types.ts deleted file mode 100644 index 914193fb85..0000000000 --- a/internal-packages/run-engine/src/run-queue/types.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { z } from "zod"; -import { RuntimeEnvironmentType } from "../../../database/src/index.js"; -import { MinimalAuthenticatedEnvironment } from "../shared/index.js"; - -export const InputPayload = z.object({ - runId: z.string(), - taskIdentifier: z.string(), - orgId: z.string(), - projectId: z.string(), - environmentId: z.string(), - environmentType: z.nativeEnum(RuntimeEnvironmentType), - queue: z.string(), - concurrencyKey: z.string().optional(), - timestamp: z.number(), -}); -export type InputPayload = z.infer; - -export const OutputPayload = InputPayload.extend({ - version: z.literal("1"), - masterQueue: z.string(), -}); -export type OutputPayload = z.infer; - -export type QueueCapacity = { - current: number; - limit: number; -}; - -export type QueueCapacities = { - queue: QueueCapacity; - env: QueueCapacity; -}; - -export type QueueWithScores = { - queue: string; - capacities: QueueCapacities; - age: number; - size: number; -}; - -export type QueueRange = { offset: number; count: number }; - -export interface RunQueueKeyProducer { - masterQueueScanPattern(masterQueue: string): string; - queueCurrentConcurrencyScanPattern(): string; - //queue - queueKey(env: MinimalAuthenticatedEnvironment, queue: string, concurrencyKey?: string): string; - queueConcurrencyLimitKey(env: MinimalAuthenticatedEnvironment, queue: string): string; - concurrencyLimitKeyFromQueue(queue: string): string; - currentConcurrencyKeyFromQueue(queue: string): string; - currentConcurrencyKey( - env: MinimalAuthenticatedEnvironment, - queue: string, - concurrencyKey?: string - ): string; - disabledConcurrencyLimitKeyFromQueue(queue: string): string; - //env oncurrency - envCurrentConcurrencyKey(env: MinimalAuthenticatedEnvironment): string; - envConcurrencyLimitKey(env: MinimalAuthenticatedEnvironment): string; - envConcurrencyLimitKeyFromQueue(queue: string): string; - envCurrentConcurrencyKeyFromQueue(queue: string): string; - //task concurrency - taskIdentifierCurrentConcurrencyKey( - env: MinimalAuthenticatedEnvironment, - taskIdentifier: string - ): string; - taskIdentifierCurrentConcurrencyKeyPrefixFromQueue(queue: string): string; - taskIdentifierCurrentConcurrencyKeyFromQueue(queue: string, taskIdentifier: string): string; - //project concurrency - projectCurrentConcurrencyKey(env: MinimalAuthenticatedEnvironment): string; - projectCurrentConcurrencyKeyFromQueue(queue: string): string; - //message payload - messageKeyPrefixFromQueue(queue: string): string; - messageKey(orgId: string, messageId: string): string; - //utils - stripKeyPrefix(key: string): string; - extractComponentsFromQueue(queue: string): { - orgId: string; - projectId: string; - envId: string; - queue: string; - concurrencyKey: string | undefined; - }; -} - -export type PriorityStrategyChoice = string | { abort: true }; - -export interface RunQueuePriorityStrategy { - /** - * chooseQueue is called to select the next queue to process a message from - * - * @param queues - * @param parentQueue - * @param consumerId - * - * @returns The queue to process the message from, or an object with `abort: true` if no queue is available - */ - chooseQueue( - queues: Array, - parentQueue: string, - consumerId: string, - previousRange: QueueRange - ): { choice: PriorityStrategyChoice; nextRange: QueueRange }; - - /** - * This function is called to get the next candidate selection for the queue - * The `range` is used to select the set of queues that will be considered for the next selection (passed to chooseQueue) - * The `selectionId` is used to identify the selection and should be passed to chooseQueue - * - * @param parentQueue The parent queue that holds the candidate queues - * @param consumerId The consumerId that is making the request - * - * @returns The scores and the selectionId for the next candidate selection - */ - nextCandidateSelection(parentQueue: string, consumerId: string): Promise<{ range: QueueRange }>; -} diff --git a/internal-packages/run-engine/src/shared/asyncWorker.ts b/internal-packages/run-engine/src/shared/asyncWorker.ts deleted file mode 100644 index 016662e1d5..0000000000 --- a/internal-packages/run-engine/src/shared/asyncWorker.ts +++ /dev/null @@ -1,34 +0,0 @@ -export class AsyncWorker { - private running = false; - private timeout?: NodeJS.Timeout; - - constructor(private readonly fn: () => Promise, private readonly interval: number) {} - - start() { - if (this.running) { - return; - } - - this.running = true; - - this.#run(); - } - - stop() { - this.running = false; - } - - async #run() { - if (!this.running) { - return; - } - - try { - await this.fn(); - } catch (e) { - console.error(e); - } - - this.timeout = setTimeout(this.#run.bind(this), this.interval); - } -} diff --git a/internal-packages/run-engine/src/shared/index.ts b/internal-packages/run-engine/src/shared/index.ts deleted file mode 100644 index 6bd3e304e3..0000000000 --- a/internal-packages/run-engine/src/shared/index.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { Attributes } from "@opentelemetry/api"; -import { Prisma } from "../../../database/src"; - -export type AuthenticatedEnvironment = Prisma.RuntimeEnvironmentGetPayload<{ - include: { project: true; organization: true; orgMember: true }; -}>; - -export type MinimalAuthenticatedEnvironment = { - id: AuthenticatedEnvironment["id"]; - type: AuthenticatedEnvironment["type"]; - maximumConcurrencyLimit: AuthenticatedEnvironment["maximumConcurrencyLimit"]; - project: { - id: AuthenticatedEnvironment["project"]["id"]; - }; - organization: { - id: AuthenticatedEnvironment["organization"]["id"]; - }; -}; - -const SemanticEnvResources = { - ENV_ID: "$trigger.env.id", - ENV_TYPE: "$trigger.env.type", - ENV_SLUG: "$trigger.env.slug", - ORG_ID: "$trigger.org.id", - ORG_SLUG: "$trigger.org.slug", - ORG_TITLE: "$trigger.org.title", - PROJECT_ID: "$trigger.project.id", - PROJECT_NAME: "$trigger.project.name", - USER_ID: "$trigger.user.id", -}; - -export function attributesFromAuthenticatedEnv(env: MinimalAuthenticatedEnvironment): Attributes { - return { - [SemanticEnvResources.ENV_ID]: env.id, - [SemanticEnvResources.ENV_TYPE]: env.type, - [SemanticEnvResources.ORG_ID]: env.organization.id, - [SemanticEnvResources.PROJECT_ID]: env.project.id, - }; -} diff --git a/internal-packages/run-engine/tsconfig.json b/internal-packages/run-engine/tsconfig.json deleted file mode 100644 index 515b521967..0000000000 --- a/internal-packages/run-engine/tsconfig.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2019", - "lib": ["ES2019", "DOM", "DOM.Iterable"], - "module": "CommonJS", - "moduleResolution": "Node", - "moduleDetection": "force", - "verbatimModuleSyntax": false, - "types": ["vitest/globals"], - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "isolatedModules": true, - "preserveWatchOutput": true, - "skipLibCheck": true, - "noEmit": true, - "strict": true, - "paths": { - "@internal/testcontainers": ["../../internal-packages/testcontainers/src/index"], - "@internal/testcontainers/*": ["../../internal-packages/testcontainers/src/*"], - "@internal/zod-worker": ["../../internal-packages/zod-worker/src/index"], - "@internal/zod-worker/*": ["../../internal-packages/zod-worker/src/*"], - "@trigger.dev/core": ["../../packages/core/src/index"], - "@trigger.dev/core/*": ["../../packages/core/src/*"], - "@trigger.dev/database": ["../database/src/index"], - "@trigger.dev/database/*": ["../database/src/*"] - } - }, - "exclude": ["node_modules"] -} diff --git a/internal-packages/run-engine/vitest.config.ts b/internal-packages/run-engine/vitest.config.ts deleted file mode 100644 index 4afd926425..0000000000 --- a/internal-packages/run-engine/vitest.config.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { defineConfig } from "vitest/config"; - -export default defineConfig({ - test: { - include: ["**/*.test.ts"], - globals: true, - }, -}); From e1bba093851961faf170dc45ab54942182c6e6fc Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 8 Oct 2024 16:50:58 +0100 Subject: [PATCH 108/114] Remove the RunEngine prisma schema changes --- .../database/prisma/schema.prisma | 144 +----------------- 1 file changed, 1 insertion(+), 143 deletions(-) diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 37bd8b49a3..d346d4584c 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -470,8 +470,6 @@ model Project { alertStorages ProjectAlertStorage[] bulkActionGroups BulkActionGroup[] BackgroundWorkerFile BackgroundWorkerFile[] - waitpoints Waitpoint[] - taskRunWaitpoints TaskRunWaitpoint[] } enum ProjectVersion { @@ -1113,8 +1111,6 @@ model TaskAttempt { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt - executionSnapshot TaskRunExecutionSnapshot[] - @@unique([taskId, number]) } @@ -1658,8 +1654,6 @@ model TaskRun { number Int @default(0) friendlyId String @unique - engine RunEngineVersion @default(V1) - status TaskRunStatus @default(PENDING) idempotencyKey String? @@ -1681,12 +1675,8 @@ model TaskRun { project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) projectId String - // The specific queue this run is in queue String - /// The main queue that this run is part of - masterQueue String @default("main") - createdAt DateTime @default(now()) updatedAt DateTime @updatedAt @@ -1718,16 +1708,9 @@ model TaskRun { expiredAt DateTime? maxAttempts Int? - ///When this run is finished, the waitpoint will be marked as completed - associatedWaitpoint Waitpoint? - - ///If there are any blocked waitpoints, the run won't be executed - blockedByWaitpoints TaskRunWaitpoint[] - batchItems BatchTaskRunItem[] dependency TaskRunDependency? CheckpointRestoreEvent CheckpointRestoreEvent[] - executionSnapshot TaskRunExecutionSnapshot[] alerts ProjectAlert[] @@ -1857,131 +1840,6 @@ enum TaskRunStatus { TIMED_OUT } -enum RunEngineVersion { - /// The original version that uses marqs v1 and Graphile - V1 - V2 -} - -/// Used by the RunEngine during TaskRun execution -/// It has the required information to transactionally progress a run through states, -/// and prevent side effects like heartbeats failing a run that has progressed. -/// It is optimised for performance and is designed to be cleared at some point, -/// so there are no cascading relationships to other models. -model TaskRunExecutionSnapshot { - id String @id @default(cuid()) - - /// This should never be V1 - engine RunEngineVersion @default(V2) - - /// The execution status - executionStatus TaskRunExecutionStatus - /// For debugging - description String - - /// Run - runId String - run TaskRun @relation(fields: [runId], references: [id]) - runStatus TaskRunStatus - - /// Attempt - currentAttemptId String? - currentAttempt TaskAttempt? @relation(fields: [currentAttemptId], references: [id]) - currentAttemptStatus TaskAttemptStatus? - - /// todo Checkpoint - - /// These are only ever appended, so we don't need updatedAt - createdAt DateTime @default(now()) - - ///todo machine spec? - - ///todo worker - - /// Used to get the latest state quickly - @@index([runId, createdAt(sort: Desc)]) -} - -enum TaskRunExecutionStatus { - RUN_CREATED - DEQUEUED_FOR_EXECUTION - EXECUTING - BLOCKED_BY_WAITPOINTS - FINISHED -} - -/// A Waitpoint blocks a run from continuing until it's completed -/// If there's a waitpoint blocking a run, it shouldn't be in the queue -model Waitpoint { - id String @id @default(cuid()) - - type WaitpointType - status WaitpointStatus @default(PENDING) - - completedAt DateTime? - - /// If it's an Event type waitpoint, this is the event. It can also be provided for the DATETIME type - idempotencyKey String - userProvidedIdempotencyKey Boolean - - /// If an idempotencyKey is no longer active, we store it here and generate a new one for the idempotencyKey field. - /// This is a workaround because Prisma doesn't support partial indexes. - inactiveIdempotencyKey String? - - /// If it's a RUN type waitpoint, this is the associated run - completedByTaskRunId String? @unique - completedByTaskRun TaskRun? @relation(fields: [completedByTaskRunId], references: [id], onDelete: SetNull) - - /// If it's a DATETIME type waitpoint, this is the date - completedAfter DateTime? - - /// The runs this waitpoint is blocking - blockingTaskRuns TaskRunWaitpoint[] - - /// When completed, an output can be stored here - output String? - outputType String @default("application/json") - - project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) - projectId String - - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - - @@unique([projectId, idempotencyKey]) -} - -enum WaitpointType { - RUN - DATETIME - EVENT -} - -enum WaitpointStatus { - PENDING - COMPLETED -} - -model TaskRunWaitpoint { - id String @id @default(cuid()) - - taskRun TaskRun @relation(fields: [taskRunId], references: [id]) - taskRunId String - - waitpoint Waitpoint @relation(fields: [waitpointId], references: [id]) - waitpointId String - - project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) - projectId String - - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - - @@unique([taskRunId, waitpointId]) - @@index([taskRunId]) - @@index([waitpointId]) -} - model TaskRunTag { id String @id @default(cuid()) name String @@ -2011,7 +1869,7 @@ model TaskRunDependency { checkpointEvent CheckpointRestoreEvent? @relation(fields: [checkpointEventId], references: [id], onDelete: Cascade, onUpdate: Cascade) checkpointEventId String? @unique - /// An attempt that is dependent on this task run. + /// An attempt that is dependent on this task run. dependentAttempt TaskRunAttempt? @relation(fields: [dependentAttemptId], references: [id]) dependentAttemptId String? From 7d05c9700dcbe6b394e14bfc3ecacace9e00f575 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 8 Oct 2024 16:54:30 +0100 Subject: [PATCH 109/114] Delete triggerTaskV2 --- .../app/v3/services/triggerTaskV2.server.ts | 591 ------------------ 1 file changed, 591 deletions(-) delete mode 100644 apps/webapp/app/v3/services/triggerTaskV2.server.ts diff --git a/apps/webapp/app/v3/services/triggerTaskV2.server.ts b/apps/webapp/app/v3/services/triggerTaskV2.server.ts deleted file mode 100644 index 4ea9394d2c..0000000000 --- a/apps/webapp/app/v3/services/triggerTaskV2.server.ts +++ /dev/null @@ -1,591 +0,0 @@ -import { - IOPacket, - QueueOptions, - SemanticInternalAttributes, - TriggerTaskRequestBody, - packetRequiresOffloading, -} from "@trigger.dev/core/v3"; -import { env } from "~/env.server"; -import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; -import { autoIncrementCounter } from "~/services/autoIncrementCounter.server"; -import { workerQueue } from "~/services/worker.server"; -import { marqs, sanitizeQueueName } from "~/v3/marqs/index.server"; -import { eventRepository } from "../eventRepository.server"; -import { generateFriendlyId } from "../friendlyIdentifiers"; -import { uploadToObjectStore } from "../r2.server"; -import { startActiveSpan } from "../tracer.server"; -import { getEntitlement } from "~/services/platform.v3.server"; -import { BaseService, ServiceValidationError } from "./baseService.server"; -import { logger } from "~/services/logger.server"; -import { isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus"; -import { createTag, MAX_TAGS_PER_RUN } from "~/models/taskRunTag.server"; -import { findCurrentWorkerFromEnvironment } from "../models/workerDeployment.server"; -import { handleMetadataPacket } from "~/utils/packets"; -import { RunEngine } from "@internal/run-engine"; -import { prisma } from "~/db.server"; - -export type TriggerTaskServiceOptions = { - idempotencyKey?: string; - triggerVersion?: string; - traceContext?: Record; - spanParentAsLink?: boolean; - parentAsLinkType?: "replay" | "trigger"; - batchId?: string; - customIcon?: string; -}; - -export class OutOfEntitlementError extends Error { - constructor() { - super("You can't trigger a task because you have run out of credits."); - } -} - -//todo move this to a singleton somewhere -const engine = new RunEngine({ - prisma, - redis: { - port: env.REDIS_PORT, - host: env.REDIS_HOST, - username: env.REDIS_USERNAME, - password: env.REDIS_PASSWORD, - enableAutoPipelining: true, - ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }), - }, - zodWorker: { - connectionString: env.DATABASE_URL, - concurrency: env.WORKER_CONCURRENCY, - pollInterval: env.WORKER_POLL_INTERVAL, - noPreparedStatements: env.DATABASE_URL !== env.DIRECT_URL, - schema: env.WORKER_SCHEMA, - maxPoolSize: env.WORKER_CONCURRENCY + 1, - shutdownTimeoutInMs: env.GRACEFUL_SHUTDOWN_TIMEOUT, - }, -}); - -export class TriggerTaskService extends BaseService { - public async call( - taskId: string, - environment: AuthenticatedEnvironment, - body: TriggerTaskRequestBody, - options: TriggerTaskServiceOptions = {} - ) { - return await this.traceWithEnv("call()", environment, async (span) => { - span.setAttribute("taskId", taskId); - - const idempotencyKey = options.idempotencyKey ?? body.options?.idempotencyKey; - const delayUntil = await parseDelay(body.options?.delay); - - const ttl = - typeof body.options?.ttl === "number" - ? stringifyDuration(body.options?.ttl) - : body.options?.ttl ?? (environment.type === "DEVELOPMENT" ? "10m" : undefined); - - const existingRun = idempotencyKey - ? await this._prisma.taskRun.findUnique({ - where: { - runtimeEnvironmentId_taskIdentifier_idempotencyKey: { - runtimeEnvironmentId: environment.id, - idempotencyKey, - taskIdentifier: taskId, - }, - }, - }) - : undefined; - - if (existingRun) { - span.setAttribute("runId", existingRun.friendlyId); - - return existingRun; - } - - if (environment.type !== "DEVELOPMENT") { - const result = await getEntitlement(environment.organizationId); - if (result && result.hasAccess === false) { - throw new OutOfEntitlementError(); - } - } - - if ( - body.options?.tags && - typeof body.options.tags !== "string" && - body.options.tags.length > MAX_TAGS_PER_RUN - ) { - throw new ServiceValidationError( - `Runs can only have ${MAX_TAGS_PER_RUN} tags, you're trying to set ${body.options.tags.length}.` - ); - } - - const runFriendlyId = generateFriendlyId("run"); - - const payloadPacket = await this.#handlePayloadPacket( - body.payload, - body.options?.payloadType ?? "application/json", - runFriendlyId, - environment - ); - - const metadataPacket = body.options?.metadata - ? handleMetadataPacket( - body.options?.metadata, - body.options?.metadataType ?? "application/json" - ) - : undefined; - - const dependentAttempt = body.options?.dependentAttempt - ? await this._prisma.taskRunAttempt.findUnique({ - where: { friendlyId: body.options.dependentAttempt }, - include: { - taskRun: { - select: { - id: true, - status: true, - taskIdentifier: true, - rootTaskRunId: true, - depth: true, - }, - }, - }, - }) - : undefined; - - if ( - dependentAttempt && - (isFinalAttemptStatus(dependentAttempt.status) || - isFinalRunStatus(dependentAttempt.taskRun.status)) - ) { - logger.debug("Dependent attempt or run is in a terminal state", { - dependentAttempt: dependentAttempt, - }); - - if (isFinalAttemptStatus(dependentAttempt.status)) { - throw new ServiceValidationError( - `Cannot trigger ${taskId} as the parent attempt has a status of ${dependentAttempt.status}` - ); - } else { - throw new ServiceValidationError( - `Cannot trigger ${taskId} as the parent run has a status of ${dependentAttempt.taskRun.status}` - ); - } - } - - const parentAttempt = body.options?.parentAttempt - ? await this._prisma.taskRunAttempt.findUnique({ - where: { friendlyId: body.options.parentAttempt }, - include: { - taskRun: { - select: { - id: true, - status: true, - taskIdentifier: true, - rootTaskRunId: true, - depth: true, - }, - }, - }, - }) - : undefined; - - const dependentBatchRun = body.options?.dependentBatch - ? await this._prisma.batchTaskRun.findUnique({ - where: { friendlyId: body.options.dependentBatch }, - include: { - dependentTaskAttempt: { - include: { - taskRun: { - select: { - id: true, - status: true, - taskIdentifier: true, - rootTaskRunId: true, - depth: true, - }, - }, - }, - }, - }, - }) - : undefined; - - if ( - dependentBatchRun && - dependentBatchRun.dependentTaskAttempt && - (isFinalAttemptStatus(dependentBatchRun.dependentTaskAttempt.status) || - isFinalRunStatus(dependentBatchRun.dependentTaskAttempt.taskRun.status)) - ) { - logger.debug("Dependent batch run task attempt or run has been canceled", { - dependentBatchRunId: dependentBatchRun.id, - status: dependentBatchRun.status, - attempt: dependentBatchRun.dependentTaskAttempt, - }); - - if (isFinalAttemptStatus(dependentBatchRun.dependentTaskAttempt.status)) { - throw new ServiceValidationError( - `Cannot trigger ${taskId} as the parent attempt has a status of ${dependentBatchRun.dependentTaskAttempt.status}` - ); - } else { - throw new ServiceValidationError( - `Cannot trigger ${taskId} as the parent run has a status of ${dependentBatchRun.dependentTaskAttempt.taskRun.status}` - ); - } - } - - const parentBatchRun = body.options?.parentBatch - ? await this._prisma.batchTaskRun.findUnique({ - where: { friendlyId: body.options.parentBatch }, - include: { - dependentTaskAttempt: { - include: { - taskRun: { - select: { - id: true, - status: true, - taskIdentifier: true, - rootTaskRunId: true, - }, - }, - }, - }, - }, - }) - : undefined; - - return await eventRepository.traceEvent( - taskId, - { - context: options.traceContext, - spanParentAsLink: options.spanParentAsLink, - parentAsLinkType: options.parentAsLinkType, - kind: "SERVER", - environment, - taskSlug: taskId, - attributes: { - properties: { - [SemanticInternalAttributes.SHOW_ACTIONS]: true, - }, - style: { - icon: options.customIcon ?? "task", - }, - runIsTest: body.options?.test ?? false, - batchId: options.batchId, - idempotencyKey, - }, - incomplete: true, - immediate: true, - }, - async (event, traceContext, traceparent) => { - const run = await autoIncrementCounter.incrementInTransaction( - `v3-run:${environment.id}:${taskId}`, - async (num, tx) => { - const lockedToBackgroundWorker = body.options?.lockToVersion - ? await tx.backgroundWorker.findUnique({ - where: { - projectId_runtimeEnvironmentId_version: { - projectId: environment.projectId, - runtimeEnvironmentId: environment.id, - version: body.options?.lockToVersion, - }, - }, - }) - : undefined; - - let queueName = sanitizeQueueName( - await this.#getQueueName(taskId, environment, body.options?.queue?.name) - ); - - // Check that the queuename is not an empty string - if (!queueName) { - queueName = sanitizeQueueName(`task/${taskId}`); - } - - event.setAttribute("queueName", queueName); - span.setAttribute("queueName", queueName); - - //upsert tags - let tagIds: string[] = []; - const bodyTags = - typeof body.options?.tags === "string" ? [body.options.tags] : body.options?.tags; - if (bodyTags && bodyTags.length > 0) { - for (const tag of bodyTags) { - const tagRecord = await createTag({ - tag, - projectId: environment.projectId, - }); - if (tagRecord) { - tagIds.push(tagRecord.id); - } - } - } - - const depth = dependentAttempt - ? dependentAttempt.taskRun.depth + 1 - : parentAttempt - ? parentAttempt.taskRun.depth + 1 - : dependentBatchRun?.dependentTaskAttempt - ? dependentBatchRun.dependentTaskAttempt.taskRun.depth + 1 - : 0; - - event.setAttribute("runId", runFriendlyId); - span.setAttribute("runId", runFriendlyId); - - const taskRun = await engine.trigger( - { - number: num, - friendlyId: runFriendlyId, - environment: environment, - idempotencyKey, - taskIdentifier: taskId, - payload: payloadPacket.data ?? "", - payloadType: payloadPacket.dataType, - context: body.context, - traceContext: traceContext, - traceId: event.traceId, - spanId: event.spanId, - parentSpanId: - options.parentAsLinkType === "replay" ? undefined : traceparent?.spanId, - lockedToVersionId: lockedToBackgroundWorker?.id, - concurrencyKey: body.options?.concurrencyKey, - queueName, - queue: body.options?.queue, - //todo multiple worker pools support - masterQueue: "main", - isTest: body.options?.test ?? false, - delayUntil, - queuedAt: delayUntil ? undefined : new Date(), - maxAttempts: body.options?.maxAttempts, - ttl, - tags: tagIds, - parentTaskRunId: parentAttempt?.taskRun.id, - parentTaskRunAttemptId: parentAttempt?.id, - rootTaskRunId: parentAttempt?.taskRun.rootTaskRunId ?? parentAttempt?.taskRun.id, - batchId: dependentBatchRun?.id ?? parentBatchRun?.id, - resumeParentOnCompletion: !!(dependentAttempt ?? dependentBatchRun), - depth, - metadata: metadataPacket?.data, - metadataType: metadataPacket?.dataType, - seedMetadata: metadataPacket?.data, - seedMetadataType: metadataPacket?.dataType, - }, - this._prisma - ); - - return taskRun; - }, - async (_, tx) => { - const counter = await tx.taskRunNumberCounter.findUnique({ - where: { - taskIdentifier_environmentId: { - taskIdentifier: taskId, - environmentId: environment.id, - }, - }, - select: { lastNumber: true }, - }); - - return counter?.lastNumber; - }, - this._prisma - ); - - return run; - } - ); - }); - } - - async #getQueueName(taskId: string, environment: AuthenticatedEnvironment, queueName?: string) { - if (queueName) { - return queueName; - } - - const defaultQueueName = `task/${taskId}`; - - const worker = await findCurrentWorkerFromEnvironment(environment); - - if (!worker) { - logger.debug("Failed to get queue name: No worker found", { - taskId, - environmentId: environment.id, - }); - - return defaultQueueName; - } - - const task = await this._prisma.backgroundWorkerTask.findUnique({ - where: { - workerId_slug: { - workerId: worker.id, - slug: taskId, - }, - }, - }); - - if (!task) { - console.log("Failed to get queue name: No task found", { - taskId, - environmentId: environment.id, - }); - - return defaultQueueName; - } - - const queueConfig = QueueOptions.optional().nullable().safeParse(task.queueConfig); - - if (!queueConfig.success) { - console.log("Failed to get queue name: Invalid queue config", { - taskId, - environmentId: environment.id, - queueConfig: task.queueConfig, - }); - - return defaultQueueName; - } - - return queueConfig.data?.name ?? defaultQueueName; - } - - async #handlePayloadPacket( - payload: any, - payloadType: string, - pathPrefix: string, - environment: AuthenticatedEnvironment - ) { - return await startActiveSpan("handlePayloadPacket()", async (span) => { - const packet = this.#createPayloadPacket(payload, payloadType); - - if (!packet.data) { - return packet; - } - - const { needsOffloading, size } = packetRequiresOffloading( - packet, - env.TASK_PAYLOAD_OFFLOAD_THRESHOLD - ); - - if (!needsOffloading) { - return packet; - } - - const filename = `${pathPrefix}/payload.json`; - - await uploadToObjectStore(filename, packet.data, packet.dataType, environment); - - return { - data: filename, - dataType: "application/store", - }; - }); - } - - #createPayloadPacket(payload: any, payloadType: string): IOPacket { - if (payloadType === "application/json") { - return { data: JSON.stringify(payload), dataType: "application/json" }; - } - - if (typeof payload === "string") { - return { data: payload, dataType: payloadType }; - } - - return { dataType: payloadType }; - } -} - -export async function parseDelay(value?: string | Date): Promise { - if (!value) { - return; - } - - if (value instanceof Date) { - return value; - } - - try { - const date = new Date(value); - - // Check if the date is valid - if (isNaN(date.getTime())) { - return parseNaturalLanguageDuration(value); - } - - if (date.getTime() <= Date.now()) { - return; - } - - return date; - } catch (error) { - return parseNaturalLanguageDuration(value); - } -} - -export function parseNaturalLanguageDuration(duration: string): Date | undefined { - const regexPattern = /^(\d+w)?(\d+d)?(\d+h)?(\d+m)?(\d+s)?$/; - - const result: Date = new Date(); - let hasMatch = false; - - const elements = duration.match(regexPattern); - if (elements) { - if (elements[1]) { - const weeks = Number(elements[1].slice(0, -1)); - if (weeks >= 0) { - result.setDate(result.getDate() + 7 * weeks); - hasMatch = true; - } - } - if (elements[2]) { - const days = Number(elements[2].slice(0, -1)); - if (days >= 0) { - result.setDate(result.getDate() + days); - hasMatch = true; - } - } - if (elements[3]) { - const hours = Number(elements[3].slice(0, -1)); - if (hours >= 0) { - result.setHours(result.getHours() + hours); - hasMatch = true; - } - } - if (elements[4]) { - const minutes = Number(elements[4].slice(0, -1)); - if (minutes >= 0) { - result.setMinutes(result.getMinutes() + minutes); - hasMatch = true; - } - } - if (elements[5]) { - const seconds = Number(elements[5].slice(0, -1)); - if (seconds >= 0) { - result.setSeconds(result.getSeconds() + seconds); - hasMatch = true; - } - } - } - - if (hasMatch) { - return result; - } - - return undefined; -} - -function stringifyDuration(seconds: number): string | undefined { - if (seconds <= 0) { - return; - } - - const units = { - w: Math.floor(seconds / 604800), - d: Math.floor((seconds % 604800) / 86400), - h: Math.floor((seconds % 86400) / 3600), - m: Math.floor((seconds % 3600) / 60), - s: Math.floor(seconds % 60), - }; - - // Filter the units having non-zero values and join them - const result: string = Object.entries(units) - .filter(([unit, val]) => val != 0) - .map(([unit, val]) => `${val}${unit}`) - .join(""); - - return result; -} From 60e7b5f5851c79235ae9214a2744c18b6c78a018 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 8 Oct 2024 16:55:06 +0100 Subject: [PATCH 110/114] Remove zodworker test script (no tests) --- internal-packages/zod-worker/package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal-packages/zod-worker/package.json b/internal-packages/zod-worker/package.json index a10886d659..f58c0d2836 100644 --- a/internal-packages/zod-worker/package.json +++ b/internal-packages/zod-worker/package.json @@ -19,7 +19,6 @@ "vitest": "^1.4.0" }, "scripts": { - "typecheck": "tsc --noEmit", - "test": "vitest" + "typecheck": "tsc --noEmit" } } From 1254649d51abca18f4b629d9cfa47539f7377017 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 8 Oct 2024 17:02:33 +0100 Subject: [PATCH 111/114] Update test-containers readme --- internal-packages/testcontainers/README.md | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/internal-packages/testcontainers/README.md b/internal-packages/testcontainers/README.md index c6547d1bed..51c2240d6c 100644 --- a/internal-packages/testcontainers/README.md +++ b/internal-packages/testcontainers/README.md @@ -1,10 +1,3 @@ -# Redis worker +# Test container -This is a simple worker that pulls tasks from a Redis queue (also in this package). - -Features - -- Configurable settings for concurrency and pull speed. -- Job payloads. -- A schema so only defined jobs can be added to the queue. -- The ability to have future dates for jobs. +This is package exposes some useful vitest utilities for writing tests with Postgres, Prisma, and Redis. From d328d71d8f366c72019c15a42c43dcb343d3b08f Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 8 Oct 2024 17:07:07 +0100 Subject: [PATCH 112/114] Generate the client first --- internal-packages/testcontainers/src/utils.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal-packages/testcontainers/src/utils.ts b/internal-packages/testcontainers/src/utils.ts index 73c46ab4ff..04acfe85aa 100644 --- a/internal-packages/testcontainers/src/utils.ts +++ b/internal-packages/testcontainers/src/utils.ts @@ -9,6 +9,14 @@ export async function createPostgresContainer() { // Run migrations const databasePath = path.resolve(__dirname, "../../database"); + execSync(`npx prisma generate --schema ${databasePath}/prisma/schema.prisma`, { + env: { + ...process.env, + DATABASE_URL: container.getConnectionUri(), + DIRECT_URL: container.getConnectionUri(), + }, + }); + execSync(`npx prisma db push --schema ${databasePath}/prisma/schema.prisma`, { env: { ...process.env, From 1f6c86dfec7f681c302dcdff94cc317e3df8b904 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 8 Oct 2024 17:08:18 +0100 Subject: [PATCH 113/114] Use a specific version of the prisma package --- internal-packages/testcontainers/src/utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal-packages/testcontainers/src/utils.ts b/internal-packages/testcontainers/src/utils.ts index 04acfe85aa..27aa721b0f 100644 --- a/internal-packages/testcontainers/src/utils.ts +++ b/internal-packages/testcontainers/src/utils.ts @@ -9,7 +9,7 @@ export async function createPostgresContainer() { // Run migrations const databasePath = path.resolve(__dirname, "../../database"); - execSync(`npx prisma generate --schema ${databasePath}/prisma/schema.prisma`, { + execSync(`npx prisma@5.4.1 generate --schema ${databasePath}/prisma/schema.prisma`, { env: { ...process.env, DATABASE_URL: container.getConnectionUri(), @@ -17,7 +17,7 @@ export async function createPostgresContainer() { }, }); - execSync(`npx prisma db push --schema ${databasePath}/prisma/schema.prisma`, { + execSync(`npx prisma@5.4.1 db push --schema ${databasePath}/prisma/schema.prisma`, { env: { ...process.env, DATABASE_URL: container.getConnectionUri(), From 08cde55d9a83e8f5d79e6cf3553dc3bb4aa87a6e Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 8 Oct 2024 17:17:06 +0100 Subject: [PATCH 114/114] Generate the prisma client before running the unit tests --- .github/workflows/unit-tests.yml | 3 +++ internal-packages/testcontainers/src/utils.ts | 8 -------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index ba239b5fb2..93dd07deda 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -27,5 +27,8 @@ jobs: - name: ๐Ÿ“ฅ Download deps run: pnpm install --frozen-lockfile + - name: ๐Ÿ“€ Generate Prisma Client + run: pnpm run generate + - name: ๐Ÿงช Run Unit Tests run: pnpm run test diff --git a/internal-packages/testcontainers/src/utils.ts b/internal-packages/testcontainers/src/utils.ts index 27aa721b0f..67fd022525 100644 --- a/internal-packages/testcontainers/src/utils.ts +++ b/internal-packages/testcontainers/src/utils.ts @@ -9,14 +9,6 @@ export async function createPostgresContainer() { // Run migrations const databasePath = path.resolve(__dirname, "../../database"); - execSync(`npx prisma@5.4.1 generate --schema ${databasePath}/prisma/schema.prisma`, { - env: { - ...process.env, - DATABASE_URL: container.getConnectionUri(), - DIRECT_URL: container.getConnectionUri(), - }, - }); - execSync(`npx prisma@5.4.1 db push --schema ${databasePath}/prisma/schema.prisma`, { env: { ...process.env,