Skip to content

Requirements implementation #1097

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Aug 7, 2025
Merged

Requirements implementation #1097

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
aa6d8f9
requirements implementation and example
vazirim Aug 7, 2025
fc33a2d
cleanup
vazirim Aug 7, 2025
c4dc0d2
cleanup
vazirim Aug 7, 2025
61be22c
cleanup
vazirim Aug 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions examples/requirements/email.pdl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
description: Hello world calling a model
defs:
eval:
function:
requirement: string
response: string
return:
lastOf:
- model: ollama_chat/mistral-small:latest
def: result
input: |
Does the following email end with Kind regards. Answer with a JSON object and a result field with value True or False only.
Email: ${ response }
parser: json
- ${ result.result }

fix:
function:
requirement: string
response: string
return:
lastOf:
- model: ollama_chat/mistral-small:latest
def: instruction
input: |
A model responded with the following response: ${ response }
In order to satisfy the following requirement: ${ requirement }
what should be added to the prompt as an instruction?

Respond with only with the instruction.
- ${ pdl_context }

notes: Olivia helped the lab over the last few weeks by organizing intern events, advertising the speaker series, and handling issues with snack delivery.
name: Olivia
text:
- "Write an email to ${ name } using the notes following: ${ notes }"
- model: ollama_chat/granite3.2:2b
requirements:
- description: The email should end with Kind regards
evaluate: ${ eval }
transformContext: ${ fix }
retry: 5
175 changes: 175 additions & 0 deletions src/pdl/pdl-schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -194,6 +199,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -417,6 +427,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -554,6 +569,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -726,6 +746,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -849,6 +874,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -966,6 +996,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -1116,6 +1151,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -1255,6 +1295,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -1364,6 +1409,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -1563,6 +1613,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -1685,6 +1740,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -1798,6 +1858,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -2068,6 +2133,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -2184,6 +2254,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -2770,6 +2845,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -2942,6 +3022,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -3137,6 +3222,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -3262,6 +3352,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -3793,6 +3888,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -3947,6 +4047,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down Expand Up @@ -4073,6 +4178,71 @@
"title": "RepeatBlock",
"type": "object"
},
"RequirementType": {
"additionalProperties": false,
"description": "Single requirement definition.",
"properties": {
"description": {
"anyOf": [
{
"$ref": "#/$defs/LocalizedExpression_TypeVar_"
},
{},
{
"type": "string"
}
],
"title": "Description"
},
"evaluate": {
"anyOf": [
{
"$ref": "#/$defs/LocalizedExpression_TypeVar_"
},
{
"$ref": "#/$defs/FunctionBlock"
},
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Evaluate"
},
"transformContext": {
"anyOf": [
{
"$ref": "#/$defs/LocalizedExpression_TypeVar_"
},
{
"$ref": "#/$defs/FunctionBlock"
},
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Transformcontext"
}
},
"required": [
"description",
"evaluate",
"transformContext"
],
"title": "RequirementType",
"type": "object"
},
"RequirementsType": {
"items": {
"$ref": "#/$defs/RequirementType"
},
"type": "array"
},
"TextBlock": {
"additionalProperties": false,
"description": "Create the concatenation of the stringify version of the result of each block of the list of blocks.",
Expand Down Expand Up @@ -4129,6 +4299,11 @@
"default": null,
"description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n "
},
"requirements": {
"$ref": "#/$defs/RequirementsType",
"default": [],
"description": "Specify any requirements that the result of the block must satisfy.\n "
},
"role": {
"$ref": "#/$defs/OptionalStr",
"default": null,
Expand Down
Loading