From aa6d8f9627721eb1735303bf78b8d876b3c37aa0 Mon Sep 17 00:00:00 2001 From: Mandana Vaziri Date: Thu, 7 Aug 2025 13:22:41 -0400 Subject: [PATCH 1/4] requirements implementation and example Signed-off-by: Mandana Vaziri --- examples/requirements/email.pdl | 42 ++++++++ src/pdl/pdl-schema.json | 175 ++++++++++++++++++++++++++++++++ src/pdl/pdl_ast.py | 31 ++++++ src/pdl/pdl_interpreter.py | 22 +++- tests/test_examples_run.yaml | 1 + 5 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 examples/requirements/email.pdl diff --git a/examples/requirements/email.pdl b/examples/requirements/email.pdl new file mode 100644 index 000000000..c949b33d3 --- /dev/null +++ b/examples/requirements/email.pdl @@ -0,0 +1,42 @@ +description: Hello world calling a model +defs: + eval: + function: + requirement: string + response: string + return: + lastOf: + - model: ollama_chat/mistral-small:latest + def: result + input: | + Does the following email end with Kind regards. Answer with a JSON object and a result field with value True or False only. + Email: ${ response } + parser: json + - ${ result.result } + + fix: + function: + requirement: string + response: string + return: + lastOf: + - model: ollama_chat/mistral-small:latest + def: instruction + input: | + A model responded with the following response: ${ response } + In order to satisfy the following requirement: ${ requirement } + what should be added to the prompt as an instruction? + + Respond with only with the instruction. + - ${ pdl_context } + + notes: Olivia helped the lab over the last few weeks by organizing intern events, advertising the speaker series, and handling issues with snack delivery. + name: Olivia +text: +- "Write an email to ${ name } using the notes following: ${ notes }" +- model: ollama_chat/granite3.2:2b + requirements: + - description: The email should end with Kind regards + evaluate: ${ eval } + transformContext: ${ fix } + retry: 5 diff --git a/src/pdl/pdl-schema.json b/src/pdl/pdl-schema.json index 230ad79bd..6658cfcb6 100644 --- a/src/pdl/pdl-schema.json +++ b/src/pdl/pdl-schema.json @@ -76,6 +76,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -194,6 +199,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -417,6 +427,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -554,6 +569,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -726,6 +746,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -849,6 +874,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -966,6 +996,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -1116,6 +1151,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -1255,6 +1295,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -1364,6 +1409,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -1563,6 +1613,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -1685,6 +1740,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -1798,6 +1858,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -2068,6 +2133,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -2184,6 +2254,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -2770,6 +2845,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -2942,6 +3022,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -3137,6 +3222,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -3262,6 +3352,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -3793,6 +3888,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -3947,6 +4047,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, @@ -4073,6 +4178,71 @@ "title": "RepeatBlock", "type": "object" }, + "RequirementType": { + "additionalProperties": false, + "description": "Single requirement definition.", + "properties": { + "description": { + "anyOf": [ + { + "$ref": "#/$defs/LocalizedExpression_TypeVar_" + }, + {}, + { + "type": "string" + } + ], + "title": "Description" + }, + "evaluate": { + "anyOf": [ + { + "$ref": "#/$defs/LocalizedExpression_TypeVar_" + }, + { + "$ref": "#/$defs/FunctionBlock" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Evaluate" + }, + "transformContext": { + "anyOf": [ + { + "$ref": "#/$defs/LocalizedExpression_TypeVar_" + }, + { + "$ref": "#/$defs/FunctionBlock" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Transformcontext" + } + }, + "required": [ + "description", + "evaluate", + "transformContext" + ], + "title": "RequirementType", + "type": "object" + }, + "RequirementsType": { + "items": { + "$ref": "#/$defs/RequirementType" + }, + "type": "array" + }, "TextBlock": { "additionalProperties": false, "description": "Create the concatenation of the stringify version of the result of each block of the list of blocks.", @@ -4129,6 +4299,11 @@ "default": null, "description": "Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials.\n " }, + "requirements": { + "$ref": "#/$defs/RequirementsType", + "default": [], + "description": "Specify any requirements that the result of the block must satisfy.\n " + }, "role": { "$ref": "#/$defs/OptionalStr", "default": null, diff --git a/src/pdl/pdl_ast.py b/src/pdl/pdl_ast.py index 65e066784..496378e18 100644 --- a/src/pdl/pdl_ast.py +++ b/src/pdl/pdl_ast.py @@ -329,6 +329,26 @@ class ContributeValue(BaseModel): ) """Type of the contribute field.""" +class RequirementType(BaseModel): + """Single requirement definition.""" + + model_config = ConfigDict(extra="forbid") + + description: ExpressionType + """English description of the requirement""" + + evaluate: Optional[ExpressionType["FunctionBlock"]] + """Evaluation function for the requirement""" + + transformContext: Optional[ExpressionType["FunctionBlock"]] + """Function to transform the context for the requirement""" + + +RequirementsType = TypeAliasType( + "RequirementsType", Sequence[RequirementType] +) +"""Type of requirements field""" + class PdlTiming(BaseModel): """Internal data structure to record timing information in the trace.""" @@ -406,6 +426,11 @@ class Block(BaseModel): trace_error_on_retry: OptionalBoolOrStr = None """Whether to add the errors while retrying to the trace. Set this to true to use retry feature for multiple LLM trials. """ + + requirements: RequirementsType = [] + """Specify any requirements that the result of the block must satisfy. + """ + role: RoleType = None """Role associated to the block and sub-blocks. Typical roles are `system`, `user`, and `assistant`, @@ -1165,6 +1190,12 @@ def __init__( self.message = message + + + + + + MAX_NEW_TOKENS = 1024 MIN_NEW_TOKENS = 1 REPETITION_PENALTY = 1.05 diff --git a/src/pdl/pdl_interpreter.py b/src/pdl/pdl_interpreter.py index 23552d8a2..b59dd76f9 100644 --- a/src/pdl/pdl_interpreter.py +++ b/src/pdl/pdl_interpreter.py @@ -323,7 +323,7 @@ def process_block( ) from exc result = PdlConst(v) background = SingletonContext( - PdlDict( + PdlDict( { "role": state.role, "content": result, @@ -476,6 +476,22 @@ def process_advanced_block( result, background, new_scope, trace = process_block_body( state, scope, block, loc ) + if block.requirements is not []: + requirements_satisfied = True + for req in block.requirements: + evalfn, _ = process_expr(scope, getattr(req, "evaluate"), loc) + eval = evalfn(requirement=getattr(req, "description"), response=result) + if eval.result() == False: + requirements_satisfied = False + transfn, _ = process_expr(scope, getattr(req, "transformContext"), loc) + new_context = transfn(pdl_context=scope["pdl_context"], requirement=getattr(req, "description"), response=result) + scope = scope | { + "pdl_context": new_context + } + if requirements_satisfied is False: + print("\nTrying again!") + continue + result = lazy_apply(id_with_set_first_use_nanos(block.pdl__timing), result) add_done_callback( id_with_set_first_use_nanos(block.pdl__timing), background @@ -2238,6 +2254,10 @@ def parse_result(parser: ParserType, text: str) -> JSONReturnType: match parser: case "json": try: + if text == 'False': + return json.loads("false") + if text == 'True': + return json.loads("true") result = json_repair.loads(text) # type: ignore[reportAssignmentType] except Exception as exc: raise PDLRuntimeParserError( diff --git a/tests/test_examples_run.yaml b/tests/test_examples_run.yaml index 78ab19b4e..6c3d355ab 100644 --- a/tests/test_examples_run.yaml +++ b/tests/test_examples_run.yaml @@ -29,6 +29,7 @@ skip: - examples/optimizer/mbpp.pdl - examples/optimizer/fever.pdl - examples/optimizer/gsm8k.pdl + - examples/requirements/email.pdl with_inputs: examples/tutorial/programs/chatbot.pdl: stdin: | From fc33a2d38607e45e2977320cffce17a27fa2b51c Mon Sep 17 00:00:00 2001 From: Mandana Vaziri Date: Thu, 7 Aug 2025 13:27:58 -0400 Subject: [PATCH 2/4] cleanup Signed-off-by: Mandana Vaziri --- src/pdl/pdl_ast.py | 15 ++++----------- src/pdl/pdl_interpreter.py | 30 ++++++++++++++++++------------ 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/src/pdl/pdl_ast.py b/src/pdl/pdl_ast.py index 496378e18..8d190ac8f 100644 --- a/src/pdl/pdl_ast.py +++ b/src/pdl/pdl_ast.py @@ -329,6 +329,7 @@ class ContributeValue(BaseModel): ) """Type of the contribute field.""" + class RequirementType(BaseModel): """Single requirement definition.""" @@ -336,17 +337,15 @@ class RequirementType(BaseModel): description: ExpressionType """English description of the requirement""" - + evaluate: Optional[ExpressionType["FunctionBlock"]] """Evaluation function for the requirement""" transformContext: Optional[ExpressionType["FunctionBlock"]] """Function to transform the context for the requirement""" - -RequirementsType = TypeAliasType( - "RequirementsType", Sequence[RequirementType] -) + +RequirementsType = TypeAliasType("RequirementsType", Sequence[RequirementType]) """Type of requirements field""" @@ -1190,12 +1189,6 @@ def __init__( self.message = message - - - - - - MAX_NEW_TOKENS = 1024 MIN_NEW_TOKENS = 1 REPETITION_PENALTY = 1.05 diff --git a/src/pdl/pdl_interpreter.py b/src/pdl/pdl_interpreter.py index b59dd76f9..53627dbfc 100644 --- a/src/pdl/pdl_interpreter.py +++ b/src/pdl/pdl_interpreter.py @@ -323,7 +323,7 @@ def process_block( ) from exc result = PdlConst(v) background = SingletonContext( - PdlDict( + PdlDict( { "role": state.role, "content": result, @@ -442,7 +442,7 @@ def set_error_to_scope_for_retry( return scope -def process_advanced_block( +def process_advanced_block( # noqa:C901 state: InterpreterState, scope: ScopeType, block: AdvancedBlockType, @@ -476,18 +476,24 @@ def process_advanced_block( result, background, new_scope, trace = process_block_body( state, scope, block, loc ) - if block.requirements is not []: + if block.requirements != []: requirements_satisfied = True for req in block.requirements: evalfn, _ = process_expr(scope, getattr(req, "evaluate"), loc) - eval = evalfn(requirement=getattr(req, "description"), response=result) - if eval.result() == False: + evaluation = evalfn( + requirement=getattr(req, "description"), response=result + ) + if evaluation.result() is False: requirements_satisfied = False - transfn, _ = process_expr(scope, getattr(req, "transformContext"), loc) - new_context = transfn(pdl_context=scope["pdl_context"], requirement=getattr(req, "description"), response=result) - scope = scope | { - "pdl_context": new_context - } + transfn, _ = process_expr( + scope, getattr(req, "transformContext"), loc + ) + new_context = transfn( + pdl_context=scope["pdl_context"], + requirement=getattr(req, "description"), + response=result, + ) + scope = scope | {"pdl_context": new_context} if requirements_satisfied is False: print("\nTrying again!") continue @@ -2254,9 +2260,9 @@ def parse_result(parser: ParserType, text: str) -> JSONReturnType: match parser: case "json": try: - if text == 'False': + if text == "False": return json.loads("false") - if text == 'True': + if text == "True": return json.loads("true") result = json_repair.loads(text) # type: ignore[reportAssignmentType] except Exception as exc: From c4dc0d2aeec9e75b51d3262432c645ebdf6772f0 Mon Sep 17 00:00:00 2001 From: Mandana Vaziri Date: Thu, 7 Aug 2025 14:24:01 -0400 Subject: [PATCH 3/4] cleanup Signed-off-by: Mandana Vaziri --- src/pdl/pdl_dumper.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/pdl/pdl_dumper.py b/src/pdl/pdl_dumper.py index 3e6f2ace5..d9b1b3860 100644 --- a/src/pdl/pdl_dumper.py +++ b/src/pdl/pdl_dumper.py @@ -61,6 +61,7 @@ RepeatBlock, StructuredBlock, TextBlock, + RequirementType ) from .pdl_lazy import PdlLazy from .pdl_schema_utils import OLD_PDLTYPE_TO_JSONSCHEMA_NAME @@ -297,6 +298,8 @@ def block_to_dict( # noqa: C901 d["pdl__result"] = data_to_dict(block.pdl__result.result(), json_compatible) if block.parser is not None: d["parser"] = parser_to_dict(block.parser) + if block.requirements is not None: + d["requirements"] = [requirement_to_dict(b, json_compatible) for b in block.requirements] # if block.pdl__location is not None: # d["pdl__location"] = location_to_dict(block.pdl__location) if block.fallback is not None: @@ -397,6 +400,14 @@ def usage_to_dict(usage: PdlUsage) -> dict: d["prompt_tokens"] = usage.prompt_tokens return d +def requirement_to_dict(req: RequirementType, json_compatible: bool) -> dict: + d: dict = {} + d["description"] = req.description + d["evaluate"] = expr_to_dict(req.evaluate, json_compatible) + d["transformContext"] = expr_to_dict(req.transformContext, json_compatible) + return d + + def pattern_to_dict(pattern: PatternType): if not isinstance(pattern, pdl_ast.Pattern): From 61be22cb3c97cc30973c85a26163b7e60fb16283 Mon Sep 17 00:00:00 2001 From: Mandana Vaziri Date: Thu, 7 Aug 2025 14:28:25 -0400 Subject: [PATCH 4/4] cleanup Signed-off-by: Mandana Vaziri --- src/pdl/pdl_dumper.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/pdl/pdl_dumper.py b/src/pdl/pdl_dumper.py index d9b1b3860..52d087c71 100644 --- a/src/pdl/pdl_dumper.py +++ b/src/pdl/pdl_dumper.py @@ -59,9 +59,9 @@ ReadBlock, RegexParser, RepeatBlock, + RequirementType, StructuredBlock, TextBlock, - RequirementType ) from .pdl_lazy import PdlLazy from .pdl_schema_utils import OLD_PDLTYPE_TO_JSONSCHEMA_NAME @@ -299,7 +299,9 @@ def block_to_dict( # noqa: C901 if block.parser is not None: d["parser"] = parser_to_dict(block.parser) if block.requirements is not None: - d["requirements"] = [requirement_to_dict(b, json_compatible) for b in block.requirements] + d["requirements"] = [ + requirement_to_dict(b, json_compatible) for b in block.requirements + ] # if block.pdl__location is not None: # d["pdl__location"] = location_to_dict(block.pdl__location) if block.fallback is not None: @@ -400,6 +402,7 @@ def usage_to_dict(usage: PdlUsage) -> dict: d["prompt_tokens"] = usage.prompt_tokens return d + def requirement_to_dict(req: RequirementType, json_compatible: bool) -> dict: d: dict = {} d["description"] = req.description @@ -408,7 +411,6 @@ def requirement_to_dict(req: RequirementType, json_compatible: bool) -> dict: return d - def pattern_to_dict(pattern: PatternType): if not isinstance(pattern, pdl_ast.Pattern): return pattern