lastOf support

Signed-off-by: Nick Mitchell <[email protected]>

Author: starpit

pdl-live-react/src-tauri/src/pdl/ast.rs

@@ -1,6 +1,6 @@
 use ::std::collections::HashMap;
 use serde::{Deserialize, Serialize};
-use serde_json::{Number, Value};
+use serde_json::{to_string, Number, Value};
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
 //why doesn't this work? #[serde(rename_all_fields(serialize = "lowercase"))]
@@ -74,6 +74,86 @@ impl CallBlock {
     }
 }
 
+pub trait SequencingBlock {
+    fn kind(&self) -> &str;
+    fn description(&self) -> &Option<String>;
+    fn role(&self) -> &Option<Role>;
+    fn def(&self) -> &Option<String>;
+    fn defs(&self) -> &Option<HashMap<String, PdlBlock>>;
+    fn items(&self) -> &Vec<PdlBlock>;
+    fn with_items(&self, items: Vec<PdlBlock>) -> Self;
+    fn parser(&self) -> &Option<PdlParser>;
+    fn to_block(&self) -> PdlBlock;
+    fn result_for(&self, output_results: Vec<PdlResult>) -> PdlResult;
+    fn messages_for<T: Clone>(&self, output_messages: Vec<T>) -> Vec<T>;
+}
+
+/// Return the value of the last block if the list of blocks
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct LastOfBlock {
+    /// Sequence of blocks to execute
+    #[serde(rename = "lastOf")]
+    pub last_of: Vec<PdlBlock>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub description: Option<String>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub role: Option<Role>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub defs: Option<HashMap<String, PdlBlock>>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub parser: Option<PdlParser>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub def: Option<String>,
+}
+impl SequencingBlock for LastOfBlock {
+    fn kind(&self) -> &str {
+        "lastOf"
+    }
+    fn description(&self) -> &Option<String> {
+        &self.description
+    }
+    fn role(&self) -> &Option<Role> {
+        &self.role
+    }
+    fn def(&self) -> &Option<String> {
+        return &self.def;
+    }
+    fn defs(&self) -> &Option<HashMap<String, PdlBlock>> {
+        &self.defs
+    }
+    fn items(&self) -> &Vec<PdlBlock> {
+        &self.last_of
+    }
+    fn with_items(&self, items: Vec<PdlBlock>) -> Self {
+        let mut b = self.clone();
+        b.last_of = items;
+        b
+    }
+    fn parser(&self) -> &Option<PdlParser> {
+        &self.parser
+    }
+    fn to_block(&self) -> PdlBlock {
+        PdlBlock::LastOf(self.clone())
+    }
+    fn result_for(&self, output_results: Vec<PdlResult>) -> PdlResult {
+        match output_results.last() {
+            Some(result) => result.clone(),
+            None => "".into(),
+        }
+    }
+    fn messages_for<T: Clone>(&self, output_messages: Vec<T>) -> Vec<T> {
+        match output_messages.last() {
+            Some(m) => vec![m.clone()],
+            None => vec![],
+        }
+    }
+}
+
 /// Create the concatenation of the stringify version of the result of
 /// each block of the list of blocks.
 #[derive(Serialize, Deserialize, Debug, Clone)]
@@ -96,6 +176,49 @@ pub struct TextBlock {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub def: Option<String>,
 }
+impl SequencingBlock for TextBlock {
+    fn kind(&self) -> &str {
+        "text"
+    }
+    fn description(&self) -> &Option<String> {
+        &self.description
+    }
+    fn role(&self) -> &Option<Role> {
+        &self.role
+    }
+    fn def(&self) -> &Option<String> {
+        return &self.def;
+    }
+    fn defs(&self) -> &Option<HashMap<String, PdlBlock>> {
+        &self.defs
+    }
+    fn items(&self) -> &Vec<PdlBlock> {
+        &self.text
+    }
+    fn with_items(&self, items: Vec<PdlBlock>) -> Self {
+        let mut b = self.clone();
+        b.text = items;
+        b
+    }
+    fn parser(&self) -> &Option<PdlParser> {
+        &self.parser
+    }
+    fn to_block(&self) -> PdlBlock {
+        PdlBlock::Text(self.clone())
+    }
+    fn result_for(&self, output_results: Vec<PdlResult>) -> PdlResult {
+        PdlResult::String(
+            output_results
+                .into_iter()
+                .map(|m| m.to_string())
+                .collect::<Vec<_>>()
+                .join("\n"),
+        )
+    }
+    fn messages_for<T: Clone>(&self, output_messages: Vec<T>) -> Vec<T> {
+        output_messages
+    }
+}
 
 impl TextBlock {
     pub fn new(text: Vec<PdlBlock>) -> Self {
@@ -366,6 +489,7 @@ pub enum PdlBlock {
     Message(MessageBlock),
     Repeat(RepeatBlock),
     Text(TextBlock),
+    LastOf(LastOfBlock),
     Model(ModelBlock),
     Function(FunctionBlock),
     PythonCode(PythonCodeBlock),
@@ -389,3 +513,44 @@ impl From<&str> for Box<PdlBlock> {
         Box::new(PdlBlock::String(s.into()))
     }
 }
+
+pub type Scope = HashMap<String, PdlResult>;
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct Closure {
+    pub scope: Scope,
+    pub function: FunctionBlock,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(untagged)]
+pub enum PdlResult {
+    Number(Number),
+    String(String),
+    Bool(bool),
+    Block(PdlBlock),
+    Closure(Closure),
+    List(Vec<PdlResult>),
+    Dict(HashMap<String, PdlResult>),
+}
+impl ::std::fmt::Display for PdlResult {
+    fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
+        let s = to_string(&self).unwrap(); // TODO: .map_err(|e| e.to_string())?;
+        write!(f, "{}", s)
+    }
+}
+impl From<&str> for PdlResult {
+    fn from(s: &str) -> Self {
+        PdlResult::String(s.to_string())
+    }
+}
+impl From<String> for PdlResult {
+    fn from(s: String) -> Self {
+        PdlResult::String(s)
+    }
+}
+impl From<Number> for PdlResult {
+    fn from(n: Number) -> Self {
+        PdlResult::Number(n)
+    }
+}

pdl-live-react/src-tauri/src/pdl/interpreter.rs

@@ -21,57 +21,16 @@ use ollama_rs::{
     Ollama,
 };
 
-use serde::{Deserialize, Serialize};
-use serde_json::{from_str, to_string, Number, Value};
+use serde_json::{from_str, to_string, Value};
 use serde_norway::{from_reader, from_str as from_yaml_str};
 
 use crate::pdl::ast::{
-    ArrayBlock, CallBlock, FunctionBlock, IfBlock, ListOrString, MessageBlock, ModelBlock,
-    ObjectBlock, PdlBlock, PdlParser, PdlUsage, PythonCodeBlock, ReadBlock, RepeatBlock, Role,
-    StringOrBoolean, TextBlock,
+    ArrayBlock, CallBlock, Closure, FunctionBlock, IfBlock, ListOrString, MessageBlock, ModelBlock,
+    ObjectBlock, PdlBlock, PdlParser, PdlResult, PdlUsage, PythonCodeBlock, ReadBlock, RepeatBlock,
+    Role, Scope, SequencingBlock, StringOrBoolean,
 };
 
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct Closure {
-    pub scope: Scope,
-    pub function: FunctionBlock,
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone)]
-#[serde(untagged)]
-pub enum PdlResult {
-    Number(Number),
-    String(String),
-    Bool(bool),
-    Block(PdlBlock),
-    Closure(Closure),
-    List(Vec<PdlResult>),
-    Dict(HashMap<String, PdlResult>),
-}
-impl ::std::fmt::Display for PdlResult {
-    fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
-        let s = to_string(&self).unwrap(); // TODO: .map_err(|e| e.to_string())?;
-        write!(f, "{}", s)
-    }
-}
-impl From<&str> for PdlResult {
-    fn from(s: &str) -> Self {
-        PdlResult::String(s.to_string())
-    }
-}
-impl From<String> for PdlResult {
-    fn from(s: String) -> Self {
-        PdlResult::String(s)
-    }
-}
-impl From<Number> for PdlResult {
-    fn from(n: Number) -> Self {
-        PdlResult::Number(n)
-    }
-}
-
 type Context = Vec<ChatMessage>;
-type Scope = HashMap<String, PdlResult>;
 type PdlError = Box<dyn Error + Send + Sync>;
 type Interpretation = Result<(PdlResult, Context, PdlBlock), PdlError>;
 type InterpretationSync = Result<(PdlResult, Context, PdlBlock), Box<dyn Error>>;
@@ -146,14 +105,15 @@ impl<'a> Interpreter<'a> {
             PdlBlock::PythonCode(block) => self.run_python_code(block, context).await,
             PdlBlock::Read(block) => self.run_read(block, context).await,
             PdlBlock::Repeat(block) => self.run_repeat(block, context).await,
-            PdlBlock::Text(block) => self.run_text(block, context).await,
+            PdlBlock::LastOf(block) => self.run_sequence(block, context).await,
+            PdlBlock::Text(block) => self.run_sequence(block, context).await,
             PdlBlock::Array(block) => self.run_array(block, context).await,
             PdlBlock::Message(block) => self.run_message(block, context).await,
             _ => Err(Box::from(format!("Unsupported block {:?}", program))),
         }?;
 
         if match program {
-            PdlBlock::Call(_) | PdlBlock::Model(_) | PdlBlock::Text(_) => false,
+            PdlBlock::Call(_) | PdlBlock::Model(_) => false,
             _ => self.emit,
         } {
             println!("{}", pretty_print(&messages));
@@ -733,60 +693,58 @@ impl<'a> Interpreter<'a> {
         Ok(())
     }
 
-    /// Run a PdlBlock::Text
-    async fn run_text(&mut self, block: &TextBlock, context: Context) -> Interpretation {
+    /// Run a sequencing block (e.g. TextBlock, LastOfBlock)
+    async fn run_sequence(
+        &mut self,
+        block: &impl SequencingBlock,
+        context: Context,
+    ) -> Interpretation {
         if self.debug {
-            eprintln!(
-                "Text {:?}",
-                block
-                    .description
-                    .clone()
-                    .unwrap_or("<no description>".to_string())
-            );
+            let description = if let Some(d) = block.description() {
+                d
+            } else {
+                &"<no description>".to_string()
+            };
+            eprintln!("{} {description}", block.kind());
         }
 
         let mut input_messages = context.clone();
         let mut output_results = vec![];
         let mut output_messages = vec![];
         let mut output_blocks = vec![];
 
-        self.process_defs(&block.defs).await?;
+        self.process_defs(block.defs()).await?;
 
-        let mut iter = block.text.iter();
+        let mut iter = block.items().iter();
         while let Some(block) = iter.next() {
             // run each element of the Text block
             let (this_result, this_messages, trace) =
-                self.run(&block, input_messages.clone()).await?;
+                self.run_quiet(&block, input_messages.clone()).await?;
             input_messages.extend(this_messages.clone());
             output_results.push(this_result);
+
             output_messages.extend(this_messages);
             output_blocks.push(trace);
         }
         self.scope.pop();
 
-        let mut trace = block.clone();
-        trace.text = output_blocks;
-
-        let result_string = output_results
-            .into_iter()
-            .map(|m| m.to_string())
-            .collect::<Vec<_>>()
-            .join("\n");
-
-        // FIXME, use output_results
+        let trace = block.with_items(output_blocks);
         let result = self.def(
-            &block.def,
-            &PdlResult::String(result_string.clone()),
-            &block.parser,
+            trace.def(),
+            &trace.result_for(output_results),
+            trace.parser(),
         )?;
-
+        let result_messages = trace.messages_for::<ChatMessage>(output_messages);
         Ok((
             result,
-            match &block.role {
-                Some(role) => vec![ChatMessage::new(self.to_ollama_role(role), result_string)],
-                None => output_messages,
+            match block.role() {
+                Some(role) => result_messages
+                    .into_iter()
+                    .map(|m| ChatMessage::new(self.to_ollama_role(role), m.content))
+                    .collect(),
+                None => result_messages,
             },
-            PdlBlock::Text(trace),
+            trace.to_block(),
         ))
     }
 

pdl-live-react/src-tauri/src/pdl/interpreter_tests.rs

@@ -138,6 +138,23 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn last_of_parser_json() -> Result<(), Box<dyn Error>> {
+        let json = "{\"key\":\"value\"}";
+        let program = json!({
+            "lastOf": [
+                { "def": "foo", "parser": "json", "text": [json] },
+                "${ foo.key }"
+            ]
+        });
+
+        let (_, messages, _) = run_json(program, false)?;
+        assert_eq!(messages.len(), 1);
+        assert_eq!(messages[0].role, MessageRole::User);
+        assert_eq!(messages[0].content, "value");
+        Ok(())
+    }
+
     #[test]
     fn text_call_function_no_args() -> Result<(), Box<dyn Error>> {
         let program = json!({

pdl-live-react/src-tauri/tests/cli/json-parser-lastOf.pdl

@@ -0,0 +1,6 @@
+lastOf:
+  - text:
+      - '{"key": "value"}'
+    parser: json
+    def: foo
+  - ${ foo.key }