feat: some regex parser support for rust interpreter

Split mode TODO!

Signed-off-by: Nick Mitchell <[email protected]>
Signed-off-by: Claudio Spiess <[email protected]>

Author: starpit

pdl-live-react/src-tauri/Cargo.lock

@@ -50,6 +50,7 @@ fs4 = "0.13.1"
 derive_builder = "0.20.2"
 iana-time-zone = "0.1.63"
 async-openai = "0.28.1"
+regex = "1.11.1"
 
 [target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies]
 tauri-plugin-cli = "2"

pdl-live-react/src-tauri/Cargo.toml

@@ -19,13 +19,18 @@ pub enum Role {
     Tool,
 }
 
+/// Function used to parse to value (https://docs.python.org/3/library/re.html).
 #[derive(Serialize, Deserialize, Debug, Clone)]
-#[serde(rename_all_fields(serialize = "lowercase"))]
 pub enum RegexMode {
+    #[serde(rename = "search")]
     Search,
+    #[serde(rename = "match")]
     Match,
+    #[serde(rename = "fullmatch")]
     Fullmatch,
+    #[serde(rename = "split")]
     Split,
+    #[serde(rename = "findall")]
     Findall,
 }
 
@@ -40,7 +45,7 @@ pub struct RegexParser {
 
     /// Expected type of the parsed value
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub spec: Option<Value>,
+    pub spec: Option<IndexMap<String, PdlType>>,
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
@@ -744,3 +749,15 @@ impl From<Number> for PdlResult {
         PdlResult::Number(n)
     }
 }
+impl PartialEq for PdlResult {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (PdlResult::Number(a), PdlResult::Number(b)) => a == b,
+            (PdlResult::String(a), PdlResult::String(b)) => a == b,
+            (PdlResult::Bool(a), PdlResult::Bool(b)) => a == b,
+            (PdlResult::List(a), PdlResult::List(b)) => a == b,
+            (PdlResult::Dict(a), PdlResult::Dict(b)) => a == b,
+            _ => false,
+        }
+    }
+}

pdl-live-react/src-tauri/src/pdl/ast.rs

@@ -27,8 +27,8 @@ use crate::pdl::ast::{
     IncludeBlock, ListOrString, MessageBlock, Metadata, MetadataBuilder, ModelBlock, ObjectBlock,
     PdlBlock,
     PdlBlock::Advanced,
-    PdlParser, PdlResult, PdlUsage, PythonCodeBlock, ReadBlock, RepeatBlock, Role, Scope,
-    SequencingBlock, StringOrBoolean, StringOrNull, Timing,
+    PdlParser, PdlResult, PdlUsage, PythonCodeBlock, ReadBlock, RegexMode, RegexParser,
+    RepeatBlock, Role, Scope, SequencingBlock, StringOrBoolean, StringOrNull, Timing,
 };
 
 type Messages = Vec<ChatMessage>;
@@ -1172,12 +1172,13 @@ impl<'a> Interpreter<'a> {
                                 Some(Value::String(s)) => Some(ChatMessage::user(s.clone())),
                                 _ => None,
                             },
-                            _ => None,
+                            m => Some(ChatMessage::user(m.to_string())),
                         })
                         .collect(),
-                    _ => vec![],
+                    _ => vec![ChatMessage::user(to_string(m)?)],
                 },
-                _ => vec![],
+                Value::Array(a) => vec![ChatMessage::user(to_string(a)?)],
+                m => vec![ChatMessage::user(m.to_string())],
             };
             Ok((result, messages, Data(trace)))
         }
@@ -1288,7 +1289,38 @@ impl<'a> Interpreter<'a> {
                     .collect::<Result<_, _>>()?,
             )),
             PdlParser::Yaml => from_yaml_str(result).map_err(|e| Box::from(e)),
-            PdlParser::Regex(_) => todo!(),
+            PdlParser::Regex(RegexParser { regex, mode, spec }) => {
+                use regex::Regex;
+                let re = Regex::new(regex)?;
+                let expected_captures: Vec<&str> = if let Some(spec) = spec {
+                    spec.keys().map(|k| k.as_str()).collect()
+                } else {
+                    vec![]
+                };
+
+                match mode {
+                    Some(RegexMode::Findall) => Ok(PdlResult::List(
+                        re.captures_iter(result)
+                            .flat_map(|cap| {
+                                expected_captures.iter().filter_map(move |k| {
+                                    cap.name(k).and_then(|m| Some(m.as_str().into()))
+                                })
+                            })
+                            .collect(),
+                    )),
+                    Some(RegexMode::Split) => todo!(),
+                    _ => Ok(PdlResult::Dict(
+                        re.captures_iter(result)
+                            .flat_map(|cap| {
+                                expected_captures.iter().filter_map(move |k| {
+                                    cap.name(k)
+                                        .and_then(|m| Some((k.to_string(), m.as_str().into())))
+                                })
+                            })
+                            .collect(),
+                    )),
+                }
+            }
         }
     }
 

pdl-live-react/src-tauri/src/pdl/interpreter.rs

@@ -4,7 +4,7 @@ mod tests {
     use serde_json::json;
 
     use crate::pdl::{
-        ast::{Block, Body::*, ModelBlockBuilder, PdlBlock, PdlBlock::Advanced, Scope},
+        ast::{Block, Body::*, ModelBlockBuilder, PdlBlock, PdlBlock::Advanced, PdlResult, Scope},
         interpreter::{RunOptions, load_scope, run_json_sync as run_json, run_sync as run},
     };
 
@@ -679,6 +679,67 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn regex_findall() -> Result<(), Box<dyn Error>> {
+        let program = json!({
+            "data": "aaa999bbb888",
+            "parser": {
+                "regex": "[^0-9]*(?P<answer1>[0-9]+)[^0-9]*(?P<answer2>[0-9]+)$",
+                "mode": "findall",
+                "spec": {
+                    "answer1": "str",
+                    "answer2": "str"
+                }
+            }
+        });
+
+        let (_, messages, _) = run_json(program, streaming(), initial_scope())?;
+        assert_eq!(messages.len(), 1);
+        assert_eq!(messages[0].role, MessageRole::User);
+        assert_eq!(messages[0].content, "[\"999\",\"888\"]");
+        Ok(())
+    }
+
+    #[test]
+    fn regex_plain_1() -> Result<(), Box<dyn Error>> {
+        let program = json!({
+            "data": "aaa999bbb888",
+            "parser": {
+                "regex": "[^0-9]*(?P<answer1>[0-9]+)[^0-9]*(?P<answer2>[0-9]+)$",
+                "spec": {
+                    "answer1": "str",
+                    "answer2": "str"
+                }
+            }
+        });
+
+        let (result, _, _) = run_json(program, streaming(), initial_scope())?;
+        let mut m = ::std::collections::HashMap::new();
+        m.insert("answer1".into(), "999".into());
+        m.insert("answer2".into(), "888".into());
+        assert_eq!(result, PdlResult::Dict(m));
+        Ok(())
+    }
+
+    #[test]
+    fn regex_plain_2() -> Result<(), Box<dyn Error>> {
+        let program = json!({
+            "data": "aaa999bbb888",
+            "parser": {
+                "regex": "[^0-9]*(?P<answer1>[0-9]+)[^0-9]*(?P<answer2>[0-9]+)$",
+                "spec": {
+                    "answer1": "str",
+                }
+            }
+        });
+
+        let (result, _, _) = run_json(program, streaming(), initial_scope())?;
+        let mut m = ::std::collections::HashMap::new();
+        m.insert("answer1".into(), "999".into());
+        assert_eq!(result, PdlResult::Dict(m));
+        Ok(())
+    }
+
     #[test]
     fn bee_1() -> Result<(), Box<dyn Error>> {
         let program = crate::compile::beeai::compile("./tests/data/bee_1.py", false)?;

pdl-live-react/src-tauri/src/pdl/interpreter_tests.rs

@@ -0,0 +1,7 @@
+data: aaa999bbb888
+parser:
+  regex: '[^0-9]*(?P<answer1>[0-9]+)[^0-9]*(?P<answer2>[0-9]+)$'
+  mode: findall
+  spec:
+    answer1: str
+    answer2: str