feat: update beeai compiler to support compiling directly from python source

This refactors the pip install logic (small changes) to allow for more flexible specification of which "internal" requirements.txt to use --- internal in the sense that it is specified in our tauri.conf.json to be embedded into the app.

Signed-off-by: Nick Mitchell <[email protected]>

Author: starpit

pdl-live-react/beeai-requirements.txt

@@ -0,0 +1,2 @@
+-e git+https://github.com/starpit/bee-agent-framework.git@nick-meta-combo#egg=beeai_framework&subdirectory=python
+#beeai_framework==0.1

pdl-live-react/package.json

@@ -14,8 +14,9 @@
     "tauri": "tauri",
     "test:quality": "concurrently -n 'lint,types,formatting' 'npm run lint' 'tsc --build --noEmit' \"prettier --check 'tests/**/*.ts'  'src/**/*.{ts,tsx,css}'\"",
     "test:ui": "playwright install --with-deps && playwright test",
+    "test:bee": "./src-tauri/target/debug/pdl compile beeai demos/beeai.py -g --output - | jq", 
     "types": "(cd .. && python -m src.pdl.pdl --schema > src/pdl/pdl-schema.json) && json2ts ../src/pdl/pdl-schema.json src/pdl_ast.d.ts --unreachableDefinitions && npm run format",
-    "test": "concurrently -n 'quality,playwright' 'npm run test:quality' 'npm run test:ui'",
+    "test": "concurrently -n 'quality,playwright,bee' 'npm run test:quality' 'npm run test:ui' 'npm run test:bee'",
     "pdl": "./src-tauri/target/debug/pdl",
     "view": "npm run pdl view",
     "start": "npm run tauri dev"

pdl-live-react/src-tauri/src/cli/run.rs

@@ -3,7 +3,7 @@ use duct::cmd;
 use futures::executor::block_on;
 use yaml_rust2::yaml::LoadError;
 
-use crate::interpreter::pip::pip_install_interpreter_if_needed;
+use crate::interpreter::pip::pip_install_internal_if_needed;
 use crate::interpreter::pull::pull_if_needed;
 
 #[cfg(desktop)]
@@ -21,7 +21,8 @@ pub fn run_pdl_program(
 
     // async the model pull and pip installs
     let pull_future = pull_if_needed(&source_file_path);
-    let bin_path_future = pip_install_interpreter_if_needed(app_handle);
+    let bin_path_future =
+        pip_install_internal_if_needed(app_handle, &"interpreter/requirements.txt");
 
     // wait for any model pulls to finish
     block_on(pull_future).map_err(|e| match e {

pdl-live-react/src-tauri/src/cli/setup.rs

@@ -50,7 +50,12 @@ pub fn cli(app: &mut tauri::App) -> Result<(), Box<dyn ::std::error::Error>> {
                                 value: Value::Bool(debug),
                                 ..
                             }),
-                        ) => compile::beeai::compile(source_file_path, output_file_path, debug),
+                        ) => compile::beeai::compile(
+                            app.handle().clone(),
+                            source_file_path,
+                            output_file_path,
+                            debug,
+                        ),
                         _ => Err(Box::from("Invalid compile subcommand")),
                     }
                 }

pdl-live-react/src-tauri/src/compile/beeai.rs

@@ -1,12 +1,18 @@
 use ::std::collections::HashMap;
 use ::std::error::Error;
+use ::std::ffi::OsStr;
 use ::std::fs::File;
 use ::std::io::BufReader;
+use ::std::path::{Path, PathBuf};
 
+use duct::cmd;
+use futures::executor::block_on;
 use serde::Deserialize;
 use serde_json::{from_reader, json, to_string, Value};
+use tempfile::Builder;
 
 use crate::interpreter::ast::{PdlBaseType, PdlBlock, PdlOptionalType, PdlParser, PdlType};
+use crate::interpreter::pip::pip_install_internal_if_needed;
 
 macro_rules! zip {
     ($x: expr) => ($x);
@@ -276,18 +282,70 @@ fn tool_imports(object: &String) -> (&str, &str) {
     }
 }
 
+fn python_source_to_json(
+    app_handle: tauri::AppHandle,
+    source_file_path: &String,
+    debug: &bool,
+) -> Result<PathBuf, Box<dyn Error>> {
+    if *debug {
+        eprintln!("Compiling from Python source");
+    }
+    let bin_path = block_on(pip_install_internal_if_needed(
+        app_handle,
+        &"interpreter/beeai-requirements.txt",
+    ))?;
+
+    let dry_run_file_path = Builder::new()
+        .prefix(&"pdl-bee")
+        .suffix(".json")
+        .tempfile()?;
+    let (_f, dry_run_file) = dry_run_file_path.keep()?;
+
+    let args = vec![source_file_path];
+
+    cmd(bin_path.join("python"), &args)
+        .env("DRY_RUN", "True")
+        .env("DRY_RUN_FILE", &dry_run_file)
+        .stdout_null()
+        .run()?;
+
+    if *debug {
+        eprintln!(
+            "Finished generating BeeAi JSON snapshot to {:?}",
+            &dry_run_file
+        )
+    }
+    Ok(dry_run_file)
+}
+
 pub fn compile(
+    app_handle: tauri::AppHandle,
     source_file_path: &String,
     output_path: &String,
     debug: &bool,
 ) -> Result<(), Box<dyn Error>> {
-    println!("Compiling beeai {} to {}", source_file_path, output_path);
+    if *debug {
+        eprintln!("Compiling beeai {} to {}", source_file_path, output_path);
+    }
 
-    // Open the file in read-only mode with buffer.
-    let file = File::open(source_file_path)?;
-    let reader = BufReader::new(file);
+    let file = match Path::new(source_file_path)
+        .extension()
+        .and_then(OsStr::to_str)
+    {
+        Some("py") => {
+            let json_snapshot_file = python_source_to_json(app_handle, source_file_path, debug)?;
+            File::open(json_snapshot_file)
+        }
+        _ => {
+            if *debug {
+                eprintln!("Compiling from JSON snapshot");
+            }
+            File::open(source_file_path)
+        }
+    }?;
 
     // Read the JSON contents of the file as an instance of `User`.
+    let reader = BufReader::new(file);
     let bee: BeeAiProgram = from_reader(reader)?;
 
     let inputs: Vec<PdlBlock> = bee

pdl-live-react/src-tauri/src/interpreter/pip.rs

@@ -37,13 +37,14 @@ pub async fn pip_install_if_needed(
 }
 
 #[cfg(desktop)]
-pub async fn pip_install_interpreter_if_needed(
+pub async fn pip_install_internal_if_needed(
     app_handle: tauri::AppHandle,
+    requirements: &str,
 ) -> Result<PathBuf, tauri::Error> {
     // the interpreter requirements.txt
     let requirements_path = app_handle
         .path()
-        .resolve("interpreter/requirements.txt", BaseDirectory::Resource)?;
+        .resolve(requirements, BaseDirectory::Resource)?;
 
     let cache_path = app_handle.path().cache_dir()?.join("pdl");
 

pdl-live-react/src-tauri/tauri.conf.json

@@ -88,7 +88,8 @@
     "active": true,
     "targets": "all",
     "resources": {
-      "../requirements.txt": "interpreter/requirements.txt"
+      "../requirements.txt": "interpreter/requirements.txt",
+      "../beeai-requirements.txt": "interpreter/beeai-requirements.txt"
     },
     "icon": [
       "icons/32x32.png",