Rename processed_data to computed

Author: ml-evs

pydatalab/schemas/cell.json

@@ -239,8 +239,8 @@
           "title": "Bokeh Plot Data",
           "type": "object"
         },
-        "processed_data": {
-          "title": "Processed Data",
+        "computed": {
+          "title": "Computed",
           "type": "object"
         },
         "metadata": {

pydatalab/schemas/equipment.json

@@ -203,8 +203,8 @@
           "title": "Bokeh Plot Data",
           "type": "object"
         },
-        "processed_data": {
-          "title": "Processed Data",
+        "computed": {
+          "title": "Computed",
           "type": "object"
         },
         "metadata": {

pydatalab/schemas/sample.json

@@ -292,8 +292,8 @@
           "title": "Bokeh Plot Data",
           "type": "object"
         },
-        "processed_data": {
-          "title": "Processed Data",
+        "computed": {
+          "title": "Computed",
           "type": "object"
         },
         "metadata": {

pydatalab/schemas/startingmaterial.json

@@ -345,8 +345,8 @@
           "title": "Bokeh Plot Data",
           "type": "object"
         },
-        "processed_data": {
-          "title": "Processed Data",
+        "computed": {
+          "title": "Computed",
           "type": "object"
         },
         "metadata": {

pydatalab/src/pydatalab/apps/chat/blocks.py

@@ -208,7 +208,7 @@ def _prepare_item_json_for_chat(self, item_id: str, item_data: dict | None = Non
             str(file["immutable_id"]): file["name"] for file in item_data.get("files", [])
         }
 
-        big_data_keys = ["bokeh_plot_data", "b64_encoded_image"]
+        big_data_keys = ["bokeh_plot_data", "b64_encoded_image", "computed"]
         for block in item_data.get("blocks_obj", {}).values():
             block_fields_to_remove = ["item_id", "block_id", "collection_id"] + big_data_keys
             [block.pop(field, None) for field in block_fields_to_remove]
@@ -218,13 +218,12 @@ def _prepare_item_json_for_chat(self, item_id: str, item_data: dict | None = Non
                 "acquisition_parameters",
                 "carrier_offset_Hz",
                 "nscans",
-                "processed_data",
                 "processed_data_shape",
                 "processing_parameters",
                 "pulse_program",
                 "selected_process",
             ]
-            [block.pop(field, None) for field in NMR_fields_to_remove]
+            [block["metadata"].pop(field, None) for field in NMR_fields_to_remove]
 
             # replace file_id with the actual filename
             file_id = block.pop("file_id", None)

pydatalab/src/pydatalab/apps/xrd/blocks.py

@@ -262,8 +262,8 @@ def generate_xrd_plot(self, filenames: list[str | Path] | None = None) -> None:
                 pattern_df["normalized intensity (staggered)"] += ind
                 pattern_dfs.append(pattern_df)
 
-            self.data["processed_data"] = {}
-            self.data["processed_data"]["peak_data"] = peak_information
+            self.data["computed"] = {}
+            self.data["computed"]["peak_data"] = peak_information
 
         elif filenames is None:
             file_info = get_file_info_by_id(self.data["file_id"], update_if_live=True)
@@ -285,11 +285,9 @@ def generate_xrd_plot(self, filenames: list[str | Path] | None = None) -> None:
                 f"{self.data.get('wavelength', self.defaults['wavelength'])} Å"
             )
             peak_model = PeakInformation(**peak_data)
-            if "processed_data" not in self.data:
-                self.data["processed_data"] = {"peak_data": {}}
-            self.data["processed_data"]["peak_data"][str(file_info["immutable_id"])] = (
-                peak_model.dict()
-            )
+            if "computed" not in self.data:
+                self.data["computed"] = {"peak_data": {}}
+            self.data["computed"]["peak_data"][str(file_info["immutable_id"])] = peak_model.dict()
             pattern_dfs = [pattern_df]
 
         else:
@@ -306,9 +304,9 @@ def generate_xrd_plot(self, filenames: list[str | Path] | None = None) -> None:
                 pattern_dfs.append(pattern_df)
 
                 peak_model = PeakInformation(**peak_data)
-                if "processed_data" not in self.data:
-                    self.data["processed_data"] = {"peak_data": {}}
-                self.data["processed_data"]["peak_data"][f] = peak_model.dict()
+                if "computed" not in self.data:
+                    self.data["computed"] = {"peak_data": {}}
+                self.data["computed"]["peak_data"][f] = peak_model.dict()
                 pattern_dfs = [pattern_df]
 
         if pattern_dfs:

pydatalab/src/pydatalab/blocks/base.py

@@ -328,7 +328,7 @@ def update_from_web(self, data: dict):
         )
         self.data.update(
             self.block_db_model(**data).dict(
-                exclude={"processed_data", "metadata", "bokeh_plot_data", "b64_encoded_image"},
+                exclude={"computed", "metadata", "bokeh_plot_data", "b64_encoded_image"},
                 exclude_unset=True,
             )
         )

pydatalab/src/pydatalab/models/blocks.py

@@ -47,8 +47,8 @@ class DataBlockResponse(BaseModel):
     bokeh_plot_data: dict | None
     """A JSON-encoded string containing the Bokeh plot data, if any."""
 
-    processed_data: dict | None = None
-    """Any processed data associated with the block, small enough to store and filter directly in the database,
+    computed: dict | None = None
+    """Any processed or computed data associated with the block, small enough to store and filter directly in the database,
     i.e., strings or a few hundred numbers not exceeding 16KB in size.
     Examples could include peak positions, and widths, but not the full spectrum.
     """

pydatalab/tests/server/test_blocks.py

@@ -258,9 +258,9 @@ def test_xrd_block_lifecycle(admin_client, default_sample_dict):
 
     web_block = response.json["new_block_data"]
     assert "bokeh_plot_data" in web_block
-    assert "processed_data" in web_block
+    assert "computed" in web_block
     assert web_block["wavelength"] == 2.0
-    assert "peak_data" in web_block["processed_data"]
+    assert "peak_data" in web_block["computed"]
     assert "file_id" in web_block
     assert web_block["file_id"] == file_id
     assert web_block.get("errors") is None
@@ -269,7 +269,7 @@ def test_xrd_block_lifecycle(admin_client, default_sample_dict):
     db = block.to_db()
     # 'computed' keys should be dropped when loading from web
     assert "bokeh_plot_data" not in db
-    assert "processed_data" not in db
+    assert "computed" not in db
     assert db["wavelength"] == 2.0
 
     # But they should still be in the database
@@ -280,8 +280,8 @@ def test_xrd_block_lifecycle(admin_client, default_sample_dict):
     assert response.json["status"] == "success"
     assert "blocks_obj" in item_data
     block = item_data["blocks_obj"][block_id]
-    assert "processed_data" in block
-    assert "peak_data" in block["processed_data"]
+    assert "computed" in block
+    assert "peak_data" in block["computed"]
     assert block["wavelength"] == 2.0
 
 
@@ -461,7 +461,7 @@ def test_create_sample_with_example_files(
         assert response.json["new_block_data"]["bokeh_plot_data"] is not None
 
     if block_type == "xrd":
-        assert response.json["new_block_data"]["processed_data"]["peak_data"] is not None
+        assert response.json["new_block_data"]["computed"]["peak_data"] is not None
 
     response = admin_client.get(f"/get-item-data/{sample_id}")
     assert response.status_code == 200
@@ -472,4 +472,4 @@ def test_create_sample_with_example_files(
 
     if block_type == "xrd":
         doc = database.items.find_one({"item_id": sample_id}, projection={"blocks_obj": 1})
-        assert doc["blocks_obj"][block_id]["processed_data"]["peak_data"] is not None
+        assert doc["blocks_obj"][block_id]["computed"]["peak_data"] is not None

webapp/src/server_fetch_utils.js

@@ -506,7 +506,8 @@ export async function updateBlockFromServer(
   // - Will strip known "large data" keys, even if not formalised, e.g., bokeh_plot_data.
   //
   delete block_data.bokeh_plot_data;
-  delete block_data.processed_data;
+  delete block_data.b64_encoded_images;
+  delete block_data.computed;
   delete block_data.metadata;
 
   store.commit("setBlockUpdating", block_id);
@@ -581,8 +582,9 @@ export function saveItem(item_id) {
     for (const block_id in item_data.blocks_obj) {
       const block = item_data.blocks_obj[block_id];
       block.bokeh_plot_data = null;
-      block.processed_data = null;
+      block.computed = null;
       block.metadata = null;
+      block.b64_encoded_images = null;
     }
   }