Restructure NMR block to avoid saving all data in db

Author: ml-evs

pydatalab/src/pydatalab/apps/nmr/blocks.py

@@ -23,6 +23,7 @@ class NMRBlock(DataBlock):
     description = "A data block for loading and visualizing 1D NMR data from Bruker projects or JCAMP-DX files."
 
     accepted_file_extensions = BRUKER_FILE_EXTENSIONS + JCAMP_FILE_EXTENSIONS
+    processed_data: dict | None = None
     defaults = {"process number": 1}
     _supports_collections = False
 
@@ -34,7 +35,7 @@ def read_bruker_nmr_data(
         self,
         filename: str | Path | None = None,
         file_info: dict | None = None,
-    ) -> tuple[dict | None, dict] | None:
+    ) -> tuple[dict | None, dict]:
         """Loads a Bruker project from the passed or attached zip file
         and parses it into a serialized dataframe and metadata dictionary.
 
@@ -119,7 +120,6 @@ def read_bruker_nmr_data(
         metadata["title"] = topspin_title
 
         self.data["metadata"] = metadata
-        self.data["processed_data"] = serialized_df
 
         return serialized_df, metadata
 
@@ -202,24 +202,26 @@ def read_jcamp_nmr_data(
             pass
 
         serialized_df = df.to_dict() if (df is not None) else None
-
-        self.data["processed_data"] = serialized_df
         self.data["metadata"] = metadata
 
+        return serialized_df, metadata
+
     def load_nmr_data(self, file_info: dict):
         location, name, ext = self._extract_file_info(file_info=file_info)
 
         if ext == ".zip":
-            self.read_bruker_nmr_data(file_info=file_info)
+            df, metadata = self.read_bruker_nmr_data(file_info=file_info)
 
         elif ext in (".jdx", ".dx"):
-            self.read_jcamp_nmr_data(file_info=file_info)
+            df, metadata = self.read_jcamp_nmr_data(file_info=file_info)
 
         else:
             raise RuntimeError(
                 f"Unsupported file extension for NMR reader: {ext} (must be one of {self.accepted_file_extensions})"
             )
 
+        return df
+
     def generate_nmr_plot(self, parse: bool = True):
         """Generate an NMR plot and store processed data for the
         data files attached to this block.
@@ -232,7 +234,7 @@ def generate_nmr_plot(self, parse: bool = True):
             file_info = get_file_info_by_id(self.data["file_id"], update_if_live=True)
             name, ext = os.path.splitext(file_info["name"])
 
-            self.load_nmr_data(file_info)
+            self.processed_data = self.load_nmr_data(file_info)
 
         processed_data_shape = self.data.get("metadata", {}).get("processed_data_shape", [])
         if not processed_data_shape or len(processed_data_shape) > 1:
@@ -241,14 +243,14 @@ def generate_nmr_plot(self, parse: bool = True):
             )
             return
 
-        if "processed_data" not in self.data or not self.data["processed_data"]:
+        if not self.processed_data:
             self.data["bokeh_plot_data"] = None
             warnings.warn(
                 "No compatible processed data available for plotting, only metadata will be displayed."
             )
             return
 
-        df = pd.DataFrame(self.data["processed_data"])
+        df = pd.DataFrame(self.processed_data)
         df["normalized intensity"] = df.intensity / df.intensity.max()
 
         self.data["bokeh_plot_data"] = self.make_nmr_plot(df, self.data["metadata"])

pydatalab/tests/apps/test_nmr.py

@@ -91,28 +91,30 @@ def test_nmr_block(
     nmr_1d_solution_path, nmr_1d_solution_path_renamed, nmr_1d_solid_path, nmr_2d_matpass_path
 ):
     block = NMRBlock(item_id="nmr-block")
-    block.read_bruker_nmr_data(nmr_1d_solid_path)
+    block.processed_data, block.data["metadata"] = block.read_bruker_nmr_data(nmr_1d_solid_path)
     assert block.data["metadata"]["topspin_title"].split("\n")[0] == "7Li 40 kHz 40 C hahn-echo"
     block.generate_nmr_plot(parse=False)
     plot = block.data["bokeh_plot_data"]
     assert plot is not None
 
     block = NMRBlock(item_id="nmr-block")
-    block.read_bruker_nmr_data(nmr_1d_solution_path)
+    block.processed_data, block.data["metadata"] = block.read_bruker_nmr_data(nmr_1d_solution_path)
     assert block.data["metadata"]["topspin_title"].split("\n")[0] == "31P reference, 85% H3PO4"
     block.generate_nmr_plot(parse=False)
     plot = block.data["bokeh_plot_data"]
     assert plot is not None
 
     block = NMRBlock(item_id="nmr-block")
-    block.read_bruker_nmr_data(nmr_1d_solution_path_renamed)
+    block.processed_data, block.data["metadata"] = block.read_bruker_nmr_data(
+        nmr_1d_solution_path_renamed
+    )
     assert block.data["metadata"]["topspin_title"].split("\n")[0] == "31P reference, 85% H3PO4"
     block.generate_nmr_plot(parse=False)
     plot = block.data["bokeh_plot_data"]
     assert plot is not None
 
     block = NMRBlock(item_id="nmr-block")
-    block.read_bruker_nmr_data(nmr_2d_matpass_path)
+    block.processed_data, block.data["metadata"] = block.read_bruker_nmr_data(nmr_2d_matpass_path)
     assert block.data["metadata"]["topspin_title"].split("\n")[0] == "7Li 40kHz 40 C MATPASS"
     # catch warning about processed data
     with pytest.warns(UserWarning, match="Only metadata"):