Feat: support Markdown in operator specs (#2115)

* Remove operators module files after successfully starting the server

* Try converting MD to RST on-the-fly with pypandoc (way too slow)

* Directly include the MD descriptions

* Try without black formatting

* Add MD but with indents

* Properly expose Markdown descriptions in code generation

* Fix mustache spacings

* Fix mustache

* Improve type hinting in operator.mustache

* Use a dedicated indented string for pin docstrings

* Remove superfluous newlines after the operator description in the docstring

* Fix mustache for pin description in main operator docstring

* Use black to reformat generated modules (does not scramble descriptions)

* Code quality fix to mustache

* Manage operator description separately

* Print timings of generation

* try pypandoc

* Add Markdown2RstTranslator in ansys.dpf.core.operators.build and a test

* Move Markdown2RstTranslator to ansys.dpf.core.operators.translator

* Add pypandoc to requirements_test.txt

* Add pypandoc_binary to requirements_test.txt

Author: PProfizi

.ci/code_generation.py

@@ -8,6 +8,9 @@
 from ansys.dpf import core
 from ansys.dpf.core.operators import build
 
+core.set_default_server_context(core.AvailableServerContexts.premium)
+core.start_local_server(config=core.AvailableServerConfigs.LegacyGrpcServer)
+
 local_dir = Path(__file__).parent
 TARGET_PATH = local_dir.parent / "src" / "ansys" / "dpf" / "core" / "operators"
 files = TARGET_PATH.glob("*")
@@ -26,7 +29,4 @@
     except:
         pass
 
-core.set_default_server_context(core.AvailableServerContexts.premium)
-core.start_local_server(config=core.AvailableServerConfigs.LegacyGrpcServer)
-
 build.build_operators()

requirements/requirements_test.txt

@@ -3,6 +3,7 @@ coverage==7.6.12
 graphviz==0.20.1
 imageio==2.36.1
 imageio-ffmpeg==0.6.0
+pypandoc_binary==1.15
 pytest==8.3.4
 pytest-cov==6.0.0
 pytest-order==1.3.0

src/ansys/dpf/core/operators/build.py

@@ -3,6 +3,7 @@
 import os
 from datetime import datetime
 from textwrap import wrap
+import time
 
 import black
 import chevron
@@ -11,18 +12,15 @@
 from ansys.dpf.core.dpf_operator import available_operator_names
 from ansys.dpf.core.outputs import _make_printable_type
 from ansys.dpf.core.mapping_types import map_types_to_python
+from ansys.dpf.core.operators.translator import Markdown2RstTranslator
 
 
-def build_docstring(specification):
+def build_docstring(specification_description):
     """Used to generate class docstrings."""
     docstring = ""
-    if specification.description:
-        docstring += "\n".join(
-            wrap(specification.description, subsequent_indent="    ")
-        )
-        docstring += "\n\n"
-    docstring = docstring.rstrip()
-    return docstring.replace('"', "'")
+    if specification_description:
+        docstring += specification_description.replace("\n", "\n    ")
+    return docstring
 
 
 def map_types(cpp_types):
@@ -83,6 +81,9 @@ def build_pin_data(pins, output=False):
         if multiple_types and output:
             printable_type_names = [_make_printable_type(name) for name in type_names]
 
+        document = specification.document
+        document_pin_docstring = document.replace("\n", "\n        ")
+
         pin_data = {
             "id": id,
             "name": pin_name,
@@ -97,13 +98,8 @@ def build_pin_data(pins, output=False):
             "main_type": main_type,
             "built_in_main_type": main_type in built_in_types,
             "optional": specification.optional,
-            "document": "\n".join(
-                wrap(
-                    specification.document.capitalize().lstrip(' '),
-                    subsequent_indent="        ",
-                    width=45,
-                )
-            ),
+            "document": document,
+            "document_pin_docstring": document_pin_docstring,
             "ellipsis": 0 if specification.ellipsis else -1,
         }
 
@@ -125,7 +121,7 @@ def build_pin_data(pins, output=False):
 
 
 def build_operator(
-    specification, operator_name, class_name, capital_class_name, category
+    specification, operator_name, class_name, capital_class_name, category, specification_description
 ):
 
     input_pins = []
@@ -137,11 +133,7 @@ def build_operator(
         output_pins = build_pin_data(specification.outputs, output=True)
     multiple_output_types = any(pin["multiple_types"] for pin in output_pins)
 
-    docstring = build_docstring(specification)
-
-    specification_description = "\n".join(
-        wrap(specification.description, subsequent_indent="            ")
-    )
+    docstring = build_docstring(specification_description)
 
     date_and_time = datetime.now().strftime("%m/%d/%Y, %H:%M:%S")
 
@@ -173,6 +165,7 @@ def build_operator(
 
 def build_operators():
     print(f"Generating operators for server {dpf.SERVER.version}")
+    time_0 = time.time()
 
     this_path = os.path.dirname(os.path.abspath(__file__))
 
@@ -192,6 +185,9 @@ def build_operators():
         "invert", "invert_fc",
     ]
     categories = set()
+
+    translator = Markdown2RstTranslator()
+
     for operator_name in available_operators:
         if succeeded == done + 100:
             done += 100
@@ -226,6 +222,9 @@ def build_operators():
         # Get python class name from scripting name
         capital_class_name = common._snake_to_camel_case(scripting_name)
 
+        # Convert Markdown descriptions to RST
+        specification_description = translator.convert(specification.description)
+
         # Write to operator file
         operator_file = os.path.join(category_path, scripting_name + ".py")
         with open(operator_file, "wb") as f:
@@ -236,6 +235,7 @@ def build_operators():
                     scripting_name,
                     capital_class_name,
                     category,
+                    specification_description,
                 )
                 exec(operator_str, globals())
                 f.write(operator_str.encode())
@@ -269,6 +269,7 @@ def build_operators():
 
     if succeeded == len(available_operators) - hidden:
         print("Success")
+        print(f"Took {time.time() - time_0}")
         exit(0)
     else:
         print("Terminated with errors")

src/ansys/dpf/core/operators/operator.mustache

@@ -4,6 +4,8 @@
 Autogenerated DPF operator classes.
 """
 
+from __future__ import annotations
+
 from warnings import warn
 from ansys.dpf.core.dpf_operator import Operator
 from ansys.dpf.core.inputs import Input, _Inputs
@@ -17,36 +19,39 @@ from ansys.dpf.core.outputs import _Outputs
 from ansys.dpf.core.outputs import _modify_output_spec_with_one_type
 {{/multiple_output_types}}
 from ansys.dpf.core.operators.specification import PinSpecification, Specification
+from ansys.dpf.core.config import Config
+from ansys.dpf.core.server_types import AnyServerType
+
 
 class {{class_name}}(Operator):
-    """{{{docstring}}}
+    r"""{{{docstring}}}
 
     Parameters
     ----------
     {{#input_pins}}
     {{#optional}}
-    {{name}} :{{#types_for_docstring}} {{types_for_docstring}},{{/types_for_docstring}} optional
+    {{name}}:{{#types_for_docstring}} {{types_for_docstring}},{{/types_for_docstring}} optional
     {{/optional}}
     {{^optional}}
-    {{name}} :{{#types_for_docstring}} {{types_for_docstring}}{{/types_for_docstring}}
+    {{name}}:{{#types_for_docstring}} {{types_for_docstring}}{{/types_for_docstring}}
     {{/optional}}
-    {{#document}}
-        {{{document}}}
-    {{/document}}
+    {{#document_pin_docstring}}
+        {{{document_pin_docstring}}}
+    {{/document_pin_docstring}}
     {{/input_pins}}
 
     Returns
     -------
     {{#output_pins}}
     {{#optional}}
-    {{name}} :{{#types_for_docstring}} {{types_for_docstring}},{{/types_for_docstring}} optional
+    {{name}}:{{#types_for_docstring}} {{types_for_docstring}},{{/types_for_docstring}} optional
     {{/optional}}
     {{^optional}}
-    {{name}} :{{#types_for_docstring}} {{types_for_docstring}}{{/types_for_docstring}}
+    {{name}}:{{#types_for_docstring}} {{types_for_docstring}}{{/types_for_docstring}}
     {{/optional}}
-    {{#document}}
-        {{{document}}}
-    {{/document}}
+    {{#document_pin_docstring}}
+        {{{document_pin_docstring}}}
+    {{/document_pin_docstring}}
     {{/output_pins}}
 
     Examples
@@ -87,8 +92,8 @@ class {{class_name}}(Operator):
         {{/input_pins}}
 
     @staticmethod
-    def _spec():
-        description = """{{specification_description}}"""
+    def _spec() -> Specification:
+        description = r"""{{{specification_description}}}"""
         spec = Specification(
             description=description,
             map_input_pin_spec={
@@ -102,7 +107,7 @@ class {{class_name}}(Operator):
                     type_names=["any"],
                     {{/has_types}}
                     optional={{optional}},
-                    document="""{{{document}}}""",
+                    document=r"""{{{document}}}""",
                     {{#has_derived_class}}
                     name_derived_class=["{{{derived_type_name}}}"],
                     {{/has_derived_class}}
@@ -117,7 +122,7 @@ class {{class_name}}(Operator):
                     type_names={{{types}}},
                     {{/has_types}}
                     optional={{optional}},
-                    document="""{{{document}}}""",
+                    document=r"""{{{document}}}""",
                     {{#has_derived_class}}
                     name_derived_class=["{{{derived_type_name}}}"],
                     {{/has_derived_class}}
@@ -128,7 +133,7 @@ class {{class_name}}(Operator):
         return spec
 
     @staticmethod
-    def default_config(server=None):
+    def default_config(server: AnyServerType = None) -> Config:
         """Returns the default config of the operator.
 
         This config can then be changed to the user needs and be used to
@@ -137,32 +142,40 @@ class {{class_name}}(Operator):
 
         Parameters
         ----------
-        server : server.DPFServer, optional
+        server:
             Server with channel connected to the remote or local instance. When
             ``None``, attempts to use the global server.
+
+        Returns
+        -------
+        config:
+            A new Config instance equivalent to the default config for this operator.
         """
         return Operator.default_config(name="{{operator_name}}", server=server)
 
     @property
-    def inputs(self):
+    def inputs(self) -> Inputs{{capital_class_name}}:
         """Enables to connect inputs to the operator
 
         Returns
         --------
-        inputs : Inputs{{capital_class_name}}
+        inputs:
+            An instance of Inputs{{capital_class_name}}.
         """
         return super().inputs
 
     @property
-    def outputs(self):
+    def outputs(self) -> Outputs{{capital_class_name}}:
         """Enables to get outputs of the operator by evaluating it
 
         Returns
         --------
-        outputs : Outputs{{capital_class_name}}
+        outputs:
+            An instance of Outputs{{capital_class_name}}.
         """
         return super().outputs
 
+
 class Inputs{{capital_class_name}}(_Inputs):
     """Intermediate class used to connect user inputs to
     {{class_name}} operator.
@@ -183,19 +196,20 @@ class Inputs{{capital_class_name}}(_Inputs):
         self._{{name}} = Input({{class_name}}._spec().input_pin({{id}}), {{id}}, op, {{ellipsis}})
         self._inputs.append(self._{{name}})
         {{/input_pins}}
-
     {{#input_pins}}
+
     @property
-    def {{name}}(self):
-        """Allows to connect {{name}} input to the operator.
-        {{#document}}
+    def {{name}}(self) -> Input:
+        r"""Allows to connect {{name}} input to the operator.
+        {{#document_pin_docstring}}
 
-        {{{document}}}
-        {{/document}}
+        {{{document_pin_docstring}}}
+        {{/document_pin_docstring}}
 
-        Parameters
-        ----------
-        my_{{name}} :{{#types_for_docstring}} {{types_for_docstring}}{{/types_for_docstring}}
+        Returns
+        -------
+        input:
+            An Input instance for this pin.
 
         Examples
         --------
@@ -206,8 +220,9 @@ class Inputs{{capital_class_name}}(_Inputs):
         >>> op.inputs.{{name}}(my_{{name}})
         """
         return self._{{name}}
-
     {{/input_pins}}
+
+
 class Outputs{{capital_class_name}}(_Outputs):
     """Intermediate class used to get outputs from
     {{class_name}} operator.
@@ -236,27 +251,29 @@ class Outputs{{capital_class_name}}(_Outputs):
         self._outputs.append(self._{{name}})
         {{/multiple_types}}
         {{/output_pins}}
-    {{#output_pins}}{{^multiple_types}}
 
+{{#output_pins}}
+{{^multiple_types}}
     @property
-    def {{name}}(self):
-        """Allows to get {{name}} output of the operator
+    def {{name}}(self) -> Output:
+        r"""Allows to get {{name}} output of the operator
+        {{#document_pin_docstring}}
+
+        {{{document_pin_docstring}}}
+        {{/document_pin_docstring}}
 
         Returns
-        ----------
-        {{#derived_type_name}}
-        my_{{name}} : {{derived_type_name}}
-        {{/derived_type_name}}
-        {{^derived_type_name}}
-        my_{{name}} :{{#types_for_docstring}} {{types_for_docstring}}{{/types_for_docstring}}
-        {{/derived_type_name}}
+        -------
+        output:
+            An Output instance for this pin.
 
         Examples
         --------
         >>> from ansys.dpf import core as dpf
         >>> op = dpf.operators.{{category}}.{{class_name}}()
-        >>> # Connect inputs : op.inputs. ...
+        >>> # Get the output from op.outputs. ...
         >>> result_{{name}} = op.outputs.{{name}}()
-        """ # noqa: E501
+        """
         return self._{{name}}
-    {{/multiple_types}}{{/output_pins}}
+{{/multiple_types}}
+{{/output_pins}}