Merge #927

927: Make embedders deseralize to correct type r=curquiza a=sanders41

# Pull Request

## Related issue
Fixes #926

## What does this PR do?
- Checks the model before deserializing to make sure the correct model is used.
- Updates test to catch any potential regressions.

## PR checklist
Please check if your PR fulfills the following requirements:
- [x] Does this PR fix an existing issue, or have you listed the changes applied in the PR description (and why they are needed)?
- [x] Have you read the contributing guidelines?
- [x] Have you made sure that the title is accurate and descriptive of the changes?

Thank you so much for contributing to Meilisearch!


Co-authored-by: Paul Sanders <[email protected]>

Author: meili-bors[bot]

meilisearch/index.py

@@ -10,7 +10,16 @@
 from meilisearch.config import Config
 from meilisearch.errors import version_error_hint_message
 from meilisearch.models.document import Document, DocumentsResults
-from meilisearch.models.index import Embedders, Faceting, IndexStats, Pagination, TypoTolerance
+from meilisearch.models.index import (
+    Embedders,
+    Faceting,
+    HuggingFaceEmbedder,
+    IndexStats,
+    OpenAiEmbedder,
+    Pagination,
+    TypoTolerance,
+    UserProvidedEmbedder,
+)
 from meilisearch.models.task import Task, TaskInfo, TaskResults
 from meilisearch.task import TaskHandler
 
@@ -865,7 +874,23 @@ def get_settings(self) -> Dict[str, Any]:
         MeilisearchApiError
             An error containing details about why Meilisearch can't process your request. Meilisearch error codes are described here: https://www.meilisearch.com/docs/reference/errors/error_codes#meilisearch-errors
         """
-        return self.http.get(f"{self.config.paths.index}/{self.uid}/{self.config.paths.setting}")
+        settings = self.http.get(
+            f"{self.config.paths.index}/{self.uid}/{self.config.paths.setting}"
+        )
+
+        if settings.get("embedders"):
+            embedders: dict[str, OpenAiEmbedder | HuggingFaceEmbedder | UserProvidedEmbedder] = {}
+            for k, v in settings["embedders"].items():
+                if v.get("source") == "openAi":
+                    embedders[k] = OpenAiEmbedder(**v)
+                elif v.get("source") == "huggingFace":
+                    embedders[k] = HuggingFaceEmbedder(**v)
+                else:
+                    embedders[k] = UserProvidedEmbedder(**v)
+
+            settings["embedders"] = embedders
+
+        return settings
 
     def update_settings(self, body: Mapping[str, Any]) -> TaskInfo:
         """Update settings of the index.
@@ -1777,7 +1802,17 @@ def get_embedders(self) -> Embedders | None:
         if not response:
             return None
 
-        return Embedders(embedders=response)
+        embedders: dict[str, OpenAiEmbedder | HuggingFaceEmbedder | UserProvidedEmbedder] = {}
+        for k, v in response.items():
+            print(v.get("source"))
+            if v.get("source") == "openAi":
+                embedders[k] = OpenAiEmbedder(**v)
+            elif v.get("source") == "huggingFace":
+                embedders[k] = HuggingFaceEmbedder(**v)
+            else:
+                embedders[k] = UserProvidedEmbedder(**v)
+
+        return Embedders(embedders=embedders)
 
     def update_embedders(self, body: Union[Mapping[str, Any], None]) -> TaskInfo:
         """Update embedders of the index.

tests/conftest.py

@@ -7,6 +7,7 @@
 
 import meilisearch
 from meilisearch.errors import MeilisearchApiError
+from meilisearch.models.index import HuggingFaceEmbedder, OpenAiEmbedder, UserProvidedEmbedder
 from tests import common
 
 
@@ -230,8 +231,7 @@ def enable_vector_search():
 @fixture
 def new_embedders():
     return {
-        "default": {
-            "source": "userProvided",
-            "dimensions": 1,
-        }
+        "default": UserProvidedEmbedder(dimensions=1).model_dump(by_alias=True),
+        "open_ai": OpenAiEmbedder().model_dump(by_alias=True),
+        "hugging_face": HuggingFaceEmbedder().model_dump(by_alias=True),
     }

tests/settings/test_settings.py

@@ -1,7 +1,17 @@
-NEW_SETTINGS = {
-    "rankingRules": ["typo", "words"],
-    "searchableAttributes": ["title", "overview"],
-}
+# pylint: disable=redefined-outer-name
+import pytest
+
+from meilisearch.models.index import HuggingFaceEmbedder, OpenAiEmbedder, UserProvidedEmbedder
+
+
+@pytest.fixture
+def new_settings(new_embedders):
+    return {
+        "rankingRules": ["typo", "words"],
+        "searchableAttributes": ["title", "overview"],
+        "embedders": new_embedders,
+    }
+
 
 DEFAULT_RANKING_RULES = ["words", "typo", "proximity", "attribute", "sort", "exactness"]
 
@@ -31,36 +41,41 @@ def test_get_settings_default(empty_index):
     assert response["synonyms"] == {}
 
 
-def test_update_settings(empty_index):
+@pytest.mark.usefixtures("enable_vector_search")
+def test_update_settings(new_settings, empty_index):
     """Tests updating some settings."""
     index = empty_index()
-    response = index.update_settings(NEW_SETTINGS)
+    response = index.update_settings(new_settings)
     update = index.wait_for_task(response.task_uid)
     assert update.status == "succeeded"
     response = index.get_settings()
-    for rule in NEW_SETTINGS["rankingRules"]:
+    for rule in new_settings["rankingRules"]:
         assert rule in response["rankingRules"]
     assert response["distinctAttribute"] is None
-    for attribute in NEW_SETTINGS["searchableAttributes"]:
+    for attribute in new_settings["searchableAttributes"]:
         assert attribute in response["searchableAttributes"]
     assert response["displayedAttributes"] == ["*"]
     assert response["stopWords"] == []
     assert response["synonyms"] == {}
+    assert isinstance(response["embedders"]["default"], UserProvidedEmbedder)
+    assert isinstance(response["embedders"]["open_ai"], OpenAiEmbedder)
+    assert isinstance(response["embedders"]["hugging_face"], HuggingFaceEmbedder)
 
 
-def test_reset_settings(empty_index):
+@pytest.mark.usefixtures("enable_vector_search")
+def test_reset_settings(new_settings, empty_index):
     """Tests resetting all the settings to their default value."""
     index = empty_index()
     # Update settings first
-    response = index.update_settings(NEW_SETTINGS)
+    response = index.update_settings(new_settings)
     update = index.wait_for_task(response.task_uid)
     assert update.status == "succeeded"
     # Check the settings have been correctly updated
     response = index.get_settings()
-    for rule in NEW_SETTINGS["rankingRules"]:
+    for rule in new_settings["rankingRules"]:
         assert rule in response["rankingRules"]
     assert response["distinctAttribute"] is None
-    for attribute in NEW_SETTINGS["searchableAttributes"]:
+    for attribute in new_settings["searchableAttributes"]:
         assert attribute in response["searchableAttributes"]
     assert response["displayedAttributes"] == ["*"]
     assert response["stopWords"] == []
@@ -80,3 +95,4 @@ def test_reset_settings(empty_index):
     assert response["searchableAttributes"] == ["*"]
     assert response["stopWords"] == []
     assert response["synonyms"] == {}
+    assert response.get("embedders") is None

tests/settings/test_settings_embedders.py

@@ -1,6 +1,7 @@
+# pylint: disable=redefined-outer-name
 import pytest
 
-from meilisearch.models.index import Embedders
+from meilisearch.models.index import HuggingFaceEmbedder, OpenAiEmbedder, UserProvidedEmbedder
 
 
 @pytest.mark.usefixtures("enable_vector_search")
@@ -19,7 +20,9 @@ def test_update_embedders_with_user_provided_source(new_embedders, empty_index):
     update = index.wait_for_task(response_update.task_uid)
     response_get = index.get_embedders()
     assert update.status == "succeeded"
-    assert response_get == Embedders(embedders=new_embedders)
+    assert isinstance(response_get.embedders["default"], UserProvidedEmbedder)
+    assert isinstance(response_get.embedders["open_ai"], OpenAiEmbedder)
+    assert isinstance(response_get.embedders["hugging_face"], HuggingFaceEmbedder)
 
 
 @pytest.mark.usefixtures("enable_vector_search")
@@ -30,15 +33,19 @@ def test_reset_embedders(new_embedders, empty_index):
     # Update the settings
     response_update = index.update_embedders(new_embedders)
     update1 = index.wait_for_task(response_update.task_uid)
+    assert update1.status == "succeeded"
     # Get the setting after update
     response_get = index.get_embedders()
+    assert isinstance(response_get.embedders["default"], UserProvidedEmbedder)
+    assert isinstance(response_get.embedders["open_ai"], OpenAiEmbedder)
+    assert isinstance(response_get.embedders["hugging_face"], HuggingFaceEmbedder)
     # Reset the setting
     response_reset = index.reset_embedders()
     update2 = index.wait_for_task(response_reset.task_uid)
     # Get the setting after reset
-    response_last = index.get_embedders()
-
-    assert update1.status == "succeeded"
-    assert response_get == Embedders(embedders=new_embedders)
     assert update2.status == "succeeded"
+    assert isinstance(response_get.embedders["default"], UserProvidedEmbedder)
+    assert isinstance(response_get.embedders["open_ai"], OpenAiEmbedder)
+    assert isinstance(response_get.embedders["hugging_face"], HuggingFaceEmbedder)
+    response_last = index.get_embedders()
     assert response_last is None