From 17259d3a5d4acdb611ce35e7bb3c79dd414ed403 Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Mon, 12 Feb 2024 13:22:30 -0500 Subject: [PATCH 1/3] Make embedders deseralize to correct type --- meilisearch/index.py | 39 +++++++++++++++++++++-- tests/conftest.py | 8 ++--- tests/settings/test_settings.py | 39 ++++++++++++++++------- tests/settings/test_settings_embedders.py | 23 +++++++++---- 4 files changed, 84 insertions(+), 25 deletions(-) diff --git a/meilisearch/index.py b/meilisearch/index.py index ddb68ab4..2033ed41 100644 --- a/meilisearch/index.py +++ b/meilisearch/index.py @@ -10,7 +10,16 @@ from meilisearch.config import Config from meilisearch.errors import version_error_hint_message from meilisearch.models.document import Document, DocumentsResults -from meilisearch.models.index import Embedders, Faceting, IndexStats, Pagination, TypoTolerance +from meilisearch.models.index import ( + Embedders, + Faceting, + HuggingFaceEmbedder, + IndexStats, + OpenAiEmbedder, + Pagination, + TypoTolerance, + UserProvidedEmbedder, +) from meilisearch.models.task import Task, TaskInfo, TaskResults from meilisearch.task import TaskHandler @@ -865,7 +874,21 @@ def get_settings(self) -> Dict[str, Any]: MeilisearchApiError An error containing details about why Meilisearch can't process your request. Meilisearch error codes are described here: https://www.meilisearch.com/docs/reference/errors/error_codes#meilisearch-errors """ - return self.http.get(f"{self.config.paths.index}/{self.uid}/{self.config.paths.setting}") + settings = self.http.get(f"{self.config.paths.index}/{self.uid}/{self.config.paths.setting}") + + if settings.get("embedders"): + embedders: dict[str, OpenAiEmbedder | HuggingFaceEmbedder | UserProvidedEmbedder] = {} + for k, v in settings["embedders"].items(): + if v.get("source") == "openAi": + embedders[k] = OpenAiEmbedder(**v) + elif v.get("source") == "huggingFace": + embedders[k] = HuggingFaceEmbedder(**v) + else: + embedders[k] = UserProvidedEmbedder(**v) + + settings["embedders"] = embedders + + return settings def update_settings(self, body: Mapping[str, Any]) -> TaskInfo: """Update settings of the index. @@ -1777,7 +1800,17 @@ def get_embedders(self) -> Embedders | None: if not response: return None - return Embedders(embedders=response) + embedders: dict[str, OpenAiEmbedder | HuggingFaceEmbedder | UserProvidedEmbedder] = {} + for k, v in response.items(): + print(v.get("source")) + if v.get("source") == "openAi": + embedders[k] = OpenAiEmbedder(**v) + elif v.get("source") == "huggingFace": + embedders[k] = HuggingFaceEmbedder(**v) + else: + embedders[k] = UserProvidedEmbedder(**v) + + return Embedders(embedders=embedders) def update_embedders(self, body: Union[Mapping[str, Any], None]) -> TaskInfo: """Update embedders of the index. diff --git a/tests/conftest.py b/tests/conftest.py index 043bca1f..a88999a8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,6 +7,7 @@ import meilisearch from meilisearch.errors import MeilisearchApiError +from meilisearch.models.index import HuggingFaceEmbedder, OpenAiEmbedder, UserProvidedEmbedder from tests import common @@ -230,8 +231,7 @@ def enable_vector_search(): @fixture def new_embedders(): return { - "default": { - "source": "userProvided", - "dimensions": 1, - } + "default": UserProvidedEmbedder(dimensions=1).model_dump(by_alias=True), + "open_ai": OpenAiEmbedder().model_dump(by_alias=True), + "hugging_face": HuggingFaceEmbedder().model_dump(by_alias=True), } diff --git a/tests/settings/test_settings.py b/tests/settings/test_settings.py index 14ed09e1..684dea62 100644 --- a/tests/settings/test_settings.py +++ b/tests/settings/test_settings.py @@ -1,7 +1,16 @@ -NEW_SETTINGS = { - "rankingRules": ["typo", "words"], - "searchableAttributes": ["title", "overview"], -} +import pytest + +from meilisearch.models.index import HuggingFaceEmbedder, OpenAiEmbedder, UserProvidedEmbedder + + +@pytest.fixture +def new_settings(new_embedders): + return { + "rankingRules": ["typo", "words"], + "searchableAttributes": ["title", "overview"], + "embedders": new_embedders, + } + DEFAULT_RANKING_RULES = ["words", "typo", "proximity", "attribute", "sort", "exactness"] @@ -31,36 +40,41 @@ def test_get_settings_default(empty_index): assert response["synonyms"] == {} -def test_update_settings(empty_index): +@pytest.mark.usefixtures("enable_vector_search") +def test_update_settings(new_settings, empty_index): """Tests updating some settings.""" index = empty_index() - response = index.update_settings(NEW_SETTINGS) + response = index.update_settings(new_settings) update = index.wait_for_task(response.task_uid) assert update.status == "succeeded" response = index.get_settings() - for rule in NEW_SETTINGS["rankingRules"]: + for rule in new_settings["rankingRules"]: assert rule in response["rankingRules"] assert response["distinctAttribute"] is None - for attribute in NEW_SETTINGS["searchableAttributes"]: + for attribute in new_settings["searchableAttributes"]: assert attribute in response["searchableAttributes"] assert response["displayedAttributes"] == ["*"] assert response["stopWords"] == [] assert response["synonyms"] == {} + assert isinstance(response["embedders"]["default"], UserProvidedEmbedder) + assert isinstance(response["embedders"]["open_ai"], OpenAiEmbedder) + assert isinstance(response["embedders"]["hugging_face"], HuggingFaceEmbedder) -def test_reset_settings(empty_index): +@pytest.mark.usefixtures("enable_vector_search") +def test_reset_settings(new_settings, empty_index): """Tests resetting all the settings to their default value.""" index = empty_index() # Update settings first - response = index.update_settings(NEW_SETTINGS) + response = index.update_settings(new_settings) update = index.wait_for_task(response.task_uid) assert update.status == "succeeded" # Check the settings have been correctly updated response = index.get_settings() - for rule in NEW_SETTINGS["rankingRules"]: + for rule in new_settings["rankingRules"]: assert rule in response["rankingRules"] assert response["distinctAttribute"] is None - for attribute in NEW_SETTINGS["searchableAttributes"]: + for attribute in new_settings["searchableAttributes"]: assert attribute in response["searchableAttributes"] assert response["displayedAttributes"] == ["*"] assert response["stopWords"] == [] @@ -80,3 +94,4 @@ def test_reset_settings(empty_index): assert response["searchableAttributes"] == ["*"] assert response["stopWords"] == [] assert response["synonyms"] == {} + assert response.get("embedders") is None diff --git a/tests/settings/test_settings_embedders.py b/tests/settings/test_settings_embedders.py index 4d27ffb1..5e60d81f 100644 --- a/tests/settings/test_settings_embedders.py +++ b/tests/settings/test_settings_embedders.py @@ -1,6 +1,11 @@ import pytest -from meilisearch.models.index import Embedders +from meilisearch.models.index import ( + Embedders, + HuggingFaceEmbedder, + OpenAiEmbedder, + UserProvidedEmbedder, +) @pytest.mark.usefixtures("enable_vector_search") @@ -19,7 +24,9 @@ def test_update_embedders_with_user_provided_source(new_embedders, empty_index): update = index.wait_for_task(response_update.task_uid) response_get = index.get_embedders() assert update.status == "succeeded" - assert response_get == Embedders(embedders=new_embedders) + assert isinstance(response_get.embedders["default"], UserProvidedEmbedder) + assert isinstance(response_get.embedders["open_ai"], OpenAiEmbedder) + assert isinstance(response_get.embedders["hugging_face"], HuggingFaceEmbedder) @pytest.mark.usefixtures("enable_vector_search") @@ -30,15 +37,19 @@ def test_reset_embedders(new_embedders, empty_index): # Update the settings response_update = index.update_embedders(new_embedders) update1 = index.wait_for_task(response_update.task_uid) + assert update1.status == "succeeded" # Get the setting after update response_get = index.get_embedders() + assert isinstance(response_get.embedders["default"], UserProvidedEmbedder) + assert isinstance(response_get.embedders["open_ai"], OpenAiEmbedder) + assert isinstance(response_get.embedders["hugging_face"], HuggingFaceEmbedder) # Reset the setting response_reset = index.reset_embedders() update2 = index.wait_for_task(response_reset.task_uid) # Get the setting after reset - response_last = index.get_embedders() - - assert update1.status == "succeeded" - assert response_get == Embedders(embedders=new_embedders) assert update2.status == "succeeded" + assert isinstance(response_get.embedders["default"], UserProvidedEmbedder) + assert isinstance(response_get.embedders["open_ai"], OpenAiEmbedder) + assert isinstance(response_get.embedders["hugging_face"], HuggingFaceEmbedder) + response_last = index.get_embedders() assert response_last is None From b5159b5d4cc86131d1a9cc94fc8d459b754b85ce Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Mon, 12 Feb 2024 13:25:46 -0500 Subject: [PATCH 2/3] Run black --- meilisearch/index.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/meilisearch/index.py b/meilisearch/index.py index 2033ed41..080b9c0f 100644 --- a/meilisearch/index.py +++ b/meilisearch/index.py @@ -874,7 +874,9 @@ def get_settings(self) -> Dict[str, Any]: MeilisearchApiError An error containing details about why Meilisearch can't process your request. Meilisearch error codes are described here: https://www.meilisearch.com/docs/reference/errors/error_codes#meilisearch-errors """ - settings = self.http.get(f"{self.config.paths.index}/{self.uid}/{self.config.paths.setting}") + settings = self.http.get( + f"{self.config.paths.index}/{self.uid}/{self.config.paths.setting}" + ) if settings.get("embedders"): embedders: dict[str, OpenAiEmbedder | HuggingFaceEmbedder | UserProvidedEmbedder] = {} From f2f6929d2a55138d899b8e6667495a704ce7c08f Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Mon, 12 Feb 2024 13:26:51 -0500 Subject: [PATCH 3/3] Fix pylint warning --- tests/settings/test_settings.py | 1 + tests/settings/test_settings_embedders.py | 8 ++------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/settings/test_settings.py b/tests/settings/test_settings.py index 684dea62..b7d80702 100644 --- a/tests/settings/test_settings.py +++ b/tests/settings/test_settings.py @@ -1,3 +1,4 @@ +# pylint: disable=redefined-outer-name import pytest from meilisearch.models.index import HuggingFaceEmbedder, OpenAiEmbedder, UserProvidedEmbedder diff --git a/tests/settings/test_settings_embedders.py b/tests/settings/test_settings_embedders.py index 5e60d81f..93267a9f 100644 --- a/tests/settings/test_settings_embedders.py +++ b/tests/settings/test_settings_embedders.py @@ -1,11 +1,7 @@ +# pylint: disable=redefined-outer-name import pytest -from meilisearch.models.index import ( - Embedders, - HuggingFaceEmbedder, - OpenAiEmbedder, - UserProvidedEmbedder, -) +from meilisearch.models.index import HuggingFaceEmbedder, OpenAiEmbedder, UserProvidedEmbedder @pytest.mark.usefixtures("enable_vector_search")