Add multi-modal search

src/types/experimental-features.ts

@@ -4,12 +4,13 @@
  * @see `meilisearch::routes::features::RuntimeTogglableFeatures`
  */
 export type RuntimeTogglableFeatures = {
-  metrics?: boolean | null;
-  logsRoute?: boolean | null;
-  editDocumentsByFunction?: boolean | null;
+  chatCompletions?: boolean | null;
+  compositeEmbedders?: boolean | null;
   containsFilter?: boolean | null;
-  network?: boolean | null;
+  editDocumentsByFunction?: boolean | null;
   getTaskDocumentsRoute?: boolean | null;
-  compositeEmbedders?: boolean | null;
-  chatCompletions?: boolean | null;
+  logsRoute?: boolean | null;
+  metrics?: boolean | null;
+  multimodal?: boolean | null;
+  network?: boolean | null;
 };

src/types/types.ts

@@ -237,6 +237,7 @@ export type SearchParams = Query &
     distinct?: string;
     retrieveVectors?: boolean;
     locales?: Locale[];
+    media?: Record<string, string>;
   };
 
 // Search parameters for searches made with the GET method
@@ -587,6 +588,18 @@ export type RestEmbedder = {
   headers?: Record<string, string>;
   documentTemplateMaxBytes?: number;
   binaryQuantized?: boolean;
+  indexingFragments?: Record<
+    string,
+    {
+      value: RecordAny;
+    }
+  >;
+  searchFragments?: Record<
+    string,
+    {
+      value: RecordAny;
+    }
+  >;
 };
 
 export type OllamaEmbedder = {

tests/experimental-features.test.ts

@@ -6,27 +6,29 @@ const ms = await getClient("Master");
 
 afterAll(async () => {
   await ms.updateExperimentalFeatures({
-    metrics: false,
-    logsRoute: false,
-    editDocumentsByFunction: false,
+    chatCompletions: false,
+    compositeEmbedders: false,
     containsFilter: false,
-    network: false,
+    editDocumentsByFunction: false,
     getTaskDocumentsRoute: false,
-    compositeEmbedders: false,
-    chatCompletions: false,
+    logsRoute: false,
+    metrics: false,
+    multimodal: false,
+    network: false,
   } satisfies { [TKey in keyof RuntimeTogglableFeatures]-?: false });
 });
 
 test(`${ms.updateExperimentalFeatures.name} and ${ms.getExperimentalFeatures.name} methods`, async () => {
   const features: { [TKey in keyof RuntimeTogglableFeatures]-?: true } = {
-    metrics: true,
-    logsRoute: true,
-    editDocumentsByFunction: true,
+    chatCompletions: true,
+    compositeEmbedders: true,
     containsFilter: true,
-    network: true,
+    editDocumentsByFunction: true,
     getTaskDocumentsRoute: true,
-    compositeEmbedders: true,
-    chatCompletions: true,
+    logsRoute: true,
+    metrics: true,
+    multimodal: true,
+    network: true,
   };
 
   const updateResponse = await ms.updateExperimentalFeatures(features);

tests/fixtures/movies.json

@@ -0,0 +1,90 @@
+[
+  {
+    "id": 11,
+    "title": "Star Wars",
+    "overview": "Princess Leia is captured and held hostage by the evil Imperial forces in their effort to take over the galactic Empire. Venturesome Luke Skywalker and dashing captain Han Solo team together with the loveable robot duo R2-D2 and C-3PO to rescue the beautiful princess and restore peace and justice in the Empire.",
+    "genres": ["Adventure", "Action", "Science Fiction"],
+    "poster": "https://image.tmdb.org/t/p/w500/6FfCtAuVAW8XJjZ7eWeLibRLWTw.jpg",
+    "release_date": 233366400
+  },
+  {
+    "id": 12,
+    "title": "Finding Nemo",
+    "overview": "Nemo, an adventurous young clownfish, is unexpectedly taken from his Great Barrier Reef home to a dentist's office aquarium. It's up to his worrisome father Marlin and a friendly but forgetful fish Dory to bring Nemo home -- meeting vegetarian sharks, surfer dude turtles, hypnotic jellyfish, hungry seagulls, and more along the way.",
+    "genres": ["Animation", "Family"],
+    "poster": "https://image.tmdb.org/t/p/w500/eHuGQ10FUzK1mdOY69wF5pGgEf5.jpg",
+    "release_date": 1054252800
+  },
+  {
+    "id": 13,
+    "title": "Forrest Gump",
+    "overview": "A man with a low IQ has accomplished great things in his life and been present during significant historic events—in each case, far exceeding what anyone imagined he could do. But despite all he has achieved, his one true love eludes him.",
+    "genres": ["Comedy", "Drama", "Romance"],
+    "poster": "https://image.tmdb.org/t/p/w500/h5J4W4veyxMXDMjeNxZI46TsHOb.jpg",
+    "release_date": 773452800
+  },
+  {
+    "id": 18,
+    "title": "The Fifth Element",
+    "overview": "In 2257, a taxi driver is unintentionally given the task of saving a young girl who is part of the key that will ensure the survival of humanity.",
+    "genres": ["Adventure", "Fantasy", "Action", "Thriller", "Science Fiction"],
+    "poster": "https://image.tmdb.org/t/p/w500/fPtlCO1yQtnoLHOwKtWz7db6RGU.jpg",
+    "release_date": 862531200
+  },
+  {
+    "id": 22,
+    "title": "Pirates of the Caribbean: The Curse of the Black Pearl",
+    "overview": "Jack Sparrow, a freewheeling 18th-century pirate, quarrels with a rival pirate bent on pillaging Port Royal. When the governor's daughter is kidnapped, Sparrow decides to help the girl's love save her.",
+    "genres": ["Adventure", "Fantasy", "Action"],
+    "poster": "https://image.tmdb.org/t/p/w500/z8onk7LV9Mmw6zKz4hT6pzzvmvl.jpg",
+    "release_date": 1057708800
+  },
+  {
+    "id": 24,
+    "title": "Kill Bill: Vol. 1",
+    "overview": "An assassin is shot by her ruthless employer, Bill, and other members of their assassination circle – but she lives to plot her vengeance.",
+    "genres": ["Action", "Crime"],
+    "poster": "https://image.tmdb.org/t/p/w500/v7TaX8kXMXs5yFFGR41guUDNcnB.jpg",
+    "release_date": 1065744000
+  },
+  {
+    "id": 35,
+    "title": "The Simpsons Movie",
+    "overview": "After Homer accidentally pollutes the town's water supply, Springfield is encased in a gigantic dome by the EPA and the Simpsons are declared fugitives.",
+    "genres": ["Animation", "Comedy", "Family"],
+    "poster": "https://image.tmdb.org/t/p/w500/s3b8TZWwmkYc2KoJ5zk77qB6PzY.jpg",
+    "release_date": 1185321600
+  },
+  {
+    "id": 62,
+    "title": "2001: A Space Odyssey",
+    "overview": "Humanity finds a mysterious object buried beneath the lunar surface and sets off to find its origins with the help of HAL 9000, the world's most advanced super computer.",
+    "genres": ["Science Fiction", "Mystery", "Adventure"],
+    "poster": "https://image.tmdb.org/t/p/w500/ve72VxNqjGM69Uky4WTo2bK6rfq.jpg",
+    "release_date": -55209600
+  },
+  {
+    "id": 65,
+    "title": "8 Mile",
+    "overview": "The setting is Detroit in 1995. The city is divided by 8 Mile, a road that splits the town in half along racial lines. A young white rapper, Jimmy \"B-Rabbit\" Smith Jr. summons strength within himself to cross over these arbitrary boundaries to fulfill his dream of success in hip hop. With his pal Future and the three one third in place, all he has to do is not choke.",
+    "genres": ["Music", "Drama"],
+    "poster": "https://image.tmdb.org/t/p/w500/7BmQj8qE1FLuLTf7Xjf9sdIHzoa.jpg",
+    "release_date": 1036713600
+  },
+  {
+    "id": 81,
+    "title": "Nausicaä of the Valley of the Wind",
+    "overview": "After a global war, the seaside kingdom known as the Valley of the Wind remains one of the last strongholds on Earth untouched by a poisonous jungle and the powerful insects that guard it. Led by the courageous Princess Nausicaä, the people of the Valley engage in an epic struggle to restore the bond between humanity and Earth.",
+    "genres": ["Adventure", "Animation", "Fantasy"],
+    "poster": "https://image.tmdb.org/t/p/w500/sIpcATxMrKHRRUJAGI5UIUT7XMG.jpg",
+    "release_date": 447811200
+  },
+  {
+    "id": 98,
+    "title": "Gladiator",
+    "overview": "In the year 180, the death of emperor Marcus Aurelius throws the Roman Empire into chaos.  Maximus is one of the Roman army's most capable and trusted generals and a key advisor to the emperor.  As Marcus' devious son Commodus ascends to the throne, Maximus is set to be executed.  He escapes, but is captured by slave traders.  Renamed Spaniard and forced to become a gladiator, Maximus must battle to the death with other men for the amusement of paying audiences.",
+    "genres": ["Action", "Drama", "Adventure"],
+    "poster": "https://image.tmdb.org/t/p/w500/ehGpN04mLJIrSnxcZBMvHeG0eDc.jpg",
+    "release_date": 957139200
+  }
+]

tests/multi_modal_search.test.ts

@@ -0,0 +1,167 @@
+import { beforeAll, describe, expect, test } from "vitest";
+import { getClient } from "./utils/meilisearch-test-utils.js";
+import type { Embedder } from "../src/types/types.js";
+import movies from "./fixtures/movies.json" assert { type: "json" };
+import type { Meilisearch } from "../src/index.js";
+
+const VOYAGE_API_KEY = import.meta.env.VITE_VOYAGE_API_KEY as string;
+
+const INDEX_UID = "multi-modal-search-test";
+const EMBEDDER_NAME = "multimodal";
+const EMBEDDER_CONFIG = {
+  source: "rest",
+  url: "https://api.voyageai.com/v1/multimodalembeddings",
+  apiKey: VOYAGE_API_KEY,
+  dimensions: 1024,
+  indexingFragments: {
+    textAndPoster: {
+      value: {
+        content: [
+          {
+            type: "text",
+            text: "{{q}}",
+          },
+          {
+            type: "image_url",
+            image_url: "{{media.poster}}",
+          },
+        ],
+      },
+    },
+    // text: {
+    //   value: {
+    //     // The shape of the data here depends on the model used
+    //     content: [
+    //       {
+    //         type: "text",
+    //         text: "A movie titled {{doc.title}} whose description starts with {{doc.overview|truncatewords:20}}.",
+    //       },
+    //     ],
+    //   },
+    // },
+    // poster: {
+    //   value: {
+    //     // The shape of the data here depends on the model used
+    //     content: [
+    //       {
+    //         type: "image_url",
+    //         image_url: "{{doc.poster}}",
+    //       },
+    //     ],
+    //   },
+    // },
+  },
+  searchFragments: {
+    // poster: {
+    //   value: {
+    //     content: [
+    //       {
+    //         type: "image_url",
+    //         image_url: "{{media.poster}}",
+    //       },
+    //     ],
+    //   },
+    // },
+    // text: {
+    //   value: {
+    //     content: [
+    //       {
+    //         type: "text",
+    //         // uses the `q` field from search queries
+    //         text: "{{q}}",
+    //       },
+    //     ],
+    //   },
+    // },
+    textAndPoster: {
+      value: {
+        content: [
+          {
+            type: "text",
+            text: "{{q}}",
+          },
+          {
+            type: "image_url",
+            image_url: "{{media.poster}}",
+          },
+        ],
+      },
+    },
+  },
+  request: {
+    inputs: ["{{fragment}}", "{{..}}"],
+    model: "voyage-multimodal-3",
+  },
+  response: {
+    // Maps how Voyage API returns the embedding
+    data: [
+      {
+        embedding: "{{embedding}}",
+      },
+      "{{..}}",
+    ],
+  },
+} satisfies Embedder;
+
+describe.skipIf(!VOYAGE_API_KEY)("Multi-modal search", () => {
+  let searchClient: Meilisearch;
+
+  beforeAll(async () => {
+    const client = await getClient("Admin");
+    await client.updateExperimentalFeatures({
+      multimodal: true,
+    });
+    await client.createIndex(INDEX_UID).waitTask();
+    await client.index(INDEX_UID).updateSettings({
+      searchableAttributes: ["title", "overview"],
+      embedders: {
+        [EMBEDDER_NAME]: EMBEDDER_CONFIG,
+      },
+    });
+    await client.index(INDEX_UID).addDocuments(movies).waitTask();
+    searchClient = await getClient("Search");
+  });
+
+  test("should work with text query", async () => {
+    const response = await searchClient
+      .index(INDEX_UID)
+      .search("A movie with lightsabers in space", {
+        hybrid: {
+          embedder: EMBEDDER_NAME,
+          semanticRatio: 1,
+        },
+      });
+    expect(response.hits[0].title).toBe("Star Wars");
+  });
+
+  test("should work with image query", async () => {
+    const theFifthElementPoster = movies[3].poster;
+
+    const response = await searchClient.index(INDEX_UID).search(null, {
+      media: {
+        poster: theFifthElementPoster,
+      },
+      hybrid: {
+        embedder: EMBEDDER_NAME,
+        semanticRatio: 1,
+      },
+    });
+    expect(response.hits[0].title).toBe("The Fifth Element");
+  });
+
+  test("should work with text and image query", async () => {
+    const spaceImageUrl =
+      "https://science.nasa.gov/wp-content/uploads/2023/06/webb-flickr-52259221868-30e1c78f0c-4k-jpg.webp";
+    const response = await searchClient.index(INDEX_UID).search(null, {
+      q: "A futuristic movie",
+      media: {
+        poster: spaceImageUrl,
+      },
+      hybrid: {
+        embedder: EMBEDDER_NAME,
+        semanticRatio: 1,
+      },
+    });
+    expect(response.hits[0].title).toBe("Star Wars");
+  });
+});

vite.config.ts

@@ -1,4 +1,4 @@
-import { defineConfig } from "vite";
+import { defineConfig, loadEnv } from "vite";
 import pkg from "./package.json" with { type: "json" };
 
 const indexInput = "src/index.ts";
@@ -57,6 +57,7 @@ export default defineConfig(({ mode }) => {
       fileParallelism: false,
       testTimeout: 100_000, // 100 seconds
       coverage: { include: ["src/**/*.ts"] },
+      env: loadEnv("test", process.cwd()),
     },
   };
 });