Support to load Transformers.js v3 and make it default (#336)

* Support to load Transformers.js v3 and make it default

* Update tests

* Update tests

Author: whitphx

pyodide-e2e/src/tests/import.test.ts

@@ -1,6 +1,7 @@
 import type { PyodideInterface } from "pyodide";
 import { beforeEach, describe, it, expect } from "vitest";
 import { setupPyodideForTest } from "./utils";
+import { version } from "os";
 
 describe("import_transformers_js", () => {
   let pyodide: PyodideInterface;
@@ -9,42 +10,51 @@ describe("import_transformers_js", () => {
     pyodide = await setupPyodideForTest();
   });
 
-  it("can import specific version of Transformers.js", async () => {
-    await pyodide.runPythonAsync(
-      `from transformers_js_py import import_transformers_js`,
-    );
-
-    await pyodide.runPythonAsync(
-      `transformers = await import_transformers_js()`,
-    );
-    const versionLatest = await pyodide.runPythonAsync(
-      `transformers.env.version`,
-    );
+  it("loads the latest version of Transformers.js by default", async () => {
     const expectedLatestVersion = await fetch(
-      "https://registry.npmjs.org/@xenova/transformers/latest",
+      "https://registry.npmjs.org/@huggingface/transformers/latest",
     )
       .then((response) => response.json())
       .then((json) => json.version);
-    expect(versionLatest).toBe(expectedLatestVersion);
 
     await pyodide.runPythonAsync(
-      `transformers = await import_transformers_js("2.4.2")`,
-    );
-    const version242 = await pyodide.runPythonAsync(`transformers.env.version`);
-    expect(version242).toBe("2.4.2");
+      `
+from transformers_js_py import import_transformers_js
 
-    await pyodide.runPythonAsync(
-      `transformers = await import_transformers_js("2.7.0")`,
+transformers = await import_transformers_js()
+`,
     );
-    const version270 = await pyodide.runPythonAsync(`transformers.env.version`);
-    expect(version270).toBe("2.7.0");
 
-    await pyodide.runPythonAsync(
-      `transformers = await import_transformers_js("https://cdn.jsdelivr.net/npm/@xenova/[email protected]")`,
-    );
-    const version2177 = await pyodide.runPythonAsync(
-      `transformers.env.version`,
+    expect(pyodide.runPython(`transformers.env.version`)).toBe(
+      expectedLatestVersion,
     );
-    expect(version2177).toBe("2.17.1");
+  });
+
+  [
+    { versionOrUrl: "2.4.2", expectedVersion: "2.4.2" },
+    { versionOrUrl: "2.7.0", expectedVersion: "2.7.0" },
+    {
+      versionOrUrl: "https://cdn.jsdelivr.net/npm/@xenova/[email protected]",
+      expectedVersion: "2.17.1",
+    },
+    { versionOrUrl: "3.0.0", expectedVersion: "3.0.0" },
+    {
+      versionOrUrl:
+        "https://cdn.jsdelivr.net/npm/@huggingface/[email protected]",
+      expectedVersion: "3.1.0",
+    },
+  ].forEach(({ versionOrUrl, expectedVersion }) => {
+    it(`can import specific version of Transformers.js. (${versionOrUrl} => ${expectedVersion})`, async () => {
+      await pyodide.runPythonAsync(
+        `from transformers_js_py import import_transformers_js`,
+      );
+
+      await pyodide.runPythonAsync(
+        `transformers = await import_transformers_js("${versionOrUrl}")`,
+      );
+      expect(pyodide.runPython(`transformers.env.version`)).toBe(
+        expectedVersion,
+      );
+    });
   });
 });

transformers_js_py/proxies.py

@@ -242,7 +242,9 @@ async def import_transformers_js(version_or_url: str = "latest"):
                 return new URL(versionOrUrl);
             } catch {
                 const version = versionOrUrl;
-                return new URL('https://cdn.jsdelivr.net/npm/@xenova/transformers@' + version);
+                const v3 = version === "latest" || version.startsWith("3.");
+                const packageName = v3 ? '@huggingface/transformers' : '@xenova/transformers';
+                return new URL(`https://cdn.jsdelivr.net/npm/${packageName}@${version}`);
             }
         }