Merge pull request #13 from LlmKira/dev-20250304

🔧 fix(model): enhance Windows model loading with robust path handling

Author: sudoskys

.github/workflows/python_test.yml

@@ -6,8 +6,7 @@ on:
     branches:
       - main
       - dev
-      - develop
-      - '**-develop'
+      - 'dev-**'
 
 jobs:
   Testing:
@@ -27,6 +26,7 @@ jobs:
       - name: Install dependencies
         run: |
           pdm install --no-lock -G testing
-      - name: Run Tests
+
+      - name: Run Regular Tests
         run: |
           pdm run -v pytest tests

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fast-langdetect"
-version = "0.3.0"
+version = "0.3.1"
 description = "Quickly detect text language and segment language"
 authors = [
     { name = "sudoskys", email = "[email protected]" },

src/fast_langdetect/infer.py

@@ -130,30 +130,61 @@ def load_with_download(self, model_path: Path, proxy: Optional[str] = None) -> A
         return self.load_local(model_path)
 
     def _load_windows_compatible(self, model_path: Path) -> Any:
-        """Handle Windows path compatibility issues."""
-        if re.match(r'^[A-Za-z0-9_/\\:.]*$', str(model_path)):
-            return fasttext.load_model(str(model_path))
+        """
+        Handle Windows path compatibility issues when loading FastText models.
         
-        # Create a temporary file to handle special characters in the path
-        with tempfile.NamedTemporaryFile(delete=False) as tmp:
-            tmp_path = tmp.name
-            shutil.copy2(model_path, tmp_path)
+        Attempts multiple strategies in order:
+        1. Direct loading if path contains only safe characters
+        2. Loading via relative path if possible
+        3. Copying to temporary file as last resort
         
+        :param model_path: Path to the model file
+        :return: Loaded FastText model
+        :raises DetectError: If all loading strategies fail
+        """
+        model_path_str = str(model_path.resolve())
+        
+        # Try to load model directly
         try:
-            model = fasttext.load_model(tmp_path)
-            return model
+            return fasttext.load_model(model_path_str)
+        except Exception as e:
+            logger.debug(f"fast-langdetect: Load model failed: {e}")
+        
+        # Try to load model using relative path
+        try:
+            cwd = Path.cwd()
+            rel_path = os.path.relpath(model_path, cwd)
+            return fasttext.load_model(rel_path)
+        except Exception as e:
+            logger.debug(f"fast-langdetect: Failed to load model using relative path: {e}")
+        
+        # Use temporary file as last resort
+        logger.debug(f"fast-langdetect: Using temporary file to load model: {model_path}")
+        tmp_path = None
+        try:
+            # Use NamedTemporaryFile to create a temporary file
+            tmp_fd, tmp_path = tempfile.mkstemp(suffix='.bin')
+            os.close(tmp_fd)  # Close file descriptor
+            
+            # Copy model file to temporary location
+            shutil.copy2(model_path, tmp_path)
+            return fasttext.load_model(tmp_path)
+        except Exception as e:
+            raise DetectError(f"Failed to load model using temporary file: {e}")
         finally:
-            try:
-                os.unlink(tmp_path)
-            except (OSError, PermissionError) as e:
-                logger.warning(f"fast-langdetect: Failed to delete temporary file {tmp_path}: {e}")
-                # Schedule file for deletion on next reboot on Windows
-                if platform.system() == "Windows":
-                    try:
-                        import _winapi
-                        _winapi.MoveFileEx(tmp_path, None, _winapi.MOVEFILE_DELAY_UNTIL_REBOOT)
-                    except (ImportError, AttributeError, OSError) as we:
-                        logger.warning(f"fast-langdetect: Failed to schedule file deletion: {we}")
+            # Clean up temporary file
+            if tmp_path and os.path.exists(tmp_path):
+                try:
+                    os.unlink(tmp_path)
+                except (OSError, PermissionError) as e:
+                    logger.warning(f"fast-langdetect: Failed to delete temporary file {tmp_path}: {e}")
+                    # Plan to delete on next reboot on Windows
+                    if platform.system() == "Windows":
+                        try:
+                            import _winapi
+                            _winapi.MoveFileEx(tmp_path, None, _winapi.MOVEFILE_DELAY_UNTIL_REBOOT)
+                        except (ImportError, AttributeError, OSError) as we:
+                            logger.warning(f"fast-langdetect: Failed to schedule file deletion: {we}")
 
     def _load_unix(self, model_path: Path) -> Any:
         """Load model on Unix-like systems."""

tests/test_chinese_path.py

@@ -0,0 +1,40 @@
+import os
+import tempfile
+import shutil
+import platform
+from pathlib import Path
+import pytest
+from fast_langdetect.infer import LangDetectConfig, LangDetector
+
+@pytest.mark.skipif(platform.system() != "Windows", reason="Windows path test")
+def test_model_loading_with_chinese_path():
+    # 创建包含中文字符的临时目录
+    temp_dir = Path(tempfile.gettempdir()) / "测试_模型_路径"
+    os.makedirs(temp_dir, exist_ok=True)
+    
+    try:
+        # 使用项目中已有的模型文件
+        # 查找项目根目录
+        project_root = Path(__file__).parent.parent
+        model_path = project_root / "src" / "fast_langdetect" / "resources" / "lid.176.ftz"
+        
+        if not model_path.exists():
+            pytest.skip(f"Model file does not exist: {model_path}")
+            
+        # 复制模型文件到中文路径
+        chinese_model_path = temp_dir / "测试模型.ftz"
+        shutil.copy2(model_path, chinese_model_path)
+        
+        # 正确使用自定义模型路径
+        config = LangDetectConfig(
+            custom_model_path=str(chinese_model_path),
+            allow_fallback=False
+            )
+        detector = LangDetector(config)
+        result = detector.detect("This is a test")
+        
+        assert "lang" in result
+        assert "score" in result
+    finally:
+        # 清理
+        shutil.rmtree(temp_dir, ignore_errors=True)