Add huggingface top5 image classification automodel (huggingface#268)

Author: AmosLewis

generate_sharktank.py

@@ -43,6 +43,7 @@ def create_hash(file_name):
 def save_torch_model(torch_model_list):
     from tank.model_utils import get_hf_model
     from tank.model_utils import get_vision_model
+    from tank.model_utils import get_hf_img_cls_model
 
     with open(torch_model_list) as csvfile:
         torch_reader = csv.reader(csvfile, delimiter=",")
@@ -51,15 +52,19 @@ def save_torch_model(torch_model_list):
             torch_model_name = row[0]
             tracing_required = row[1]
             model_type = row[2]
+            is_dynamic = row[3]
 
             tracing_required = False if tracing_required == "False" else True
+            is_dynamic = False if is_dynamic == "False" else True
 
             model = None
             input = None
             if model_type == "vision":
                 model, input, _ = get_vision_model(torch_model_name)
             elif model_type == "hf":
                 model, input, _ = get_hf_model(torch_model_name)
+            elif model_type == "hf_img_cls":
+                model, input, _ = get_hf_img_cls_model(torch_model_name)
 
             torch_model_name = torch_model_name.replace("/", "_")
             torch_model_dir = os.path.join(
@@ -85,12 +90,13 @@ def save_torch_model(torch_model_list):
             )
             np.save(os.path.join(torch_model_dir, "hash"), np.array(mlir_hash))
             # Generate torch dynamic models.
-            mlir_importer.import_debug(
-                is_dynamic=True,
-                tracing_required=tracing_required,
-                dir=torch_model_dir,
-                model_name=torch_model_name + "_dynamic",
-            )
+            if is_dynamic:
+                mlir_importer.import_debug(
+                    is_dynamic=True,
+                    tracing_required=tracing_required,
+                    dir=torch_model_dir,
+                    model_name=torch_model_name + "_dynamic",
+                )
 
 
 def save_tf_model(tf_model_list):

requirements.txt

@@ -7,6 +7,9 @@ tqdm
 # SHARK Downloader
 gsutil
 
+# generate_sharktank
+transformers==4.18.0
+
 # Testing
 pytest
 pytest-xdist

tank/facebook_deit-small-distilled-patch16-224_torch/facebook_deit-small-distilled-patch16-224_torch_test.py

@@ -0,0 +1,68 @@
+from shark.iree_utils._common import check_device_drivers, device_driver_info
+from shark.shark_inference import SharkInference
+from shark.shark_downloader import download_torch_model
+
+import unittest
+import pytest
+import numpy as np
+
+
+class DeitModuleTester:
+    def __init__(
+        self,
+        benchmark=False,
+    ):
+        self.benchmark = benchmark
+
+    def create_and_check_module(self, dynamic, device):
+        model, func_name, inputs, golden_out = download_torch_model(
+            "facebook/deit-small-distilled-patch16-224", dynamic
+        )
+
+        shark_module = SharkInference(
+            model,
+            func_name,
+            device=device,
+            mlir_dialect="linalg",
+            is_benchmark=self.benchmark,
+        )
+        shark_module.compile()
+        result = shark_module.forward(inputs)
+
+        print(np.allclose(golden_out[0], result[0], rtol=1e-02, atol=1e-03))
+
+
+class DeitModuleTest(unittest.TestCase):
+    @pytest.fixture(autouse=True)
+    def configure(self, pytestconfig):
+        self.module_tester = DeitModuleTester(self)
+        self.module_tester.benchmark = pytestconfig.getoption("benchmark")
+
+    def test_module_static_cpu(self):
+        dynamic = False
+        device = "cpu"
+        self.module_tester.create_and_check_module(dynamic, device)
+
+    @pytest.mark.skipif(
+        check_device_drivers("gpu"), reason=device_driver_info("gpu")
+    )
+    def test_module_static_gpu(self):
+        dynamic = False
+        device = "gpu"
+        self.module_tester.create_and_check_module(dynamic, device)
+
+    @pytest.mark.skipif(
+        check_device_drivers("vulkan"), reason=device_driver_info("vulkan")
+    )
+    def test_module_static_vulkan(self):
+        dynamic = False
+        device = "vulkan"
+        self.module_tester.create_and_check_module(dynamic, device)
+
+
+if __name__ == "__main__":
+    # dynamic = False
+    # device = "cpu"
+    # module_tester = DeiteModuleTester()
+    # module_tester.create_and_check_module(dynamic, device)
+    unittest.main()

tank/google_vit-base-patch16-224_tf/google_vit-base-patch16-224_tf_test.py

@@ -63,8 +63,8 @@ def test_module_static_vulkan(self):
 
 
 if __name__ == "__main__":
-    dynamic = False
-    device = "cpu"
-    module_tester = VitBaseModuleTester()
-    module_tester.create_and_check_module(dynamic, device)
-    # unittest.main()
+    # dynamic = False
+    # device = "cpu"
+    # module_tester = VitBaseModuleTester()
+    # module_tester.create_and_check_module(dynamic, device)
+    unittest.main()

tank/google_vit-base-patch16-224_torch/google_vit-base-patch16-224_torch_test.py

@@ -0,0 +1,68 @@
+from shark.iree_utils._common import check_device_drivers, device_driver_info
+from shark.shark_inference import SharkInference
+from shark.shark_downloader import download_torch_model
+
+import unittest
+import pytest
+import numpy as np
+
+
+class VitBaseModuleTester:
+    def __init__(
+        self,
+        benchmark=False,
+    ):
+        self.benchmark = benchmark
+
+    def create_and_check_module(self, dynamic, device):
+        model, func_name, inputs, golden_out = download_torch_model(
+            "google/vit-base-patch16-224", dynamic
+        )
+
+        shark_module = SharkInference(
+            model,
+            func_name,
+            device=device,
+            mlir_dialect="linalg",
+            is_benchmark=self.benchmark,
+        )
+        shark_module.compile()
+        result = shark_module.forward(inputs)
+
+        print(np.allclose(golden_out[0], result[0], rtol=1e-02, atol=1e-03))
+
+
+class VitBaseModuleTest(unittest.TestCase):
+    @pytest.fixture(autouse=True)
+    def configure(self, pytestconfig):
+        self.module_tester = VitBaseModuleTester(self)
+        self.module_tester.benchmark = pytestconfig.getoption("benchmark")
+
+    def test_module_static_cpu(self):
+        dynamic = False
+        device = "cpu"
+        self.module_tester.create_and_check_module(dynamic, device)
+
+    @pytest.mark.skipif(
+        check_device_drivers("gpu"), reason=device_driver_info("gpu")
+    )
+    def test_module_static_gpu(self):
+        dynamic = False
+        device = "gpu"
+        self.module_tester.create_and_check_module(dynamic, device)
+
+    @pytest.mark.skipif(
+        check_device_drivers("vulkan"), reason=device_driver_info("vulkan")
+    )
+    def test_module_static_vulkan(self):
+        dynamic = False
+        device = "vulkan"
+        self.module_tester.create_and_check_module(dynamic, device)
+
+
+if __name__ == "__main__":
+    # dynamic = False
+    # device = "cpu"
+    # module_tester = VitBaseModuleTester()
+    # module_tester.create_and_check_module(dynamic, device)
+    unittest.main()

tank/microsoft_beit-base-patch16-224-pt22k-ft22k_torch/microsoft_beit-base-patch16-224-pt22k-ft22k_torch.py

@@ -0,0 +1,68 @@
+from shark.iree_utils._common import check_device_drivers, device_driver_info
+from shark.shark_inference import SharkInference
+from shark.shark_downloader import download_torch_model
+
+import unittest
+import pytest
+import numpy as np
+
+
+class BeitModuleTester:
+    def __init__(
+        self,
+        benchmark=False,
+    ):
+        self.benchmark = benchmark
+
+    def create_and_check_module(self, dynamic, device):
+        model, func_name, inputs, golden_out = download_torch_model(
+            "microsoft/beit-base-patch16-224-pt22k-ft22k", dynamic
+        )
+
+        shark_module = SharkInference(
+            model,
+            func_name,
+            device=device,
+            mlir_dialect="linalg",
+            is_benchmark=self.benchmark,
+        )
+        shark_module.compile()
+        result = shark_module.forward(inputs)
+
+        print(np.allclose(golden_out[0], result[0], rtol=1e-02, atol=1e-03))
+
+
+class BeitModuleTest(unittest.TestCase):
+    @pytest.fixture(autouse=True)
+    def configure(self, pytestconfig):
+        self.module_tester = BeitModuleTester(self)
+        self.module_tester.benchmark = pytestconfig.getoption("benchmark")
+
+    def test_module_static_cpu(self):
+        dynamic = False
+        device = "cpu"
+        self.module_tester.create_and_check_module(dynamic, device)
+
+    @pytest.mark.skipif(
+        check_device_drivers("gpu"), reason=device_driver_info("gpu")
+    )
+    def test_module_static_gpu(self):
+        dynamic = False
+        device = "gpu"
+        self.module_tester.create_and_check_module(dynamic, device)
+
+    @pytest.mark.skipif(
+        check_device_drivers("vulkan"), reason=device_driver_info("vulkan")
+    )
+    def test_module_static_vulkan(self):
+        dynamic = False
+        device = "vulkan"
+        self.module_tester.create_and_check_module(dynamic, device)
+
+
+if __name__ == "__main__":
+    # dynamic = False
+    # device = "cpu"
+    # module_tester = BeitModuleTester()
+    # module_tester.create_and_check_module(dynamic, device)
+    unittest.main()

tank/microsoft_resnet-50_torch/microsoft_resnet-50_torch_test.py

@@ -0,0 +1,68 @@
+from shark.iree_utils._common import check_device_drivers, device_driver_info
+from shark.shark_inference import SharkInference
+from shark.shark_downloader import download_torch_model
+
+import unittest
+import pytest
+import numpy as np
+
+
+class ResnetModuleTester:
+    def __init__(
+        self,
+        benchmark=False,
+    ):
+        self.benchmark = benchmark
+
+    def create_and_check_module(self, dynamic, device):
+        model, func_name, inputs, golden_out = download_torch_model(
+            "microsoft/resnet-50", dynamic
+        )
+
+        shark_module = SharkInference(
+            model,
+            func_name,
+            device=device,
+            mlir_dialect="linalg",
+            is_benchmark=self.benchmark,
+        )
+        shark_module.compile()
+        result = shark_module.forward(inputs)
+
+        print(np.allclose(golden_out[0], result[0], rtol=1e-01, atol=1e-03))
+
+
+class ResnetModuleTest(unittest.TestCase):
+    @pytest.fixture(autouse=True)
+    def configure(self, pytestconfig):
+        self.module_tester = ResnetModuleTester(self)
+        self.module_tester.benchmark = pytestconfig.getoption("benchmark")
+
+    def test_module_static_cpu(self):
+        dynamic = False
+        device = "cpu"
+        self.module_tester.create_and_check_module(dynamic, device)
+
+    @pytest.mark.skipif(
+        check_device_drivers("gpu"), reason=device_driver_info("gpu")
+    )
+    def test_module_static_gpu(self):
+        dynamic = False
+        device = "gpu"
+        self.module_tester.create_and_check_module(dynamic, device)
+
+    @pytest.mark.skipif(
+        check_device_drivers("vulkan"), reason=device_driver_info("vulkan")
+    )
+    def test_module_static_vulkan(self):
+        dynamic = False
+        device = "vulkan"
+        self.module_tester.create_and_check_module(dynamic, device)
+
+
+if __name__ == "__main__":
+    # dynamic = False
+    # device = "cpu"
+    # module_tester = ResnetModuleTester()
+    # module_tester.create_and_check_module(dynamic, device)
+    unittest.main()