Add mnasnet to torch models and minor fixes. (huggingface#577)

* Minor fixes to benchmark runner

* Add Mnasnet to tank.

Author: monorimet

shark/shark_benchmark_runner.py

@@ -260,19 +260,12 @@ def get_metadata(self, modelname):
                     return [param_count, model_tags, model_notes]
 
     def compare_bench_results(self, baseline: str, result: str):
-        # Takes two numbers represented as strings and returns "<n>x slower/faster", as in "result is <n>x slower than baseline".
+        # Takes a baseline and a result string and calculates a comparison, e.g. "1.04x baseline".
         a = float(baseline)
         b = float(result)
-        if a < b:
-            # result slower than baseline
-            comparison = (b - a) / a
-            comp_str = f"{round(comparison, 2)}x slower"
-        elif a > b:
-            # result faster than baseline
-            comparison = a / b
-            comp_str = f"{round(comparison, 2)}x faster"
-        else:
-            comp_str = "equal"
+        # result faster than baseline
+        comparison = a / b
+        comp_str = f"{round(comparison, 2)}x baseline"
         return comp_str
 
     def benchmark_all_csv(
@@ -327,7 +320,7 @@ def benchmark_all_csv(
                         bench_result["ms/iter"],
                     ) = self.benchmark_frontend(modelname)
                     self.frontend_result = bench_result["ms/iter"]
-                    bench_result["vs. PyTorch/TF"] = "="
+                    bench_result["vs. PyTorch/TF"] = "baseline"
                     (
                         bench_result["param_count"],
                         bench_result["tags"],

tank/all_models.csv

@@ -32,3 +32,4 @@ resnet50,linalg,torch,1e-2,1e-3,default
 squeezenet1_0,linalg,torch,1e-2,1e-3,default
 wide_resnet50_2,linalg,torch,1e-2,1e-3,default
 efficientnet-v2-s,mhlo,tf,1e-02,1e-3,default
+mnasnet1_0,linalg,torch,1e-2,1e-3,default

tank/model_metadata.csv

@@ -28,4 +28,4 @@ roberta-base,False,False,-,-,-
 xlm-roberta-base,False,False,-,-,-
 facebook/convnext-tiny-224,False,False,-,-,-
 efficientnet-v2-s,False,False,22M,"image-classification,cnn","Includes MBConv and Fused-MBConv"
-
+mnasnet1_0,False,True,-,"cnn, torchvision, mobile, architecture-search","Outperforms other mobile CNNs on Accuracy vs. Latency"

tank/model_utils.py

@@ -15,6 +15,7 @@
     "squeezenet1_0",
     "wide_resnet50_2",
     "mobilenet_v3_small",
+    "mnasnet1_0",
 ]
 hf_img_cls_models = [
     "google/vit-base-patch16-224",
@@ -149,6 +150,7 @@ def get_vision_model(torch_model):
         "squeezenet1_0": models.squeezenet1_0(pretrained=True),
         "wide_resnet50_2": models.wide_resnet50_2(pretrained=True),
         "mobilenet_v3_small": models.mobilenet_v3_small(pretrained=True),
+        "mnasnet1_0": models.mnasnet1_0(pretrained=True),
     }
     if isinstance(torch_model, str):
         torch_model = vision_models_dict[torch_model]
@@ -160,6 +162,8 @@ def get_vision_model(torch_model):
 
 ################################################################################
 
+####################### Other PyTorch HF Models ###############################
+
 # Utility function for comparing two tensors (torch).
 def compare_tensors(torch_tensor, numpy_tensor, rtol=1e-02, atol=1e-03):
     # torch_to_numpy = torch_tensor.detach().numpy()

tank/test_models.py

@@ -227,13 +227,21 @@ def postprocess_outputs(self, golden_out, result):
 
 
 def run_test(module_tester, dynamic, device):
+    import multiprocessing
+
     tempdir = tempfile.TemporaryDirectory(
         prefix=module_tester.tmp_prefix, dir="./shark_tmp/"
     )
     module_tester.temp_dir = tempdir.name
 
     with ireec.tools.TempFileSaver(tempdir.name):
-        module_tester.create_and_check_module(dynamic, device)
+        p = multiprocessing.Process(
+            target=module_tester.create_and_check_module,
+            args=(dynamic, device),
+        )
+        p.start()
+        p.join()
+        return p
 
 
 class SharkModuleTest(unittest.TestCase):
@@ -339,10 +347,7 @@ def test_module(self, dynamic, device, config):
             pytest.xfail(
                 reason="Numerics Issues: https://github.com/nod-ai/SHARK/issues/388"
             )
-        if config["model_name"] == "mobilenet_v3_small" and device in [
-            "metal",
-            "vulkan",
-        ]:
+        if config["model_name"] == "mobilenet_v3_small":
             pytest.xfail(
                 reason="Numerics Issues: https://github.com/nod-ai/SHARK/issues/388"
             )
@@ -417,9 +422,5 @@ def test_module(self, dynamic, device, config):
         # We must create a new process each time we benchmark a model to allow
         # for Tensorflow to release GPU resources. Using the same process to
         # benchmark multiple models leads to OOM.
-        p = multiprocessing.Process(
-            target=run_test, args=(self.module_tester, dynamic, device)
-        )
-        p.start()
-        p.join()
-        assert not p.exitcode
+
+        run_test(self.module_tester, dynamic, device)

tank/torch_model_list.csv

@@ -16,3 +16,4 @@ microsoft/resnet-50,True,hf_img_cls,False,23M,"image-classification,cnn,residual
 facebook/deit-small-distilled-patch16-224,True,hf_img_cls,False,22M,"image-classification,vision-transformer,cnn",N/A
 microsoft/beit-base-patch16-224-pt22k-ft22k,True,hf_img_cls,False,86M,"image-classification,transformer-encoder,bert-variant,vision-transformer",N/A
 nvidia/mit-b0,True,hf_img_cls,False,3.7M,"image-classification,transformer-encoder",SegFormer
+mnasnet1_0,False,vision,True,-,"cnn, torchvision, mobile, architecture-search","Outperforms other mobile CNNs on Accuracy vs. Latency"