diff --git a/pkg/workloads/cortex/onnx_serve/api.py b/pkg/workloads/cortex/onnx_serve/api.py
index d42588f72a..1eb3263fdb 100644
--- a/pkg/workloads/cortex/onnx_serve/api.py
+++ b/pkg/workloads/cortex/onnx_serve/api.py
@@ -226,7 +226,7 @@ def start(args):
 
         local_cache["api"] = api
         local_cache["ctx"] = ctx
-        if api.get("request_handler_impl_key") is not None:
+        if api.get("request_handler") is not None:
             package.install_packages(ctx.python_packages, ctx.storage)
             local_cache["request_handler"] = ctx.get_request_handler_impl(api["name"])
 
diff --git a/pkg/workloads/cortex/tf_api/api.py b/pkg/workloads/cortex/tf_api/api.py
index 57b56d779f..91b2818ed2 100644
--- a/pkg/workloads/cortex/tf_api/api.py
+++ b/pkg/workloads/cortex/tf_api/api.py
@@ -447,16 +447,16 @@ def start(args):
     local_cache["ctx"] = ctx
 
     try:
-        if api.get("request_handler_impl_key") is not None:
+        if api.get("request_handler") is not None or util.is_resource_ref(api["model"]):
+            package.install_packages(ctx.python_packages, ctx.storage)
+
+        if api.get("request_handler") is not None:
             local_cache["request_handler"] = ctx.get_request_handler_impl(api["name"])
 
         if not util.is_resource_ref(api["model"]):
-            if api.get("request_handler") is not None:
-                package.install_packages(ctx.python_packages, ctx.storage)
             if not os.path.isdir(args.model_dir):
                 ctx.storage.download_and_unzip_external(api["model"], args.model_dir)
         else:
-            package.install_packages(ctx.python_packages, ctx.storage)
             model_name = util.get_resource_ref(api["model"])
             model = ctx.models[model_name]
             estimator = ctx.estimators[model["estimator"]]