Merge branch 'dev' into pthummar/add_features_e2e_tests

Author: pdthummar

.github/workflows/ci_consumption_workflow.yml

@@ -1,7 +1,7 @@
 # This workflow will run all tests in tests/consumption_tests in Docker using a consumption image
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: CI Consumption tests
+name: CI Consumption E2E tests
 
 on:
   workflow_dispatch:
@@ -28,10 +28,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev]
-          python setup.py build
-          python setup.py webhost --branch-name=dev
-          python setup.py extension
+          python -m pip install -U -e .[dev]
       - name: Running 3.7 Tests
         if: matrix.python-version == 3.7
         env:

.github/workflows/ci_docker_con_workflow.yml

@@ -29,10 +29,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev]
-          python setup.py build
-          python setup.py webhost --branch-name=dev
-          python setup.py extension
+          python -m pip install -U -e .[dev]
       - name: Running 3.7 Tests
         if: matrix.python-version == 3.7
         env:

.github/workflows/ci_docker_ded_workflow.yml

@@ -30,10 +30,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev]
-          python setup.py build
-          python setup.py webhost --branch-name=dev
-          python setup.py extension
+          python -m pip install -U -e .[dev]
       - name: Running 3.7 Tests
         if: matrix.python-version == 3.7
         env:

.github/workflows/ci_e2e_workflow.yml

@@ -58,7 +58,7 @@ jobs:
           }
 
           python -m pip install --upgrade pip
-          python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev]
+          python -m pip install -U -e .[dev]
           python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
 
           # Retry a couple times to avoid certificate issue
@@ -76,7 +76,7 @@ jobs:
           AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString37 }}
           AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString37 }}
         run: |
-          python -m pytest -n auto  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
+          python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
       - name: Running 3.8 Tests
         if: matrix.python-version == 3.8
         env:
@@ -88,7 +88,7 @@ jobs:
           AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString38 }}
           AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString38 }}
         run: |
-          python -m pytest -n auto  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
+          python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
       - name: Running 3.9 Tests
         if: matrix.python-version == 3.9
         env:
@@ -100,7 +100,7 @@ jobs:
           AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString39 }}
           AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString39 }}
         run: |
-          python -m pytest -n auto  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
+          python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
       - name: Running 3.10 Tests
         if: matrix.python-version == 3.10
         env:
@@ -112,7 +112,7 @@ jobs:
           AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString310 }}
           AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString310 }}
         run: |
-          python -m pytest -n auto  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
+          python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
       - name: Running 3.11 Tests
         if: matrix.python-version == 3.11
         env:
@@ -124,7 +124,7 @@ jobs:
           AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString311 }}
           AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString311 }}
         run: |
-          python -m pytest -n auto  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
+          python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
       - name: Codecov
         uses: codecov/[email protected]
         with:

.github/workflows/ci_ut_workflow.yml

@@ -55,7 +55,7 @@ jobs:
         }
 
         python -m pip install --upgrade pip
-        python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev]
+        python -m pip install -U -e .[dev]
         python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
 
         # Retry a couple times to avoid certificate issue
@@ -66,7 +66,7 @@ jobs:
       env:
         AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }} # needed for installing azure-functions-durable while running setup.py
       run: |
-        python -m pytest -n auto --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch tests/unittests
+        python -m pytest -n auto --dist loadfile --reruns 4 -vv --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch tests/unittests
     - name: Codecov
       uses: codecov/codecov-action@v3
       with:

azure_functions_worker/bindings/context.py

@@ -1,5 +1,8 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
+import threading
+from typing import Type
+
 from . import TraceContext
 from . import RetryContext
 
@@ -10,18 +13,24 @@ def __init__(self,
                  func_name: str,
                  func_dir: str,
                  invocation_id: str,
+                 thread_local_storage: Type[threading.local],
                  trace_context: TraceContext,
                  retry_context: RetryContext) -> None:
         self.__func_name = func_name
         self.__func_dir = func_dir
         self.__invocation_id = invocation_id
+        self.__thread_local_storage = thread_local_storage
         self.__trace_context = trace_context
         self.__retry_context = retry_context
 
     @property
     def invocation_id(self) -> str:
         return self.__invocation_id
 
+    @property
+    def thread_local_storage(self) -> Type[threading.local]:
+        return self.__thread_local_storage
+
     @property
     def function_name(self) -> str:
         return self.__func_name

azure_functions_worker/constants.py

@@ -30,6 +30,9 @@
 """
 UNIX_SHARED_MEMORY_DIRECTORIES = "FUNCTIONS_UNIX_SHARED_MEMORY_DIRECTORIES"
 
+# Flag to enable loading functions at init request
+PYTHON_LOAD_FUNCTIONS_INIT = "PYTHON_LOAD_FUNCTIONS_INIT"
+
 # Setting Defaults
 PYTHON_THREADPOOL_THREAD_COUNT_DEFAULT = 1
 PYTHON_THREADPOOL_THREAD_COUNT_MIN = 1

azure_functions_worker/dispatcher.py

@@ -26,7 +26,7 @@
                         PYTHON_THREADPOOL_THREAD_COUNT_MAX_37,
                         PYTHON_THREADPOOL_THREAD_COUNT_MIN,
                         PYTHON_ENABLE_DEBUG_LOGGING, SCRIPT_FILE_NAME,
-                        PYTHON_LANGUAGE_RUNTIME)
+                        PYTHON_LANGUAGE_RUNTIME, PYTHON_LOAD_FUNCTIONS_INIT)
 from .extension import ExtensionManager
 from .logging import disable_console_logging, enable_console_logging
 from .logging import enable_debug_logging_recommendation
@@ -288,6 +288,10 @@ async def _handle__worker_init_request(self, request):
         if not DependencyManager.is_in_linux_consumption():
             DependencyManager.prioritize_customer_dependencies()
 
+        if DependencyManager.is_in_linux_consumption() \
+                and is_envvar_true(PYTHON_LOAD_FUNCTIONS_INIT):
+            import azure.functions  # NoQA
+
         return protos.StreamingMessage(
             request_id=self.request_id,
             worker_init_response=protos.WorkerInitResponse(
@@ -450,6 +454,10 @@ async def _handle__invocation_request(self, request):
                     shmem_mgr=self._shmem_mgr)
 
             fi_context = self._get_context(invoc_request, fi.name, fi.directory)
+
+            # Use local thread storage to store the invocation ID
+            # for a customer's threads
+            fi_context.thread_local_storage.invocation_id = invocation_id
             if fi.requires_context:
                 args['context'] = fi_context
 
@@ -525,10 +533,11 @@ async def _handle__function_environment_reload_request(self, request):
             func_env_reload_request = \
                 request.function_environment_reload_request
 
-            # Import before clearing path cache so that the default
-            # azure.functions modules is available in sys.modules for
-            # customer use
-            import azure.functions  # NoQA
+            if not is_envvar_true(PYTHON_LOAD_FUNCTIONS_INIT):
+                # Import before clearing path cache so that the default
+                # azure.functions modules is available in sys.modules for
+                # customer use
+                import azure.functions  # NoQA
 
             # Append function project root to module finding sys.path
             if func_env_reload_request.function_app_directory:
@@ -660,7 +669,7 @@ def _get_context(invoc_request: protos.InvocationRequest, name: str,
 
         return bindings.Context(
             name, directory, invoc_request.invocation_id,
-            trace_context, retry_context)
+            _invocation_id_local, trace_context, retry_context)
 
     @disable_feature_by(constants.PYTHON_ROLLBACK_CWD_PATH)
     def _change_cwd(self, new_cwd: str):
@@ -728,12 +737,12 @@ def _create_sync_call_tp(
     def _run_sync_func(self, invocation_id, context, func, params):
         # This helper exists because we need to access the current
         # invocation_id from ThreadPoolExecutor's threads.
-        _invocation_id_local.v = invocation_id
+        context.thread_local_storage.invocation_id = invocation_id
         try:
             return ExtensionManager.get_sync_invocation_wrapper(context,
                                                                 func)(params)
         finally:
-            _invocation_id_local.v = None
+            context.thread_local_storage.invocation_id = None
 
     async def _run_async_func(self, context, func, params):
         return await ExtensionManager.get_async_invocation_wrapper(
@@ -837,7 +846,7 @@ def get_current_invocation_id() -> Optional[str]:
             if task_invocation_id is not None:
                 return task_invocation_id
 
-    return getattr(_invocation_id_local, 'v', None)
+    return getattr(_invocation_id_local, 'invocation_id', None)
 
 
 _invocation_id_local = threading.local()

azure_functions_worker/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 
-VERSION = '4.10.1'
+VERSION = '4.11.0'

python/prodV4/worker.py

@@ -53,13 +53,13 @@ def determine_user_pkg_paths():
     func_worker_dir = str(Path(__file__).absolute().parent)
     env = os.environ
 
-    if is_azure_environment():
-        user_pkg_paths = determine_user_pkg_paths()
-        joined_pkg_paths = os.pathsep.join(user_pkg_paths)
+    # Setting up python path for all environments to prioritize
+    # third-party user packages over worker packages in PYTHONPATH
+    user_pkg_paths = determine_user_pkg_paths()
+    joined_pkg_paths = os.pathsep.join(user_pkg_paths)
+    env['PYTHONPATH'] = f'{joined_pkg_paths}:{func_worker_dir}'
 
-        # On cloud, we prioritize third-party user packages
-        # over worker packages in PYTHONPATH
-        env['PYTHONPATH'] = f'{joined_pkg_paths}:{func_worker_dir}'
+    if is_azure_environment():
         os.execve(sys.executable,
                   [sys.executable, '-m', 'azure_functions_worker']
                   + sys.argv[1:],