diff --git a/.ci/scripts/backend-test-linux.sh b/.ci/scripts/backend-test-linux.sh new file mode 100755 index 00000000000..ac10f3b94b8 --- /dev/null +++ b/.ci/scripts/backend-test-linux.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +set -eux + +SUITE=$1 +FLOW=$2 +ARTIFACT_DIR=$3 + +echo "Running backend test job for suite $SUITE, flow $FLOW." +echo "Saving job artifacts to $ARTIFACT_DIR." + +# The generic Linux job chooses to use base env, not the one setup by the image +eval "$(conda shell.bash hook)" +CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") +conda activate "${CONDA_ENV}" + +# Setup swiftshader and Vulkan SDK which are required to build the Vulkan delegate +#source .ci/scripts/setup-vulkan-linux-deps.sh + +# We need the runner to test the built library. +PYTHON_EXECUTABLE=python .ci/scripts/setup-linux.sh --build-tool cmake --build-mode Release + +python -m executorch.backends.test.suite.runner $SUITE --flow $FLOW --report "$ARTIFACT_DIR/test_results.csv" diff --git a/.ci/scripts/setup-linux.sh b/.ci/scripts/setup-linux.sh index a090571ab49..feb8a128b17 100755 --- a/.ci/scripts/setup-linux.sh +++ b/.ci/scripts/setup-linux.sh @@ -11,6 +11,7 @@ set -exu source "$(dirname "${BASH_SOURCE[0]}")/utils.sh" read -r BUILD_TOOL BUILD_MODE EDITABLE < <(parse_args "$@") +echo "Build tool: $BUILD_TOOL, Mode: $BUILD_MODE" # As Linux job is running inside a Docker container, all of its dependencies # have already been installed, so we use PyTorch build from source here instead diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 4658fdc0d26..87e2da638b8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -36,3 +36,24 @@ jobs: uses: ./.github/workflows/_link_check.yml with: ref: ${{ github.sha }} + + backend-test-linux: + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + strategy: + fail-fast: false + matrix: + flow: [xnnpack, xnnpack_static_int8_per_channel] + suite: [models, operators] + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + runner: linux.2xlarge + docker-image: ci-image:executorch-ubuntu-22.04-clang12 + submodules: recursive + timeout: 90 + upload-artifact: test-report-${{ matrix.flow }}-${{ matrix.suite }} + script: | + set -eux + # Intentionally suppressing exit code for now. + # TODO (gjcomer) Remove this when jobs are stable. + .ci/scripts/backend-test-linux.sh "${{ matrix.suite }}" "${{ matrix.flow }}" "${RUNNER_ARTIFACT_DIR}" || EXIT_CODE=$? + echo "Test run complete with exit code $EXIT_CODE." diff --git a/backends/test/suite/runner.py b/backends/test/suite/runner.py index 7a1fb64989a..3729d94cdf3 100644 --- a/backends/test/suite/runner.py +++ b/backends/test/suite/runner.py @@ -15,6 +15,7 @@ # Set of unsupported ops that should cause tests to be skipped UNSUPPORTED_PORTABLE_OPS = { "aten::_embedding_bag", + "aten::_adaptive_avg_pool2d", "aten::median", "aten::median.dim", "aten::round.decimals",