Skip to content

Commit 7ded926

Browse files
GregoryComerGregoryComer
committed
[Windows] Add CI wheel build jobs for Windows
ghstack-source-id: 094157c ghstack-comment-id: 3240461597 Pull-Request: #13837
1 parent 66799e3 commit 7ded926

File tree

8 files changed

+246
-4
lines changed

8 files changed

+246
-4
lines changed

‎.ci/scripts/test_model.ps1‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ function ExportModel-Xnnpack {
3434
[bool]$quantize
3535
)
3636

37-
if $(quantize) {
37+
if ($quantize) {
3838
python -m examples.xnnpack.aot_compiler --model_name="${MODEL_NAME}" --delegate --quantize | Write-Host
3939
$modelFile = "$($modelName)_xnnpack_q8.pte"
4040
} else {

‎.ci/scripts/wheel/pre_build_script.sh‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,26 @@ set -euxo pipefail
99

1010
# This script is run before building ExecuTorch binaries
1111

12+
# Clone nested submodules for tokenizers - this is a workaround for recursive
13+
# submodule clone failing due to path length limitations on Windows. Eventually,
14+
# we should update the core job in test-infra to enable long paths before
15+
# checkout to avoid needing to do this.
16+
pushd extension/llm/tokenizers
17+
git submodule update --init
18+
popd
19+
20+
# On Windows, enable symlinks and re-checkout the current revision to create
21+
# the symlinked src/ directory. This is needed to build the wheel.
22+
UNAME_S=$(uname -s)
23+
if [[ $UNAME_S == *"MINGW"* || $UNAME_S == *"MSYS"* ]]; then
24+
echo "Enabling symlinks on Windows"
25+
git config core.symlinks true
26+
git checkout -f HEAD
27+
fi
28+
1229
# Manually install build requirements because `python setup.py bdist_wheel` does
1330
# not install them. TODO(dbort): Switch to using `python -m build --wheel`,
1431
# which does install them. Though we'd need to disable build isolation to be
1532
# able to see the installed torch package.
1633

17-
"${GITHUB_WORKSPACE}/${REPOSITORY}/install_requirements.sh" --example
34+
"${GITHUB_WORKSPACE}/${REPOSITORY}/install_requirements.sh" --example

‎.ci/scripts/wheel/test_windows.py‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
#!/usr/bin/env python
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
from typing import List
9+
10+
import sys
11+
import executorch
12+
13+
import torch
14+
from executorch.backends.xnnpack.partition.xnnpack_partitioner import XnnpackPartitioner
15+
from executorch.examples.models import Backend, Model, MODEL_NAME_TO_MODEL
16+
from executorch.examples.models.model_factory import EagerModelFactory
17+
from executorch.examples.xnnpack import MODEL_NAME_TO_OPTIONS
18+
from executorch.examples.xnnpack.quantization.utils import quantize as quantize_xnn
19+
from executorch.exir import EdgeCompileConfig, to_edge_transform_and_lower
20+
from executorch.extension.pybindings.portable_lib import (
21+
_load_for_executorch_from_buffer,
22+
)
23+
from test_base import ModelTest
24+
25+
26+
def test_model_xnnpack(model: Model, quantize: bool) -> None:
27+
model_instance, example_inputs, _, _ = EagerModelFactory.create_model(
28+
*MODEL_NAME_TO_MODEL[str(model)]
29+
)
30+
31+
model_instance.eval()
32+
ref_outputs = model_instance(*example_inputs)
33+
34+
if quantize:
35+
quant_type = MODEL_NAME_TO_OPTIONS[str(model)].quantization
36+
model_instance = torch.export.export_for_training(
37+
model_instance, example_inputs
38+
)
39+
model_instance = quantize_xnn(
40+
model_instance.module(), example_inputs, quant_type
41+
)
42+
43+
lowered = to_edge_transform_and_lower(
44+
torch.export.export(model_instance, example_inputs),
45+
partitioner=[XnnpackPartitioner()],
46+
compile_config=EdgeCompileConfig(
47+
_check_ir_validity=False,
48+
),
49+
).to_executorch()
50+
51+
loaded_model = _load_for_executorch_from_buffer(lowered.buffer)
52+
et_outputs = loaded_model([*example_inputs])
53+
54+
if isinstance(ref_outputs, torch.Tensor):
55+
ref_outputs = (ref_outputs,)
56+
57+
assert len(ref_outputs) == len(et_outputs)
58+
for i in range(len(ref_outputs)):
59+
assert torch.allclose(ref_outputs[i], et_outputs[i], atol=1e-5)
60+
61+
62+
def run_tests(model_tests: List[ModelTest]) -> None:
63+
for model_test in model_tests:
64+
if model_test.backend == Backend.Xnnpack:
65+
test_model_xnnpack(model_test.model, quantize=False)
66+
else:
67+
raise RuntimeError(f"Unsupported backend {model_test.backend}.")
68+
69+
70+
if __name__ == "__main__":
71+
run_tests(
72+
model_tests=[
73+
ModelTest(
74+
model=Model.Mv3,
75+
backend=Backend.Xnnpack,
76+
),
77+
]
78+
)

‎.ci/scripts/wheel/vc_env_helper.bat‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
REM This is lightly modified from the torchvision Windows build logic.
2+
REM See https://github.com/pytorch/vision/blob/main/packaging/windows/internal/vc_env_helper.bat
3+
4+
@echo on
5+
6+
set VC_VERSION_LOWER=17
7+
set VC_VERSION_UPPER=18
8+
9+
for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [%VC_VERSION_LOWER%^,%VC_VERSION_UPPER%^) -property installationPath`) do (
10+
if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" (
11+
set "VS15INSTALLDIR=%%i"
12+
set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat"
13+
goto vswhere
14+
)
15+
)
16+
17+
:vswhere
18+
if "%VSDEVCMD_ARGS%" == "" (
19+
call "%VS15VCVARSALL%" x64 || exit /b 1
20+
) else (
21+
call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS% || exit /b 1
22+
)
23+
24+
@echo on
25+
26+
if "%CU_VERSION%" == "xpu" call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
27+
28+
set DISTUTILS_USE_SDK=1
29+
30+
set args=%1
31+
shift
32+
:start
33+
if [%1] == [] goto done
34+
set args=%args% %1
35+
shift
36+
goto start
37+
38+
:done
39+
if "%args%" == "" (
40+
echo Usage: vc_env_helper.bat [command] [args]
41+
echo e.g. vc_env_helper.bat cl /c test.cpp
42+
)
43+
44+
echo "Evaluating symlink status. CWD: %CD%"
45+
set work_dir=%CD%
46+
if exist setup.py (
47+
echo "Creating symlink..."
48+
REM Setup a symlink to shorten the path length.
49+
REM Note that the ET directory has to be named "executorch".
50+
cd %GITHUB_WORKSPACE%
51+
if not exist et\ (
52+
mkdir et
53+
)
54+
cd et
55+
echo Work dir: %work_dir%
56+
if not exist executorch\ (
57+
mklink /d executorch %work_dir%
58+
)
59+
cd executorch
60+
)
61+
echo "Post symlink CWD: %CD%"
62+
63+
%args% || exit /b 1

‎.github/workflows/build-wheels-windows.yml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: Build Windows Wheels
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- nightly
8+
- main
9+
- release/*
10+
tags:
11+
# NOTE: Binary build pipelines should only get triggered on release candidate builds
12+
# Release candidate tags look like: v1.11.0-rc1
13+
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
14+
workflow_dispatch:
15+
16+
permissions:
17+
id-token: write
18+
contents: read
19+
20+
jobs:
21+
generate-matrix:
22+
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
23+
with:
24+
package-type: wheel
25+
os: windows
26+
test-infra-repository: pytorch/test-infra
27+
test-infra-ref: main
28+
with-cuda: disabled
29+
with-rocm: disabled
30+
python-versions: '["3.10", "3.11", "3.12"]'
31+
32+
build:
33+
needs: generate-matrix
34+
strategy:
35+
fail-fast: false
36+
matrix:
37+
include:
38+
- repository: pytorch/executorch
39+
pre-script: .ci\\scripts\\wheel\\pre_build_script.sh
40+
env-script: .ci\\scripts\\wheel\\vc_env_helper.bat
41+
post-script: .ci\\scripts\\wheel\\post_build_script.sh
42+
smoke-test-script: .ci/scripts/wheel/test_windows.py
43+
package-name: executorch
44+
name: ${{ matrix.repository }}
45+
uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main
46+
with:
47+
repository: ${{ matrix.repository }}
48+
ref: ""
49+
test-infra-repository: pytorch/test-infra
50+
test-infra-ref: main
51+
build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
52+
pre-script: ${{ matrix.pre-script }}
53+
env-script: ${{ matrix.env-script }}
54+
post-script: ${{ matrix.post-script }}
55+
package-name: ${{ matrix.package-name }}
56+
smoke-test-script: ${{ matrix.smoke-test-script }}
57+
trigger-event: ${{ github.event_name }}
58+
wheel-build-params: "--verbose"
59+
submodules: true

‎examples/models/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ def __str__(self) -> str:
4444

4545

4646
class Backend(str, Enum):
47+
Xnnpack = ("xnnpack",)
4748
XnnpackQuantizationDelegation = "xnnpack-quantization-delegation"
4849
CoreMlExportOnly = "coreml"
4950
CoreMlExportAndTest = "coreml-test" # AOT export + test with runner

‎setup.py‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,20 @@ def string(cls) -> str:
142142
@classmethod
143143
def write_to_python_file(cls, path: str) -> None:
144144
"""Creates a file similar to PyTorch core's `torch/version.py`."""
145+
146+
print(f"Writing version.py to {path}, cwd: {os.getcwd()}", file=sys.stderr)
147+
148+
os.makedirs(os.path.dirname(path), exist_ok=True)
149+
150+
try:
151+
path1 = os.path.dirname(path)
152+
path2 = os.path.dirname(path1)
153+
154+
print(f"Files at {path2}: {list(os.listdir(path2))}", file=sys.stderr)
155+
print(f"Files at {path1}: {list(os.listdir(path1))}", file=sys.stderr)
156+
finally:
157+
pass
158+
145159
lines = [
146160
"from typing import Optional",
147161
'__all__ = ["__version__", "git_version"]',
@@ -152,6 +166,9 @@ def write_to_python_file(cls, path: str) -> None:
152166
with open(path, "w") as fp:
153167
fp.write("\n".join(lines) + "\n")
154168

169+
file_dir = os.path.dirname(path)
170+
print(f"Files: {list(os.listdir(file_dir))}", file=sys.stderr)
171+
155172

156173
# The build type is determined by the DEBUG environment variable. If DEBUG is
157174
# set to a non-empty value, the build type is Debug. Otherwise, the build type
@@ -773,7 +790,7 @@ def run(self): # noqa C901
773790
# platform-specific files using InstallerBuildExt.
774791
ext_modules=[
775792
BuiltFile(
776-
src_dir="%CMAKE_CACHE_DIR%/third-party/flatc_proj/bin/",
793+
src_dir="%CMAKE_CACHE_DIR%/third-party/flatc_ep/bin/",
777794
src_name="flatc",
778795
dst="executorch/data/bin/",
779796
is_executable=True,

‎third-party/CMakeLists.txt‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,17 @@ else()
2525
set(_executorch_external_project_additional_args CMAKE_GENERATOR "Unix Makefiles")
2626
endif()
2727

28+
# DEBUG
29+
if(WIN32)
30+
set_directory_properties(PROPERTIES EP_BASE "C:\\actions-runner\\_work\\")
31+
endif()
32+
2833
# We use ExternalProject to build flatc from source to force it target the host.
2934
# Otherwise, flatc will target the project's toolchain (i.e. iOS, or Android).
3035
ExternalProject_Add(
3136
flatbuffers_ep
32-
PREFIX ${CMAKE_CURRENT_BINARY_DIR}/flatc_proj
37+
PREFIX ${CMAKE_CURRENT_BINARY_DIR}/flatc_ep
38+
BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/flatc_ep/src/build
3339
SOURCE_DIR ${PROJECT_SOURCE_DIR}/third-party/flatbuffers
3440
CMAKE_ARGS -DFLATBUFFERS_BUILD_FLATC=ON
3541
-DFLATBUFFERS_INSTALL=ON
@@ -82,6 +88,7 @@ ExternalProject_Add(
8288
flatcc_ep
8389
PREFIX ${CMAKE_CURRENT_BINARY_DIR}/flatcc_ep
8490
SOURCE_DIR ${PROJECT_SOURCE_DIR}/third-party/flatcc
91+
BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/flatcc_ep/src/build
8592
CMAKE_ARGS -DFLATCC_RTONLY=OFF
8693
-DFLATCC_TEST=OFF
8794
-DFLATCC_REFLECTION=OFF

0 commit comments

Comments
 (0)