From 82d81f2a8bd3349be0cac9eed49b95830e49ba62 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 28 Dec 2018 00:23:31 +0100 Subject: [PATCH 01/17] fixup! ci/lib.sh: add support for Azure Pipelines TASKDEFINITIONSURI is not set in parallel tasks... And COLLECTIONURI is not set in the Linux32 job... Signed-off-by: Johannes Schindelin --- ci/lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/lib.sh b/ci/lib.sh index 21d430f15772fb..0dee59706c42b8 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -19,7 +19,7 @@ then BREW_INSTALL_PACKAGES="git-lfs gettext" export GIT_PROVE_OPTS="--timer --jobs 3 --state=failed,slow,save" export GIT_TEST_OPTS="--verbose-log -x --immediate" -elif test -n "$SYSTEM_TASKDEFINITIONSURI" +elif test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI" then # We are running in Azure Pipelines CI_BRANCH="$BUILD_SOURCEBRANCH" From a2379f703128b5c019dcef245e1eb4c898662f24 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 15 Jan 2019 19:46:25 +0100 Subject: [PATCH 02/17] fixup! Add a build definition for Azure DevOps For some reason, the indentation of the PowerShell tasks got all messed up. Fix that. While at it, do not define `c()` for only one user, and do not initialize `/etc/passwd` (it does not seem to have a noticeable effect). Signed-off-by: Johannes Schindelin --- azure-pipelines.yml | 120 +++++++++++++++++++++----------------------- 1 file changed, 56 insertions(+), 64 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 757c43c5f48a76..b015d6609fb9d3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -167,81 +167,73 @@ phases: timeoutInMinutes: 240 steps: - powershell: | - if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { - net use s: \\gitfileshare.file.core.windows.net\test-cache "$GITFILESHAREPWD" /user:AZURE\gitfileshare /persistent:no - cmd /c mklink /d "$(Build.SourcesDirectory)\test-cache" S:\ - } + if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { + net use s: \\gitfileshare.file.core.windows.net\test-cache "$GITFILESHAREPWD" /user:AZURE\gitfileshare /persistent:no + cmd /c mklink /d "$(Build.SourcesDirectory)\test-cache" S:\ + } displayName: 'Mount test-cache' env: GITFILESHAREPWD: $(gitfileshare.pwd) - powershell: | - # Helper to check the error level of the latest command (exit with error when appropriate) - function c() { if (!$?) { exit(1) } } - - # Add build agent's MinGit to PATH - $env:PATH = $env:AGENT_HOMEDIRECTORY +"\externals\\git\cmd;" +$env:PATH - - # Helper to initialize (or update) a Git worktree - function init ($path, $url, $set_origin) { - if (Test-Path $path) { - cd $path; c - if (Test-Path .git) { - git init; c - } else { - git status - } - } else { - git init $path; c - cd $path; c - } - git config core.autocrlf false; c - git config core.untrackedCache true; c - if (($set_origin -ne 0) -and !(git config remote.origin.url)) { - git remote add origin $url; c - } - git fetch --depth=1 $url master; c - git reset --hard FETCH_HEAD; c - git clean -df; c - } - - # Initialize Git for Windows' SDK - $sdk_path = "$(Build.SourcesDirectory)\git-sdk-64" - init "$sdk_path" "https://dev.azure.com/git-for-windows/git-sdk-64/_git/git-sdk-64" 0 - init usr\src\build-extra https://github.com/git-for-windows/build-extra 1 - - # Let Git ignore the SDK and the test-cache - "/git-sdk-64/`n/test-cache/`n" | Out-File -NoNewLine -Encoding ascii -Append "$(Build.SourcesDirectory)\.git\info\exclude" - - # Help MSYS2 runtime startup by initializing /etc/passwd - & "$sdk_path\git-cmd" --command=usr\\bin\\bash.exe -lc "mkpasswd -c >>/etc/passwd"; c + # Helper to check the error level of the latest command (exit with error when appropriate) + function c() { if (!$?) { exit(1) } } + + # Add build agent's MinGit to PATH + $env:PATH = $env:AGENT_HOMEDIRECTORY +"\externals\\git\cmd;" +$env:PATH + + # Helper to initialize (or update) a Git worktree + function init ($path, $url, $set_origin) { + if (Test-Path $path) { + cd $path; c + if (Test-Path .git) { + git init; c + } else { + git status + } + } else { + git init $path; c + cd $path; c + } + git config core.autocrlf false; c + git config core.untrackedCache true; c + if (($set_origin -ne 0) -and !(git config remote.origin.url)) { + git remote add origin $url; c + } + git fetch --depth=1 $url master; c + git reset --hard FETCH_HEAD; c + git clean -df; c + } + + # Initialize Git for Windows' SDK + $sdk_path = "$(Build.SourcesDirectory)\git-sdk-64" + init "$sdk_path" "https://dev.azure.com/git-for-windows/git-sdk-64/_git/git-sdk-64" 0 + init usr\src\build-extra https://github.com/git-for-windows/build-extra 1 + + # Let Git ignore the SDK and the test-cache + "/git-sdk-64/`n/test-cache/`n" | Out-File -NoNewLine -Encoding ascii -Append "$(Build.SourcesDirectory)\.git\info\exclude" displayName: 'Initialize the Git for Windows SDK' - powershell: | - # Helper to check the error level of the latest command (exit with error when appropriate) - function c() { if (!$?) { exit(1) } } - - cd "$(Build.SourcesDirectory)"; c - - git-sdk-64\git-cmd --command=usr\\bin\\bash.exe -lc @" - export MAKEFLAGS=-j10 - export DEVELOPER=1 - export NO_PERL=1 - export NO_SVN_TESTS=1 - export GIT_TEST_SKIP_REBASE_P=1 - - ci/run-build-and-tests.sh || { - ci/print-test-failures.sh - exit 1 - } - "@ - c + git-sdk-64\git-cmd --command=usr\\bin\\bash.exe -lc @" + export MAKEFLAGS=-j10 + export DEVELOPER=1 + export NO_PERL=1 + export NO_SVN_TESTS=1 + export GIT_TEST_SKIP_REBASE_P=1 + + ci/run-build-and-tests.sh || { + ci/print-test-failures.sh + exit 1 + } + "@ + if (!$?) { exit(1) } displayName: 'Build & Test' env: HOME: $(Build.SourcesDirectory) MSYSTEM: MINGW64 - powershell: | - if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { - cmd /c rmdir "$(Build.SourcesDirectory)\test-cache" - } + if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { + cmd /c rmdir "$(Build.SourcesDirectory)\test-cache" + } displayName: 'Unmount test-cache' condition: true env: From ca997ef3e9d592aa54d66766f913fb7019c49197 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 15 Jan 2019 16:56:06 +0100 Subject: [PATCH 03/17] fixup! Add a build definition for Azure DevOps At some point we switched from Hosted Linux Preview to Ubuntu 16.04, at which point it became unnecessary to install the `docker` package for the Linux-32 job. Signed-off-by: Johannes Schindelin --- azure-pipelines.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b015d6609fb9d3..ec44642333c643 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -256,21 +256,6 @@ phases: - bash: | test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 - sudo apt-get update && - sudo rm /var/lib/apt/lists/lock && - sudo apt-get -y install \ - apt-transport-https \ - ca-certificates \ - curl \ - software-properties-common && - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - && - sudo add-apt-repository \ - "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ - $(lsb_release -cs) \ - stable" && - sudo apt-get update && - sudo apt-get -y install docker-ce && - sudo AGENT_OS="$AGENT_OS" BUILD_BUILDNUMBER="$BUILD_BUILDNUMBER" BUILD_REPOSITORY_URI="$BUILD_REPOSITORY_URI" BUILD_SOURCEBRANCH="$BUILD_SOURCEBRANCH" BUILD_SOURCEVERSION="$BUILD_SOURCEVERSION" SYSTEM_PHASENAME="$SYSTEM_PHASENAME" SYSTEM_TASKDEFINITIONSURI="$SYSTEM_TASKDEFINITIONSURI" SYSTEM_TEAMPROJECT="$SYSTEM_TEAMPROJECT" CC=$CC MAKEFLAGS=-j3 bash -lxc ci/run-linux32-docker.sh || exit 1 sudo chmod a+r t/out/TEST-*.xml From e1a2142157de45d691134ce679f1ab5cb21a47f4 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 15 Jan 2019 17:25:59 +0100 Subject: [PATCH 04/17] fixup! Add a build definition for Azure DevOps Remove a trailing empty line. Signed-off-by: Johannes Schindelin --- ci/mount-fileshare.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/mount-fileshare.sh b/ci/mount-fileshare.sh index 5fb5f74b705cb4..a967be9af53f3c 100755 --- a/ci/mount-fileshare.sh +++ b/ci/mount-fileshare.sh @@ -23,4 +23,3 @@ Darwin) ;; esac || die "Could not mount $4" - From 760ff732cb658a5a95422cf67c80f87a32422dd1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 19 Dec 2018 14:54:35 +0100 Subject: [PATCH 05/17] fixup! Add a build definition for Azure DevOps Let's use the new YAML schema. See https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema Signed-off-by: Johannes Schindelin --- azure-pipelines.yml | 49 ++++++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ec44642333c643..2c0834d66a359a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -2,12 +2,11 @@ resources: - repo: self fetchDepth: 1 -phases: -- phase: linux_clang +jobs: +- job: linux_clang displayName: linux-clang condition: succeeded() - queue: - name: Hosted Ubuntu 1604 + pool: Hosted Ubuntu 1604 steps: - bash: | test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 @@ -37,11 +36,10 @@ phases: publishRunAttachments: false condition: succeededOrFailed() -- phase: linux_gcc +- job: linux_gcc displayName: linux-gcc condition: succeeded() - queue: - name: Hosted Ubuntu 1604 + pool: Hosted Ubuntu 1604 steps: - bash: | test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 @@ -69,11 +67,10 @@ phases: publishRunAttachments: false condition: succeededOrFailed() -- phase: osx_clang +- job: osx_clang displayName: osx-clang condition: succeeded() - queue: - name: Hosted macOS + pool: Hosted macOS steps: - bash: | test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 @@ -99,11 +96,10 @@ phases: publishRunAttachments: false condition: succeededOrFailed() -- phase: osx_gcc +- job: osx_gcc displayName: osx-gcc condition: succeeded() - queue: - name: Hosted macOS + pool: Hosted macOS steps: - bash: | test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 @@ -127,11 +123,10 @@ phases: publishRunAttachments: false condition: succeededOrFailed() -- phase: gettext_poison +- job: gettext_poison displayName: GETTEXT_POISON condition: succeeded() - queue: - name: Hosted Ubuntu 1604 + pool: Hosted Ubuntu 1604 steps: - bash: | test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 @@ -160,11 +155,10 @@ phases: publishRunAttachments: false condition: succeededOrFailed() -- phase: windows +- job: windows displayName: Windows - queue: - name: Hosted - timeoutInMinutes: 240 + pool: Hosted + timeoutInMinutes: 240 steps: - powershell: | if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { @@ -247,11 +241,10 @@ phases: publishRunAttachments: false condition: succeededOrFailed() -- phase: linux32 +- job: linux32 displayName: Linux32 condition: succeeded() - queue: - name: Hosted Ubuntu 1604 + pool: Hosted Ubuntu 1604 steps: - bash: | test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 @@ -273,11 +266,10 @@ phases: publishRunAttachments: false condition: succeededOrFailed() -- phase: static_analysis +- job: static_analysis displayName: StaticAnalysis condition: succeeded() - queue: - name: Hosted Ubuntu 1604 + pool: Hosted Ubuntu 1604 steps: - bash: | test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 @@ -295,11 +287,10 @@ phases: env: GITFILESHAREPWD: $(gitfileshare.pwd) -- phase: documentation +- job: documentation displayName: Documentation condition: succeeded() - queue: - name: Hosted Ubuntu 1604 + pool: Hosted Ubuntu 1604 steps: - bash: | test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 From 6934fe704605621336b854a85a265ab094d578f2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 19 Dec 2018 13:23:29 +0100 Subject: [PATCH 06/17] fixup! Add a build definition for Azure DevOps Some improvements (suggested via review on the Git mailing list) have not been incorporated into Git for Windows' `master` yet, other changes were left-overs from debugging that need reverting. Signed-off-by: Johannes Schindelin --- azure-pipelines.yml | 17 +++++++---------- ci/mount-fileshare.sh | 2 +- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2c0834d66a359a..c5d5f5bca7aa95 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -12,12 +12,11 @@ jobs: test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 sudo apt-get update && - sudo rm /var/lib/apt/lists/lock && sudo apt-get -y install git gcc make libssl-dev libcurl4-openssl-dev libexpat-dev tcl tk gettext git-email zlib1g-dev apache2-bin && export CC=clang || exit 1 - ci/install-dependencies.sh + ci/install-dependencies.sh || exit 1 ci/run-build-and-tests.sh || { ci/print-test-failures.sh exit 1 @@ -44,11 +43,11 @@ jobs: - bash: | test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 + sudo add-apt-repository ppa:ubuntu-toolchain-r/test && sudo apt-get update && - sudo rm /var/lib/apt/lists/lock && - sudo apt-get -y install git gcc make libssl-dev libcurl4-openssl-dev libexpat-dev tcl tk gettext git-email zlib1g-dev apache2-bin || exit 1 + sudo apt-get -y install git gcc make libssl-dev libcurl4-openssl-dev libexpat-dev tcl tk gettext git-email zlib1g-dev apache2 language-pack-is git-svn gcc-8 || exit 1 - ci/install-dependencies.sh + ci/install-dependencies.sh || exit 1 ci/run-build-and-tests.sh || { ci/print-test-failures.sh exit 1 @@ -77,7 +76,7 @@ jobs: export CC=clang - ci/install-dependencies.sh + ci/install-dependencies.sh || exit 1 ci/run-build-and-tests.sh || { ci/print-test-failures.sh exit 1 @@ -104,7 +103,7 @@ jobs: - bash: | test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 - ci/install-dependencies.sh + ci/install-dependencies.sh || exit 1 ci/run-build-and-tests.sh || { ci/print-test-failures.sh exit 1 @@ -132,7 +131,6 @@ jobs: test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 sudo apt-get update && - sudo rm /var/lib/apt/lists/lock && sudo apt-get -y install git gcc make libssl-dev libcurl4-openssl-dev libexpat-dev tcl tk gettext git-email zlib1g-dev && export jobname=GETTEXT_POISON || exit 1 @@ -157,6 +155,7 @@ jobs: - job: windows displayName: Windows + condition: succeeded() pool: Hosted timeoutInMinutes: 240 steps: @@ -275,7 +274,6 @@ jobs: test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 sudo apt-get update && - sudo rm /var/lib/apt/lists/lock && sudo apt-get install -y coccinelle && export jobname=StaticAnalysis && @@ -296,7 +294,6 @@ jobs: test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 sudo apt-get update && - sudo rm /var/lib/apt/lists/lock && sudo apt-get install -y asciidoc xmlto asciidoctor && export ALREADY_HAVE_ASCIIDOCTOR=yes. && diff --git a/ci/mount-fileshare.sh b/ci/mount-fileshare.sh index a967be9af53f3c..26b58a80960f78 100755 --- a/ci/mount-fileshare.sh +++ b/ci/mount-fileshare.sh @@ -6,7 +6,7 @@ die () { } test $# = 4 || -die "Usage: $0 " mkdir -p "$4" || die "Could not create $4" From 02451caa3c490ec81ec81b02f9a5f3ca8a671145 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 19 Dec 2018 13:23:29 +0100 Subject: [PATCH 07/17] fixup! Add a build definition for Azure DevOps We do not need build-extra. So let's not clone it. Signed-off-by: Johannes Schindelin --- azure-pipelines.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c5d5f5bca7aa95..54675d3eecbbb2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -200,7 +200,6 @@ jobs: # Initialize Git for Windows' SDK $sdk_path = "$(Build.SourcesDirectory)\git-sdk-64" init "$sdk_path" "https://dev.azure.com/git-for-windows/git-sdk-64/_git/git-sdk-64" 0 - init usr\src\build-extra https://github.com/git-for-windows/build-extra 1 # Let Git ignore the SDK and the test-cache "/git-sdk-64/`n/test-cache/`n" | Out-File -NoNewLine -Encoding ascii -Append "$(Build.SourcesDirectory)\.git\info\exclude" From a4c8e460197f3ce9a8a7b189c517a833c99edcc9 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 19 Dec 2018 13:23:29 +0100 Subject: [PATCH 08/17] fixup! Add a build definition for Azure DevOps It is not strictly necessary to start a command in PowerShell with `&`, but it is safer. Signed-off-by: Johannes Schindelin --- azure-pipelines.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 54675d3eecbbb2..f3cabb0dd0ce7c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -179,22 +179,22 @@ jobs: if (Test-Path $path) { cd $path; c if (Test-Path .git) { - git init; c + & git init; c } else { - git status + & git status } } else { - git init $path; c + & git init $path; c cd $path; c } - git config core.autocrlf false; c - git config core.untrackedCache true; c + & git config core.autocrlf false; c + & git config core.untrackedCache true; c if (($set_origin -ne 0) -and !(git config remote.origin.url)) { - git remote add origin $url; c + & git remote add origin $url; c } - git fetch --depth=1 $url master; c - git reset --hard FETCH_HEAD; c - git clean -df; c + & git fetch --depth=1 $url master; c + & git reset --hard FETCH_HEAD; c + & git clean -df; c } # Initialize Git for Windows' SDK @@ -205,7 +205,7 @@ jobs: "/git-sdk-64/`n/test-cache/`n" | Out-File -NoNewLine -Encoding ascii -Append "$(Build.SourcesDirectory)\.git\info\exclude" displayName: 'Initialize the Git for Windows SDK' - powershell: | - git-sdk-64\git-cmd --command=usr\\bin\\bash.exe -lc @" + & "git-sdk-64\git-cmd.exe" --command=usr\\bin\\bash.exe -lc @" export MAKEFLAGS=-j10 export DEVELOPER=1 export NO_PERL=1 From f6fb98bdd2633627761a43db0d6a258cf0793897 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 15 Jan 2019 19:33:20 +0100 Subject: [PATCH 09/17] ci: move the Windows job to the top The Windows job currently takes a whopping ~1h20m to complete. Which is *far* longer than the next-longest job takes (linux-gcc, ~35m). As such, it makes sense to start the Windows job first, to minimize the overall run time (which is now pretty safely the run time of the Windows job). This affects only the Azure Pipelines configuration, not the Travis one, of course, as Travis cannot run our Windows job: 1h20m is distinctly longer than the 50 minute timeout of Travis' free tier. This commit is best viewed with `--color-moved`. Signed-off-by: Johannes Schindelin --- azure-pipelines.yml | 172 ++++++++++++++++++++++---------------------- 1 file changed, 86 insertions(+), 86 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f3cabb0dd0ce7c..e44d2733a47cdb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -3,6 +3,92 @@ resources: fetchDepth: 1 jobs: +- job: windows + displayName: Windows + condition: succeeded() + pool: Hosted + timeoutInMinutes: 240 + steps: + - powershell: | + if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { + net use s: \\gitfileshare.file.core.windows.net\test-cache "$GITFILESHAREPWD" /user:AZURE\gitfileshare /persistent:no + cmd /c mklink /d "$(Build.SourcesDirectory)\test-cache" S:\ + } + displayName: 'Mount test-cache' + env: + GITFILESHAREPWD: $(gitfileshare.pwd) + - powershell: | + # Helper to check the error level of the latest command (exit with error when appropriate) + function c() { if (!$?) { exit(1) } } + + # Add build agent's MinGit to PATH + $env:PATH = $env:AGENT_HOMEDIRECTORY +"\externals\\git\cmd;" +$env:PATH + + # Helper to initialize (or update) a Git worktree + function init ($path, $url, $set_origin) { + if (Test-Path $path) { + cd $path; c + if (Test-Path .git) { + & git init; c + } else { + & git status + } + } else { + & git init $path; c + cd $path; c + } + & git config core.autocrlf false; c + & git config core.untrackedCache true; c + if (($set_origin -ne 0) -and !(git config remote.origin.url)) { + & git remote add origin $url; c + } + & git fetch --depth=1 $url master; c + & git reset --hard FETCH_HEAD; c + & git clean -df; c + } + + # Initialize Git for Windows' SDK + $sdk_path = "$(Build.SourcesDirectory)\git-sdk-64" + init "$sdk_path" "https://dev.azure.com/git-for-windows/git-sdk-64/_git/git-sdk-64" 0 + + # Let Git ignore the SDK and the test-cache + "/git-sdk-64/`n/test-cache/`n" | Out-File -NoNewLine -Encoding ascii -Append "$(Build.SourcesDirectory)\.git\info\exclude" + displayName: 'Initialize the Git for Windows SDK' + - powershell: | + & "git-sdk-64\git-cmd.exe" --command=usr\\bin\\bash.exe -lc @" + export MAKEFLAGS=-j10 + export DEVELOPER=1 + export NO_PERL=1 + export NO_SVN_TESTS=1 + export GIT_TEST_SKIP_REBASE_P=1 + + ci/run-build-and-tests.sh || { + ci/print-test-failures.sh + exit 1 + } + "@ + if (!$?) { exit(1) } + displayName: 'Build & Test' + env: + HOME: $(Build.SourcesDirectory) + MSYSTEM: MINGW64 + - powershell: | + if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { + cmd /c rmdir "$(Build.SourcesDirectory)\test-cache" + } + displayName: 'Unmount test-cache' + condition: true + env: + GITFILESHAREPWD: $(gitfileshare.pwd) + - task: PublishTestResults@2 + displayName: 'Publish Test Results **/TEST-*.xml' + inputs: + mergeTestResults: true + testRunTitle: 'windows' + platform: Windows + publishRunAttachments: false + condition: succeededOrFailed() + - job: linux_clang displayName: linux-clang condition: succeeded() @@ -153,92 +239,6 @@ jobs: publishRunAttachments: false condition: succeededOrFailed() -- job: windows - displayName: Windows - condition: succeeded() - pool: Hosted - timeoutInMinutes: 240 - steps: - - powershell: | - if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { - net use s: \\gitfileshare.file.core.windows.net\test-cache "$GITFILESHAREPWD" /user:AZURE\gitfileshare /persistent:no - cmd /c mklink /d "$(Build.SourcesDirectory)\test-cache" S:\ - } - displayName: 'Mount test-cache' - env: - GITFILESHAREPWD: $(gitfileshare.pwd) - - powershell: | - # Helper to check the error level of the latest command (exit with error when appropriate) - function c() { if (!$?) { exit(1) } } - - # Add build agent's MinGit to PATH - $env:PATH = $env:AGENT_HOMEDIRECTORY +"\externals\\git\cmd;" +$env:PATH - - # Helper to initialize (or update) a Git worktree - function init ($path, $url, $set_origin) { - if (Test-Path $path) { - cd $path; c - if (Test-Path .git) { - & git init; c - } else { - & git status - } - } else { - & git init $path; c - cd $path; c - } - & git config core.autocrlf false; c - & git config core.untrackedCache true; c - if (($set_origin -ne 0) -and !(git config remote.origin.url)) { - & git remote add origin $url; c - } - & git fetch --depth=1 $url master; c - & git reset --hard FETCH_HEAD; c - & git clean -df; c - } - - # Initialize Git for Windows' SDK - $sdk_path = "$(Build.SourcesDirectory)\git-sdk-64" - init "$sdk_path" "https://dev.azure.com/git-for-windows/git-sdk-64/_git/git-sdk-64" 0 - - # Let Git ignore the SDK and the test-cache - "/git-sdk-64/`n/test-cache/`n" | Out-File -NoNewLine -Encoding ascii -Append "$(Build.SourcesDirectory)\.git\info\exclude" - displayName: 'Initialize the Git for Windows SDK' - - powershell: | - & "git-sdk-64\git-cmd.exe" --command=usr\\bin\\bash.exe -lc @" - export MAKEFLAGS=-j10 - export DEVELOPER=1 - export NO_PERL=1 - export NO_SVN_TESTS=1 - export GIT_TEST_SKIP_REBASE_P=1 - - ci/run-build-and-tests.sh || { - ci/print-test-failures.sh - exit 1 - } - "@ - if (!$?) { exit(1) } - displayName: 'Build & Test' - env: - HOME: $(Build.SourcesDirectory) - MSYSTEM: MINGW64 - - powershell: | - if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { - cmd /c rmdir "$(Build.SourcesDirectory)\test-cache" - } - displayName: 'Unmount test-cache' - condition: true - env: - GITFILESHAREPWD: $(gitfileshare.pwd) - - task: PublishTestResults@2 - displayName: 'Publish Test Results **/TEST-*.xml' - inputs: - mergeTestResults: true - testRunTitle: 'windows' - platform: Windows - publishRunAttachments: false - condition: succeededOrFailed() - - job: linux32 displayName: Linux32 condition: succeeded() From e241605ef3a8ad03c8ad22b1985fbd8929f3b4db Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 3 Jan 2019 11:53:43 +0100 Subject: [PATCH 10/17] ci: use git-sdk-64-minimal build artifact Instead of a shallow fetch followed by a sparse checkout, we are better off by using a separate, dedicated Pipeline that bundles the SDK as a build artifact, and then consuming that build artifact here. In fact, since this artifact will be used a lot, we spent substantial time on figuring out a minimal subset of the Git for Windows SDK, just enough to build and test Git. The result is a size reduction from around 1GB (compressed) to around 55MB (compressed). This also comes with the change where we now call `usr\bin\bash.exe` directly, as `git-cmd.exe` is not included in the minimal SDK. That reduces the time to initialize Git for Windows' SDK from anywhere between 2m30s-7m to a little over 1m. Note: in theory, we could also use the DownloadBuildArtifacts@0 task here. However, restricted permissions that are in effect when building from forks would let this fail for PR builds, defeating the whole purpose of the Azure Pipelines support for git.git. Signed-off-by: Johannes Schindelin --- azure-pipelines.yml | 42 ++++++++---------------------------------- 1 file changed, 8 insertions(+), 34 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e44d2733a47cdb..25acf343994c57 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,44 +18,18 @@ jobs: env: GITFILESHAREPWD: $(gitfileshare.pwd) - powershell: | - # Helper to check the error level of the latest command (exit with error when appropriate) - function c() { if (!$?) { exit(1) } } - - # Add build agent's MinGit to PATH - $env:PATH = $env:AGENT_HOMEDIRECTORY +"\externals\\git\cmd;" +$env:PATH - - # Helper to initialize (or update) a Git worktree - function init ($path, $url, $set_origin) { - if (Test-Path $path) { - cd $path; c - if (Test-Path .git) { - & git init; c - } else { - & git status - } - } else { - & git init $path; c - cd $path; c - } - & git config core.autocrlf false; c - & git config core.untrackedCache true; c - if (($set_origin -ne 0) -and !(git config remote.origin.url)) { - & git remote add origin $url; c - } - & git fetch --depth=1 $url master; c - & git reset --hard FETCH_HEAD; c - & git clean -df; c - } - - # Initialize Git for Windows' SDK - $sdk_path = "$(Build.SourcesDirectory)\git-sdk-64" - init "$sdk_path" "https://dev.azure.com/git-for-windows/git-sdk-64/_git/git-sdk-64" 0 + $urlbase = "https://dev.azure.com/git-for-windows/git/_apis/build/builds" + $id = ((Invoke-WebRequest -UseBasicParsing "${urlbase}?definitions=22&statusFilter=completed&resultFilter=succeeded&`$top=1").content | ConvertFrom-JSON).value[0].id + $downloadUrl = ((Invoke-WebRequest -UseBasicParsing "${urlbase}/$id/artifacts").content | ConvertFrom-JSON).value[1].resource.downloadUrl + (New-Object Net.WebClient).DownloadFile($downloadUrl,"git-sdk-64-minimal.zip") + Expand-Archive git-sdk-64-minimal.zip -DestinationPath . -Force + Remove-Item git-sdk-64-minimal.zip # Let Git ignore the SDK and the test-cache - "/git-sdk-64/`n/test-cache/`n" | Out-File -NoNewLine -Encoding ascii -Append "$(Build.SourcesDirectory)\.git\info\exclude" + "/git-sdk-64-minimal/`n/test-cache/`n" | Out-File -NoNewLine -Encoding ascii -Append "$(Build.SourcesDirectory)\.git\info\exclude" displayName: 'Initialize the Git for Windows SDK' - powershell: | - & "git-sdk-64\git-cmd.exe" --command=usr\\bin\\bash.exe -lc @" + & git-sdk-64-minimal\usr\bin\bash.exe -lc @" export MAKEFLAGS=-j10 export DEVELOPER=1 export NO_PERL=1 From a7f8b0d0ee2d338f352c3502ecd0f92c3ea9ae0d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 26 Dec 2018 19:15:34 +0100 Subject: [PATCH 11/17] mingw: be more generous when wrapping up the setitimer() emulation Every once in a while, the Azure Pipeline fails with some semi-random error: timer thread did not terminate timely This error message means that the thread that is used to emulate the setitimer() function did not terminate within 1,000 milliseconds. The most likely explanation (and therefore the one we should assume to be true, according to Occam's Razor) is that the timeout of one second is simply not enough because we try to run so many tasks in parallel. So let's give it ten seconds instead of only one. That should be enough. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 29c6c2cae9b444..517dee27e95c71 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2404,7 +2404,7 @@ static void stop_timer_thread(void) if (timer_event) SetEvent(timer_event); /* tell thread to terminate */ if (timer_thread) { - int rc = WaitForSingleObject(timer_thread, 1000); + int rc = WaitForSingleObject(timer_thread, 10000); if (rc == WAIT_TIMEOUT) error("timer thread did not terminate timely"); else if (rc != WAIT_OBJECT_0) From 3036485ddb9f02d710721e5a0bae815835194761 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 27 Dec 2018 11:19:36 +0100 Subject: [PATCH 12/17] fixup! mingw: special-case arguments to `sh` While working on parallelizing the tests in Azure Pipelines, an issue was discovered with the `is_msys2_sh()` function: it expects the path components to be separated by exactly one dir separator. That does not need to be the case, though, e.g. when the components in the `PATH` variable have trailing slashes. Let's make the code much more robust in this respect. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 517dee27e95c71..e0c02fd9ecc5d1 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1628,6 +1628,21 @@ struct pinfo_t { static struct pinfo_t *pinfo = NULL; CRITICAL_SECTION pinfo_cs; +/* Used to match and chomp off path components */ +static inline int match_last_path_component(const char *path, size_t *len, + const char *component) +{ + size_t component_len = strlen(component); + if (*len < component_len + 1 || + !is_dir_sep(path[*len - component_len - 1]) || + strncasecmp(path + *len - component_len, component, component_len)) + return 0; + *len -= component_len + 1; + while (*len > 0 && is_dir_sep(path[*len - 1])) + (*len)--; + return 1; +} + static int is_msys2_sh(const char *cmd) { if (cmd && !strcmp(cmd, "sh")) { @@ -1642,13 +1657,10 @@ static int is_msys2_sh(const char *cmd) ret = 0; else { size_t len = strlen(p); - ret = len > 15 && - is_dir_sep(p[len - 15]) && - !strncasecmp(p + len - 14, "usr", 3) && - is_dir_sep(p[len - 11]) && - !strncasecmp(p + len - 10, "bin", 3) && - is_dir_sep(p[len - 7]) && - !strcasecmp(p + len - 6, "sh.exe"); + + ret = match_last_path_component(p, &len, "sh.exe") && + match_last_path_component(p, &len, "bin") && + match_last_path_component(p, &len, "usr"); free(p); } return ret; From 3ee87b31c9c93734f259bd64f93dc413a1acc268 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 28 Dec 2018 00:02:57 +0100 Subject: [PATCH 13/17] fixup! tests: optionally write results as JUnit-style .xml Make sure to write the .xml in UTF-8 encoding. We also need to make sure that invalid UTF-8 encoding is turned into valid UTF-8 (using the Replacement Character, \uFFFD) because t9902's trace contains such invalid byte sequences, and the task that uploads the test results would refuse to do anything if it was asked to parse an .xml file with invalid UTF-8 in it. Signed-off-by: Johannes Schindelin --- Makefile | 1 + t/helper/test-tool.c | 1 + t/helper/test-tool.h | 1 + t/helper/test-xml-encode.c | 80 ++++++++++++++++++++++++++++++++++++++ t/test-lib.sh | 13 +------ 5 files changed, 84 insertions(+), 12 deletions(-) create mode 100644 t/helper/test-xml-encode.c diff --git a/Makefile b/Makefile index 575c59d7ee0ddd..6444c2af8a3442 100644 --- a/Makefile +++ b/Makefile @@ -757,6 +757,7 @@ TEST_BUILTINS_OBJS += test-submodule-config.o TEST_BUILTINS_OBJS += test-submodule-nested-repo-config.o TEST_BUILTINS_OBJS += test-subprocess.o TEST_BUILTINS_OBJS += test-urlmatch-normalization.o +TEST_BUILTINS_OBJS += test-xml-encode.o TEST_BUILTINS_OBJS += test-wildmatch.o TEST_BUILTINS_OBJS += test-windows-named-pipe.o TEST_BUILTINS_OBJS += test-write-cache.o diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index 52304d0748ce52..935c97638c271e 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -51,6 +51,7 @@ static struct test_cmd cmds[] = { { "submodule-nested-repo-config", cmd__submodule_nested_repo_config }, { "subprocess", cmd__subprocess }, { "urlmatch-normalization", cmd__urlmatch_normalization }, + { "xml-encode", cmd__xml_encode }, { "wildmatch", cmd__wildmatch }, #ifdef GIT_WINDOWS_NATIVE { "windows-named-pipe", cmd__windows_named_pipe }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index 955490ec9f29f1..b702e4152ec052 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -47,6 +47,7 @@ int cmd__submodule_config(int argc, const char **argv); int cmd__submodule_nested_repo_config(int argc, const char **argv); int cmd__subprocess(int argc, const char **argv); int cmd__urlmatch_normalization(int argc, const char **argv); +int cmd__xml_encode(int argc, const char **argv); int cmd__wildmatch(int argc, const char **argv); #ifdef GIT_WINDOWS_NATIVE int cmd__windows_named_pipe(int argc, const char **argv); diff --git a/t/helper/test-xml-encode.c b/t/helper/test-xml-encode.c new file mode 100644 index 00000000000000..367c4875e65251 --- /dev/null +++ b/t/helper/test-xml-encode.c @@ -0,0 +1,80 @@ +#include "test-tool.h" + +static const char *utf8_replace_character = "�"; + +/* + * Encodes (possibly incorrect) UTF-8 on to , to be embedded + * in an XML file. + */ +int cmd__xml_encode(int argc, const char **argv) +{ + unsigned char buf[1024], tmp[4], *tmp2 = NULL; + ssize_t cur = 0, len = 1, remaining = 0; + unsigned char ch; + + for (;;) { + if (++cur == len) { + len = xread(0, buf, sizeof(buf)); + if (!len) + return 0; + if (len < 0) + die_errno("Could not read "); + cur = 0; + } + ch = buf[cur]; + + if (tmp2) { + if ((ch & 0xc0) != 0x80) { + fputs(utf8_replace_character, stdout); + tmp2 = 0; + cur--; + continue; + } + *tmp2 = ch; + tmp2++; + if (--remaining == 0) { + fwrite(tmp, tmp2 - tmp, 1, stdout); + tmp2 = 0; + } + continue; + } + + if (!(ch & 0x80)) { + /* 0xxxxxxx */ + if (ch == '&') + fputs("&", stdout); + else if (ch == '\'') + fputs("'", stdout); + else if (ch == '"') + fputs(""", stdout); + else if (ch == '<') + fputs("<", stdout); + else if (ch == '>') + fputs(">", stdout); + else if (ch >= 0x20) + fputc(ch, stdout); + else if (ch == 0x09 || ch == 0x0a || ch == 0x0d) + fprintf(stdout, "&#x%02x;", ch); + else + fputs(utf8_replace_character, stdout); + } else if ((ch & 0xe0) == 0xc0) { + /* 110XXXXx 10xxxxxx */ + tmp[0] = ch; + remaining = 1; + tmp2 = tmp + 1; + } else if ((ch & 0xf0) == 0xe0) { + /* 1110XXXX 10Xxxxxx 10xxxxxx */ + tmp[0] = ch; + remaining = 2; + tmp2 = tmp + 1; + } else if ((ch & 0xf8) == 0xf0) { + /* 11110XXX 10XXxxxx 10xxxxxx 10xxxxxx */ + tmp[0] = ch; + remaining = 3; + tmp2 = tmp + 1; + } else + fputs(utf8_replace_character, stdout); + } + + return 0; +} diff --git a/t/test-lib.sh b/t/test-lib.sh index 51f502b1614167..88ac140ca7893f 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -888,18 +888,7 @@ write_junit_xml () { } xml_attr_encode () { - # We do not translate CR to because BSD sed does not handle - # \r in the regex. In practice, the output should not even have any - # carriage returns. - printf '%s\n' "$@" | - sed -e 's/&/\&/g' -e "s/'/\'/g" -e 's/"/\"/g' \ - -e 's//\>/g' \ - -e "s/$(printf \\x1c)/\\�/g" \ - -e "s/$(printf \\x1d)/\\�/g" \ - -e "s/$(printf \\x1e)/\\�/g" \ - -e "s/$(printf \\x1f)/\\�/g" \ - -e 's/ /\ /g' -e 's/$/\ /' -e '$s/ $//' | - tr -d '\012\015' + printf '%s\n' "$@" | test-tool xml-encode } write_junit_xml_testcase () { From 898de195f55db5d44657c928c6cc59bf7476098a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 27 Dec 2018 23:12:00 +0100 Subject: [PATCH 14/17] tests: avoid calling Perl just to determine file sizes It is a bit ridiculous to spin up a full-blown Perl instance (especially on Windows, where that means spinning up a full POSIX emulation layer, AKA the MSYS2 runtime) just to tell how large a given file is. So let's just use the test-tool to do that job instead. Signed-off-by: Johannes Schindelin --- t/helper/test-path-utils.c | 12 ++++++++++++ t/t0021-conversion.sh | 2 +- t/t1050-large.sh | 2 +- t/t5315-pack-objects-compression.sh | 2 +- t/t9303-fast-import-compression.sh | 2 +- 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/t/helper/test-path-utils.c b/t/helper/test-path-utils.c index ae091d9b3e63cb..30211d6d641538 100644 --- a/t/helper/test-path-utils.c +++ b/t/helper/test-path-utils.c @@ -291,6 +291,18 @@ int cmd__path_utils(int argc, const char **argv) return !!res; } + if (argc > 2 && !strcmp(argv[1], "file-size")) { + int res = 0, i; + struct stat st; + + for (i = 2; i < argc; i++) + if (stat(argv[i], &st)) + res = error_errno("Cannot stat '%s'", argv[i]); + else + printf("%"PRIuMAX"\n", (uintmax_t)st.st_size); + return !!res; + } + fprintf(stderr, "%s: unknown function name: %s\n", argv[0], argv[1] ? argv[1] : "(there was none)"); return 1; diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh index 518ea6be6fbebc..27c3d0afb29106 100755 --- a/t/t0021-conversion.sh +++ b/t/t0021-conversion.sh @@ -24,7 +24,7 @@ generate_random_characters () { } file_size () { - perl -e 'print -s $ARGV[0]' "$1" + test-tool path-utils file-size "$1" } filter_git () { diff --git a/t/t1050-large.sh b/t/t1050-large.sh index 1a9b21b2934b0a..dcb4dbba673eb2 100755 --- a/t/t1050-large.sh +++ b/t/t1050-large.sh @@ -8,7 +8,7 @@ test_description='adding and checking out large blobs' # This should be moved to test-lib.sh together with the # copy in t0021 after both topics have graduated to 'master'. file_size () { - perl -e 'print -s $ARGV[0]' "$1" + test-tool path-utils file-size "$1" } test_expect_success setup ' diff --git a/t/t5315-pack-objects-compression.sh b/t/t5315-pack-objects-compression.sh index 34c47dae09966b..df970d75845e78 100755 --- a/t/t5315-pack-objects-compression.sh +++ b/t/t5315-pack-objects-compression.sh @@ -7,7 +7,7 @@ test_description='pack-object compression configuration' # This should be moved to test-lib.sh together with the # copy in t0021 after both topics have graduated to 'master'. file_size () { - perl -e 'print -s $ARGV[0]' "$1" + test-tool path-utils file-size "$1" } test_expect_success setup ' diff --git a/t/t9303-fast-import-compression.sh b/t/t9303-fast-import-compression.sh index 856219f46a7fc4..5045f02a5331fc 100755 --- a/t/t9303-fast-import-compression.sh +++ b/t/t9303-fast-import-compression.sh @@ -6,7 +6,7 @@ test_description='compression setting of fast-import utility' # This should be moved to test-lib.sh together with the # copy in t0021 after both topics have graduated to 'master'. file_size () { - perl -e 'print -s $ARGV[0]' "$1" + test-tool path-utils file-size "$1" } import_large () { From 1b512ad99fe3a62515a8e0b608a6166af7c2adad Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 27 Dec 2018 23:15:15 +0100 Subject: [PATCH 15/17] fixup! tests: include detailed trace logs with --write-junit-xml upon failure Let's use the new `test-tool path-utils file-size` command in the new JUnit XML feature. Signed-off-by: Johannes Schindelin --- t/test-lib.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/t/test-lib.sh b/t/test-lib.sh index 88ac140ca7893f..b7912d6ef44c13 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -902,7 +902,8 @@ write_junit_xml_testcase () { junit_have_testcase=t if test -n "$GIT_TEST_TEE_OUTPUT_FILE" then - GIT_TEST_TEE_OFFSET=$(perl -e 'print -s $ARGV[0]' "$GIT_TEST_TEE_OUTPUT_FILE") + GIT_TEST_TEE_OFFSET=$(test-tool path-utils file-size \ + "$GIT_TEST_TEE_OUTPUT_FILE") fi } From dd3edb09d085a5cb6ab7fdd77caeffc19b9b7716 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 15 Jan 2019 16:58:01 +0100 Subject: [PATCH 16/17] fixup! travis: fix skipping tagged releases Add the comment that we added to the patch series intended for upstream git.git. Signed-off-by: Johannes Schindelin --- ci/lib.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/lib.sh b/ci/lib.sh index 0dee59706c42b8..bf5f1c9c0f1b4d 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -2,7 +2,8 @@ if test true = "$TRAVIS" then - # We are running within Travis CI + # When building a PR, TRAVIS_BRANCH refers to the *target* branch. Not + # what we want here. We want the source branch instead. CI_BRANCH="${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}" CI_COMMIT="$TRAVIS_COMMIT" CI_JOB_ID="$TRAVIS_JOB_ID" From 3346fdcbb5f353c93247cb95a2493a2ae271159b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 15 Jan 2019 19:23:03 +0100 Subject: [PATCH 17/17] ci: parallelize testing on Windows The fact that Git's test suite is implemented in Unix shell script that is as portable as we can muster, combined with the fact that Unix shell scripting is foreign to Windows (and therefore has to be emulated), results in pretty abysmal speed of the test suite on that platform, for pretty much no other reason than that language choice. For comparison: while the Linux build & test is typically done within about 8 minutes, the Windows build & test typically lasts about 80 minutes in Azure Pipelines. To help with that, let's use the Azure Pipeline feature where you can parallelize jobs, make jobs depend on each other, and pass artifacts between them. The tests are distributed using the following heuristic: listing all test scripts ordered by size in descending order (as a cheap way to estimate the overall run time), every Nth script is run (where N is the total number of parallel jobs), starting at the index corresponding to the parallel job. This slicing is performed by a new function that is added to the `test-tool`. To optimize the overall runtime of the entire Pipeline, we need to move the Windows jobs to the beginning (otherwise there would be a very decent chance for the Pipeline to be run only the Windows build, while all the parallel Windows test jobs wait for this single one). We use Azure Pipelines Artifacts for both the minimal Git for Windows SDK as well as the built executables, as deduplication and caching close to the agents makes that really fast. For comparison: while downloading and unpacking the minimal Git for Windows SDK via PowerShell takes only one minute (down from anywhere between 2.5 to 7 when using a shallow clone), uploading it as Pipeline Artifact takes less than 30s and downloading and unpacking less than 20s (sometimes even as little as only twelve seconds). Signed-off-by: Johannes Schindelin --- Makefile | 10 +++++ azure-pipelines.yml | 84 +++++++++++++++++++++++++++++++++----- ci/make-test-artifacts.sh | 12 ++++++ ci/run-test-slice.sh | 17 ++++++++ t/helper/test-path-utils.c | 31 ++++++++++++++ 5 files changed, 144 insertions(+), 10 deletions(-) create mode 100755 ci/make-test-artifacts.sh create mode 100755 ci/run-test-slice.sh diff --git a/Makefile b/Makefile index 6444c2af8a3442..4b3e7896ebb7cf 100644 --- a/Makefile +++ b/Makefile @@ -2958,6 +2958,16 @@ rpm:: @false .PHONY: rpm +artifacts-tar:: $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) $(OTHER_PROGRAMS) \ + GIT-BUILD-OPTIONS $(TEST_PROGRAMS) $(test_bindir_programs) \ + $(NO_INSTALL) $(MOFILES) + $(QUIET_SUBDIR0)templates $(QUIET_SUBDIR1) \ + SHELL_PATH='$(SHELL_PATH_SQ)' PERL_PATH='$(PERL_PATH_SQ)' + test -n "$(ARTIFACTS_DIRECTORY)" + mkdir -p "$(ARTIFACTS_DIRECTORY)" + $(TAR) czf "$(ARTIFACTS_DIRECTORY)/artifacts.tar.gz" $^ templates/blt/ +.PHONY: artifacts-tar + htmldocs = git-htmldocs-$(GIT_VERSION) manpages = git-manpages-$(GIT_VERSION) .PHONY: dist-doc distclean diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 25acf343994c57..6cd27b3483997f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -3,8 +3,8 @@ resources: fetchDepth: 1 jobs: -- job: windows - displayName: Windows +- job: windows_build + displayName: Windows Build condition: succeeded() pool: Hosted timeoutInMinutes: 240 @@ -27,25 +27,89 @@ jobs: # Let Git ignore the SDK and the test-cache "/git-sdk-64-minimal/`n/test-cache/`n" | Out-File -NoNewLine -Encoding ascii -Append "$(Build.SourcesDirectory)\.git\info\exclude" - displayName: 'Initialize the Git for Windows SDK' + displayName: 'Download git-sdk-64-minimal' - powershell: | & git-sdk-64-minimal\usr\bin\bash.exe -lc @" - export MAKEFLAGS=-j10 - export DEVELOPER=1 - export NO_PERL=1 - export NO_SVN_TESTS=1 - export GIT_TEST_SKIP_REBASE_P=1 + ci/make-test-artifacts.sh artifacts + "@ + if (!$?) { exit(1) } + displayName: Build + env: + HOME: $(Build.SourcesDirectory) + MSYSTEM: MINGW64 + MAKEFLAGS: -j10 + DEVELOPER: 1 + NO_PERL: 1 + - task: PublishPipelineArtifact@0 + displayName: 'Publish Pipeline Artifact: test artifacts' + inputs: + artifactName: 'windows-artifacts' + targetPath: '$(Build.SourcesDirectory)\artifacts' + - task: PublishPipelineArtifact@0 + displayName: 'Publish Pipeline Artifact: git-sdk-64-minimal' + inputs: + artifactName: 'git-sdk-64-minimal' + targetPath: '$(Build.SourcesDirectory)\git-sdk-64-minimal' + - powershell: | + if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { + cmd /c rmdir "$(Build.SourcesDirectory)\test-cache" + } + displayName: 'Unmount test-cache' + condition: true + env: + GITFILESHAREPWD: $(gitfileshare.pwd) + +- job: windows_test + displayName: Windows Test + dependsOn: windows_build + condition: succeeded() + pool: Hosted + timeoutInMinutes: 240 + strategy: + parallel: 10 + steps: + - powershell: | + if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { + net use s: \\gitfileshare.file.core.windows.net\test-cache "$GITFILESHAREPWD" /user:AZURE\gitfileshare /persistent:no + cmd /c mklink /d "$(Build.SourcesDirectory)\test-cache" S:\ + } + displayName: 'Mount test-cache' + env: + GITFILESHAREPWD: $(gitfileshare.pwd) + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact: test artifacts' + inputs: + artifactName: 'windows-artifacts' + targetPath: '$(Build.SourcesDirectory)' + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact: git-sdk-64-minimal' + inputs: + artifactName: 'git-sdk-64-minimal' + targetPath: '$(Build.SourcesDirectory)\git-sdk-64-minimal' + - powershell: | + & git-sdk-64-minimal\usr\bin\bash.exe -lc @" + test -f artifacts.tar.gz || { + echo No test artifacts found\; skipping >&2 + exit 0 + } + tar xf artifacts.tar.gz || exit 1 + + # Let Git ignore the SDK and the test-cache + printf '%s\n' /git-sdk-64-minimal/ /test-cache/ >>.git/info/exclude - ci/run-build-and-tests.sh || { + ci/run-test-slice.sh `$SYSTEM_JOBPOSITIONINPHASE `$SYSTEM_TOTALJOBSINPHASE || { ci/print-test-failures.sh exit 1 } "@ if (!$?) { exit(1) } - displayName: 'Build & Test' + displayName: 'Test (parallel)' env: HOME: $(Build.SourcesDirectory) MSYSTEM: MINGW64 + MAKEFLAGS: -j10 + NO_SVN_TESTS: 1 + GIT_TEST_SKIP_REBASE_P: 1 - powershell: | if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { cmd /c rmdir "$(Build.SourcesDirectory)\test-cache" diff --git a/ci/make-test-artifacts.sh b/ci/make-test-artifacts.sh new file mode 100755 index 00000000000000..646967481f6d78 --- /dev/null +++ b/ci/make-test-artifacts.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# +# Build Git and store artifacts for testing +# + +mkdir -p "$1" # in case ci/lib.sh decides to quit early + +. ${0%/*}/lib.sh + +make artifacts-tar ARTIFACTS_DIRECTORY="$1" + +check_unignored_build_artifacts diff --git a/ci/run-test-slice.sh b/ci/run-test-slice.sh new file mode 100755 index 00000000000000..f8c2c3106a2ef4 --- /dev/null +++ b/ci/run-test-slice.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# +# Test Git in parallel +# + +. ${0%/*}/lib.sh + +case "$CI_OS_NAME" in +windows*) cmd //c mklink //j t\\.prove "$(cygpath -aw "$cache_dir/.prove")";; +*) ln -s "$cache_dir/.prove" t/.prove;; +esac + +make --quiet -C t T="$(cd t && + ./helper/test-tool path-utils slice-tests "$1" "$2" t[0-9]*.sh | + tr '\n' ' ')" + +check_unignored_build_artifacts diff --git a/t/helper/test-path-utils.c b/t/helper/test-path-utils.c index 30211d6d641538..f4c57750902656 100644 --- a/t/helper/test-path-utils.c +++ b/t/helper/test-path-utils.c @@ -177,6 +177,14 @@ static int is_dotgitmodules(const char *path) return is_hfs_dotgitmodules(path) || is_ntfs_dotgitmodules(path); } +static int cmp_by_st_size(const void *a, const void *b) +{ + intptr_t x = (intptr_t)((struct string_list_item *)a)->util; + intptr_t y = (intptr_t)((struct string_list_item *)b)->util; + + return x > y ? -1 : (x < y ? +1 : 0); +} + int cmd__path_utils(int argc, const char **argv) { if (argc == 3 && !strcmp(argv[1], "normalize_path_copy")) { @@ -303,6 +311,29 @@ int cmd__path_utils(int argc, const char **argv) return !!res; } + if (argc > 5 && !strcmp(argv[1], "slice-tests")) { + int res = 0; + long offset, stride, i; + struct string_list list = STRING_LIST_INIT_NODUP; + struct stat st; + + offset = strtol(argv[2], NULL, 10); + stride = strtol(argv[3], NULL, 10); + if (stride < 1) + stride = 1; + for (i = 4; i < argc; i++) + if (stat(argv[i], &st)) + res = error_errno("Cannot stat '%s'", argv[i]); + else + string_list_append(&list, argv[i])->util = + (void *)(intptr_t)st.st_size; + QSORT(list.items, list.nr, cmp_by_st_size); + for (i = offset; i < list.nr; i+= stride) + printf("%s\n", list.items[i].string); + + return !!res; + } + fprintf(stderr, "%s: unknown function name: %s\n", argv[0], argv[1] ? argv[1] : "(there was none)"); return 1;