mac: fix setup and re-enable CI

.github/workflows/selfdrive_tests.yaml

@@ -69,52 +69,52 @@ jobs:
                           rm -rf /tmp/scons_cache/* && \
                           scons -j$(nproc) --cache-populate"
 
-  #build_mac:
-  #  name: build macos
-  #  runs-on: macos-10.15
-  #  timeout-minutes: 60
-  #  steps:
-  #  - uses: actions/checkout@v2
-  #    with:
-  #      submodules: true
-  #  - name: Determine pre-existing Homebrew packages
-  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
-  #    run: |
-  #      echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
-  #      ls -1 /usr/local/Cellar >> $GITHUB_ENV
-  #      echo 'EOF' >> $GITHUB_ENV
-  #  - name: Cache dependencies
-  #    id: dependency-cache
-  #    uses: actions/cache@v2
-  #    with:
-  #      path: |
-  #        ~/.pyenv
-  #        ~/Library/Caches/pip
-  #        ~/Library/Caches/pipenv
-  #        /usr/local/Cellar
-  #        ~/github_brew_cache_entries.txt
-  #      key: macos-cache-${{ hashFiles('tools/mac_setup.sh') }}
-  #  - name: Brew link restored dependencies
-  #    if: steps.dependency-cache.outputs.cache-hit == 'true'
-  #    run: |
-  #      while read pkg; do
-  #        brew link --force "$pkg" # `--force` for keg-only packages
-  #      done < ~/github_brew_cache_entries.txt
-  #  - name: Install dependencies
-  #    run: ./tools/mac_setup.sh
-  #  - name: Build openpilot
-  #    run: eval "$(pyenv init -)" && scons -j$(nproc)
-  #  - name: Remove pre-existing Homebrew packages for caching
-  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
-  #    run: |
-  #      cd /usr/local/Cellar
-  #      new_cellar=$(ls -1)
-  #      comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
-  #        if [[ $pkg != "zstd" ]]; then # caching step needs zstd
-  #          rm -rf "$pkg"
-  #        fi
-  #      done
-  #      comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt
+  build_mac:
+   name: build macos
+   runs-on: macos-11
+   timeout-minutes: 60
+   steps:
+   - uses: actions/checkout@v2
+     with:
+       submodules: true
+   - name: Determine pre-existing Homebrew packages
+     if: steps.dependency-cache.outputs.cache-hit != 'true'
+     run: |
+       echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
+       ls -1 /usr/local/Cellar >> $GITHUB_ENV
+       echo 'EOF' >> $GITHUB_ENV
+   - name: Cache dependencies
+     id: dependency-cache
+     uses: actions/cache@v2
+     with:
+       path: |
+         ~/.pyenv
+         ~/Library/Caches/pip
+         ~/Library/Caches/pipenv
+         /usr/local/Cellar
+         ~/github_brew_cache_entries.txt
+       key: macos-cache-${{ hashFiles('tools/mac_setup.sh') }}
+   - name: Brew link restored dependencies
+     if: steps.dependency-cache.outputs.cache-hit == 'true'
+     run: |
+       while read pkg; do
+         brew link --force "$pkg" # `--force` for keg-only packages
+       done < ~/github_brew_cache_entries.txt
+   - name: Install dependencies
+     run: ./tools/mac_setup.sh
+   - name: Build openpilot
+     run: pipenv run scons -j$(nproc)
+   - name: Remove pre-existing Homebrew packages for caching
+     if: steps.dependency-cache.outputs.cache-hit != 'true'
+     run: |
+       cd /usr/local/Cellar
+       new_cellar=$(ls -1)
+       comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
+         if [[ $pkg != "zstd" ]]; then # caching step needs zstd
+           rm -rf "$pkg"
+         fi
+       done
+       comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt
 
   build_webcam:
     name: build webcam

Pipfile

@@ -35,7 +35,7 @@ tenacity = "*"
 
 [packages]
 atomicwrites = "*"
-casadi = "*"
+casadi = {version = "*", markers="platform_system != 'Darwin'"}
 cffi = "*"
 crcmod = "*"
 cryptography = "*"
@@ -51,7 +51,7 @@ libusb1 = "*"
 nose = "*"
 numpy = "*"
 onnx = "*"
-onnxruntime-gpu = "*"
+onnxruntime-gpu = {version = "*", markers="platform_system != 'Darwin'"}
 pillow = "*"
 psutil = "*"
 pycapnp = "==1.1.0"

tools/mac_setup.sh

@@ -1,21 +1,23 @@
 #!/bin/bash -e
-
+PYTHON_VERSION=3.8.10
+PYTHON_VER=3.8
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 ROOT="$(cd $DIR/../ && pwd)"
+ARCH=$(uname -m)
 
 # Install brew if required
 if [[ $(command -v brew) == "" ]]; then
   echo "Installing Hombrew"
   /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
 fi
 
+## TODO remove protobuf,protobuf-c,swig when
+## casadi pip package is available
 brew bundle --file=- <<-EOS
 brew "cmake"
 brew "git-lfs"
 brew "zlib"
 brew "bzip2"
-brew "rust"
-brew "rustup-init"
 brew "capnp"
 brew "coreutils"
 brew "eigen"
@@ -29,6 +31,9 @@ brew "openssl"
 brew "pyenv"
 brew "qt@5"
 brew "zeromq"
+brew "protobuf"
+brew "protobuf-c"
+brew "swig"
 cask "gcc-arm-embedded"
 EOS
 
@@ -38,12 +43,6 @@ elif [[ $SHELL == "/bin/bash" ]]; then
   RC_FILE="$HOME/.bash_profile"
 fi
 
-# TODO: get rid of this somehow
-# Build requirements for macOS
-# https://github.com/pyenv/pyenv/issues/1740
-# https://github.com/pyca/cryptography/blob/main/docs/installation.rst
-rustup-init -y
-
 export LDFLAGS="$LDFLAGS -L/usr/local/opt/zlib/lib"
 export LDFLAGS="$LDFLAGS -L/usr/local/opt/bzip2/lib"
 export LDFLAGS="$LDFLAGS -L/usr/local/opt/[email protected]/lib"
@@ -55,15 +54,35 @@ export PATH="$PATH:/usr/local/bin"
 
 # openpilot environment
 if [ -z "$OPENPILOT_ENV" ] && [ -n "$RC_FILE" ] && [ -z "$CI" ]; then
-  echo "export PATH=\"\$PATH:$HOME/.cargo/bin\"" >> $RC_FILE
   echo "source $ROOT/tools/openpilot_env.sh" >> $RC_FILE
-  export PATH="$PATH:\"\$HOME/.cargo/bin\""
   source "$ROOT/tools/openpilot_env.sh"
   echo "Added openpilot_env to RC file: $RC_FILE"
 fi
 
 # install python dependencies
-$ROOT/update_requirements.sh
+$ROOT/update_requirements.sh || true
+
+# install casadi
+VENV=`pipenv --venv`
+if [ ! -f "$VENV/include/casadi/casadi.hpp" ]; then
+  echo "-- casadi manual install"
+  cd /tmp/ && curl -L https://github.com/casadi/casadi/archive/refs/tags/ge6.tar.gz --output casadi.tar.gz
+  tar -xzf casadi.tar.gz
+  cd casadi-ge6/ && mkdir -p build && cd build
+  cmake .. \
+    -DWITH_PYTHON=ON \
+    -DCMAKE_INSTALL_PREFIX:PATH=$VENV \
+    -DPYTHON_PREFIX:PATH=$VENV/lib/python$PYTHON_VER/site-packages \
+    -DPYTHON_LIBRARY:FILEPATH=$HOME/.pyenv/versions/$PYTHON_VERSION/lib/libpython$PYTHON_VER.dylib \
+    -DPYTHON_EXECUTABLE:FILEPATH=$HOME/.pyenv/versions/$PYTHON_VERSION/bin/python \
+    -DPYTHON_INCLUDE_DIR:PATH=$HOME/.pyenv/versions/$PYTHON_VERSION/include/python$PYTHON_VER \
+    -DCMAKE_CXX_FLAGS="-ferror-limit=0" -DCMAKE_C_FLAGS="-ferror-limit=0"
+  CFLAGS="-ferror-limit=0" make -j$(nproc) && make install
+else
+  echo "----   'casadi found in venv. skipping build   ----"
+fi
+
+cd $ROOT
 
 echo
 echo "----   FINISH OPENPILOT SETUP   ----"

tools/openpilot_env.sh

@@ -1,8 +1,20 @@
 if [ -z "$OPENPILOT_ENV" ]; then
   OP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
+  ARCH="$(uname -m)"
+  HOST="$(uname -s)"
+
   export PYTHONPATH="$OP_ROOT:$PYTHONPATH"
   export PATH="$HOME/.pyenv/bin:$PATH"
 
+  # for qt resource compiler "rcc"
+  if [[ "$HOST" == "Darwin" ]]; then
+    if [[ "$ARCH" == "arm64" ]]; then
+      export PATH="/opt/homebrew/opt/qt@5/bin:$PATH"
+    else # x86_64 intel
+      export PATH="/opt/local/opt/qt@5/bin:$PATH"
+    fi
+  fi
+
   # Pyenv suggests we place the below two lines in .profile before we source
   # .bashrc, but there is no simple way to guarantee we do this correctly
   # programmatically across heterogeneous systems. For end-user convenience,

update_requirements.sh

@@ -1,5 +1,5 @@
 #!/bin/bash -e
-
+HOST="$(uname -s)"
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 cd $DIR
 
@@ -17,14 +17,25 @@ if ! pyenv prefix ${PYENV_PYTHON_VERSION} &> /dev/null; then
   CONFIGURE_OPTS="--enable-shared" pyenv install -f ${PYENV_PYTHON_VERSION}
 fi
 
-if ! command -v pipenv &> /dev/null; then
-  echo "pipenv install ..."
-  pip install pipenv
-fi
+if [[ "$HOST" != "Darwin" ]]; then
+  if ! command -v pipenv &> /dev/null; then
+    echo "pipenv install ..."
+    pip install pipenv
+  fi
+
+  echo "update pip"
+  pip install pip==21.3.1
+  pip install pipenv==2021.11.23
+else  # MacOS
+  if ! command -v pipenv &> /dev/null; then
+    echo "pipenv install ..."
+    python3 -m pip install pipenv
+  fi
 
-echo "update pip"
-pip install pip==21.3.1
-pip install pipenv==2021.11.23
+  echo "update pip"
+  python3 -m pip install pip==21.3.1
+  python3 -m pip install pipenv==2021.11.23
+fi
 
 if [ -d "./xx" ]; then
   export PIPENV_SYSTEM=1
@@ -37,8 +48,12 @@ else
   RUN=""
 fi
 
-echo "pip packages install ..."
-pipenv install --dev --deploy --clear
+echo "pip packages install for $HOST ..."
+if [[ "$HOST" == "Darwin" ]]; then
+  pipenv install --dev --deploy --clear --skip-lock
+else
+  pipenv install --dev --deploy --clear
+fi
 pyenv rehash
 
 if [ -f "$DIR/.pre-commit-config.yaml" ]; then