Merge pull request #130 from CrayLabs/develop

v0.2.0

Author: Sam Partee

.github/workflows/build_wheels.yml

@@ -58,43 +58,3 @@ jobs:
       - uses: actions/upload-artifact@v2
         with:
           path: ./wheelhouse/*.whl
-
-  build_sdist:
-    name: Build source distribution
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - uses: actions/setup-python@v2
-        name: Install Python
-        with:
-          python-version: '3.8'
-
-      - name: Build sdist
-        run: |
-          python -m pip install cmake>=3.10
-          python setup.py sdist
-
-      - uses: actions/upload-artifact@v2
-        with:
-          path: dist/*.tar.gz
-
-
-  upload_pypi:
-    needs: [build_wheels, build_sdist]
-    runs-on: ubuntu-latest
-    # upload to PyPI on every tag starting with 'v'
-    # if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
-    # Publish when a GitHub Release is created
-    if: github.event_name == 'release' && github.event.action == 'published'
-    steps:
-      - uses: actions/download-artifact@v2
-        with:
-          name: artifact
-          path: dist
-
-      - uses: pypa/gh-action-pypi-publish@master
-        with:
-          user: __token__
-          password: ${{ secrets.PYPI }}
-          #repository_url: https://test.pypi.org/legacy/

.github/workflows/release.yml

@@ -0,0 +1,98 @@
+name: deploy-release
+
+on:
+  release:
+    types: [published]
+
+env:
+  HOMEBREW_NO_ANALYTICS: "ON" # Make Homebrew installation a little quicker
+  HOMEBREW_NO_AUTO_UPDATE: "ON"
+  HOMEBREW_NO_BOTTLE_SOURCE_FALLBACK: "ON"
+  HOMEBREW_NO_GITHUB_API: "ON"
+  HOMEBREW_NO_INSTALL_CLEANUP: "ON"
+  CIBW_SKIP: "pp* *i686*" # skip building for PyPy
+  CIBW_ARCHES: x86_64
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04, macos-10.15]
+        gcc_v: [8] # Version of GFortran we want to use.
+    env:
+      FC: gfortran-${{ matrix.gcc_v }}
+      GCC_V: ${{ matrix.gcc_v }}
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+
+      - name: Install cibuildwheel
+        run: python -m pip install cibuildwheel==1.10.0
+
+      - name: Set up common
+        run: python -m pip install --upgrade cmake
+
+      - name: Install GFortran Linux
+        if: contains(matrix.os, 'ubuntu')
+        run: |
+          sudo add-apt-repository ppa:ubuntu-toolchain-r/test
+          sudo apt-get update
+          sudo apt-get install -y gcc-${GCC_V} gfortran-${GCC_V}
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_V} 100 \
+          --slave /usr/bin/gfortran gfortran /usr/bin/gfortran-${GCC_V}
+          sudo apt-get install -y libtool automake
+
+      - name: Install GFortran macOS
+        if: contains(matrix.os, 'macos')
+        run: brew install gcc@${GCC_V} || brew upgrade gcc@${GCC_V} || true
+
+      - name: Install automake macOS
+        if: contains(matrix.os, 'macos')
+        run: brew install automake || true
+
+      - name: Build wheels
+        run: python -m cibuildwheel --output-dir wheelhouse
+
+      - uses: actions/upload-artifact@v2
+        with:
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v2
+        name: Install Python
+        with:
+          python-version: '3.8'
+
+      - name: Build sdist
+        run: |
+          python -m pip install cmake>=3.10
+          python setup.py sdist
+
+      - uses: actions/upload-artifact@v2
+        with:
+          path: dist/*.tar.gz
+
+
+  upload_pypi:
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: artifact
+          path: dist
+
+      - uses: pypa/gh-action-pypi-publish@master
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI }}
+          #repository_url: https://test.pypi.org/legacy/

CMakeLists.txt

@@ -4,16 +4,15 @@ cmake_minimum_required(VERSION 3.10)
 
 option(BUILD_PYTHON "Build the python module" ON)
 
-set(CMAKE_BUILD_TYPE Release)
+set(CMAKE_BUILD_TYPE RELEASE)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_EXE_LINKER_FLAGS "-lpthread")
 set(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install)
 
 find_library(REDISPP redis++ PATHS ${CMAKE_SOURCE_DIR}/install/lib NO_DEFAULT_PATH REQUIRED)
 find_library(HIREDIS hiredis PATHS ${CMAKE_SOURCE_DIR}/install/lib NO_DEFAULT_PATH REQUIRED)
-find_library(PROTOBUF protobuf PATHS ${CMAKE_SOURCE_DIR}/install/lib NO_DEFAULT_PATH REQUIRED)
 
-set(EXT_CLIENT_LIBRARIES ${REDISPP} ${HIREDIS} ${PROTOBUF})
+set(EXT_CLIENT_LIBRARIES ${REDISPP} ${HIREDIS})
 
 set(CLIENT_SRC
     src/c/c_client.cpp
@@ -30,15 +29,15 @@ set(CLIENT_SRC
     src/cpp/redisserver.cpp
     src/cpp/rediscluster.cpp
     src/cpp/redis.cpp
-    utils/protobuf/smartredis.pb.cc
+    src/cpp/metadatafield.cpp
+    src/cpp/stringfield.cpp
     src/fortran/fortran_c_interop.F90
     src/fortran/dataset.F90
     src/fortran/client.F90)
 
 include_directories(SYSTEM
-    install/include/
     include
-    utils/protobuf
+    install/include
     src/fortran
 )
 
@@ -57,12 +56,6 @@ install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/"
         PATTERN "*.h" PATTERN "*.tcc" PATTERN "*.inc"
 )
 
-install(DIRECTORY "${CMAKE_SOURCE_DIR}/utils/protobuf/"
-        DESTINATION "include"
-        FILES_MATCHING
-        PATTERN "*.h" PATTERN "*.tcc"
-)
-
 if(BUILD_PYTHON)
 	message("-- Python client build enabled")
 	add_subdirectory(${CMAKE_SOURCE_DIR}/third-party/pybind

Makefile

@@ -45,6 +45,7 @@ test-deps-gpu:
 .PHONY: build-tests
 build-tests: lib
 	./build-scripts/build_cpp_tests.sh
+	./build-scripts/build_cpp_unit_tests.sh
 	./build-scripts/build_c_tests.sh
 	./build-scripts/build_fortran_tests.sh
 
@@ -53,7 +54,12 @@ build-tests: lib
 .PHONY: build-test-cpp
 build-test-cpp: lib
 	./build-scripts/build_cpp_tests.sh
+	./build-scripts/build_cpp_unit_tests.sh
 
+# help: build-unit-test-cpp            - build the C++ unit tests
+.PHONY: build-unit-test-cpp
+build-unit-test-cpp: lib
+	./build-scripts/build_cpp_unit_tests.sh
 
 # help: build-test-c                   - build the C tests
 .PHONY: build-test-c
@@ -169,14 +175,14 @@ cov:
 .PHONY: test
 test: build-tests
 test:
-	@python -m pytest -vv ./tests
+	@PYTHONFAULTHANDLER=1 python -m pytest -vv ./tests
 
 
 # help: test-verbose                   - Build and run all tests [verbosely]
 .PHONY: test-verbose
 test-verbose: build-tests
 test-verbose:
-	@python -m pytest -vv -s ./tests
+	@PYTHONFAULTHANDLER=1 python -m pytest -vv -s ./tests
 
 # help: test-c                         - Build and run all C tests
 .PHONY: test-c
@@ -187,14 +193,20 @@ test-c:
 # help: test-cpp                       - Build and run all C++ tests
 .PHONY: test-cpp
 test-cpp: build-test-cpp
+test-cpp: build-unit-test-cpp
 test-cpp:
 	@python -m pytest -vv -s ./tests/cpp/
 
+# help: unit-test-cpp                  - Build and run unit tests for C++
+.PHONY: unit-test-cpp
+unit-test-cpp: build-unit-test-cpp
+unit-test-cpp:
+	@python -m pytest -vv -s ./tests/cpp/unit-tests/
 
 # help: test-py                        - run python tests
 .PHONY: test-py
 test-py:
-	@python -m pytest -vv ./tests/python/
+	@PYTHONFAULTHANDLER=1 python -m pytest -vv ./tests/python/
 
 # help: test-fortran                   - run fortran tests
 .PHONY: test-fortran
@@ -204,5 +216,9 @@ test-fortran: build-test-fortran
 # help: testpy-cov                     - run python tests with coverage
 .PHONY: testpy-cov
 testpy-cov:
-	@python -m pytest --cov=./src/python/module/smartredis/ -vv ./tests/python/
+	@PYTHONFAULTHANDLER=1 python -m pytest --cov=./src/python/module/smartredis/ -vv ./tests/python/
 
+# help: testcpp-cov                    - run cpp unit tests with coverage
+.PHONY: testcpp-cov
+testcpp-cov: unit-test-cpp
+	./build-scripts/build_cpp_cov.sh

README.md

@@ -18,7 +18,9 @@
 
 [![License](https://img.shields.io/github/license/CrayLabs/SmartSim)](https://github.com/CrayLabs/SmartRedis/blob/master/LICENSE.md)
 ![GitHub last commit](https://img.shields.io/github/last-commit/CrayLabs/SmartRedis)
+![PyPI - Wheel](https://img.shields.io/pypi/wheel/smartredis)
 ![GitHub tag (latest by date)](https://img.shields.io/github/v/tag/CrayLabs/SmartRedis)
+![PyPI - Python Version](https://img.shields.io/pypi/pyversions/smartredis)
 ![Language](https://img.shields.io/github/languages/top/CrayLabs/SmartRedis)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 
@@ -32,7 +34,7 @@ SmartRedis provides clients in the following languages:
 
 | Language   | Version/Standard  |
 |------------|:-----------------:|
-| Python     |   3.7+            |
+| Python     |   3.7, 3.8, 3.9   |
 | C++        |   C++17           |
 | C          |   C99             |
 | Fortran    |   Fortran 2018    |
@@ -60,8 +62,6 @@ SmartRedis utilizes the following libraries.
  - [NumPy](https://github.com/numpy/numpy)
  - [Hiredis](https://github.com/redis/hiredis) 1.0.0
  - [Redis-plus-plus](https://github.com/sewenew/redis-plus-plus)  1.2.3
- - [protobuf](https://github.com/protocolbuffers/protobuf)  3.11.3
-
 
 ## Publications
 

build-scripts/build-catch.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# Install Catch
+if [[ -f ./catch/catch.hpp ]]; then
+    echo "Catch has already been download and installed"
+else
+    echo "Installing Catch"
+    if [[ ! -d "./catch" ]]; then
+        wget https://github.com/catchorg/Catch2/releases/download/v2.13.6/catch.hpp -P catch
+    else
+        echo "Catch downloaded"
+    fi
+    echo "Finished installing Catch"
+fi

build-scripts/build-lcov.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Install LCOV
+if [[ -f ./lcov/install/usr/local/bin/lcov ]]; then
+    echo "LCOV has already been download and installed"
+else
+    echo "Installing LCOV"
+    if [[ ! -d "./lcov" ]]; then
+        git clone https://github.com/linux-test-project/lcov.git --branch v1.15 --depth=1 lcov
+    else
+        echo "LCOV downloaded"
+    fi
+    cd lcov
+    mkdir install
+    echo "Building LCOV v1.15"
+    CC=gcc CXX=g++ DESTDIR="install/" make install
+    cd ../
+fi

build-scripts/build-redisai-cpu.sh

@@ -1,11 +1,14 @@
 #!/bin/bash
 
+# get the number of processors
+NPROC=$(python -c "import multiprocessing as mp; print(mp.cpu_count())")
+
 #Install RedisAI
 if [[ -f ./RedisAI/install-cpu/redisai.so ]]; then
     echo "RedisAI CPU has already been downloaded and installed"
 else
     if [[ ! -d "./RedisAI" ]]; then
-        git clone --recursive https://github.com/RedisAI/RedisAI.git --branch v1.2.2 --depth=1 RedisAI
+        GIT_LFS_SKIP_SMUDGE=1 git clone --recursive https://github.com/RedisAI/RedisAI.git --branch v1.2.3 --depth=1 RedisAI
         cd RedisAI
         cd ..
     else
@@ -15,7 +18,7 @@ else
     echo "Downloading RedisAI CPU dependencies"
     CC=gcc CXX=g++ WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 bash get_deps.sh cpu
     echo "Building RedisAI"
-    CC=gcc CXX=g++ GPU=0 WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 WITH_UNIT_TESTS=0 make -C opt clean build
+    CC=gcc CXX=g++ GPU=0 WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 WITH_UNIT_TESTS=0 make -j $NPROC -C opt clean build
 
     if [ -f "./install-cpu/redisai.so" ]; then
         echo "Finished installing RedisAI"

build-scripts/build-redisai-gpu.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# get the number of processors
+NPROC=$(python -c "import multiprocessing as mp; print(mp.cpu_count())")
+
 #Install RedisAI
 if [[ -f ./RedisAI/install-gpu/redisai.so ]]; then
     echo "RedisAI GPU has already been downloaded and installed"
@@ -43,17 +46,17 @@ else
 
 
     if [[ ! -d "./RedisAI" ]]; then
-        git clone --recursive https://github.com/RedisAI/RedisAI.git --branch v1.2.2 --depth=1 RedisAI
+        GIT_LFS_SKIP_SMUDGE=1 git clone --recursive https://github.com/RedisAI/RedisAI.git --branch v1.2.3 --depth=1 RedisAI
         cd RedisAI
         cd ..
     else
         echo "RedisAI downloaded"
     fi
     cd RedisAI
     echo "Downloading RedisAI CPU dependencies"
-    CC=gcc CXX=g++ WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 bash get_deps.sh cpu
+    CC=gcc CXX=g++ WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 bash get_deps.sh gpu
     echo "Building RedisAI"
-    CC=gcc CXX=g++ GPU=1 WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 WITH_UNIT_TESTS=0 make -C opt clean build
+    CC=gcc CXX=g++ GPU=1 WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 WITH_UNIT_TESTS=0 make -j $NPROC -C opt clean build
 
     if [ -f "./install-gpu/redisai.so" ]; then
         echo "Finished installing RedisAI"