From 33cef0058a6f2ef8a17e403acd150ecd6b6f0851 Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 13 Feb 2020 15:39:38 +0800 Subject: [PATCH 1/9] init --- elasticdl/Makefile | 18 +++++++++--------- elasticdl/docker/Dockerfile.dev | 5 +++++ elasticdl/proto/elasticdl.proto | 1 + 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/elasticdl/Makefile b/elasticdl/Makefile index 981b1a0fc..1b1ea3e84 100644 --- a/elasticdl/Makefile +++ b/elasticdl/Makefile @@ -1,21 +1,21 @@ PYTHON = python - -EDL_PROTO_FILE = elasticdl/proto/elasticdl.proto - +PROTO_PATH=elasticdl/proto GO_FILE_DIR = elasticdl/pkg/proto -GO_PB_FILE = $(GO_FILE_DIR)/elasticdl.pb.go - CXX = g++ CAPI_DIR = elasticdl/pkg/kernel/capi -all: python_pb $(GO_PB_FILE) kernel +all: python_pb go_pb kernel python_pb: - $(PYTHON) -m grpc_tools.protoc -I . $(EDL_PROTO_FILE) --python_out=. --grpc_python_out=. + $(PYTHON) -m grpc_tools.protoc -I${TF_PATH} -I${PROTO_PATH} \ + ${PROTO_PATH}/elasticdl.proto --python_out=${PROTO_PATH} --grpc_python_out=${PROTO_PATH} -$(GO_PB_FILE): $(EDL_PROTO_FILE) +go_pb: $(EDL_PROTO_FILE) mkdir -p $(GO_FILE_DIR) - protoc -I . $< --go_out=plugins=grpc:. + protoc -I${TF_PATH} ${TF_PATH}/tensorflow/core/framework/resource_handle.proto --go_out=${GOPATH}/src + protoc -I${TF_PATH} ${TF_PATH}/tensorflow/core/framework/tensor_shape.proto --go_out=${GOPATH}/src + protoc -I${TF_PATH} ${TF_PATH}/tensorflow/core/framework/tensor.proto --go_out=${GOPATH}/src + protoc -I${TF_PATH} -I${PROTO_PATH} ${PROTO_PATH}/elasticdl.proto --go_out=plugins=grpc:. kernel: ${CXX} -O3 -std=c++11 ${CAPI_DIR}/kernel_api.cc -c -o ${CAPI_DIR}/libkernel_api.o diff --git a/elasticdl/docker/Dockerfile.dev b/elasticdl/docker/Dockerfile.dev index 04a164392..64c1818a5 100644 --- a/elasticdl/docker/Dockerfile.dev +++ b/elasticdl/docker/Dockerfile.dev @@ -49,6 +49,11 @@ RUN /install-protobuf.bash && rm /install-protobuf.bash # Install elasticdl.org/elasticdl Go package ENV ELASTICDLPATH $GOPATH/src/elasticdl.org/elasticdl +ENV TF_PATH /tmp/tensorflow +RUN mkdir -p $TF_PATH/tensorflow/core/framework && \ + curl -OL https://raw.githubusercontent.com/tensorflow/tensorflow/master/tensorflow/core/framework/tensor.proto && \ + curl -OL https://raw.githubusercontent.com/tensorflow/tensorflow/master/tensorflow/core/framework/tensor_shape.proto && \ + curl -OL https://raw.githubusercontent.com/tensorflow/tensorflow/master/tensorflow/core/framework/resource_handle.proto COPY . /tmp/elasticdl RUN cd /tmp/elasticdl && make -f elasticdl/Makefile && \ mkdir -p $ELASTICDLPATH && \ diff --git a/elasticdl/proto/elasticdl.proto b/elasticdl/proto/elasticdl.proto index 33dd3e37e..316967bb6 100644 --- a/elasticdl/proto/elasticdl.proto +++ b/elasticdl/proto/elasticdl.proto @@ -1,6 +1,7 @@ syntax = "proto3"; import "google/protobuf/empty.proto"; +import "tensorflow/core/framework/tensor.proto"; package proto; option go_package = "elasticdl/pkg/proto"; From 7b2462ca6a303ac371d95bccb2ba590a110d163e Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 13 Feb 2020 15:58:47 +0800 Subject: [PATCH 2/9] update --- elasticdl/docker/Dockerfile.dev | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/elasticdl/docker/Dockerfile.dev b/elasticdl/docker/Dockerfile.dev index 64c1818a5..0ee2b6795 100644 --- a/elasticdl/docker/Dockerfile.dev +++ b/elasticdl/docker/Dockerfile.dev @@ -50,10 +50,8 @@ RUN /install-protobuf.bash && rm /install-protobuf.bash # Install elasticdl.org/elasticdl Go package ENV ELASTICDLPATH $GOPATH/src/elasticdl.org/elasticdl ENV TF_PATH /tmp/tensorflow -RUN mkdir -p $TF_PATH/tensorflow/core/framework && \ - curl -OL https://raw.githubusercontent.com/tensorflow/tensorflow/master/tensorflow/core/framework/tensor.proto && \ - curl -OL https://raw.githubusercontent.com/tensorflow/tensorflow/master/tensorflow/core/framework/tensor_shape.proto && \ - curl -OL https://raw.githubusercontent.com/tensorflow/tensorflow/master/tensorflow/core/framework/resource_handle.proto +RUN cd /tmp && git clone --depth=1 https://github.com/tensorflow/tensorflow.git +RUN mkdir -p ${GOPATH}/src COPY . /tmp/elasticdl RUN cd /tmp/elasticdl && make -f elasticdl/Makefile && \ mkdir -p $ELASTICDLPATH && \ @@ -61,4 +59,4 @@ RUN cd /tmp/elasticdl && make -f elasticdl/Makefile && \ cd $GOPATH/src/elasticdl.org/elasticdl && \ export GO111MODULE=on && \ go mod init elasticdl.org/elasticdl && \ - go install ./... && cd / && rm -rf /tmp/elasticdl + go install ./... && cd / && rm -rf /tmp/elasticdl && rm -rf /tmp/tensorflow From a9f8154fccabdc4bf2f8a7d741b90944ae8cef2e Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 13 Feb 2020 16:19:20 +0800 Subject: [PATCH 3/9] refine cmd --- elasticdl/docker/Dockerfile.dev | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/elasticdl/docker/Dockerfile.dev b/elasticdl/docker/Dockerfile.dev index 0ee2b6795..a7a658260 100644 --- a/elasticdl/docker/Dockerfile.dev +++ b/elasticdl/docker/Dockerfile.dev @@ -51,10 +51,9 @@ RUN /install-protobuf.bash && rm /install-protobuf.bash ENV ELASTICDLPATH $GOPATH/src/elasticdl.org/elasticdl ENV TF_PATH /tmp/tensorflow RUN cd /tmp && git clone --depth=1 https://github.com/tensorflow/tensorflow.git -RUN mkdir -p ${GOPATH}/src COPY . /tmp/elasticdl -RUN cd /tmp/elasticdl && make -f elasticdl/Makefile && \ - mkdir -p $ELASTICDLPATH && \ +RUN cd /tmp/elasticdl && mkdir -p $ELASTICDLPATH && \ + make -f elasticdl/Makefile && \ cp -r /tmp/elasticdl/elasticdl/pkg $ELASTICDLPATH && \ cd $GOPATH/src/elasticdl.org/elasticdl && \ export GO111MODULE=on && \ From 25f5e1aac6b0d181c8482e84332b61579550f95e Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 13 Feb 2020 18:22:06 +0800 Subject: [PATCH 4/9] polish again --- elasticdl/Makefile | 3 ++- elasticdl/docker/Dockerfile.dev | 1 - elasticdl/docker/scripts/install-go.bash | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/elasticdl/Makefile b/elasticdl/Makefile index 1b1ea3e84..5b0f30cb2 100644 --- a/elasticdl/Makefile +++ b/elasticdl/Makefile @@ -10,8 +10,9 @@ python_pb: $(PYTHON) -m grpc_tools.protoc -I${TF_PATH} -I${PROTO_PATH} \ ${PROTO_PATH}/elasticdl.proto --python_out=${PROTO_PATH} --grpc_python_out=${PROTO_PATH} -go_pb: $(EDL_PROTO_FILE) +go_pb: mkdir -p $(GO_FILE_DIR) + protoc -I${TF_PATH} ${TF_PATH}/tensorflow/core/framework/types.proto --go_out=${GOPATH}/src protoc -I${TF_PATH} ${TF_PATH}/tensorflow/core/framework/resource_handle.proto --go_out=${GOPATH}/src protoc -I${TF_PATH} ${TF_PATH}/tensorflow/core/framework/tensor_shape.proto --go_out=${GOPATH}/src protoc -I${TF_PATH} ${TF_PATH}/tensorflow/core/framework/tensor.proto --go_out=${GOPATH}/src diff --git a/elasticdl/docker/Dockerfile.dev b/elasticdl/docker/Dockerfile.dev index a7a658260..bec2180b6 100644 --- a/elasticdl/docker/Dockerfile.dev +++ b/elasticdl/docker/Dockerfile.dev @@ -56,6 +56,5 @@ RUN cd /tmp/elasticdl && mkdir -p $ELASTICDLPATH && \ make -f elasticdl/Makefile && \ cp -r /tmp/elasticdl/elasticdl/pkg $ELASTICDLPATH && \ cd $GOPATH/src/elasticdl.org/elasticdl && \ - export GO111MODULE=on && \ go mod init elasticdl.org/elasticdl && \ go install ./... && cd / && rm -rf /tmp/elasticdl && rm -rf /tmp/tensorflow diff --git a/elasticdl/docker/scripts/install-go.bash b/elasticdl/docker/scripts/install-go.bash index 69d32c86c..f4454181e 100755 --- a/elasticdl/docker/scripts/install-go.bash +++ b/elasticdl/docker/scripts/install-go.bash @@ -6,7 +6,6 @@ GO_MIRROR_URL=$1 curl --silent ${GO_MIRROR_URL}/go1.13.4.linux-amd64.tar.gz | tar -C /usr/local -xzf - -go env -w GO111MODULE=on go env -w GOPROXY=https://goproxy.io,direct go get github.com/golang/protobuf/protoc-gen-go From dd4044cc749134c50c808d503d41e408cdad3d7b Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 13 Feb 2020 18:35:00 +0800 Subject: [PATCH 5/9] test ci --- elasticdl/docker/Dockerfile.dev | 1 - 1 file changed, 1 deletion(-) diff --git a/elasticdl/docker/Dockerfile.dev b/elasticdl/docker/Dockerfile.dev index bec2180b6..0bfa5d322 100644 --- a/elasticdl/docker/Dockerfile.dev +++ b/elasticdl/docker/Dockerfile.dev @@ -56,5 +56,4 @@ RUN cd /tmp/elasticdl && mkdir -p $ELASTICDLPATH && \ make -f elasticdl/Makefile && \ cp -r /tmp/elasticdl/elasticdl/pkg $ELASTICDLPATH && \ cd $GOPATH/src/elasticdl.org/elasticdl && \ - go mod init elasticdl.org/elasticdl && \ go install ./... && cd / && rm -rf /tmp/elasticdl && rm -rf /tmp/tensorflow From eb4b5bb932a67f1871ec1145ef3579bfe03ccd9b Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 13 Feb 2020 18:53:42 +0800 Subject: [PATCH 6/9] test ci --- elasticdl/docker/Dockerfile.dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticdl/docker/Dockerfile.dev b/elasticdl/docker/Dockerfile.dev index 0bfa5d322..384a91891 100644 --- a/elasticdl/docker/Dockerfile.dev +++ b/elasticdl/docker/Dockerfile.dev @@ -56,4 +56,4 @@ RUN cd /tmp/elasticdl && mkdir -p $ELASTICDLPATH && \ make -f elasticdl/Makefile && \ cp -r /tmp/elasticdl/elasticdl/pkg $ELASTICDLPATH && \ cd $GOPATH/src/elasticdl.org/elasticdl && \ - go install ./... && cd / && rm -rf /tmp/elasticdl && rm -rf /tmp/tensorflow + go install ./... && cd / && rm -rf /tmp/elasticdl From da525908b380aa53243ad30b43216089e713dc80 Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 13 Feb 2020 19:11:06 +0800 Subject: [PATCH 7/9] trigger ci --- elasticdl/docker/scripts/install-go.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/elasticdl/docker/scripts/install-go.bash b/elasticdl/docker/scripts/install-go.bash index f4454181e..9650bbc26 100755 --- a/elasticdl/docker/scripts/install-go.bash +++ b/elasticdl/docker/scripts/install-go.bash @@ -15,5 +15,6 @@ go get golang.org/x/tools/cmd/cover go get github.com/mattn/goveralls go get github.com/rakyll/gotest go get github.com/wangkuiyi/goyaccfmt +go get github.com/stretchr/testify/assert cp $GOPATH/bin/* /usr/local/bin/ From f7c02a009931cb8cb59eb5fe7499081e8deac6b8 Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 13 Feb 2020 19:44:52 +0800 Subject: [PATCH 8/9] fix ci --- elasticdl/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elasticdl/Makefile b/elasticdl/Makefile index 5b0f30cb2..d1584eeeb 100644 --- a/elasticdl/Makefile +++ b/elasticdl/Makefile @@ -7,8 +7,8 @@ CAPI_DIR = elasticdl/pkg/kernel/capi all: python_pb go_pb kernel python_pb: - $(PYTHON) -m grpc_tools.protoc -I${TF_PATH} -I${PROTO_PATH} \ - ${PROTO_PATH}/elasticdl.proto --python_out=${PROTO_PATH} --grpc_python_out=${PROTO_PATH} + $(PYTHON) -m grpc_tools.protoc -I${TF_PATH} -I. \ + ${PROTO_PATH}/elasticdl.proto --python_out=. --grpc_python_out=. go_pb: mkdir -p $(GO_FILE_DIR) From 6b605c2dda989fe32d15a4024c3427f1478f2c67 Mon Sep 17 00:00:00 2001 From: qijun Date: Fri, 14 Feb 2020 10:18:10 +0800 Subject: [PATCH 9/9] follow comments --- elasticdl/Makefile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/elasticdl/Makefile b/elasticdl/Makefile index d1584eeeb..446ad3de7 100644 --- a/elasticdl/Makefile +++ b/elasticdl/Makefile @@ -1,22 +1,21 @@ PYTHON = python -PROTO_PATH=elasticdl/proto GO_FILE_DIR = elasticdl/pkg/proto CXX = g++ CAPI_DIR = elasticdl/pkg/kernel/capi all: python_pb go_pb kernel -python_pb: +python_pb: elasticdl/proto/elasticdl.proto $(PYTHON) -m grpc_tools.protoc -I${TF_PATH} -I. \ - ${PROTO_PATH}/elasticdl.proto --python_out=. --grpc_python_out=. + elasticdl/proto/elasticdl.proto --python_out=. --grpc_python_out=. -go_pb: +go_pb: elasticdl/proto/elasticdl.proto mkdir -p $(GO_FILE_DIR) protoc -I${TF_PATH} ${TF_PATH}/tensorflow/core/framework/types.proto --go_out=${GOPATH}/src protoc -I${TF_PATH} ${TF_PATH}/tensorflow/core/framework/resource_handle.proto --go_out=${GOPATH}/src protoc -I${TF_PATH} ${TF_PATH}/tensorflow/core/framework/tensor_shape.proto --go_out=${GOPATH}/src protoc -I${TF_PATH} ${TF_PATH}/tensorflow/core/framework/tensor.proto --go_out=${GOPATH}/src - protoc -I${TF_PATH} -I${PROTO_PATH} ${PROTO_PATH}/elasticdl.proto --go_out=plugins=grpc:. + protoc -I${TF_PATH} -Ielasticdl/proto elasticdl/proto/elasticdl.proto --go_out=plugins=grpc:. kernel: ${CXX} -O3 -std=c++11 ${CAPI_DIR}/kernel_api.cc -c -o ${CAPI_DIR}/libkernel_api.o