From eeedfa9f45091ae6bfc1289c5a0441e194ee06b3 Mon Sep 17 00:00:00 2001 From: Anthony Maton Date: Tue, 14 Feb 2017 08:58:40 +0100 Subject: [PATCH 01/10] Update Dockerfile - Add ca-certificates as a dependency to update the CA certs database to allow HTTPS resolution (required to clone with git and wget the busybx - Add bzip2 as a dependency to decompress the busybox bzip2 archive --- Dockerfile | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9d727c1..fd47758 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ MAINTAINER Virginie Van den Schrieck, virginie.vandenschrieck@pythia-project.org RUN cat /etc/resolv.conf RUN apt-get update \ && apt-get install -y gcc libc6-dev make curl wget xz-utils\ - --no-install-recommends \ + ca-certificates bzip2 --no-install-recommends \ && rm -rf /var/lib/apt/lists/* @@ -37,8 +37,7 @@ RUN apt-get update \ RUN apt-get update \ && apt-get install -y fakeroot squashfs-tools libc6-dev-i386 bc - - + #Install make4 WORKDIR /home/ @@ -49,7 +48,7 @@ RUN ./configure && make && make install &&make distclean #Install Pythia WORKDIR /home -RUN git clone https://github.com/pythia-project/pythia.git +RUN git clone https://github.com/pythia-project/pythia.git RUN ls && pwd WORKDIR /home/pythia/ RUN git submodule update --init --recursive && make @@ -60,5 +59,3 @@ RUN echo "tmpfs /dev/shm tmpfs defaults,nosuid,nodev 0 0" >> /etc/fstab && echo #TODO manually when running in privileged mode : mount /dev/shm - - From 317f5e89e8489a0bcd0370673652e6c296408f70 Mon Sep 17 00:00:00 2001 From: Anthony Maton Date: Tue, 14 Feb 2017 09:14:27 +0100 Subject: [PATCH 02/10] Update pythia github URL It is done to avoid unexpected behavior if github changes its redirection system --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fd47758..c8bb360 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,7 +48,7 @@ RUN ./configure && make && make install &&make distclean #Install Pythia WORKDIR /home -RUN git clone https://github.com/pythia-project/pythia.git +RUN git clone https://github.com/pythia-project/pythia-core.git RUN ls && pwd WORKDIR /home/pythia/ RUN git submodule update --init --recursive && make From 763983a60f703fa14bc6ef1e9e69716f8bb7fe50 Mon Sep 17 00:00:00 2001 From: Anthony Maton Date: Tue, 14 Feb 2017 09:16:39 +0100 Subject: [PATCH 03/10] Add destination to the clone instruction --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c8bb360..306fb68 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,7 +48,7 @@ RUN ./configure && make && make install &&make distclean #Install Pythia WORKDIR /home -RUN git clone https://github.com/pythia-project/pythia-core.git +RUN git clone https://github.com/pythia-project/pythia-core.git pythia RUN ls && pwd WORKDIR /home/pythia/ RUN git submodule update --init --recursive && make From 8ceec0a5d4ebca640e708c052b78967a6b32434b Mon Sep 17 00:00:00 2001 From: Anthony Maton Date: Wed, 15 Feb 2017 09:22:09 +0100 Subject: [PATCH 04/10] Add basic Docker quickinstall documentation --- README.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 11fa88d..060f417 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,29 @@ Once successfully installed, you can try to execute a simple task: and you will see, among others, ``Hello world!`` printed in your terminal. +## Use with Docker +Docker allow the pythia-core framework to run on MacOS or Windows installation. + +Start by cloning the git repository and build the docker image: + + > git clone --recursive https://github.com/pythia-project/pythia-core.git + > cd pythia-core + > docker build -t pythia-core . + +Once the image is successfully built, you can now start the image: + + > docker run -dit -p 8080:8080 --security-opt seccomp:unconfined --privileged pythia-core + > docker exec -it --privileged CONTAINER_ID bash + > mount /dev/shm + > cd out && touch input.txt + > ./pythia execute -input="input.txt" -tasks="tasks/hello-world.task" + +You can obtain the container id using docker ps. +You should see among others, ``Hello world!`` printed in your terminal. + ## Contributors - Sébastien Combéfis - Vianney le Clément de Saint-Marcq - Charles Vandevoorde -- Virginie Van den Schrieck \ No newline at end of file +- Virginie Van den Schrieck From 742fdf4f14acb65daf99fa76adfe70f382065ed0 Mon Sep 17 00:00:00 2001 From: Anthony Maton Date: Thu, 16 Feb 2017 09:42:07 +0100 Subject: [PATCH 05/10] Remove custom make4.1 triggering SEGFAULT --- Dockerfile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 306fb68..ab7fa59 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,13 +38,6 @@ RUN apt-get update \ RUN apt-get update \ && apt-get install -y fakeroot squashfs-tools libc6-dev-i386 bc -#Install make4 - -WORKDIR /home/ -RUN curl http://gnu.xl-mirror.nl/make/make-4.1.tar.gz | tar -v -C . -xz -WORKDIR /home/make-4.1/ -RUN ./configure && make && make install &&make distclean - #Install Pythia WORKDIR /home From 2a5f4ae3aefe51602ffee154910455ae031abcd5 Mon Sep 17 00:00:00 2001 From: Anthony Maton Date: Fri, 31 Mar 2017 11:43:36 +0200 Subject: [PATCH 06/10] Mount /dev/shm automatically instead of manually This change force to run the container in privilege mode which was already the case since the "mount /dev/shm" was required to use pythia-core --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ab7fa59..5cbdc5f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,7 +48,7 @@ RUN git submodule update --init --recursive && make #Change fstab to have shm in no-exec mode for UML -RUN echo "tmpfs /dev/shm tmpfs defaults,nosuid,nodev 0 0" >> /etc/fstab && echo "">>/etc/fstab -#TODO manually when running in privileged mode : mount /dev/shm +RUN echo "tmpfs /dev/shm tmpfs defaults,nosuid,nodev 0 0" >> /etc/fstab && echo "">>/etc/fstab +RUN mount /dev/shm From 23eee21bceb7b4fa0a0bff8f14a1e75af04b6076 Mon Sep 17 00:00:00 2001 From: Anthony Maton Date: Fri, 31 Mar 2017 13:01:47 +0200 Subject: [PATCH 07/10] Add a docker-compose for better management --- Dockerfile | 2 +- docker-compose.yml | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile index 5cbdc5f..68bbcf3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,5 +50,5 @@ RUN git submodule update --init --recursive && make #Change fstab to have shm in no-exec mode for UML RUN echo "tmpfs /dev/shm tmpfs defaults,nosuid,nodev 0 0" >> /etc/fstab && echo "">>/etc/fstab -RUN mount /dev/shm +ENTRYPOINT sh -c "mount /dev/shm && ./out/pythia queue & ./out/pythia pool & ./out/pythia server" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1e499fc --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,12 @@ +version: '3' +services: + core: + privileged: true + security_opt: + - seccomp:unconfined + build: . + ports: + - "9000:9000" + - "8080:8080" + volumes: + - .:/home/pythia From ff124ce850931f5d7e756fcc1d145ab416e26af8 Mon Sep 17 00:00:00 2001 From: Anthony Maton Date: Fri, 31 Mar 2017 13:04:25 +0200 Subject: [PATCH 08/10] Adjust README to the new workflow --- README.md | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 060f417..c8c5490 100644 --- a/README.md +++ b/README.md @@ -29,23 +29,16 @@ and you will see, among others, ``Hello world!`` printed in your terminal. ## Use with Docker Docker allow the pythia-core framework to run on MacOS or Windows installation. - +It requires docker-compose to also be present (it's shipped by default for OS X and Windows and needs to be downloaded for Linux) Start by cloning the git repository and build the docker image: > git clone --recursive https://github.com/pythia-project/pythia-core.git > cd pythia-core - > docker build -t pythia-core . + > docker-compose --verbose build Once the image is successfully built, you can now start the image: - > docker run -dit -p 8080:8080 --security-opt seccomp:unconfined --privileged pythia-core - > docker exec -it --privileged CONTAINER_ID bash - > mount /dev/shm - > cd out && touch input.txt - > ./pythia execute -input="input.txt" -tasks="tasks/hello-world.task" - -You can obtain the container id using docker ps. -You should see among others, ``Hello world!`` printed in your terminal. + > docker-compose up ## Contributors From 533b410ce7a117b39c77b3681e3a80e1df3bd482 Mon Sep 17 00:00:00 2001 From: Anthony Maton Date: Mon, 3 Apr 2017 10:55:16 +0200 Subject: [PATCH 09/10] Separate each component in a separate container --- Dockerfile | 4 ++-- docker-compose.yml | 26 ++++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 68bbcf3..96bea2e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ MAINTAINER Virginie Van den Schrieck, virginie.vandenschrieck@pythia-project.org RUN cat /etc/resolv.conf RUN apt-get update \ && apt-get install -y gcc libc6-dev make curl wget xz-utils\ - ca-certificates bzip2 --no-install-recommends \ + ca-certificates bzip2 --no-install-recommends strace \ && rm -rf /var/lib/apt/lists/* @@ -51,4 +51,4 @@ RUN git submodule update --init --recursive && make RUN echo "tmpfs /dev/shm tmpfs defaults,nosuid,nodev 0 0" >> /etc/fstab && echo "">>/etc/fstab -ENTRYPOINT sh -c "mount /dev/shm && ./out/pythia queue & ./out/pythia pool & ./out/pythia server" \ No newline at end of file +ENTRYPOINT sh -c "mount /dev/shm && strace ./out/pythia queue & strace ./out/pythia pool & strace ./out/pythia server" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 1e499fc..42ca4e4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,34 @@ version: '3' services: - core: + pool: + privileged: true + security_opt: + - seccomp:unconfined + build: . + volumes: + - .:/home/pythia + links: + - queue + entrypoint: ./out/pythia -queue "queue:9000" pool & mount -av + queue: privileged: true security_opt: - seccomp:unconfined build: . ports: - "9000:9000" - - "8080:8080" volumes: - .:/home/pythia + entrypoint: ./out/pythia -queue "0.0.0.0:9000" queue + server: + privileged: true + security_opt: + - seccomp:unconfined + build: . + ports: + - "8080:8080" + volumes: + - ./:/home/pythia + links: + - queue + entrypoint: ./out/pythia -queue "queue:9000" server From c99568dbd63080f2844208223fdcdac00c7e1198 Mon Sep 17 00:00:00 2001 From: Anthony Maton Date: Mon, 3 Apr 2017 15:01:33 +0200 Subject: [PATCH 10/10] Adapt dockerfile to the new development environment --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 96bea2e..4b49e2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ MAINTAINER Virginie Van den Schrieck, virginie.vandenschrieck@pythia-project.org RUN cat /etc/resolv.conf RUN apt-get update \ && apt-get install -y gcc libc6-dev make curl wget xz-utils\ + --no-install-recommends \ ca-certificates bzip2 --no-install-recommends strace \ && rm -rf /var/lib/apt/lists/* @@ -51,4 +52,4 @@ RUN git submodule update --init --recursive && make RUN echo "tmpfs /dev/shm tmpfs defaults,nosuid,nodev 0 0" >> /etc/fstab && echo "">>/etc/fstab -ENTRYPOINT sh -c "mount /dev/shm && strace ./out/pythia queue & strace ./out/pythia pool & strace ./out/pythia server" \ No newline at end of file +# ENTRYPOINT sh -c "mount /dev/shm && strace ./out/pythia queue & strace ./out/pythia pool & strace ./out/pythia server"