Docker Compose (fastapi, milvus) logic

.github/workflows/test.yaml

@@ -1,4 +1,4 @@
-name: Python package
+name: test
 
 on: [push]
 
@@ -27,6 +27,5 @@ jobs:
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-      - name: Test with pytest
-        run: |
-           pytest --verbose tests/running_piper_test.py::TestVenv
+      - name: Test with makefile
+        run: make test

MANIFEST.in

@@ -0,0 +1,4 @@
+include piper/base/backend/templates/*.j2
+include piper/base/docker/images/*.j2
+include piper/base/docker_compose/templates/*.j2
+include piper/base/virtualenv/templates/*.j2

Makefile

@@ -0,0 +1,16 @@
+SHELL:=/usr/bin/env bash
+
+.PHONY: lint
+lint:
+	flake8 piper
+
+.PHONY: unit
+unit:
+	pytest -vs tests/running_piper_test.py::TestCompose
+
+.PHONY: package
+package:
+	pip check
+
+.PHONY: test
+test: package unit

Readme.rst

@@ -5,6 +5,7 @@
 • `Chat (Community & Support) <https://t.me/pipertool>`_
 • `Tutorials <http://pipertool.org>`_
 
+|Contribs| |License| |Release| |Python Version| |Docs| |Build status| |Contact| |Blog|
 
 **Piper** is an **open-source** platform for data science and machine
 learning prototyping. Concentrate only on your goals. Key features:
@@ -31,6 +32,48 @@ How Piper works
 
 Quick start
 ===========
+Quick start pipertool package compose env
+===========
+
+In root directory project run command in terminal
+
+- sudo -u root /bin/bash
+
+- create and activate venv
+
+- pip install -r requirements.txt
+
+- in configuration.py rename for correctly path for new directory
+
+- python setup.py install
+
+- piper --env-type compose start
+
+- 0.0.0.0:7585 - FastApi
+
+- 0.0.0.0:9001 - Milvus Console (minioadmin/minioadmin)
+
+- piper --env-type compose stop
+
+- pip uninstall piper
+
+Quick start pipertool package compose env
+===========
+
+In root directory project run command in terminal
+
+- sudo -u root /bin/bash
+
+- create and activate venv
+
+- pip install -r requirements.txt
+
+- in configuration.py rename for correctly path for new directory
+
+- python main.py
+
+- await click CTRL+C from compose env
+
 
 
 Installation
@@ -88,6 +131,28 @@ By submitting a pull request to this project, you agree to license your contribu
    :alt: Piper logo
 
 
-.. |Contribs| image:: https://tatradev.com
-   :target: https://github.com/TatraDev/piper/graphs/contributors
+.. |Contribs| image:: https://img.shields.io/badge/contributors-3-brightgreen
+   :target: https://github.com/TatraDev/pipertool/graphs/contributors
    :alt: Contributors
+
+.. |Python Version| image:: https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10-blue
+   :target: https://pypi.org/project/pipertool
+   :alt: Python Version
+
+.. |Release| image:: https://img.shields.io/badge/release-v0.0.2-blue
+   :target: https://github.com/TatraDev/pipertool/releases
+
+.. |Build status| image:: https://github.com/TatraDev/pipertool/workflows/test/badge.svg?branch=venv_logic&event=push
+    :target: https://github.com/TatraDev/pipertool/actions?query=workflow%3Atest
+
+.. |Contact| image:: https://img.shields.io/badge/telegram-write%20me-blue.svg
+    :target:  https://t.me/pipertool
+
+.. |Blog| image:: https://img.shields.io/badge/site-my%20blog-yellow.svg
+    :target:  https://tatradev.com
+
+.. |License| image:: https://img.shields.io/badge/License-Apache_2.0-yellow.svg
+    :target:  https://www.apache.org/licenses/LICENSE-2.0
+
+.. |Docs| image:: https://img.shields.io/badge/docs-passing-brightgreen
+    :target:  http://pipertool.org

main.py

@@ -5,7 +5,8 @@
 from loguru import logger
 
 from piper.configurations import get_configuration
-from piper.envs import CurrentEnv, DockerEnv, VirtualEnv
+from piper.envs import ComposeEnv, CurrentEnv, DockerEnv, VirtualEnv
+
 # from piper.services import (SpacyNER, StringValue, TesseractRecognizer,
 #                             TestMessageAdder)
 # from piper.utils import tesrct_utils as tu
@@ -21,7 +22,7 @@
     #     result = loop.run_until_complete(adder(x))
     #     print(result)
 
-    # x = StringValue(value="hello, world") 
+    # x = StringValue(value="hello, world")
     # adder = TestMessageAdder(appender="!", port=cfg.docker_app_port)
     # result = loop.run_until_complete(adder(x))
     # print(result)
@@ -37,11 +38,21 @@
     #     result = loop.run_until_complete(recognizer())
     #     logger.info(f'result of recognition is {result}')
 
-    with VirtualEnv() as env:
-        env.copy_struct_project()
-        env.create_files_for_venv()
-        env.create_files_for_tests()
-
+    # with VirtualEnv() as env:
+    #     env.copy_struct_project()
+    #     env.create_files_for_venv()
+    #     env.create_files_for_tests()
+
+    with ComposeEnv() as env:
+        try:
+            env.copy_struct_project()
+            env.create_files_for_compose(testing=True)
+            env.start_compose()
+        except KeyboardInterrupt:
+            logger.info('Ctrl+C pressed. Except KeyboardInterrupt.')
+            env.stop_compose()
+            sys.exit(1)
+
         # sys.exit()
 
         # sn = SpacyNER()
@@ -55,4 +66,4 @@
         #         result1_str = "\n".join(str(x) for x in result1)
         #         logger.info(f'result of NER for model {avalable_model} is {result1_str}')
         #     else:
-        #         logger.info(f'module didn`t get NER data')
+        #         logger.info(f'module didn`t get NER data')

piper/__main__.py

@@ -0,0 +1,34 @@
+import sys
+
+import click
+
+from piper.envs import ComposeEnv
+
+
+@click.command()
+@click.argument('type_command')
+@click.option(
+    '--env-type', '-e',
+    help='your current interpretation',
+)
+def main(type_command: str, env_type: str):
+    if env_type == 'compose':
+        if type_command == 'start':
+            print("type_command == 'start'")
+            with ComposeEnv() as env:
+                env.copy_struct_project()
+                env.create_files_for_compose()
+                env.start_compose()
+        elif type_command == 'stop':
+            print("type_command == 'stop'")
+            with ComposeEnv() as env:
+                env.stop_compose()
+    else:
+        raise NotImplementedError(f'{env_type} not released in this version pipertool')
+
+
+if __name__ == '__main__':
+    args = sys.argv
+    if "--help" in args or len(args) == 1:
+        print("CVE")
+    main()

piper/base/docker_compose/compose_executors.py

@@ -0,0 +1,79 @@
+import inspect
+import subprocess
+from typing import Dict
+
+from piper.base.docker_compose.utils import ComposeServices
+from piper.configurations import get_configuration
+from piper.utils.logger_utils import logger
+
+
+def copy_piper(path: str):
+    cfg = get_configuration()
+    from distutils.dir_util import copy_tree
+    copy_tree(cfg.piper_path, f"{path}/piper")
+
+
+def copy_scripts(path: str, scripts: Dict[str, str]):
+    for script_name, script_path in scripts.items():
+        with open(f"{path}/{script_name}.py", "w") as output:
+            with open(script_path, "r") as current_file:
+                output.write(current_file.read())
+
+
+def write_requirements(path, requirements):
+    with open(f"{path}/requirements.txt", "w") as output:
+        output.write("\n".join(requirements))
+
+
+class ComposeExecutor:
+    requirements = ["gunicorn", "fastapi", "uvicorn", "aiohttp", "Jinja2", "pydantic", "pymilvus", "numpy", "loguru"]
+
+    def __init__(self):
+        logger.info('ComposeExecutor init with is_compose_env()')
+
+        cfg = get_configuration()
+        self.project_output_path = cfg.path
+
+    def scripts(self):
+        return {"service": inspect.getfile(self.__class__)}
+
+    def copy_struct_project(self):
+        copy_piper(self.project_output_path)
+        copy_scripts(self.project_output_path, self.scripts())
+
+    def create_files_for_compose(self, testing: bool = False):
+        logger.info('ComposeExecutor create_fast_api_files_venv()')
+
+        compose_service = ComposeServices(
+            name_path=self.project_output_path,
+        )
+
+        main_fastapi = compose_service.render_script_fastapi()
+        with open(f"{self.project_output_path}/main.py", "w") as output:
+            output.write(main_fastapi)
+
+        docker_compose = compose_service.render_compose_services()
+        with open(f"{self.project_output_path}/docker-compose.yaml", "w") as output:
+            output.write(docker_compose)
+
+        bash_start = compose_service.render_bash_start(testing=testing)
+        with open(f"{self.project_output_path}/bash-start.sh", "w") as output:
+            output.write(bash_start)
+
+        bash_stop = compose_service.render_bash_stop()
+        with open(f"{self.project_output_path}/bash-stop.sh", "w") as output:
+            output.write(bash_stop)
+
+        dockerfile = compose_service.render_dockerfile()
+        with open(f"{self.project_output_path}/Dockerfile", "w") as output:
+            output.write(dockerfile)
+
+        write_requirements(self.project_output_path, self.requirements)
+
+    def start_compose(self):
+        process_chmod_start = subprocess.run(f'chmod +x {self.project_output_path}bash-start.sh', shell=True)
+        process_run = subprocess.run(f'{self.project_output_path}bash-start.sh', shell=True)
+
+    def stop_compose(self):
+        process_chmod_stop = subprocess.run(f'chmod +x {self.project_output_path}bash-stop.sh', shell=True)
+        process_run = subprocess.run(f'{self.project_output_path}bash-stop.sh', shell=True)

piper/base/docker_compose/templates/bash-start-compose.j2

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+sudo docker-compose -f {{ name_path }}docker-compose.yaml up --build {{ compose_flag }}

piper/base/docker_compose/templates/bash-stop-compose.j2

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+sudo docker-compose -f {{ name_path }}docker-compose.yaml down

piper/base/docker_compose/templates/compose-services.j2

@@ -0,0 +1,60 @@
+version: '3'
+
+services:
+  etcd:
+    container_name: milvus-etcd
+    image: quay.io/coreos/etcd:v3.5.0
+    environment:
+      - ETCD_AUTO_COMPACTION_MODE=revision
+      - ETCD_AUTO_COMPACTION_RETENTION=1000
+      - ETCD_QUOTA_BACKEND_BYTES=4294967296
+    volumes:
+      - data_milvus:/etcd
+    command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
+
+  minio:
+    container_name: milvus-minio
+    image: minio/minio:RELEASE.2022-03-17T06-34-49Z
+    ports:
+      - "9001:9001"
+    environment:
+      MINIO_ACCESS_KEY: minioadmin
+      MINIO_SECRET_KEY: minioadmin
+    volumes:
+      - data_milvus:/minio_data
+    command: minio server /data --console-address ":9001"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+  standalone:
+    container_name: milvus-standalone
+    image: milvusdb/milvus:v2.1.1
+    command: ["milvus", "run", "standalone"]
+    environment:
+      ETCD_ENDPOINTS: etcd:2379
+      MINIO_ADDRESS: minio:9000
+    volumes:
+      - data_milvus:/var/lib/milvus
+    ports:
+      - "19530:19530"
+      - "9091:9091"
+    depends_on:
+      - "etcd"
+      - "minio"
+
+  microservice:
+    build:
+      context: .
+    restart: always
+    ports:
+      - 7585:80
+    depends_on:
+      - "etcd"
+      - "minio"
+      - "standalone"
+
+volumes:
+  data_milvus:

piper/base/docker_compose/templates/dockerfile.j2

@@ -0,0 +1,8 @@
+FROM tiangolo/uvicorn-gunicorn-fastapi:python3.8
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY main.py .