diff --git a/main.py b/main.py index 833b6f0..18c758e 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,9 @@ from piper.services import TestMessageAdder, StringValue -from envs import CurrentEnv +from piper.envs import CurrentEnv import asyncio +from loguru import logger +logger.add("file.log", level="INFO") if __name__ == '__main__': loop = asyncio.get_event_loop() diff --git a/piper/base/backend/templates/fast-api.j2 b/piper/base/backend/templates/fast-api.j2 index 2c1d540..732f64b 100644 --- a/piper/base/backend/templates/fast-api.j2 +++ b/piper/base/backend/templates/fast-api.j2 @@ -1,13 +1,17 @@ import time -from fastapi import FastAPI, Request +from fastapi import FastAPI, Request, status from piper.envs import CurrentEnv {% for script_name in scripts.keys() %} from {{ script_name }} import * {% endfor %} -app = FastAPI() +app = FastAPI(debug=True) + +@app.post('/health_check', status_code = status.HTTP_200_OK) +async def hl(): + return {"message": "health check"} with CurrentEnv(): service = {{ service_class }}( {% for k, v in service_kwargs.items() %} {{ k }}={{ v }}, {% endfor %} ) diff --git a/piper/base/executors.py b/piper/base/executors.py index a74fb9b..8316bfc 100644 --- a/piper/base/executors.py +++ b/piper/base/executors.py @@ -5,6 +5,7 @@ import inspect import aiohttp +from loguru import logger import docker from pydantic import BaseModel @@ -12,7 +13,10 @@ from piper.base.backend.utils import render_fast_api_backend from piper.envs import is_docker_env, is_current_env, get_env from piper.configurations import get_configuration +from piper.utils import docker_utils as du +import requests +import sys class BaseExecutor: pass @@ -55,14 +59,14 @@ async def run(self, *args, **kwargs): pass async def __call__(self, *args, **kwargs): - print(get_env()) - print(is_current_env()) + print('get_env()', get_env()) + print('is_current_env()', is_current_env()) if is_current_env(): return await self.run(*args, **kwargs) else: function = "run" request_dict = inputs_to_dict(*args, **kwargs) - print(request_dict) + print('request_dict', request_dict) async with aiohttp.ClientSession() as session: async with session.post(f'http://{self.host}:{self.port}/{function}', json=request_dict) as resp: return await resp.json() @@ -112,30 +116,67 @@ def run_container(image: str, ports: Dict[int, int]): return container +def wait_for_fast_api_app_start(host, external_port, wait_on_iter, n_iters): + ''' + wait for fast api app will be loaded + external_port - + wait_on_iter - seconds between health_check requests + n_iters - total health_check requests + ''' + logger.info('waiting for FastAPI app start') + i = 0 + while True: + try: + r = requests.post(f"http://{host}:{external_port}/health_check/") + print(r.status_code, r.reason) + if r.status_code == 200: + break + except Exception as e: + time.sleep(wait_on_iter) + + if i == n_iters: + logger.error('FastAPI app can`t start or n_iters too small') + sys.exit() + i += 1 + class FastAPIExecutor(HTTPExecutor): - requirements = ["gunicorn", "fastapi", "uvicorn", "aiohttp", "docker", "Jinja2", "pydantic"] + requirements = ["gunicorn", "fastapi", "uvicorn", "aiohttp", "docker", "Jinja2", "pydantic", "loguru"] base_handler = "run" def __init__(self, port: int = 8080, **service_kwargs): self.container = None + self.image_tag = 'piper:latest' + self.container_name = "piper_FastAPI" + if is_docker_env(): + docker_client = docker.DockerClient(base_url='unix://var/run/docker.sock') cfg = get_configuration() - path = cfg.path + project_output_path = cfg.path + + copy_piper(project_output_path) + copy_scripts(project_output_path, self.scripts()) - copy_piper(path) - copy_scripts(path, self.scripts()) + self.create_fast_api_files(project_output_path, **service_kwargs) - self.create_fast_api_files(path, **service_kwargs) + # create and run docker container + # if container exits it will be recreated! + du.create_image_and_container_by_dockerfile( + docker_client, + project_output_path, + self.image_tag, + self.container_name, + port + ) - image_tag = self.__class__.__name__.lower() - docker_image = PythonImage(tag=image_tag, - python_docker_version="3.9", - cmd=f"./run.sh") - build_image(path, docker_image) - self.container = run_container(image_tag, {8080: port}) + wait_for_fast_api_app_start('localhost', 8788, 0.5, 10) else: # TODO: Local ENVIRONMENT checks pass + + # a = super().__init__('localhost', port, 'hl') + # a.__call__() + # print('hl', a) + super().__init__('localhost', port, self.base_handler) def rm_container(self): diff --git a/piper/configurations.py b/piper/configurations.py index 0c0c632..201c53d 100644 --- a/piper/configurations.py +++ b/piper/configurations.py @@ -1,5 +1,6 @@ class Configuration: path = "/Users/olegsokolov/PycharmProjects/piper/applications" + path = "/home/pavel/repo/piper_new_out/" piper_path = "piper" default_env = "docker" env = None diff --git a/piper/utils/__init__.py b/piper/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/piper/utils/docker_utils.py b/piper/utils/docker_utils.py new file mode 100644 index 0000000..9b36983 --- /dev/null +++ b/piper/utils/docker_utils.py @@ -0,0 +1,175 @@ +import docker +import time +import sys +from loguru import logger + +def get_image(docker_client, image_name): + try: + cur_image = docker_client.images.get(image_name) + return cur_image + except docker.errors.ImageNotFound: + logger.info(f'image with tag {image_name} not found') + return False + + +def delete_image(docker_client, image_tag): + try: + docker_client.images.remove(image_tag, force=True) + return True + except Exception as e: + logger.error('error while remove image', e) + return False + + +def get_container(docker_client, container_name): + try: + cur_container = docker_client.containers.get(container_name) + return cur_container + except docker.errors.NotFound: + logger.info(f'container with name {container_name} not found') + return False + except Exception as e: + logger.error(f'non defined exeption {e}') + return False + +def get_container_with_status(docker_client, container_name): + try: + cur_container = docker_client.containers.get(container_name) + if cur_container: + status = cur_container.status + cont_id = cur_container.id + return cur_container, status, cont_id + except docker.errors.NotFound: + logger.info(f'container with name {container_name} not found') + return False + except Exception as e: + logger.error(f'non defined exeption {e}') + return False + +def stop_container(docker_client, container_name): + try: + cur_container = docker_client.containers.get(container_name) + cur_container.stop() + return True + except docker.errors.NotFound: + logger.error(f'container for stop with name {container_name} not found') + return False + except docker.errors.APIError: + logger.error(f'error while stop container {container_name}') + return False + + +def remove_container(docker_client, container_name): + try: + cur_container = docker_client.containers.get(container_name) + cur_container.remove(v=True, force=True) + return True + except docker.errors.NotFound: + logger.error(f'container for stop with name {container_name} not found') + return False + except docker.errors.APIError as de: + logger.error(f'error while remove container {container_name}') + logger.error(de) + return False + + +def stop_and_rm_container(docker_client, container_name): + # get container + cur_container = get_container(docker_client, container_name) + + # get container status + if not cur_container: + logger.info(f'container {container_name} not found') + return 'deleted' + else: + status = cur_container.status + cont_id = cur_container.id + logger.info('status', status, type(status)) + + if status == 'running': + logger.info(f'container {container_name} started already. Stop it!') + # stop + stop_result = stop_container(docker_client, cont_id) + logger.info('stoped', stop_result) + status = 'exited' + + if status == 'exited': + logger.info(f'container {container_name} exists already. Remove it!') + # rm + remove_result = remove_container(docker_client, cont_id) + logger.info('removed, remove_result is ', remove_result) + status = 'deleted' + return status + + +def image_find_and_rm(docker_client, image_tag): + cur_img = get_image(docker_client, image_tag) + logger.info(cur_img) + if cur_img: + logger.info(f'image {image_tag} exists') + logger.info(f'cur_img is {cur_img}, ID is {cur_img.id}') + del_result = delete_image(docker_client, image_tag) + logger.info(f'del_result of image {del_result}') + return del_result + else: + # не нужно ничего удалять, контейнера нет + return True + + +def create_image_and_container_by_dockerfile(docker_client, path, image_tag, container_name, port): + # should be deleted + status = stop_and_rm_container(docker_client, container_name) + + cur_cont = get_container(docker_client, container_name) + if cur_cont: + logger.error(f'container not deleted, {cur_cont}') + sys.exit() + + # remove image + if status == 'deleted': + # remove image + del_result = image_find_and_rm(docker_client, image_tag) + if del_result: + # create new image + image, logs = docker_client.images.build( + path=path, + tag=image_tag, + quiet=False, + rm=True, # creates image only without container + forcerm=True, + timeout=20 + ) + for log in logs: + logger.debug(log) + logger.info(f'image {image} created') + + # run container + try: + container = docker_client.containers.run(image_tag, name=container_name, detach=True, ports={8080:port}) + for log in container.logs(): + logger.debug(log) + logger.info(f'container {container} created') + + i = 0 + while True: + i += 1 + # logger.info(get_container_with_status(docker_client, container_name)) + container.reload() + logger.info(f'container.status {container.status}') + if container.status == 'running': + break + + if i == 20: + logger.error(f'container {container_name} can`t start, status is {container.status}') + sys.exit() + time.sleep(0.5) + + + except docker.errors.APIError as api_e: + logger.error(f'eroror while run container {container_name}') + logger.error(str(api_e)) + sys.exit() + else: + logger.error(f'error while del image {image_tag}') + sys.exit() + diff --git a/requirements.txt b/requirements.txt index c9e848d..fa6642d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ docker aiohttp Jinja2 -pydantic \ No newline at end of file +pydantic +loguru \ No newline at end of file