Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion main.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from piper.services import TestMessageAdder, StringValue, TesseractRecognizer, SpacyNER
from piper.envs import CurrentEnv, DockerEnv
from piper.envs import CurrentEnv, DockerEnv, VirtualEnv
from piper.configurations import get_configuration
import time
import asyncio
Expand Down Expand Up @@ -34,6 +34,11 @@
result = loop.run_until_complete(recognizer())
logger.info(f'result of recognition is {result}')

with VirtualEnv() as env:
env.copy_struct_project()
env.create_files_for_venv()
env.create_files_for_tests()

# sys.exit()

# sn = SpacyNER()
Expand Down
3 changes: 2 additions & 1 deletion piper/base/backend/utils.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import jinja2
import os

import jinja2


def render_fast_api_backend(**kwargs):
"""
Expand Down
1 change: 1 addition & 0 deletions piper/base/docker/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import os

import jinja2


Expand Down
Empty file.
6 changes: 6 additions & 0 deletions piper/base/virtualenv/templates/bash-create-tests.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#!/usr/bin/env bash

source {{ name_venv }}/bin/activate
pip3 install pytest

python3 -m pytest
12 changes: 12 additions & 0 deletions piper/base/virtualenv/templates/bash-create-venv.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/usr/bin/env bash

NAMEVENV={{ name_path }}{{ name_venv }}

pip3 install virtualenv

python3 -m virtualenv $NAMEVENV
source $NAMEVENV/bin/activate

pip3 install -r {{ name_path }}requirements.txt

python3 {{ name_path }}main.py {{ number }}
8 changes: 8 additions & 0 deletions piper/base/virtualenv/templates/python-script-tests.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
import pytest

from piper_new_out.main import sum_factorial


def test_base_func_from_main():
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Это просто не в темплейт надо, а отнаследоваться от VenvExecutor и добавить в call факториала подсчет, тогда отличный тест как раз выйдет, что все ок на просто примере оборачивается)

res_from_main = sum_factorial({{ number }})
assert res_from_main == 4037913
20 changes: 20 additions & 0 deletions piper/base/virtualenv/templates/python-script-venv.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
import argparse


def sum_factorial(number: int) -> int:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

А это как некий мок?
Может просто в лог выводить "Found no implementation! Fix it, please!"

partial_factorial = 1
partial_sum = 0
for i in range(1, number + 1):
partial_factorial *= i
partial_sum += partial_factorial
return partial_sum


if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("number", type=int)
args = parser.parse_args()
if args.number:
print(f'Sum of factorial to {args.number} = {sum_factorial(args.number)}')
else:
print('No positional arguments in cli')
79 changes: 79 additions & 0 deletions piper/base/virtualenv/utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
import os

import jinja2

from piper.utils.logger_utils import logger


class VenvPython:

def __init__(
self,
name_path: str,
name_venv: str,
number: int,
):
self.name_path = name_path
self.name_venv = name_venv
self.number = number

def render_venv_bash(self):
"""
Render bash script for create and activate venv
"""
logger.info('Render bash script for create and activate venv')

template_dir = os.path.join(os.path.dirname(__file__), 'templates')
jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir),
trim_blocks=True,
lstrip_blocks=True)
template = jinja_env.get_template('bash-create-venv.j2')
return template.render(
name_path=self.name_path,
name_venv=self.name_venv,
number=self.number,
)

@staticmethod
def render_venv_python():
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Вот это хорошая мысль, и нейминг, возможно мы ее как-то расширим для разных уровней вложенности энвов друг в друга

"""
Render main file for virtual env logic
"""
logger.info('Render main file for virtual env logic')

template_dir = os.path.join(os.path.dirname(__file__), 'templates')
jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir),
trim_blocks=True,
lstrip_blocks=True)
template = jinja_env.get_template('python-script-venv.j2')
return template.render()

def render_tests_bash(self):
"""
Render bash script for create and activate venv
"""
logger.info('Render bash script for create and activate venv')

template_dir = os.path.join(os.path.dirname(__file__), 'templates')
jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir),
trim_blocks=True,
lstrip_blocks=True)
template = jinja_env.get_template('bash-create-tests.j2')
return template.render(
name_venv=self.name_venv,
)

def render_tests_python(self):
"""
Render bash script for create and activate venv
"""
logger.info('Render bash script for create and activate venv')

template_dir = os.path.join(os.path.dirname(__file__), 'templates')
jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir),
trim_blocks=True,
lstrip_blocks=True)
template = jinja_env.get_template('python-script-tests.j2')
return template.render(
number=self.number,
)
94 changes: 94 additions & 0 deletions piper/base/virtualenv/venv_executors.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
import inspect
import os
import subprocess
from typing import Dict

from piper.base.virtualenv.utils import VenvPython
from piper.configurations import get_configuration
from piper.utils.logger_utils import logger


def copy_piper(path: str):
cfg = get_configuration()
from distutils.dir_util import copy_tree
copy_tree(cfg.piper_path, f"{path}/piper")


def copy_scripts(path: str, scripts: Dict[str, str]):
for script_name, script_path in scripts.items():
with open(f"{path}/{script_name}.py", "w") as output:
with open(script_path, "r") as current_file:
output.write(current_file.read())


def write_requirements(path, requirements):
with open(f"{path}/requirements.txt", "w") as output:
output.write("\n".join(requirements))


class VirtualEnvExecutor:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Пока так отлично, но на будущее куда-то себе пометить мысль, что это некий микс выходит энва и экзекьютора, то есть может сделать именно енвом эту логику, а экзекьюторы более абстрактными без исполнения и только на тестах выяснять что все ок исполняется

requirements = ["gunicorn", "fastapi", "uvicorn", "aiohttp", "docker", "Jinja2", "pydantic", "loguru"]

def __init__(self):
logger.info('VirtualEnvExecutor init with is_virtual_env()')

cfg = get_configuration()
self.project_output_path = cfg.path
self.name_venv = cfg.name_venv
self.number = cfg.number

def scripts(self):
return {"service": inspect.getfile(self.__class__)}

def copy_struct_project(self):
copy_piper(self.project_output_path)
copy_scripts(self.project_output_path, self.scripts())

def create_files_for_venv(self):
logger.info('VirtualEnvExecutor create_fast_api_files_venv()')

venv_python_image = VenvPython(
name_path=self.project_output_path,
name_venv=self.name_venv,
number=self.number,
)

venv_main = venv_python_image.render_venv_python()
with open(f"{self.project_output_path}/main.py", "w") as output:
output.write(venv_main)

venv_bash = venv_python_image.render_venv_bash()
with open(f"{self.project_output_path}/create_venv.sh", "w") as output:
output.write(venv_bash)

write_requirements(self.project_output_path, self.requirements)

process_chmod = subprocess.run(f'chmod +x {self.project_output_path}create_venv.sh', shell=True)
process_run = subprocess.run(f'{self.project_output_path}create_venv.sh', shell=True)

def create_files_for_tests(self):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Этот тест он для чего? он точно внутри должен быть, а не где-то в папке tests в корне проекта?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Этот тест генерится уже для конечного пользователя. Создается он во вновь созданной директории проекта, которая может быть (и чаще всего) отлично от исходной диреткории нашего шаблона. Тест првоеряет корректность созданного и активированного виртуального окружения и запуск функционала из файла main.py

logger.info('VirtualEnvExecutor create_files_for_tests()')

with open(f"{self.project_output_path}/__init__.py", "w") as output:
pass

tests_directory = f"{self.project_output_path}/tests"
if not os.path.exists(tests_directory):
os.makedirs(tests_directory)

with open(f"{self.project_output_path}/tests/__init__.py", "w") as output:
pass

venv_python_image = VenvPython(
name_path=self.project_output_path,
name_venv=self.name_venv,
number=self.number,
)

test_main = venv_python_image.render_tests_python()
with open(f"{self.project_output_path}/tests/test_main.py", "w") as output:
output.write(test_main)

test_bash = venv_python_image.render_tests_bash()
with open(f"{self.project_output_path}/test_venv.sh", "w") as output:
output.write(test_bash)
4 changes: 4 additions & 0 deletions piper/configurations.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ class Configuration:
piper_path = "piper"
default_env = "docker"
docker_app_port = 8788

name_venv = "venv_test"
number = 10

env = None

# start time and counter
Expand Down
24 changes: 24 additions & 0 deletions piper/envs/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
from piper.base.virtualenv.venv_executors import VirtualEnvExecutor
from piper.configurations import get_configuration
from piper.utils.logger_utils import logger

cfg = get_configuration()

Expand All @@ -7,6 +9,8 @@ def init_default_env():
# INITIALIZE ENVIRONMENT FROM CONFIGURATION
if cfg.default_env == "docker":
set_env(DockerEnv())
elif cfg.default_env == "virtualenv":
set_env(VirtualEnv())
else:
set_env(CurrentEnv())

Expand Down Expand Up @@ -52,9 +56,29 @@ def __exit__(self, *args, **kws):
set_env(self._old_environment)


class VirtualEnv:

def __init__(self):
self.__resource = VirtualEnvExecutor()

def __enter__(self):
logger.info("Entering VirtualEnv")
self._old_environment = get_env()
set_env(self)
return self.__resource

def __exit__(self, *args, **kws):
logger.info("Exiting VirtualEnv")
set_env(self._old_environment)


def is_current_env():
return get_env().__class__.__name__ == "CurrentEnv"


def is_docker_env():
return get_env().__class__.__name__ == "DockerEnv"


def is_virtual_env():
return get_env().__class__.__name__ == "VirtualEnv"
4 changes: 2 additions & 2 deletions piper/services/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from piper.base.executors import FastAPIExecutor, FastAPITesseractExecutor
from piper.base.executors import FastAPIExecutor, FastAPITesseractExecutor, VirtualEnvExecutor
from fastapi.responses import JSONResponse

from pydantic import BaseModel
Expand Down Expand Up @@ -30,7 +30,7 @@ def __init__(self, appender="TEST", **kwargs):

async def run(self, message: StringValue) -> StringValue:
return StringValue(value=(message.value + self.appender))


class TesseractRecognizer(FastAPITesseractExecutor):
'''
Expand Down
Loading