diff --git a/docs/workloads/batch/configuration.md b/docs/workloads/batch/configuration.md index c01a9fed41..b91f2838a1 100644 --- a/docs/workloads/batch/configuration.md +++ b/docs/workloads/batch/configuration.md @@ -23,6 +23,10 @@ predictor: env: # dictionary of environment variables log_level: # log level that can be "debug", "info", "warning" or "error" (default: "info") shm_size: # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null) + dependencies: # (optional) + pip: # relative path to requirements.txt (default: requirements.txt) + conda: # relative path to conda-packages.txt (default: conda-packages.txt) + shell: # relative path to a shell script for system package installation (default: dependencies.sh) ``` ### TensorFlow Predictor @@ -50,6 +54,10 @@ predictor: env: # dictionary of environment variables log_level: # log level that can be "debug", "info", "warning" or "error" (default: "info") shm_size: # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null) + dependencies: # (optional) + pip: # relative path to requirements.txt (default: requirements.txt) + conda: # relative path to conda-packages.txt (default: conda-packages.txt) + shell: # relative path to a shell script for system package installation (default: dependencies.sh) ``` ### ONNX Predictor @@ -71,6 +79,10 @@ predictor: env: # dictionary of environment variables log_level: # log level that can be "debug", "info", "warning" or "error" (default: "info") shm_size: # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null) + dependencies: # (optional) + pip: # relative path to requirements.txt (default: requirements.txt) + conda: # relative path to conda-packages.txt (default: conda-packages.txt) + shell: # relative path to a shell script for system package installation (default: dependencies.sh) ``` ## Compute diff --git a/docs/workloads/dependencies/python-packages.md b/docs/workloads/dependencies/python-packages.md index 4d6f44533e..bed17208e8 100644 --- a/docs/workloads/dependencies/python-packages.md +++ b/docs/workloads/dependencies/python-packages.md @@ -2,7 +2,8 @@ ## PyPI packages -You can install your required PyPI packages and import them in your Python files using pip. Cortex looks for a `requirements.txt` file in the top level Cortex project directory (i.e. the directory which contains `cortex.yaml`): +You can install your required PyPI packages and import them in your Python files using pip. Cortex looks for +a `requirements.txt` file in the top level Cortex project directory (i.e. the directory which contains `cortex.yaml`): ```text ./my-classifier/ @@ -14,11 +15,13 @@ You can install your required PyPI packages and import them in your Python files If you want to use `conda` to install your python packages, see the [Conda section](#conda-packages) below. -Note that some packages are pre-installed by default (see "pre-installed packages" for your Predictor type in the Realtime API Predictor documentation and Batch API Predictor documentation). +Note that some packages are pre-installed by default (see "pre-installed packages" for your Predictor type in the +Realtime API Predictor documentation and Batch API Predictor documentation). ## Private PyPI packages -To install packages from a private PyPI index, create a `pip.conf` inside the same directory as `requirements.txt`, and add the following contents: +To install packages from a private PyPI index, create a `pip.conf` inside the same directory as `requirements.txt`, and +add the following contents: ```text [global] @@ -35,7 +38,8 @@ You may now add packages to `requirements.txt` which are found in the private in ## GitHub packages -You can also install public/private packages from git registries (such as GitHub) by adding them to `requirements.txt`. Here's an example for GitHub: +You can also install public/private packages from git registries (such as GitHub) by adding them to `requirements.txt`. +Here's an example for GitHub: ```text # requirements.txt @@ -47,11 +51,14 @@ git+https://github.com//.git@#egg=@github.com//.git@#egg= ``` -On GitHub, you can generate a personal access token by following [these steps](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line). +On GitHub, you can generate a personal access token by +following [these steps](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line) +. ## Installing with Setup -Python packages can also be installed by providing a `setup.py` that describes your project's modules. Here's an example directory structure: +Python packages can also be installed by providing a `setup.py` that describes your project's modules. Here's an example +directory structure: ```text ./my-classifier/ @@ -65,6 +72,7 @@ Python packages can also be installed by providing a `setup.py` that describes y ``` In this case, `requirements.txt` will have this form: + ```text # requirements.txt @@ -73,7 +81,9 @@ In this case, `requirements.txt` will have this form: ## Conda packages -Cortex supports installing Conda packages. We recommend only using Conda when your required packages are not available in PyPI. Cortex looks for a `conda-packages.txt` file in the top level Cortex project directory (i.e. the directory which contains `cortex.yaml`): +Cortex supports installing Conda packages. We recommend only using Conda when your required packages are not available +in PyPI. Cortex looks for a `conda-packages.txt` file in the top level Cortex project directory (i.e. the directory +which contains `cortex.yaml`): ```text ./my-classifier/ @@ -83,16 +93,48 @@ Cortex supports installing Conda packages. We recommend only using Conda when yo └── conda-packages.txt ``` -The `conda-packages.txt` file follows the format of `conda list --export`. Each line of `conda-packages.txt` should follow this pattern: `[channel::]package[=version[=buildid]]`. +The `conda-packages.txt` file follows the format of `conda list --export`. Each line of `conda-packages.txt` should +follow this pattern: `[channel::]package[=version[=buildid]]`. Here's an example of `conda-packages.txt`: + ```text conda-forge::rdkit conda-forge::pygpu ``` -In situations where both `requirements.txt` and `conda-packages.txt` are provided, Cortex installs Conda packages in `conda-packages.txt` followed by PyPI packages in `requirements.txt`. Conda and Pip package managers install packages and dependencies independently. You may run into situations where Conda and pip package managers install different versions of the same package because they install and resolve dependencies independently from one another. To resolve package version conflicts, it may be in your best interest to specify their exact versions in `conda-packages.txt`. +In situations where both `requirements.txt` and `conda-packages.txt` are provided, Cortex installs Conda packages +in `conda-packages.txt` followed by PyPI packages in `requirements.txt`. Conda and Pip package managers install packages +and dependencies independently. You may run into situations where Conda and pip package managers install different +versions of the same package because they install and resolve dependencies independently from one another. To resolve +package version conflicts, it may be in your best interest to specify their exact versions in `conda-packages.txt`. -The current version of Python is `3.6.9`. Updating Python to a different version is possible with Conda, but there are no guarantees that Cortex's web server will continue functioning correctly. If there's a change in Python's version, the necessary core packages for the web server will be reinstalled. If you are using a custom base image, any other Python packages that are built in to the image won't be accessible at runtime. +The current version of Python is `3.6.9`. Updating Python to a different version is possible with Conda, but there are +no guarantees that Cortex's web server will continue functioning correctly. If there's a change in Python's version, the +necessary core packages for the web server will be reinstalled. If you are using a custom base image, any other Python +packages that are built in to the image won't be accessible at runtime. Check the [best practices](https://www.anaconda.com/using-pip-in-a-conda-environment/) on using `pip` inside `conda`. + +## Customizing Dependency Paths + +Cortex allows you to specify different dependency paths other than the default ones. This can be useful when deploying +different versions of the same API (e.g. CPU vs GPU dependencies). + +To customize the path for your dependencies, you can specify `predictor.dependencies` in your API's configuration file. You can set +one or more fields to specify the path for each dependency type. Each path should be a relative path with respect to the current file. + +For example: + +```yaml +# cortex.yaml + +- name: my-classifier + kind: RealtimeAPI + predictor: + (...) + dependencies: + pip: requirement-gpu.txt + conda: conda-packages-gpu.txt + shell: dependencies-gpu.sh +``` diff --git a/docs/workloads/dependencies/system-packages.md b/docs/workloads/dependencies/system-packages.md index 6c7ca481bc..a9ed1bdb8c 100644 --- a/docs/workloads/dependencies/system-packages.md +++ b/docs/workloads/dependencies/system-packages.md @@ -40,3 +40,22 @@ conda install -n env python=3.8.5 # re-install cortex core dependencies /usr/local/cortex/install-core-dependencies.sh ``` + +## Customizing Dependency Paths + +Cortex allows you to specify a path for this script other than `dependencies.sh`. This can be useful when deploying +different versions of the same API (e.g. CPU vs GPU dependencies). The path should be a relative path with respect +to the API configuration file, and is specified via `predictor.dependencies.shell`. + +For example: + +```yaml +# cortex.yaml + +- name: my-classifier + kind: RealtimeAPI + predictor: + (...) + dependencies: + shell: dependencies-gpu.sh +``` diff --git a/docs/workloads/realtime/configuration.md b/docs/workloads/realtime/configuration.md index 4c41e98ac1..c81ee14be8 100644 --- a/docs/workloads/realtime/configuration.md +++ b/docs/workloads/realtime/configuration.md @@ -19,6 +19,10 @@ predictor: type: python path: # path to a python file with a PythonPredictor class definition, relative to the Cortex root (required) + dependencies: # (optional) + pip: # relative path to requirements.txt (default: requirements.txt) + conda: # relative path to conda-packages.txt (default: conda-packages.txt) + shell: # relative path to a shell script for system package installation (default: dependencies.sh) multi_model_reloading: # use this to serve one or more models with live reloading (optional) path: # S3/GCS path to an exported model directory (e.g. s3://my-bucket/exported_model/) (either this, 'dir', or 'paths' must be provided if 'multi_model_reloading' is specified) paths: # list of S3/GCS paths to exported model directories (either this, 'dir', or 'path' must be provided if 'multi_model_reloading' is specified) @@ -48,6 +52,10 @@ predictor: predictor: type: tensorflow path: # path to a python file with a TensorFlowPredictor class definition, relative to the Cortex root (required) + dependencies: # (optional) + pip: # relative path to requirements.txt (default: requirements.txt) + conda: # relative path to conda-packages.txt (default: conda-packages.txt) + shell: # relative path to a shell script for system package installation (default: dependencies.sh) models: # (required) path: # S3/GCS path to an exported SavedModel directory (e.g. s3://my-bucket/exported_model/) (either this, 'dir', or 'paths' must be provided) paths: # list of S3/GCS paths to exported SavedModel directories (either this, 'dir', or 'path' must be provided) @@ -80,6 +88,10 @@ predictor: predictor: type: onnx path: # path to a python file with an ONNXPredictor class definition, relative to the Cortex root (required) + dependencies: # (optional) + pip: # relative path to requirements.txt (default: requirements.txt) + conda: # relative path to conda-packages.txt (default: conda-packages.txt) + shell: # relative path to a shell script for system package installation (default: dependencies.sh) models: # (required) path: # S3/GCS path to an exported model directory (e.g. s3://my-bucket/exported_model/) (either this, 'dir', or 'paths' must be provided) paths: # list of S3/GCS paths to exported model directories (either this, 'dir', or 'path' must be provided) diff --git a/pkg/cortex/serve/init/bootloader.sh b/pkg/cortex/serve/init/bootloader.sh index e84e5160ac..74cc214c1c 100755 --- a/pkg/cortex/serve/init/bootloader.sh +++ b/pkg/cortex/serve/init/bootloader.sh @@ -55,20 +55,20 @@ function install_deps() { eval $source_env_file_cmd # execute script if present in project's directory - if [ -f "/mnt/project/dependencies.sh" ]; then - bash -e /mnt/project/dependencies.sh + if [ -f "/mnt/project/${CORTEX_DEPENDENCIES_SHELL}" ]; then + bash -e "/mnt/project/${CORTEX_DEPENDENCIES_SHELL}" fi # install from conda-packages.txt - if [ -f "/mnt/project/conda-packages.txt" ]; then + if [ -f "/mnt/project/${CORTEX_DEPENDENCIES_CONDA}" ]; then # look for packages in defaults and then conda-forge to improve chances of finding the package (specifically for python reinstalls) conda config --append channels conda-forge - conda install -y --file /mnt/project/conda-packages.txt + conda install -y --file "/mnt/project/${CORTEX_DEPENDENCIES_CONDA}" fi # install pip packages - if [ -f "/mnt/project/requirements.txt" ]; then - pip --no-cache-dir install -r /mnt/project/requirements.txt + if [ -f "/mnt/project/${CORTEX_DEPENDENCIES_PIP}" ]; then + pip --no-cache-dir install -r "/mnt/project/${CORTEX_DEPENDENCIES_PIP}" fi # install core cortex dependencies if required diff --git a/pkg/operator/operator/k8s.go b/pkg/operator/operator/k8s.go index 86aa48c965..bfa998afe2 100644 --- a/pkg/operator/operator/k8s.go +++ b/pkg/operator/operator/k8s.go @@ -593,6 +593,18 @@ func getEnvVars(api *spec.API, container string) []kcore.EnvVar { Name: "CORTEX_PROJECT_DIR", Value: path.Join(_emptyDirMountPath, "project"), }, + kcore.EnvVar{ + Name: "CORTEX_DEPENDENCIES_PIP", + Value: api.Predictor.Dependencies.Pip, + }, + kcore.EnvVar{ + Name: "CORTEX_DEPENDENCIES_CONDA", + Value: api.Predictor.Dependencies.Conda, + }, + kcore.EnvVar{ + Name: "CORTEX_DEPENDENCIES_SHELL", + Value: api.Predictor.Dependencies.Shell, + }, ) if api.Kind == userconfig.RealtimeAPIKind { diff --git a/pkg/types/spec/validations.go b/pkg/types/spec/validations.go index bbef70d836..b7e951e2db 100644 --- a/pkg/types/spec/validations.go +++ b/pkg/types/spec/validations.go @@ -243,6 +243,7 @@ func predictorValidation() *cr.StructFieldValidation { multiModelValidation("Models"), multiModelValidation("MultiModelReloading"), serverSideBatchingValidation(), + dependencyPathValidation(), }, }, } @@ -639,6 +640,35 @@ func serverSideBatchingValidation() *cr.StructFieldValidation { } } +func dependencyPathValidation() *cr.StructFieldValidation { + return &cr.StructFieldValidation{ + StructField: "Dependencies", + StructValidation: &cr.StructValidation{ + Required: false, + StructFieldValidations: []*cr.StructFieldValidation{ + { + StructField: "Pip", + StringValidation: &cr.StringValidation{ + Default: "requirements.txt", + }, + }, + { + StructField: "Conda", + StringValidation: &cr.StringValidation{ + Default: "conda-packages.txt", + }, + }, + { + StructField: "Shell", + StringValidation: &cr.StringValidation{ + Default: "dependencies.sh", + }, + }, + }, + }, + } +} + var resourceStructValidation = cr.StructValidation{ AllowExtraFields: true, StructFieldValidations: resourceStructValidations, diff --git a/pkg/types/userconfig/api.go b/pkg/types/userconfig/api.go index 1be57ae6c4..e8e62a619d 100644 --- a/pkg/types/userconfig/api.go +++ b/pkg/types/userconfig/api.go @@ -61,6 +61,7 @@ type Predictor struct { TensorFlowServingImage string `json:"tensorflow_serving_image" yaml:"tensorflow_serving_image"` Config map[string]interface{} `json:"config" yaml:"config"` Env map[string]string `json:"env" yaml:"env"` + Dependencies *Dependencies `json:"dependencies" yaml:"dependencies"` } type TaskDefinition struct { @@ -97,6 +98,12 @@ type ServerSideBatching struct { BatchInterval time.Duration `json:"batch_interval" yaml:"batch_interval"` } +type Dependencies struct { + Pip string `json:"pip" yaml:"pip"` + Conda string `json:"conda" yaml:"conda"` + Shell string `json:"shell" yaml:"shell"` +} + type Networking struct { Endpoint *string `json:"endpoint" yaml:"endpoint"` }