From 6fb6c44b2380405dfc9953514313204d1788f8d3 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Wed, 12 May 2021 14:47:59 +0300 Subject: [PATCH 01/82] Add dd --- design-spec/async.yaml | 38 +++++++++++++++ design-spec/batch.yaml | 23 ++++++++++ design-spec/realtime.yaml | 97 +++++++++++++++++++++++++++++++++++++++ design-spec/task.yaml | 23 ++++++++++ 4 files changed, 181 insertions(+) create mode 100644 design-spec/async.yaml create mode 100644 design-spec/batch.yaml create mode 100644 design-spec/realtime.yaml create mode 100644 design-spec/task.yaml diff --git a/design-spec/async.yaml b/design-spec/async.yaml new file mode 100644 index 0000000000..0c77623221 --- /dev/null +++ b/design-spec/async.yaml @@ -0,0 +1,38 @@ +# new + +- name: + kind: AsyncAPI + shm_size: + node_groups: + log_level: + config: + containers: + - name: api + image: + env: + port: + command: + args: + healthcheck: + compute: + cpu: + gpu: + inf: + mem: + autoscaling: + min_replicas: + max_replicas: + init_replicas: + max_replica_concurrency: + window: + downscale_stabilization_period: + upscale_stabilization_period: + max_downscale_factor: + max_upscale_factor: + downscale_tolerance: + upscale_tolerance: + update_strategy: + max_surge: + max_unavailable: + networking: + endpoint: diff --git a/design-spec/batch.yaml b/design-spec/batch.yaml new file mode 100644 index 0000000000..7721fefc02 --- /dev/null +++ b/design-spec/batch.yaml @@ -0,0 +1,23 @@ +# new + +- name: + kind: BatchAPI + shm_size: + node_groups: + log_level: + config: + containers: + - name: batch + image: + env: + port: + command: + args: + healthcheck: + compute: + cpu: + gpu: + inf: + mem: + networking: + endpoint: diff --git a/design-spec/realtime.yaml b/design-spec/realtime.yaml new file mode 100644 index 0000000000..dc13faef19 --- /dev/null +++ b/design-spec/realtime.yaml @@ -0,0 +1,97 @@ +# old + +- name: + kind: RealtimeAPI + handler: + type: python + path: + protobuf_path: + dependencies: + pip: + conda: + shell: + multi_model_reloading: + path: + paths: + - name: + path: + dir: + cache_size: + disk_cache_size: + server_side_batching: + max_batch_size: + batch_interval: + processes_per_replica: + threads_per_process: + config: + python_path: + image: + env: + log_level: + shm_size: + compute: + cpu: + gpu: + inf: + mem: + node_groups: + autoscaling: + min_replicas: + max_replicas: + init_replicas: + max_replica_concurrency: + target_replica_concurrency: + window: + downscale_stabilization_period: + upscale_stabilization_period: + max_downscale_factor: + max_upscale_factor: + downscale_tolerance: + upscale_tolerance: + update_strategy: + max_surge: + max_unavailable: + networking: + endpoint: + +--- + +# new + +- name: + kind: RealtimeAPI + shm_size: + node_groups: + log_level: + config: + containers: + - name: api + image: + env: + port: + command: + args: + healthcheck: + compute: + cpu: + gpu: + inf: + mem: + autoscaling: + min_replicas: + max_replicas: + init_replicas: + max_replica_queue_length: + max_replica_concurrency: + window: + downscale_stabilization_period: + upscale_stabilization_period: + max_downscale_factor: + max_upscale_factor: + downscale_tolerance: + upscale_tolerance: + update_strategy: + max_surge: + max_unavailable: + networking: + endpoint: diff --git a/design-spec/task.yaml b/design-spec/task.yaml new file mode 100644 index 0000000000..15c6b63941 --- /dev/null +++ b/design-spec/task.yaml @@ -0,0 +1,23 @@ +# new + +- name: + kind: TaskAPI + shm_size: + node_groups: + log_level: + config: + containers: + - name: api + image: + env: + port: + command: + args: + healthcheck: + compute: + cpu: + gpu: + inf: + mem: + networking: + endpoint: From 3eb736b17c3469bf88d966221abf9ae8e1d7f0a2 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Thu, 13 May 2021 15:22:56 +0300 Subject: [PATCH 02/82] CaaS WIP --- design-spec/realtime.yaml | 32 +- pkg/consts/consts.go | 22 - pkg/operator/endpoints/deploy.go | 8 +- pkg/operator/resources/resources.go | 25 +- pkg/operator/resources/validations.go | 12 +- pkg/types/spec/api.go | 5 - pkg/types/spec/errors.go | 315 +------ pkg/types/spec/utils.go | 383 +------- pkg/types/spec/validations.go | 880 ++----------------- pkg/types/userconfig/api.go | 149 +--- pkg/types/userconfig/config_key.go | 4 + pkg/types/userconfig/handler_type.go | 88 -- pkg/types/userconfig/model_structure_type.go | 25 - 13 files changed, 165 insertions(+), 1783 deletions(-) delete mode 100644 pkg/types/userconfig/handler_type.go delete mode 100644 pkg/types/userconfig/model_structure_type.go diff --git a/design-spec/realtime.yaml b/design-spec/realtime.yaml index dc13faef19..95f4740b76 100644 --- a/design-spec/realtime.yaml +++ b/design-spec/realtime.yaml @@ -60,29 +60,27 @@ - name: kind: RealtimeAPI - shm_size: - node_groups: - log_level: - config: - containers: - - name: api - image: - env: - port: - command: - args: - healthcheck: - compute: - cpu: - gpu: - inf: - mem: + pod: + shm_size: + node_groups: + containers: + - name: api + image: + env: + command: + args: + compute: + cpu: + gpu: + inf: + mem: autoscaling: min_replicas: max_replicas: init_replicas: max_replica_queue_length: max_replica_concurrency: + target_replica_concurrency: window: downscale_stabilization_period: upscale_stabilization_period: diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 8636517447..cee47c60b2 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -17,35 +17,13 @@ limitations under the License. package consts import ( - "fmt" "os" - - "github.com/cortexlabs/cortex/pkg/lib/sets/strset" ) var ( CortexVersion = "master" // CORTEX_VERSION CortexVersionMinor = "master" // CORTEX_VERSION_MINOR - SingleModelName = "_cortex_default" - - DefaultImagePythoHandlerCPU = fmt.Sprintf("%s/python-handler-cpu:%s", DefaultRegistry(), CortexVersion) - DefaultImagePythonHandlerGPU = fmt.Sprintf("%s/python-handler-gpu:%s-cuda10.2-cudnn8", DefaultRegistry(), CortexVersion) - DefaultImagePythonHandlerInf = fmt.Sprintf("%s/python-handler-inf:%s", DefaultRegistry(), CortexVersion) - DefaultImageTensorFlowServingCPU = fmt.Sprintf("%s/tensorflow-serving-cpu:%s", DefaultRegistry(), CortexVersion) - DefaultImageTensorFlowServingGPU = fmt.Sprintf("%s/tensorflow-serving-gpu:%s", DefaultRegistry(), CortexVersion) - DefaultImageTensorFlowServingInf = fmt.Sprintf("%s/tensorflow-serving-inf:%s", DefaultRegistry(), CortexVersion) - DefaultImageTensorFlowHandler = fmt.Sprintf("%s/tensorflow-handler:%s", DefaultRegistry(), CortexVersion) - DefaultImagePathsSet = strset.New( - DefaultImagePythoHandlerCPU, - DefaultImagePythonHandlerGPU, - DefaultImagePythonHandlerInf, - DefaultImageTensorFlowServingCPU, - DefaultImageTensorFlowServingGPU, - DefaultImageTensorFlowServingInf, - DefaultImageTensorFlowHandler, - ) - DefaultMaxReplicaConcurrency = int64(1024) NeuronCoresPerInf = int64(4) AuthHeader = "X-Cortex-Authorization" diff --git a/pkg/operator/endpoints/deploy.go b/pkg/operator/endpoints/deploy.go index 0b24ee08d0..57784ad775 100644 --- a/pkg/operator/endpoints/deploy.go +++ b/pkg/operator/endpoints/deploy.go @@ -42,13 +42,7 @@ func Deploy(w http.ResponseWriter, r *http.Request) { return } - projectBytes, err := files.ReadReqFile(r, "project.zip") - if err != nil { - respondError(w, r, err) - return - } - - response, err := resources.Deploy(projectBytes, configFileName, configBytes, force) + response, err := resources.Deploy(configFileName, configBytes, force) if err != nil { respondError(w, r, err) return diff --git a/pkg/operator/resources/resources.go b/pkg/operator/resources/resources.go index 065e434684..78fea8b5e9 100644 --- a/pkg/operator/resources/resources.go +++ b/pkg/operator/resources/resources.go @@ -90,39 +90,20 @@ func GetDeployedResourceByNameOrNil(resourceName string) (*operator.DeployedReso }, nil } -func Deploy(projectBytes []byte, configFileName string, configBytes []byte, force bool) ([]schema.DeployResult, error) { - projectID := hash.Bytes(projectBytes) - projectFileMap, err := archive.UnzipMemToMem(projectBytes) - if err != nil { - return nil, err - } - - projectFiles := ProjectFiles{ - ProjectByteMap: projectFileMap, - } +func Deploy(configFileName string, configBytes []byte, force bool) ([]schema.DeployResult, error) { + projectID := hash.Bytes(configBytes) apiConfigs, err := spec.ExtractAPIConfigs(configBytes, configFileName) if err != nil { return nil, err } - err = ValidateClusterAPIs(apiConfigs, projectFiles) + err = ValidateClusterAPIs(apiConfigs) if err != nil { err = errors.Append(err, fmt.Sprintf("\n\napi configuration schema can be found at https://docs.cortex.dev/v/%s/", consts.CortexVersionMinor)) return nil, err } - projectKey := spec.ProjectKey(projectID, config.ClusterConfig.ClusterUID) - isProjectUploaded, err := config.AWS.IsS3File(config.ClusterConfig.Bucket, projectKey) - if err != nil { - return nil, err - } - if !isProjectUploaded { - if err = config.AWS.UploadBytesToS3(projectBytes, config.ClusterConfig.Bucket, projectKey); err != nil { - return nil, err - } - } - // This is done if user specifies RealtimeAPIs in same file as TrafficSplitter apiConfigs = append(ExclusiveFilterAPIsByKind(apiConfigs, userconfig.TrafficSplitterKind), InclusiveFilterAPIsByKind(apiConfigs, userconfig.TrafficSplitterKind)...) diff --git a/pkg/operator/resources/validations.go b/pkg/operator/resources/validations.go index 49b0ccab0b..d9292d7f64 100644 --- a/pkg/operator/resources/validations.go +++ b/pkg/operator/resources/validations.go @@ -70,7 +70,7 @@ func (projectFiles ProjectFiles) HasDir(path string) bool { return false } -func ValidateClusterAPIs(apis []userconfig.API, projectFiles spec.ProjectFiles) error { +func ValidateClusterAPIs(apis []userconfig.API) error { if len(apis) == 0 { return spec.ErrorNoAPIs() } @@ -109,7 +109,7 @@ func ValidateClusterAPIs(apis []userconfig.API, projectFiles spec.ProjectFiles) if api.Kind == userconfig.RealtimeAPIKind || api.Kind == userconfig.BatchAPIKind || api.Kind == userconfig.TaskAPIKind || api.Kind == userconfig.AsyncAPIKind { - if err := spec.ValidateAPI(api, nil, projectFiles, config.AWS, config.K8s); err != nil { + if err := spec.ValidateAPI(api, config.AWS, config.K8s); err != nil { return errors.Wrap(err, api.Identify()) } @@ -139,7 +139,7 @@ func ValidateClusterAPIs(apis []userconfig.API, projectFiles spec.ProjectFiles) for i := range apis { api := &apis[i] if api.Kind != userconfig.TrafficSplitterKind { - if err := validateK8sCompute(api.Compute, maxMemMap); err != nil { + if err := validateK8sCompute(api, maxMemMap); err != nil { return err } } @@ -188,12 +188,14 @@ var _nvidiaDCGMExporterMemReserve = kresource.MustParse("50Mi") var _inferentiaCPUReserve = kresource.MustParse("100m") var _inferentiaMemReserve = kresource.MustParse("100Mi") -func validateK8sCompute(compute *userconfig.Compute, maxMemMap map[string]kresource.Quantity) error { +func validateK8sCompute(api *userconfig.API, maxMemMap map[string]kresource.Quantity) error { + compute := spec.GetTotalComputeFromContainers(api.Containers) + allErrors := []error{} successfulLoops := 0 clusterNodeGroupNames := strset.New(config.ClusterConfig.GetNodeGroupNames()...) - apiNodeGroupNames := compute.NodeGroups + apiNodeGroupNames := api.NodeGroups if apiNodeGroupNames != nil { for _, ngName := range apiNodeGroupNames { diff --git a/pkg/types/spec/api.go b/pkg/types/spec/api.go index de0c375055..06f0971f02 100644 --- a/pkg/types/spec/api.go +++ b/pkg/types/spec/api.go @@ -46,11 +46,6 @@ type API struct { ProjectKey string `json:"project_key"` } -type CuratedModelResource struct { - *userconfig.ModelResource - Versions []int64 `json:"versions"` -} - /* APIID (uniquely identifies an api configuration for a given deployment) * SpecID (uniquely identifies api configuration specified by user) diff --git a/pkg/types/spec/errors.go b/pkg/types/spec/errors.go index 1c9aff53be..db68bd5538 100644 --- a/pkg/types/spec/errors.go +++ b/pkg/types/spec/errors.go @@ -22,7 +22,6 @@ import ( "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/errors" - "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/k8s" libmath "github.com/cortexlabs/cortex/pkg/lib/math" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" @@ -43,6 +42,8 @@ const ( ErrOneOfPrerequisitesNotDefined = "spec.one_of_prerequisites_not_defined" ErrConfigGreaterThanOtherConfig = "spec.config_greater_than_other_config" + ErrPortCanOnlyDefinedOnce = "spec.port_can_only_be_defined_once" + ErrMinReplicasGreaterThanMax = "spec.min_replicas_greater_than_max" ErrInitReplicasGreaterThanMax = "spec.init_replicas_greater_than_max" ErrInitReplicasLessThanMin = "spec.init_replicas_less_than_min" @@ -50,32 +51,8 @@ const ( ErrInvalidSurgeOrUnavailable = "spec.invalid_surge_or_unavailable" ErrSurgeAndUnavailableBothZero = "spec.surge_and_unavailable_both_zero" - ErrModelCachingNotSupportedWhenMultiprocessingEnabled = "spec.model_caching_not_supported_when_multiprocessing_enabled" - ErrShmSizeCannotExceedMem = "spec.shm_size_cannot_exceed_mem" - ErrFileNotFound = "spec.file_not_found" - ErrDirIsEmpty = "spec.dir_is_empty" - ErrMustBeRelativeProjectPath = "spec.must_be_relative_project_path" - ErrPythonPathNotFound = "spec.python_path_not_found" - - ErrS3FileNotFound = "spec.s3_file_not_found" - ErrS3DirNotFound = "spec.s3_dir_not_found" - ErrS3DirIsEmpty = "spec.s3_dir_is_empty" - - ErrModelPathNotDirectory = "spec.model_path_not_directory" - ErrInvalidBucketScheme = "spec.invalid_bucket_scheme" - ErrInvalidPythonModelPath = "spec.invalid_python_model_path" - ErrInvalidTensorFlowModelPath = "spec.invalid_tensorflow_model_path" - - ErrDuplicateModelNames = "spec.duplicate_model_names" - ErrReservedModelName = "spec.reserved_model_name" - - ErrProtoNumServicesMismatch = "spec.proto_num_services_mismatch" - ErrProtoMissingPackageName = "spec.proto_missing_package_name" - ErrProtoInvalidPackageName = "spec.proto_invalid_package_name" - ErrProtoInvalidNetworkingEndpoint = "spec.proto_invalid_networking_endpoint" - ErrFieldMustBeDefinedForHandlerType = "spec.field_must_be_defined_for_handler_type" ErrFieldNotSupportedByHandlerType = "spec.field_not_supported_by_handler_type" ErrNoAvailableNodeComputeLimit = "spec.no_available_node_compute_limit" @@ -96,11 +73,6 @@ const ( ErrInvalidONNXHandlerType = "spec.invalid_onnx_handler_type" ) -var _modelCurrentStructure = ` - but its current structure is - -%s` - func ErrorMalformedConfig() error { return errors.WithStack(&errors.Error{ Kind: ErrMalformedConfig, @@ -197,6 +169,13 @@ func ErrorConfigGreaterThanOtherConfig(tooBigKey string, tooBigVal interface{}, }) } +func ErrorPortCanOnlyDefinedOnce() error { + return errors.WithStack(&errors.Error{ + Kind: ErrPortCanOnlyDefinedOnce, + Message: fmt.Sprintf("%s field must be specified for one container only", userconfig.PortKey), + }) +} + func ErrorMinReplicasGreaterThanMax(min int32, max int32) error { return errors.WithStack(&errors.Error{ Kind: ErrMinReplicasGreaterThanMax, @@ -239,243 +218,6 @@ func ErrorShmSizeCannotExceedMem(parentFieldName string, shmSize k8s.Quantity, m }) } -func ErrorModelCachingNotSupportedWhenMultiprocessingEnabled(desiredProcesses int32) error { - const maxNumProcesses int32 = 1 - return errors.WithStack(&errors.Error{ - Kind: ErrModelCachingNotSupportedWhenMultiprocessingEnabled, - Message: fmt.Sprintf("when dynamic model caching is enabled (%s < provided models), the max value %s can take is %d, while currently it's set to %d", - userconfig.ModelsCacheSizeKey, userconfig.ProcessesPerReplicaKey, maxNumProcesses, desiredProcesses), - }) -} - -func ErrorFileNotFound(path string) error { - return errors.WithStack(&errors.Error{ - Kind: ErrFileNotFound, - Message: fmt.Sprintf("%s: not found or insufficient permissions", path), - }) -} - -func ErrorDirIsEmpty(path string) error { - return errors.WithStack(&errors.Error{ - Kind: ErrDirIsEmpty, - Message: fmt.Sprintf("%s: directory is empty", path), - }) -} - -func ErrorMustBeRelativeProjectPath(path string) error { - return errors.WithStack(&errors.Error{ - Kind: ErrMustBeRelativeProjectPath, - Message: fmt.Sprintf("%s: must be a relative path (relative to the directory containing your API configuration file)", path), - }) -} - -func ErrorPythonPathNotFound(pythonPath string) error { - return errors.WithStack(&errors.Error{ - Kind: ErrPythonPathNotFound, - Message: fmt.Sprintf("%s: path does not exist, or has been excluded from your project directory", pythonPath), - }) -} - -func ErrorS3FileNotFound(path string) error { - return errors.WithStack(&errors.Error{ - Kind: ErrS3FileNotFound, - Message: fmt.Sprintf("%s: file not found or insufficient permissions", path), - }) -} - -func ErrorS3DirNotFound(path string) error { - return errors.WithStack(&errors.Error{ - Kind: ErrS3DirNotFound, - Message: fmt.Sprintf("%s: dir not found or insufficient permissions", path), - }) -} - -func ErrorS3DirIsEmpty(path string) error { - return errors.WithStack(&errors.Error{ - Kind: ErrS3DirIsEmpty, - Message: fmt.Sprintf("%s: S3 directory is empty", path), - }) -} - -func ErrorModelPathNotDirectory(modelPath string) error { - return errors.WithStack(&errors.Error{ - Kind: ErrModelPathNotDirectory, - Message: fmt.Sprintf("%s: model path must be a directory", modelPath), - }) -} - -func ErrorInvalidBucketScheme(path string) error { - return errors.WithStack(&errors.Error{ - Kind: ErrInvalidBucketScheme, - Message: fmt.Sprintf("%s: path must be an S3 path (e.g. s3://bucket/my-dir/)", path), - }) -} - -var _pythonModelTemplates = ` - %s - ├── 1523423423/ (Version prefix) - | └── * // Model-specific files (i.e. model.h5, model.pkl, labels.json, etc) - └── 2434389194/ (Version prefix) - └── * // Model-specific files (i.e. model.h5, model.pkl, labels.json, etc) - -or like - - %s - └── * // Model-specific files (i.e. model.h5, model.pkl, labels.json, etc) -` - -func ErrorInvalidPythonModelPath(modelPath string, modelSubPaths []string) error { - message := fmt.Sprintf("%s: invalid %s model path. ", modelPath, userconfig.PythonHandlerType.CasedString()) - message += " " + fmt.Sprintf("For models provided for the %s handler type, the path must be a directory with one of the following structures:\n", userconfig.PythonHandlerType) - - message += fmt.Sprintf(_pythonModelTemplates, modelPath, modelPath) - - if len(modelSubPaths) > 0 { - message += "\n" + "but its current structure is (limited to 50 sub-paths)" + "\n\n" - if len(modelSubPaths) > 50 { - message += s.Indent(files.FileTree(modelSubPaths[:50], "", files.DirsSorted), " ") - message += "\n ..." - } else { - message += s.Indent(files.FileTree(modelSubPaths, "", files.DirsSorted), " ") - } - } else { - message += "\n" + "but its current directory is empty" - } - - return errors.WithStack(&errors.Error{ - Kind: ErrInvalidPythonModelPath, - Message: message, - }) -} - -var _tfVersionedExpectedStructMessage = ` - %s - ├── 1523423423/ (Version prefix, usually a timestamp) - | ├── saved_model.pb - | └── variables/ - | ├── variables.index - | ├── variables.data-00000-of-00003 - | ├── variables.data-00001-of-00003 - | └── variables.data-00002-of-... - └── 2434389194/ (Version prefix, usually a timestamp) - ├── saved_model.pb - └── variables/ - ├── variables.index - ├── variables.data-00000-of-00003 - ├── variables.data-00001-of-00003 - └── variables.data-00002-of-... - -or like - - %s - ├── saved_model.pb - └── variables/ - ├── variables.index - ├── variables.data-00000-of-00003 - ├── variables.data-00001-of-00003 - └── variables.data-00002-of-... -` -var _neuronTfVersionedExpectedStructMessage = ` - %s - ├── 1523423423/ (Version prefix, usually a timestamp) - | └── saved_model.pb - └── 2434389194/ (Version prefix, usually a timestamp) - └── saved_model.pb - -or like - - %s - └── saved_model.pb -` - -func ErrorInvalidTensorFlowModelPath(modelPath string, neuronExport bool, modelSubPaths []string) error { - handlerType := userconfig.TensorFlowHandlerType.CasedString() - if neuronExport { - handlerType = "Neuron " + handlerType - } - message := fmt.Sprintf("%s: invalid %s model path.", modelPath, handlerType) - message += " " + fmt.Sprintf("For models provided for the %s handler type, the path must be a directory with one of the following structures:\n", userconfig.TensorFlowHandlerType) - - if !neuronExport { - message += fmt.Sprintf(_tfVersionedExpectedStructMessage, modelPath, modelPath) - } else { - message += fmt.Sprintf(_neuronTfVersionedExpectedStructMessage, modelPath, modelPath) - } - - if len(modelSubPaths) > 0 { - message += "\n" + "but its current structure is (limited to 50 sub-paths)" + "\n\n" - if len(modelSubPaths) > 50 { - message += s.Indent(files.FileTree(modelSubPaths[:50], "", files.DirsSorted), " ") - message += "\n ..." - } else { - message += s.Indent(files.FileTree(modelSubPaths, "", files.DirsSorted), " ") - } - } else { - message += "\n" + "but its current directory is empty" - } - - return errors.WithStack(&errors.Error{ - Kind: ErrInvalidTensorFlowModelPath, - Message: message, - }) -} - -func ErrorDuplicateModelNames(duplicateModel string) error { - return errors.WithStack(&errors.Error{ - Kind: ErrDuplicateModelNames, - Message: fmt.Sprintf("cannot have multiple models with the same name (%s)", duplicateModel), - }) -} - -func ErrorReservedModelName(reservedModel string) error { - return errors.WithStack(&errors.Error{ - Kind: ErrReservedModelName, - Message: fmt.Sprintf("%s: is a reserved name; please specify a different model name", reservedModel), - }) -} - -func ErrorProtoNumServicesMismatch(requested int) error { - return errors.WithStack(&errors.Error{ - Kind: ErrProtoNumServicesMismatch, - Message: fmt.Sprintf("can only have one service defined; there are currently %d services defined", requested), - }) -} - -func ErrorProtoMissingPackageName(allowedPackageName string) error { - return errors.WithStack(&errors.Error{ - Kind: ErrProtoMissingPackageName, - Message: fmt.Sprintf("your protobuf definition must have the %s package defined", allowedPackageName), - }) -} - -func ErrorProtoInvalidPackageName(requestedPackageName, allowedPackageName string) error { - return errors.WithStack(&errors.Error{ - Kind: ErrProtoInvalidPackageName, - Message: fmt.Sprintf("found invalid package %s; your package must be named %s", requestedPackageName, allowedPackageName), - }) -} - -func ErrorProtoInvalidNetworkingEndpoint(allowedValue string) error { - return errors.WithStack(&errors.Error{ - Kind: ErrProtoInvalidNetworkingEndpoint, - Message: fmt.Sprintf("because of the protobuf definition from section %s and field %s, the only permitted value is %s", userconfig.HandlerKey, userconfig.ProtobufPathKey, allowedValue), - }) -} - -func ErrorFieldMustBeDefinedForHandlerType(fieldKey string, handlerType userconfig.HandlerType) error { - return errors.WithStack(&errors.Error{ - Kind: ErrFieldMustBeDefinedForHandlerType, - Message: fmt.Sprintf("%s field must be defined for the %s handler type", fieldKey, handlerType.String()), - }) -} - -func ErrorFieldNotSupportedByHandlerType(fieldKey string, handlerType userconfig.HandlerType) error { - return errors.WithStack(&errors.Error{ - Kind: ErrFieldNotSupportedByHandlerType, - Message: fmt.Sprintf("%s is not a supported field for the %s handler type", fieldKey, handlerType.String()), - }) -} - func ErrorCortexPrefixedEnvVarNotAllowed() error { return errors.WithStack(&errors.Error{ Kind: ErrCortexPrefixedEnvVarNotAllowed, @@ -526,38 +268,6 @@ func ErrorInvalidNumberOfInfs(requestedInfs int64) error { }) } -func ErrorInsufficientBatchConcurrencyLevel(maxBatchSize int32, processesPerReplica int32, threadsPerProcess int32) error { - return errors.WithStack(&errors.Error{ - Kind: ErrInsufficientBatchConcurrencyLevel, - Message: fmt.Sprintf( - "%s (%d) must be less than or equal to %s * %s (%d * %d = %d)", - userconfig.MaxBatchSizeKey, maxBatchSize, userconfig.ProcessesPerReplicaKey, userconfig.ThreadsPerProcessKey, processesPerReplica, threadsPerProcess, processesPerReplica*threadsPerProcess, - ), - }) -} - -func ErrorInsufficientBatchConcurrencyLevelInf(maxBatchSize int32, threadsPerProcess int32) error { - return errors.WithStack(&errors.Error{ - Kind: ErrInsufficientBatchConcurrencyLevelInf, - Message: fmt.Sprintf( - "%s (%d) must be less than or equal to %s (%d)", - userconfig.MaxBatchSizeKey, maxBatchSize, userconfig.ThreadsPerProcessKey, threadsPerProcess, - ), - }) -} - -func ErrorConcurrencyMismatchServerSideBatchingPython(maxBatchsize int32, threadsPerProcess int32) error { - return errors.WithStack( - &errors.Error{ - Kind: ErrConcurrencyMismatchServerSideBatchingPython, - Message: fmt.Sprintf( - "%s (%d) must be equal to %s (%d) when using server side batching with the python handler", - userconfig.ThreadsPerProcessKey, threadsPerProcess, userconfig.MaxBatchSizeKey, maxBatchsize, - ), - }, - ) -} - func ErrorIncorrectTrafficSplitterWeightTotal(totalWeight int32) error { return errors.WithStack(&errors.Error{ Kind: ErrIncorrectTrafficSplitterWeight, @@ -597,10 +307,3 @@ func ErrorUnexpectedDockerSecretData(reason string, secretData map[string][]byte Message: fmt.Sprintf("docker registry secret named \"%s\" was found, but contains unexpected data (%s); got: %s", _dockerPullSecretName, reason, s.UserStr(secretDataStrMap)), }) } - -func ErrorInvalidONNXHandlerType() error { - return errors.WithStack(&errors.Error{ - Kind: ErrInvalidONNXHandlerType, - Message: "the onnx handler type has been replaced by the python handler type; please use the python handler type instead (all onnx models are fully supported by the python handler type)", - }) -} diff --git a/pkg/types/spec/utils.go b/pkg/types/spec/utils.go index eb0a6b6091..d4af31ade0 100644 --- a/pkg/types/spec/utils.go +++ b/pkg/types/spec/utils.go @@ -17,17 +17,10 @@ limitations under the License. package spec import ( - "path/filepath" - "strconv" "strings" - "github.com/cortexlabs/cortex/pkg/consts" - "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" - "github.com/cortexlabs/cortex/pkg/lib/files" - "github.com/cortexlabs/cortex/pkg/lib/pointer" - "github.com/cortexlabs/cortex/pkg/lib/sets/strset" - "github.com/cortexlabs/cortex/pkg/lib/slices" + "github.com/cortexlabs/cortex/pkg/lib/k8s" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/types/userconfig" ) @@ -50,17 +43,37 @@ func FindDuplicateNames(apis []userconfig.API) []userconfig.API { return nil } -func checkDuplicateModelNames(models []CuratedModelResource) error { - names := strset.New() +func GetTotalComputeFromContainers(containers []userconfig.Container) userconfig.Compute { + compute := userconfig.Compute{} - for _, model := range models { - if names.Has(model.Name) { - return ErrorDuplicateModelNames(model.Name) + for _, container := range containers { + if container.Compute == nil { + continue + } + + if container.Compute.CPU != nil { + newCPUQuantity := k8s.NewQuantity(container.Compute.CPU.Value()) + if compute.CPU != nil { + compute.CPU = newCPUQuantity + } else if newCPUQuantity != nil { + compute.CPU.AddQty(*newCPUQuantity) + } } - names.Add(model.Name) + + if container.Compute.Mem != nil { + newMemQuantity := k8s.NewQuantity(container.Compute.Mem.Value()) + if compute.CPU != nil { + compute.Mem = newMemQuantity + } else if newMemQuantity != nil { + compute.Mem.AddQty(*newMemQuantity) + } + } + + compute.GPU += container.Compute.GPU + compute.Inf += container.Compute.Inf } - return nil + return compute } func surgeOrUnavailableValidator(str string) (string, error) { @@ -85,346 +98,6 @@ func surgeOrUnavailableValidator(str string) (string, error) { return str, nil } -func checkForInvalidBucketScheme(modelPath string) (string, error) { - if !strings.HasPrefix(modelPath, "s3://") { - return "", ErrorInvalidBucketScheme(modelPath) - } - return modelPath, nil -} - -type errorForHandlerTypeFn func(string, []string) error - -func generateErrorForHandlerTypeFn(api *userconfig.API) errorForHandlerTypeFn { - return func(modelPrefix string, modelPaths []string) error { - switch api.Handler.Type { - case userconfig.PythonHandlerType: - return ErrorInvalidPythonModelPath(modelPrefix, modelPaths) - case userconfig.TensorFlowHandlerType: - return ErrorInvalidTensorFlowModelPath(modelPrefix, api.Compute.Inf > 0, modelPaths) - } - return nil - } -} - -func validateDirModels( - modelPath string, - signatureKey *string, - awsClient *aws.Client, - errorForHandlerType errorForHandlerTypeFn, - extraValidators []modelValidator) ([]CuratedModelResource, error) { - - var bucket string - var dirPrefix string - var modelDirPaths []string - var err error - - modelPath = s.EnsureSuffix(modelPath, "/") - - awsClientForBucket, err := aws.NewFromClientS3Path(modelPath, awsClient) - if err != nil { - return nil, err - } - - bucket, dirPrefix, err = aws.SplitS3Path(modelPath) - if err != nil { - return nil, err - } - - s3Objects, err := awsClientForBucket.ListS3PathDir(modelPath, false, nil, nil) - if err != nil { - return nil, err - } - modelDirPaths = aws.ConvertS3ObjectsToKeys(s3Objects...) - - if len(modelDirPaths) == 0 { - return nil, errorForHandlerType(dirPrefix, modelDirPaths) - } - - modelNames := []string{} - modelDirPathLength := len(slices.RemoveEmpties(strings.Split(dirPrefix, "/"))) - for _, path := range modelDirPaths { - splitPath := slices.RemoveEmpties(strings.Split(path, "/")) - modelNames = append(modelNames, splitPath[modelDirPathLength]) - } - modelNames = slices.UniqueStrings(modelNames) - - modelResources := make([]CuratedModelResource, len(modelNames)) - for i, modelName := range modelNames { - modelNameWrapStr := modelName - if modelName == consts.SingleModelName { - modelNameWrapStr = "" - } - - modelPrefix := filepath.Join(dirPrefix, modelName) - modelPrefix = s.EnsureSuffix(modelPrefix, "/") - - modelStructureType := determineBaseModelStructure(modelDirPaths, modelPrefix) - if modelStructureType == userconfig.UnknownModelStructureType { - return nil, errors.Wrap(errorForHandlerType(modelPrefix, nil), modelNameWrapStr) - } - - var versions []string - if modelStructureType == userconfig.VersionedModelType { - versions = getModelVersionsFromPaths(modelDirPaths, modelPrefix) - for _, version := range versions { - versionedModelPrefix := filepath.Join(modelPrefix, version) - versionedModelPrefix = s.EnsureSuffix(versionedModelPrefix, "/") - - for _, validator := range extraValidators { - err := validator(modelDirPaths, modelPrefix, pointer.String(versionedModelPrefix)) - if err != nil { - return nil, errors.Wrap(err, modelNameWrapStr) - } - } - } - } else { - for _, validator := range extraValidators { - err := validator(modelDirPaths, modelPrefix, nil) - if err != nil { - return nil, errors.Wrap(err, modelNameWrapStr) - } - } - } - - intVersions, err := slices.StringToInt64(versions) - if err != nil { - return nil, errors.Wrap(err, modelNameWrapStr) - } - - fullModelPath := s.EnsureSuffix(aws.S3Path(bucket, modelPrefix), "/") - - modelResources[i] = CuratedModelResource{ - ModelResource: &userconfig.ModelResource{ - Name: modelName, - Path: fullModelPath, - SignatureKey: signatureKey, - }, - Versions: intVersions, - } - } - - return modelResources, nil -} - -func validateModels( - models []userconfig.ModelResource, - defaultSignatureKey *string, - awsClient *aws.Client, - errorForHandlerType errorForHandlerTypeFn, - extraValidators []modelValidator) ([]CuratedModelResource, error) { - - var bucket string - var modelPrefix string - var modelPaths []string - - modelResources := make([]CuratedModelResource, len(models)) - for i, model := range models { - modelNameWrapStr := model.Name - if model.Name == consts.SingleModelName { - modelNameWrapStr = "" - } - - modelPath := s.EnsureSuffix(model.Path, "/") - - awsClientForBucket, err := aws.NewFromClientS3Path(model.Path, awsClient) - if err != nil { - return nil, errors.Wrap(err, modelNameWrapStr) - } - - bucket, modelPrefix, err = aws.SplitS3Path(model.Path) - if err != nil { - return nil, errors.Wrap(err, modelNameWrapStr) - } - modelPrefix = s.EnsureSuffix(modelPrefix, "/") - - s3Objects, err := awsClientForBucket.ListS3PathDir(modelPath, false, nil, nil) - if err != nil { - return nil, errors.Wrap(err, modelNameWrapStr) - } - modelPaths = aws.ConvertS3ObjectsToKeys(s3Objects...) - - if len(modelPaths) == 0 { - return nil, errors.Wrap(errorForHandlerType(modelPrefix, modelPaths), modelNameWrapStr) - } - - modelStructureType := determineBaseModelStructure(modelPaths, modelPrefix) - if modelStructureType == userconfig.UnknownModelStructureType { - return nil, errors.Wrap(ErrorInvalidPythonModelPath(modelPath, []string{}), modelNameWrapStr) - } - - var versions []string - if modelStructureType == userconfig.VersionedModelType { - versions = getModelVersionsFromPaths(modelPaths, modelPrefix) - for _, version := range versions { - versionedModelPrefix := filepath.Join(modelPrefix, version) - versionedModelPrefix = s.EnsureSuffix(versionedModelPrefix, "/") - - for _, validator := range extraValidators { - err := validator(modelPaths, modelPrefix, pointer.String(versionedModelPrefix)) - if err != nil { - return nil, errors.Wrap(err, modelNameWrapStr) - } - } - } - } else { - for _, validator := range extraValidators { - err := validator(modelPaths, modelPrefix, nil) - if err != nil { - return nil, errors.Wrap(err, modelNameWrapStr) - } - } - } - - intVersions, err := slices.StringToInt64(versions) - if err != nil { - return nil, errors.Wrap(err, modelNameWrapStr) - } - - var signatureKey *string - if model.SignatureKey != nil { - signatureKey = model.SignatureKey - } else if defaultSignatureKey != nil { - signatureKey = defaultSignatureKey - } - - fullModelPath := s.EnsureSuffix(aws.S3Path(bucket, modelPrefix), "/") - - modelResources[i] = CuratedModelResource{ - ModelResource: &userconfig.ModelResource{ - Name: model.Name, - Path: fullModelPath, - SignatureKey: signatureKey, - }, - Versions: intVersions, - } - } - - return modelResources, nil -} - -func tensorflowModelValidator(paths []string, prefix string, versionedPrefix *string) error { - var filteredFilePaths []string - if versionedPrefix != nil { - filteredFilePaths = files.FilterPathsWithDirPrefix(paths, *versionedPrefix) - } else { - filteredFilePaths = files.FilterPathsWithDirPrefix(paths, prefix) - } - - errFunc := func() error { - if versionedPrefix != nil { - return ErrorInvalidTensorFlowModelPath(prefix, false, files.FilterPathsWithDirPrefix(paths, prefix)) - } - return ErrorInvalidTensorFlowModelPath(prefix, false, filteredFilePaths) - } - - filesToHave := []string{"saved_model.pb", "variables/variables.index"} - for _, fileToHave := range filesToHave { - var fullFilePath string - if versionedPrefix != nil { - fullFilePath = filepath.Join(*versionedPrefix, fileToHave) - } else { - fullFilePath = filepath.Join(prefix, fileToHave) - } - if !slices.HasString(filteredFilePaths, fullFilePath) { - return errFunc() - } - } - - filesWithPrefix := []string{"variables/variables.data-00000-of"} - for _, fileWithPrefix := range filesWithPrefix { - var prefixPath string - if versionedPrefix != nil { - prefixPath = filepath.Join(*versionedPrefix, fileWithPrefix) - } else { - prefixPath = filepath.Join(prefix, fileWithPrefix) - } - prefixPaths := slices.FilterStrs(filteredFilePaths, func(path string) bool { - return strings.HasPrefix(path, prefixPath) - }) - if len(prefixPaths) == 0 { - return errFunc() - } - } - - return nil -} - -func tensorflowNeuronModelValidator(paths []string, prefix string, versionedPrefix *string) error { - var filteredFilePaths []string - if versionedPrefix != nil { - filteredFilePaths = files.FilterPathsWithDirPrefix(paths, *versionedPrefix) - } else { - filteredFilePaths = files.FilterPathsWithDirPrefix(paths, prefix) - } - - errFunc := func() error { - if versionedPrefix != nil { - return ErrorInvalidTensorFlowModelPath(prefix, true, files.FilterPathsWithDirPrefix(paths, prefix)) - } - return ErrorInvalidTensorFlowModelPath(prefix, true, filteredFilePaths) - } - - filesToHave := []string{"saved_model.pb"} - for _, fileToHave := range filesToHave { - var fullFilePath string - if versionedPrefix != nil { - fullFilePath = filepath.Join(*versionedPrefix, fileToHave) - } else { - fullFilePath = filepath.Join(prefix, fileToHave) - } - if !slices.HasString(filteredFilePaths, fullFilePath) { - return errFunc() - } - } - - return nil -} - -func determineBaseModelStructure(paths []string, prefix string) userconfig.ModelStructureType { - filteredPaths := files.FilterPathsWithDirPrefix(paths, prefix) - prefixLength := len(slices.RemoveEmpties(strings.Split(prefix, "/"))) - - numFailedVersionChecks := 0 - numPassedVersionChecks := 0 - for _, path := range filteredPaths { - splitPath := slices.RemoveEmpties(strings.Split(path, "/")) - versionStr := splitPath[prefixLength] - _, err := strconv.ParseInt(versionStr, 10, 64) - if err != nil { - numFailedVersionChecks++ - continue - } - numPassedVersionChecks++ - if len(splitPath) == prefixLength { - return userconfig.UnknownModelStructureType - } - } - - if numFailedVersionChecks > 0 && numPassedVersionChecks > 0 { - return userconfig.UnknownModelStructureType - } - if numFailedVersionChecks > 0 { - return userconfig.NonVersionedModelType - } - if numPassedVersionChecks > 0 { - return userconfig.VersionedModelType - } - return userconfig.UnknownModelStructureType -} - -func getModelVersionsFromPaths(paths []string, prefix string) []string { - filteredPaths := files.FilterPathsWithDirPrefix(paths, prefix) - prefixLength := len(slices.RemoveEmpties(strings.Split(prefix, "/"))) - - versions := []string{} - for _, path := range filteredPaths { - splitPath := slices.RemoveEmpties(strings.Split(path, "/")) - versions = append(versions, splitPath[prefixLength]) - } - - return slices.UniqueStrings(versions) -} - func verifyTotalWeight(apis []*userconfig.TrafficSplit) error { totalWeight := int32(0) for _, api := range apis { diff --git a/pkg/types/spec/validations.go b/pkg/types/spec/validations.go index 000313aede..4c8db743e5 100644 --- a/pkg/types/spec/validations.go +++ b/pkg/types/spec/validations.go @@ -17,9 +17,9 @@ limitations under the License. package spec import ( - "bytes" "context" "fmt" + "strconv" "strings" "time" @@ -29,18 +29,15 @@ import ( cr "github.com/cortexlabs/cortex/pkg/lib/configreader" "github.com/cortexlabs/cortex/pkg/lib/docker" "github.com/cortexlabs/cortex/pkg/lib/errors" - "github.com/cortexlabs/cortex/pkg/lib/files" libjson "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/k8s" libmath "github.com/cortexlabs/cortex/pkg/lib/math" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/regex" - s "github.com/cortexlabs/cortex/pkg/lib/strings" libtime "github.com/cortexlabs/cortex/pkg/lib/time" "github.com/cortexlabs/cortex/pkg/lib/urls" "github.com/cortexlabs/cortex/pkg/types/userconfig" dockertypes "github.com/docker/docker/api/types" - pbparser "github.com/emicklei/proto" kresource "k8s.io/apimachinery/pkg/api/resource" ) @@ -54,31 +51,27 @@ func apiValidation(resource userconfig.Resource) *cr.StructValidation { switch resource.Kind { case userconfig.RealtimeAPIKind: structFieldValidations = append(resourceStructValidations, - handlerValidation(), + podValidation(), networkingValidation(), - computeValidation(), autoscalingValidation(), updateStrategyValidation(), ) case userconfig.AsyncAPIKind: structFieldValidations = append(resourceStructValidations, - handlerValidation(), + podValidation(), networkingValidation(), - computeValidation(), autoscalingValidation(), updateStrategyValidation(), ) case userconfig.BatchAPIKind: structFieldValidations = append(resourceStructValidations, - handlerValidation(), + podValidation(), networkingValidation(), - computeValidation(), ) case userconfig.TaskAPIKind: structFieldValidations = append(resourceStructValidations, - taskDefinitionValidation(), + podValidation(), networkingValidation(), - computeValidation(), ) case userconfig.TrafficSplitterKind: structFieldValidations = append(resourceStructValidations, @@ -146,91 +139,11 @@ func multiAPIsValidation() *cr.StructFieldValidation { } } -func handlerValidation() *cr.StructFieldValidation { +func podValidation() *cr.StructFieldValidation { return &cr.StructFieldValidation{ - StructField: "Handler", + StructField: "Pod", StructValidation: &cr.StructValidation{ - Required: true, StructFieldValidations: []*cr.StructFieldValidation{ - { - StructField: "Type", - StringValidation: &cr.StringValidation{ - Required: true, - AllowedValues: userconfig.HandlerTypeStrings(), - HiddenAllowedValues: []string{"onnx"}, - }, - Parser: func(str string) (interface{}, error) { - if str == "onnx" { - return nil, ErrorInvalidONNXHandlerType() - } - return userconfig.HandlerTypeFromString(str), nil - }, - }, - { - StructField: "Path", - StringValidation: &cr.StringValidation{ - Required: true, - AllowedSuffixes: []string{ - ".py", - ".pickle", - }, - }, - }, - { - StructField: "ProtobufPath", - StringPtrValidation: &cr.StringPtrValidation{ - Default: nil, - AllowExplicitNull: true, - AlphaNumericDotUnderscore: true, - Suffix: ".proto", - }, - }, - { - StructField: "PythonPath", - StringPtrValidation: &cr.StringPtrValidation{ - AllowEmpty: false, - DisallowedValues: []string{".", "./", "./."}, - Validator: func(path string) (string, error) { - if files.IsAbsOrTildePrefixed(path) { - return "", ErrorMustBeRelativeProjectPath(path) - } - path = strings.TrimPrefix(path, "./") - path = s.EnsureSuffix(path, "/") - return path, nil - }, - }, - }, - { - StructField: "Image", - StringValidation: &cr.StringValidation{ - Required: false, - AllowEmpty: true, - DockerImageOrEmpty: true, - }, - }, - { - StructField: "TensorFlowServingImage", - StringValidation: &cr.StringValidation{ - Required: false, - AllowEmpty: true, - DockerImageOrEmpty: true, - }, - }, - { - StructField: "ProcessesPerReplica", - Int32Validation: &cr.Int32Validation{ - Default: 1, - GreaterThanOrEqualTo: pointer.Int32(1), - LessThanOrEqualTo: pointer.Int32(100), - }, - }, - { - StructField: "ThreadsPerProcess", - Int32Validation: &cr.Int32Validation{ - Default: 1, - GreaterThanOrEqualTo: pointer.Int32(1), - }, - }, { StructField: "ShmSize", StringPtrValidation: &cr.StringPtrValidation{ @@ -240,112 +153,74 @@ func handlerValidation() *cr.StructFieldValidation { Parser: k8s.QuantityParser(&k8s.QuantityValidation{}), }, { - StructField: "LogLevel", - StringValidation: &cr.StringValidation{ - Default: "info", - AllowedValues: userconfig.LogLevelTypes(), - }, - Parser: func(str string) (interface{}, error) { - return userconfig.LogLevelFromString(str), nil - }, - }, - { - StructField: "Config", - InterfaceMapValidation: &cr.InterfaceMapValidation{ - StringKeysOnly: true, - AllowEmpty: true, - AllowExplicitNull: true, - ConvertNullToEmpty: true, - Default: map[string]interface{}{}, - }, - }, - { - StructField: "Env", - StringMapValidation: &cr.StringMapValidation{ - Default: map[string]string{}, - AllowEmpty: true, + StructField: "NodeGroups", + StringListValidation: &cr.StringListValidation{ + Required: false, + Default: nil, + AllowExplicitNull: true, + AllowEmpty: false, + ElementStringValidation: &cr.StringValidation{ + AlphaNumericDashUnderscore: true, + }, }, }, - multiModelValidation("Models"), - multiModelValidation("MultiModelReloading"), - serverSideBatchingValidation(), - dependencyPathValidation(), + containersValidation(), }, }, } } -func taskDefinitionValidation() *cr.StructFieldValidation { +func containersValidation() *cr.StructFieldValidation { return &cr.StructFieldValidation{ - StructField: "TaskDefinition", - StructValidation: &cr.StructValidation{ - Required: true, - StructFieldValidations: []*cr.StructFieldValidation{ - { - StructField: "Path", - StringValidation: &cr.StringValidation{ - Required: true, - }, - }, - { - StructField: "PythonPath", - StringPtrValidation: &cr.StringPtrValidation{ - AllowEmpty: false, - DisallowedValues: []string{".", "./", "./."}, - Validator: func(path string) (string, error) { - if files.IsAbsOrTildePrefixed(path) { - return "", ErrorMustBeRelativeProjectPath(path) - } - path = strings.TrimPrefix(path, "./") - path = s.EnsureSuffix(path, "/") - return path, nil + StructField: "Containers", + StructListValidation: &cr.StructListValidation{ + Required: true, + TreatNullAsEmpty: true, + MinLength: 1, + StructValidation: &cr.StructValidation{ + StructFieldValidations: []*cr.StructFieldValidation{ + { + StructField: "Name", + StringValidation: &cr.StringValidation{ + Required: true, + AllowEmpty: false, + AlphaNumericDashUnderscore: true, }, }, - }, - { - StructField: "Image", - StringValidation: &cr.StringValidation{ - Required: false, - AllowEmpty: true, - DockerImageOrEmpty: true, - }, - }, - { - StructField: "ShmSize", - StringPtrValidation: &cr.StringPtrValidation{ - Default: nil, - AllowExplicitNull: true, - }, - Parser: k8s.QuantityParser(&k8s.QuantityValidation{}), - }, - { - StructField: "LogLevel", - StringValidation: &cr.StringValidation{ - Default: "info", - AllowedValues: userconfig.LogLevelTypes(), + { + StructField: "Image", + StringValidation: &cr.StringValidation{ + Required: true, + AllowEmpty: false, + DockerImageOrEmpty: true, + }, }, - Parser: func(str string) (interface{}, error) { - return userconfig.LogLevelFromString(str), nil + { + StructField: "Env", + StringMapValidation: &cr.StringMapValidation{ + Required: false, + Default: map[string]string{}, + AllowEmpty: true, + }, }, - }, - { - StructField: "Config", - InterfaceMapValidation: &cr.InterfaceMapValidation{ - StringKeysOnly: true, - AllowEmpty: true, - AllowExplicitNull: true, - ConvertNullToEmpty: true, - Default: map[string]interface{}{}, + { + StructField: "Command", + StringListValidation: &cr.StringListValidation{ + Required: false, + AllowExplicitNull: true, + AllowEmpty: true, + }, }, - }, - { - StructField: "Env", - StringMapValidation: &cr.StringMapValidation{ - Default: map[string]string{}, - AllowEmpty: true, + { + StructField: "Args", + StringListValidation: &cr.StringListValidation{ + Required: false, + AllowExplicitNull: true, + AllowEmpty: true, + }, }, + computeValidation(), }, - dependencyPathValidation(), }, }, } @@ -408,18 +283,6 @@ func computeValidation() *cr.StructFieldValidation { GreaterThanOrEqualTo: pointer.Int64(0), }, }, - { - StructField: "NodeGroups", - StringListValidation: &cr.StringListValidation{ - Required: false, - Default: nil, - AllowExplicitNull: true, - AllowEmpty: false, - ElementStringValidation: &cr.StringValidation{ - AlphaNumericDashUnderscore: true, - }, - }, - }, }, }, } @@ -556,150 +419,6 @@ func updateStrategyValidation() *cr.StructFieldValidation { } } -func multiModelValidation(fieldName string) *cr.StructFieldValidation { - return &cr.StructFieldValidation{ - StructField: fieldName, - StructValidation: &cr.StructValidation{ - Required: false, - DefaultNil: true, - StructFieldValidations: []*cr.StructFieldValidation{ - { - StructField: "Path", - StringPtrValidation: &cr.StringPtrValidation{ - Required: false, - Validator: checkForInvalidBucketScheme, - }, - }, - multiModelPathsValidation(), - { - StructField: "Dir", - StringPtrValidation: &cr.StringPtrValidation{ - Required: false, - Validator: checkForInvalidBucketScheme, - }, - }, - { - StructField: "SignatureKey", - StringPtrValidation: &cr.StringPtrValidation{ - Required: false, - }, - }, - { - StructField: "CacheSize", - Int32PtrValidation: &cr.Int32PtrValidation{ - Required: false, - GreaterThan: pointer.Int32(0), - }, - }, - { - StructField: "DiskCacheSize", - Int32PtrValidation: &cr.Int32PtrValidation{ - Required: false, - GreaterThan: pointer.Int32(0), - }, - }, - }, - }, - } -} - -func multiModelPathsValidation() *cr.StructFieldValidation { - return &cr.StructFieldValidation{ - StructField: "Paths", - StructListValidation: &cr.StructListValidation{ - Required: false, - TreatNullAsEmpty: true, - StructValidation: &cr.StructValidation{ - StructFieldValidations: []*cr.StructFieldValidation{ - { - StructField: "Name", - StringValidation: &cr.StringValidation{ - Required: true, - AllowEmpty: false, - DisallowedValues: []string{consts.SingleModelName}, - AlphaNumericDashUnderscore: true, - }, - }, - { - StructField: "Path", - StringValidation: &cr.StringValidation{ - Required: true, - AllowEmpty: false, - Validator: checkForInvalidBucketScheme, - }, - }, - { - StructField: "SignatureKey", - StringPtrValidation: &cr.StringPtrValidation{ - Required: false, - AllowEmpty: false, - }, - }, - }, - }, - }, - } -} - -func serverSideBatchingValidation() *cr.StructFieldValidation { - return &cr.StructFieldValidation{ - StructField: "ServerSideBatching", - StructValidation: &cr.StructValidation{ - Required: false, - DefaultNil: true, - AllowExplicitNull: true, - StructFieldValidations: []*cr.StructFieldValidation{ - { - StructField: "MaxBatchSize", - Int32Validation: &cr.Int32Validation{ - Required: true, - GreaterThanOrEqualTo: pointer.Int32(2), - LessThanOrEqualTo: pointer.Int32(1024), // this is an arbitrary limit - }, - }, - { - StructField: "BatchInterval", - StringValidation: &cr.StringValidation{ - Required: true, - }, - Parser: cr.DurationParser(&cr.DurationValidation{ - GreaterThan: pointer.Duration(libtime.MustParseDuration("0s")), - }), - }, - }, - }, - } -} - -func dependencyPathValidation() *cr.StructFieldValidation { - return &cr.StructFieldValidation{ - StructField: "Dependencies", - StructValidation: &cr.StructValidation{ - Required: false, - StructFieldValidations: []*cr.StructFieldValidation{ - { - StructField: "Pip", - StringValidation: &cr.StringValidation{ - Default: "requirements.txt", - }, - }, - { - StructField: "Conda", - StringValidation: &cr.StringValidation{ - Default: "conda-packages.txt", - }, - }, - { - StructField: "Shell", - StringValidation: &cr.StringValidation{ - Default: "dependencies.sh", - }, - }, - }, - }, - } -} - var resourceStructValidation = cr.StructValidation{ AllowExtraFields: true, StructFieldValidations: resourceStructValidations, @@ -746,16 +465,7 @@ func ExtractAPIConfigs(configBytes []byte, configFileName string) ([]userconfig. if !ok { return nil, errors.ErrorUnexpected("unable to cast api spec to json") // unexpected } - api.SubmittedAPISpec = interfaceMap - - if resourceStruct.Kind == userconfig.RealtimeAPIKind || - resourceStruct.Kind == userconfig.BatchAPIKind || - resourceStruct.Kind == userconfig.TaskAPIKind || - resourceStruct.Kind == userconfig.AsyncAPIKind { - api.ApplyDefaultDockerPaths() - } - apis[i] = api } @@ -764,33 +474,16 @@ func ExtractAPIConfigs(configBytes []byte, configFileName string) ([]userconfig. func ValidateAPI( api *userconfig.API, - models *[]CuratedModelResource, - projectFiles ProjectFiles, awsClient *aws.Client, k8sClient *k8s.Client, ) error { - // if models is nil, we need to set it to an empty slice to avoid nil pointer exceptions - if models == nil { - models = &[]CuratedModelResource{} - } - if api.Networking.Endpoint == nil && (api.Handler == nil || (api.Handler != nil && api.Handler.ProtobufPath == nil)) { api.Networking.Endpoint = pointer.String("/" + api.Name) } - switch api.Kind { - case userconfig.TaskAPIKind: - if err := validateTaskDefinition(api, projectFiles, awsClient, k8sClient); err != nil { - return errors.Wrap(err, userconfig.TaskDefinitionKey) - } - default: - if err := validateHandler(api, models, projectFiles, awsClient, k8sClient); err != nil { - if errors.GetKind(err) == ErrProtoInvalidNetworkingEndpoint { - return errors.Wrap(err, userconfig.NetworkingKey, userconfig.EndpointKey) - } - return errors.Wrap(err, userconfig.HandlerKey) - } + if err := validateContainers(api, awsClient, k8sClient); err != nil { + return errors.Wrap(err, userconfig.ContainersKey) } if api.Autoscaling != nil { @@ -799,10 +492,6 @@ func ValidateAPI( } } - if err := validateCompute(api); err != nil { - return errors.Wrap(err, userconfig.ComputeKey) - } - if api.UpdateStrategy != nil { if err := validateUpdateStrategy(api.UpdateStrategy); err != nil { return errors.Wrap(err, userconfig.UpdateStrategyKey) @@ -815,46 +504,6 @@ func ValidateAPI( } } - if api.TaskDefinition != nil && api.TaskDefinition.ShmSize != nil && api.Compute.Mem != nil { - if api.TaskDefinition.ShmSize.Cmp(api.Compute.Mem.Quantity) > 0 { - return ErrorShmSizeCannotExceedMem(userconfig.TaskDefinitionKey, *api.TaskDefinition.ShmSize, *api.Compute.Mem) - } - } - - return nil -} - -func validateTaskDefinition( - api *userconfig.API, - projectFiles ProjectFiles, - awsClient *aws.Client, - k8sClient *k8s.Client, -) error { - taskDefinition := api.TaskDefinition - - if err := validateDockerImagePath(taskDefinition.Image, awsClient, k8sClient); err != nil { - return errors.Wrap(err, userconfig.ImageKey) - } - - for key := range taskDefinition.Env { - if strings.HasPrefix(key, "CORTEX_") { - return errors.Wrap(ErrorCortexPrefixedEnvVarNotAllowed(), userconfig.EnvKey, key) - } - } - - if !projectFiles.HasFile(taskDefinition.Path) { - return errors.Wrap(files.ErrorFileDoesNotExist(taskDefinition.Path), userconfig.PathKey) - } - - if taskDefinition.PythonPath != nil { - if !projectFiles.HasDir(*taskDefinition.PythonPath) { - return errors.Wrap( - ErrorPythonPathNotFound(*taskDefinition.PythonPath), - userconfig.PythonPathKey, - ) - } - } - return nil } @@ -882,406 +531,35 @@ func ValidateTrafficSplitter(api *userconfig.API) error { return nil } -func validateHandler( +func validateContainers( api *userconfig.API, - models *[]CuratedModelResource, - projectFiles ProjectFiles, awsClient *aws.Client, k8sClient *k8s.Client, ) error { - handler := api.Handler - - if !projectFiles.HasFile(handler.Path) { - return errors.Wrap(files.ErrorFileDoesNotExist(handler.Path), userconfig.PathKey) - } - - if handler.PythonPath != nil { - if err := validatePythonPath(handler, projectFiles); err != nil { - return errors.Wrap(err, userconfig.PythonPathKey) - } - } - - if handler.IsGRPC() { - if api.Kind != userconfig.RealtimeAPIKind { - return ErrorKeyIsNotSupportedForKind(userconfig.ProtobufPathKey, api.Kind) - } - - if err := validateProtobufPath(api, projectFiles); err != nil { - return err - } - } - - if err := validateMultiModelsFields(api); err != nil { - return err - } - - switch handler.Type { - case userconfig.PythonHandlerType: - if err := validatePythonHandler(api, models, awsClient); err != nil { - return err - } - case userconfig.TensorFlowHandlerType: - if err := validateTensorFlowHandler(api, models, awsClient); err != nil { - return err - } - if err := validateDockerImagePath(handler.TensorFlowServingImage, awsClient, k8sClient); err != nil { - return errors.Wrap(err, userconfig.TensorFlowServingImageKey) - } - } - - if api.Kind == userconfig.BatchAPIKind || api.Kind == userconfig.AsyncAPIKind { - if handler.MultiModelReloading != nil { - return ErrorKeyIsNotSupportedForKind(userconfig.MultiModelReloadingKey, api.Kind) - } - - if handler.ServerSideBatching != nil { - return ErrorKeyIsNotSupportedForKind(userconfig.ServerSideBatchingKey, api.Kind) - } - - if handler.ProcessesPerReplica > 1 { - return ErrorKeyIsNotSupportedForKind(userconfig.ProcessesPerReplicaKey, api.Kind) - } - - if handler.ThreadsPerProcess > 1 { - return ErrorKeyIsNotSupportedForKind(userconfig.ThreadsPerProcessKey, api.Kind) - } - } - - if err := validateDockerImagePath(handler.Image, awsClient, k8sClient); err != nil { - return errors.Wrap(err, userconfig.ImageKey) - } - - for key := range handler.Env { - if strings.HasPrefix(key, "CORTEX_") { - return errors.Wrap(ErrorCortexPrefixedEnvVarNotAllowed(), userconfig.EnvKey, key) - } - } - - return nil -} - -func validateMultiModelsFields(api *userconfig.API) error { - handler := api.Handler - - var models *userconfig.MultiModels - if api.Handler.Models != nil { - if api.Handler.Type == userconfig.PythonHandlerType { - return ErrorFieldNotSupportedByHandlerType(userconfig.ModelsKey, api.Handler.Type) - } - models = api.Handler.Models - } - if api.Handler.MultiModelReloading != nil { - if api.Handler.Type != userconfig.PythonHandlerType { - return ErrorFieldNotSupportedByHandlerType(userconfig.MultiModelReloadingKey, api.Handler.Type) - } - models = api.Handler.MultiModelReloading - } - - if models == nil { - if api.Handler.Type != userconfig.PythonHandlerType { - return ErrorFieldMustBeDefinedForHandlerType(userconfig.ModelsKey, api.Handler.Type) - } - return nil - } - - if models.Path == nil && len(models.Paths) == 0 && models.Dir == nil { - return errors.Wrap(ErrorSpecifyOnlyOneField(userconfig.ModelsPathKey, userconfig.ModelsPathsKey, userconfig.ModelsDirKey), userconfig.ModelsKey) - } - if models.Path != nil && len(models.Paths) > 0 && models.Dir != nil { - return errors.Wrap(ErrorSpecifyOnlyOneField(userconfig.ModelsPathKey, userconfig.ModelsPathsKey, userconfig.ModelsDirKey), userconfig.ModelsKey) - } - - if models.Path != nil && len(models.Paths) > 0 { - return errors.Wrap(ErrorConflictingFields(userconfig.ModelsPathKey, userconfig.ModelsPathsKey), userconfig.ModelsKey) - } - if models.Dir != nil && len(models.Paths) > 0 { - return errors.Wrap(ErrorConflictingFields(userconfig.ModelsPathsKey, userconfig.ModelsDirKey), userconfig.ModelsKey) - } - if models.Dir != nil && models.Path != nil { - return errors.Wrap(ErrorConflictingFields(userconfig.ModelsPathKey, userconfig.ModelsDirKey), userconfig.ModelsKey) - } - - if models.CacheSize != nil && api.Kind != userconfig.RealtimeAPIKind { - return errors.Wrap(ErrorKeyIsNotSupportedForKind(userconfig.ModelsCacheSizeKey, api.Kind), userconfig.ModelsKey) - } - if models.DiskCacheSize != nil && api.Kind != userconfig.RealtimeAPIKind { - return errors.Wrap(ErrorKeyIsNotSupportedForKind(userconfig.ModelsDiskCacheSizeKey, api.Kind), userconfig.ModelsKey) - } - - if (models.CacheSize == nil && models.DiskCacheSize != nil) || - (models.CacheSize != nil && models.DiskCacheSize == nil) { - return errors.Wrap(ErrorSpecifyAllOrNone(userconfig.ModelsCacheSizeKey, userconfig.ModelsDiskCacheSizeKey), userconfig.ModelsKey) - } - - if models.CacheSize != nil && models.DiskCacheSize != nil { - if *models.CacheSize > *models.DiskCacheSize { - return errors.Wrap(ErrorConfigGreaterThanOtherConfig(userconfig.ModelsCacheSizeKey, *models.CacheSize, userconfig.ModelsDiskCacheSizeKey, *models.DiskCacheSize), userconfig.ModelsKey) - } - - if handler.ProcessesPerReplica > 1 { - return ErrorModelCachingNotSupportedWhenMultiprocessingEnabled(handler.ProcessesPerReplica) - } - } - - return nil -} - -func validatePythonHandler(api *userconfig.API, models *[]CuratedModelResource, awsClient *aws.Client) error { - handler := api.Handler - - if handler.Models != nil { - return ErrorFieldNotSupportedByHandlerType(userconfig.ModelsKey, handler.Type) - } - - if handler.ServerSideBatching != nil { - if handler.ServerSideBatching.MaxBatchSize != handler.ThreadsPerProcess { - return ErrorConcurrencyMismatchServerSideBatchingPython( - handler.ServerSideBatching.MaxBatchSize, - handler.ThreadsPerProcess, - ) - } - } - if handler.TensorFlowServingImage != "" { - return ErrorFieldNotSupportedByHandlerType(userconfig.TensorFlowServingImageKey, handler.Type) - } - - if handler.MultiModelReloading == nil { - return nil - } - mmr := handler.MultiModelReloading - if mmr.SignatureKey != nil { - return errors.Wrap(ErrorFieldNotSupportedByHandlerType(userconfig.ModelsSignatureKeyKey, handler.Type), userconfig.MultiModelReloadingKey) - } - - hasSingleModel := mmr.Path != nil - hasMultiModels := !hasSingleModel - - var modelWrapError func(error) error - var modelResources []userconfig.ModelResource - - if hasSingleModel { - modelWrapError = func(err error) error { - return errors.Wrap(err, userconfig.MultiModelReloadingKey, userconfig.ModelsPathKey) - } - modelResources = []userconfig.ModelResource{ - { - Name: consts.SingleModelName, - Path: *mmr.Path, - }, - } - *mmr.Path = s.EnsureSuffix(*mmr.Path, "/") - } - if hasMultiModels { - if mmr.SignatureKey != nil { - return errors.Wrap(ErrorFieldNotSupportedByHandlerType(userconfig.ModelsSignatureKeyKey, handler.Type), userconfig.MultiModelReloadingKey) - } - - if len(mmr.Paths) > 0 { - modelWrapError = func(err error) error { - return errors.Wrap(err, userconfig.MultiModelReloadingKey, userconfig.ModelsPathsKey) - } - - for _, path := range mmr.Paths { - if path.SignatureKey != nil { - return errors.Wrap( - ErrorFieldNotSupportedByHandlerType(userconfig.ModelsSignatureKeyKey, handler.Type), - userconfig.MultiModelReloadingKey, - userconfig.ModelsKey, - userconfig.ModelsPathsKey, - path.Name, - ) - } - (*path).Path = s.EnsureSuffix((*path).Path, "/") - modelResources = append(modelResources, *path) - } - } - - if mmr.Dir != nil { - modelWrapError = func(err error) error { - return errors.Wrap(err, userconfig.MultiModelReloadingKey, userconfig.ModelsDirKey) - } - } - } - - var err error - if hasMultiModels && mmr.Dir != nil { - *models, err = validateDirModels(*mmr.Dir, nil, awsClient, generateErrorForHandlerTypeFn(api), nil) - } else { - *models, err = validateModels(modelResources, nil, awsClient, generateErrorForHandlerTypeFn(api), nil) - } - if err != nil { - return modelWrapError(err) - } - - if hasMultiModels { - for _, model := range *models { - if model.Name == consts.SingleModelName { - return modelWrapError(ErrorReservedModelName(model.Name)) - } - } - } - - if err := checkDuplicateModelNames(*models); err != nil { - return modelWrapError(err) - } - - return nil -} - -func validateTensorFlowHandler(api *userconfig.API, models *[]CuratedModelResource, awsClient *aws.Client) error { - handler := api.Handler - - if handler.ServerSideBatching != nil { - if api.Compute.Inf == 0 && handler.ServerSideBatching.MaxBatchSize > handler.ProcessesPerReplica*handler.ThreadsPerProcess { - return ErrorInsufficientBatchConcurrencyLevel(handler.ServerSideBatching.MaxBatchSize, handler.ProcessesPerReplica, handler.ThreadsPerProcess) - } - if api.Compute.Inf > 0 && handler.ServerSideBatching.MaxBatchSize > handler.ThreadsPerProcess { - return ErrorInsufficientBatchConcurrencyLevelInf(handler.ServerSideBatching.MaxBatchSize, handler.ThreadsPerProcess) - } - } - - if handler.MultiModelReloading != nil { - return ErrorFieldNotSupportedByHandlerType(userconfig.MultiModelReloadingKey, userconfig.PythonHandlerType) - } + containers := api.Containers - hasSingleModel := handler.Models.Path != nil - hasMultiModels := !hasSingleModel - - var modelWrapError func(error) error - var modelResources []userconfig.ModelResource - - if hasSingleModel { - modelWrapError = func(err error) error { - return errors.Wrap(err, userconfig.ModelsPathKey) - } - modelResources = []userconfig.ModelResource{ - { - Name: consts.SingleModelName, - Path: *handler.Models.Path, - SignatureKey: handler.Models.SignatureKey, - }, - } - *handler.Models.Path = s.EnsureSuffix(*handler.Models.Path, "/") - } - if hasMultiModels { - if len(handler.Models.Paths) > 0 { - modelWrapError = func(err error) error { - return errors.Wrap(err, userconfig.ModelsKey, userconfig.ModelsPathsKey) - } - - for _, path := range handler.Models.Paths { - if path.SignatureKey == nil && handler.Models.SignatureKey != nil { - path.SignatureKey = handler.Models.SignatureKey - } - (*path).Path = s.EnsureSuffix((*path).Path, "/") - modelResources = append(modelResources, *path) - } + numPorts := 0 + for i, container := range containers { + if err := validateDockerImagePath(container.Image, awsClient, k8sClient); err != nil { + return errors.Wrap(err, strconv.FormatInt(int64(i), 10), userconfig.ImageKey) } - if handler.Models.Dir != nil { - modelWrapError = func(err error) error { - return errors.Wrap(err, userconfig.ModelsKey, userconfig.ModelsDirKey) - } + if container.Port != nil { + numPorts++ } - } - - var validators []modelValidator - if api.Compute.Inf == 0 { - validators = append(validators, tensorflowModelValidator) - } else { - validators = append(validators, tensorflowNeuronModelValidator) - } - var err error - if hasMultiModels && handler.Models.Dir != nil { - *models, err = validateDirModels(*handler.Models.Dir, handler.Models.SignatureKey, awsClient, generateErrorForHandlerTypeFn(api), validators) - } else { - *models, err = validateModels(modelResources, handler.Models.SignatureKey, awsClient, generateErrorForHandlerTypeFn(api), validators) - } - if err != nil { - return modelWrapError(err) - } - - if hasMultiModels { - for _, model := range *models { - if model.Name == consts.SingleModelName { - return modelWrapError(ErrorReservedModelName(model.Name)) + for key := range container.Env { + if strings.HasPrefix(key, "CORTEX_") { + return errors.Wrap(ErrorCortexPrefixedEnvVarNotAllowed(), strconv.FormatInt(int64(i), 10), userconfig.EnvKey, key) } } } - - if err := checkDuplicateModelNames(*models); err != nil { - return modelWrapError(err) + if numPorts != 1 { + return ErrorPortCanOnlyDefinedOnce() } - return nil -} - -func validateProtobufPath(api *userconfig.API, projectFiles ProjectFiles) error { - apiName := api.Name - protobufPath := *api.Handler.ProtobufPath - - if !projectFiles.HasFile(protobufPath) { - return errors.Wrap(files.ErrorFileDoesNotExist(protobufPath), userconfig.ProtobufPathKey) - } - protoBytes, err := projectFiles.GetFile(protobufPath) - if err != nil { - return errors.Wrap(err, userconfig.ProtobufPathKey, *api.Handler.ProtobufPath) - } - - protoReader := bytes.NewReader(protoBytes) - parser := pbparser.NewParser(protoReader) - proto, err := parser.Parse() - if err != nil { - return errors.Wrap(errors.WithStack(err), userconfig.ProtobufPathKey, *api.Handler.ProtobufPath) - } - - var packageName string - var serviceName string - - numServices := 0 - pbparser.Walk(proto, - pbparser.WithPackage(func(pkg *pbparser.Package) { - packageName = pkg.Name - }), - pbparser.WithService(func(service *pbparser.Service) { - numServices++ - serviceName = service.Name - }), - ) - - if numServices != 1 { - return errors.Wrap(ErrorProtoNumServicesMismatch(numServices), userconfig.ProtobufPathKey, *api.Handler.ProtobufPath) - } - - var requiredPackageName string - requiredPackageName = strings.ReplaceAll(apiName, "-", "_") - - if api.Handler.ServerSideBatching != nil { - return ErrorConflictingFields(userconfig.ProtobufPathKey, userconfig.ServerSideBatchingKey) - } - - if packageName == "" { - return errors.Wrap(ErrorProtoMissingPackageName(requiredPackageName), userconfig.ProtobufPathKey, *api.Handler.ProtobufPath) - } - if packageName != requiredPackageName { - return errors.Wrap(ErrorProtoInvalidPackageName(packageName, requiredPackageName), userconfig.ProtobufPathKey, *api.Handler.ProtobufPath) - } - - requiredEndpoint := "/" + requiredPackageName + "." + serviceName - if api.Networking.Endpoint == nil { - api.Networking.Endpoint = pointer.String(requiredEndpoint) - } - if *api.Networking.Endpoint != requiredEndpoint { - return ErrorProtoInvalidNetworkingEndpoint(requiredEndpoint) - } - - return nil -} - -func validatePythonPath(handler *userconfig.Handler, projectFiles ProjectFiles) error { - if !projectFiles.HasDir(*handler.PythonPath) { - return ErrorPythonPathNotFound(*handler.PythonPath) + if err := validateCompute(GetTotalComputeFromContainers(containers)); err != nil { + return err } return nil @@ -1322,9 +600,7 @@ func validateAutoscaling(api *userconfig.API) error { return nil } -func validateCompute(api *userconfig.API) error { - compute := api.Compute - +func validateCompute(compute userconfig.Compute) error { if compute.GPU > 0 && compute.Inf > 0 { return ErrorComputeResourceConflict(userconfig.GPUKey, userconfig.InfKey) } diff --git a/pkg/types/userconfig/api.go b/pkg/types/userconfig/api.go index a38e09de1b..13cf707434 100644 --- a/pkg/types/userconfig/api.go +++ b/pkg/types/userconfig/api.go @@ -21,7 +21,6 @@ import ( "strings" "time" - "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" s "github.com/cortexlabs/cortex/pkg/lib/strings" @@ -32,11 +31,16 @@ import ( type API struct { Resource + + NodeGroups []string `json:"node_groups" yaml:"node_groups"` + ShmSize *k8s.Quantity `json:"shm_size" yaml:"shm_size"` + PythonPath *string `json:"python_path" yaml:"python_path"` + LogLevel LogLevel `json:"log_level" yaml:"log_level"` + Config map[string]interface{} `json:"config" yaml:"config"` + + Containers []Container `json:"containers" yaml:"containers"` APIs []*TrafficSplit `json:"apis" yaml:"apis"` - Handler *Handler `json:"handler" yaml:"handler"` - TaskDefinition *TaskDefinition `json:"definition" yaml:"definition"` Networking *Networking `json:"networking" yaml:"networking"` - Compute *Compute `json:"compute" yaml:"compute"` Autoscaling *Autoscaling `json:"autoscaling" yaml:"autoscaling"` UpdateStrategy *UpdateStrategy `json:"update_strategy" yaml:"update_strategy"` Index int `json:"index" yaml:"-"` @@ -44,45 +48,16 @@ type API struct { SubmittedAPISpec interface{} `json:"submitted_api_spec" yaml:"submitted_api_spec"` } -type Handler struct { - Type HandlerType `json:"type" yaml:"type"` - Path string `json:"path" yaml:"path"` - ProtobufPath *string `json:"protobuf_path" yaml:"protobuf_path"` - - MultiModelReloading *MultiModels `json:"multi_model_reloading" yaml:"multi_model_reloading"` - Models *MultiModels `json:"models" yaml:"models"` - - ServerSideBatching *ServerSideBatching `json:"server_side_batching" yaml:"server_side_batching"` - ProcessesPerReplica int32 `json:"processes_per_replica" yaml:"processes_per_replica"` - ThreadsPerProcess int32 `json:"threads_per_process" yaml:"threads_per_process"` - ShmSize *k8s.Quantity `json:"shm_size" yaml:"shm_size"` - PythonPath *string `json:"python_path" yaml:"python_path"` - LogLevel LogLevel `json:"log_level" yaml:"log_level"` - Image string `json:"image" yaml:"image"` - TensorFlowServingImage string `json:"tensorflow_serving_image" yaml:"tensorflow_serving_image"` - Config map[string]interface{} `json:"config" yaml:"config"` - Env map[string]string `json:"env" yaml:"env"` - Dependencies *Dependencies `json:"dependencies" yaml:"dependencies"` -} +type Container struct { + Name string `json:"name" yaml:"name"` + Image string `json:"image" yaml:"image"` + Env map[string]string `json:"env" yaml:"env"` -type TaskDefinition struct { - Path string `json:"path" yaml:"path"` - PythonPath *string `json:"python_path" yaml:"python_path"` - Image string `json:"image" yaml:"image"` - ShmSize *k8s.Quantity `json:"shm_size" yaml:"shm_size"` - LogLevel LogLevel `json:"log_level" yaml:"log_level"` - Config map[string]interface{} `json:"config" yaml:"config"` - Env map[string]string `json:"env" yaml:"env"` - Dependencies *Dependencies `json:"dependencies" yaml:"dependencies"` -} + Port *int32 `json:"port" yaml:"port"` + Command []string `json:"command" yaml:"command"` + Args []string `json:"args" yaml:"args"` -type MultiModels struct { - Path *string `json:"path" yaml:"path"` - Paths []*ModelResource `json:"paths" yaml:"paths"` - Dir *string `json:"dir" yaml:"dir"` - CacheSize *int32 `json:"cache_size" yaml:"cache_size"` - DiskCacheSize *int32 `json:"disk_cache_size" yaml:"disk_cache_size"` - SignatureKey *string `json:"signature_key" yaml:"signature_key"` + Compute *Compute `json:"compute" yaml:"compute"` } type TrafficSplit struct { @@ -91,33 +66,15 @@ type TrafficSplit struct { Shadow bool `json:"shadow" yaml:"shadow"` } -type ModelResource struct { - Name string `json:"name" yaml:"name"` - Path string `json:"path" yaml:"path"` - SignatureKey *string `json:"signature_key" yaml:"signature_key"` -} - -type ServerSideBatching struct { - MaxBatchSize int32 `json:"max_batch_size" yaml:"max_batch_size"` - BatchInterval time.Duration `json:"batch_interval" yaml:"batch_interval"` -} - -type Dependencies struct { - Pip string `json:"pip" yaml:"pip"` - Conda string `json:"conda" yaml:"conda"` - Shell string `json:"shell" yaml:"shell"` -} - type Networking struct { Endpoint *string `json:"endpoint" yaml:"endpoint"` } type Compute struct { - CPU *k8s.Quantity `json:"cpu" yaml:"cpu"` - Mem *k8s.Quantity `json:"mem" yaml:"mem"` - GPU int64 `json:"gpu" yaml:"gpu"` - Inf int64 `json:"inf" yaml:"inf"` - NodeGroups []string `json:"node_groups" yaml:"node_groups"` + CPU *k8s.Quantity `json:"cpu" yaml:"cpu"` + Mem *k8s.Quantity `json:"mem" yaml:"mem"` + GPU int64 `json:"gpu" yaml:"gpu"` + Inf int64 `json:"inf" yaml:"inf"` } type Autoscaling struct { @@ -144,72 +101,6 @@ func (api *API) Identify() string { return IdentifyAPI(api.FileName, api.Name, api.Kind, api.Index) } -func (api *API) ModelNames() []string { - names := []string{} - for _, model := range api.Handler.Models.Paths { - names = append(names, model.Name) - } - return names -} - -func (api *API) ApplyDefaultDockerPaths() { - usesGPU := api.Compute.GPU > 0 - usesInf := api.Compute.Inf > 0 - - switch api.Kind { - case RealtimeAPIKind, BatchAPIKind, AsyncAPIKind: - api.applyHandlerDefaultDockerPaths(usesGPU, usesInf) - case TaskAPIKind: - api.applyTaskDefaultDockerPaths(usesGPU, usesInf) - } -} - -func (api *API) applyHandlerDefaultDockerPaths(usesGPU, usesInf bool) { - handler := api.Handler - switch handler.Type { - case PythonHandlerType: - if handler.Image == "" { - if usesGPU { - handler.Image = consts.DefaultImagePythonHandlerGPU - } else if usesInf { - handler.Image = consts.DefaultImagePythonHandlerInf - } else { - handler.Image = consts.DefaultImagePythoHandlerCPU - } - } - case TensorFlowHandlerType: - if handler.Image == "" { - handler.Image = consts.DefaultImageTensorFlowHandler - } - if handler.TensorFlowServingImage == "" { - if usesGPU { - handler.TensorFlowServingImage = consts.DefaultImageTensorFlowServingGPU - } else if usesInf { - handler.TensorFlowServingImage = consts.DefaultImageTensorFlowServingInf - } else { - handler.TensorFlowServingImage = consts.DefaultImageTensorFlowServingCPU - } - } - } -} - -func (api *API) applyTaskDefaultDockerPaths(usesGPU, usesInf bool) { - task := api.TaskDefinition - if task.Image == "" { - if usesGPU { - task.Image = consts.DefaultImagePythonHandlerGPU - } else if usesInf { - task.Image = consts.DefaultImagePythonHandlerInf - } else { - task.Image = consts.DefaultImagePythoHandlerCPU - } - } -} - -func (handler *Handler) IsGRPC() bool { - return handler.ProtobufPath != nil -} - func IdentifyAPI(filePath string, name string, kind Kind, index int) string { str := "" diff --git a/pkg/types/userconfig/config_key.go b/pkg/types/userconfig/config_key.go index 70113ae0b7..ccc0fb9b3d 100644 --- a/pkg/types/userconfig/config_key.go +++ b/pkg/types/userconfig/config_key.go @@ -32,6 +32,10 @@ const ( WeightKey = "weight" ShadowKey = "shadow" + // Containers + ContainersKey = "containers" + PortKey = "port" + // Handler TypeKey = "type" PathKey = "path" diff --git a/pkg/types/userconfig/handler_type.go b/pkg/types/userconfig/handler_type.go deleted file mode 100644 index c4cacb2d83..0000000000 --- a/pkg/types/userconfig/handler_type.go +++ /dev/null @@ -1,88 +0,0 @@ -/* -Copyright 2021 Cortex Labs, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package userconfig - -type HandlerType int - -const ( - UnknownHandlerType HandlerType = iota - PythonHandlerType - TensorFlowHandlerType -) - -var _handlerTypes = []string{ - "unknown", - "python", - "tensorflow", -} - -var _casedHandlerTypes = []string{ - "unknown", - "Python", - "TensorFlow", -} - -func HandlerTypeFromString(s string) HandlerType { - for i := 0; i < len(_handlerTypes); i++ { - if s == _handlerTypes[i] { - return HandlerType(i) - } - } - return UnknownHandlerType -} - -func HandlerTypeStrings() []string { - return _handlerTypes[1:] -} - -func (t HandlerType) String() string { - return _handlerTypes[t] -} - -func (t HandlerType) CasedString() string { - return _casedHandlerTypes[t] -} - -// MarshalText satisfies TextMarshaler -func (t HandlerType) MarshalText() ([]byte, error) { - return []byte(t.String()), nil -} - -// UnmarshalText satisfies TextUnmarshaler -func (t *HandlerType) UnmarshalText(text []byte) error { - enum := string(text) - for i := 0; i < len(_handlerTypes); i++ { - if enum == _handlerTypes[i] { - *t = HandlerType(i) - return nil - } - } - - *t = UnknownHandlerType - return nil -} - -// UnmarshalBinary satisfies BinaryUnmarshaler -// Needed for msgpack -func (t *HandlerType) UnmarshalBinary(data []byte) error { - return t.UnmarshalText(data) -} - -// MarshalBinary satisfies BinaryMarshaler -func (t HandlerType) MarshalBinary() ([]byte, error) { - return []byte(t.String()), nil -} diff --git a/pkg/types/userconfig/model_structure_type.go b/pkg/types/userconfig/model_structure_type.go deleted file mode 100644 index 038978687b..0000000000 --- a/pkg/types/userconfig/model_structure_type.go +++ /dev/null @@ -1,25 +0,0 @@ -/* -Copyright 2021 Cortex Labs, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package userconfig - -type ModelStructureType int - -const ( - UnknownModelStructureType ModelStructureType = iota - NonVersionedModelType - VersionedModelType -) From 22067dcd004c0e0ab2173b4db3a092417990e57f Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Thu, 13 May 2021 15:33:14 +0300 Subject: [PATCH 03/82] Remove gRPC --- cli/cmd/lib_realtime_apis.go | 261 +----------------- pkg/operator/operator/k8s.go | 5 - pkg/operator/resources/errors.go | 34 +-- pkg/operator/resources/realtimeapi/api.go | 7 - .../resources/realtimeapi/k8s_specs.go | 80 ++---- pkg/operator/resources/resources.go | 79 +----- pkg/operator/resources/validations.go | 28 +- pkg/operator/schema/schema.go | 49 +--- pkg/types/spec/validations.go | 2 +- 9 files changed, 50 insertions(+), 495 deletions(-) diff --git a/cli/cmd/lib_realtime_apis.go b/cli/cmd/lib_realtime_apis.go index b61ef1fc10..8f384bd658 100644 --- a/cli/cmd/lib_realtime_apis.go +++ b/cli/cmd/lib_realtime_apis.go @@ -17,25 +17,17 @@ limitations under the License. package cmd import ( - "crypto/tls" "fmt" - "io/ioutil" - "net/http" - "strconv" "strings" "time" "github.com/cortexlabs/cortex/cli/types/cliconfig" - "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/console" - "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/lib/table" libtime "github.com/cortexlabs/cortex/pkg/lib/time" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/metrics" - "github.com/cortexlabs/cortex/pkg/types/status" - "github.com/cortexlabs/cortex/pkg/types/userconfig" ) func realtimeAPITable(realtimeAPI schema.APIResponse, env cliconfig.Environment) (string, error) { @@ -51,19 +43,7 @@ func realtimeAPITable(realtimeAPI schema.APIResponse, env cliconfig.Environment) out += "\n" + console.Bold("metrics dashboard: ") + *realtimeAPI.DashboardURL + "\n" } - if realtimeAPI.Spec.Handler.IsGRPC() { - out += "\n" + console.Bold("insecure endpoint: ") + fmt.Sprintf("%s:%d", realtimeAPI.Endpoint, realtimeAPI.GRPCPorts["insecure"]) - out += "\n" + console.Bold("secure endpoint: ") + fmt.Sprintf("%s:%d", realtimeAPI.Endpoint, realtimeAPI.GRPCPorts["secure"]) + "\n" - } else { - out += "\n" + console.Bold("endpoint: ") + realtimeAPI.Endpoint + "\n" - } - - if !(realtimeAPI.Spec.Handler.Type == userconfig.PythonHandlerType && realtimeAPI.Spec.Handler.MultiModelReloading == nil) && realtimeAPI.Spec.Handler.ProtobufPath == nil { - decribedModels := describeModelInput(realtimeAPI.Status, realtimeAPI.RealtimeModelMetadata.TFModelSummary, realtimeAPI.RealtimeModelMetadata.PythonModelSummary) - if decribedModels != "" { - out += "\n" + decribedModels - } - } + out += "\n" + console.Bold("endpoint: ") + realtimeAPI.Endpoint + "\n" out += "\n" + apiHistoryTable(realtimeAPI.APIVersions) @@ -159,242 +139,3 @@ func code5XXStr(metrics *metrics.Metrics) string { } return s.Int(metrics.NetworkStats.Code5XX) } - -func describeModelInput(status *status.Status, apiTFLiveReloadingSummary *schema.TFLiveReloadingSummary, apiModelSummary *schema.PythonModelSummary) string { - if status.Updated.Ready+status.Stale.Ready == 0 { - return "the models' metadata schema will be available when the api is live\n" - } - - if apiTFLiveReloadingSummary != nil { - t, err := parseAPITFLiveReloadingSummary(apiTFLiveReloadingSummary) - if err != nil { - return "error parsing the model's input schema: " + errors.Message(err) + "\n" - } - return t - } - - if apiModelSummary != nil { - t, err := parseAPIModelSummary(apiModelSummary) - if err != nil { - return "error parsing the models' metadata schema: " + errors.Message(err) + "\n" - } - return t - } - - return "" -} - -func getModelFromModelID(modelID string) (modelName string, modelVersion int64, err error) { - splitIndex := strings.LastIndex(modelID, "-") - modelName = modelID[:splitIndex] - modelVersion, err = strconv.ParseInt(modelID[splitIndex+1:], 10, 64) - return -} - -func makeRequest(request *http.Request) (http.Header, []byte, error) { - client := http.Client{ - Timeout: 5 * time.Second, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }, - } - - response, err := client.Do(request) - if err != nil { - return nil, nil, errors.Wrap(err, errStrFailedToConnect(*request.URL)) - } - defer response.Body.Close() - - if response.StatusCode != 200 { - bodyBytes, err := ioutil.ReadAll(response.Body) - if err != nil { - return nil, nil, errors.Wrap(err, _errStrRead) - } - return nil, nil, ErrorResponseUnknown(string(bodyBytes), response.StatusCode) - } - - bodyBytes, err := ioutil.ReadAll(response.Body) - if err != nil { - return nil, nil, errors.Wrap(err, _errStrRead) - } - return response.Header, bodyBytes, nil -} - -func parseAPIModelSummary(summary *schema.PythonModelSummary) (string, error) { - rows := make([][]interface{}, 0) - - for modelName, modelMetadata := range summary.ModelMetadata { - latestVersion := int64(0) - for _, version := range modelMetadata.Versions { - v, err := strconv.ParseInt(version, 10, 64) - if err != nil { - return "", err - } - if v > latestVersion { - latestVersion = v - } - } - latestStrVersion := strconv.FormatInt(latestVersion, 10) - - for idx, version := range modelMetadata.Versions { - var latestTag string - if latestStrVersion == version { - latestTag = " (latest)" - } - - timestamp := modelMetadata.Timestamps[idx] - date := time.Unix(timestamp, 0) - - rows = append(rows, []interface{}{ - modelName, - version + latestTag, - date.Format(_timeFormat), - }) - } - } - - _, usesCortexDefaultModelName := summary.ModelMetadata[consts.SingleModelName] - - t := table.Table{ - Headers: []table.Header{ - { - Title: "model name", - MaxWidth: 32, - Hidden: usesCortexDefaultModelName, - }, - { - Title: "model version", - MaxWidth: 25, - }, - { - Title: "edit time", - MaxWidth: 32, - }, - }, - Rows: rows, - } - - return t.MustFormat(), nil -} - -func parseAPITFLiveReloadingSummary(summary *schema.TFLiveReloadingSummary) (string, error) { - latestVersions := make(map[string]int64) - - numRows := 0 - models := make(map[string]schema.GenericModelMetadata, 0) - for modelID, modelMetadata := range summary.ModelMetadata { - timestamp := modelMetadata.Timestamp - modelName, modelVersion, err := getModelFromModelID(modelID) - if err != nil { - return "", err - } - if _, ok := models[modelName]; !ok { - models[modelName] = schema.GenericModelMetadata{ - Versions: []string{strconv.FormatInt(modelVersion, 10)}, - Timestamps: []int64{timestamp}, - } - } else { - model := models[modelName] - model.Versions = append(model.Versions, strconv.FormatInt(modelVersion, 10)) - model.Timestamps = append(model.Timestamps, timestamp) - models[modelName] = model - } - if _, ok := latestVersions[modelName]; !ok { - latestVersions[modelName] = modelVersion - } else if modelVersion > latestVersions[modelName] { - latestVersions[modelName] = modelVersion - } - numRows += len(modelMetadata.InputSignatures) - } - - rows := make([][]interface{}, 0, numRows) - for modelName, model := range models { - latestVersion := latestVersions[modelName] - - for _, modelVersion := range model.Versions { - modelID := fmt.Sprintf("%s-%s", modelName, modelVersion) - - inputSignatures := summary.ModelMetadata[modelID].InputSignatures - timestamp := summary.ModelMetadata[modelID].Timestamp - versionInt, err := strconv.ParseInt(modelVersion, 10, 64) - if err != nil { - return "", err - } - - var applicableTags string - if versionInt == latestVersion { - applicableTags = " (latest)" - } - - date := time.Unix(timestamp, 0) - - for inputName, inputSignature := range inputSignatures { - shapeStr := make([]string, len(inputSignature.Shape)) - for idx, dim := range inputSignature.Shape { - shapeStr[idx] = s.ObjFlatNoQuotes(dim) - } - shapeRowEntry := "" - if len(shapeStr) == 1 && shapeStr[0] == "scalar" { - shapeRowEntry = "scalar" - } else if len(shapeStr) == 1 && shapeStr[0] == "unknown" { - shapeRowEntry = "unknown" - } else { - shapeRowEntry = "(" + strings.Join(shapeStr, ", ") + ")" - } - rows = append(rows, []interface{}{ - modelName, - modelVersion + applicableTags, - inputName, - inputSignature.Type, - shapeRowEntry, - date.Format(_timeFormat), - }) - } - } - } - - usesCortexDefaultModelName := false - for modelID := range summary.ModelMetadata { - modelName, _, err := getModelFromModelID(modelID) - if err != nil { - return "", err - } - if modelName == consts.SingleModelName { - usesCortexDefaultModelName = true - break - } - } - - t := table.Table{ - Headers: []table.Header{ - { - Title: "model name", - MaxWidth: 32, - Hidden: usesCortexDefaultModelName, - }, - { - Title: "model version", - MaxWidth: 25, - }, - { - Title: "model input", - MaxWidth: 32, - }, - { - Title: "type", - MaxWidth: 10, - }, - { - Title: "shape", - MaxWidth: 20, - }, - { - Title: "edit time", - MaxWidth: 32, - }, - }, - Rows: rows, - } - - return t.MustFormat(), nil -} diff --git a/pkg/operator/operator/k8s.go b/pkg/operator/operator/k8s.go index daa2f43396..f9536596ce 100644 --- a/pkg/operator/operator/k8s.go +++ b/pkg/operator/operator/k8s.go @@ -61,10 +61,5 @@ func APIEndpoint(api *spec.API) (string, error) { } baseAPIEndpoint = strings.Replace(baseAPIEndpoint, "https://", "http://", 1) - if api.Handler != nil && api.Handler.IsGRPC() { - baseAPIEndpoint = strings.Replace(baseAPIEndpoint, "http://", "", 1) - return baseAPIEndpoint, nil - } - return urls.Join(baseAPIEndpoint, *api.Networking.Endpoint), nil } diff --git a/pkg/operator/resources/errors.go b/pkg/operator/resources/errors.go index ecc99381aa..6ad110ff44 100644 --- a/pkg/operator/resources/errors.go +++ b/pkg/operator/resources/errors.go @@ -27,17 +27,15 @@ import ( ) const ( - ErrOperationIsOnlySupportedForKind = "resources.operation_is_only_supported_for_kind" - ErrAPINotDeployed = "resources.api_not_deployed" - ErrAPIIDNotFound = "resources.api_id_not_found" - ErrCannotChangeTypeOfDeployedAPI = "resources.cannot_change_kind_of_deployed_api" - ErrCannotChangeProtocolWhenUsedByTrafficSplitter = "resources.cannot_change_protocol_when_used_by_traffic_splitter" - ErrNoAvailableNodeComputeLimit = "resources.no_available_node_compute_limit" - ErrJobIDRequired = "resources.job_id_required" - ErrRealtimeAPIUsedByTrafficSplitter = "resources.realtime_api_used_by_traffic_splitter" - ErrAPIsNotDeployed = "resources.apis_not_deployed" - ErrGRPCNotSupportedForTrafficSplitter = "resources.grpc_not_supported_for_traffic_splitter" - ErrInvalidNodeGroupSelector = "resources.invalid_node_group_selector" + ErrOperationIsOnlySupportedForKind = "resources.operation_is_only_supported_for_kind" + ErrAPINotDeployed = "resources.api_not_deployed" + ErrAPIIDNotFound = "resources.api_id_not_found" + ErrCannotChangeTypeOfDeployedAPI = "resources.cannot_change_kind_of_deployed_api" + ErrNoAvailableNodeComputeLimit = "resources.no_available_node_compute_limit" + ErrJobIDRequired = "resources.job_id_required" + ErrRealtimeAPIUsedByTrafficSplitter = "resources.realtime_api_used_by_traffic_splitter" + ErrAPIsNotDeployed = "resources.apis_not_deployed" + ErrInvalidNodeGroupSelector = "resources.invalid_node_group_selector" ) func ErrorOperationIsOnlySupportedForKind(resource operator.DeployedResource, supportedKind userconfig.Kind, supportedKinds ...userconfig.Kind) error { @@ -75,13 +73,6 @@ func ErrorCannotChangeKindOfDeployedAPI(name string, newKind, prevKind userconfi }) } -func ErrorCannotChangeProtocolWhenUsedByTrafficSplitter(protocolChangingAPIName string, trafficSplitters []string) error { - return errors.WithStack(&errors.Error{ - Kind: ErrCannotChangeProtocolWhenUsedByTrafficSplitter, - Message: fmt.Sprintf("cannot change the serving protocol (http -> grpc) of api %s because it is used by the following %s: %s", protocolChangingAPIName, strings.PluralS("TrafficSplitter", len(trafficSplitters)), strings.StrsSentence(trafficSplitters, "")), - }) -} - func ErrorNoAvailableNodeComputeLimit(resource string, reqStr string, maxStr string) error { message := fmt.Sprintf("no instances can satisfy the requested %s quantity - requested %s %s but instances only have %s %s available", resource, reqStr, resource, maxStr, resource) if maxStr == "0" { @@ -111,13 +102,6 @@ func ErrorAPIsNotDeployed(notDeployedAPIs []string) error { }) } -func ErrorGRPCNotSupportedForTrafficSplitter(grpcAPIName string) error { - return errors.WithStack(&errors.Error{ - Kind: ErrGRPCNotSupportedForTrafficSplitter, - Message: fmt.Sprintf("api %s (of kind %s) is served using the grpc protocol and therefore, it cannot be used for the %s kind", grpcAPIName, userconfig.RealtimeAPIKind, userconfig.TrafficSplitterKind), - }) -} - func ErrorInvalidNodeGroupSelector(selected string, availableNodeGroups []string) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidNodeGroupSelector, diff --git a/pkg/operator/resources/realtimeapi/api.go b/pkg/operator/resources/realtimeapi/api.go index 8dee7d132d..ccf42044e9 100644 --- a/pkg/operator/resources/realtimeapi/api.go +++ b/pkg/operator/resources/realtimeapi/api.go @@ -257,19 +257,12 @@ func GetAPIByName(deployedResource *operator.DeployedResource) ([]schema.APIResp dashboardURL := pointer.String(getDashboardURL(api.Name)) - grpcPorts := map[string]int64{} - if api.Handler != nil && api.Handler.IsGRPC() { - grpcPorts["insecure"] = 80 - grpcPorts["secure"] = 443 - } - return []schema.APIResponse{ { Spec: *api, Status: status, Metrics: metrics, Endpoint: apiEndpoint, - GRPCPorts: grpcPorts, DashboardURL: dashboardURL, }, }, nil diff --git a/pkg/operator/resources/realtimeapi/k8s_specs.go b/pkg/operator/resources/realtimeapi/k8s_specs.go index 124e501ce8..9fa79e69a0 100644 --- a/pkg/operator/resources/realtimeapi/k8s_specs.go +++ b/pkg/operator/resources/realtimeapi/k8s_specs.go @@ -44,25 +44,19 @@ func tensorflowAPISpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.D containers, volumes := workloads.TensorFlowHandlerContainers(api) containers = append(containers, workloads.RequestMonitorContainer(api)) - servingProtocol := "http" - if api.Handler != nil && api.Handler.IsGRPC() { - servingProtocol = "grpc" - } - return k8s.Deployment(&k8s.DeploymentSpec{ Name: workloads.K8sName(api.Name), Replicas: getRequestedReplicasFromDeployment(api, prevDeployment), MaxSurge: pointer.String(api.UpdateStrategy.MaxSurge), MaxUnavailable: pointer.String(api.UpdateStrategy.MaxUnavailable), Labels: map[string]string{ - "apiName": api.Name, - "apiKind": api.Kind.String(), - "apiID": api.ID, - "specID": api.SpecID, - "deploymentID": api.DeploymentID, - "handlerID": api.HandlerID, - "servingProtocol": servingProtocol, - "cortex.dev/api": "true", + "apiName": api.Name, + "apiKind": api.Kind.String(), + "apiID": api.ID, + "specID": api.SpecID, + "deploymentID": api.DeploymentID, + "handlerID": api.HandlerID, + "cortex.dev/api": "true", }, Annotations: api.ToK8sAnnotations(), Selector: map[string]string{ @@ -71,12 +65,11 @@ func tensorflowAPISpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.D }, PodSpec: k8s.PodSpec{ Labels: map[string]string{ - "apiName": api.Name, - "apiKind": api.Kind.String(), - "deploymentID": api.DeploymentID, - "handlerID": api.HandlerID, - "servingProtocol": servingProtocol, - "cortex.dev/api": "true", + "apiName": api.Name, + "apiKind": api.Kind.String(), + "deploymentID": api.DeploymentID, + "handlerID": api.HandlerID, + "cortex.dev/api": "true", }, Annotations: map[string]string{ "traffic.sidecar.istio.io/excludeOutboundIPRanges": "0.0.0.0/0", @@ -102,25 +95,19 @@ func pythonAPISpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Deplo containers, volumes := workloads.PythonHandlerContainers(api) containers = append(containers, workloads.RequestMonitorContainer(api)) - servingProtocol := "http" - if api.Handler != nil && api.Handler.IsGRPC() { - servingProtocol = "grpc" - } - return k8s.Deployment(&k8s.DeploymentSpec{ Name: workloads.K8sName(api.Name), Replicas: getRequestedReplicasFromDeployment(api, prevDeployment), MaxSurge: pointer.String(api.UpdateStrategy.MaxSurge), MaxUnavailable: pointer.String(api.UpdateStrategy.MaxUnavailable), Labels: map[string]string{ - "apiName": api.Name, - "apiKind": api.Kind.String(), - "apiID": api.ID, - "specID": api.SpecID, - "deploymentID": api.DeploymentID, - "handlerID": api.HandlerID, - "servingProtocol": servingProtocol, - "cortex.dev/api": "true", + "apiName": api.Name, + "apiKind": api.Kind.String(), + "apiID": api.ID, + "specID": api.SpecID, + "deploymentID": api.DeploymentID, + "handlerID": api.HandlerID, + "cortex.dev/api": "true", }, Annotations: api.ToK8sAnnotations(), Selector: map[string]string{ @@ -129,12 +116,11 @@ func pythonAPISpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Deplo }, PodSpec: k8s.PodSpec{ Labels: map[string]string{ - "apiName": api.Name, - "apiKind": api.Kind.String(), - "deploymentID": api.DeploymentID, - "handlerID": api.HandlerID, - "servingProtocol": servingProtocol, - "cortex.dev/api": "true", + "apiName": api.Name, + "apiKind": api.Kind.String(), + "deploymentID": api.DeploymentID, + "handlerID": api.HandlerID, + "cortex.dev/api": "true", }, Annotations: map[string]string{ "traffic.sidecar.istio.io/excludeOutboundIPRanges": "0.0.0.0/0", @@ -157,21 +143,16 @@ func pythonAPISpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Deplo } func serviceSpec(api *spec.API) *kcore.Service { - servingProtocol := "http" - if api.Handler != nil && api.Handler.IsGRPC() { - servingProtocol = "grpc" - } return k8s.Service(&k8s.ServiceSpec{ Name: workloads.K8sName(api.Name), - PortName: servingProtocol, + PortName: "http", Port: workloads.DefaultPortInt32, TargetPort: workloads.DefaultPortInt32, Annotations: api.ToK8sAnnotations(), Labels: map[string]string{ - "apiName": api.Name, - "apiKind": api.Kind.String(), - "servingProtocol": servingProtocol, - "cortex.dev/api": "true", + "apiName": api.Name, + "apiKind": api.Kind.String(), + "cortex.dev/api": "true", }, Selector: map[string]string{ "apiName": api.Name, @@ -184,11 +165,6 @@ func virtualServiceSpec(api *spec.API) *istioclientnetworking.VirtualService { servingProtocol := "http" rewritePath := pointer.String("/") - if api.Handler != nil && api.Handler.IsGRPC() { - servingProtocol = "grpc" - rewritePath = nil - } - return k8s.VirtualService(&k8s.VirtualServiceSpec{ Name: workloads.K8sName(api.Name), Gateways: []string{"apis-gateway"}, diff --git a/pkg/operator/resources/resources.go b/pkg/operator/resources/resources.go index 78fea8b5e9..10168d077c 100644 --- a/pkg/operator/resources/resources.go +++ b/pkg/operator/resources/resources.go @@ -31,7 +31,6 @@ import ( "github.com/cortexlabs/cortex/pkg/lib/parallel" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/telemetry" - "github.com/cortexlabs/cortex/pkg/lib/urls" "github.com/cortexlabs/cortex/pkg/operator/lib/routines" "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/operator/resources/asyncapi" @@ -138,40 +137,6 @@ func UpdateAPI(apiConfig *userconfig.API, projectID string, force bool) (*schema return nil, "", ErrorCannotChangeKindOfDeployedAPI(apiConfig.Name, apiConfig.Kind, deployedResource.Kind) } - if deployedResource != nil { - prevAPISpec, err := operator.DownloadAPISpec(deployedResource.Name, deployedResource.ID()) - if err != nil { - return nil, "", err - } - - if deployedResource.Kind == userconfig.RealtimeAPIKind && prevAPISpec != nil && !prevAPISpec.Handler.IsGRPC() && apiConfig.Handler.IsGRPC() { - realtimeAPIName := deployedResource.Name - - virtualServices, err := config.K8s.ListVirtualServicesByLabel("apiKind", userconfig.TrafficSplitterKind.String()) - if err != nil { - return nil, "", err - } - - trafficSplitterList, err := trafficsplitter.GetAllAPIs(virtualServices) - if err != nil { - return nil, "", err - } - - dependentTrafficSplitters := []string{} - for _, trafficSplitter := range trafficSplitterList { - for _, api := range trafficSplitter.Spec.APIs { - if realtimeAPIName == api.Name { - dependentTrafficSplitters = append(dependentTrafficSplitters, api.Name) - } - } - } - - if len(dependentTrafficSplitters) > 0 { - return nil, "", ErrorCannotChangeProtocolWhenUsedByTrafficSplitter(realtimeAPIName, dependentTrafficSplitters) - } - } - } - telemetry.Event("operator.deploy", apiConfig.TelemetryEvent()) var api *spec.API @@ -277,39 +242,12 @@ func patchAPI(apiConfig *userconfig.API, force bool) (*spec.API, string, error) } } - err = ValidateClusterAPIs([]userconfig.API{*apiConfig}, projectFiles) + err = ValidateClusterAPIs([]userconfig.API{*apiConfig}) if err != nil { err = errors.Append(err, fmt.Sprintf("\n\napi configuration schema can be found at https://docs.cortex.dev/v/%s/", consts.CortexVersionMinor)) return nil, "", err } - if deployedResource.Kind == userconfig.RealtimeAPIKind && !prevAPISpec.Handler.IsGRPC() && apiConfig.Handler.IsGRPC() { - realtimeAPIName := deployedResource.Name - - virtualServices, err := config.K8s.ListVirtualServicesByLabel("apiKind", userconfig.TrafficSplitterKind.String()) - if err != nil { - return nil, "", err - } - - trafficSplitterList, err := trafficsplitter.GetAllAPIs(virtualServices) - if err != nil { - return nil, "", err - } - - dependentTrafficSplitters := []string{} - for _, trafficSplitter := range trafficSplitterList { - for _, api := range trafficSplitter.Spec.APIs { - if realtimeAPIName == api.Name { - dependentTrafficSplitters = append(dependentTrafficSplitters, api.Name) - } - } - } - - if len(dependentTrafficSplitters) > 0 { - return nil, "", ErrorCannotChangeProtocolWhenUsedByTrafficSplitter(realtimeAPIName, dependentTrafficSplitters) - } - } - switch deployedResource.Kind { case userconfig.RealtimeAPIKind: return realtimeapi.UpdateAPI(apiConfig, prevAPISpec.ProjectID, force) @@ -584,21 +522,6 @@ func GetAPI(apiName string) ([]schema.APIResponse, error) { } } - // best effort - if config.K8s != nil && apiResponse[0].Spec.Kind == userconfig.RealtimeAPIKind && !apiResponse[0].Spec.Handler.IsGRPC() && (apiResponse[0].Spec.Handler.MultiModelReloading != nil || apiResponse[0].Spec.Handler.Models != nil) { - internalAPIEndpoint := config.K8s.InternalServiceEndpoint("api-"+apiResponse[0].Spec.Name, _defaultAPIPortInt32) - - infoAPIEndpoint := urls.Join(internalAPIEndpoint, "info") - tfModelSummary, pythonModelSummary, err := realtimeapi.GetModelsMetadata(apiResponse[0].Status, apiResponse[0].Spec.Handler, infoAPIEndpoint) - if err != nil { - operatorLogger.Warn(errors.Wrap(err, fmt.Sprintf("api %s", apiResponse[0].Spec.Name))) - return apiResponse, nil - } - - apiResponse[0].RealtimeModelMetadata.TFModelSummary = tfModelSummary - apiResponse[0].RealtimeModelMetadata.PythonModelSummary = pythonModelSummary - } - return apiResponse, nil } diff --git a/pkg/operator/resources/validations.go b/pkg/operator/resources/validations.go index d9292d7f64..910346c502 100644 --- a/pkg/operator/resources/validations.go +++ b/pkg/operator/resources/validations.go @@ -31,8 +31,6 @@ import ( "github.com/cortexlabs/cortex/pkg/types/userconfig" istioclientnetworking "istio.io/client-go/pkg/apis/networking/v1beta1" kresource "k8s.io/apimachinery/pkg/api/resource" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - klabels "k8s.io/apimachinery/pkg/labels" ) type ProjectFiles struct { @@ -86,22 +84,6 @@ func ValidateClusterAPIs(apis []userconfig.API) error { } } - virtualServicesForGrpc, err := config.K8s.ListVirtualServices(&v1.ListOptions{ - LabelSelector: klabels.SelectorFromSet( - map[string]string{ - "servingProtocol": "grpc", - }).String(), - }) - if err != nil { - return err - } - grpcDeployedRealtimeAPIs := strset.New() - for _, virtualService := range virtualServicesForGrpc { - if virtualService.Labels["apiKind"] == userconfig.RealtimeAPIKind.String() { - grpcDeployedRealtimeAPIs.Add(virtualService.Labels["apiName"]) - } - } - realtimeAPIs := InclusiveFilterAPIsByKind(apis, userconfig.RealtimeAPIKind) for i := range apis { @@ -122,7 +104,7 @@ func ValidateClusterAPIs(apis []userconfig.API) error { if err := spec.ValidateTrafficSplitter(api); err != nil { return errors.Wrap(err, api.Identify()) } - if err := checkIfAPIExists(api.APIs, realtimeAPIs, httpDeployedRealtimeAPIs, grpcDeployedRealtimeAPIs); err != nil { + if err := checkIfAPIExists(api.APIs, realtimeAPIs, httpDeployedRealtimeAPIs); err != nil { return errors.Wrap(err, api.Identify()) } if err := validateEndpointCollisions(api, virtualServices); err != nil { @@ -333,16 +315,10 @@ func ExclusiveFilterAPIsByKind(apis []userconfig.API, kindsToExclude ...userconf } // checkIfAPIExists checks if referenced apis in trafficsplitter are either defined in yaml or already deployed. -// Also prevents traffic splitting apis that use grpc. -func checkIfAPIExists(trafficSplitterAPIs []*userconfig.TrafficSplit, apis []userconfig.API, httpDeployedRealtimeAPIs strset.Set, grpcDeployedRealtimeAPIs strset.Set) error { +func checkIfAPIExists(trafficSplitterAPIs []*userconfig.TrafficSplit, apis []userconfig.API, httpDeployedRealtimeAPIs strset.Set) error { var missingAPIs []string // check if apis named in trafficsplitter are either defined in same yaml or already deployed for _, trafficSplitAPI := range trafficSplitterAPIs { - // don't allow apis that use grpc - if grpcDeployedRealtimeAPIs.Has(trafficSplitAPI.Name) { - return ErrorGRPCNotSupportedForTrafficSplitter(trafficSplitAPI.Name) - } - // check if already deployed deployed := httpDeployedRealtimeAPIs.Has(trafficSplitAPI.Name) diff --git a/pkg/operator/schema/schema.go b/pkg/operator/schema/schema.go index 9486e1f225..d2b1433340 100644 --- a/pkg/operator/schema/schema.go +++ b/pkg/operator/schema/schema.go @@ -50,16 +50,14 @@ type DeployResult struct { } type APIResponse struct { - Spec spec.API `json:"spec"` - Status *status.Status `json:"status,omitempty"` - Metrics *metrics.Metrics `json:"metrics,omitempty"` - Endpoint string `json:"endpoint"` - GRPCPorts map[string]int64 `json:"grpc_ports,omitempty"` - DashboardURL *string `json:"dashboard_url,omitempty"` - BatchJobStatuses []status.BatchJobStatus `json:"batch_job_statuses,omitempty"` - TaskJobStatuses []status.TaskJobStatus `json:"task_job_statuses,omitempty"` - RealtimeModelMetadata RealtimeModelMetadata `json:"realtime_model_metadata"` - APIVersions []APIVersion `json:"api_versions,omitempty"` + Spec spec.API `json:"spec"` + Status *status.Status `json:"status,omitempty"` + Metrics *metrics.Metrics `json:"metrics,omitempty"` + Endpoint string `json:"endpoint"` + DashboardURL *string `json:"dashboard_url,omitempty"` + BatchJobStatuses []status.BatchJobStatus `json:"batch_job_statuses,omitempty"` + TaskJobStatuses []status.TaskJobStatus `json:"task_job_statuses,omitempty"` + APIVersions []APIVersion `json:"api_versions,omitempty"` } type BatchJobResponse struct { @@ -87,37 +85,6 @@ type ErrorResponse struct { Message string `json:"message"` } -type RealtimeModelMetadata struct { - TFModelSummary *TFLiveReloadingSummary `json:"tf_model_summary"` - PythonModelSummary *PythonModelSummary `json:"python_model_summary"` -} - -type TFLiveReloadingSummary struct { - ModelMetadata map[string]TFModelIDMetadata `json:"model_metadata"` -} - -type TFModelIDMetadata struct { - DiskPath string `json:"disk_path"` - SignatureKey string `json:"signature_key"` - InputSignatures map[string]InputSignature `json:"input_signatures"` - Timestamp int64 `json:"timestamp"` - SignatureDef map[string]interface{} `json:"signature_def"` -} - -type InputSignature struct { - Shape []interface{} `json:"shape"` - Type string `json:"type"` -} - -type PythonModelSummary struct { - ModelMetadata map[string]GenericModelMetadata `json:"model_metadata"` -} - -type GenericModelMetadata struct { - Versions []string `json:"versions"` - Timestamps []int64 `json:"timestamps"` -} - type APIVersion struct { APIID string `json:"api_id"` LastUpdated int64 `json:"last_updated"` diff --git a/pkg/types/spec/validations.go b/pkg/types/spec/validations.go index 4c8db743e5..0fcb18ae08 100644 --- a/pkg/types/spec/validations.go +++ b/pkg/types/spec/validations.go @@ -478,7 +478,7 @@ func ValidateAPI( k8sClient *k8s.Client, ) error { - if api.Networking.Endpoint == nil && (api.Handler == nil || (api.Handler != nil && api.Handler.ProtobufPath == nil)) { + if api.Networking.Endpoint == nil { api.Networking.Endpoint = pointer.String("/" + api.Name) } From aa6b36742fa71bb38ab32411a24194aa9fabd78e Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Thu, 13 May 2021 18:02:25 +0300 Subject: [PATCH 04/82] WIP CaaS --- cli/cmd/cluster.go | 20 - cli/cmd/deploy.go | 86 ---- pkg/consts/consts.go | 8 +- pkg/lib/configreader/errors.go | 8 - pkg/lib/configreader/validators.go | 20 - pkg/operator/lib/autoscaler/autoscaler.go | 4 +- .../resources/realtimeapi/k8s_specs.go | 20 +- pkg/operator/resources/resources.go | 19 - pkg/operator/resources/validations.go | 43 +- pkg/types/spec/api.go | 16 +- pkg/types/spec/errors.go | 21 +- pkg/types/spec/project_files.go | 28 -- pkg/types/spec/utils.go | 34 -- pkg/types/spec/validations.go | 91 ++--- pkg/types/userconfig/api.go | 381 +++++------------- pkg/types/userconfig/config_key.go | 70 +--- pkg/workloads/k8s.go | 24 -- 17 files changed, 181 insertions(+), 712 deletions(-) delete mode 100644 pkg/types/spec/project_files.go diff --git a/cli/cmd/cluster.go b/cli/cmd/cluster.go index 574ab3ca40..b4e056edcf 100644 --- a/cli/cmd/cluster.go +++ b/cli/cmd/cluster.go @@ -33,7 +33,6 @@ import ( "github.com/cortexlabs/cortex/cli/types/cliconfig" "github.com/cortexlabs/cortex/cli/types/flags" "github.com/cortexlabs/cortex/pkg/consts" - "github.com/cortexlabs/cortex/pkg/lib/archive" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/console" "github.com/cortexlabs/cortex/pkg/lib/docker" @@ -51,7 +50,6 @@ import ( "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" "github.com/cortexlabs/cortex/pkg/types/clusterstate" - "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/cortexlabs/yaml" "github.com/spf13/cobra" ) @@ -769,24 +767,6 @@ var _clusterExportCmd = &cobra.Command{ if err != nil { exit.Error(err) } - - if apiResponse.Spec.Kind != userconfig.TrafficSplitterKind { - zipFileLocation := path.Join(baseDir, path.Base(apiResponse.Spec.ProjectKey)) - err = awsClient.DownloadFileFromS3(info.ClusterConfig.Bucket, apiResponse.Spec.ProjectKey, zipFileLocation) - if err != nil { - exit.Error(err) - } - - _, err = archive.UnzipFileToDir(zipFileLocation, baseDir) - if err != nil { - exit.Error(err) - } - - err := os.Remove(zipFileLocation) - if err != nil { - exit.Error(err) - } - } } }, } diff --git a/cli/cmd/deploy.go b/cli/cmd/deploy.go index 28a0e7aca7..9f17140da1 100644 --- a/cli/cmd/deploy.go +++ b/cli/cmd/deploy.go @@ -18,20 +18,15 @@ package cmd import ( "fmt" - "path" "strings" "github.com/cortexlabs/cortex/cli/cluster" "github.com/cortexlabs/cortex/cli/types/flags" - "github.com/cortexlabs/cortex/pkg/lib/archive" - "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/files" libjson "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/print" - "github.com/cortexlabs/cortex/pkg/lib/prompt" - s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/lib/table" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/operator/schema" @@ -147,49 +142,6 @@ func getConfigPath(args []string) string { return files.RelToAbsPath(configPath, _cwd) } -func findProjectFiles(configPath string) ([]string, error) { - projectRoot := files.Dir(configPath) - - ignoreFns := []files.IgnoreFn{ - files.IgnoreSpecificFiles(configPath), - files.IgnoreCortexDebug, - files.IgnoreHiddenFiles, - files.IgnoreHiddenFolders, - files.IgnorePythonGeneratedFiles, - } - - cortexIgnorePath := path.Join(projectRoot, ".cortexignore") - if files.IsFile(cortexIgnorePath) { - cortexIgnore, err := files.GitIgnoreFn(cortexIgnorePath) - if err != nil { - return nil, err - } - ignoreFns = append(ignoreFns, cortexIgnore) - } - - if !_flagDeployDisallowPrompt { - ignoreFns = append(ignoreFns, files.PromptForFilesAboveSize(_warningFileBytes, "do you want to upload %s (%s)?")) - } - ignoreFns = append(ignoreFns, - files.ErrorOnBigFilesFn(_maxFileSizeBytes), - // must be the last appended IgnoreFn - files.ErrorOnProjectSizeLimit(_maxProjectSizeBytes), - ) - - projectPaths, err := files.ListDirRecursive(projectRoot, false, ignoreFns...) - if err != nil { - return nil, err - } - - // Include .env file containing environment variables - dotEnvPath := path.Join(projectRoot, ".env") - if files.IsFile(dotEnvPath) { - projectPaths = append(projectPaths, dotEnvPath) - } - - return projectPaths, nil -} - func getDeploymentBytes(configPath string) (map[string][]byte, error) { configBytes, err := files.ReadFileBytes(configPath) if err != nil { @@ -200,44 +152,6 @@ func getDeploymentBytes(configPath string) (map[string][]byte, error) { "config": configBytes, } - projectRoot := files.Dir(configPath) - - projectPaths, err := findProjectFiles(configPath) - if err != nil { - return nil, err - } - - canSkipPromptMsg := "you can skip this prompt next time with `cortex deploy --yes`\n" - rootDirMsg := "this directory" - if projectRoot != _cwd { - rootDirMsg = fmt.Sprintf("./%s", files.DirPathRelativeToCWD(projectRoot)) - } - - didPromptFileCount := false - if !_flagDeployDisallowPrompt && len(projectPaths) >= _warningFileCount { - msg := fmt.Sprintf("cortex will zip %d files in %s and upload them to the cluster; we recommend that you upload large files/directories (e.g. models) to s3 and download them in your api's __init__ function, and avoid sending unnecessary files by removing them from this directory or referencing them in a .cortexignore file. Would you like to continue?", len(projectPaths), rootDirMsg) - prompt.YesOrExit(msg, canSkipPromptMsg, "") - didPromptFileCount = true - } - - projectZipBytes, _, err := archive.ZipToMem(&archive.Input{ - FileLists: []archive.FileListInput{ - { - Sources: projectPaths, - RemovePrefix: projectRoot, - }, - }, - }) - if err != nil { - return nil, errors.Wrap(err, "failed to zip project folder") - } - - if !_flagDeployDisallowPrompt && !didPromptFileCount && len(projectZipBytes) >= _warningProjectBytes { - msg := fmt.Sprintf("cortex will zip %d files in %s (%s) and upload them to the cluster, though we recommend you upload large files (e.g. models) to s3 and download them in your api's __init__ function. Would you like to continue?", len(projectPaths), rootDirMsg, s.IntToBase2Byte(len(projectZipBytes))) - prompt.YesOrExit(msg, canSkipPromptMsg, "") - } - - uploadBytes["project.zip"] = projectZipBytes return uploadBytes, nil } diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index cee47c60b2..59c5802a53 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -24,9 +24,11 @@ var ( CortexVersion = "master" // CORTEX_VERSION CortexVersionMinor = "master" // CORTEX_VERSION_MINOR - DefaultMaxReplicaConcurrency = int64(1024) - NeuronCoresPerInf = int64(4) - AuthHeader = "X-Cortex-Authorization" + DefaultMaxReplicaQueueLength = int64(1024) + DefaultMaxReplicaConcurrency = int64(1024) + DefaultTargetReplicaConcurrency = float64(8) + NeuronCoresPerInf = int64(4) + AuthHeader = "X-Cortex-Authorization" DefaultInClusterConfigPath = "/configs/cluster/cluster.yaml" MaxBucketLifecycleRules = 100 diff --git a/pkg/lib/configreader/errors.go b/pkg/lib/configreader/errors.go index c6509d09be..d11c4f274e 100644 --- a/pkg/lib/configreader/errors.go +++ b/pkg/lib/configreader/errors.go @@ -72,7 +72,6 @@ const ( ErrEmailInvalid = "configreader.email_invalid" ErrCortexResourceOnlyAllowed = "configreader.cortex_resource_only_allowed" ErrCortexResourceNotAllowed = "configreader.cortex_resource_not_allowed" - ErrImageVersionMismatch = "configreader.image_version_mismatch" ErrFieldCantBeSpecified = "configreader.field_cant_be_specified" ) @@ -436,13 +435,6 @@ func ErrorCortexResourceNotAllowed(resourceName string) error { }) } -func ErrorImageVersionMismatch(image, tag, cortexVersion string) error { - return errors.WithStack(&errors.Error{ - Kind: ErrImageVersionMismatch, - Message: fmt.Sprintf("the specified image (%s) has a tag (%s) which does not match your Cortex version (%s); please update the image tag, remove the image registry path from your configuration file (to use the default value), or update your CLI (pip install cortex==%s)", image, tag, cortexVersion, cortexVersion), - }) -} - func ErrorFieldCantBeSpecified(errMsg string) error { message := errMsg if message == "" { diff --git a/pkg/lib/configreader/validators.go b/pkg/lib/configreader/validators.go index 166f35e01d..c15a6f4ae7 100644 --- a/pkg/lib/configreader/validators.go +++ b/pkg/lib/configreader/validators.go @@ -18,11 +18,9 @@ package configreader import ( "regexp" - "strings" "time" "github.com/cortexlabs/cortex/pkg/lib/aws" - "github.com/cortexlabs/cortex/pkg/lib/docker" "github.com/cortexlabs/cortex/pkg/lib/files" ) @@ -120,21 +118,3 @@ func DurationParser(v *DurationValidation) func(string) (interface{}, error) { return d, nil } } - -func ValidateImageVersion(image, cortexVersion string) (string, error) { - if !strings.HasPrefix(image, "quay.io/cortexlabs/") && !strings.HasPrefix(image, "quay.io/cortexlabsdev/") && !strings.HasPrefix(image, "docker.io/cortexlabs/") && !strings.HasPrefix(image, "docker.io/cortexlabsdev/") && !strings.HasPrefix(image, "cortexlabs/") && !strings.HasPrefix(image, "cortexlabsdev/") { - return image, nil - } - - tag := docker.ExtractImageTag(image) - // in docker, missing tag implies "latest" - if tag == "" { - tag = "latest" - } - - if !strings.HasPrefix(tag, cortexVersion) { - return "", ErrorImageVersionMismatch(image, tag, cortexVersion) - } - - return image, nil -} diff --git a/pkg/operator/lib/autoscaler/autoscaler.go b/pkg/operator/lib/autoscaler/autoscaler.go index cd42ad66ed..0fc7c7197d 100644 --- a/pkg/operator/lib/autoscaler/autoscaler.go +++ b/pkg/operator/lib/autoscaler/autoscaler.go @@ -133,7 +133,7 @@ func AutoscaleFn(initialDeployment *kapps.Deployment, apiSpec *spec.API, getInFl return nil } - rawRecommendation := *avgInFlight / *autoscalingSpec.TargetReplicaConcurrency + rawRecommendation := *avgInFlight / autoscalingSpec.TargetReplicaConcurrency recommendation := int32(math.Ceil(rawRecommendation)) if rawRecommendation < float64(currentReplicas) && rawRecommendation > float64(currentReplicas)*(1-autoscalingSpec.DownscaleTolerance) { @@ -199,7 +199,7 @@ func AutoscaleFn(initialDeployment *kapps.Deployment, apiSpec *spec.API, getInFl apiLogger.Debugw(fmt.Sprintf("%s autoscaler tick", apiName), "autoscaling", map[string]interface{}{ "avg_in_flight": *avgInFlight, - "target_replica_concurrency": *autoscalingSpec.TargetReplicaConcurrency, + "target_replica_concurrency": autoscalingSpec.TargetReplicaConcurrency, "raw_recommendation": rawRecommendation, "current_replicas": currentReplicas, "downscale_tolerance": autoscalingSpec.DownscaleTolerance, diff --git a/pkg/operator/resources/realtimeapi/k8s_specs.go b/pkg/operator/resources/realtimeapi/k8s_specs.go index 9fa79e69a0..4b22848d45 100644 --- a/pkg/operator/resources/realtimeapi/k8s_specs.go +++ b/pkg/operator/resources/realtimeapi/k8s_specs.go @@ -162,9 +162,6 @@ func serviceSpec(api *spec.API) *kcore.Service { } func virtualServiceSpec(api *spec.API) *istioclientnetworking.VirtualService { - servingProtocol := "http" - rewritePath := pointer.String("/") - return k8s.VirtualService(&k8s.VirtualServiceSpec{ Name: workloads.K8sName(api.Name), Gateways: []string{"apis-gateway"}, @@ -174,17 +171,16 @@ func virtualServiceSpec(api *spec.API) *istioclientnetworking.VirtualService { Port: uint32(workloads.DefaultPortInt32), }}, PrefixPath: api.Networking.Endpoint, - Rewrite: rewritePath, + Rewrite: pointer.String("/"), Annotations: api.ToK8sAnnotations(), Labels: map[string]string{ - "apiName": api.Name, - "apiKind": api.Kind.String(), - "servingProtocol": servingProtocol, - "apiID": api.ID, - "specID": api.SpecID, - "deploymentID": api.DeploymentID, - "handlerID": api.HandlerID, - "cortex.dev/api": "true", + "apiName": api.Name, + "apiKind": api.Kind.String(), + "apiID": api.ID, + "specID": api.SpecID, + "deploymentID": api.DeploymentID, + "handlerID": api.HandlerID, + "cortex.dev/api": "true", }, }) } diff --git a/pkg/operator/resources/resources.go b/pkg/operator/resources/resources.go index 10168d077c..1cffbba47b 100644 --- a/pkg/operator/resources/resources.go +++ b/pkg/operator/resources/resources.go @@ -23,7 +23,6 @@ import ( "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/consts" batch "github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1" - "github.com/cortexlabs/cortex/pkg/lib/archive" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/hash" @@ -219,29 +218,11 @@ func patchAPI(apiConfig *userconfig.API, force bool) (*spec.API, string, error) return nil, "", ErrorCannotChangeKindOfDeployedAPI(apiConfig.Name, apiConfig.Kind, deployedResource.Kind) } - var projectFiles ProjectFiles - prevAPISpec, err := operator.DownloadAPISpec(deployedResource.Name, deployedResource.ID()) if err != nil { return nil, "", err } - if deployedResource.Kind != userconfig.TrafficSplitterKind { - bytes, err := config.AWS.ReadBytesFromS3(config.ClusterConfig.Bucket, prevAPISpec.ProjectKey) - if err != nil { - return nil, "", err - } - - projectFileMap, err := archive.UnzipMemToMem(bytes) - if err != nil { - return nil, "", err - } - - projectFiles = ProjectFiles{ - ProjectByteMap: projectFileMap, - } - } - err = ValidateClusterAPIs([]userconfig.API{*apiConfig}) if err != nil { err = errors.Append(err, fmt.Sprintf("\n\napi configuration schema can be found at https://docs.cortex.dev/v/%s/", consts.CortexVersionMinor)) diff --git a/pkg/operator/resources/validations.go b/pkg/operator/resources/validations.go index 910346c502..ef1fcac615 100644 --- a/pkg/operator/resources/validations.go +++ b/pkg/operator/resources/validations.go @@ -18,11 +18,9 @@ package resources import ( "fmt" - "strings" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/lib/errors" - "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" s "github.com/cortexlabs/cortex/pkg/lib/strings" @@ -33,41 +31,6 @@ import ( kresource "k8s.io/apimachinery/pkg/api/resource" ) -type ProjectFiles struct { - ProjectByteMap map[string][]byte -} - -func (projectFiles ProjectFiles) AllPaths() []string { - pFiles := make([]string, 0, len(projectFiles.ProjectByteMap)) - for path := range projectFiles.ProjectByteMap { - pFiles = append(pFiles, path) - } - return pFiles -} - -func (projectFiles ProjectFiles) GetFile(path string) ([]byte, error) { - bytes, ok := projectFiles.ProjectByteMap[path] - if !ok { - return nil, files.ErrorFileDoesNotExist(path) - } - return bytes, nil -} - -func (projectFiles ProjectFiles) HasFile(path string) bool { - _, ok := projectFiles.ProjectByteMap[path] - return ok -} - -func (projectFiles ProjectFiles) HasDir(path string) bool { - path = s.EnsureSuffix(path, "/") - for projectFilePath := range projectFiles.ProjectByteMap { - if strings.HasPrefix(projectFilePath, path) { - return true - } - } - return false -} - func ValidateClusterAPIs(apis []userconfig.API) error { if len(apis) == 0 { return spec.ErrorNoAPIs() @@ -171,13 +134,11 @@ var _inferentiaCPUReserve = kresource.MustParse("100m") var _inferentiaMemReserve = kresource.MustParse("100Mi") func validateK8sCompute(api *userconfig.API, maxMemMap map[string]kresource.Quantity) error { - compute := spec.GetTotalComputeFromContainers(api.Containers) - allErrors := []error{} successfulLoops := 0 clusterNodeGroupNames := strset.New(config.ClusterConfig.GetNodeGroupNames()...) - apiNodeGroupNames := api.NodeGroups + apiNodeGroupNames := api.Pod.NodeGroups if apiNodeGroupNames != nil { for _, ngName := range apiNodeGroupNames { @@ -187,6 +148,8 @@ func validateK8sCompute(api *userconfig.API, maxMemMap map[string]kresource.Quan } } + compute := userconfig.GetTotalComputeFromContainers(api.Pod.Containers) + for _, instanceMetadata := range config.InstancesMetadata { if apiNodeGroupNames != nil { matchedNodeGroups := 0 diff --git a/pkg/types/spec/api.go b/pkg/types/spec/api.go index 06f0971f02..818388d225 100644 --- a/pkg/types/spec/api.go +++ b/pkg/types/spec/api.go @@ -43,7 +43,6 @@ type API struct { LastUpdated int64 `json:"last_updated"` MetadataRoot string `json:"metadata_root"` ProjectID string `json:"project_id"` - ProjectKey string `json:"project_key"` } /* @@ -65,12 +64,8 @@ func GetAPISpec(apiConfig *userconfig.API, projectID string, deploymentID string var buf bytes.Buffer buf.WriteString(s.Obj(apiConfig.Resource)) - buf.WriteString(s.Obj(apiConfig.Handler)) - buf.WriteString(s.Obj(apiConfig.TaskDefinition)) + buf.WriteString(s.Obj(apiConfig.Pod)) buf.WriteString(projectID) - if apiConfig.Compute != nil { - buf.WriteString(s.Obj(apiConfig.Compute.Normalized())) - } handlerID := hash.Bytes(buf.Bytes()) buf.Reset() @@ -94,7 +89,6 @@ func GetAPISpec(apiConfig *userconfig.API, projectID string, deploymentID string LastUpdated: time.Now().Unix(), MetadataRoot: MetadataRoot(apiConfig.Name, clusterUID), ProjectID: projectID, - ProjectKey: ProjectKey(projectID, clusterUID), } } @@ -139,14 +133,6 @@ func MetadataRoot(apiName string, clusterUID string) string { ) } -func ProjectKey(projectID string, clusterUID string) string { - return filepath.Join( - clusterUID, - "projects", - projectID+".zip", - ) -} - // Extract the timestamp from an API ID func TimeFromAPIID(apiID string) (time.Time, error) { timeIDStr := strings.Split(apiID, "-")[0] diff --git a/pkg/types/spec/errors.go b/pkg/types/spec/errors.go index db68bd5538..28f4a9e1a0 100644 --- a/pkg/types/spec/errors.go +++ b/pkg/types/spec/errors.go @@ -23,7 +23,6 @@ import ( "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/k8s" - libmath "github.com/cortexlabs/cortex/pkg/lib/math" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/types/userconfig" @@ -42,8 +41,6 @@ const ( ErrOneOfPrerequisitesNotDefined = "spec.one_of_prerequisites_not_defined" ErrConfigGreaterThanOtherConfig = "spec.config_greater_than_other_config" - ErrPortCanOnlyDefinedOnce = "spec.port_can_only_be_defined_once" - ErrMinReplicasGreaterThanMax = "spec.min_replicas_greater_than_max" ErrInitReplicasGreaterThanMax = "spec.init_replicas_greater_than_max" ErrInitReplicasLessThanMin = "spec.init_replicas_less_than_min" @@ -61,7 +58,6 @@ const ( ErrRegistryAccountIDMismatch = "spec.registry_account_id_mismatch" ErrKeyIsNotSupportedForKind = "spec.key_is_not_supported_for_kind" ErrComputeResourceConflict = "spec.compute_resource_conflict" - ErrInvalidNumberOfInfProcesses = "spec.invalid_number_of_inf_processes" ErrInvalidNumberOfInfs = "spec.invalid_number_of_infs" ErrInsufficientBatchConcurrencyLevel = "spec.insufficient_batch_concurrency_level" ErrInsufficientBatchConcurrencyLevelInf = "spec.insufficient_batch_concurrency_level_inf" @@ -169,13 +165,6 @@ func ErrorConfigGreaterThanOtherConfig(tooBigKey string, tooBigVal interface{}, }) } -func ErrorPortCanOnlyDefinedOnce() error { - return errors.WithStack(&errors.Error{ - Kind: ErrPortCanOnlyDefinedOnce, - Message: fmt.Sprintf("%s field must be specified for one container only", userconfig.PortKey), - }) -} - func ErrorMinReplicasGreaterThanMax(min int32, max int32) error { return errors.WithStack(&errors.Error{ Kind: ErrMinReplicasGreaterThanMax, @@ -214,7 +203,7 @@ func ErrorSurgeAndUnavailableBothZero() error { func ErrorShmSizeCannotExceedMem(parentFieldName string, shmSize k8s.Quantity, mem k8s.Quantity) error { return errors.WithStack(&errors.Error{ Kind: ErrShmSizeCannotExceedMem, - Message: fmt.Sprintf("%s.shm_size (%s) cannot exceed compute.mem (%s)", parentFieldName, shmSize.UserString, mem.UserString), + Message: fmt.Sprintf("%s.shm_size (%s) cannot exceed total compute mem (%s)", parentFieldName, shmSize.UserString, mem.UserString), }) } @@ -253,14 +242,6 @@ func ErrorComputeResourceConflict(resourceA, resourceB string) error { }) } -func ErrorInvalidNumberOfInfProcesses(processesPerReplica int64, numInf int64, numNeuronCores int64) error { - acceptableProcesses := libmath.FactorsInt64(numNeuronCores) - return errors.WithStack(&errors.Error{ - Kind: ErrInvalidNumberOfInfProcesses, - Message: fmt.Sprintf("cannot evenly distribute %d Inferentia %s (%d NeuronCores total) over %d processes - acceptable numbers of processes are %s", numInf, s.PluralS("ASIC", numInf), numNeuronCores, processesPerReplica, s.UserStrsOr(acceptableProcesses)), - }) -} - func ErrorInvalidNumberOfInfs(requestedInfs int64) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidNumberOfInfs, diff --git a/pkg/types/spec/project_files.go b/pkg/types/spec/project_files.go deleted file mode 100644 index e4c04af845..0000000000 --- a/pkg/types/spec/project_files.go +++ /dev/null @@ -1,28 +0,0 @@ -/* -Copyright 2021 Cortex Labs, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package spec - -type ProjectFiles interface { - // Return all paths in the project, relative to the project root containing cortex.yaml - AllPaths() []string - // Return the contents of a file, given the path (relative to the project root) - GetFile(string) ([]byte, error) - // Return whether the project contains a file path (relative to the project root) - HasFile(string) bool - // Return whether the project contains a directory path (relative to the project root) - HasDir(string) bool -} diff --git a/pkg/types/spec/utils.go b/pkg/types/spec/utils.go index d4af31ade0..7f609f956a 100644 --- a/pkg/types/spec/utils.go +++ b/pkg/types/spec/utils.go @@ -20,7 +20,6 @@ import ( "strings" "github.com/cortexlabs/cortex/pkg/lib/errors" - "github.com/cortexlabs/cortex/pkg/lib/k8s" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/types/userconfig" ) @@ -43,39 +42,6 @@ func FindDuplicateNames(apis []userconfig.API) []userconfig.API { return nil } -func GetTotalComputeFromContainers(containers []userconfig.Container) userconfig.Compute { - compute := userconfig.Compute{} - - for _, container := range containers { - if container.Compute == nil { - continue - } - - if container.Compute.CPU != nil { - newCPUQuantity := k8s.NewQuantity(container.Compute.CPU.Value()) - if compute.CPU != nil { - compute.CPU = newCPUQuantity - } else if newCPUQuantity != nil { - compute.CPU.AddQty(*newCPUQuantity) - } - } - - if container.Compute.Mem != nil { - newMemQuantity := k8s.NewQuantity(container.Compute.Mem.Value()) - if compute.CPU != nil { - compute.Mem = newMemQuantity - } else if newMemQuantity != nil { - compute.Mem.AddQty(*newMemQuantity) - } - } - - compute.GPU += container.Compute.GPU - compute.Inf += container.Compute.Inf - } - - return compute -} - func surgeOrUnavailableValidator(str string) (string, error) { if strings.HasSuffix(str, "%") { parsed, ok := s.ParseInt32(strings.TrimSuffix(str, "%")) diff --git a/pkg/types/spec/validations.go b/pkg/types/spec/validations.go index 0fcb18ae08..dfdfbc66d5 100644 --- a/pkg/types/spec/validations.go +++ b/pkg/types/spec/validations.go @@ -31,7 +31,6 @@ import ( "github.com/cortexlabs/cortex/pkg/lib/errors" libjson "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/k8s" - libmath "github.com/cortexlabs/cortex/pkg/lib/math" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/regex" libtime "github.com/cortexlabs/cortex/pkg/lib/time" @@ -314,6 +313,16 @@ func autoscalingValidation() *cr.StructFieldValidation { GreaterThan: pointer.Int32(0), }, }, + { + StructField: "MaxReplicaQueueLength", + Int64Validation: &cr.Int64Validation{ + Default: consts.DefaultMaxReplicaQueueLength, + GreaterThan: pointer.Int64(0), + // our configured nginx can theoretically accept up to 32768 connections, but during testing, + // it has been observed that the number is just slightly lower, so it has been offset by 2678 + LessThanOrEqualTo: pointer.Int64(30000), + }, + }, { StructField: "MaxReplicaConcurrency", Int64Validation: &cr.Int64Validation{ @@ -326,7 +335,8 @@ func autoscalingValidation() *cr.StructFieldValidation { }, { StructField: "TargetReplicaConcurrency", - Float64PtrValidation: &cr.Float64PtrValidation{ + Float64Validation: &cr.Float64Validation{ + Default: consts.DefaultTargetReplicaConcurrency, GreaterThan: pointer.Float64(0), }, }, @@ -482,8 +492,10 @@ func ValidateAPI( api.Networking.Endpoint = pointer.String("/" + api.Name) } - if err := validateContainers(api, awsClient, k8sClient); err != nil { - return errors.Wrap(err, userconfig.ContainersKey) + if api.Pod != nil { + if err := validatePod(api, awsClient, k8sClient); err != nil { + return errors.Wrap(err, userconfig.PodKey) + } } if api.Autoscaling != nil { @@ -498,12 +510,6 @@ func ValidateAPI( } } - if api.Handler != nil && api.Handler.ShmSize != nil && api.Compute.Mem != nil { - if api.Handler.ShmSize.Cmp(api.Compute.Mem.Quantity) > 0 { - return ErrorShmSizeCannotExceedMem(userconfig.HandlerKey, *api.Handler.ShmSize, *api.Compute.Mem) - } - } - return nil } @@ -531,50 +537,54 @@ func ValidateTrafficSplitter(api *userconfig.API) error { return nil } -func validateContainers( +func validatePod( api *userconfig.API, awsClient *aws.Client, k8sClient *k8s.Client, ) error { - containers := api.Containers + containers := api.Pod.Containers + totalCompute := userconfig.GetTotalComputeFromContainers(containers) + + if api.Pod.ShmSize != nil { + if totalCompute.Mem != nil && api.Pod.ShmSize.Cmp(totalCompute.Mem.Quantity) > 0 { + return ErrorShmSizeCannotExceedMem(userconfig.HandlerKey, *api.Pod.ShmSize, *totalCompute.Mem) + } + } - numPorts := 0 + if err := validateCompute(totalCompute); err != nil { + return errors.Wrap(err, userconfig.ComputeKey) + } + + if err := validateContainers(containers, awsClient, k8sClient); err != nil { + return errors.Wrap(err, userconfig.ContainersKey) + } + + return nil +} + +func validateContainers( + containers []userconfig.Container, + awsClient *aws.Client, + k8sClient *k8s.Client, +) error { for i, container := range containers { if err := validateDockerImagePath(container.Image, awsClient, k8sClient); err != nil { return errors.Wrap(err, strconv.FormatInt(int64(i), 10), userconfig.ImageKey) } - if container.Port != nil { - numPorts++ - } - for key := range container.Env { if strings.HasPrefix(key, "CORTEX_") { return errors.Wrap(ErrorCortexPrefixedEnvVarNotAllowed(), strconv.FormatInt(int64(i), 10), userconfig.EnvKey, key) } } } - if numPorts != 1 { - return ErrorPortCanOnlyDefinedOnce() - } - - if err := validateCompute(GetTotalComputeFromContainers(containers)); err != nil { - return err - } - - return nil } func validateAutoscaling(api *userconfig.API) error { autoscaling := api.Autoscaling - handler := api.Handler - - if autoscaling.TargetReplicaConcurrency == nil { - autoscaling.TargetReplicaConcurrency = pointer.Float64(float64(handler.ProcessesPerReplica * handler.ThreadsPerProcess)) - } - if *autoscaling.TargetReplicaConcurrency > float64(autoscaling.MaxReplicaConcurrency) { - return ErrorConfigGreaterThanOtherConfig(userconfig.TargetReplicaConcurrencyKey, *autoscaling.TargetReplicaConcurrency, userconfig.MaxReplicaConcurrencyKey, autoscaling.MaxReplicaConcurrency) + if autoscaling.TargetReplicaConcurrency > float64(autoscaling.MaxReplicaConcurrency) { + return ErrorConfigGreaterThanOtherConfig(userconfig.TargetReplicaConcurrencyKey, autoscaling.TargetReplicaConcurrency, userconfig.MaxReplicaConcurrencyKey, autoscaling.MaxReplicaConcurrency) } if autoscaling.MinReplicas > autoscaling.MaxReplicas { @@ -589,14 +599,6 @@ func validateAutoscaling(api *userconfig.API) error { return ErrorInitReplicasLessThanMin(autoscaling.InitReplicas, autoscaling.MinReplicas) } - if api.Compute.Inf > 0 { - numNeuronCores := api.Compute.Inf * consts.NeuronCoresPerInf - processesPerReplica := int64(handler.ProcessesPerReplica) - if !libmath.IsDivisibleByInt64(numNeuronCores, processesPerReplica) { - return ErrorInvalidNumberOfInfProcesses(processesPerReplica, api.Compute.Inf, numNeuronCores) - } - } - return nil } @@ -625,13 +627,6 @@ func validateDockerImagePath( awsClient *aws.Client, k8sClient *k8s.Client, ) error { - if consts.DefaultImagePathsSet.Has(image) { - return nil - } - if _, err := cr.ValidateImageVersion(image, consts.CortexVersion); err != nil { - return err - } - dockerClient, err := docker.GetDockerClient() if err != nil { return err diff --git a/pkg/types/userconfig/api.go b/pkg/types/userconfig/api.go index 13cf707434..472d20c946 100644 --- a/pkg/types/userconfig/api.go +++ b/pkg/types/userconfig/api.go @@ -22,7 +22,6 @@ import ( "time" "github.com/cortexlabs/cortex/pkg/lib/k8s" - "github.com/cortexlabs/cortex/pkg/lib/sets/strset" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/lib/urls" "github.com/cortexlabs/yaml" @@ -32,13 +31,7 @@ import ( type API struct { Resource - NodeGroups []string `json:"node_groups" yaml:"node_groups"` - ShmSize *k8s.Quantity `json:"shm_size" yaml:"shm_size"` - PythonPath *string `json:"python_path" yaml:"python_path"` - LogLevel LogLevel `json:"log_level" yaml:"log_level"` - Config map[string]interface{} `json:"config" yaml:"config"` - - Containers []Container `json:"containers" yaml:"containers"` + Pod *Pod `json:"pod" yaml:"pod"` APIs []*TrafficSplit `json:"apis" yaml:"apis"` Networking *Networking `json:"networking" yaml:"networking"` Autoscaling *Autoscaling `json:"autoscaling" yaml:"autoscaling"` @@ -48,12 +41,17 @@ type API struct { SubmittedAPISpec interface{} `json:"submitted_api_spec" yaml:"submitted_api_spec"` } +type Pod struct { + NodeGroups []string `json:"node_groups" yaml:"node_groups"` + ShmSize *k8s.Quantity `json:"shm_size" yaml:"shm_size"` + Containers []Container `json:"containers" yaml:"containers"` +} + type Container struct { Name string `json:"name" yaml:"name"` Image string `json:"image" yaml:"image"` Env map[string]string `json:"env" yaml:"env"` - Port *int32 `json:"port" yaml:"port"` Command []string `json:"command" yaml:"command"` Args []string `json:"args" yaml:"args"` @@ -81,8 +79,9 @@ type Autoscaling struct { MinReplicas int32 `json:"min_replicas" yaml:"min_replicas"` MaxReplicas int32 `json:"max_replicas" yaml:"max_replicas"` InitReplicas int32 `json:"init_replicas" yaml:"init_replicas"` - TargetReplicaConcurrency *float64 `json:"target_replica_concurrency" yaml:"target_replica_concurrency"` + MaxReplicaQueueLength int64 `json:"max_replica_queue_length" yaml:"max_replica_queue_length"` MaxReplicaConcurrency int64 `json:"max_replica_concurrency" yaml:"max_replica_concurrency"` + TargetReplicaConcurrency float64 `json:"target_replica_concurrency" yaml:"target_replica_concurrency"` Window time.Duration `json:"window" yaml:"window"` DownscaleStabilizationPeriod time.Duration `json:"downscale_stabilization_period" yaml:"downscale_stabilization_period"` UpscaleStabilizationPeriod time.Duration `json:"upscale_stabilization_period" yaml:"upscale_stabilization_period"` @@ -123,11 +122,6 @@ func IdentifyAPI(filePath string, name string, kind Kind, index int) string { // InitReplicas was left out deliberately func (api *API) ToK8sAnnotations() map[string]string { annotations := map[string]string{} - if api.Handler != nil { - annotations[ProcessesPerReplicaAnnotationKey] = s.Int32(api.Handler.ProcessesPerReplica) - annotations[ThreadsPerProcessAnnotationKey] = s.Int32(api.Handler.ThreadsPerProcess) - } - if api.Networking != nil { annotations[EndpointAnnotationKey] = *api.Networking.Endpoint } @@ -135,7 +129,8 @@ func (api *API) ToK8sAnnotations() map[string]string { if api.Autoscaling != nil { annotations[MinReplicasAnnotationKey] = s.Int32(api.Autoscaling.MinReplicas) annotations[MaxReplicasAnnotationKey] = s.Int32(api.Autoscaling.MaxReplicas) - annotations[TargetReplicaConcurrencyAnnotationKey] = s.Float64(*api.Autoscaling.TargetReplicaConcurrency) + annotations[MaxReplicaQueueLengthAnnotationKey] = s.Int64(api.Autoscaling.MaxReplicaQueueLength) + annotations[TargetReplicaConcurrencyAnnotationKey] = s.Float64(api.Autoscaling.TargetReplicaConcurrency) annotations[MaxReplicaConcurrencyAnnotationKey] = s.Int64(api.Autoscaling.MaxReplicaConcurrency) annotations[WindowAnnotationKey] = api.Autoscaling.Window.String() annotations[DownscaleStabilizationPeriodAnnotationKey] = api.Autoscaling.DownscaleStabilizationPeriod.String() @@ -163,11 +158,11 @@ func AutoscalingFromAnnotations(k8sObj kmeta.Object) (*Autoscaling, error) { } a.MaxReplicas = maxReplicas - targetReplicaConcurrency, err := k8s.ParseFloat64Annotation(k8sObj, TargetReplicaConcurrencyAnnotationKey) + maxReplicaQueueLength, err := k8s.ParseInt64Annotation(k8sObj, MaxReplicaQueueLengthAnnotationKey) if err != nil { return nil, err } - a.TargetReplicaConcurrency = &targetReplicaConcurrency + a.MaxReplicaQueueLength = maxReplicaQueueLength maxReplicaConcurrency, err := k8s.ParseInt64Annotation(k8sObj, MaxReplicaConcurrencyAnnotationKey) if err != nil { @@ -175,6 +170,12 @@ func AutoscalingFromAnnotations(k8sObj kmeta.Object) (*Autoscaling, error) { } a.MaxReplicaConcurrency = maxReplicaConcurrency + targetReplicaConcurrency, err := k8s.ParseFloat64Annotation(k8sObj, TargetReplicaConcurrencyAnnotationKey) + if err != nil { + return nil, err + } + a.TargetReplicaConcurrency = targetReplicaConcurrency + window, err := k8s.ParseDurationAnnotation(k8sObj, WindowAnnotationKey) if err != nil { return nil, err @@ -232,14 +233,9 @@ func (api *API) UserStr() string { } } - if api.TaskDefinition != nil { - sb.WriteString(fmt.Sprintf("%s:\n", TaskDefinitionKey)) - sb.WriteString(s.Indent(api.TaskDefinition.UserStr(), " ")) - } - - if api.Handler != nil { + if api.Pod != nil { sb.WriteString(fmt.Sprintf("%s:\n", HandlerKey)) - sb.WriteString(s.Indent(api.Handler.UserStr(), " ")) + sb.WriteString(s.Indent(api.Pod.UserStr(), " ")) } if api.Networking != nil { @@ -247,11 +243,6 @@ func (api *API) UserStr() string { sb.WriteString(s.Indent(api.Networking.UserStr(), " ")) } - if api.Compute != nil { - sb.WriteString(fmt.Sprintf("%s:\n", ComputeKey)) - sb.WriteString(s.Indent(api.Compute.UserStr(), " ")) - } - if api.Autoscaling != nil { sb.WriteString(fmt.Sprintf("%s:\n", AutoscalingKey)) sb.WriteString(s.Indent(api.Autoscaling.UserStr(), " ")) @@ -265,14 +256,6 @@ func (api *API) UserStr() string { return sb.String() } -func (dependencies Dependencies) UserStr() string { - var sb strings.Builder - sb.WriteString(fmt.Sprintf("%s: %s\n", PipKey, dependencies.Pip)) - sb.WriteString(fmt.Sprintf("%s: %s\n", CondaKey, dependencies.Conda)) - sb.WriteString(fmt.Sprintf("%s: %s\n", ShellKey, dependencies.Shell)) - return sb.String() -} - func (trafficSplit *TrafficSplit) UserStr() string { var sb strings.Builder sb.WriteString(fmt.Sprintf("%s: %s\n", NameKey, trafficSplit.Name)) @@ -281,132 +264,48 @@ func (trafficSplit *TrafficSplit) UserStr() string { return sb.String() } -func (task *TaskDefinition) UserStr() string { +func (pod *Pod) UserStr() string { var sb strings.Builder - sb.WriteString(fmt.Sprintf("%s: %s\n", PathKey, task.Path)) - if task.PythonPath != nil { - sb.WriteString(fmt.Sprintf("%s: %s\n", PythonPathKey, *task.PythonPath)) + if pod.ShmSize != nil { + sb.WriteString(fmt.Sprintf("%s: %s\n", ShmSizeKey, pod.ShmSize.UserString)) } - sb.WriteString(fmt.Sprintf("%s: %s\n", ImageKey, task.Image)) - if task.ShmSize != nil { - sb.WriteString(fmt.Sprintf("%s: %s\n", ShmSizeKey, task.ShmSize.String())) - } - sb.WriteString(fmt.Sprintf("%s: %s\n", LogLevelKey, task.LogLevel)) - if len(task.Config) > 0 { - sb.WriteString(fmt.Sprintf("%s:\n", ConfigKey)) - d, _ := yaml.Marshal(&task.Config) - sb.WriteString(s.Indent(string(d), " ")) + if pod.NodeGroups == nil { + sb.WriteString(fmt.Sprintf("%s: null\n", NodeGroupsKey)) + } else { + sb.WriteString(fmt.Sprintf("%s: %s\n", NodeGroupsKey, s.ObjFlatNoQuotes(pod.NodeGroups))) } - if len(task.Env) > 0 { - sb.WriteString(fmt.Sprintf("%s:\n", EnvKey)) - d, _ := yaml.Marshal(&task.Env) - sb.WriteString(s.Indent(string(d), " ")) + + sb.WriteString(fmt.Sprintf("%s:\n", ContainersKey)) + for _, container := range pod.Containers { + containerUserStr := s.Indent(container.UserStr(), " ") + containerUserStr = containerUserStr[:2] + "-" + containerUserStr[3:] + sb.WriteString(containerUserStr) } - sb.WriteString(fmt.Sprintf("%s:\n", DependenciesKey)) - sb.WriteString(s.Indent(task.Dependencies.UserStr(), " ")) return sb.String() } -func (handler *Handler) UserStr() string { +func (container *Container) UserStr() string { var sb strings.Builder - sb.WriteString(fmt.Sprintf("%s: %s\n", TypeKey, handler.Type)) - sb.WriteString(fmt.Sprintf("%s: %s\n", PathKey, handler.Path)) - - if handler.ProtobufPath != nil { - sb.WriteString(fmt.Sprintf("%s: %s\n", ProtobufPathKey, *handler.ProtobufPath)) - } - - if handler.Models != nil { - sb.WriteString(fmt.Sprintf("%s:\n", ModelsKey)) - sb.WriteString(s.Indent(handler.Models.UserStr(), " ")) - } - if handler.MultiModelReloading != nil { - sb.WriteString(fmt.Sprintf("%s:\n", MultiModelReloadingKey)) - sb.WriteString(s.Indent(handler.MultiModelReloading.UserStr(), " ")) - } - - if handler.Type == TensorFlowHandlerType && handler.ServerSideBatching != nil { - sb.WriteString(fmt.Sprintf("%s:\n", ServerSideBatchingKey)) - sb.WriteString(s.Indent(handler.ServerSideBatching.UserStr(), " ")) - } - - sb.WriteString(fmt.Sprintf("%s: %s\n", ProcessesPerReplicaKey, s.Int32(handler.ProcessesPerReplica))) - sb.WriteString(fmt.Sprintf("%s: %s\n", ThreadsPerProcessKey, s.Int32(handler.ThreadsPerProcess))) - - if handler.ShmSize != nil { - sb.WriteString(fmt.Sprintf("%s: %s\n", ShmSizeKey, handler.ShmSize.UserString)) - } - - if len(handler.Config) > 0 { - sb.WriteString(fmt.Sprintf("%s:\n", ConfigKey)) - d, _ := yaml.Marshal(&handler.Config) - sb.WriteString(s.Indent(string(d), " ")) - } - sb.WriteString(fmt.Sprintf("%s: %s\n", ImageKey, handler.Image)) - if handler.TensorFlowServingImage != "" { - sb.WriteString(fmt.Sprintf("%s: %s\n", TensorFlowServingImageKey, handler.TensorFlowServingImage)) - } - if handler.PythonPath != nil { - sb.WriteString(fmt.Sprintf("%s: %s\n", PythonPathKey, *handler.PythonPath)) - } - - sb.WriteString(fmt.Sprintf("%s: %s\n", LogLevelKey, handler.LogLevel)) + sb.WriteString(fmt.Sprintf("%s: %s\n", ContainerNameKey, container.Name)) + sb.WriteString(fmt.Sprintf("%s: %s\n", ImageKey, container.Image)) - if len(handler.Env) > 0 { + if len(container.Env) > 0 { sb.WriteString(fmt.Sprintf("%s:\n", EnvKey)) - d, _ := yaml.Marshal(&handler.Env) + d, _ := yaml.Marshal(&container.Env) sb.WriteString(s.Indent(string(d), " ")) } - sb.WriteString(fmt.Sprintf("%s:\n", DependenciesKey)) - sb.WriteString(s.Indent(handler.Dependencies.UserStr(), " ")) - - return sb.String() -} - -func (models *MultiModels) UserStr() string { - var sb strings.Builder - if len(models.Paths) > 0 { - sb.WriteString(fmt.Sprintf("%s:\n", ModelsPathsKey)) - for _, model := range models.Paths { - modelUserStr := s.Indent(model.UserStr(), " ") - modelUserStr = modelUserStr[:2] + "-" + modelUserStr[3:] - sb.WriteString(modelUserStr) - } - } else if models.Path != nil { - sb.WriteString(fmt.Sprintf("%s: %s\n", ModelsPathKey, *models.Path)) - } else { - sb.WriteString(fmt.Sprintf("%s: %s\n", ModelsDirKey, *models.Dir)) - } - if models.SignatureKey != nil { - sb.WriteString(fmt.Sprintf("%s: %s\n", ModelsSignatureKeyKey, *models.SignatureKey)) - } - if models.CacheSize != nil { - sb.WriteString(fmt.Sprintf("%s: %s\n", ModelsCacheSizeKey, s.Int32(*models.CacheSize))) - } - if models.DiskCacheSize != nil { - sb.WriteString(fmt.Sprintf("%s: %s\n", ModelsDiskCacheSizeKey, s.Int32(*models.DiskCacheSize))) - } - return sb.String() -} + sb.WriteString(fmt.Sprintf("%s: %s\n", CommandKey, s.ObjFlatNoQuotes(container.Command))) + sb.WriteString(fmt.Sprintf("%s: %s\n", ArgsKey, s.ObjFlatNoQuotes(container.Args))) -func (model *ModelResource) UserStr() string { - var sb strings.Builder - sb.WriteString(fmt.Sprintf("%s: %s\n", ModelsNameKey, model.Name)) - sb.WriteString(fmt.Sprintf("%s: %s\n", ModelsPathKey, model.Path)) - if model.SignatureKey != nil { - sb.WriteString(fmt.Sprintf("%s: %s\n", ModelsSignatureKeyKey, *model.SignatureKey)) + if container.Compute != nil { + sb.WriteString(fmt.Sprintf("%s:\n", ComputeKey)) + sb.WriteString(s.Indent(container.Compute.UserStr(), " ")) } - return sb.String() -} -func (batch *ServerSideBatching) UserStr() string { - var sb strings.Builder - sb.WriteString(fmt.Sprintf("%s: %s\n", MaxBatchSizeKey, s.Int32(batch.MaxBatchSize))) - sb.WriteString(fmt.Sprintf("%s: %s\n", BatchIntervalKey, batch.BatchInterval)) return sb.String() } @@ -418,34 +317,6 @@ func (networking *Networking) UserStr() string { return sb.String() } -// Represent compute using the smallest base units e.g. bytes for Mem, milli for CPU -func (compute *Compute) Normalized() string { - var sb strings.Builder - if compute.CPU == nil { - sb.WriteString(fmt.Sprintf("%s: null\n", CPUKey)) - } else { - sb.WriteString(fmt.Sprintf("%s: %d\n", CPUKey, compute.CPU.MilliValue())) - } - if compute.GPU > 0 { - sb.WriteString(fmt.Sprintf("%s: %s\n", GPUKey, s.Int64(compute.GPU))) - } - if compute.Inf > 0 { - sb.WriteString(fmt.Sprintf("%s: %s\n", InfKey, s.Int64(compute.Inf))) - } - if compute.Mem == nil { - sb.WriteString(fmt.Sprintf("%s: null\n", MemKey)) - } else { - sb.WriteString(fmt.Sprintf("%s: %d\n", MemKey, compute.Mem.Value())) - } - if compute.NodeGroups == nil { - sb.WriteString(fmt.Sprintf("%s: null\n", NodeGroupsKey)) - } else { - sb.WriteString(fmt.Sprintf("%s: %s\n", NodeGroupsKey, s.ObjFlatNoQuotes(compute.NodeGroups))) - } - - return sb.String() -} - func (compute *Compute) UserStr() string { var sb strings.Builder if compute.CPU == nil { @@ -464,11 +335,6 @@ func (compute *Compute) UserStr() string { } else { sb.WriteString(fmt.Sprintf("%s: %s\n", MemKey, compute.Mem.UserString)) } - if compute.NodeGroups == nil { - sb.WriteString(fmt.Sprintf("%s: null # automatic node-group selection\n", NodeGroupsKey)) - } else { - sb.WriteString(fmt.Sprintf("%s: %s\n", NodeGroupsKey, s.ObjFlatNoQuotes(compute.NodeGroups))) - } return sb.String() } @@ -497,14 +363,6 @@ func (compute Compute) Equals(c2 *Compute) bool { return false } - if compute.NodeGroups == nil && c2.NodeGroups != nil || compute.NodeGroups != nil && c2.NodeGroups == nil { - return false - } - - if !strset.New(compute.NodeGroups...).IsEqual(strset.New(c2.NodeGroups...)) { - return false - } - return true } @@ -513,8 +371,9 @@ func (autoscaling *Autoscaling) UserStr() string { sb.WriteString(fmt.Sprintf("%s: %s\n", MinReplicasKey, s.Int32(autoscaling.MinReplicas))) sb.WriteString(fmt.Sprintf("%s: %s\n", MaxReplicasKey, s.Int32(autoscaling.MaxReplicas))) sb.WriteString(fmt.Sprintf("%s: %s\n", InitReplicasKey, s.Int32(autoscaling.InitReplicas))) + sb.WriteString(fmt.Sprintf("%s: %s\n", MaxReplicaQueueLengthKey, s.Int64(autoscaling.MaxReplicaQueueLength))) sb.WriteString(fmt.Sprintf("%s: %s\n", MaxReplicaConcurrencyKey, s.Int64(autoscaling.MaxReplicaConcurrency))) - sb.WriteString(fmt.Sprintf("%s: %s\n", TargetReplicaConcurrencyKey, s.Float64(*autoscaling.TargetReplicaConcurrency))) + sb.WriteString(fmt.Sprintf("%s: %s\n", TargetReplicaConcurrencyKey, s.Float64(autoscaling.TargetReplicaConcurrency))) sb.WriteString(fmt.Sprintf("%s: %s\n", WindowKey, autoscaling.Window.String())) sb.WriteString(fmt.Sprintf("%s: %s\n", DownscaleStabilizationPeriodKey, autoscaling.DownscaleStabilizationPeriod.String())) sb.WriteString(fmt.Sprintf("%s: %s\n", UpscaleStabilizationPeriodKey, autoscaling.UpscaleStabilizationPeriod.String())) @@ -541,6 +400,39 @@ func ZeroCompute() Compute { } } +func GetTotalComputeFromContainers(containers []Container) Compute { + compute := Compute{} + + for _, container := range containers { + if container.Compute == nil { + continue + } + + if container.Compute.CPU != nil { + newCPUQuantity := k8s.NewQuantity(container.Compute.CPU.Value()) + if compute.CPU != nil { + compute.CPU = newCPUQuantity + } else if newCPUQuantity != nil { + compute.CPU.AddQty(*newCPUQuantity) + } + } + + if container.Compute.Mem != nil { + newMemQuantity := k8s.NewQuantity(container.Compute.Mem.Value()) + if compute.CPU != nil { + compute.Mem = newMemQuantity + } else if newMemQuantity != nil { + compute.Mem.AddQty(*newMemQuantity) + } + } + + compute.GPU += container.Compute.GPU + compute.Inf += container.Compute.Inf + } + + return compute +} + func (api *API) TelemetryEvent() map[string]interface{} { event := map[string]interface{}{"kind": api.Kind} @@ -559,100 +451,27 @@ func (api *API) TelemetryEvent() map[string]interface{} { } } - if api.Compute != nil { - event["compute._is_defined"] = true - if api.Compute.CPU != nil { - event["compute.cpu._is_defined"] = true - event["compute.cpu"] = float64(api.Compute.CPU.MilliValue()) / 1000 - } - if api.Compute.Mem != nil { - event["compute.mem._is_defined"] = true - event["compute.mem"] = api.Compute.Mem.Value() - } - event["compute.gpu"] = api.Compute.GPU - event["compute.inf"] = api.Compute.Inf - event["compute.node_groups._is_defined"] = len(api.Compute.NodeGroups) > 0 - event["compute.node_groups._len"] = len(api.Compute.NodeGroups) - } - - if api.Handler != nil { - event["handler._is_defined"] = true - event["handler.type"] = api.Handler.Type - event["handler.processes_per_replica"] = api.Handler.ProcessesPerReplica - event["handler.threads_per_process"] = api.Handler.ThreadsPerProcess - - if api.Handler.ShmSize != nil { - event["handler.shm_size"] = api.Handler.ShmSize.String() - } - - event["handler.log_level"] = api.Handler.LogLevel - - if api.Handler.ProtobufPath != nil { - event["handler.protobuf_path._is_defined"] = true - } - if api.Handler.PythonPath != nil { - event["handler.python_path._is_defined"] = true - } - if !strings.HasPrefix(api.Handler.Image, "cortexlabs/") { - event["handler.image._is_custom"] = true + if api.Pod != nil { + event["pod._is_defined"] = true + if api.Pod.ShmSize != nil { + event["pod.shm_size"] = api.Pod.ShmSize.String() } - if !strings.HasPrefix(api.Handler.TensorFlowServingImage, "cortexlabs/") { - event["handler.tensorflow_serving_image._is_custom"] = true + event["pod.node_groups._is_defined"] = len(api.Pod.NodeGroups) > 0 + event["pod.node_groups._len"] = len(api.Pod.NodeGroups) + event["pod.containers._len"] = len(api.Pod.Containers) + + totalCompute := GetTotalComputeFromContainers(api.Pod.Containers) + event["pod.containers.compute._is_defined"] = true + if totalCompute.CPU != nil { + event["pod.containers.compute.cpu._is_defined"] = true + event["pod.containers.compute.cpu"] = float64(totalCompute.CPU.MilliValue()) / 1000 } - if len(api.Handler.Config) > 0 { - event["handler.config._is_defined"] = true - event["handler.config._len"] = len(api.Handler.Config) - } - if len(api.Handler.Env) > 0 { - event["handler.env._is_defined"] = true - event["handler.env._len"] = len(api.Handler.Env) - } - - var models *MultiModels - if api.Handler.Models != nil { - models = api.Handler.Models - } - if api.Handler.MultiModelReloading != nil { - models = api.Handler.MultiModelReloading - } - - if models != nil { - event["handler.models._is_defined"] = true - if models.Path != nil { - event["handler.models.path._is_defined"] = true - } - if len(models.Paths) > 0 { - event["handler.models.paths._is_defined"] = true - event["handler.models.paths._len"] = len(models.Paths) - var numSignatureKeysDefined int - for _, mmPath := range models.Paths { - if mmPath.SignatureKey != nil { - numSignatureKeysDefined++ - } - } - event["handler.models.paths._num_signature_keys_defined"] = numSignatureKeysDefined - } - if models.Dir != nil { - event["handler.models.dir._is_defined"] = true - } - if models.CacheSize != nil { - event["handler.models.cache_size._is_defined"] = true - event["handler.models.cache_size"] = *models.CacheSize - } - if models.DiskCacheSize != nil { - event["handler.models.disk_cache_size._is_defined"] = true - event["handler.models.disk_cache_size"] = *models.DiskCacheSize - } - if models.SignatureKey != nil { - event["handler.models.signature_key._is_defined"] = true - } - } - - if api.Handler.ServerSideBatching != nil { - event["handler.server_side_batching._is_defined"] = true - event["handler.server_side_batching.max_batch_size"] = api.Handler.ServerSideBatching.MaxBatchSize - event["handler.server_side_batching.batch_interval"] = api.Handler.ServerSideBatching.BatchInterval.Seconds() + if totalCompute.Mem != nil { + event["pod.containers.compute.mem._is_defined"] = true + event["pod.containers.compute.mem"] = totalCompute.Mem.Value() } + event["pod.containers.compute.gpu"] = totalCompute.GPU + event["pod.containers.compute.inf"] = totalCompute.Inf } if api.UpdateStrategy != nil { @@ -666,11 +485,9 @@ func (api *API) TelemetryEvent() map[string]interface{} { event["autoscaling.min_replicas"] = api.Autoscaling.MinReplicas event["autoscaling.max_replicas"] = api.Autoscaling.MaxReplicas event["autoscaling.init_replicas"] = api.Autoscaling.InitReplicas - if api.Autoscaling.TargetReplicaConcurrency != nil { - event["autoscaling.target_replica_concurrency._is_defined"] = true - event["autoscaling.target_replica_concurrency"] = *api.Autoscaling.TargetReplicaConcurrency - } + event["autoscaling.max_replica_queue_length"] = api.Autoscaling.MaxReplicaQueueLength event["autoscaling.max_replica_concurrency"] = api.Autoscaling.MaxReplicaConcurrency + event["autoscaling.target_replica_concurrency"] = api.Autoscaling.TargetReplicaConcurrency event["autoscaling.window"] = api.Autoscaling.Window.Seconds() event["autoscaling.downscale_stabilization_period"] = api.Autoscaling.DownscaleStabilizationPeriod.Seconds() event["autoscaling.upscale_stabilization_period"] = api.Autoscaling.UpscaleStabilizationPeriod.Seconds() diff --git a/pkg/types/userconfig/config_key.go b/pkg/types/userconfig/config_key.go index ccc0fb9b3d..216b3fdbec 100644 --- a/pkg/types/userconfig/config_key.go +++ b/pkg/types/userconfig/config_key.go @@ -32,66 +32,35 @@ const ( WeightKey = "weight" ShadowKey = "shadow" - // Containers + // Pod + PodKey = "pod" + NodeGroupsKey = "node_groups" + ShmSizeKey = "shm_size" ContainersKey = "containers" - PortKey = "port" - - // Handler - TypeKey = "type" - PathKey = "path" - ProtobufPathKey = "protobuf_path" - ServerSideBatchingKey = "server_side_batching" - PythonPathKey = "python_path" - ImageKey = "image" - TensorFlowServingImageKey = "tensorflow_serving_image" - ProcessesPerReplicaKey = "processes_per_replica" - ThreadsPerProcessKey = "threads_per_process" - ShmSizeKey = "shm_size" - LogLevelKey = "log_level" - ConfigKey = "config" - EnvKey = "env" - - // Handler/TaskDefinition.Dependencies - DependenciesKey = "dependencies" - PipKey = "pip" - ShellKey = "shell" - CondaKey = "conda" - // MultiModelReloading - MultiModelReloadingKey = "multi_model_reloading" - - // MultiModels - ModelsKey = "models" - ModelsPathKey = "path" - ModelsPathsKey = "paths" - ModelsDirKey = "dir" - ModelsSignatureKeyKey = "signature_key" - ModelsCacheSizeKey = "cache_size" - ModelsDiskCacheSizeKey = "disk_cache_size" - - // ServerSideBatching - MaxBatchSizeKey = "max_batch_size" - BatchIntervalKey = "batch_interval" + // Containers + ContainerNameKey = "name" + ImageKey = "image" + EnvKey = "env" + CommandKey = "command" + ArgsKey = "args" - // ModelResource - ModelsNameKey = "name" + // Compute + CPUKey = "cpu" + MemKey = "mem" + GPUKey = "gpu" + InfKey = "inf" // Networking EndpointKey = "endpoint" - // Compute - CPUKey = "cpu" - MemKey = "mem" - GPUKey = "gpu" - InfKey = "inf" - NodeGroupsKey = "node_groups" - // Autoscaling MinReplicasKey = "min_replicas" MaxReplicasKey = "max_replicas" InitReplicasKey = "init_replicas" - TargetReplicaConcurrencyKey = "target_replica_concurrency" + MaxReplicaQueueLengthKey = "max_replica_queue_length" MaxReplicaConcurrencyKey = "max_replica_concurrency" + TargetReplicaConcurrencyKey = "target_replica_concurrency" WindowKey = "window" DownscaleStabilizationPeriodKey = "downscale_stabilization_period" UpscaleStabilizationPeriodKey = "upscale_stabilization_period" @@ -106,12 +75,11 @@ const ( // K8s annotation EndpointAnnotationKey = "networking.cortex.dev/endpoint" - ProcessesPerReplicaAnnotationKey = "handler.cortex.dev/processes-per-replica" - ThreadsPerProcessAnnotationKey = "handler.cortex.dev/threads-per-process" MinReplicasAnnotationKey = "autoscaling.cortex.dev/min-replicas" MaxReplicasAnnotationKey = "autoscaling.cortex.dev/max-replicas" - TargetReplicaConcurrencyAnnotationKey = "autoscaling.cortex.dev/target-replica-concurrency" + MaxReplicaQueueLengthAnnotationKey = "autoscaling.cortex.dev/max-replica-queue-length" MaxReplicaConcurrencyAnnotationKey = "autoscaling.cortex.dev/max-replica-concurrency" + TargetReplicaConcurrencyAnnotationKey = "autoscaling.cortex.dev/target-replica-concurrency" WindowAnnotationKey = "autoscaling.cortex.dev/window" DownscaleStabilizationPeriodAnnotationKey = "autoscaling.cortex.dev/downscale-stabilization-period" UpscaleStabilizationPeriodAnnotationKey = "autoscaling.cortex.dev/upscale-stabilization-period" diff --git a/pkg/workloads/k8s.go b/pkg/workloads/k8s.go index d40ed6a622..625e14bd31 100644 --- a/pkg/workloads/k8s.go +++ b/pkg/workloads/k8s.go @@ -88,14 +88,6 @@ func TaskInitContainer(api *spec.API, job *spec.TaskJob) kcore.Container { downloadConfig := downloadContainerConfig{ LastLog: fmt.Sprintf(_downloaderLastLog, "task"), DownloadArgs: []downloadContainerArg{ - { - From: aws.S3Path(config.ClusterConfig.Bucket, api.ProjectKey), - To: path.Join(_emptyDirMountPath, "project"), - Unzip: true, - ItemName: "the project code", - HideFromLog: true, - HideUnzippingLog: true, - }, { From: aws.S3Path(config.ClusterConfig.Bucket, api.Key), To: APISpecPath, @@ -145,14 +137,6 @@ func BatchInitContainer(api *spec.API, job *spec.BatchJob) kcore.Container { downloadConfig := downloadContainerConfig{ LastLog: fmt.Sprintf(_downloaderLastLog, api.Handler.Type.String()), DownloadArgs: []downloadContainerArg{ - { - From: aws.S3Path(config.ClusterConfig.Bucket, api.ProjectKey), - To: path.Join(_emptyDirMountPath, "project"), - Unzip: true, - ItemName: "the project code", - HideFromLog: true, - HideUnzippingLog: true, - }, { From: aws.S3Path(config.ClusterConfig.Bucket, api.Key), To: APISpecPath, @@ -193,14 +177,6 @@ func InitContainer(api *spec.API) kcore.Container { downloadConfig := downloadContainerConfig{ LastLog: fmt.Sprintf(_downloaderLastLog, api.Handler.Type.String()), DownloadArgs: []downloadContainerArg{ - { - From: aws.S3Path(config.ClusterConfig.Bucket, api.ProjectKey), - To: path.Join(_emptyDirMountPath, "project"), - Unzip: true, - ItemName: "the project code", - HideFromLog: true, - HideUnzippingLog: true, - }, { From: aws.S3Path(config.ClusterConfig.Bucket, api.HandlerKey), To: APISpecPath, From 74dc9c17c8acc7f579bda1e118cda43bce0509b2 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Thu, 13 May 2021 21:18:41 +0300 Subject: [PATCH 05/82] WIP CaaS --- cli/cmd/cluster.go | 5 - cli/cmd/errors.go | 13 - cli/cmd/prepare_debug.go | 112 -- cli/cmd/root.go | 2 - go.mod | 1 - .../batch/batchjob_controller_helpers.go | 19 +- .../batch/batchjob_controller_test.go | 14 +- pkg/lib/configreader/errors.go | 8 + pkg/lib/configreader/validators.go | 20 + pkg/operator/operator/logging.go | 35 +- pkg/operator/resources/asyncapi/api.go | 2 +- pkg/operator/resources/asyncapi/k8s_specs.go | 23 +- pkg/operator/resources/job/batchapi/job.go | 2 +- .../resources/job/taskapi/k8s_specs.go | 14 +- pkg/operator/resources/realtimeapi/api.go | 66 -- .../resources/realtimeapi/k8s_specs.go | 76 +- pkg/types/spec/api.go | 14 +- pkg/types/spec/validations.go | 2 + pkg/types/userconfig/api.go | 9 + pkg/workloads/helpers.go | 98 +- pkg/workloads/init.go | 174 +++ pkg/workloads/k8s.go | 1033 +++-------------- 22 files changed, 480 insertions(+), 1262 deletions(-) delete mode 100644 cli/cmd/prepare_debug.go create mode 100644 pkg/workloads/init.go diff --git a/cli/cmd/cluster.go b/cli/cmd/cluster.go index b4e056edcf..1f2348bf11 100644 --- a/cli/cmd/cluster.go +++ b/cli/cmd/cluster.go @@ -714,11 +714,6 @@ var _clusterExportCmd = &cobra.Command{ OperatorEndpoint: "https://" + *loadBalancer.DNSName, } - info, err := cluster.Info(operatorConfig) - if err != nil { - exit.Error(err) - } - var apisResponse []schema.APIResponse if len(args) == 0 { apisResponse, err = cluster.GetAPIs(operatorConfig) diff --git a/cli/cmd/errors.go b/cli/cmd/errors.go index 9a3db6e652..bbc196d61b 100644 --- a/cli/cmd/errors.go +++ b/cli/cmd/errors.go @@ -26,7 +26,6 @@ import ( s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/lib/urls" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" - "github.com/cortexlabs/cortex/pkg/types/userconfig" ) const ( @@ -69,7 +68,6 @@ const ( ErrDeployFromTopLevelDir = "cli.deploy_from_top_level_dir" ErrAPINameMustBeProvided = "cli.api_name_must_be_provided" ErrAPINotFoundInConfig = "cli.api_not_found_in_config" - ErrNotSupportedForKindAndType = "cli.not_supported_for_kind_and_type" ErrClusterUIDsLimitInBucket = "cli.cluster_uids_limit_in_bucket" ) @@ -286,17 +284,6 @@ func ErrorAPINotFoundInConfig(apiName string) error { }) } -func ErrorNotSupportedForKindAndType(kind userconfig.Kind, handlerType userconfig.HandlerType) error { - return errors.WithStack(&errors.Error{ - Kind: ErrNotSupportedForKindAndType, - Message: fmt.Sprintf("this command is still in beta and currently only supports %s with type %s", userconfig.RealtimeAPIKind.String(), userconfig.PythonHandlerType.String()), - Metadata: map[string]interface{}{ - "apiKind": kind.String(), - "handlerType": handlerType.String(), - }, - }) -} - func ErrorClusterUIDsLimitInBucket(bucket string) error { return errors.WithStack(&errors.Error{ Kind: ErrClusterUIDsLimitInBucket, diff --git a/cli/cmd/prepare_debug.go b/cli/cmd/prepare_debug.go deleted file mode 100644 index 757d5c9fe1..0000000000 --- a/cli/cmd/prepare_debug.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Copyright 2021 Cortex Labs, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "fmt" - "path" - - "github.com/cortexlabs/cortex/pkg/consts" - "github.com/cortexlabs/cortex/pkg/lib/errors" - "github.com/cortexlabs/cortex/pkg/lib/exit" - "github.com/cortexlabs/cortex/pkg/lib/files" - libjson "github.com/cortexlabs/cortex/pkg/lib/json" - "github.com/cortexlabs/cortex/pkg/lib/telemetry" - "github.com/cortexlabs/cortex/pkg/types/spec" - "github.com/cortexlabs/cortex/pkg/types/userconfig" - "github.com/spf13/cobra" -) - -func prepareDebugInit() { - _prepareDebugCmd.Flags().SortFlags = false -} - -var _prepareDebugCmd = &cobra.Command{ - Use: "prepare-debug CONFIG_FILE [API_NAME]", - Short: "prepare artifacts to debug containers", - Args: cobra.RangeArgs(1, 2), - Run: func(cmd *cobra.Command, args []string) { - telemetry.Event("cli.prepare-debug") - - configPath := args[0] - - var apiName string - if len(args) == 2 { - apiName = args[1] - } - - configBytes, err := files.ReadFileBytes(configPath) - if err != nil { - exit.Error(err) - } - - projectRoot := files.Dir(files.UserRelToAbsPath(configPath)) - - apis, err := spec.ExtractAPIConfigs(configBytes, args[0]) - if err != nil { - exit.Error(err) - } - - if apiName == "" && len(apis) > 1 { - exit.Error(errors.Wrap(ErrorAPINameMustBeProvided(), configPath)) - } - - var apiToPrepare userconfig.API - if apiName == "" { - apiToPrepare = apis[0] - } else { - found := false - for i := range apis { - api := apis[i] - if api.Name == apiName { - found = true - apiToPrepare = api - break - } - } - if !found { - exit.Error(errors.Wrap(ErrorAPINotFoundInConfig(apiName), configPath)) - } - } - - if apiToPrepare.Kind != userconfig.RealtimeAPIKind { - exit.Error(ErrorNotSupportedForKindAndType(apiToPrepare.Kind, userconfig.UnknownHandlerType)) - } - if apiToPrepare.Handler.Type != userconfig.PythonHandlerType { - exit.Error(ErrorNotSupportedForKindAndType(apiToPrepare.Kind, apiToPrepare.Handler.Type)) - } - - apiSpec := spec.API{ - API: &apiToPrepare, - } - - debugFileName := apiSpec.Name + ".debug.json" - jsonStr, err := libjson.Pretty(apiSpec) - if err != nil { - exit.Error(err) - } - files.WriteFile([]byte(jsonStr), path.Join(projectRoot, debugFileName)) - - fmt.Println(fmt.Sprintf( - ` -docker run --rm -p 9000:8888 \ --e "CORTEX_VERSION=%s" \ --e "CORTEX_API_SPEC=/mnt/project/%s" \ --v %s:/mnt/project \ -%s`, consts.CortexVersion, debugFileName, path.Clean(projectRoot), apiToPrepare.Handler.Image)) - }, -} diff --git a/cli/cmd/root.go b/cli/cmd/root.go index 9cae06557e..bee317fabf 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -112,7 +112,6 @@ func init() { clusterInit() completionInit() deleteInit() - prepareDebugInit() deployInit() envInit() getInit() @@ -160,7 +159,6 @@ func Execute() { _rootCmd.AddCommand(_logsCmd) _rootCmd.AddCommand(_refreshCmd) _rootCmd.AddCommand(_deleteCmd) - _rootCmd.AddCommand(_prepareDebugCmd) _rootCmd.AddCommand(_clusterCmd) diff --git a/go.mod b/go.mod index 52075059e0..10cbf7d015 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,6 @@ require ( github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0 github.com/docker/go-connections v0.4.0 // indirect - github.com/emicklei/proto v1.9.0 github.com/fatih/color v1.10.0 github.com/getsentry/sentry-go v0.10.0 github.com/go-logr/logr v0.3.0 diff --git a/pkg/crds/controllers/batch/batchjob_controller_helpers.go b/pkg/crds/controllers/batch/batchjob_controller_helpers.go index 91d588a201..e32fa4cac9 100644 --- a/pkg/crds/controllers/batch/batchjob_controller_helpers.go +++ b/pkg/crds/controllers/batch/batchjob_controller_helpers.go @@ -272,23 +272,8 @@ func (r *BatchJobReconciler) desiredWorkerJob(batchJob batch.BatchJob, apiSpec s var containers []kcore.Container var volumes []kcore.Volume - switch apiSpec.Handler.Type { - case userconfig.PythonHandlerType: - containers, volumes = workloads.PythonHandlerJobContainers(&apiSpec) - case userconfig.TensorFlowHandlerType: - containers, volumes = workloads.TensorFlowHandlerJobContainers(&apiSpec) - default: - return nil, fmt.Errorf("unexpected handler type (%s)", apiSpec.Handler.Type) - } - - for i, container := range containers { - if container.Name == workloads.APIContainerName { - containers[i].Env = append(container.Env, kcore.EnvVar{ - Name: "CORTEX_JOB_SPEC", - Value: workloads.BatchSpecPath, - }) - } - } + containers, volumes = workloads.UserPodContainers(apiSpec) + // TODO add the proxy as well job := k8s.Job( &k8s.JobSpec{ diff --git a/pkg/crds/controllers/batch/batchjob_controller_test.go b/pkg/crds/controllers/batch/batchjob_controller_test.go index c475f7c5f9..454359235e 100644 --- a/pkg/crds/controllers/batch/batchjob_controller_test.go +++ b/pkg/crds/controllers/batch/batchjob_controller_test.go @@ -42,12 +42,16 @@ func uploadTestAPISpec(apiName string, apiID string) error { Name: apiName, Kind: userconfig.BatchAPIKind, }, - Handler: &userconfig.Handler{ - Type: userconfig.PythonHandlerType, - Image: "quay.io/cortexlabs/python-handler-cpu:master", - Dependencies: &userconfig.Dependencies{}, + Pod: &userconfig.Pod{ + // TODO use a real image + Containers: []userconfig.Container{ + { + Name: "api", + Image: "quay.io/cortexlabs/batch-container-test:master", + Command: []string{"/bin/run"}, + }, + }, }, - Compute: &userconfig.Compute{}, }, ID: apiID, SpecID: random.String(5), diff --git a/pkg/lib/configreader/errors.go b/pkg/lib/configreader/errors.go index d11c4f274e..c6509d09be 100644 --- a/pkg/lib/configreader/errors.go +++ b/pkg/lib/configreader/errors.go @@ -72,6 +72,7 @@ const ( ErrEmailInvalid = "configreader.email_invalid" ErrCortexResourceOnlyAllowed = "configreader.cortex_resource_only_allowed" ErrCortexResourceNotAllowed = "configreader.cortex_resource_not_allowed" + ErrImageVersionMismatch = "configreader.image_version_mismatch" ErrFieldCantBeSpecified = "configreader.field_cant_be_specified" ) @@ -435,6 +436,13 @@ func ErrorCortexResourceNotAllowed(resourceName string) error { }) } +func ErrorImageVersionMismatch(image, tag, cortexVersion string) error { + return errors.WithStack(&errors.Error{ + Kind: ErrImageVersionMismatch, + Message: fmt.Sprintf("the specified image (%s) has a tag (%s) which does not match your Cortex version (%s); please update the image tag, remove the image registry path from your configuration file (to use the default value), or update your CLI (pip install cortex==%s)", image, tag, cortexVersion, cortexVersion), + }) +} + func ErrorFieldCantBeSpecified(errMsg string) error { message := errMsg if message == "" { diff --git a/pkg/lib/configreader/validators.go b/pkg/lib/configreader/validators.go index c15a6f4ae7..166f35e01d 100644 --- a/pkg/lib/configreader/validators.go +++ b/pkg/lib/configreader/validators.go @@ -18,9 +18,11 @@ package configreader import ( "regexp" + "strings" "time" "github.com/cortexlabs/cortex/pkg/lib/aws" + "github.com/cortexlabs/cortex/pkg/lib/docker" "github.com/cortexlabs/cortex/pkg/lib/files" ) @@ -118,3 +120,21 @@ func DurationParser(v *DurationValidation) func(string) (interface{}, error) { return d, nil } } + +func ValidateImageVersion(image, cortexVersion string) (string, error) { + if !strings.HasPrefix(image, "quay.io/cortexlabs/") && !strings.HasPrefix(image, "quay.io/cortexlabsdev/") && !strings.HasPrefix(image, "docker.io/cortexlabs/") && !strings.HasPrefix(image, "docker.io/cortexlabsdev/") && !strings.HasPrefix(image, "cortexlabs/") && !strings.HasPrefix(image, "cortexlabsdev/") { + return image, nil + } + + tag := docker.ExtractImageTag(image) + // in docker, missing tag implies "latest" + if tag == "" { + tag = "latest" + } + + if !strings.HasPrefix(tag, cortexVersion) { + return "", ErrorImageVersionMismatch(image, tag, cortexVersion) + } + + return image, nil +} diff --git a/pkg/operator/operator/logging.go b/pkg/operator/operator/logging.go index 190d40f8eb..f49746f64b 100644 --- a/pkg/operator/operator/logging.go +++ b/pkg/operator/operator/logging.go @@ -114,7 +114,7 @@ func GetRealtimeAPILogger(apiName string, apiID string) (*zap.SugaredLogger, err return nil, err } - return initializeLogger(loggerCacheKey, apiSpec.Handler.LogLevel, map[string]interface{}{ + return initializeLogger(loggerCacheKey, userconfig.InfoLogLevel, map[string]interface{}{ "apiName": apiSpec.Name, "apiKind": apiSpec.Kind.String(), "apiID": apiSpec.ID, @@ -128,7 +128,7 @@ func GetRealtimeAPILoggerFromSpec(apiSpec *spec.API) (*zap.SugaredLogger, error) return logger, nil } - return initializeLogger(loggerCacheKey, apiSpec.Handler.LogLevel, map[string]interface{}{ + return initializeLogger(loggerCacheKey, userconfig.InfoLogLevel, map[string]interface{}{ "apiName": apiSpec.Name, "apiKind": apiSpec.Kind.String(), "apiID": apiSpec.ID, @@ -142,34 +142,7 @@ func GetJobLogger(jobKey spec.JobKey) (*zap.SugaredLogger, error) { return logger, nil } - apiName := jobKey.APIName - var logLevel userconfig.LogLevel - switch jobKey.Kind { - case userconfig.BatchAPIKind: - jobSpec, err := DownloadBatchJobSpec(jobKey) - if err != nil { - return nil, err - } - apiSpec, err := DownloadAPISpec(apiName, jobSpec.APIID) - if err != nil { - return nil, err - } - logLevel = apiSpec.Handler.LogLevel - case userconfig.TaskAPIKind: - jobSpec, err := DownloadTaskJobSpec(jobKey) - if err != nil { - return nil, err - } - apiSpec, err := DownloadAPISpec(apiName, jobSpec.APIID) - if err != nil { - return nil, err - } - logLevel = apiSpec.TaskDefinition.LogLevel - default: - return nil, errors.ErrorUnexpected("unexpected kind", jobKey.Kind.String()) - } - - return initializeLogger(loggerCacheKey, logLevel, map[string]interface{}{ + return initializeLogger(loggerCacheKey, userconfig.InfoLogLevel, map[string]interface{}{ "apiName": jobKey.APIName, "apiKind": jobKey.Kind.String(), "jobID": jobKey.ID, @@ -183,7 +156,7 @@ func GetJobLoggerFromSpec(apiSpec *spec.API, jobKey spec.JobKey) (*zap.SugaredLo return logger, nil } - return initializeLogger(loggerCacheKey, apiSpec.Handler.LogLevel, map[string]interface{}{ + return initializeLogger(loggerCacheKey, userconfig.InfoLogLevel, map[string]interface{}{ "apiName": jobKey.APIName, "apiKind": jobKey.Kind.String(), "jobID": jobKey.ID, diff --git a/pkg/operator/resources/asyncapi/api.go b/pkg/operator/resources/asyncapi/api.go index 534c75d672..8ce8abcae8 100644 --- a/pkg/operator/resources/asyncapi/api.go +++ b/pkg/operator/resources/asyncapi/api.go @@ -334,7 +334,7 @@ func getK8sResources(apiConfig userconfig.API) (resources, error) { } func applyK8sResources(api spec.API, prevK8sResources resources, queueURL string) error { - apiDeployment := apiDeploymentSpec(api, prevK8sResources.apiDeployment, queueURL) + apiDeployment := deploymentSpec(api, prevK8sResources.apiDeployment, queueURL) gatewayDeployment := gatewayDeploymentSpec(api, prevK8sResources.gatewayDeployment, queueURL) gatewayHPA, err := gatewayHPASpec(api) if err != nil { diff --git a/pkg/operator/resources/asyncapi/k8s_specs.go b/pkg/operator/resources/asyncapi/k8s_specs.go index c367522f9f..5653ceed7b 100644 --- a/pkg/operator/resources/asyncapi/k8s_specs.go +++ b/pkg/operator/resources/asyncapi/k8s_specs.go @@ -17,12 +17,9 @@ limitations under the License. package asyncapi import ( - "fmt" - "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/types/spec" - "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/cortexlabs/cortex/pkg/workloads" "istio.io/client-go/pkg/apis/networking/v1beta1" kapps "k8s.io/api/apps/v1" @@ -53,7 +50,7 @@ func gatewayDeploymentSpec(api spec.API, prevDeployment *kapps.Deployment, queue }, }, } - container := workloads.AsyncGatewayContainers(api, queueURL, volumeMounts) + container := workloads.AsyncGatewayContainer(api, queueURL, volumeMounts) return *k8s.Deployment(&k8s.DeploymentSpec{ Name: getGatewayK8sName(api.Name), @@ -90,7 +87,7 @@ func gatewayDeploymentSpec(api spec.API, prevDeployment *kapps.Deployment, queue Containers: []kcore.Container{container}, NodeSelector: workloads.NodeSelectors(), Tolerations: workloads.GenerateResourceTolerations(), - Affinity: workloads.GenerateNodeAffinities(api.Compute.NodeGroups), + Affinity: workloads.GenerateNodeAffinities(api.Pod.NodeGroups), Volumes: volumes, ServiceAccountName: workloads.ServiceAccountName, }, @@ -173,20 +170,14 @@ func gatewayVirtualServiceSpec(api spec.API) v1beta1.VirtualService { }) } -func apiDeploymentSpec(api spec.API, prevDeployment *kapps.Deployment, queueURL string) kapps.Deployment { +func deploymentSpec(api spec.API, prevDeployment *kapps.Deployment, queueURL string) kapps.Deployment { var ( containers []kcore.Container volumes []kcore.Volume ) - switch api.Handler.Type { - case userconfig.PythonHandlerType: - containers, volumes = workloads.AsyncPythonHandlerContainers(api, queueURL) - case userconfig.TensorFlowHandlerType: - containers, volumes = workloads.AsyncTensorflowHandlerContainers(api, queueURL) - default: - panic(fmt.Sprintf("invalid handler type: %s", api.Handler.Type)) - } + containers, volumes = workloads.UserPodContainers(api) + // TODO add the proxy as well return *k8s.Deployment(&k8s.DeploymentSpec{ Name: workloads.K8sName(api.Name), @@ -222,12 +213,12 @@ func apiDeploymentSpec(api spec.API, prevDeployment *kapps.Deployment, queueURL RestartPolicy: "Always", TerminationGracePeriodSeconds: pointer.Int64(_terminationGracePeriodSeconds), InitContainers: []kcore.Container{ - workloads.InitContainer(&api), + workloads.InitContainer(api), }, Containers: containers, NodeSelector: workloads.NodeSelectors(), Tolerations: workloads.GenerateResourceTolerations(), - Affinity: workloads.GenerateNodeAffinities(api.Compute.NodeGroups), + Affinity: workloads.GenerateNodeAffinities(api.Pod.NodeGroups), Volumes: volumes, ServiceAccountName: workloads.ServiceAccountName, }, diff --git a/pkg/operator/resources/job/batchapi/job.go b/pkg/operator/resources/job/batchapi/job.go index 8f46418279..481edd9562 100644 --- a/pkg/operator/resources/job/batchapi/job.go +++ b/pkg/operator/resources/job/batchapi/job.go @@ -145,7 +145,7 @@ func SubmitJob(apiName string, submission *schema.BatchJobSubmission) (*spec.Bat Timeout: timeout, DeadLetterQueue: deadLetterQueue, TTL: &kmeta.Duration{Duration: _batchJobTTL}, - NodeGroups: apiSpec.Compute.NodeGroups, + NodeGroups: apiSpec.Pod.NodeGroups, }, } diff --git a/pkg/operator/resources/job/taskapi/k8s_specs.go b/pkg/operator/resources/job/taskapi/k8s_specs.go index 1df9a94ad7..84292f77a0 100644 --- a/pkg/operator/resources/job/taskapi/k8s_specs.go +++ b/pkg/operator/resources/job/taskapi/k8s_specs.go @@ -59,17 +59,7 @@ func virtualServiceSpec(api *spec.API) *istioclientnetworking.VirtualService { } func k8sJobSpec(api *spec.API, job *spec.TaskJob) *kbatch.Job { - containers, volumes := workloads.TaskContainers(api) - for i, container := range containers { - if container.Name == workloads.APIContainerName { - containers[i].Env = append(container.Env, - kcore.EnvVar{ - Name: "CORTEX_TASK_SPEC", - Value: workloads.TaskSpecPath, - }, - ) - } - } + containers, volumes := workloads.UserPodContainers(*api) return k8s.Job(&k8s.JobSpec{ Name: job.JobKey.K8sName(), @@ -104,7 +94,7 @@ func k8sJobSpec(api *spec.API, job *spec.TaskJob) *kbatch.Job { Containers: containers, NodeSelector: workloads.NodeSelectors(), Tolerations: workloads.GenerateResourceTolerations(), - Affinity: workloads.GenerateNodeAffinities(api.Compute.NodeGroups), + Affinity: workloads.GenerateNodeAffinities(api.Pod.NodeGroups), Volumes: volumes, ServiceAccountName: workloads.ServiceAccountName, }, diff --git a/pkg/operator/resources/realtimeapi/api.go b/pkg/operator/resources/realtimeapi/api.go index ccf42044e9..7f01aa957c 100644 --- a/pkg/operator/resources/realtimeapi/api.go +++ b/pkg/operator/resources/realtimeapi/api.go @@ -18,17 +18,14 @@ package realtimeapi import ( "fmt" - "net/http" "path/filepath" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/lib/cron" "github.com/cortexlabs/cortex/pkg/lib/errors" - "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/parallel" "github.com/cortexlabs/cortex/pkg/lib/pointer" - "github.com/cortexlabs/cortex/pkg/lib/requests" autoscalerlib "github.com/cortexlabs/cortex/pkg/operator/lib/autoscaler" "github.com/cortexlabs/cortex/pkg/operator/lib/routines" "github.com/cortexlabs/cortex/pkg/operator/operator" @@ -444,66 +441,3 @@ func getDashboardURL(apiName string) string { return dashboardURL } - -func GetModelsMetadata(status *status.Status, handler *userconfig.Handler, apiEndpoint string) (*schema.TFLiveReloadingSummary, *schema.PythonModelSummary, error) { - if status.Updated.Ready+status.Stale.Ready == 0 { - return nil, nil, nil - } - - cachingEnabled := handler.Models != nil && handler.Models.CacheSize != nil && handler.Models.DiskCacheSize != nil - if handler.Type == userconfig.TensorFlowHandlerType && !cachingEnabled { - tfLiveReloadingSummary, err := getTFLiveReloadingSummary(apiEndpoint) - if err != nil { - return nil, nil, err - } - return tfLiveReloadingSummary, nil, nil - } - - if handler.Type == userconfig.PythonHandlerType && handler.MultiModelReloading != nil { - pythonModelSummary, err := getPythonModelSummary(apiEndpoint) - if err != nil { - return nil, nil, err - } - return nil, pythonModelSummary, nil - } - - return nil, nil, nil -} - -func getPythonModelSummary(apiEndpoint string) (*schema.PythonModelSummary, error) { - req, err := http.NewRequest("GET", apiEndpoint, nil) - if err != nil { - return nil, errors.Wrap(err, "unable to request api summary") - } - req.Header.Set("Content-Type", "application/json") - _, response, err := requests.MakeRequest(req) - if err != nil { - return nil, err - } - - var pythonModelSummary schema.PythonModelSummary - err = json.DecodeWithNumber(response, &pythonModelSummary) - if err != nil { - return nil, errors.Wrap(err, "unable to parse api summary response") - } - return &pythonModelSummary, nil -} - -func getTFLiveReloadingSummary(apiEndpoint string) (*schema.TFLiveReloadingSummary, error) { - req, err := http.NewRequest("GET", apiEndpoint, nil) - if err != nil { - return nil, errors.Wrap(err, "unable to request api summary") - } - req.Header.Set("Content-Type", "application/json") - _, response, err := requests.MakeRequest(req) - if err != nil { - return nil, err - } - - var tfLiveReloadingSummary schema.TFLiveReloadingSummary - err = json.DecodeWithNumber(response, &tfLiveReloadingSummary) - if err != nil { - return nil, errors.Wrap(err, "unable to parse api summary response") - } - return &tfLiveReloadingSummary, nil -} diff --git a/pkg/operator/resources/realtimeapi/k8s_specs.go b/pkg/operator/resources/realtimeapi/k8s_specs.go index 4b22848d45..3ab34f06bf 100644 --- a/pkg/operator/resources/realtimeapi/k8s_specs.go +++ b/pkg/operator/resources/realtimeapi/k8s_specs.go @@ -20,7 +20,6 @@ import ( "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/types/spec" - "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/cortexlabs/cortex/pkg/workloads" istioclientnetworking "istio.io/client-go/pkg/apis/networking/v1beta1" kapps "k8s.io/api/apps/v1" @@ -30,74 +29,12 @@ import ( var _terminationGracePeriodSeconds int64 = 60 // seconds func deploymentSpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Deployment { - switch api.Handler.Type { - case userconfig.TensorFlowHandlerType: - return tensorflowAPISpec(api, prevDeployment) - case userconfig.PythonHandlerType: - return pythonAPISpec(api, prevDeployment) - default: - return nil // unexpected - } -} - -func tensorflowAPISpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Deployment { - containers, volumes := workloads.TensorFlowHandlerContainers(api) - containers = append(containers, workloads.RequestMonitorContainer(api)) - - return k8s.Deployment(&k8s.DeploymentSpec{ - Name: workloads.K8sName(api.Name), - Replicas: getRequestedReplicasFromDeployment(api, prevDeployment), - MaxSurge: pointer.String(api.UpdateStrategy.MaxSurge), - MaxUnavailable: pointer.String(api.UpdateStrategy.MaxUnavailable), - Labels: map[string]string{ - "apiName": api.Name, - "apiKind": api.Kind.String(), - "apiID": api.ID, - "specID": api.SpecID, - "deploymentID": api.DeploymentID, - "handlerID": api.HandlerID, - "cortex.dev/api": "true", - }, - Annotations: api.ToK8sAnnotations(), - Selector: map[string]string{ - "apiName": api.Name, - "apiKind": api.Kind.String(), - }, - PodSpec: k8s.PodSpec{ - Labels: map[string]string{ - "apiName": api.Name, - "apiKind": api.Kind.String(), - "deploymentID": api.DeploymentID, - "handlerID": api.HandlerID, - "cortex.dev/api": "true", - }, - Annotations: map[string]string{ - "traffic.sidecar.istio.io/excludeOutboundIPRanges": "0.0.0.0/0", - }, - K8sPodSpec: kcore.PodSpec{ - RestartPolicy: "Always", - TerminationGracePeriodSeconds: pointer.Int64(_terminationGracePeriodSeconds), - InitContainers: []kcore.Container{ - workloads.InitContainer(api), - }, - Containers: containers, - NodeSelector: workloads.NodeSelectors(), - Tolerations: workloads.GenerateResourceTolerations(), - Affinity: workloads.GenerateNodeAffinities(api.Compute.NodeGroups), - Volumes: volumes, - ServiceAccountName: workloads.ServiceAccountName, - }, - }, - }) -} - -func pythonAPISpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Deployment { - containers, volumes := workloads.PythonHandlerContainers(api) - containers = append(containers, workloads.RequestMonitorContainer(api)) + containers, volumes := workloads.UserPodContainers(*api) + // TODO add the proxy as well return k8s.Deployment(&k8s.DeploymentSpec{ Name: workloads.K8sName(api.Name), - Replicas: getRequestedReplicasFromDeployment(api, prevDeployment), + Replicas: getRequestedReplicasFromDeployment(*api, prevDeployment), MaxSurge: pointer.String(api.UpdateStrategy.MaxSurge), MaxUnavailable: pointer.String(api.UpdateStrategy.MaxUnavailable), Labels: map[string]string{ @@ -129,12 +66,13 @@ func pythonAPISpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Deplo RestartPolicy: "Always", TerminationGracePeriodSeconds: pointer.Int64(_terminationGracePeriodSeconds), InitContainers: []kcore.Container{ - workloads.InitContainer(api), + workloads.KubexitInitContainer(), + workloads.InitContainer(*api), }, Containers: containers, NodeSelector: workloads.NodeSelectors(), Tolerations: workloads.GenerateResourceTolerations(), - Affinity: workloads.GenerateNodeAffinities(api.Compute.NodeGroups), + Affinity: workloads.GenerateNodeAffinities(api.Pod.NodeGroups), Volumes: volumes, ServiceAccountName: workloads.ServiceAccountName, }, @@ -185,7 +123,7 @@ func virtualServiceSpec(api *spec.API) *istioclientnetworking.VirtualService { }) } -func getRequestedReplicasFromDeployment(api *spec.API, deployment *kapps.Deployment) int32 { +func getRequestedReplicasFromDeployment(api spec.API, deployment *kapps.Deployment) int32 { requestedReplicas := api.Autoscaling.InitReplicas if deployment != nil && deployment.Spec.Replicas != nil && *deployment.Spec.Replicas > 0 { diff --git a/pkg/types/spec/api.go b/pkg/types/spec/api.go index 818388d225..a3fa2eb3c9 100644 --- a/pkg/types/spec/api.go +++ b/pkg/types/spec/api.go @@ -38,11 +38,13 @@ type API struct { SpecID string `json:"spec_id"` HandlerID string `json:"handler_id"` DeploymentID string `json:"deployment_id"` - Key string `json:"key"` - HandlerKey string `json:"handler_key"` + ProjectID string `json:"project_id"` + + Key string `json:"key"` + HandlerKey string `json:"handler_key"` + LastUpdated int64 `json:"last_updated"` MetadataRoot string `json:"metadata_root"` - ProjectID string `json:"project_id"` } /* @@ -50,9 +52,9 @@ APIID (uniquely identifies an api configuration for a given deployment) * SpecID (uniquely identifies api configuration specified by user) * HandlerID (used to determine when rolling updates need to happen) * Resource - * Handler - * TaskDefinition - * Compute + * Containers + * Compute + * Pod * ProjectID * Deployment Strategy * Autoscaling diff --git a/pkg/types/spec/validations.go b/pkg/types/spec/validations.go index dfdfbc66d5..1764dd729a 100644 --- a/pkg/types/spec/validations.go +++ b/pkg/types/spec/validations.go @@ -578,6 +578,8 @@ func validateContainers( } } } + + return nil } func validateAutoscaling(api *userconfig.API) error { diff --git a/pkg/types/userconfig/api.go b/pkg/types/userconfig/api.go index 472d20c946..bbb9172bf6 100644 --- a/pkg/types/userconfig/api.go +++ b/pkg/types/userconfig/api.go @@ -22,6 +22,7 @@ import ( "time" "github.com/cortexlabs/cortex/pkg/lib/k8s" + "github.com/cortexlabs/cortex/pkg/lib/sets/strset" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/lib/urls" "github.com/cortexlabs/yaml" @@ -433,6 +434,14 @@ func GetTotalComputeFromContainers(containers []Container) Compute { return compute } +func GetContainerNames(containers []Container) strset.Set { + containerNames := strset.New() + for _, container := range containers { + containerNames.Add(container.Name) + } + return containerNames +} + func (api *API) TelemetryEvent() map[string]interface{} { event := map[string]interface{}{"kind": api.Kind} diff --git a/pkg/workloads/helpers.go b/pkg/workloads/helpers.go index c8da9229e2..5d6020d051 100644 --- a/pkg/workloads/helpers.go +++ b/pkg/workloads/helpers.go @@ -18,8 +18,10 @@ package workloads import ( "fmt" + "path" + "strings" - "github.com/cortexlabs/cortex/pkg/types/userconfig" + "github.com/cortexlabs/cortex/pkg/lib/k8s" kcore "k8s.io/api/core/v1" ) @@ -57,7 +59,7 @@ func FileExistsProbe(fileName string) *kcore.Probe { } } -func socketExistsProbe(socketName string) *kcore.Probe { +func SocketExistsProbe(socketName string) *kcore.Probe { return &kcore.Probe{ InitialDelaySeconds: 3, TimeoutSeconds: 5, @@ -72,30 +74,86 @@ func socketExistsProbe(socketName string) *kcore.Probe { } } -func nginxGracefulStopper(apiKind userconfig.Kind) *kcore.Lifecycle { - if apiKind == userconfig.RealtimeAPIKind { - return &kcore.Lifecycle{ - PreStop: &kcore.Handler{ - Exec: &kcore.ExecAction{ - // the sleep is required to wait for any k8s-related race conditions - // as described in https://medium.com/codecademy-engineering/kubernetes-nginx-and-zero-downtime-in-production-2c910c6a5ed8 - Command: []string{"/bin/sh", "-c", "sleep 5; /usr/sbin/nginx -s quit; while pgrep -x nginx; do sleep 1; done"}, +func baseClusterEnvVars() []kcore.EnvFromSource { + envVars := []kcore.EnvFromSource{ + { + ConfigMapRef: &kcore.ConfigMapEnvSource{ + LocalObjectReference: kcore.LocalObjectReference{ + Name: "env-vars", }, }, - } + }, } - return nil + + return envVars } -func waitAPIContainerToStop(apiKind userconfig.Kind) *kcore.Lifecycle { - if apiKind == userconfig.RealtimeAPIKind { - return &kcore.Lifecycle{ - PreStop: &kcore.Handler{ - Exec: &kcore.ExecAction{ - Command: []string{"/bin/sh", "-c", fmt.Sprintf("while curl localhost:%s/nginx_status; do sleep 1; done", DefaultPortStr)}, +func getKubexitEnvVars(containerName string, deathDeps []string, birthDeps []string) []kcore.EnvVar { + envVars := []kcore.EnvVar{ + { + Name: "KUBEXIT_NAME", + Value: containerName, + }, + { + Name: "KUBEXIT_GRAVEYARD", + Value: _kubexitGraveyardMountPath, + }, + } + + if deathDeps != nil { + envVars = append(envVars, + kcore.EnvVar{ + Name: "KUBEXIT_DEATH_DEPS", + Value: strings.Join(deathDeps, ","), + }, + // kcore.EnvVar{ + // Name: "KUBEXIT_IGNORE_CODE_ON_DEATH_DEPS", + // Value: "true", + // }, + ) + } + + if birthDeps != nil { + envVars = append(envVars, + kcore.EnvVar{ + Name: "KUBEXIT_BIRTH_DEPS", + Value: strings.Join(birthDeps, ","), + }, + // kcore.EnvVar{ + // Name: "KUBEXIT_IGNORE_CODE_ON_DEATH_DEPS", + // Value: "true", + // }, + ) + } + + return envVars +} + +func defaultVolumes() []kcore.Volume { + return []kcore.Volume{ + k8s.EmptyDirVolume(_emptyDirVolumeName), + k8s.EmptyDirVolume(_kubexitGraveyardName), + { + Name: "client-config", + VolumeSource: kcore.VolumeSource{ + ConfigMap: &kcore.ConfigMapVolumeSource{ + LocalObjectReference: kcore.LocalObjectReference{ + Name: "client-config", + }, }, }, - } + }, + } +} + +func defaultVolumeMounts() []kcore.VolumeMount { + return []kcore.VolumeMount{ + k8s.EmptyDirVolumeMount(_emptyDirVolumeName, _emptyDirMountPath), + k8s.EmptyDirVolumeMount(_kubexitGraveyardName, _kubexitGraveyardMountPath), + { + Name: "client-config", + MountPath: path.Join(_clientConfigDir, "cli.yaml"), + SubPath: "cli.yaml", + }, } - return nil } diff --git a/pkg/workloads/init.go b/pkg/workloads/init.go new file mode 100644 index 0000000000..50750868f6 --- /dev/null +++ b/pkg/workloads/init.go @@ -0,0 +1,174 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workloads + +import ( + "encoding/base64" + "encoding/json" + + "github.com/cortexlabs/cortex/pkg/config" + "github.com/cortexlabs/cortex/pkg/lib/aws" + "github.com/cortexlabs/cortex/pkg/types/spec" + "github.com/cortexlabs/cortex/pkg/types/userconfig" + kcore "k8s.io/api/core/v1" +) + +const ( + APISpecPath = "/mnt/spec/spec.json" + TaskSpecPath = "/mnt/spec/task.json" + BatchSpecPath = "/mnt/spec/batch.json" +) + +const ( + _downloaderInitContainerName = "downloader" + _downloaderLastLog = "downloading the serving image(s)" + _kubexitInitContainerName = "kubexit" +) + +func KubexitInitContainer() kcore.Container { + return kcore.Container{ + Name: _kubexitInitContainerName, + Image: config.ClusterConfig.ImageKubexit, + ImagePullPolicy: kcore.PullAlways, + Command: []string{"cp", "/bin/kubexit", "/mnt/kubexit"}, + VolumeMounts: defaultVolumeMounts(), + } +} + +func TaskInitContainer(api *spec.API, job *spec.TaskJob) kcore.Container { + downloadConfig := downloadContainerConfig{ + LastLog: _downloaderLastLog, + DownloadArgs: []downloadContainerArg{ + { + From: aws.S3Path(config.ClusterConfig.Bucket, api.Key), + To: APISpecPath, + Unzip: false, + ToFile: true, + ItemName: "the api spec", + HideFromLog: true, + HideUnzippingLog: true, + }, + { + From: aws.S3Path(config.ClusterConfig.Bucket, job.SpecFilePath(config.ClusterConfig.ClusterUID)), + To: TaskSpecPath, + Unzip: false, + ToFile: true, + ItemName: "the task spec", + HideFromLog: true, + HideUnzippingLog: true, + }, + }, + } + + downloadArgsBytes, _ := json.Marshal(downloadConfig) + downloadArgs := base64.URLEncoding.EncodeToString(downloadArgsBytes) + + return kcore.Container{ + Name: _downloaderInitContainerName, + Image: config.ClusterConfig.ImageDownloader, + ImagePullPolicy: kcore.PullAlways, + Args: []string{"--download=" + downloadArgs}, + EnvFrom: baseClusterEnvVars(), + Env: []kcore.EnvVar{ + { + Name: "CORTEX_LOG_LEVEL", + Value: userconfig.InfoLogLevel.String(), + }, + }, + VolumeMounts: defaultVolumeMounts(), + } +} + +func BatchInitContainer(api *spec.API, job *spec.BatchJob) kcore.Container { + downloadConfig := downloadContainerConfig{ + LastLog: _downloaderLastLog, + DownloadArgs: []downloadContainerArg{ + { + From: aws.S3Path(config.ClusterConfig.Bucket, api.Key), + To: APISpecPath, + Unzip: false, + ToFile: true, + ItemName: "the api spec", + HideFromLog: true, + HideUnzippingLog: true, + }, + { + From: aws.S3Path(config.ClusterConfig.Bucket, job.SpecFilePath(config.ClusterConfig.ClusterUID)), + To: BatchSpecPath, + Unzip: false, + ToFile: true, + ItemName: "the job spec", + HideFromLog: true, + HideUnzippingLog: true, + }, + }, + } + + downloadArgsBytes, _ := json.Marshal(downloadConfig) + downloadArgs := base64.URLEncoding.EncodeToString(downloadArgsBytes) + + return kcore.Container{ + Name: _downloaderInitContainerName, + Image: config.ClusterConfig.ImageDownloader, + ImagePullPolicy: kcore.PullAlways, + Args: []string{"--download=" + downloadArgs}, + EnvFrom: baseClusterEnvVars(), + Env: []kcore.EnvVar{ + { + Name: "CORTEX_LOG_LEVEL", + Value: userconfig.InfoLogLevel.String(), + }, + }, + VolumeMounts: defaultVolumeMounts(), + } +} + +// for async and realtime apis +func InitContainer(api spec.API) kcore.Container { + downloadConfig := downloadContainerConfig{ + LastLog: _downloaderLastLog, + DownloadArgs: []downloadContainerArg{ + { + From: aws.S3Path(config.ClusterConfig.Bucket, api.HandlerKey), + To: APISpecPath, + Unzip: false, + ToFile: true, + ItemName: "the api spec", + HideFromLog: true, + HideUnzippingLog: true, + }, + }, + } + + downloadArgsBytes, _ := json.Marshal(downloadConfig) + downloadArgs := base64.URLEncoding.EncodeToString(downloadArgsBytes) + + return kcore.Container{ + Name: _downloaderInitContainerName, + Image: config.ClusterConfig.ImageDownloader, + ImagePullPolicy: kcore.PullAlways, + Args: []string{"--download=" + downloadArgs}, + EnvFrom: baseClusterEnvVars(), + Env: []kcore.EnvVar{ + { + Name: "CORTEX_LOG_LEVEL", + Value: userconfig.InfoLogLevel.String(), + }, + }, + VolumeMounts: defaultVolumeMounts(), + } +} diff --git a/pkg/workloads/k8s.go b/pkg/workloads/k8s.go index 625e14bd31..d9a9d46455 100644 --- a/pkg/workloads/k8s.go +++ b/pkg/workloads/k8s.go @@ -17,15 +17,8 @@ limitations under the License. package workloads import ( - "encoding/base64" - "encoding/json" - "fmt" - "path" - "strings" - "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/consts" - "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/pointer" s "github.com/cortexlabs/cortex/pkg/lib/strings" @@ -39,38 +32,23 @@ import ( const ( DefaultPortInt32 = int32(8888) - DefaultPortStr = "8888" DefaultRequestMonitorPortStr = "15000" DefaultRequestMonitorPortInt32 = int32(15000) - APIContainerName = "api" ServiceAccountName = "default" - APISpecPath = "/mnt/spec/spec.json" - TaskSpecPath = "/mnt/spec/task.json" - BatchSpecPath = "/mnt/spec/batch.json" ) const ( - _clientConfigDir = "/mnt/client" - _emptyDirMountPath = "/mnt" - _emptyDirVolumeName = "mnt" - _tfServingContainerName = "serve" - _requestMonitorContainerName = "request-monitor" - _gatewayContainerName = "gateway" - _downloaderInitContainerName = "downloader" - _downloaderLastLog = "downloading the %s serving image" - _neuronRTDContainerName = "neuron-rtd" - _tfBaseServingPortInt32, _tfBaseServingPortStr = int32(9000), "9000" - _tfServingHost = "localhost" - _tfServingEmptyModelConfig = "/etc/tfs/model_config_server.conf" - _tfServingMaxNumLoadRetries = "0" // maximum retries to load a model that didn't get loaded the first time - _tfServingLoadTimeMicros = "30000000" // 30 seconds (how much time a model can take to load into memory) - _tfServingBatchConfig = "/etc/tfs/batch_config.conf" - _apiReadinessFile = "/mnt/workspace/api_readiness.txt" - _neuronRTDSocket = "/sock/neuron.sock" - _requestMonitorReadinessFile = "/request_monitor_ready.txt" - _kubexitInitContainerName = "kubexit" - _kubexitGraveyardName = "graveyard" - _kubexitGraveyardMountPath = "/graveyard" + _clientConfigDir = "/mnt/client" + _emptyDirMountPath = "/mnt" + _emptyDirVolumeName = "mnt" + + _gatewayContainerName = "gateway" + + _neuronRTDContainerName = "neuron-rtd" + _neuronRTDSocket = "/sock/neuron.sock" + + _kubexitGraveyardName = "graveyard" + _kubexitGraveyardMountPath = "/graveyard" ) var ( @@ -84,222 +62,7 @@ var ( _hugePagesMemPerInf = int64(128 * 2 * 1024 * 1024) // bytes ) -func TaskInitContainer(api *spec.API, job *spec.TaskJob) kcore.Container { - downloadConfig := downloadContainerConfig{ - LastLog: fmt.Sprintf(_downloaderLastLog, "task"), - DownloadArgs: []downloadContainerArg{ - { - From: aws.S3Path(config.ClusterConfig.Bucket, api.Key), - To: APISpecPath, - Unzip: false, - ToFile: true, - ItemName: "the api spec", - HideFromLog: true, - HideUnzippingLog: true, - }, - { - From: aws.S3Path(config.ClusterConfig.Bucket, job.SpecFilePath(config.ClusterConfig.ClusterUID)), - To: TaskSpecPath, - Unzip: false, - ToFile: true, - ItemName: "the task spec", - HideFromLog: true, - HideUnzippingLog: true, - }, - }, - } - - downloadArgsBytes, _ := json.Marshal(downloadConfig) - downloadArgs := base64.URLEncoding.EncodeToString(downloadArgsBytes) - - return kcore.Container{ - Name: _downloaderInitContainerName, - Image: config.ClusterConfig.ImageDownloader, - ImagePullPolicy: kcore.PullAlways, - Args: []string{"--download=" + downloadArgs}, - EnvFrom: baseEnvVars(), - Env: downloaderEnvVars(api), - VolumeMounts: defaultVolumeMounts(), - } -} - -func KubexitInitContainer() kcore.Container { - return kcore.Container{ - Name: _kubexitInitContainerName, - Image: config.ClusterConfig.ImageKubexit, - ImagePullPolicy: kcore.PullAlways, - Command: []string{"cp", "/bin/kubexit", "/mnt/kubexit"}, - VolumeMounts: defaultVolumeMounts(), - } -} - -func BatchInitContainer(api *spec.API, job *spec.BatchJob) kcore.Container { - downloadConfig := downloadContainerConfig{ - LastLog: fmt.Sprintf(_downloaderLastLog, api.Handler.Type.String()), - DownloadArgs: []downloadContainerArg{ - { - From: aws.S3Path(config.ClusterConfig.Bucket, api.Key), - To: APISpecPath, - Unzip: false, - ToFile: true, - ItemName: "the api spec", - HideFromLog: true, - HideUnzippingLog: true, - }, - { - From: aws.S3Path(config.ClusterConfig.Bucket, job.SpecFilePath(config.ClusterConfig.ClusterUID)), - To: BatchSpecPath, - Unzip: false, - ToFile: true, - ItemName: "the job spec", - HideFromLog: true, - HideUnzippingLog: true, - }, - }, - } - - downloadArgsBytes, _ := json.Marshal(downloadConfig) - downloadArgs := base64.URLEncoding.EncodeToString(downloadArgsBytes) - - return kcore.Container{ - Name: _downloaderInitContainerName, - Image: config.ClusterConfig.ImageDownloader, - ImagePullPolicy: kcore.PullAlways, - Args: []string{"--download=" + downloadArgs}, - EnvFrom: baseEnvVars(), - Env: downloaderEnvVars(api), - VolumeMounts: defaultVolumeMounts(), - } -} - -// for async and realtime apis -func InitContainer(api *spec.API) kcore.Container { - downloadConfig := downloadContainerConfig{ - LastLog: fmt.Sprintf(_downloaderLastLog, api.Handler.Type.String()), - DownloadArgs: []downloadContainerArg{ - { - From: aws.S3Path(config.ClusterConfig.Bucket, api.HandlerKey), - To: APISpecPath, - Unzip: false, - ToFile: true, - ItemName: "the api spec", - HideFromLog: true, - HideUnzippingLog: true, - }, - }, - } - - downloadArgsBytes, _ := json.Marshal(downloadConfig) - downloadArgs := base64.URLEncoding.EncodeToString(downloadArgsBytes) - - return kcore.Container{ - Name: _downloaderInitContainerName, - Image: config.ClusterConfig.ImageDownloader, - ImagePullPolicy: kcore.PullAlways, - Args: []string{"--download=" + downloadArgs}, - EnvFrom: baseEnvVars(), - Env: downloaderEnvVars(api), - VolumeMounts: defaultVolumeMounts(), - } -} - -func TaskContainers(api *spec.API) ([]kcore.Container, []kcore.Volume) { - apiPodResourceList := kcore.ResourceList{} - apiPodResourceLimitsList := kcore.ResourceList{} - apiPodVolumeMounts := defaultVolumeMounts() - volumes := DefaultVolumes() - var containers []kcore.Container - - if api.Compute.GPU > 0 { - apiPodResourceList["nvidia.com/gpu"] = *kresource.NewQuantity(api.Compute.GPU, kresource.DecimalSI) - apiPodResourceLimitsList["nvidia.com/gpu"] = *kresource.NewQuantity(api.Compute.GPU, kresource.DecimalSI) - } else if api.Compute.Inf > 0 { - volumes = append(volumes, kcore.Volume{ - Name: "neuron-sock", - }) - rtdVolumeMounts := []kcore.VolumeMount{ - k8s.EmptyDirVolumeMount(_emptyDirVolumeName, _emptyDirMountPath), - { - Name: "neuron-sock", - MountPath: "/sock", - }, - { - Name: _kubexitGraveyardName, - MountPath: _kubexitGraveyardMountPath, - }, - } - apiPodVolumeMounts = append(apiPodVolumeMounts, rtdVolumeMounts...) - neuronContainer := neuronRuntimeDaemonContainer(api, rtdVolumeMounts, getKubexitEnvVars(_neuronRTDContainerName, APIContainerName)) - - if api.Compute.CPU != nil { - q1, q2 := k8s.SplitInTwo(k8s.QuantityPtr(api.Compute.CPU.Quantity.DeepCopy())) - apiPodResourceList[kcore.ResourceCPU] = *q1 - neuronContainer.Resources.Requests[kcore.ResourceCPU] = *q2 - } - - if api.Compute.Mem != nil { - q1, q2 := k8s.SplitInTwo(k8s.QuantityPtr(api.Compute.Mem.Quantity.DeepCopy())) - apiPodResourceList[kcore.ResourceMemory] = *q1 - neuronContainer.Resources.Requests[kcore.ResourceMemory] = *q2 - } - - containers = append(containers, neuronContainer) - } else { - if api.Compute.CPU != nil { - apiPodResourceList[kcore.ResourceCPU] = api.Compute.CPU.DeepCopy() - } - if api.Compute.Mem != nil { - apiPodResourceList[kcore.ResourceMemory] = api.Compute.Mem.DeepCopy() - } - } - - if api.TaskDefinition.ShmSize != nil { - volumes = append(volumes, kcore.Volume{ - Name: "dshm", - VolumeSource: kcore.VolumeSource{ - EmptyDir: &kcore.EmptyDirVolumeSource{ - Medium: kcore.StorageMediumMemory, - SizeLimit: k8s.QuantityPtr(api.TaskDefinition.ShmSize.Quantity), - }, - }, - }) - apiPodVolumeMounts = append(apiPodVolumeMounts, kcore.VolumeMount{ - Name: "dshm", - MountPath: "/dev/shm", - }) - } - - containers = append(containers, kcore.Container{ - Name: APIContainerName, - Image: api.TaskDefinition.Image, - ImagePullPolicy: kcore.PullAlways, - Env: taskEnvVars(api), - EnvFrom: baseEnvVars(), - VolumeMounts: apiPodVolumeMounts, - Resources: kcore.ResourceRequirements{ - Requests: apiPodResourceList, - Limits: apiPodResourceLimitsList, - }, - Ports: []kcore.ContainerPort{ - {ContainerPort: DefaultPortInt32}, - }, - SecurityContext: &kcore.SecurityContext{ - Privileged: pointer.Bool(true), - }}, - ) - - return containers, volumes -} - -func AsyncPythonHandlerContainers(api spec.API, queueURL string) ([]kcore.Container, []kcore.Volume) { - return pythonHandlerContainers(&api, getAsyncAPIEnvVars(api, queueURL), false) -} - -func AsyncTensorflowHandlerContainers(api spec.API, queueURL string) ([]kcore.Container, []kcore.Volume) { - return tensorFlowHandlerContainers(&api, getAsyncAPIEnvVars(api, queueURL), false) -} - -func AsyncGatewayContainers(api spec.API, queueURL string, volumeMounts []kcore.VolumeMount) kcore.Container { +func AsyncGatewayContainer(api spec.API, queueURL string, volumeMounts []kcore.VolumeMount) kcore.Container { return kcore.Container{ Name: _gatewayContainerName, Image: config.ClusterConfig.ImageAsyncGateway, @@ -316,7 +79,7 @@ func AsyncGatewayContainers(api spec.API, queueURL string, volumeMounts []kcore. Env: []kcore.EnvVar{ { Name: "CORTEX_LOG_LEVEL", - Value: strings.ToUpper(api.Handler.LogLevel.String()), + Value: userconfig.InfoLogLevel.String(), }, }, Resources: kcore.ResourceRequirements{ @@ -345,682 +108,106 @@ func AsyncGatewayContainers(api spec.API, queueURL string, volumeMounts []kcore. } } -func PythonHandlerContainers(api *spec.API) ([]kcore.Container, []kcore.Volume) { - return pythonHandlerContainers(api, apiContainerEnvVars(api), false) -} - -func PythonHandlerJobContainers(api *spec.API) ([]kcore.Container, []kcore.Volume) { - return pythonHandlerContainers(api, apiContainerEnvVars(api), true) -} - -func pythonHandlerContainers(api *spec.API, envVars []kcore.EnvVar, isJob bool) ([]kcore.Container, []kcore.Volume) { - apiPodResourceList := kcore.ResourceList{} - apiPodResourceLimitsList := kcore.ResourceList{} - apiPodVolumeMounts := defaultVolumeMounts() - volumes := DefaultVolumes() +func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { + volumes := defaultVolumes() - var containers []kcore.Container - - var neuronRTDEnvVars []kcore.EnvVar - if isJob { - envVars = append(envVars, getKubexitEnvVars(APIContainerName)...) - volumes = append(volumes, k8s.EmptyDirVolume(_kubexitGraveyardName)) - apiPodVolumeMounts = append(apiPodVolumeMounts, - kcore.VolumeMount{Name: _kubexitGraveyardName, MountPath: _kubexitGraveyardMountPath}, - ) - neuronRTDEnvVars = getKubexitEnvVars(_neuronRTDContainerName, APIContainerName) - } - - if api.Compute.Inf == 0 { - if api.Compute.CPU != nil { - userPodCPURequest := k8s.QuantityPtr(api.Compute.CPU.Quantity.DeepCopy()) - if api.Kind == userconfig.RealtimeAPIKind { - userPodCPURequest.Sub(_requestMonitorCPURequest) - } - apiPodResourceList[kcore.ResourceCPU] = *userPodCPURequest - } - - if api.Compute.Mem != nil { - userPodMemRequest := k8s.QuantityPtr(api.Compute.Mem.Quantity.DeepCopy()) - if api.Kind == userconfig.RealtimeAPIKind { - userPodMemRequest.Sub(_requestMonitorMemRequest) - } - apiPodResourceList[kcore.ResourceMemory] = *userPodMemRequest - } - - if api.Compute.GPU > 0 { - apiPodResourceList["nvidia.com/gpu"] = *kresource.NewQuantity(api.Compute.GPU, kresource.DecimalSI) - apiPodResourceLimitsList["nvidia.com/gpu"] = *kresource.NewQuantity(api.Compute.GPU, kresource.DecimalSI) - } - } else { - volumes = append(volumes, kcore.Volume{ - Name: "neuron-sock", - }) - rtdVolumeMounts := []kcore.VolumeMount{ - { - Name: "neuron-sock", - MountPath: "/sock", - }, - } - - apiPodVolumeMounts = append(apiPodVolumeMounts, rtdVolumeMounts...) - - if isJob { - rtdVolumeMounts = append(rtdVolumeMounts, - k8s.EmptyDirVolumeMount(_emptyDirVolumeName, _emptyDirMountPath), - kcore.VolumeMount{Name: _kubexitGraveyardName, MountPath: _kubexitGraveyardMountPath}, - ) - } - - neuronContainer := neuronRuntimeDaemonContainer(api, rtdVolumeMounts, neuronRTDEnvVars) - - if api.Compute.CPU != nil { - userPodCPURequest := k8s.QuantityPtr(api.Compute.CPU.Quantity.DeepCopy()) - if api.Kind == userconfig.RealtimeAPIKind { - userPodCPURequest.Sub(_requestMonitorCPURequest) - } - q1, q2 := k8s.SplitInTwo(userPodCPURequest) - apiPodResourceList[kcore.ResourceCPU] = *q1 - neuronContainer.Resources.Requests[kcore.ResourceCPU] = *q2 - } - - if api.Compute.Mem != nil { - userPodMemRequest := k8s.QuantityPtr(api.Compute.Mem.Quantity.DeepCopy()) - if api.Kind == userconfig.RealtimeAPIKind { - userPodMemRequest.Sub(_requestMonitorMemRequest) - } - q1, q2 := k8s.SplitInTwo(userPodMemRequest) - apiPodResourceList[kcore.ResourceMemory] = *q1 - neuronContainer.Resources.Requests[kcore.ResourceMemory] = *q2 - } - containers = append(containers, neuronContainer) - } - - if api.Handler.ShmSize != nil { + defaultMounts := []kcore.VolumeMount{} + if api.Pod.ShmSize != nil { volumes = append(volumes, kcore.Volume{ Name: "dshm", VolumeSource: kcore.VolumeSource{ EmptyDir: &kcore.EmptyDirVolumeSource{ Medium: kcore.StorageMediumMemory, - SizeLimit: k8s.QuantityPtr(api.Handler.ShmSize.Quantity), + SizeLimit: k8s.QuantityPtr(api.Pod.ShmSize.Quantity), }, }, }) - apiPodVolumeMounts = append(apiPodVolumeMounts, kcore.VolumeMount{ + defaultMounts = append(defaultMounts, kcore.VolumeMount{ Name: "dshm", MountPath: "/dev/shm", }) } - containers = append(containers, kcore.Container{ - Name: APIContainerName, - Image: api.Handler.Image, - ImagePullPolicy: kcore.PullAlways, - Env: envVars, - EnvFrom: baseEnvVars(), - VolumeMounts: apiPodVolumeMounts, - ReadinessProbe: FileExistsProbe(_apiReadinessFile), - Lifecycle: nginxGracefulStopper(api.Kind), - Resources: kcore.ResourceRequirements{ - Requests: apiPodResourceList, - Limits: apiPodResourceLimitsList, - }, - Ports: []kcore.ContainerPort{ - {ContainerPort: DefaultPortInt32}, - }, - SecurityContext: &kcore.SecurityContext{ - Privileged: pointer.Bool(true), - }}, - ) - - return containers, volumes -} - -func TensorFlowHandlerContainers(api *spec.API) ([]kcore.Container, []kcore.Volume) { - return tensorFlowHandlerContainers(api, apiContainerEnvVars(api), false) -} - -func TensorFlowHandlerJobContainers(api *spec.API) ([]kcore.Container, []kcore.Volume) { - return tensorFlowHandlerContainers(api, apiContainerEnvVars(api), true) -} - -func tensorFlowHandlerContainers(api *spec.API, envVars []kcore.EnvVar, isJob bool) ([]kcore.Container, []kcore.Volume) { - apiResourceList := kcore.ResourceList{} - tfServingResourceList := kcore.ResourceList{} - tfServingLimitsList := kcore.ResourceList{} - volumeMounts := defaultVolumeMounts() - volumes := DefaultVolumes() + var containers []kcore.Container + containerNames := userconfig.GetContainerNames(api.Pod.Containers) + for _, container := range api.Pod.Containers { + containerResourceList := kcore.ResourceList{} + containerResourceLimitsList := kcore.ResourceList{} - var neuronRTDEnvVars []kcore.EnvVar - tfServingEnvVars := tensorflowServingEnvVars(api) - if isJob { - envVars = append(envVars, getKubexitEnvVars(APIContainerName)...) - tfServingEnvVars = append(tfServingEnvVars, getKubexitEnvVars(_tfServingContainerName, APIContainerName)...) + if container.Compute.CPU != nil { + containerResourceList[kcore.ResourceCPU] = *k8s.QuantityPtr(container.Compute.CPU.Quantity.DeepCopy()) + } - volumes = append(volumes, k8s.EmptyDirVolume(_kubexitGraveyardName)) - volumeMounts = append(volumeMounts, - kcore.VolumeMount{Name: _kubexitGraveyardName, MountPath: _kubexitGraveyardMountPath}, - ) - neuronRTDEnvVars = getKubexitEnvVars(_neuronRTDContainerName, APIContainerName) - } + if container.Compute.Mem != nil { + containerResourceList[kcore.ResourceMemory] = *k8s.QuantityPtr(container.Compute.Mem.Quantity.DeepCopy()) + } - var containers []kcore.Container - if api.Compute.Inf == 0 { - if api.Compute.CPU != nil { - userPodCPURequest := k8s.QuantityPtr(api.Compute.CPU.Quantity.DeepCopy()) - if api.Kind == userconfig.RealtimeAPIKind { - userPodCPURequest.Sub(_requestMonitorCPURequest) - } - q1, q2 := k8s.SplitInTwo(userPodCPURequest) - apiResourceList[kcore.ResourceCPU] = *q1 - tfServingResourceList[kcore.ResourceCPU] = *q2 + if container.Compute.GPU > 0 { + containerResourceList["nvidia.com/gpu"] = *kresource.NewQuantity(container.Compute.GPU, kresource.DecimalSI) + containerResourceLimitsList["nvidia.com/gpu"] = *kresource.NewQuantity(container.Compute.GPU, kresource.DecimalSI) } - if api.Compute.Mem != nil { - userPodMemRequest := k8s.QuantityPtr(api.Compute.Mem.Quantity.DeepCopy()) - if api.Kind == userconfig.RealtimeAPIKind { - userPodMemRequest.Sub(_requestMonitorMemRequest) + containerVolumeMounts := append(defaultVolumeMounts(), defaultMounts...) + if container.Compute.Inf > 0 { + volumes = append(volumes, kcore.Volume{ + Name: "neuron-sock", + }) + rtdVolumeMounts := []kcore.VolumeMount{ + { + Name: "neuron-sock", + MountPath: "/sock", + }, } - q1, q2 := k8s.SplitInTwo(userPodMemRequest) - apiResourceList[kcore.ResourceMemory] = *q1 - tfServingResourceList[kcore.ResourceMemory] = *q2 - } - if api.Compute.GPU > 0 { - tfServingResourceList["nvidia.com/gpu"] = *kresource.NewQuantity(api.Compute.GPU, kresource.DecimalSI) - tfServingLimitsList["nvidia.com/gpu"] = *kresource.NewQuantity(api.Compute.GPU, kresource.DecimalSI) - } - } else { - volumes = append(volumes, kcore.Volume{ - Name: "neuron-sock", - }) - rtdVolumeMounts := []kcore.VolumeMount{ - { - Name: "neuron-sock", - MountPath: "/sock", - }, - } - volumeMounts = append(volumeMounts, rtdVolumeMounts...) + containerVolumeMounts = append(containerVolumeMounts, rtdVolumeMounts...) - if isJob { rtdVolumeMounts = append(rtdVolumeMounts, k8s.EmptyDirVolumeMount(_emptyDirVolumeName, _emptyDirMountPath), kcore.VolumeMount{Name: _kubexitGraveyardName, MountPath: _kubexitGraveyardMountPath}, ) - } - neuronContainer := neuronRuntimeDaemonContainer(api, rtdVolumeMounts, neuronRTDEnvVars) - - if api.Compute.CPU != nil { - userPodCPURequest := k8s.QuantityPtr(api.Compute.CPU.Quantity.DeepCopy()) - if api.Kind == userconfig.RealtimeAPIKind { - userPodCPURequest.Sub(_requestMonitorCPURequest) - } - q1, q2, q3 := k8s.SplitInThree(userPodCPURequest) - apiResourceList[kcore.ResourceCPU] = *q1 - tfServingResourceList[kcore.ResourceCPU] = *q2 - neuronContainer.Resources.Requests[kcore.ResourceCPU] = *q3 + neuronRTDEnvVars := getKubexitEnvVars(_neuronRTDContainerName, containerNames.Slice(), nil) + containers = append(containers, neuronRuntimeDaemonContainer(container.Compute.Inf, rtdVolumeMounts, neuronRTDEnvVars)) } - if api.Compute.Mem != nil { - userPodMemRequest := k8s.QuantityPtr(api.Compute.Mem.Quantity.DeepCopy()) - if api.Kind == userconfig.RealtimeAPIKind { - userPodMemRequest.Sub(_requestMonitorMemRequest) - } - q1, q2, q3 := k8s.SplitInThree(userPodMemRequest) - apiResourceList[kcore.ResourceMemory] = *q1 - tfServingResourceList[kcore.ResourceMemory] = *q2 - neuronContainer.Resources.Requests[kcore.ResourceMemory] = *q3 - } - - containers = append(containers, neuronContainer) - } - - if api.Handler.ShmSize != nil { - volumes = append(volumes, kcore.Volume{ - Name: "dshm", - VolumeSource: kcore.VolumeSource{ - EmptyDir: &kcore.EmptyDirVolumeSource{ - Medium: kcore.StorageMediumMemory, - SizeLimit: k8s.QuantityPtr(api.Handler.ShmSize.Quantity), - }, - }, - }) - volumeMounts = append(volumeMounts, kcore.VolumeMount{ - Name: "dshm", - MountPath: "/dev/shm", - }) - } - - containers = append(containers, kcore.Container{ - Name: APIContainerName, - Image: api.Handler.Image, - ImagePullPolicy: kcore.PullAlways, - Env: envVars, - EnvFrom: baseEnvVars(), - VolumeMounts: volumeMounts, - ReadinessProbe: FileExistsProbe(_apiReadinessFile), - Lifecycle: nginxGracefulStopper(api.Kind), - Resources: kcore.ResourceRequirements{ - Requests: apiResourceList, - }, - Ports: []kcore.ContainerPort{ - {ContainerPort: DefaultPortInt32}, - }, - SecurityContext: &kcore.SecurityContext{ - Privileged: pointer.Bool(true), - }}, - tensorflowServingContainer( - api, - volumeMounts, - kcore.ResourceRequirements{ - Limits: tfServingLimitsList, - Requests: tfServingResourceList, - }, - tfServingEnvVars, - ), - ) - - return containers, volumes -} - -func taskEnvVars(api *spec.API) []kcore.EnvVar { - envVars := apiContainerEnvVars(api) - envVars = append(envVars, - kcore.EnvVar{ - Name: "CORTEX_TASK_SPEC", - Value: TaskSpecPath, - }, - ) - envVars = append(envVars, getKubexitEnvVars(APIContainerName)...) - return envVars -} - -func getAsyncAPIEnvVars(api spec.API, queueURL string) []kcore.EnvVar { - envVars := apiContainerEnvVars(&api) - - envVars = append(envVars, - kcore.EnvVar{ - Name: "CORTEX_QUEUE_URL", - Value: queueURL, - }, - kcore.EnvVar{ - Name: "CORTEX_ASYNC_WORKLOAD_PATH", - Value: aws.S3Path(config.ClusterConfig.Bucket, fmt.Sprintf("%s/workloads/%s", config.ClusterConfig.ClusterUID, api.Name)), - }, - ) + containerDeathDependencies := containerNames.Copy() + containerDeathDependencies.Remove(container.Name) + containerEnvVars := getKubexitEnvVars(container.Name, containerDeathDependencies.Slice(), []string{_neuronRTDContainerName}) - return envVars -} - -func requestMonitorEnvVars(api *spec.API) []kcore.EnvVar { - if api.Kind == userconfig.TaskAPIKind { - return []kcore.EnvVar{ - { - Name: "CORTEX_LOG_LEVEL", - Value: strings.ToUpper(api.TaskDefinition.LogLevel.String()), - }, - } - } - return []kcore.EnvVar{ - { - Name: "CORTEX_LOG_LEVEL", - Value: strings.ToUpper(api.Handler.LogLevel.String()), - }, - } -} - -func downloaderEnvVars(api *spec.API) []kcore.EnvVar { - if api.Kind == userconfig.TaskAPIKind { - return []kcore.EnvVar{ - { - Name: "CORTEX_LOG_LEVEL", - Value: strings.ToUpper(api.TaskDefinition.LogLevel.String()), - }, - } - } - return []kcore.EnvVar{ - { - Name: "CORTEX_LOG_LEVEL", - Value: strings.ToUpper(api.Handler.LogLevel.String()), - }, - } -} - -func tensorflowServingEnvVars(api *spec.API) []kcore.EnvVar { - envVars := []kcore.EnvVar{ - { - Name: "TF_CPP_MIN_LOG_LEVEL", - Value: s.Int(userconfig.TFNumericLogLevelFromLogLevel(api.Handler.LogLevel)), - }, - { - Name: "TF_PROCESSES", - Value: s.Int32(api.Handler.ProcessesPerReplica), - }, - { - Name: "CORTEX_TF_BASE_SERVING_PORT", - Value: _tfBaseServingPortStr, - }, - { - Name: "TF_EMPTY_MODEL_CONFIG", - Value: _tfServingEmptyModelConfig, - }, - { - Name: "TF_MAX_NUM_LOAD_RETRIES", - Value: _tfServingMaxNumLoadRetries, - }, - { - Name: "TF_LOAD_RETRY_INTERVAL_MICROS", - Value: _tfServingLoadTimeMicros, - }, - { - Name: "TF_GRPC_MAX_CONCURRENT_STREAMS", - Value: fmt.Sprintf(`--grpc_channel_arguments="grpc.max_concurrent_streams=%d"`, api.Handler.ThreadsPerProcess+10), - }, - } - - if api.Handler.ServerSideBatching != nil { - var numBatchedThreads int32 - if api.Compute.Inf > 0 { - // because there are processes_per_replica TF servers - numBatchedThreads = 1 - } else { - numBatchedThreads = api.Handler.ProcessesPerReplica + for k, v := range container.Env { + containerEnvVars = append(containerEnvVars, kcore.EnvVar{ + Name: k, + Value: v, + }) } - - envVars = append(envVars, - kcore.EnvVar{ - Name: "TF_MAX_BATCH_SIZE", - Value: s.Int32(api.Handler.ServerSideBatching.MaxBatchSize), - }, - kcore.EnvVar{ - Name: "TF_BATCH_TIMEOUT_MICROS", - Value: s.Int64(api.Handler.ServerSideBatching.BatchInterval.Microseconds()), - }, - kcore.EnvVar{ - Name: "TF_NUM_BATCHED_THREADS", - Value: s.Int32(numBatchedThreads), - }, - ) - } - - if api.Compute.Inf > 0 { - envVars = append(envVars, - kcore.EnvVar{ - Name: "NEURONCORE_GROUP_SIZES", - Value: s.Int64(api.Compute.Inf * consts.NeuronCoresPerInf / int64(api.Handler.ProcessesPerReplica)), - }, - kcore.EnvVar{ - Name: "NEURON_RTD_ADDRESS", - Value: fmt.Sprintf("unix:%s", _neuronRTDSocket), - }, - ) - } - - return envVars -} - -func apiContainerEnvVars(api *spec.API) []kcore.EnvVar { - envVars := []kcore.EnvVar{ - { - Name: "CORTEX_TELEMETRY_SENTRY_USER_ID", - Value: config.OperatorMetadata.OperatorID, - }, - { - Name: "CORTEX_TELEMETRY_SENTRY_ENVIRONMENT", - Value: "api", - }, - { - Name: "CORTEX_DEBUGGING", - Value: "false", - }, - { + containerEnvVars = append(containerEnvVars, kcore.EnvVar{ Name: "HOST_IP", ValueFrom: &kcore.EnvVarSource{ FieldRef: &kcore.ObjectFieldSelector{ FieldPath: "status.hostIP", }, }, - }, - { - Name: "CORTEX_API_SPEC", - Value: APISpecPath, - }, - } - - if api.Handler != nil { - for name, val := range api.Handler.Env { - envVars = append(envVars, kcore.EnvVar{ - Name: name, - Value: val, - }) - } - if api.Handler.Type == userconfig.TensorFlowHandlerType { - envVars = append(envVars, - kcore.EnvVar{ - Name: "CORTEX_TF_BASE_SERVING_PORT", - Value: _tfBaseServingPortStr, - }, - kcore.EnvVar{ - Name: "CORTEX_TF_SERVING_HOST", - Value: _tfServingHost, - }, - ) - } - } - - if api.TaskDefinition != nil { - for name, val := range api.TaskDefinition.Env { - envVars = append(envVars, kcore.EnvVar{ - Name: name, - Value: val, - }) - } - } - - return envVars -} - -func tensorflowServingContainer(api *spec.API, volumeMounts []kcore.VolumeMount, resources kcore.ResourceRequirements, envVars []kcore.EnvVar) kcore.Container { - var cmdArgs []string - ports := []kcore.ContainerPort{ - { - ContainerPort: _tfBaseServingPortInt32, - }, - } - - if api.Compute.Inf > 0 { - numPorts := api.Handler.ProcessesPerReplica - for i := int32(1); i < numPorts; i++ { - ports = append(ports, kcore.ContainerPort{ - ContainerPort: _tfBaseServingPortInt32 + i, - }) - } - } - - if api.Compute.Inf == 0 { - // the entrypoint is different for Inferentia-based APIs - cmdArgs = []string{ - "--port=" + _tfBaseServingPortStr, - "--model_config_file=" + _tfServingEmptyModelConfig, - "--max_num_load_retries=" + _tfServingMaxNumLoadRetries, - "--load_retry_interval_micros=" + _tfServingLoadTimeMicros, - fmt.Sprintf(`--grpc_channel_arguments="grpc.max_concurrent_streams=%d"`, api.Handler.ProcessesPerReplica*api.Handler.ThreadsPerProcess+10), - } - if api.Handler.ServerSideBatching != nil { - cmdArgs = append(cmdArgs, - "--enable_batching=true", - "--batching_parameters_file="+_tfServingBatchConfig, - ) - } - } - - var probeHandler kcore.Handler - if len(ports) == 1 { - probeHandler = kcore.Handler{ - TCPSocket: &kcore.TCPSocketAction{ - Port: intstr.IntOrString{ - IntVal: _tfBaseServingPortInt32, - }, - }, - } - } else { - probeHandler = kcore.Handler{ - Exec: &kcore.ExecAction{ - Command: []string{"/bin/bash", "-c", `test $(nc -zv localhost ` + fmt.Sprintf("%d-%d", _tfBaseServingPortInt32, _tfBaseServingPortInt32+int32(len(ports))-1) + ` 2>&1 | wc -l) -eq ` + fmt.Sprintf("%d", len(ports))}, - }, - } - } - - return kcore.Container{ - Name: _tfServingContainerName, - Image: api.Handler.TensorFlowServingImage, - ImagePullPolicy: kcore.PullAlways, - Args: cmdArgs, - Env: envVars, - EnvFrom: baseEnvVars(), - VolumeMounts: volumeMounts, - ReadinessProbe: &kcore.Probe{ - InitialDelaySeconds: 5, - TimeoutSeconds: 5, - PeriodSeconds: 5, - SuccessThreshold: 1, - FailureThreshold: 2, - Handler: probeHandler, - }, - Lifecycle: waitAPIContainerToStop(api.Kind), - Resources: resources, - Ports: ports, - } -} - -func neuronRuntimeDaemonContainer(api *spec.API, volumeMounts []kcore.VolumeMount, envVars []kcore.EnvVar) kcore.Container { - totalHugePages := api.Compute.Inf * _hugePagesMemPerInf - return kcore.Container{ - Name: _neuronRTDContainerName, - Image: config.ClusterConfig.ImageNeuronRTD, - ImagePullPolicy: kcore.PullAlways, - Env: envVars, - SecurityContext: &kcore.SecurityContext{ - Capabilities: &kcore.Capabilities{ - Add: []kcore.Capability{ - "SYS_ADMIN", - "IPC_LOCK", - }, - }, - }, - VolumeMounts: volumeMounts, - ReadinessProbe: socketExistsProbe(_neuronRTDSocket), - Lifecycle: waitAPIContainerToStop(api.Kind), - Resources: kcore.ResourceRequirements{ - Requests: kcore.ResourceList{ - "hugepages-2Mi": *kresource.NewQuantity(totalHugePages, kresource.BinarySI), - "aws.amazon.com/neuron": *kresource.NewQuantity(api.Compute.Inf, kresource.DecimalSI), - }, - Limits: kcore.ResourceList{ - "hugepages-2Mi": *kresource.NewQuantity(totalHugePages, kresource.BinarySI), - "aws.amazon.com/neuron": *kresource.NewQuantity(api.Compute.Inf, kresource.DecimalSI), - }, - }, - } -} - -func RequestMonitorContainer(api *spec.API) kcore.Container { - requests := kcore.ResourceList{} - if api.Compute != nil { - if api.Compute.CPU != nil { - requests[kcore.ResourceCPU] = _requestMonitorCPURequest - } - if api.Compute.Mem != nil { - requests[kcore.ResourceMemory] = _requestMonitorMemRequest - } - } - - return kcore.Container{ - Name: _requestMonitorContainerName, - Image: config.ClusterConfig.ImageRequestMonitor, - ImagePullPolicy: kcore.PullAlways, - Args: []string{"-p", DefaultRequestMonitorPortStr}, - Ports: []kcore.ContainerPort{ - {Name: "metrics", ContainerPort: DefaultRequestMonitorPortInt32}, - }, - Env: requestMonitorEnvVars(api), - EnvFrom: baseEnvVars(), - VolumeMounts: defaultVolumeMounts(), - ReadinessProbe: FileExistsProbe(_requestMonitorReadinessFile), - Resources: kcore.ResourceRequirements{ - Requests: requests, - }, - } -} - -func baseEnvVars() []kcore.EnvFromSource { - envVars := []kcore.EnvFromSource{ - { - ConfigMapRef: &kcore.ConfigMapEnvSource{ - LocalObjectReference: kcore.LocalObjectReference{ - Name: "env-vars", - }, - }, - }, - } - - return envVars -} - -func getKubexitEnvVars(containerName string, deathDeps ...string) []kcore.EnvVar { - envVars := []kcore.EnvVar{ - { - Name: "KUBEXIT_NAME", - Value: containerName, - }, - { - Name: "KUBEXIT_GRAVEYARD", - Value: _kubexitGraveyardMountPath, - }, - } + }) - if deathDeps != nil { - envVars = append(envVars, - kcore.EnvVar{ - Name: "KUBEXIT_DEATH_DEPS", - Value: strings.Join(deathDeps, ","), - }, - kcore.EnvVar{ - Name: "KUBEXIT_IGNORE_CODE_ON_DEATH_DEPS", - Value: "true", - }, + containers = append(containers, kcore.Container{ + Name: container.Name, + Image: container.Image, + Command: append([]string{"/mnt/kubexit"}, container.Command...), + Args: container.Args, + Env: containerEnvVars, + VolumeMounts: containerVolumeMounts, + Resources: kcore.ResourceRequirements{ + Requests: containerResourceList, + Limits: containerResourceLimitsList, + }, + ImagePullPolicy: kcore.PullAlways, + SecurityContext: &kcore.SecurityContext{ + Privileged: pointer.Bool(true), + }}, ) } - return envVars -} - -func DefaultVolumes() []kcore.Volume { - return []kcore.Volume{ - k8s.EmptyDirVolume(_emptyDirVolumeName), - { - Name: "client-config", - VolumeSource: kcore.VolumeSource{ - ConfigMap: &kcore.ConfigMapVolumeSource{ - LocalObjectReference: kcore.LocalObjectReference{ - Name: "client-config", - }, - }, - }, - }, - } -} - -func defaultVolumeMounts() []kcore.VolumeMount { - return []kcore.VolumeMount{ - k8s.EmptyDirVolumeMount(_emptyDirVolumeName, _emptyDirMountPath), - { - Name: "client-config", - MountPath: path.Join(_clientConfigDir, "cli.yaml"), - SubPath: "cli.yaml", - }, - } + return containers, volumes } func NodeSelectors() map[string]string { @@ -1120,3 +307,79 @@ func GenerateNodeAffinities(apiNodeGroups []string) *kcore.Affinity { }, } } + +func neuronRuntimeDaemonContainer(computeInf int64, volumeMounts []kcore.VolumeMount, envVars []kcore.EnvVar) kcore.Container { + totalHugePages := computeInf * _hugePagesMemPerInf + return kcore.Container{ + Name: _neuronRTDContainerName, + Image: config.ClusterConfig.ImageNeuronRTD, + ImagePullPolicy: kcore.PullAlways, + Env: envVars, + SecurityContext: &kcore.SecurityContext{ + Capabilities: &kcore.Capabilities{ + Add: []kcore.Capability{ + "SYS_ADMIN", + "IPC_LOCK", + }, + }, + }, + VolumeMounts: volumeMounts, + ReadinessProbe: SocketExistsProbe(_neuronRTDSocket), + Resources: kcore.ResourceRequirements{ + Requests: kcore.ResourceList{ + "hugepages-2Mi": *kresource.NewQuantity(totalHugePages, kresource.BinarySI), + "aws.amazon.com/neuron": *kresource.NewQuantity(computeInf, kresource.DecimalSI), + }, + Limits: kcore.ResourceList{ + "hugepages-2Mi": *kresource.NewQuantity(totalHugePages, kresource.BinarySI), + "aws.amazon.com/neuron": *kresource.NewQuantity(computeInf, kresource.DecimalSI), + }, + }, + } +} + +// func getAsyncAPIEnvVars(api spec.API, queueURL string) []kcore.EnvVar { +// envVars := apiContainerEnvVars(&api) + +// envVars = append(envVars, +// kcore.EnvVar{ +// Name: "CORTEX_QUEUE_URL", +// Value: queueURL, +// }, +// kcore.EnvVar{ +// Name: "CORTEX_ASYNC_WORKLOAD_PATH", +// Value: aws.S3Path(config.ClusterConfig.Bucket, fmt.Sprintf("%s/workloads/%s", config.ClusterConfig.ClusterUID, api.Name)), +// }, +// ) + +// return envVars +// } + +// func RequestMonitorContainer(api *spec.API) kcore.Container { +// requests := kcore.ResourceList{} +// if api.Compute != nil { +// if api.Compute.CPU != nil { +// requests[kcore.ResourceCPU] = _requestMonitorCPURequest +// } +// if api.Compute.Mem != nil { +// requests[kcore.ResourceMemory] = _requestMonitorMemRequest +// } +// } + +// return kcore.Container{ +// Name: _requestMonitorContainerName, +// Image: config.ClusterConfig.ImageRequestMonitor, +// ImagePullPolicy: kcore.PullAlways, +// Args: []string{"-p", DefaultRequestMonitorPortStr}, +// Ports: []kcore.ContainerPort{ +// {Name: "metrics", ContainerPort: DefaultRequestMonitorPortInt32}, +// }, +// Env: requestMonitorEnvVars(api), +// EnvFrom: baseEnvVars(), +// VolumeMounts: defaultVolumeMounts(), +// ReadinessProbe: FileExistsProbe(_requestMonitorReadinessFile), +// Resources: kcore.ResourceRequirements{ +// Requests: requests, +// }, +// } +// } From 05caab036cd2157cf56f6b8233230d2ee4aafb13 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Thu, 13 May 2021 21:34:06 +0300 Subject: [PATCH 06/82] Prevent use of reserved/duplicate container names --- pkg/consts/consts.go | 4 ++++ pkg/types/spec/errors.go | 8 ++++++++ pkg/types/spec/validations.go | 8 ++++++++ 3 files changed, 20 insertions(+) diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 59c5802a53..7cccdad274 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -33,6 +33,10 @@ var ( DefaultInClusterConfigPath = "/configs/cluster/cluster.yaml" MaxBucketLifecycleRules = 100 AsyncWorkloadsExpirationDays = int64(7) + + ReservedContainerNames = []string{ + "neuron-rtd", + } ) func DefaultRegistry() string { diff --git a/pkg/types/spec/errors.go b/pkg/types/spec/errors.go index 28f4a9e1a0..0441ef1b6d 100644 --- a/pkg/types/spec/errors.go +++ b/pkg/types/spec/errors.go @@ -34,6 +34,7 @@ const ( ErrDuplicateName = "spec.duplicate_name" ErrDuplicateEndpointInOneDeploy = "spec.duplicate_endpoint_in_one_deploy" ErrDuplicateEndpoint = "spec.duplicate_endpoint" + ErrDuplicateContainerName = "spec.duplicate_container_name" ErrConflictingFields = "spec.conflicting_fields" ErrSpecifyOnlyOneField = "spec.specify_only_one_field" ErrSpecifyOneOrTheOther = "spec.specify_one_or_the_other" @@ -114,6 +115,13 @@ func ErrorDuplicateEndpoint(apiName string) error { }) } +func ErrorDuplicateContainerName(containerName string) error { + return errors.WithStack(&errors.Error{ + Kind: ErrDuplicateContainerName, + Message: fmt.Sprintf("container name %s must be unique", containerName), + }) +} + func ErrorConflictingFields(fieldKeyA, fieldKeyB string) error { return errors.WithStack(&errors.Error{ Kind: ErrConflictingFields, diff --git a/pkg/types/spec/validations.go b/pkg/types/spec/validations.go index 1764dd729a..9ea09ca35b 100644 --- a/pkg/types/spec/validations.go +++ b/pkg/types/spec/validations.go @@ -33,6 +33,7 @@ import ( "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/regex" + "github.com/cortexlabs/cortex/pkg/lib/slices" libtime "github.com/cortexlabs/cortex/pkg/lib/time" "github.com/cortexlabs/cortex/pkg/lib/urls" "github.com/cortexlabs/cortex/pkg/types/userconfig" @@ -184,6 +185,7 @@ func containersValidation() *cr.StructFieldValidation { Required: true, AllowEmpty: false, AlphaNumericDashUnderscore: true, + DisallowedValues: consts.ReservedContainerNames, }, }, { @@ -567,7 +569,13 @@ func validateContainers( awsClient *aws.Client, k8sClient *k8s.Client, ) error { + containerNames := []string{} + for i, container := range containers { + if slices.HasString(containerNames, container.Name) { + return ErrorDuplicateContainerName(container.Name) + } + if err := validateDockerImagePath(container.Image, awsClient, k8sClient); err != nil { return errors.Wrap(err, strconv.FormatInt(int64(i), 10), userconfig.ImageKey) } From 9b682605e686edfd58641a41c54d6a4b3c512536 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Fri, 14 May 2021 01:45:57 +0300 Subject: [PATCH 07/82] Moving around consts --- pkg/consts/consts.go | 12 +++++++----- pkg/operator/resources/resources.go | 4 ---- pkg/operator/resources/trafficsplitter/api.go | 3 ++- pkg/operator/resources/trafficsplitter/k8s_specs.go | 4 ---- pkg/workloads/k8s.go | 6 ++---- 5 files changed, 11 insertions(+), 18 deletions(-) diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 7cccdad274..d9b3372cad 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -24,11 +24,13 @@ var ( CortexVersion = "master" // CORTEX_VERSION CortexVersionMinor = "master" // CORTEX_VERSION_MINOR - DefaultMaxReplicaQueueLength = int64(1024) - DefaultMaxReplicaConcurrency = int64(1024) - DefaultTargetReplicaConcurrency = float64(8) - NeuronCoresPerInf = int64(4) - AuthHeader = "X-Cortex-Authorization" + ProxyListeningPort, ProxyListeningPortStr = int64(8888), "8888" + DefaultMaxReplicaQueueLength = int64(1024) + DefaultMaxReplicaConcurrency = int64(1024) + DefaultTargetReplicaConcurrency = float64(8) + NeuronCoresPerInf = int64(4) + + AuthHeader = "X-Cortex-Authorization" DefaultInClusterConfigPath = "/configs/cluster/cluster.yaml" MaxBucketLifecycleRules = 100 diff --git a/pkg/operator/resources/resources.go b/pkg/operator/resources/resources.go index 1cffbba47b..ef6eb5482d 100644 --- a/pkg/operator/resources/resources.go +++ b/pkg/operator/resources/resources.go @@ -51,10 +51,6 @@ import ( var operatorLogger = logging.GetLogger() -const ( - _defaultAPIPortInt32 = int32(8888) -) - // Returns an error if resource doesn't exist func GetDeployedResourceByName(resourceName string) (*operator.DeployedResource, error) { resource, err := GetDeployedResourceByNameOrNil(resourceName) diff --git a/pkg/operator/resources/trafficsplitter/api.go b/pkg/operator/resources/trafficsplitter/api.go index 155280ba10..1ce3844b0d 100644 --- a/pkg/operator/resources/trafficsplitter/api.go +++ b/pkg/operator/resources/trafficsplitter/api.go @@ -21,6 +21,7 @@ import ( "path/filepath" "github.com/cortexlabs/cortex/pkg/config" + "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/parallel" @@ -112,7 +113,7 @@ func getTrafficSplitterDestinations(trafficSplitter *spec.API) []k8s.Destination destinations[i] = k8s.Destination{ ServiceName: workloads.K8sName(api.Name), Weight: api.Weight, - Port: uint32(_defaultPortInt32), + Port: uint32(consts.ProxyListeningPort), Shadow: api.Shadow, } } diff --git a/pkg/operator/resources/trafficsplitter/k8s_specs.go b/pkg/operator/resources/trafficsplitter/k8s_specs.go index bce81dba44..b1dbc8cedc 100644 --- a/pkg/operator/resources/trafficsplitter/k8s_specs.go +++ b/pkg/operator/resources/trafficsplitter/k8s_specs.go @@ -24,10 +24,6 @@ import ( istioclientnetworking "istio.io/client-go/pkg/apis/networking/v1beta1" ) -const ( - _defaultPortInt32, _defaultPortStr = int32(8888), "8888" -) - func virtualServiceSpec(trafficSplitter *spec.API) *istioclientnetworking.VirtualService { return k8s.VirtualService(&k8s.VirtualServiceSpec{ Name: workloads.K8sName(trafficSplitter.Name), diff --git a/pkg/workloads/k8s.go b/pkg/workloads/k8s.go index d9a9d46455..cdbec6093d 100644 --- a/pkg/workloads/k8s.go +++ b/pkg/workloads/k8s.go @@ -31,10 +31,8 @@ import ( ) const ( - DefaultPortInt32 = int32(8888) - DefaultRequestMonitorPortStr = "15000" - DefaultRequestMonitorPortInt32 = int32(15000) - ServiceAccountName = "default" + DefaultPortInt32, DefaultPortStr = int32(8888), "8888" + ServiceAccountName = "default" ) const ( From 24ced7dd1316a22f569c4fdbe4a19e2955569f82 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Fri, 14 May 2021 01:46:52 +0300 Subject: [PATCH 08/82] Fix make operator-local command --- pkg/config/config.go | 39 +++++++++++++++++++++++++++------- pkg/lib/configreader/reader.go | 14 ++++++++++++ 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 5408d61ff3..3367d843c8 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -24,6 +24,7 @@ import ( "github.com/cortexlabs/cortex/pkg/consts" batch "github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1" "github.com/cortexlabs/cortex/pkg/lib/aws" + cr "github.com/cortexlabs/cortex/pkg/lib/configreader" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/hash" "github.com/cortexlabs/cortex/pkg/lib/k8s" @@ -60,6 +61,20 @@ func InitConfigs(clusterConfig *clusterconfig.Config, operatorMetadata *clusterc OperatorMetadata = operatorMetadata } +func getClusterConfigFromConfigMap() (clusterconfig.Config, error) { + configMapData, err := K8s.GetConfigMapData("cluster-config") + if err != nil { + return clusterconfig.Config{}, err + } + clusterConfig := clusterconfig.Config{} + errs := cr.ParseYAMLBytes(&clusterConfig, clusterconfig.FullManagedValidation, []byte(configMapData["cluster.yaml"])) + if errors.FirstError(errs...) != nil { + return clusterconfig.Config{}, errors.FirstError(errs...) + } + + return clusterConfig, nil +} + func Init() error { var err error var clusterNamespace string @@ -103,6 +118,22 @@ func Init() error { clusterNamespace = clusterConfig.Namespace istioNamespace = clusterConfig.IstioNamespace + if K8s, err = k8s.New(clusterNamespace, OperatorMetadata.IsOperatorInCluster, nil, scheme); err != nil { + return err + } + + if K8sIstio, err = k8s.New(istioNamespace, OperatorMetadata.IsOperatorInCluster, nil, scheme); err != nil { + return err + } + + if !OperatorMetadata.IsOperatorInCluster { + cc, err := getClusterConfigFromConfigMap() + if err != nil { + return err + } + clusterConfig.Bucket = cc.Bucket + } + exists, err := AWS.DoesBucketExist(clusterConfig.Bucket) if err != nil { return err @@ -126,14 +157,6 @@ func Init() error { fmt.Println(errors.Message(err)) } - if K8s, err = k8s.New(clusterNamespace, OperatorMetadata.IsOperatorInCluster, nil, scheme); err != nil { - return err - } - - if K8sIstio, err = k8s.New(istioNamespace, OperatorMetadata.IsOperatorInCluster, nil, scheme); err != nil { - return err - } - prometheusURL := os.Getenv("CORTEX_PROMETHEUS_URL") if len(prometheusURL) == 0 { prometheusURL = fmt.Sprintf("http://prometheus.%s:9090", clusterNamespace) diff --git a/pkg/lib/configreader/reader.go b/pkg/lib/configreader/reader.go index 5b239eb1a0..b6a990a73b 100644 --- a/pkg/lib/configreader/reader.go +++ b/pkg/lib/configreader/reader.go @@ -1038,6 +1038,20 @@ func ParseYAMLFile(dest interface{}, validation *StructValidation, filePath stri return nil } +func ParseYAMLBytes(dest interface{}, validation *StructValidation, data []byte) []error { + fileInterface, err := ReadYAMLBytes(data) + if err != nil { + return []error{err} + } + + errs := Struct(dest, fileInterface, validation) + if errors.HasError(errs) { + return errs + } + + return nil +} + func ReadYAMLFile(filePath string) (interface{}, error) { fileBytes, err := files.ReadFileBytes(filePath) if err != nil { From 92adc2f3f5e05f29edad266d356832da17cb45b0 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Fri, 14 May 2021 17:09:57 +0300 Subject: [PATCH 09/82] WIP CaaS --- cmd/enqueuer/main.go | 2 +- cmd/request-monitor/main.go | 3 +- .../batch/batchjob_controller_helpers.go | 2 +- .../batch/batchjob_controller_test.go | 2 +- pkg/operator/resources/asyncapi/k8s_specs.go | 2 +- .../resources/job/taskapi/k8s_specs.go | 2 +- .../resources/realtimeapi/k8s_specs.go | 1 - pkg/types/spec/validations.go | 2 +- pkg/types/userconfig/api.go | 16 +++-- pkg/workloads/init.go | 71 +++---------------- pkg/workloads/k8s.go | 13 +++- test/apis/realtime-caas/.dockerignore | 7 ++ test/apis/realtime-caas/Dockerfile | 21 ++++++ test/apis/realtime-caas/cortex.yaml | 20 ++++++ test/apis/realtime-caas/main.py | 15 ++++ 15 files changed, 99 insertions(+), 80 deletions(-) create mode 100644 test/apis/realtime-caas/.dockerignore create mode 100644 test/apis/realtime-caas/Dockerfile create mode 100644 test/apis/realtime-caas/cortex.yaml create mode 100644 test/apis/realtime-caas/main.py diff --git a/cmd/enqueuer/main.go b/cmd/enqueuer/main.go index 95f0b1530d..c73724be1c 100644 --- a/cmd/enqueuer/main.go +++ b/cmd/enqueuer/main.go @@ -28,7 +28,7 @@ import ( ) func createLogger() (*zap.Logger, error) { - logLevelEnv := os.Getenv("CORTEX_LOG_LEVEL") + logLevelEnv := strings.ToUpper(os.Getenv("CORTEX_LOG_LEVEL")) disableJSONLogging := os.Getenv("CORTEX_DISABLE_JSON_LOGGING") var logLevelZap zapcore.Level diff --git a/cmd/request-monitor/main.go b/cmd/request-monitor/main.go index 1cb7d107be..876f3a5749 100644 --- a/cmd/request-monitor/main.go +++ b/cmd/request-monitor/main.go @@ -21,6 +21,7 @@ import ( "fmt" "net/http" "os" + "strings" "sync" "time" @@ -70,7 +71,7 @@ func (c *Counter) GetAllAndDelete() []int { func main() { var port = flag.String("p", _defaultPort, "port on which the server runs on") - logLevelEnv := os.Getenv("CORTEX_LOG_LEVEL") + logLevelEnv := strings.ToUpper(os.Getenv("CORTEX_LOG_LEVEL")) var logLevelZap zapcore.Level switch logLevelEnv { case "DEBUG": diff --git a/pkg/crds/controllers/batch/batchjob_controller_helpers.go b/pkg/crds/controllers/batch/batchjob_controller_helpers.go index e32fa4cac9..0f5b01c88b 100644 --- a/pkg/crds/controllers/batch/batchjob_controller_helpers.go +++ b/pkg/crds/controllers/batch/batchjob_controller_helpers.go @@ -308,7 +308,7 @@ func (r *BatchJobReconciler) desiredWorkerJob(batchJob batch.BatchJob, apiSpec s K8sPodSpec: kcore.PodSpec{ InitContainers: []kcore.Container{ workloads.KubexitInitContainer(), - workloads.BatchInitContainer(&apiSpec, &jobSpec), + workloads.BatchInitContainer(&jobSpec), }, Containers: containers, Volumes: volumes, diff --git a/pkg/crds/controllers/batch/batchjob_controller_test.go b/pkg/crds/controllers/batch/batchjob_controller_test.go index 454359235e..0fdf31a488 100644 --- a/pkg/crds/controllers/batch/batchjob_controller_test.go +++ b/pkg/crds/controllers/batch/batchjob_controller_test.go @@ -44,7 +44,7 @@ func uploadTestAPISpec(apiName string, apiID string) error { }, Pod: &userconfig.Pod{ // TODO use a real image - Containers: []userconfig.Container{ + Containers: []*userconfig.Container{ { Name: "api", Image: "quay.io/cortexlabs/batch-container-test:master", diff --git a/pkg/operator/resources/asyncapi/k8s_specs.go b/pkg/operator/resources/asyncapi/k8s_specs.go index 5653ceed7b..ca853ce6e5 100644 --- a/pkg/operator/resources/asyncapi/k8s_specs.go +++ b/pkg/operator/resources/asyncapi/k8s_specs.go @@ -213,7 +213,7 @@ func deploymentSpec(api spec.API, prevDeployment *kapps.Deployment, queueURL str RestartPolicy: "Always", TerminationGracePeriodSeconds: pointer.Int64(_terminationGracePeriodSeconds), InitContainers: []kcore.Container{ - workloads.InitContainer(api), + workloads.KubexitInitContainer(), }, Containers: containers, NodeSelector: workloads.NodeSelectors(), diff --git a/pkg/operator/resources/job/taskapi/k8s_specs.go b/pkg/operator/resources/job/taskapi/k8s_specs.go index 84292f77a0..a3c58f9795 100644 --- a/pkg/operator/resources/job/taskapi/k8s_specs.go +++ b/pkg/operator/resources/job/taskapi/k8s_specs.go @@ -89,7 +89,7 @@ func k8sJobSpec(api *spec.API, job *spec.TaskJob) *kbatch.Job { RestartPolicy: "Never", InitContainers: []kcore.Container{ workloads.KubexitInitContainer(), - workloads.TaskInitContainer(api, job), + workloads.TaskInitContainer(job), }, Containers: containers, NodeSelector: workloads.NodeSelectors(), diff --git a/pkg/operator/resources/realtimeapi/k8s_specs.go b/pkg/operator/resources/realtimeapi/k8s_specs.go index 3ab34f06bf..35238da0dc 100644 --- a/pkg/operator/resources/realtimeapi/k8s_specs.go +++ b/pkg/operator/resources/realtimeapi/k8s_specs.go @@ -67,7 +67,6 @@ func deploymentSpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Depl TerminationGracePeriodSeconds: pointer.Int64(_terminationGracePeriodSeconds), InitContainers: []kcore.Container{ workloads.KubexitInitContainer(), - workloads.InitContainer(*api), }, Containers: containers, NodeSelector: workloads.NodeSelectors(), diff --git a/pkg/types/spec/validations.go b/pkg/types/spec/validations.go index 9ea09ca35b..48f2166834 100644 --- a/pkg/types/spec/validations.go +++ b/pkg/types/spec/validations.go @@ -565,7 +565,7 @@ func validatePod( } func validateContainers( - containers []userconfig.Container, + containers []*userconfig.Container, awsClient *aws.Client, k8sClient *k8s.Client, ) error { diff --git a/pkg/types/userconfig/api.go b/pkg/types/userconfig/api.go index bbb9172bf6..9c0aa2d054 100644 --- a/pkg/types/userconfig/api.go +++ b/pkg/types/userconfig/api.go @@ -45,7 +45,7 @@ type API struct { type Pod struct { NodeGroups []string `json:"node_groups" yaml:"node_groups"` ShmSize *k8s.Quantity `json:"shm_size" yaml:"shm_size"` - Containers []Container `json:"containers" yaml:"containers"` + Containers []*Container `json:"containers" yaml:"containers"` } type Container struct { @@ -401,17 +401,17 @@ func ZeroCompute() Compute { } } -func GetTotalComputeFromContainers(containers []Container) Compute { +func GetTotalComputeFromContainers(containers []*Container) Compute { compute := Compute{} for _, container := range containers { - if container.Compute == nil { + if container == nil || container.Compute == nil { continue } if container.Compute.CPU != nil { newCPUQuantity := k8s.NewQuantity(container.Compute.CPU.Value()) - if compute.CPU != nil { + if compute.CPU == nil { compute.CPU = newCPUQuantity } else if newCPUQuantity != nil { compute.CPU.AddQty(*newCPUQuantity) @@ -420,7 +420,7 @@ func GetTotalComputeFromContainers(containers []Container) Compute { if container.Compute.Mem != nil { newMemQuantity := k8s.NewQuantity(container.Compute.Mem.Value()) - if compute.CPU != nil { + if compute.Mem == nil { compute.Mem = newMemQuantity } else if newMemQuantity != nil { compute.Mem.AddQty(*newMemQuantity) @@ -434,10 +434,12 @@ func GetTotalComputeFromContainers(containers []Container) Compute { return compute } -func GetContainerNames(containers []Container) strset.Set { +func GetContainerNames(containers []*Container) strset.Set { containerNames := strset.New() for _, container := range containers { - containerNames.Add(container.Name) + if container != nil { + containerNames.Add(container.Name) + } } return containerNames } diff --git a/pkg/workloads/init.go b/pkg/workloads/init.go index 50750868f6..88dd7f903c 100644 --- a/pkg/workloads/init.go +++ b/pkg/workloads/init.go @@ -19,6 +19,7 @@ package workloads import ( "encoding/base64" "encoding/json" + "strings" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/lib/aws" @@ -28,9 +29,7 @@ import ( ) const ( - APISpecPath = "/mnt/spec/spec.json" - TaskSpecPath = "/mnt/spec/task.json" - BatchSpecPath = "/mnt/spec/batch.json" + JobSpecPath = "/mnt/job_spec.json" ) const ( @@ -49,22 +48,13 @@ func KubexitInitContainer() kcore.Container { } } -func TaskInitContainer(api *spec.API, job *spec.TaskJob) kcore.Container { +func TaskInitContainer(job *spec.TaskJob) kcore.Container { downloadConfig := downloadContainerConfig{ LastLog: _downloaderLastLog, DownloadArgs: []downloadContainerArg{ - { - From: aws.S3Path(config.ClusterConfig.Bucket, api.Key), - To: APISpecPath, - Unzip: false, - ToFile: true, - ItemName: "the api spec", - HideFromLog: true, - HideUnzippingLog: true, - }, { From: aws.S3Path(config.ClusterConfig.Bucket, job.SpecFilePath(config.ClusterConfig.ClusterUID)), - To: TaskSpecPath, + To: JobSpecPath, Unzip: false, ToFile: true, ItemName: "the task spec", @@ -86,29 +76,20 @@ func TaskInitContainer(api *spec.API, job *spec.TaskJob) kcore.Container { Env: []kcore.EnvVar{ { Name: "CORTEX_LOG_LEVEL", - Value: userconfig.InfoLogLevel.String(), + Value: strings.ToUpper(userconfig.InfoLogLevel.String()), }, }, VolumeMounts: defaultVolumeMounts(), } } -func BatchInitContainer(api *spec.API, job *spec.BatchJob) kcore.Container { +func BatchInitContainer(job *spec.BatchJob) kcore.Container { downloadConfig := downloadContainerConfig{ LastLog: _downloaderLastLog, DownloadArgs: []downloadContainerArg{ - { - From: aws.S3Path(config.ClusterConfig.Bucket, api.Key), - To: APISpecPath, - Unzip: false, - ToFile: true, - ItemName: "the api spec", - HideFromLog: true, - HideUnzippingLog: true, - }, { From: aws.S3Path(config.ClusterConfig.Bucket, job.SpecFilePath(config.ClusterConfig.ClusterUID)), - To: BatchSpecPath, + To: JobSpecPath, Unzip: false, ToFile: true, ItemName: "the job spec", @@ -130,43 +111,7 @@ func BatchInitContainer(api *spec.API, job *spec.BatchJob) kcore.Container { Env: []kcore.EnvVar{ { Name: "CORTEX_LOG_LEVEL", - Value: userconfig.InfoLogLevel.String(), - }, - }, - VolumeMounts: defaultVolumeMounts(), - } -} - -// for async and realtime apis -func InitContainer(api spec.API) kcore.Container { - downloadConfig := downloadContainerConfig{ - LastLog: _downloaderLastLog, - DownloadArgs: []downloadContainerArg{ - { - From: aws.S3Path(config.ClusterConfig.Bucket, api.HandlerKey), - To: APISpecPath, - Unzip: false, - ToFile: true, - ItemName: "the api spec", - HideFromLog: true, - HideUnzippingLog: true, - }, - }, - } - - downloadArgsBytes, _ := json.Marshal(downloadConfig) - downloadArgs := base64.URLEncoding.EncodeToString(downloadArgsBytes) - - return kcore.Container{ - Name: _downloaderInitContainerName, - Image: config.ClusterConfig.ImageDownloader, - ImagePullPolicy: kcore.PullAlways, - Args: []string{"--download=" + downloadArgs}, - EnvFrom: baseClusterEnvVars(), - Env: []kcore.EnvVar{ - { - Name: "CORTEX_LOG_LEVEL", - Value: userconfig.InfoLogLevel.String(), + Value: strings.ToUpper(userconfig.InfoLogLevel.String()), }, }, VolumeMounts: defaultVolumeMounts(), diff --git a/pkg/workloads/k8s.go b/pkg/workloads/k8s.go index cdbec6093d..b86078d870 100644 --- a/pkg/workloads/k8s.go +++ b/pkg/workloads/k8s.go @@ -17,6 +17,8 @@ limitations under the License. package workloads import ( + "strings" + "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/k8s" @@ -77,7 +79,7 @@ func AsyncGatewayContainer(api spec.API, queueURL string, volumeMounts []kcore.V Env: []kcore.EnvVar{ { Name: "CORTEX_LOG_LEVEL", - Value: userconfig.InfoLogLevel.String(), + Value: strings.ToUpper(userconfig.InfoLogLevel.String()), }, }, Resources: kcore.ResourceRequirements{ @@ -127,6 +129,7 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { } var containers []kcore.Container + var podHasInf bool containerNames := userconfig.GetContainerNames(api.Pod.Containers) for _, container := range api.Pod.Containers { containerResourceList := kcore.ResourceList{} @@ -166,11 +169,17 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { neuronRTDEnvVars := getKubexitEnvVars(_neuronRTDContainerName, containerNames.Slice(), nil) containers = append(containers, neuronRuntimeDaemonContainer(container.Compute.Inf, rtdVolumeMounts, neuronRTDEnvVars)) + podHasInf = true } containerDeathDependencies := containerNames.Copy() containerDeathDependencies.Remove(container.Name) - containerEnvVars := getKubexitEnvVars(container.Name, containerDeathDependencies.Slice(), []string{_neuronRTDContainerName}) + var containerEnvVars []kcore.EnvVar + if podHasInf { + containerEnvVars = getKubexitEnvVars(container.Name, containerDeathDependencies.Slice(), []string{"neuron-rtd"}) + } else { + containerEnvVars = getKubexitEnvVars(container.Name, containerDeathDependencies.Slice(), nil) + } for k, v := range container.Env { containerEnvVars = append(containerEnvVars, kcore.EnvVar{ diff --git a/test/apis/realtime-caas/.dockerignore b/test/apis/realtime-caas/.dockerignore new file mode 100644 index 0000000000..3e4bdd9fbb --- /dev/null +++ b/test/apis/realtime-caas/.dockerignore @@ -0,0 +1,7 @@ +Dockerfile +README.md +*.pyc +*.pyo +*.pyd +__pycache__ +.pytest_cache diff --git a/test/apis/realtime-caas/Dockerfile b/test/apis/realtime-caas/Dockerfile new file mode 100644 index 0000000000..e5109eb08c --- /dev/null +++ b/test/apis/realtime-caas/Dockerfile @@ -0,0 +1,21 @@ +# Use the official lightweight Python image. +# https://hub.docker.com/_/python +FROM python:3.9-slim + +# Allow statements and log messages to immediately appear in the Knative logs +ENV PYTHONUNBUFFERED True + +# Copy local code to the container image. +ENV APP_HOME /app +WORKDIR $APP_HOME +COPY . ./ + +# Install production dependencies. +RUN pip install Flask gunicorn + +# Run the web service on container startup. Here we use the gunicorn +# webserver, with one worker process and 8 threads. +# For environments with multiple CPU cores, increase the number of workers +# to be equal to the cores available. +# Timeout is set to 0 to disable the timeouts of the workers to allow Cloud Run to handle instance scaling. +CMD exec gunicorn --bind :7000 --workers 1 --threads 8 --timeout 0 main:app diff --git a/test/apis/realtime-caas/cortex.yaml b/test/apis/realtime-caas/cortex.yaml new file mode 100644 index 0000000000..f5e570af05 --- /dev/null +++ b/test/apis/realtime-caas/cortex.yaml @@ -0,0 +1,20 @@ +- name: realtime + kind: RealtimeAPI + pod: + containers: + - name: api + image: 499593605069.dkr.ecr.us-west-2.amazonaws.com/sample/realtime-caas:latest + command: + - gunicorn + - --bind + - :7000 + - --workers + - "1" + - --threads + - "8" + - --timeout + - "0" + - main:app + compute: + cpu: 200m + mem: 512Mi diff --git a/test/apis/realtime-caas/main.py b/test/apis/realtime-caas/main.py new file mode 100644 index 0000000000..a226fc4ff4 --- /dev/null +++ b/test/apis/realtime-caas/main.py @@ -0,0 +1,15 @@ +import os + +from flask import Flask + +app = Flask(__name__) + + +@app.route("/") +def hello_world(): + name = os.environ.get("NAME", "World") + return "Hello {}!".format(name) + + +if __name__ == "__main__": + app.run(debug=True, host="0.0.0.0", port=7000) From 6bad6077fcbd5c8cc74d2209568922cf6a4302f1 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Fri, 14 May 2021 17:25:43 +0300 Subject: [PATCH 10/82] Add docker client helper fn (might have to rm later on) --- go.mod | 2 +- pkg/lib/docker/docker.go | 9 +++++++++ pkg/types/spec/validations.go | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 10cbf7d015..d7c90d37b1 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/onsi/ginkgo v1.14.1 github.com/onsi/gomega v1.10.2 - github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.0.1 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 diff --git a/pkg/lib/docker/docker.go b/pkg/lib/docker/docker.go index 0058ab80e4..a2772fbcba 100644 --- a/pkg/lib/docker/docker.go +++ b/pkg/lib/docker/docker.go @@ -41,6 +41,7 @@ import ( dockerclient "github.com/docker/docker/client" "github.com/docker/docker/pkg/jsonmessage" "github.com/docker/docker/pkg/term" + "github.com/opencontainers/go-digest" ) var NoAuth string @@ -321,6 +322,14 @@ func CheckImageAccessible(dockerClient *Client, dockerImage, authConfig string) return nil } +func GetDistributionDigest(dockerClient *Client, dockerImage, authConfig string) (digest.Digest, error) { + inspect, err := dockerClient.DistributionInspect(context.Background(), dockerImage, authConfig) + if err != nil { + return digest.Digest(""), errors.WithStack(err) + } + return inspect.Descriptor.Digest, nil +} + func CheckImageExistsLocally(dockerClient *Client, dockerImage string) error { images, err := dockerClient.ImageList(context.Background(), dockertypes.ImageListOptions{}) if err != nil { diff --git a/pkg/types/spec/validations.go b/pkg/types/spec/validations.go index 48f2166834..2e9340048e 100644 --- a/pkg/types/spec/validations.go +++ b/pkg/types/spec/validations.go @@ -38,6 +38,7 @@ import ( "github.com/cortexlabs/cortex/pkg/lib/urls" "github.com/cortexlabs/cortex/pkg/types/userconfig" dockertypes "github.com/docker/docker/api/types" + "github.com/opencontainers/go-digest" kresource "k8s.io/apimachinery/pkg/api/resource" ) @@ -724,3 +725,34 @@ func getDockerAuthStrFromK8s(dockerClient *docker.Client, k8sClient *k8s.Client) return dockerAuthStr, nil } + +func getDockerImageDigest( + image string, + awsClient *aws.Client, + k8sClient *k8s.Client, +) (digest.Digest, error) { + dockerClient, err := docker.GetDockerClient() + if err != nil { + return digest.Digest(""), err + } + + dockerAuthStr := docker.NoAuth + + if regex.IsValidECRURL(image) { + dockerAuthStr, err = docker.AWSAuthConfig(awsClient) + if err != nil { + return digest.Digest(""), err + } + } else if k8sClient != nil { + dockerAuthStr, err = getDockerAuthStrFromK8s(dockerClient, k8sClient) + if err != nil { + return digest.Digest(""), err + } + } + + distributionDigest, err := docker.GetDistributionDigest(dockerClient, image, dockerAuthStr) + if err != nil { + return digest.Digest(""), err + } + return distributionDigest, err +} From 1af3ee6be653d0ad1aa9bd2065a3e4523a5dcff8 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Fri, 14 May 2021 18:11:17 +0300 Subject: [PATCH 11/82] Use kubexit just for batch/task APIs --- pkg/operator/resources/asyncapi/k8s_specs.go | 15 +++++-------- .../resources/realtimeapi/k8s_specs.go | 15 +++++-------- pkg/workloads/helpers.go | 16 +++++++------- pkg/workloads/k8s.go | 22 ++++++++++++------- 4 files changed, 34 insertions(+), 34 deletions(-) diff --git a/pkg/operator/resources/asyncapi/k8s_specs.go b/pkg/operator/resources/asyncapi/k8s_specs.go index ca853ce6e5..50d62c61ed 100644 --- a/pkg/operator/resources/asyncapi/k8s_specs.go +++ b/pkg/operator/resources/asyncapi/k8s_specs.go @@ -212,15 +212,12 @@ func deploymentSpec(api spec.API, prevDeployment *kapps.Deployment, queueURL str K8sPodSpec: kcore.PodSpec{ RestartPolicy: "Always", TerminationGracePeriodSeconds: pointer.Int64(_terminationGracePeriodSeconds), - InitContainers: []kcore.Container{ - workloads.KubexitInitContainer(), - }, - Containers: containers, - NodeSelector: workloads.NodeSelectors(), - Tolerations: workloads.GenerateResourceTolerations(), - Affinity: workloads.GenerateNodeAffinities(api.Pod.NodeGroups), - Volumes: volumes, - ServiceAccountName: workloads.ServiceAccountName, + Containers: containers, + NodeSelector: workloads.NodeSelectors(), + Tolerations: workloads.GenerateResourceTolerations(), + Affinity: workloads.GenerateNodeAffinities(api.Pod.NodeGroups), + Volumes: volumes, + ServiceAccountName: workloads.ServiceAccountName, }, }, }) diff --git a/pkg/operator/resources/realtimeapi/k8s_specs.go b/pkg/operator/resources/realtimeapi/k8s_specs.go index 35238da0dc..831f43ae32 100644 --- a/pkg/operator/resources/realtimeapi/k8s_specs.go +++ b/pkg/operator/resources/realtimeapi/k8s_specs.go @@ -65,15 +65,12 @@ func deploymentSpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Depl K8sPodSpec: kcore.PodSpec{ RestartPolicy: "Always", TerminationGracePeriodSeconds: pointer.Int64(_terminationGracePeriodSeconds), - InitContainers: []kcore.Container{ - workloads.KubexitInitContainer(), - }, - Containers: containers, - NodeSelector: workloads.NodeSelectors(), - Tolerations: workloads.GenerateResourceTolerations(), - Affinity: workloads.GenerateNodeAffinities(api.Pod.NodeGroups), - Volumes: volumes, - ServiceAccountName: workloads.ServiceAccountName, + Containers: containers, + NodeSelector: workloads.NodeSelectors(), + Tolerations: workloads.GenerateResourceTolerations(), + Affinity: workloads.GenerateNodeAffinities(api.Pod.NodeGroups), + Volumes: volumes, + ServiceAccountName: workloads.ServiceAccountName, }, }, }) diff --git a/pkg/workloads/helpers.go b/pkg/workloads/helpers.go index 5d6020d051..2a6c230886 100644 --- a/pkg/workloads/helpers.go +++ b/pkg/workloads/helpers.go @@ -106,10 +106,10 @@ func getKubexitEnvVars(containerName string, deathDeps []string, birthDeps []str Name: "KUBEXIT_DEATH_DEPS", Value: strings.Join(deathDeps, ","), }, - // kcore.EnvVar{ - // Name: "KUBEXIT_IGNORE_CODE_ON_DEATH_DEPS", - // Value: "true", - // }, + kcore.EnvVar{ + Name: "KUBEXIT_IGNORE_CODE_ON_DEATH_DEPS", + Value: "true", + }, ) } @@ -119,10 +119,10 @@ func getKubexitEnvVars(containerName string, deathDeps []string, birthDeps []str Name: "KUBEXIT_BIRTH_DEPS", Value: strings.Join(birthDeps, ","), }, - // kcore.EnvVar{ - // Name: "KUBEXIT_IGNORE_CODE_ON_DEATH_DEPS", - // Value: "true", - // }, + kcore.EnvVar{ + Name: "KUBEXIT_IGNORE_CODE_ON_DEATH_DEPS", + Value: "true", + }, ) } diff --git a/pkg/workloads/k8s.go b/pkg/workloads/k8s.go index b86078d870..360b210139 100644 --- a/pkg/workloads/k8s.go +++ b/pkg/workloads/k8s.go @@ -167,18 +167,24 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { kcore.VolumeMount{Name: _kubexitGraveyardName, MountPath: _kubexitGraveyardMountPath}, ) - neuronRTDEnvVars := getKubexitEnvVars(_neuronRTDContainerName, containerNames.Slice(), nil) - containers = append(containers, neuronRuntimeDaemonContainer(container.Compute.Inf, rtdVolumeMounts, neuronRTDEnvVars)) podHasInf = true + if api.Kind == userconfig.BatchAPIKind || api.Kind == userconfig.TaskAPIKind { + neuronRTDEnvVars := getKubexitEnvVars(_neuronRTDContainerName, containerNames.Slice(), nil) + containers = append(containers, neuronRuntimeDaemonContainer(container.Compute.Inf, rtdVolumeMounts, neuronRTDEnvVars)) + } else { + containers = append(containers, neuronRuntimeDaemonContainer(container.Compute.Inf, rtdVolumeMounts, nil)) + } } - containerDeathDependencies := containerNames.Copy() - containerDeathDependencies.Remove(container.Name) var containerEnvVars []kcore.EnvVar - if podHasInf { - containerEnvVars = getKubexitEnvVars(container.Name, containerDeathDependencies.Slice(), []string{"neuron-rtd"}) - } else { - containerEnvVars = getKubexitEnvVars(container.Name, containerDeathDependencies.Slice(), nil) + if api.Kind == userconfig.BatchAPIKind || api.Kind == userconfig.TaskAPIKind { + containerDeathDependencies := containerNames.Copy() + containerDeathDependencies.Remove(container.Name) + if podHasInf { + containerEnvVars = getKubexitEnvVars(container.Name, containerDeathDependencies.Slice(), []string{"neuron-rtd"}) + } else { + containerEnvVars = getKubexitEnvVars(container.Name, containerDeathDependencies.Slice(), nil) + } } for k, v := range container.Env { From 6fb6bfb581de9dcae6e7fd598acaf94b32a0cce7 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Fri, 14 May 2021 21:53:11 +0300 Subject: [PATCH 12/82] WIP CaaS --- pkg/types/spec/errors.go | 8 ++++---- pkg/types/spec/validations.go | 18 ++++++++++-------- pkg/types/userconfig/api.go | 21 +++++++++++++++------ pkg/types/userconfig/config_key.go | 2 -- pkg/workloads/helpers.go | 20 ++++++++++++++------ pkg/workloads/init.go | 6 +++--- pkg/workloads/k8s.go | 21 ++++++++++++++++----- test/apis/realtime-caas/Dockerfile | 2 +- test/apis/realtime-caas/cortex.yaml | 12 +----------- test/apis/realtime-caas/main.py | 2 +- 10 files changed, 65 insertions(+), 47 deletions(-) diff --git a/pkg/types/spec/errors.go b/pkg/types/spec/errors.go index 0441ef1b6d..00d4ed7b73 100644 --- a/pkg/types/spec/errors.go +++ b/pkg/types/spec/errors.go @@ -208,17 +208,17 @@ func ErrorSurgeAndUnavailableBothZero() error { }) } -func ErrorShmSizeCannotExceedMem(parentFieldName string, shmSize k8s.Quantity, mem k8s.Quantity) error { +func ErrorShmSizeCannotExceedMem(shmSize k8s.Quantity, mem k8s.Quantity) error { return errors.WithStack(&errors.Error{ Kind: ErrShmSizeCannotExceedMem, - Message: fmt.Sprintf("%s.shm_size (%s) cannot exceed total compute mem (%s)", parentFieldName, shmSize.UserString, mem.UserString), + Message: fmt.Sprintf("shm_size (%s) cannot exceed total compute mem (%s)", shmSize.UserString, mem.UserString), }) } -func ErrorCortexPrefixedEnvVarNotAllowed() error { +func ErrorCortexPrefixedEnvVarNotAllowed(prefixes ...string) error { return errors.WithStack(&errors.Error{ Kind: ErrCortexPrefixedEnvVarNotAllowed, - Message: "environment variables starting with CORTEX_ are reserved", + Message: fmt.Sprintf("environment variables starting with %s are reserved", s.StrsOr(prefixes)), }) } diff --git a/pkg/types/spec/validations.go b/pkg/types/spec/validations.go index 2e9340048e..63885cc201 100644 --- a/pkg/types/spec/validations.go +++ b/pkg/types/spec/validations.go @@ -183,10 +183,11 @@ func containersValidation() *cr.StructFieldValidation { { StructField: "Name", StringValidation: &cr.StringValidation{ - Required: true, - AllowEmpty: false, - AlphaNumericDashUnderscore: true, - DisallowedValues: consts.ReservedContainerNames, + Required: true, + AllowEmpty: false, + DNS1035: true, + MaxLength: 63, + DisallowedValues: consts.ReservedContainerNames, }, }, { @@ -550,7 +551,7 @@ func validatePod( if api.Pod.ShmSize != nil { if totalCompute.Mem != nil && api.Pod.ShmSize.Cmp(totalCompute.Mem.Quantity) > 0 { - return ErrorShmSizeCannotExceedMem(userconfig.HandlerKey, *api.Pod.ShmSize, *totalCompute.Mem) + return ErrorShmSizeCannotExceedMem(*api.Pod.ShmSize, *totalCompute.Mem) } } @@ -574,16 +575,17 @@ func validateContainers( for i, container := range containers { if slices.HasString(containerNames, container.Name) { - return ErrorDuplicateContainerName(container.Name) + return errors.Wrap(ErrorDuplicateContainerName(container.Name), strconv.FormatInt(int64(i), 10), userconfig.ImageKey) } + containerNames = append(containerNames, container.Name) if err := validateDockerImagePath(container.Image, awsClient, k8sClient); err != nil { return errors.Wrap(err, strconv.FormatInt(int64(i), 10), userconfig.ImageKey) } for key := range container.Env { - if strings.HasPrefix(key, "CORTEX_") { - return errors.Wrap(ErrorCortexPrefixedEnvVarNotAllowed(), strconv.FormatInt(int64(i), 10), userconfig.EnvKey, key) + if strings.HasPrefix(key, "CORTEX_") || strings.HasPrefix(key, "KUBEXIT_") { + return errors.Wrap(ErrorCortexPrefixedEnvVarNotAllowed("CORTEX_", "KUBEXIT_"), strconv.FormatInt(int64(i), 10), userconfig.EnvKey, key) } } } diff --git a/pkg/types/userconfig/api.go b/pkg/types/userconfig/api.go index 9c0aa2d054..156c0c8d03 100644 --- a/pkg/types/userconfig/api.go +++ b/pkg/types/userconfig/api.go @@ -235,7 +235,7 @@ func (api *API) UserStr() string { } if api.Pod != nil { - sb.WriteString(fmt.Sprintf("%s:\n", HandlerKey)) + sb.WriteString(fmt.Sprintf("%s:\n", PodKey)) sb.WriteString(s.Indent(api.Pod.UserStr(), " ")) } @@ -279,7 +279,7 @@ func (pod *Pod) UserStr() string { sb.WriteString(fmt.Sprintf("%s:\n", ContainersKey)) for _, container := range pod.Containers { - containerUserStr := s.Indent(container.UserStr(), " ") + containerUserStr := s.Indent(container.UserStr(), " ") containerUserStr = containerUserStr[:2] + "-" + containerUserStr[3:] sb.WriteString(containerUserStr) } @@ -299,8 +299,17 @@ func (container *Container) UserStr() string { sb.WriteString(s.Indent(string(d), " ")) } - sb.WriteString(fmt.Sprintf("%s: %s\n", CommandKey, s.ObjFlatNoQuotes(container.Command))) - sb.WriteString(fmt.Sprintf("%s: %s\n", ArgsKey, s.ObjFlatNoQuotes(container.Args))) + if container.Command == nil { + sb.WriteString(fmt.Sprintf("%s: null\n", CommandKey)) + } else { + sb.WriteString(fmt.Sprintf("%s: %s\n", CommandKey, s.ObjFlatNoQuotes(container.Command))) + } + + if container.Args == nil { + sb.WriteString(fmt.Sprintf("%s: null\n", ArgsKey)) + } else { + sb.WriteString(fmt.Sprintf("%s: %s\n", ArgsKey, s.ObjFlatNoQuotes(container.Args))) + } if container.Compute != nil { sb.WriteString(fmt.Sprintf("%s:\n", ComputeKey)) @@ -410,7 +419,7 @@ func GetTotalComputeFromContainers(containers []*Container) Compute { } if container.Compute.CPU != nil { - newCPUQuantity := k8s.NewQuantity(container.Compute.CPU.Value()) + newCPUQuantity := k8s.NewMilliQuantity(container.Compute.CPU.ToDec().MilliValue()) if compute.CPU == nil { compute.CPU = newCPUQuantity } else if newCPUQuantity != nil { @@ -419,7 +428,7 @@ func GetTotalComputeFromContainers(containers []*Container) Compute { } if container.Compute.Mem != nil { - newMemQuantity := k8s.NewQuantity(container.Compute.Mem.Value()) + newMemQuantity := k8s.NewMilliQuantity(container.Compute.Mem.ToDec().MilliValue()) if compute.Mem == nil { compute.Mem = newMemQuantity } else if newMemQuantity != nil { diff --git a/pkg/types/userconfig/config_key.go b/pkg/types/userconfig/config_key.go index 216b3fdbec..598db9f97f 100644 --- a/pkg/types/userconfig/config_key.go +++ b/pkg/types/userconfig/config_key.go @@ -20,8 +20,6 @@ const ( // API NameKey = "name" KindKey = "kind" - HandlerKey = "handler" - TaskDefinitionKey = "definition" NetworkingKey = "networking" ComputeKey = "compute" AutoscalingKey = "autoscaling" diff --git a/pkg/workloads/helpers.go b/pkg/workloads/helpers.go index 2a6c230886..bfb06dcf55 100644 --- a/pkg/workloads/helpers.go +++ b/pkg/workloads/helpers.go @@ -129,10 +129,9 @@ func getKubexitEnvVars(containerName string, deathDeps []string, birthDeps []str return envVars } -func defaultVolumes() []kcore.Volume { - return []kcore.Volume{ +func defaultVolumes(requiresKubexit bool) []kcore.Volume { + volumes := []kcore.Volume{ k8s.EmptyDirVolume(_emptyDirVolumeName), - k8s.EmptyDirVolume(_kubexitGraveyardName), { Name: "client-config", VolumeSource: kcore.VolumeSource{ @@ -144,16 +143,25 @@ func defaultVolumes() []kcore.Volume { }, }, } + + if requiresKubexit { + return append(volumes, k8s.EmptyDirVolume(_kubexitGraveyardName)) + } + return volumes } -func defaultVolumeMounts() []kcore.VolumeMount { - return []kcore.VolumeMount{ +func defaultVolumeMounts(requiresKubexit bool) []kcore.VolumeMount { + volumeMounts := []kcore.VolumeMount{ k8s.EmptyDirVolumeMount(_emptyDirVolumeName, _emptyDirMountPath), - k8s.EmptyDirVolumeMount(_kubexitGraveyardName, _kubexitGraveyardMountPath), { Name: "client-config", MountPath: path.Join(_clientConfigDir, "cli.yaml"), SubPath: "cli.yaml", }, } + + if requiresKubexit { + return append(volumeMounts, k8s.EmptyDirVolumeMount(_kubexitGraveyardName, _kubexitGraveyardMountPath)) + } + return volumeMounts } diff --git a/pkg/workloads/init.go b/pkg/workloads/init.go index 88dd7f903c..ef84d13aba 100644 --- a/pkg/workloads/init.go +++ b/pkg/workloads/init.go @@ -44,7 +44,7 @@ func KubexitInitContainer() kcore.Container { Image: config.ClusterConfig.ImageKubexit, ImagePullPolicy: kcore.PullAlways, Command: []string{"cp", "/bin/kubexit", "/mnt/kubexit"}, - VolumeMounts: defaultVolumeMounts(), + VolumeMounts: defaultVolumeMounts(true), } } @@ -79,7 +79,7 @@ func TaskInitContainer(job *spec.TaskJob) kcore.Container { Value: strings.ToUpper(userconfig.InfoLogLevel.String()), }, }, - VolumeMounts: defaultVolumeMounts(), + VolumeMounts: defaultVolumeMounts(true), } } @@ -114,6 +114,6 @@ func BatchInitContainer(job *spec.BatchJob) kcore.Container { Value: strings.ToUpper(userconfig.InfoLogLevel.String()), }, }, - VolumeMounts: defaultVolumeMounts(), + VolumeMounts: defaultVolumeMounts(true), } } diff --git a/pkg/workloads/k8s.go b/pkg/workloads/k8s.go index 360b210139..1ce76cbb30 100644 --- a/pkg/workloads/k8s.go +++ b/pkg/workloads/k8s.go @@ -109,7 +109,8 @@ func AsyncGatewayContainer(api spec.API, queueURL string, volumeMounts []kcore.V } func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { - volumes := defaultVolumes() + requiresKubexit := api.Kind == userconfig.BatchAPIKind || api.Kind == userconfig.TaskAPIKind + volumes := defaultVolumes(requiresKubexit) defaultMounts := []kcore.VolumeMount{} if api.Pod.ShmSize != nil { @@ -148,7 +149,7 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { containerResourceLimitsList["nvidia.com/gpu"] = *kresource.NewQuantity(container.Compute.GPU, kresource.DecimalSI) } - containerVolumeMounts := append(defaultVolumeMounts(), defaultMounts...) + containerVolumeMounts := append(defaultVolumeMounts(requiresKubexit), defaultMounts...) if container.Compute.Inf > 0 { volumes = append(volumes, kcore.Volume{ Name: "neuron-sock", @@ -168,7 +169,7 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { ) podHasInf = true - if api.Kind == userconfig.BatchAPIKind || api.Kind == userconfig.TaskAPIKind { + if requiresKubexit { neuronRTDEnvVars := getKubexitEnvVars(_neuronRTDContainerName, containerNames.Slice(), nil) containers = append(containers, neuronRuntimeDaemonContainer(container.Compute.Inf, rtdVolumeMounts, neuronRTDEnvVars)) } else { @@ -177,7 +178,7 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { } var containerEnvVars []kcore.EnvVar - if api.Kind == userconfig.BatchAPIKind || api.Kind == userconfig.TaskAPIKind { + if requiresKubexit { containerDeathDependencies := containerNames.Copy() containerDeathDependencies.Remove(container.Name) if podHasInf { @@ -202,10 +203,15 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { }, }) + var containerCmd []string + if requiresKubexit { + containerCmd = append([]string{"/mnt/kubexit"}, container.Command...) + } + containers = append(containers, kcore.Container{ Name: container.Name, Image: container.Image, - Command: append([]string{"/mnt/kubexit"}, container.Command...), + Command: containerCmd, Args: container.Args, Env: containerEnvVars, VolumeMounts: containerVolumeMounts, @@ -213,6 +219,11 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { Requests: containerResourceList, Limits: containerResourceLimitsList, }, + Ports: []kcore.ContainerPort{ + { + ContainerPort: int32(8888), + }, + }, ImagePullPolicy: kcore.PullAlways, SecurityContext: &kcore.SecurityContext{ Privileged: pointer.Bool(true), diff --git a/test/apis/realtime-caas/Dockerfile b/test/apis/realtime-caas/Dockerfile index e5109eb08c..05845435a4 100644 --- a/test/apis/realtime-caas/Dockerfile +++ b/test/apis/realtime-caas/Dockerfile @@ -18,4 +18,4 @@ RUN pip install Flask gunicorn # For environments with multiple CPU cores, increase the number of workers # to be equal to the cores available. # Timeout is set to 0 to disable the timeouts of the workers to allow Cloud Run to handle instance scaling. -CMD exec gunicorn --bind :7000 --workers 1 --threads 8 --timeout 0 main:app +CMD exec gunicorn --bind :8888 --workers 1 --threads 8 --timeout 0 main:app diff --git a/test/apis/realtime-caas/cortex.yaml b/test/apis/realtime-caas/cortex.yaml index f5e570af05..4770d44709 100644 --- a/test/apis/realtime-caas/cortex.yaml +++ b/test/apis/realtime-caas/cortex.yaml @@ -1,20 +1,10 @@ - name: realtime kind: RealtimeAPI pod: + node_groups: [cpu] containers: - name: api image: 499593605069.dkr.ecr.us-west-2.amazonaws.com/sample/realtime-caas:latest - command: - - gunicorn - - --bind - - :7000 - - --workers - - "1" - - --threads - - "8" - - --timeout - - "0" - - main:app compute: cpu: 200m mem: 512Mi diff --git a/test/apis/realtime-caas/main.py b/test/apis/realtime-caas/main.py index a226fc4ff4..6ea9ba744f 100644 --- a/test/apis/realtime-caas/main.py +++ b/test/apis/realtime-caas/main.py @@ -12,4 +12,4 @@ def hello_world(): if __name__ == "__main__": - app.run(debug=True, host="0.0.0.0", port=7000) + app.run(debug=True, host="0.0.0.0", port=8888) From 6744ec8cfac13b15e1ba4b1d9efa6837d0539f30 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Fri, 14 May 2021 23:34:49 +0300 Subject: [PATCH 13/82] Add task example (plus some fixes) --- pkg/config/config.go | 1 - pkg/types/spec/errors.go | 35 ++++++++++--------- pkg/types/spec/validations.go | 7 +++- pkg/workloads/k8s.go | 20 +++++------ .../{realtime-caas => realtime}/.dockerignore | 0 .../{realtime-caas => realtime}/Dockerfile | 0 .../{realtime-caas => realtime}/cortex.yaml | 0 test/apis/{realtime-caas => realtime}/main.py | 0 test/apis/task/.dockerignore | 7 ++++ test/apis/task/Dockerfile | 17 +++++++++ test/apis/task/cortex.yaml | 12 +++++++ test/apis/task/hello-world/cortex.yaml | 7 ---- test/apis/task/hello-world/task.py | 18 ---------- .../task/iris-classifier-training/cortex.yaml | 7 ---- .../iris-classifier-training/requirements.txt | 2 -- .../task/iris-classifier-training/task.py | 31 ---------------- test/apis/task/main.py | 19 ++++++++++ 17 files changed, 89 insertions(+), 94 deletions(-) rename test/apis/{realtime-caas => realtime}/.dockerignore (100%) rename test/apis/{realtime-caas => realtime}/Dockerfile (100%) rename test/apis/{realtime-caas => realtime}/cortex.yaml (100%) rename test/apis/{realtime-caas => realtime}/main.py (100%) create mode 100644 test/apis/task/.dockerignore create mode 100644 test/apis/task/Dockerfile create mode 100644 test/apis/task/cortex.yaml delete mode 100644 test/apis/task/hello-world/cortex.yaml delete mode 100644 test/apis/task/hello-world/task.py delete mode 100644 test/apis/task/iris-classifier-training/cortex.yaml delete mode 100644 test/apis/task/iris-classifier-training/requirements.txt delete mode 100644 test/apis/task/iris-classifier-training/task.py create mode 100644 test/apis/task/main.py diff --git a/pkg/config/config.go b/pkg/config/config.go index 3367d843c8..2343c32ee5 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -170,7 +170,6 @@ func Init() error { } Prometheus = promv1.NewAPI(promClient) - if K8sAllNamspaces, err = k8s.New("", OperatorMetadata.IsOperatorInCluster, nil, scheme); err != nil { return err } diff --git a/pkg/types/spec/errors.go b/pkg/types/spec/errors.go index 00d4ed7b73..9d628c5c2f 100644 --- a/pkg/types/spec/errors.go +++ b/pkg/types/spec/errors.go @@ -51,23 +51,17 @@ const ( ErrShmSizeCannotExceedMem = "spec.shm_size_cannot_exceed_mem" - ErrFieldMustBeDefinedForHandlerType = "spec.field_must_be_defined_for_handler_type" - ErrFieldNotSupportedByHandlerType = "spec.field_not_supported_by_handler_type" - ErrNoAvailableNodeComputeLimit = "spec.no_available_node_compute_limit" - ErrCortexPrefixedEnvVarNotAllowed = "spec.cortex_prefixed_env_var_not_allowed" - ErrRegistryInDifferentRegion = "spec.registry_in_different_region" - ErrRegistryAccountIDMismatch = "spec.registry_account_id_mismatch" - ErrKeyIsNotSupportedForKind = "spec.key_is_not_supported_for_kind" - ErrComputeResourceConflict = "spec.compute_resource_conflict" - ErrInvalidNumberOfInfs = "spec.invalid_number_of_infs" - ErrInsufficientBatchConcurrencyLevel = "spec.insufficient_batch_concurrency_level" - ErrInsufficientBatchConcurrencyLevelInf = "spec.insufficient_batch_concurrency_level_inf" - ErrConcurrencyMismatchServerSideBatchingPython = "spec.concurrency_mismatch_server_side_batching_python" - ErrIncorrectTrafficSplitterWeight = "spec.incorrect_traffic_splitter_weight" - ErrTrafficSplitterAPIsNotUnique = "spec.traffic_splitter_apis_not_unique" - ErrOneShadowPerTrafficSplitter = "spec.one_shadow_per_traffic_splitter" - ErrUnexpectedDockerSecretData = "spec.unexpected_docker_secret_data" - ErrInvalidONNXHandlerType = "spec.invalid_onnx_handler_type" + ErrFieldCannotBeEmptyForKind = "spec.field_cannot_be_empty_for_kind" + ErrCortexPrefixedEnvVarNotAllowed = "spec.cortex_prefixed_env_var_not_allowed" + ErrRegistryInDifferentRegion = "spec.registry_in_different_region" + ErrRegistryAccountIDMismatch = "spec.registry_account_id_mismatch" + ErrKeyIsNotSupportedForKind = "spec.key_is_not_supported_for_kind" + ErrComputeResourceConflict = "spec.compute_resource_conflict" + ErrInvalidNumberOfInfs = "spec.invalid_number_of_infs" + ErrIncorrectTrafficSplitterWeight = "spec.incorrect_traffic_splitter_weight" + ErrTrafficSplitterAPIsNotUnique = "spec.traffic_splitter_apis_not_unique" + ErrOneShadowPerTrafficSplitter = "spec.one_shadow_per_traffic_splitter" + ErrUnexpectedDockerSecretData = "spec.unexpected_docker_secret_data" ) func ErrorMalformedConfig() error { @@ -215,6 +209,13 @@ func ErrorShmSizeCannotExceedMem(shmSize k8s.Quantity, mem k8s.Quantity) error { }) } +func ErrorFieldCannotBeEmptyForKind(field string, kind userconfig.Kind) error { + return errors.WithStack(&errors.Error{ + Kind: ErrFieldCannotBeEmptyForKind, + Message: fmt.Sprintf("field %s cannot be empty for %s kind", field, kind.String()), + }) +} + func ErrorCortexPrefixedEnvVarNotAllowed(prefixes ...string) error { return errors.WithStack(&errors.Error{ Kind: ErrCortexPrefixedEnvVarNotAllowed, diff --git a/pkg/types/spec/validations.go b/pkg/types/spec/validations.go index 63885cc201..fa687c487e 100644 --- a/pkg/types/spec/validations.go +++ b/pkg/types/spec/validations.go @@ -559,7 +559,7 @@ func validatePod( return errors.Wrap(err, userconfig.ComputeKey) } - if err := validateContainers(containers, awsClient, k8sClient); err != nil { + if err := validateContainers(containers, api.Kind, awsClient, k8sClient); err != nil { return errors.Wrap(err, userconfig.ContainersKey) } @@ -568,6 +568,7 @@ func validatePod( func validateContainers( containers []*userconfig.Container, + kind userconfig.Kind, awsClient *aws.Client, k8sClient *k8s.Client, ) error { @@ -579,6 +580,10 @@ func validateContainers( } containerNames = append(containerNames, container.Name) + if container.Command == nil && (kind == userconfig.BatchAPIKind || kind == userconfig.TaskAPIKind) { + return errors.Wrap(ErrorFieldCannotBeEmptyForKind(userconfig.CommandKey, kind), strconv.FormatInt(int64(i), 10), userconfig.CommandKey) + } + if err := validateDockerImagePath(container.Image, awsClient, k8sClient); err != nil { return errors.Wrap(err, strconv.FormatInt(int64(i), 10), userconfig.ImageKey) } diff --git a/pkg/workloads/k8s.go b/pkg/workloads/k8s.go index 1ce76cbb30..2b5a7cc5b6 100644 --- a/pkg/workloads/k8s.go +++ b/pkg/workloads/k8s.go @@ -112,7 +112,7 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { requiresKubexit := api.Kind == userconfig.BatchAPIKind || api.Kind == userconfig.TaskAPIKind volumes := defaultVolumes(requiresKubexit) - defaultMounts := []kcore.VolumeMount{} + containerMounts := []kcore.VolumeMount{} if api.Pod.ShmSize != nil { volumes = append(volumes, kcore.Volume{ Name: "dshm", @@ -123,7 +123,7 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { }, }, }) - defaultMounts = append(defaultMounts, kcore.VolumeMount{ + containerMounts = append(containerMounts, kcore.VolumeMount{ Name: "dshm", MountPath: "/dev/shm", }) @@ -149,7 +149,7 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { containerResourceLimitsList["nvidia.com/gpu"] = *kresource.NewQuantity(container.Compute.GPU, kresource.DecimalSI) } - containerVolumeMounts := append(defaultVolumeMounts(requiresKubexit), defaultMounts...) + containerVolumeMounts := append(defaultVolumeMounts(requiresKubexit), containerMounts...) if container.Compute.Inf > 0 { volumes = append(volumes, kcore.Volume{ Name: "neuron-sock", @@ -163,18 +163,18 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { containerVolumeMounts = append(containerVolumeMounts, rtdVolumeMounts...) - rtdVolumeMounts = append(rtdVolumeMounts, - k8s.EmptyDirVolumeMount(_emptyDirVolumeName, _emptyDirMountPath), - kcore.VolumeMount{Name: _kubexitGraveyardName, MountPath: _kubexitGraveyardMountPath}, - ) - - podHasInf = true if requiresKubexit { + rtdVolumeMounts = append(rtdVolumeMounts, + k8s.EmptyDirVolumeMount(_emptyDirVolumeName, _emptyDirMountPath), + kcore.VolumeMount{Name: _kubexitGraveyardName, MountPath: _kubexitGraveyardMountPath}, + ) neuronRTDEnvVars := getKubexitEnvVars(_neuronRTDContainerName, containerNames.Slice(), nil) containers = append(containers, neuronRuntimeDaemonContainer(container.Compute.Inf, rtdVolumeMounts, neuronRTDEnvVars)) } else { containers = append(containers, neuronRuntimeDaemonContainer(container.Compute.Inf, rtdVolumeMounts, nil)) } + + podHasInf = true } var containerEnvVars []kcore.EnvVar @@ -204,7 +204,7 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { }) var containerCmd []string - if requiresKubexit { + if requiresKubexit && container.Command[0] != "/mnt/kubexit" { containerCmd = append([]string{"/mnt/kubexit"}, container.Command...) } diff --git a/test/apis/realtime-caas/.dockerignore b/test/apis/realtime/.dockerignore similarity index 100% rename from test/apis/realtime-caas/.dockerignore rename to test/apis/realtime/.dockerignore diff --git a/test/apis/realtime-caas/Dockerfile b/test/apis/realtime/Dockerfile similarity index 100% rename from test/apis/realtime-caas/Dockerfile rename to test/apis/realtime/Dockerfile diff --git a/test/apis/realtime-caas/cortex.yaml b/test/apis/realtime/cortex.yaml similarity index 100% rename from test/apis/realtime-caas/cortex.yaml rename to test/apis/realtime/cortex.yaml diff --git a/test/apis/realtime-caas/main.py b/test/apis/realtime/main.py similarity index 100% rename from test/apis/realtime-caas/main.py rename to test/apis/realtime/main.py diff --git a/test/apis/task/.dockerignore b/test/apis/task/.dockerignore new file mode 100644 index 0000000000..3e4bdd9fbb --- /dev/null +++ b/test/apis/task/.dockerignore @@ -0,0 +1,7 @@ +Dockerfile +README.md +*.pyc +*.pyo +*.pyd +__pycache__ +.pytest_cache diff --git a/test/apis/task/Dockerfile b/test/apis/task/Dockerfile new file mode 100644 index 0000000000..bfb7703693 --- /dev/null +++ b/test/apis/task/Dockerfile @@ -0,0 +1,17 @@ +# Use the official lightweight Python image. +# https://hub.docker.com/_/python +FROM python:3.9-slim + +# Allow statements and log messages to immediately appear in the Knative logs +ENV PYTHONUNBUFFERED True + +# Copy local code to the container image. +ENV APP_HOME /app +WORKDIR $APP_HOME +COPY . ./ + +# Install production dependencies. +RUN pip install boto3==1.17.72 + +# Run task +CMD exec python main.py diff --git a/test/apis/task/cortex.yaml b/test/apis/task/cortex.yaml new file mode 100644 index 0000000000..403bbab606 --- /dev/null +++ b/test/apis/task/cortex.yaml @@ -0,0 +1,12 @@ +- name: task + kind: TaskAPI + pod: + containers: + - name: api + image: 499593605069.dkr.ecr.us-west-2.amazonaws.com/sample/task-caas:latest + command: + - python + - main.py + compute: + cpu: 100m + mem: 256Mi diff --git a/test/apis/task/hello-world/cortex.yaml b/test/apis/task/hello-world/cortex.yaml deleted file mode 100644 index ab88dea022..0000000000 --- a/test/apis/task/hello-world/cortex.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- name: hello-world - kind: TaskAPI - definition: - path: task.py - compute: - cpu: 50m - mem: 100Mi diff --git a/test/apis/task/hello-world/task.py b/test/apis/task/hello-world/task.py deleted file mode 100644 index 8d3bf2adc1..0000000000 --- a/test/apis/task/hello-world/task.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class Task: - def __call__(self, config): - print("hello world!") diff --git a/test/apis/task/iris-classifier-training/cortex.yaml b/test/apis/task/iris-classifier-training/cortex.yaml deleted file mode 100644 index 209a847c6c..0000000000 --- a/test/apis/task/iris-classifier-training/cortex.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- name: trainer - kind: TaskAPI - definition: - path: task.py - compute: - cpu: 200m - mem: 500Mi diff --git a/test/apis/task/iris-classifier-training/requirements.txt b/test/apis/task/iris-classifier-training/requirements.txt deleted file mode 100644 index bbc213cf3e..0000000000 --- a/test/apis/task/iris-classifier-training/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -boto3 -scikit-learn==0.21.3 diff --git a/test/apis/task/iris-classifier-training/task.py b/test/apis/task/iris-classifier-training/task.py deleted file mode 100644 index 651d45e4d1..0000000000 --- a/test/apis/task/iris-classifier-training/task.py +++ /dev/null @@ -1,31 +0,0 @@ -import os -import pickle - -import boto3 -from sklearn.datasets import load_iris -from sklearn.linear_model import LogisticRegression -from sklearn.model_selection import train_test_split - - -class Task: - def __call__(self, config): - # get the iris flower dataset - iris = load_iris() - data, labels = iris.data, iris.target - training_data, test_data, training_labels, test_labels = train_test_split(data, labels) - print("loaded dataset") - - # train the model - model = LogisticRegression(solver="lbfgs", multi_class="multinomial", max_iter=1000) - model.fit(training_data, training_labels) - accuracy = model.score(test_data, test_labels) - print("model trained; accuracy: {:.2f}".format(accuracy)) - - # upload the model - if config.get("dest_s3_dir"): - dest_dir = config["dest_s3_dir"] - bucket, key = dest_dir.replace("s3://", "").split("/", 1) - pickle.dump(model, open("model.pkl", "wb")) - s3 = boto3.client("s3") - s3.upload_file("model.pkl", bucket, os.path.join(key, "model.pkl")) - print(f"model uploaded to {dest_dir}/model.pkl") diff --git a/test/apis/task/main.py b/test/apis/task/main.py new file mode 100644 index 0000000000..e51a020b8b --- /dev/null +++ b/test/apis/task/main.py @@ -0,0 +1,19 @@ +import json, re, os, boto3 + +def main(): + with open("/mnt/job_spec.json", "r") as f: + job_spec = json.load(f) + print(json.dumps(job_spec, indent=2)) + + # get metadata + config = job_spec["config"] + job_id = job_spec["job_id"] + s3_path = config["s3_path"] + + # touch file on s3 + bucket, key = re.match("s3://(.+?)/(.+)", s3_path).groups() + s3 = boto3.client("s3") + s3.put_object(Bucket=bucket, Key=os.path.join(key, job_id), Body="") + +if __name__ == "__main__": + main() From f326d627f910b4459179142873ad473ec6615240 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Sat, 15 May 2021 00:15:22 +0300 Subject: [PATCH 14/82] Rename container for example API --- test/apis/task/cortex.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/apis/task/cortex.yaml b/test/apis/task/cortex.yaml index 403bbab606..a9780bbdfa 100644 --- a/test/apis/task/cortex.yaml +++ b/test/apis/task/cortex.yaml @@ -2,7 +2,7 @@ kind: TaskAPI pod: containers: - - name: api + - name: s3-pusher image: 499593605069.dkr.ecr.us-west-2.amazonaws.com/sample/task-caas:latest command: - python From 73ddf51d8d6c785fefee785fb336d6e545a5ca00 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Mon, 17 May 2021 16:57:51 +0300 Subject: [PATCH 15/82] Use istio metrics for cortex get (might have to do this in the proxy instead) --- pkg/operator/resources/realtimeapi/metrics.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/operator/resources/realtimeapi/metrics.go b/pkg/operator/resources/realtimeapi/metrics.go index 64c968ad10..0c8964e5cc 100644 --- a/pkg/operator/resources/realtimeapi/metrics.go +++ b/pkg/operator/resources/realtimeapi/metrics.go @@ -136,10 +136,10 @@ func getRequestCountMetric(promAPIv1 promv1.API, apiSpec spec.API) (float64, err func getAvgLatencyMetric(promAPIv1 promv1.API, apiSpec spec.API) (*float64, error) { query := fmt.Sprintf( - "rate(cortex_latency_sum{api_name=\"%s\", api_id=\"%s\"}[%dh]) "+ - "/ rate(cortex_latency_count{api_name=\"%s\", api_id=\"%s\"}[%dh]) >= 0", - apiSpec.Name, apiSpec.ID, _metricsWindowHours, - apiSpec.Name, apiSpec.ID, _metricsWindowHours, + "sum(rate(istio_request_duration_milliseconds_sum{destination_service_name=~\"api-%s.+\"}[%dh])) by (destination_service_name) "+ + "/ sum(rate(istio_request_duration_milliseconds_count{destination_service_name=~\"api-%s.+\"}[%dh])) by (destination_service_name)", + apiSpec.Name, _metricsWindowHours, + apiSpec.Name, _metricsWindowHours, ) values, err := queryPrometheusVec(promAPIv1, query) From c49eff54cad0d13e2afd0a3c6c598f6098bcf57c Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Mon, 17 May 2021 17:57:55 +0300 Subject: [PATCH 16/82] Use the rest of istio metrics for cortex get --- pkg/operator/resources/realtimeapi/metrics.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/operator/resources/realtimeapi/metrics.go b/pkg/operator/resources/realtimeapi/metrics.go index 0c8964e5cc..4c6c9fcef6 100644 --- a/pkg/operator/resources/realtimeapi/metrics.go +++ b/pkg/operator/resources/realtimeapi/metrics.go @@ -117,8 +117,8 @@ func GetMetrics(api *spec.API) (*metrics.Metrics, error) { func getRequestCountMetric(promAPIv1 promv1.API, apiSpec spec.API) (float64, error) { query := fmt.Sprintf( - "sum(cortex_status_code{api_name=\"%s\", api_id=\"%s\"} >= 0)", - apiSpec.Name, apiSpec.ID, + "istio_requests_total{destination_service_name=~\"api-%s.+\"} > 0", + apiSpec.Name, ) values, err := queryPrometheusVec(promAPIv1, query) @@ -157,8 +157,8 @@ func getAvgLatencyMetric(promAPIv1 promv1.API, apiSpec spec.API) (*float64, erro func getStatusCode2XXMetric(promAPIv1 promv1.API, apiSpec spec.API) (float64, error) { query := fmt.Sprintf( - "sum(cortex_status_code{api_name=\"%s\", api_id=\"%s\", response_code=\"2XX\"} >= 0)", - apiSpec.Name, apiSpec.ID, + "istio_requests_total{destination_service_name=~\"api-%s.+\", response_code=~\"2.*\"} > 0", + apiSpec.Name, ) values, err := queryPrometheusVec(promAPIv1, query) @@ -176,8 +176,8 @@ func getStatusCode2XXMetric(promAPIv1 promv1.API, apiSpec spec.API) (float64, er func getStatusCode4XXMetric(promAPIv1 promv1.API, apiSpec spec.API) (float64, error) { query := fmt.Sprintf( - "sum(cortex_status_code{api_name=\"%s\", api_id=\"%s\", response_code=\"4XX\"} >= 0)", - apiSpec.Name, apiSpec.ID, + "istio_requests_total{destination_service_name=~\"api-%s.+\", response_code=~\"4.*\"} > 0", + apiSpec.Name, ) values, err := queryPrometheusVec(promAPIv1, query) @@ -195,8 +195,8 @@ func getStatusCode4XXMetric(promAPIv1 promv1.API, apiSpec spec.API) (float64, er func getStatusCode5XXMetric(promAPIv1 promv1.API, apiSpec spec.API) (float64, error) { query := fmt.Sprintf( - "sum(cortex_status_code{api_name=\"%s\", api_id=\"%s\", response_code=\"5XX\"} >= 0)", - apiSpec.Name, apiSpec.ID, + "istio_requests_total{destination_service_name=~\"api-%s.+\", response_code=~\"5.*\"} > 0", + apiSpec.Name, ) values, err := queryPrometheusVec(promAPIv1, query) From d9b8e6e0ba0e3c9af028a0d16eff29f0a07e8ab6 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Mon, 17 May 2021 16:19:17 +0100 Subject: [PATCH 17/82] HTTP Reverse Proxy implementation (#2172) * Barebones proxy implementation * Add breaker and proxy transport * Remove stutter from proxy handler * Add prometheus metrics and graceful signal handling * Simplify handler code * Make in flight request counter variable int64 * Remove request-monitor * Add proxy Dockerfile * Rename flags --- cmd/proxy/main.go | 140 ++++++++++ cmd/request-monitor/main.go | 184 ------------- design-spec/async.yaml | 38 --- design-spec/batch.yaml | 23 -- design-spec/realtime.yaml | 95 ------- design-spec/task.yaml | 23 -- go.mod | 1 + go.sum | 1 - images/proxy/Dockerfile | 27 ++ images/request-monitor/Dockerfile | 18 -- pkg/consts/consts.go | 11 +- pkg/proxy/breaker.go | 307 +++++++++++++++++++++ pkg/proxy/breaker_test.go | 424 ++++++++++++++++++++++++++++++ pkg/proxy/consts.go | 30 +++ pkg/proxy/handler.go | 48 ++++ pkg/proxy/handler_test.go | 108 ++++++++ pkg/proxy/proxy.go | 46 ++++ pkg/proxy/proxy_test.go | 42 +++ pkg/proxy/request_stats.go | 90 +++++++ pkg/workloads/k8s.go | 5 +- test/apis/task/main.py | 2 + 21 files changed, 1274 insertions(+), 389 deletions(-) create mode 100644 cmd/proxy/main.go delete mode 100644 cmd/request-monitor/main.go delete mode 100644 design-spec/async.yaml delete mode 100644 design-spec/batch.yaml delete mode 100644 design-spec/realtime.yaml delete mode 100644 design-spec/task.yaml create mode 100644 images/proxy/Dockerfile delete mode 100644 images/request-monitor/Dockerfile create mode 100644 pkg/proxy/breaker.go create mode 100644 pkg/proxy/breaker_test.go create mode 100644 pkg/proxy/consts.go create mode 100644 pkg/proxy/handler.go create mode 100644 pkg/proxy/handler_test.go create mode 100644 pkg/proxy/proxy.go create mode 100644 pkg/proxy/proxy_test.go create mode 100644 pkg/proxy/request_stats.go diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go new file mode 100644 index 0000000000..0b0beea153 --- /dev/null +++ b/cmd/proxy/main.go @@ -0,0 +1,140 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "flag" + "net/http" + "os" + "os/signal" + "strconv" + "time" + + "github.com/cortexlabs/cortex/pkg/lib/logging" + "github.com/cortexlabs/cortex/pkg/proxy" + "go.uber.org/zap" +) + +const ( + _reportInterval = 10 * time.Second + _requestSampleInterval = 1 * time.Second +) + +func main() { + var ( + port int + metricsPort int + userContainerPort int + maxConcurrency int + maxQueueLength int + ) + + flag.IntVar(&port, "port", 8000, "port where the proxy will be served") + flag.IntVar(&metricsPort, "metrics-port", 8001, "port where the proxy will be served") + flag.IntVar(&userContainerPort, "user-port", 8080, "port where the proxy will redirect to the traffic to") + flag.IntVar(&maxConcurrency, "max-concurrency", 0, "max concurrency allowed for user container") + flag.IntVar(&maxQueueLength, "max-queue-length", 0, "max request queue length for user container") + flag.Parse() + + log := logging.GetLogger() + defer func() { + _ = log.Sync() + }() + + switch { + case maxConcurrency == 0: + log.Fatal("--max-concurrency flag is required") + case maxQueueLength == 0: + maxQueueLength = maxConcurrency * 10 + } + + target := "http://127.0.0.1:" + strconv.Itoa(port) + httpProxy := proxy.NewReverseProxy(target, maxQueueLength, maxQueueLength) + + requestCounterStats := &proxy.RequestStats{} + breaker := proxy.NewBreaker( + proxy.BreakerParams{ + QueueDepth: maxQueueLength, + MaxConcurrency: maxConcurrency, + InitialCapacity: maxConcurrency, + }, + ) + + promStats := proxy.NewPrometheusStatsReporter() + + go func() { + reportTicker := time.NewTicker(_reportInterval) + defer reportTicker.Stop() + + requestSamplingTicker := time.NewTicker(_requestSampleInterval) + defer requestSamplingTicker.Stop() + + for { + select { + case <-reportTicker.C: + go func() { + report := requestCounterStats.Report() + promStats.Report(report) + }() + case <-requestSamplingTicker.C: + go func() { + requestCounterStats.Append(breaker.InFlight()) + }() + } + } + }() + + servers := map[string]*http.Server{ + "proxy": { + Addr: ":" + strconv.Itoa(userContainerPort), + Handler: proxy.Handler(breaker, httpProxy), + }, + "metrics": { + Addr: ":" + strconv.Itoa(metricsPort), + Handler: promStats, + }, + } + + errCh := make(chan error) + for name, server := range servers { + go func(name string, server *http.Server) { + log.Infof("Starting %s server on %s", name, server.Addr) + errCh <- server.ListenAndServe() + }(name, server) + } + + sigint := make(chan os.Signal, 1) + signal.Notify(sigint, os.Interrupt) + + select { + case err := <-errCh: + log.Fatal("failed to start proxy server", zap.Error(err)) + case <-sigint: + // We received an interrupt signal, shut down. + log.Info("Received TERM signal, handling a graceful shutdown...") + + for name, server := range servers { + log.Infof("Shutting down %s server", name) + if err := server.Shutdown(context.Background()); err != nil { + // Error from closing listeners, or context timeout: + log.Warn("HTTP server Shutdown Error", zap.Error(err)) + } + } + log.Info("Shutdown complete, exiting...") + } +} diff --git a/cmd/request-monitor/main.go b/cmd/request-monitor/main.go deleted file mode 100644 index 876f3a5749..0000000000 --- a/cmd/request-monitor/main.go +++ /dev/null @@ -1,184 +0,0 @@ -/* -Copyright 2021 Cortex Labs, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "flag" - "fmt" - "net/http" - "os" - "strings" - "sync" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/client_golang/prometheus/promhttp" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -const ( - _tickInterval = 10 * time.Second - _requestSampleInterval = 1 * time.Second - _defaultPort = "15000" -) - -var ( - logger *zap.Logger - requestCounter Counter - inFlightReqGauge = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "cortex_in_flight_requests", - Help: "The number of in-flight requests for a cortex API", - }) -) - -type Counter struct { - sync.Mutex - s []int -} - -func (c *Counter) Append(val int) { - c.Lock() - defer c.Unlock() - c.s = append(c.s, val) -} - -func (c *Counter) GetAllAndDelete() []int { - var output []int - c.Lock() - defer c.Unlock() - output = c.s - c.s = []int{} - return output -} - -// ./request-monitor -p port -func main() { - var port = flag.String("p", _defaultPort, "port on which the server runs on") - - logLevelEnv := strings.ToUpper(os.Getenv("CORTEX_LOG_LEVEL")) - var logLevelZap zapcore.Level - switch logLevelEnv { - case "DEBUG": - logLevelZap = zapcore.DebugLevel - case "INFO": - logLevelZap = zapcore.InfoLevel - case "WARNING": - logLevelZap = zapcore.WarnLevel - case "ERROR": - logLevelZap = zapcore.ErrorLevel - } - - encoderConfig := zap.NewProductionEncoderConfig() - encoderConfig.MessageKey = "message" - - var err error - logger, err = zap.Config{ - Level: zap.NewAtomicLevelAt(logLevelZap), - Encoding: "json", - EncoderConfig: encoderConfig, - OutputPaths: []string{"stdout"}, - ErrorOutputPaths: []string{"stderr"}, - }.Build() - if err != nil { - panic(err) - } - defer func() { - _ = logger.Sync() - }() - - if _, err = os.OpenFile("/request_monitor_ready.txt", os.O_RDONLY|os.O_CREATE, 0666); err != nil { - panic(err) - } - - for { - if _, err := os.Stat("/mnt/workspace/api_readiness.txt"); err == nil { - break - } else if os.IsNotExist(err) { - logger.Debug("waiting for replica to be ready ...") - time.Sleep(_tickInterval) - } else { - logger.Error("error encountered while looking for /mnt/workspace/api_readiness.txt") // unexpected - time.Sleep(_tickInterval) - } - } - - requestCounter = Counter{} - go launchCrons(&requestCounter) - - http.Handle("/metrics", promhttp.Handler()) - - logger.Info(fmt.Sprintf("starting request monitor on :%s", *port)) - if err := http.ListenAndServe(fmt.Sprintf(":%s", *port), nil); err != nil { - logger.Fatal(err.Error()) - } -} - -func launchCrons(requestCounter *Counter) { - updateGaugeTicker := time.NewTicker(_tickInterval) - defer updateGaugeTicker.Stop() - - requestSamplingTimer := time.NewTimer(_requestSampleInterval) - defer requestSamplingTimer.Stop() - - for { - select { - case <-updateGaugeTicker.C: - go updateGauge(requestCounter) - case <-requestSamplingTimer.C: - go updateOpenConnections(requestCounter, requestSamplingTimer) - } - } -} - -func updateGauge(counter *Counter) { - requestCounts := counter.GetAllAndDelete() - - total := 0.0 - if len(requestCounts) > 0 { - for _, val := range requestCounts { - total += float64(val) - } - - total /= float64(len(requestCounts)) - } - logger.Debug(fmt.Sprintf("recorded %.2f in-flight requests on replica", total)) - inFlightReqGauge.Set(total) -} - -func getFileCount() int { - dir, err := os.Open("/mnt/requests") - if err != nil { - panic(err) - } - defer func() { - _ = dir.Close() - }() - - fileNames, err := dir.Readdirnames(0) - if err != nil { - panic(err) - } - return len(fileNames) -} - -func updateOpenConnections(requestCounter *Counter, timer *time.Timer) { - count := getFileCount() - requestCounter.Append(count) - timer.Reset(_requestSampleInterval) -} diff --git a/design-spec/async.yaml b/design-spec/async.yaml deleted file mode 100644 index 0c77623221..0000000000 --- a/design-spec/async.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# new - -- name: - kind: AsyncAPI - shm_size: - node_groups: - log_level: - config: - containers: - - name: api - image: - env: - port: - command: - args: - healthcheck: - compute: - cpu: - gpu: - inf: - mem: - autoscaling: - min_replicas: - max_replicas: - init_replicas: - max_replica_concurrency: - window: - downscale_stabilization_period: - upscale_stabilization_period: - max_downscale_factor: - max_upscale_factor: - downscale_tolerance: - upscale_tolerance: - update_strategy: - max_surge: - max_unavailable: - networking: - endpoint: diff --git a/design-spec/batch.yaml b/design-spec/batch.yaml deleted file mode 100644 index 7721fefc02..0000000000 --- a/design-spec/batch.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# new - -- name: - kind: BatchAPI - shm_size: - node_groups: - log_level: - config: - containers: - - name: batch - image: - env: - port: - command: - args: - healthcheck: - compute: - cpu: - gpu: - inf: - mem: - networking: - endpoint: diff --git a/design-spec/realtime.yaml b/design-spec/realtime.yaml deleted file mode 100644 index 95f4740b76..0000000000 --- a/design-spec/realtime.yaml +++ /dev/null @@ -1,95 +0,0 @@ -# old - -- name: - kind: RealtimeAPI - handler: - type: python - path: - protobuf_path: - dependencies: - pip: - conda: - shell: - multi_model_reloading: - path: - paths: - - name: - path: - dir: - cache_size: - disk_cache_size: - server_side_batching: - max_batch_size: - batch_interval: - processes_per_replica: - threads_per_process: - config: - python_path: - image: - env: - log_level: - shm_size: - compute: - cpu: - gpu: - inf: - mem: - node_groups: - autoscaling: - min_replicas: - max_replicas: - init_replicas: - max_replica_concurrency: - target_replica_concurrency: - window: - downscale_stabilization_period: - upscale_stabilization_period: - max_downscale_factor: - max_upscale_factor: - downscale_tolerance: - upscale_tolerance: - update_strategy: - max_surge: - max_unavailable: - networking: - endpoint: - ---- - -# new - -- name: - kind: RealtimeAPI - pod: - shm_size: - node_groups: - containers: - - name: api - image: - env: - command: - args: - compute: - cpu: - gpu: - inf: - mem: - autoscaling: - min_replicas: - max_replicas: - init_replicas: - max_replica_queue_length: - max_replica_concurrency: - target_replica_concurrency: - window: - downscale_stabilization_period: - upscale_stabilization_period: - max_downscale_factor: - max_upscale_factor: - downscale_tolerance: - upscale_tolerance: - update_strategy: - max_surge: - max_unavailable: - networking: - endpoint: diff --git a/design-spec/task.yaml b/design-spec/task.yaml deleted file mode 100644 index 15c6b63941..0000000000 --- a/design-spec/task.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# new - -- name: - kind: TaskAPI - shm_size: - node_groups: - log_level: - config: - containers: - - name: api - image: - env: - port: - command: - args: - healthcheck: - compute: - cpu: - gpu: - inf: - mem: - networking: - endpoint: diff --git a/go.mod b/go.mod index d7c90d37b1..80fe04706c 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,7 @@ require ( github.com/ugorji/go/codec v1.2.1 github.com/xlab/treeprint v1.0.0 github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect + go.uber.org/atomic v1.6.0 go.uber.org/zap v1.15.0 golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect golang.org/x/mod v0.4.2 // indirect diff --git a/go.sum b/go.sum index 024f8d77d6..9cba6915bb 100644 --- a/go.sum +++ b/go.sum @@ -507,7 +507,6 @@ github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3I github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/patrickmn/go-cache v1.0.0 h1:3gD5McaYs9CxjyK5AXGcq8gdeCARtd/9gJDUvVeaZ0Y= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= diff --git a/images/proxy/Dockerfile b/images/proxy/Dockerfile new file mode 100644 index 0000000000..2573e4e64d --- /dev/null +++ b/images/proxy/Dockerfile @@ -0,0 +1,27 @@ +# Build the manager binary +FROM golang:1.15 as builder + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY pkg pkg +COPY cmd/proxy cmd/proxy +WORKDIR /workspace/cmd/proxy + +# Build +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o /workspace/bin/proxy main.go + +# Use distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM gcr.io/distroless/static:nonroot +WORKDIR / +COPY --from=builder /workspace/bin/proxy . +USER 65532:65532 + +ENTRYPOINT ["/proxy"] diff --git a/images/request-monitor/Dockerfile b/images/request-monitor/Dockerfile deleted file mode 100644 index 750189ef89..0000000000 --- a/images/request-monitor/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM golang:1.15 as builder - -WORKDIR /workspace -COPY go.mod go.sum /workspace/ -RUN go mod download - -COPY cmd/request-monitor /workspace -RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux go build -installsuffix cgo -o request-monitor . - - -FROM alpine:3.12 - -RUN apk --no-cache add ca-certificates bash - -COPY --from=builder /workspace/request-monitor /root/ -RUN chmod +x /root/request-monitor - -ENTRYPOINT ["/root/request-monitor"] diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index d9b3372cad..0c6ee0d22a 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -24,11 +24,12 @@ var ( CortexVersion = "master" // CORTEX_VERSION CortexVersionMinor = "master" // CORTEX_VERSION_MINOR - ProxyListeningPort, ProxyListeningPortStr = int64(8888), "8888" - DefaultMaxReplicaQueueLength = int64(1024) - DefaultMaxReplicaConcurrency = int64(1024) - DefaultTargetReplicaConcurrency = float64(8) - NeuronCoresPerInf = int64(4) + ProxyListeningPort = int64(8888) + ProxyListeningPortStr = "8888" + DefaultMaxReplicaQueueLength = int64(1024) + DefaultMaxReplicaConcurrency = int64(1024) + DefaultTargetReplicaConcurrency = float64(8) + NeuronCoresPerInf = int64(4) AuthHeader = "X-Cortex-Authorization" diff --git a/pkg/proxy/breaker.go b/pkg/proxy/breaker.go new file mode 100644 index 0000000000..aa1e61cba5 --- /dev/null +++ b/pkg/proxy/breaker.go @@ -0,0 +1,307 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Code adapted from https://github.com/knative/serving/blob/main/pkg/queue/breaker.go +*/ + +package proxy + +import ( + "context" + "errors" + "fmt" + + "go.uber.org/atomic" +) + +var ( + // ErrRequestQueueFull indicates the breaker queue depth was exceeded. + ErrRequestQueueFull = errors.New("pending request queue full") +) + +// BreakerParams defines the parameters of the breaker. +type BreakerParams struct { + QueueDepth int + MaxConcurrency int + InitialCapacity int +} + +// Breaker is a component that enforces a concurrency limit on the +// execution of a function. It also maintains a queue of function +// executions in excess of the concurrency limit. Function call attempts +// beyond the limit of the queue are failed immediately. +type Breaker struct { + inFlight atomic.Int64 + totalSlots int64 + sem *semaphore + + // release is the callback function returned to callers by Reserve to + // allow the reservation made by Reserve to be released. + release func() +} + +// NewBreaker creates a Breaker with the desired queue depth, +// concurrency limit and initial capacity. +func NewBreaker(params BreakerParams) *Breaker { + if params.QueueDepth <= 0 { + panic(fmt.Sprintf("Queue depth must be greater than 0. Got %v.", params.QueueDepth)) + } + if params.MaxConcurrency < 0 { + panic(fmt.Sprintf("Max concurrency must be 0 or greater. Got %v.", params.MaxConcurrency)) + } + if params.InitialCapacity < 0 || params.InitialCapacity > params.MaxConcurrency { + panic(fmt.Sprintf("Initial capacity must be between 0 and max concurrency. Got %v.", params.InitialCapacity)) + } + + b := &Breaker{ + totalSlots: int64(params.QueueDepth + params.MaxConcurrency), + sem: newSemaphore(params.MaxConcurrency, params.InitialCapacity), + } + + // Allocating the closure returned by Reserve here avoids an allocation in Reserve. + b.release = func() { + b.sem.release() + b.releasePending() + } + + return b +} + +// tryAcquirePending tries to acquire a slot on the pending "queue". +func (b *Breaker) tryAcquirePending() bool { + // This is an atomic version of: + // + // if inFlight == totalSlots { + // return false + // } else { + // inFlight++ + // return true + // } + // + // We can't just use an atomic increment as we need to check if we're + // "allowed" to increment first. Since a Load and a CompareAndSwap are + // not done atomically, we need to retry until the CompareAndSwap succeeds + // (it fails if we're raced to it) or if we don't fulfill the condition + // anymore. + for { + cur := b.inFlight.Load() + if cur == b.totalSlots { + return false + } + if b.inFlight.CAS(cur, cur+1) { + return true + } + } +} + +// releasePending releases a slot on the pending "queue". +func (b *Breaker) releasePending() { + b.inFlight.Dec() +} + +// Reserve reserves an execution slot in the breaker, to permit +// richer semantics in the caller. +// The caller on success must execute the callback when done with work. +func (b *Breaker) Reserve(_ context.Context) (func(), bool) { + if !b.tryAcquirePending() { + return nil, false + } + + if !b.sem.tryAcquire() { + b.releasePending() + return nil, false + } + + return b.release, true +} + +// Maybe conditionally executes thunk based on the Breaker concurrency +// and queue parameters. If the concurrency limit and queue capacity are +// already consumed, Maybe returns immediately without calling thunk. If +// the thunk was executed, Maybe returns true, else false. +func (b *Breaker) Maybe(ctx context.Context, thunk func()) error { + if !b.tryAcquirePending() { + return ErrRequestQueueFull + } + + defer b.releasePending() + + // Wait for capacity in the active queue. + if err := b.sem.acquire(ctx); err != nil { + return err + } + // Defer releasing capacity in the active. + // It's safe to ignore the error returned by release since we + // make sure the semaphore is only manipulated here and acquire + // + release calls are equally paired. + defer b.sem.release() + + // Do the thing. + thunk() + // Report success + return nil +} + +// InFlight returns the number of requests currently in flight in this breaker. +func (b *Breaker) InFlight() int64 { + return b.inFlight.Load() +} + +// UpdateConcurrency updates the maximum number of in-flight requests. +func (b *Breaker) UpdateConcurrency(size int) { + b.sem.updateCapacity(size) +} + +// Capacity returns the number of allowed in-flight requests on this breaker. +func (b *Breaker) Capacity() int { + return b.sem.Capacity() +} + +// newSemaphore creates a semaphore with the desired initial capacity. +func newSemaphore(maxCapacity, initialCapacity int) *semaphore { + queue := make(chan struct{}, maxCapacity) + sem := &semaphore{queue: queue} + sem.updateCapacity(initialCapacity) + return sem +} + +// semaphore is an implementation of a semaphore based on packed integers and a channel. +// state is an uint64 that has two uint32s packed into it: capacity and inFlight. The +// former specifies how many request are allowed at any given time into the semaphore +// while the latter refers to the currently in-flight requests. +// Packing them both into one uint64 allows us to optimize access semantics using atomic +// operations, which can't be guaranteed on 2 individual values. +// The channel is merely used as a vehicle to be able to "wake up" individual goroutines +// if capacity becomes free. It's not consistently used in accordance to actual capacity +// but is rather a communication vehicle to ensure waiting routines are properly woken +// up. +type semaphore struct { + state atomic.Uint64 + queue chan struct{} +} + +// tryAcquire receives a token from the semaphore if there is one otherwise returns false. +func (s *semaphore) tryAcquire() bool { + for { + old := s.state.Load() + capacity, in := unpack(old) + if in >= capacity { + return false + } + in++ + if s.state.CAS(old, pack(capacity, in)) { + return true + } + } +} + +// acquire acquires capacity from the semaphore. +func (s *semaphore) acquire(ctx context.Context) error { + for { + old := s.state.Load() + capacity, in := unpack(old) + + if in >= capacity { + select { + case <-ctx.Done(): + return ctx.Err() + case <-s.queue: + } + // Force reload state. + continue + } + + in++ + if s.state.CAS(old, pack(capacity, in)) { + return nil + } + } +} + +// release releases capacity in the semaphore. +// If the semaphore capacity was reduced in between and as a result inFlight is greater +// than capacity, we don't wake up goroutines as they'd not get any capacity anyway. +func (s *semaphore) release() { + for { + old := s.state.Load() + capacity, in := unpack(old) + + if in == 0 { + panic("release and acquire are not paired") + } + + in-- + if s.state.CAS(old, pack(capacity, in)) { + if in < capacity { + select { + case s.queue <- struct{}{}: + default: + // We generate more wakeups than we might need as we don't know + // how many goroutines are waiting here. It is therefore okay + // to drop the poke on the floor here as this case would mean we + // have enough wakeups to wake up as many goroutines as this semaphore + // can take, which is guaranteed to be enough. + } + } + return + } + } +} + +// updateCapacity updates the capacity of the semaphore to the desired size. +func (s *semaphore) updateCapacity(size int) { + s64 := uint64(size) + for { + old := s.state.Load() + capacity, in := unpack(old) + + if capacity == s64 { + // Nothing to do, exit early. + return + } + + if s.state.CAS(old, pack(s64, in)) { + if s64 > capacity { + for i := uint64(0); i < s64-capacity; i++ { + select { + case s.queue <- struct{}{}: + default: + // See comment in `release` for explanation of this case. + } + } + } + return + } + } +} + +// Capacity is the capacity of the semaphore. +func (s *semaphore) Capacity() int { + capacity, _ := unpack(s.state.Load()) + return int(capacity) +} + +// unpack takes an uint64 and returns two uint32 (as uint64) comprised of the leftmost +// and the rightmost bits respectively. +func unpack(in uint64) (uint64, uint64) { + return in >> 32, in & 0xffffffff +} + +// pack takes two uint32 (as uint64 to avoid casting) and packs them into a single uint64 +// at the leftmost and the rightmost bits respectively. +// It's up to the caller to ensure that left and right actually fit into 32 bit. +func pack(left, right uint64) uint64 { + return left<<32 | right +} diff --git a/pkg/proxy/breaker_test.go b/pkg/proxy/breaker_test.go new file mode 100644 index 0000000000..391b78de62 --- /dev/null +++ b/pkg/proxy/breaker_test.go @@ -0,0 +1,424 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxy + +import ( + "context" + "fmt" + "testing" + "time" +) + +const ( + // semAcquireTimeout is a timeout for tests that try to acquire + // a token of a semaphore. + semAcquireTimeout = 10 * time.Second + + // semNoChangeTimeout is some additional wait time after a number + // of acquires is reached to assert that no more acquires get through. + semNoChangeTimeout = 50 * time.Millisecond +) + +func TestBreakerInvalidConstructor(t *testing.T) { + tests := []struct { + name string + options BreakerParams + }{{ + name: "QueueDepth = 0", + options: BreakerParams{QueueDepth: 0, MaxConcurrency: 1, InitialCapacity: 1}, + }, { + name: "MaxConcurrency negative", + options: BreakerParams{QueueDepth: 1, MaxConcurrency: -1, InitialCapacity: 1}, + }, { + name: "InitialCapacity negative", + options: BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: -1}, + }, { + name: "InitialCapacity out-of-bounds", + options: BreakerParams{QueueDepth: 1, MaxConcurrency: 5, InitialCapacity: 6}, + }} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Error("Expected a panic but the code didn't panic.") + } + }() + + NewBreaker(test.options) + }) + } +} + +func TestBreakerReserveOverload(t *testing.T) { + params := BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 1} + b := NewBreaker(params) // Breaker capacity = 2 + cb1, rr := b.Reserve(context.Background()) + if !rr { + t.Fatal("Reserve1 failed") + } + _, rr = b.Reserve(context.Background()) + if rr { + t.Fatal("Reserve2 was an unexpected success.") + } + // Release a slot. + cb1() + // And reserve it again. + cb2, rr := b.Reserve(context.Background()) + if !rr { + t.Fatal("Reserve2 failed") + } + cb2() +} + +func TestBreakerOverloadMixed(t *testing.T) { + // This tests when reservation and maybe are intermised. + params := BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 1} + b := NewBreaker(params) // Breaker capacity = 2 + reqs := newRequestor(b) + + // Bring breaker to capacity. + reqs.request() + // This happens in go-routine, so spin. + for _, in := unpack(b.sem.state.Load()); in != 1; _, in = unpack(b.sem.state.Load()) { + time.Sleep(time.Millisecond * 2) + } + _, rr := b.Reserve(context.Background()) + if rr { + t.Fatal("Reserve was an unexpected success.") + } + // Open a slot. + reqs.processSuccessfully(t) + // Now reservation should work. + cb, rr := b.Reserve(context.Background()) + if !rr { + t.Fatal("Reserve unexpectedly failed") + } + // Process the reservation. + cb() +} + +func TestBreakerOverload(t *testing.T) { + params := BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 1} + b := NewBreaker(params) // Breaker capacity = 2 + reqs := newRequestor(b) + + // Bring breaker to capacity. + reqs.request() + reqs.request() + + // Overshoot by one. + reqs.request() + reqs.expectFailure(t) + + // The remainer should succeed. + reqs.processSuccessfully(t) + reqs.processSuccessfully(t) +} + +func TestBreakerQueueing(t *testing.T) { + params := BreakerParams{QueueDepth: 2, MaxConcurrency: 1, InitialCapacity: 0} + b := NewBreaker(params) // Breaker capacity = 2 + reqs := newRequestor(b) + + // Bring breaker to capacity. Doesn't error because queue subsumes these requests. + reqs.request() + reqs.request() + + // Update concurrency to allow the requests to be processed. + b.UpdateConcurrency(1) + + // They should pass just fine. + reqs.processSuccessfully(t) + reqs.processSuccessfully(t) +} + +func TestBreakerNoOverload(t *testing.T) { + params := BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 1} + b := NewBreaker(params) // Breaker capacity = 2 + reqs := newRequestor(b) + + // Bring request to capacity. + reqs.request() + reqs.request() + + // Process one, send a new one in, at capacity again. + reqs.processSuccessfully(t) + reqs.request() + + // Process one, send a new one in, at capacity again. + reqs.processSuccessfully(t) + reqs.request() + + // Process the remainder successfully. + reqs.processSuccessfully(t) + reqs.processSuccessfully(t) +} + +func TestBreakerCancel(t *testing.T) { + params := BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 0} + b := NewBreaker(params) + reqs := newRequestor(b) + + // Cancel a request which cannot get capacity. + ctx1, cancel1 := context.WithCancel(context.Background()) + reqs.requestWithContext(ctx1) + cancel1() + reqs.expectFailure(t) + + // This request cannot get capacity either. This reproduced a bug we had when + // freeing slots on the pendingRequests channel. + ctx2, cancel2 := context.WithCancel(context.Background()) + reqs.requestWithContext(ctx2) + cancel2() + reqs.expectFailure(t) + + // Let through a request with capacity then timeout following request + b.UpdateConcurrency(1) + reqs.request() + + // Exceed capacity and assert one failure. This makes sure the Breaker is consistently + // at capacity. + reqs.request() + reqs.request() + reqs.expectFailure(t) + + // This request cannot get capacity. + ctx3, cancel3 := context.WithCancel(context.Background()) + reqs.requestWithContext(ctx3) + cancel3() + reqs.expectFailure(t) + + // The requests that were put in earlier should succeed. + reqs.processSuccessfully(t) + reqs.processSuccessfully(t) +} + +func TestBreakerUpdateConcurrency(t *testing.T) { + params := BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 0} + b := NewBreaker(params) + b.UpdateConcurrency(1) + if got, want := b.Capacity(), 1; got != want { + t.Errorf("Capacity() = %d, want: %d", got, want) + } + + b.UpdateConcurrency(0) + if got, want := b.Capacity(), 0; got != want { + t.Errorf("Capacity() = %d, want: %d", got, want) + } + +} + +// Test empty semaphore, token cannot be acquired +func TestSemaphoreAcquireHasNoCapacity(t *testing.T) { + gotChan := make(chan struct{}, 1) + + sem := newSemaphore(1, 0) + tryAcquire(sem, gotChan) + + select { + case <-gotChan: + t.Error("Token was acquired but shouldn't have been") + case <-time.After(semNoChangeTimeout): + // Test succeeds, semaphore didn't change in configured time. + } +} + +func TestSemaphoreAcquireNonBlockingHasNoCapacity(t *testing.T) { + sem := newSemaphore(1, 0) + if sem.tryAcquire() { + t.Error("Should have failed immediately") + } +} + +// Test empty semaphore, add capacity, token can be acquired +func TestSemaphoreAcquireHasCapacity(t *testing.T) { + gotChan := make(chan struct{}, 1) + want := 1 + + sem := newSemaphore(1, 0) + tryAcquire(sem, gotChan) + sem.updateCapacity(1) // Allows 1 acquire + + for i := 0; i < want; i++ { + select { + case <-gotChan: + // Successfully acquired a token. + case <-time.After(semAcquireTimeout): + t.Error("Was not able to acquire token before timeout") + } + } + + select { + case <-gotChan: + t.Errorf("Got more acquires than wanted, want = %d, got at least %d", want, want+1) + case <-time.After(semNoChangeTimeout): + // No change happened, success. + } +} + +func TestSemaphoreRelease(t *testing.T) { + sem := newSemaphore(1, 1) + sem.acquire(context.Background()) + func() { + defer func() { + if e := recover(); e != nil { + t.Error("Expected no panic, got message:", e) + } + sem.release() + }() + }() + func() { + defer func() { + if e := recover(); e == nil { + t.Error("Expected panic, but got none") + } + }() + sem.release() + }() +} + +func TestSemaphoreUpdateCapacity(t *testing.T) { + const initialCapacity = 1 + sem := newSemaphore(3, initialCapacity) + if got, want := sem.Capacity(), 1; got != want { + t.Errorf("Capacity = %d, want: %d", got, want) + } + sem.acquire(context.Background()) + sem.updateCapacity(initialCapacity + 2) + if got, want := sem.Capacity(), 3; got != want { + t.Errorf("Capacity = %d, want: %d", got, want) + } +} + +func TestPackUnpack(t *testing.T) { + wantL := uint64(256) + wantR := uint64(513) + + gotL, gotR := unpack(pack(wantL, wantR)) + + if gotL != wantL || gotR != wantR { + t.Fatalf("Got %d, %d want %d, %d", gotL, gotR, wantL, wantR) + } +} + +func tryAcquire(sem *semaphore, gotChan chan struct{}) { + go func() { + // blocking until someone puts the token into the semaphore + sem.acquire(context.Background()) + gotChan <- struct{}{} + }() +} + +// requestor is a set of test helpers around breaker testing. +type requestor struct { + breaker *Breaker + acceptedCh chan bool + barrierCh chan struct{} +} + +func newRequestor(breaker *Breaker) *requestor { + return &requestor{ + breaker: breaker, + acceptedCh: make(chan bool), + barrierCh: make(chan struct{}), + } +} + +// request is the same as requestWithContext but with a default context. +func (r *requestor) request() { + r.requestWithContext(context.Background()) +} + +// requestWithContext simulates a request in a separate goroutine. The +// request will either fail immediately (as observable via expectFailure) +// or block until processSuccessfully is called. +func (r *requestor) requestWithContext(ctx context.Context) { + go func() { + err := r.breaker.Maybe(ctx, func() { + <-r.barrierCh + }) + r.acceptedCh <- err == nil + }() +} + +// expectFailure waits for a request to finish and asserts it to be failed. +func (r *requestor) expectFailure(t *testing.T) { + t.Helper() + if <-r.acceptedCh { + t.Error("expected request to fail but it succeeded") + } +} + +// processSuccessfully allows a request to pass the barrier, waits for it to +// be finished and asserts it to succeed. +func (r *requestor) processSuccessfully(t *testing.T) { + t.Helper() + r.barrierCh <- struct{}{} + if !<-r.acceptedCh { + t.Error("expected request to succeed but it failed") + } +} + +func BenchmarkBreakerMaybe(b *testing.B) { + op := func() {} + + for _, c := range []int{1, 10, 100, 1000} { + breaker := NewBreaker(BreakerParams{QueueDepth: 10000000, MaxConcurrency: c, InitialCapacity: c}) + + b.Run(fmt.Sprintf("%d-sequential", c), func(b *testing.B) { + for j := 0; j < b.N; j++ { + breaker.Maybe(context.Background(), op) + } + }) + + b.Run(fmt.Sprintf("%d-parallel", c), func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + breaker.Maybe(context.Background(), op) + } + }) + }) + } +} + +func BenchmarkBreakerReserve(b *testing.B) { + op := func() {} + breaker := NewBreaker(BreakerParams{QueueDepth: 1, MaxConcurrency: 10000000, InitialCapacity: 10000000}) + + b.Run("sequential", func(b *testing.B) { + for j := 0; j < b.N; j++ { + free, got := breaker.Reserve(context.Background()) + op() + if got { + free() + } + } + }) + + b.Run("parallel", func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + free, got := breaker.Reserve(context.Background()) + op() + if got { + free() + } + } + }) + }) +} diff --git a/pkg/proxy/consts.go b/pkg/proxy/consts.go new file mode 100644 index 0000000000..95c415378c --- /dev/null +++ b/pkg/proxy/consts.go @@ -0,0 +1,30 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxy + +const ( + _userAgentKey = "User-Agent" + + // Since K8s 1.8, prober requests have + // User-Agent = "kube-probe/{major-version}.{minor-version}". + _kubeProbeUserAgentPrefix = "kube-probe/" + + // KubeletProbeHeaderName is the header name to augment the probes, because + // Istio with mTLS rewrites probes, but their probes pass a different + // user-agent. + _kubeletProbeHeaderName = "K-Kubelet-Probe" +) diff --git a/pkg/proxy/handler.go b/pkg/proxy/handler.go new file mode 100644 index 0000000000..ed59616f99 --- /dev/null +++ b/pkg/proxy/handler.go @@ -0,0 +1,48 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxy + +import ( + "context" + "errors" + "net/http" + "strings" +) + +func Handler(breaker *Breaker, next http.Handler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if isKubeletProbe(r) || breaker == nil { + next.ServeHTTP(w, r) + return + } + + if err := breaker.Maybe(r.Context(), func() { + next.ServeHTTP(w, r) + }); err != nil { + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, ErrRequestQueueFull) { + http.Error(w, err.Error(), http.StatusServiceUnavailable) + } else { + w.WriteHeader(http.StatusInternalServerError) + } + } + } +} + +func isKubeletProbe(r *http.Request) bool { + return strings.HasPrefix(r.Header.Get(_userAgentKey), _kubeProbeUserAgentPrefix) || + r.Header.Get(_kubeletProbeHeaderName) != "" +} diff --git a/pkg/proxy/handler_test.go b/pkg/proxy/handler_test.go new file mode 100644 index 0000000000..f5be7a5357 --- /dev/null +++ b/pkg/proxy/handler_test.go @@ -0,0 +1,108 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxy_test + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/cortexlabs/cortex/pkg/proxy" + "github.com/stretchr/testify/require" +) + +const ( + userContainerHost = "http://user-container.cortex.dev" +) + +func TestProxyHandlerQueueFull(t *testing.T) { + // This test sends three requests of which one should fail immediately as the queue + // saturates. + resp := make(chan struct{}) + blockHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-resp + }) + + breaker := proxy.NewBreaker( + proxy.BreakerParams{ + QueueDepth: 1, + MaxConcurrency: 1, + InitialCapacity: 1, + }, + ) + + h := proxy.Handler(breaker, blockHandler) + + req := httptest.NewRequest(http.MethodGet, userContainerHost, nil) + resps := make(chan *httptest.ResponseRecorder) + for i := 0; i < 3; i++ { + go func() { + rec := httptest.NewRecorder() + h(rec, req) + resps <- rec + }() + } + + // One of the three requests fails and it should be the first we see since the others + // are still held by the resp channel. + failure := <-resps + require.Equal(t, http.StatusServiceUnavailable, failure.Code) + require.True(t, strings.Contains(failure.Body.String(), "pending request queue full")) + + // Allow the remaining requests to pass. + close(resp) + for i := 0; i < 2; i++ { + res := <-resps + require.Equal(t, http.StatusOK, res.Code) + } +} + +func TestProxyHandlerBreakerTimeout(t *testing.T) { + // This test sends a request which will take a long time to complete. + // Then another one with a very short context timeout. + // Verifies that the second one fails with timeout. + seen := make(chan struct{}) + resp := make(chan struct{}) + defer close(resp) // Allow all requests to pass through. + blockHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen <- struct{}{} + <-resp + }) + breaker := proxy.NewBreaker(proxy.BreakerParams{ + QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 1, + }) + h := proxy.Handler(breaker, blockHandler) + + go func() { + h(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, userContainerHost, nil)) + }() + + // Wait until the first request has entered the handler. + <-seen + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + + rec := httptest.NewRecorder() + h(rec, httptest.NewRequest(http.MethodGet, userContainerHost, nil).WithContext(ctx)) + + require.Equal(t, http.StatusServiceUnavailable, rec.Code) + require.True(t, strings.Contains(rec.Body.String(), context.DeadlineExceeded.Error())) +} diff --git a/pkg/proxy/proxy.go b/pkg/proxy/proxy.go new file mode 100644 index 0000000000..8ec5db589a --- /dev/null +++ b/pkg/proxy/proxy.go @@ -0,0 +1,46 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxy + +import ( + "net/http" + "net/http/httputil" + "net/url" +) + +// NewReverseProxy creates a new cortex base reverse proxy +func NewReverseProxy(target string, maxIdle, maxIdlePerHost int) *httputil.ReverseProxy { + targetURL, err := url.Parse(target) + if err != nil { + panic(err) + } + + httpProxy := httputil.NewSingleHostReverseProxy(targetURL) + httpProxy.Transport = buildHTTPTransport(maxIdle, maxIdlePerHost) + + return httpProxy +} + +func buildHTTPTransport(maxIdle, maxIdlePerHost int) http.RoundTripper { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.DisableKeepAlives = false + transport.MaxIdleConns = maxIdle + transport.MaxIdleConnsPerHost = maxIdlePerHost + transport.ForceAttemptHTTP2 = false + transport.DisableCompression = true + return transport +} diff --git a/pkg/proxy/proxy_test.go b/pkg/proxy/proxy_test.go new file mode 100644 index 0000000000..a8a9582849 --- /dev/null +++ b/pkg/proxy/proxy_test.go @@ -0,0 +1,42 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxy_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/cortexlabs/cortex/pkg/proxy" + "github.com/stretchr/testify/require" +) + +func TestNewReverseProxy(t *testing.T) { + var isHandlerCalled bool + var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { + isHandlerCalled = true + } + + server := httptest.NewServer(handler) + httpProxy := proxy.NewReverseProxy(server.URL, 1000, 1000) + + resp := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "http://user-container.cortex.dev", nil) + httpProxy.ServeHTTP(resp, req) + + require.True(t, isHandlerCalled) +} diff --git a/pkg/proxy/request_stats.go b/pkg/proxy/request_stats.go new file mode 100644 index 0000000000..5e215a06bf --- /dev/null +++ b/pkg/proxy/request_stats.go @@ -0,0 +1,90 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxy + +import ( + "net/http" + "sync" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +type RequestStats struct { + sync.Mutex + counts []int64 +} + +func (s *RequestStats) Append(val int64) { + s.Lock() + defer s.Unlock() + s.counts = append(s.counts, val) +} + +func (s *RequestStats) GetAllAndDelete() []int64 { + var output []int64 + s.Lock() + defer s.Unlock() + output = s.counts + s.counts = []int64{} + return output +} + +func (s *RequestStats) Report() RequestStatsReport { + requestCounts := s.GetAllAndDelete() + + total := 0.0 + if len(requestCounts) > 0 { + for _, val := range requestCounts { + total += float64(val) + } + + total /= float64(len(requestCounts)) + } + + return RequestStatsReport{AvgInFlight: total} +} + +type RequestStatsReport struct { + AvgInFlight float64 +} + +type PrometheusStatsReporter struct { + handler http.Handler + inFlightRequests prometheus.Gauge +} + +func NewPrometheusStatsReporter() *PrometheusStatsReporter { + inFlightRequestsGauge := promauto.NewGauge(prometheus.GaugeOpts{ + Name: "cortex_in_flight_requests", + Help: "The number of in-flight requests for a cortex API", + }) + + return &PrometheusStatsReporter{ + handler: promhttp.Handler(), + inFlightRequests: inFlightRequestsGauge, + } +} + +func (r *PrometheusStatsReporter) Report(stats RequestStatsReport) { + r.inFlightRequests.Set(stats.AvgInFlight) +} + +func (r *PrometheusStatsReporter) ServeHTTP(w http.ResponseWriter, req *http.Request) { + r.handler.ServeHTTP(w, req) +} diff --git a/pkg/workloads/k8s.go b/pkg/workloads/k8s.go index 2b5a7cc5b6..b0f3d6fcf4 100644 --- a/pkg/workloads/k8s.go +++ b/pkg/workloads/k8s.go @@ -33,8 +33,9 @@ import ( ) const ( - DefaultPortInt32, DefaultPortStr = int32(8888), "8888" - ServiceAccountName = "default" + DefaultPortInt32 = int32(8888) + DefaultPortStr = "8888" + ServiceAccountName = "default" ) const ( diff --git a/test/apis/task/main.py b/test/apis/task/main.py index e51a020b8b..dcac680486 100644 --- a/test/apis/task/main.py +++ b/test/apis/task/main.py @@ -1,5 +1,6 @@ import json, re, os, boto3 + def main(): with open("/mnt/job_spec.json", "r") as f: job_spec = json.load(f) @@ -15,5 +16,6 @@ def main(): s3 = boto3.client("s3") s3.put_object(Bucket=bucket, Key=os.path.join(key, job_id), Body="") + if __name__ == "__main__": main() From c501d53a64bdd8759d5562e1708e21c7af31ab64 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Mon, 17 May 2021 19:46:39 +0100 Subject: [PATCH 18/82] Replace request-monitor with cortex proxy (#2174) --- build/images.sh | 2 +- docs/clusters/management/create.md | 2 +- pkg/types/clusterconfig/cluster_config.go | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/build/images.sh b/build/images.sh index 6e6e069f2d..96a7d75088 100644 --- a/build/images.sh +++ b/build/images.sh @@ -29,7 +29,7 @@ api_images=( dev_images=( "downloader" "manager" - "request-monitor" + "proxy" "async-gateway" "enqueuer" ) diff --git a/docs/clusters/management/create.md b/docs/clusters/management/create.md index 53a3bd6a18..5a79091710 100644 --- a/docs/clusters/management/create.md +++ b/docs/clusters/management/create.md @@ -100,7 +100,7 @@ image_operator: quay.io/cortexlabs/operator:master image_controller_manager: quay.io/cortexlabs/controller-manager:master image_manager: quay.io/cortexlabs/manager:master image_downloader: quay.io/cortexlabs/downloader:master -image_request_monitor: quay.io/cortexlabs/request-monitor:master +image_proxy: quay.io/cortexlabs/proxy:master image_async_gateway: quay.io/cortexlabs/async-gateway:master image_cluster_autoscaler: quay.io/cortexlabs/cluster-autoscaler:master image_metrics_server: quay.io/cortexlabs/metrics-server:master diff --git a/pkg/types/clusterconfig/cluster_config.go b/pkg/types/clusterconfig/cluster_config.go index 0bac1e602d..ce36dd5eef 100644 --- a/pkg/types/clusterconfig/cluster_config.go +++ b/pkg/types/clusterconfig/cluster_config.go @@ -94,7 +94,7 @@ type CoreConfig struct { ImageManager string `json:"image_manager" yaml:"image_manager"` ImageDownloader string `json:"image_downloader" yaml:"image_downloader"` ImageKubexit string `json:"image_kubexit" yaml:"image_kubexit"` - ImageRequestMonitor string `json:"image_request_monitor" yaml:"image_request_monitor"` + ImageProxy string `json:"image_proxy" yaml:"image_proxy"` ImageAsyncGateway string `json:"image_async_gateway" yaml:"image_async_gateway"` ImageEnqueuer string `json:"image_enqueuer" yaml:"image_enqueuer"` ImageClusterAutoscaler string `json:"image_cluster_autoscaler" yaml:"image_cluster_autoscaler"` @@ -354,9 +354,9 @@ var CoreConfigStructFieldValidations = []*cr.StructFieldValidation{ }, }, { - StructField: "ImageRequestMonitor", + StructField: "ImageProxy", StringValidation: &cr.StringValidation{ - Default: consts.DefaultRegistry() + "/request-monitor:" + consts.CortexVersion, + Default: consts.DefaultRegistry() + "/proxy:" + consts.CortexVersion, Validator: validateImageVersion, }, }, @@ -1363,8 +1363,8 @@ func (cc *CoreConfig) TelemetryEvent() map[string]interface{} { if !strings.HasPrefix(cc.ImageKubexit, "cortexlabs/") { event["image_kubexit._is_custom"] = true } - if !strings.HasPrefix(cc.ImageRequestMonitor, "cortexlabs/") { - event["image_request_monitor._is_custom"] = true + if !strings.HasPrefix(cc.ImageProxy, "cortexlabs/") { + event["image_proxy._is_custom"] = true } if !strings.HasPrefix(cc.ImageAsyncGateway, "cortexlabs/") { event["image_async_gateway._is_custom"] = true From 2ef23530625dcdd3130eb1f05073722ea5749e8c Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Mon, 17 May 2021 22:53:14 +0300 Subject: [PATCH 19/82] Fix make tests command --- pkg/crds/controllers/batch/batchjob_controller_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/crds/controllers/batch/batchjob_controller_test.go b/pkg/crds/controllers/batch/batchjob_controller_test.go index 0fdf31a488..112067ea3a 100644 --- a/pkg/crds/controllers/batch/batchjob_controller_test.go +++ b/pkg/crds/controllers/batch/batchjob_controller_test.go @@ -49,6 +49,7 @@ func uploadTestAPISpec(apiName string, apiID string) error { Name: "api", Image: "quay.io/cortexlabs/batch-container-test:master", Command: []string{"/bin/run"}, + Compute: &userconfig.Compute{}, }, }, }, From 497648197adec34641d0d4f6175ab23893721f88 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Wed, 19 May 2021 19:16:18 +0300 Subject: [PATCH 20/82] Operator changes for CaaS implementation (#2177) --- cmd/proxy/main.go | 80 ++++++- dev/registry.sh | 6 +- .../manifests/prometheus-monitoring.yaml.j2 | 10 +- pkg/consts/consts.go | 19 +- .../batch/batchjob_controller_test.go | 3 +- pkg/lib/strings/operations.go | 4 + pkg/operator/lib/autoscaler/autoscaler.go | 4 +- pkg/operator/resources/asyncapi/k8s_specs.go | 7 +- .../resources/job/batchapi/k8s_specs.go | 3 +- .../resources/job/taskapi/k8s_specs.go | 3 +- pkg/operator/resources/realtimeapi/api.go | 8 +- .../resources/realtimeapi/k8s_specs.go | 12 +- pkg/operator/resources/trafficsplitter/api.go | 2 +- pkg/types/spec/errors.go | 62 ++--- pkg/types/spec/validations.go | 48 +++- pkg/types/userconfig/api.go | 54 +++-- pkg/types/userconfig/config_key.go | 13 +- pkg/workloads/helpers.go | 92 ++++++-- pkg/workloads/init.go | 16 +- pkg/workloads/k8s.go | 216 ++++++++---------- test/apis/realtime/Dockerfile | 3 +- test/apis/realtime/cortex.yaml | 7 +- test/apis/realtime/main.py | 10 +- test/apis/task/main.py | 2 +- .../image-classifier-resnet50/cortex_inf.yaml | 2 +- .../cortex_inf_server_side_batching.yaml | 2 +- 26 files changed, 415 insertions(+), 273 deletions(-) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 0b0beea153..d7e4f0fa05 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -25,8 +25,13 @@ import ( "strconv" "time" + "github.com/cortexlabs/cortex/pkg/lib/aws" + "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/logging" + "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/proxy" + "github.com/cortexlabs/cortex/pkg/types/clusterconfig" + "github.com/cortexlabs/cortex/pkg/types/userconfig" "go.uber.org/zap" ) @@ -35,6 +40,28 @@ const ( _requestSampleInterval = 1 * time.Second ) +var ( + proxyLogger = logging.GetLogger() +) + +func Exit(err error, wrapStrs ...string) { + for _, str := range wrapStrs { + err = errors.Wrap(err, str) + } + + if err != nil && !errors.IsNoTelemetry(err) { + telemetry.Error(err) + } + + if err != nil && !errors.IsNoPrint(err) { + proxyLogger.Error(err) + } + + telemetry.Close() + + os.Exit(1) +} + func main() { var ( port int @@ -42,13 +69,16 @@ func main() { userContainerPort int maxConcurrency int maxQueueLength int + clusterConfigPath string ) - flag.IntVar(&port, "port", 8000, "port where the proxy will be served") - flag.IntVar(&metricsPort, "metrics-port", 8001, "port where the proxy will be served") - flag.IntVar(&userContainerPort, "user-port", 8080, "port where the proxy will redirect to the traffic to") + flag.IntVar(&port, "port", 8888, "port where the proxy is served") + flag.IntVar(&metricsPort, "metrics-port", 15000, "metrics port for prometheus") + flag.IntVar(&userContainerPort, "user-port", 8080, "port where the proxy redirects to the traffic to") flag.IntVar(&maxConcurrency, "max-concurrency", 0, "max concurrency allowed for user container") flag.IntVar(&maxQueueLength, "max-queue-length", 0, "max request queue length for user container") + flag.StringVar(&clusterConfigPath, "cluster-config", "", "cluster config path") + flag.Parse() log := logging.GetLogger() @@ -58,12 +88,44 @@ func main() { switch { case maxConcurrency == 0: - log.Fatal("--max-concurrency flag is required") + log.Fatal("-max-concurrency flag is required") case maxQueueLength == 0: - maxQueueLength = maxConcurrency * 10 + log.Fatal("-max-queue-length flag is required") + case clusterConfigPath == "": + log.Fatal("-cluster-config flag is required") + } + + clusterConfig, err := clusterconfig.NewForFile(clusterConfigPath) + if err != nil { + Exit(err) + } + + awsClient, err := aws.NewForRegion(clusterConfig.Region) + if err != nil { + Exit(err) + } + + _, userID, err := awsClient.CheckCredentials() + if err != nil { + Exit(err) + } + + err = telemetry.Init(telemetry.Config{ + Enabled: clusterConfig.Telemetry, + UserID: userID, + Properties: map[string]string{ + "kind": userconfig.RealtimeAPIKind.String(), + "image_type": "proxy", + }, + Environment: "api", + LogErrors: true, + BackoffMode: telemetry.BackoffDuplicateMessages, + }) + if err != nil { + Exit(err) } - target := "http://127.0.0.1:" + strconv.Itoa(port) + target := "http://127.0.0.1:" + strconv.Itoa(userContainerPort) httpProxy := proxy.NewReverseProxy(target, maxQueueLength, maxQueueLength) requestCounterStats := &proxy.RequestStats{} @@ -101,7 +163,7 @@ func main() { servers := map[string]*http.Server{ "proxy": { - Addr: ":" + strconv.Itoa(userContainerPort), + Addr: ":" + strconv.Itoa(port), Handler: proxy.Handler(breaker, httpProxy), }, "metrics": { @@ -123,7 +185,7 @@ func main() { select { case err := <-errCh: - log.Fatal("failed to start proxy server", zap.Error(err)) + Exit(errors.Wrap(err, "failed to start proxy server")) case <-sigint: // We received an interrupt signal, shut down. log.Info("Received TERM signal, handling a graceful shutdown...") @@ -133,8 +195,10 @@ func main() { if err := server.Shutdown(context.Background()); err != nil { // Error from closing listeners, or context timeout: log.Warn("HTTP server Shutdown Error", zap.Error(err)) + telemetry.Error(errors.Wrap(err, "HTTP server Shutdown Error")) } } log.Info("Shutdown complete, exiting...") + telemetry.Close() } } diff --git a/dev/registry.sh b/dev/registry.sh index eda792d95c..3b05e85bb3 100755 --- a/dev/registry.sh +++ b/dev/registry.sh @@ -222,7 +222,7 @@ elif [ "$cmd" = "create" ]; then # usage: registry.sh update-single IMAGE elif [ "$cmd" = "update-single" ]; then image=$sub_cmd - if [ "$image" = "operator" ] || [ "$image" = "request-monitor" ]; then + if [ "$image" = "operator" ] || [ "$image" = "proxy" ]; then cache_builder $image fi build_and_push $image @@ -245,8 +245,8 @@ elif [ "$cmd" = "update" ]; then if [[ " ${images_to_build[@]} " =~ " operator " ]]; then cache_builder operator fi - if [[ " ${images_to_build[@]} " =~ " request-monitor " ]]; then - cache_builder request-monitor + if [[ " ${images_to_build[@]} " =~ " proxy " ]]; then + cache_builder proxy fi if [[ " ${images_to_build[@]} " =~ " async-gateway " ]]; then cache_builder async-gateway diff --git a/manager/manifests/prometheus-monitoring.yaml.j2 b/manager/manifests/prometheus-monitoring.yaml.j2 index a56f7664e3..cee1eae033 100644 --- a/manager/manifests/prometheus-monitoring.yaml.j2 +++ b/manager/manifests/prometheus-monitoring.yaml.j2 @@ -34,7 +34,7 @@ spec: matchExpressions: - key: "monitoring.cortex.dev" operator: "In" - values: [ "istio", "request-monitor", "statsd-exporter", "dcgm-exporter", "kube-state-metrics" ] + values: [ "istio", "proxy", "statsd-exporter", "dcgm-exporter", "kube-state-metrics" ] serviceMonitorSelector: matchExpressions: - key: "monitoring.cortex.dev" @@ -168,9 +168,9 @@ spec: apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: - name: request-monitor-stats + name: proxy-stats labels: - monitoring.cortex.dev: "request-monitor" + monitoring.cortex.dev: "proxy" spec: selector: matchLabels: @@ -179,7 +179,7 @@ spec: - { key: prometheus-ignore, operator: DoesNotExist } namespaceSelector: any: true - jobLabel: request-monitor-stats + jobLabel: proxy-stats podMetricsEndpoints: - path: /metrics scheme: http @@ -188,7 +188,7 @@ spec: relabelings: - action: keep sourceLabels: [ __meta_kubernetes_pod_container_name ] - regex: "request-monitor" + regex: "proxy" - sourceLabels: [ __meta_kubernetes_pod_label_apiName ] action: replace targetLabel: api_name diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 0c6ee0d22a..50fca29b6c 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -24,12 +24,17 @@ var ( CortexVersion = "master" // CORTEX_VERSION CortexVersionMinor = "master" // CORTEX_VERSION_MINOR - ProxyListeningPort = int64(8888) - ProxyListeningPortStr = "8888" - DefaultMaxReplicaQueueLength = int64(1024) - DefaultMaxReplicaConcurrency = int64(1024) - DefaultTargetReplicaConcurrency = float64(8) - NeuronCoresPerInf = int64(4) + DefaultMaxQueueLength = int64(1024) + DefaultMaxConcurrency = int64(16) + + DefaultUserPodPortStr = "8080" + DefaultUserPodPortInt32 = int32(8080) + + ProxyListeningPortStr = "8888" + ProxyListeningPortInt32 = int32(8888) + + MetricsPortStr = "15000" + MetricsPortInt32 = int32(15000) AuthHeader = "X-Cortex-Authorization" @@ -38,7 +43,7 @@ var ( AsyncWorkloadsExpirationDays = int64(7) ReservedContainerNames = []string{ - "neuron-rtd", + "proxy", } ) diff --git a/pkg/crds/controllers/batch/batchjob_controller_test.go b/pkg/crds/controllers/batch/batchjob_controller_test.go index 112067ea3a..005691cb3e 100644 --- a/pkg/crds/controllers/batch/batchjob_controller_test.go +++ b/pkg/crds/controllers/batch/batchjob_controller_test.go @@ -22,6 +22,7 @@ import ( "time" batch "github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1" + "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/random" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/status" @@ -43,7 +44,7 @@ func uploadTestAPISpec(apiName string, apiID string) error { Kind: userconfig.BatchAPIKind, }, Pod: &userconfig.Pod{ - // TODO use a real image + Port: pointer.Int32(8080), Containers: []*userconfig.Container{ { Name: "api", diff --git a/pkg/lib/strings/operations.go b/pkg/lib/strings/operations.go index aedb528c99..f97cc42558 100644 --- a/pkg/lib/strings/operations.go +++ b/pkg/lib/strings/operations.go @@ -220,6 +220,10 @@ func PluralEs(str string, count interface{}) string { return PluralCustom(str, str+"es", count) } +func PluralIs(count interface{}) string { + return PluralCustom("is", "are", count) +} + func PluralCustom(singular string, plural string, count interface{}) string { countInt, _ := cast.InterfaceToInt64(count) if countInt == 1 { diff --git a/pkg/operator/lib/autoscaler/autoscaler.go b/pkg/operator/lib/autoscaler/autoscaler.go index 0fc7c7197d..e3d13cd920 100644 --- a/pkg/operator/lib/autoscaler/autoscaler.go +++ b/pkg/operator/lib/autoscaler/autoscaler.go @@ -133,7 +133,7 @@ func AutoscaleFn(initialDeployment *kapps.Deployment, apiSpec *spec.API, getInFl return nil } - rawRecommendation := *avgInFlight / autoscalingSpec.TargetReplicaConcurrency + rawRecommendation := *avgInFlight / *autoscalingSpec.TargetInFlight recommendation := int32(math.Ceil(rawRecommendation)) if rawRecommendation < float64(currentReplicas) && rawRecommendation > float64(currentReplicas)*(1-autoscalingSpec.DownscaleTolerance) { @@ -199,7 +199,7 @@ func AutoscaleFn(initialDeployment *kapps.Deployment, apiSpec *spec.API, getInFl apiLogger.Debugw(fmt.Sprintf("%s autoscaler tick", apiName), "autoscaling", map[string]interface{}{ "avg_in_flight": *avgInFlight, - "target_replica_concurrency": autoscalingSpec.TargetReplicaConcurrency, + "target_in_flight": *autoscalingSpec.TargetInFlight, "raw_recommendation": rawRecommendation, "current_replicas": currentReplicas, "downscale_tolerance": autoscalingSpec.DownscaleTolerance, diff --git a/pkg/operator/resources/asyncapi/k8s_specs.go b/pkg/operator/resources/asyncapi/k8s_specs.go index 50d62c61ed..fa27daf885 100644 --- a/pkg/operator/resources/asyncapi/k8s_specs.go +++ b/pkg/operator/resources/asyncapi/k8s_specs.go @@ -17,6 +17,7 @@ limitations under the License. package asyncapi import ( + "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/types/spec" @@ -128,8 +129,8 @@ func gatewayServiceSpec(api spec.API) kcore.Service { return *k8s.Service(&k8s.ServiceSpec{ Name: workloads.K8sName(api.Name), PortName: "http", - Port: workloads.DefaultPortInt32, - TargetPort: workloads.DefaultPortInt32, + Port: consts.ProxyListeningPortInt32, + TargetPort: consts.ProxyListeningPortInt32, Annotations: api.ToK8sAnnotations(), Labels: map[string]string{ "apiName": api.Name, @@ -152,7 +153,7 @@ func gatewayVirtualServiceSpec(api spec.API) v1beta1.VirtualService { Destinations: []k8s.Destination{{ ServiceName: workloads.K8sName(api.Name), Weight: 100, - Port: uint32(workloads.DefaultPortInt32), + Port: uint32(consts.ProxyListeningPortInt32), }}, PrefixPath: api.Networking.Endpoint, Rewrite: pointer.String("/"), diff --git a/pkg/operator/resources/job/batchapi/k8s_specs.go b/pkg/operator/resources/job/batchapi/k8s_specs.go index 7499e891db..5ae8b5aff8 100644 --- a/pkg/operator/resources/job/batchapi/k8s_specs.go +++ b/pkg/operator/resources/job/batchapi/k8s_specs.go @@ -21,6 +21,7 @@ import ( "path" "github.com/cortexlabs/cortex/pkg/config" + "github.com/cortexlabs/cortex/pkg/consts" batch "github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/parallel" @@ -40,7 +41,7 @@ func virtualServiceSpec(api *spec.API) *istioclientnetworking.VirtualService { Destinations: []k8s.Destination{{ ServiceName: _operatorService, Weight: 100, - Port: uint32(workloads.DefaultPortInt32), + Port: uint32(consts.ProxyListeningPortInt32), }}, PrefixPath: api.Networking.Endpoint, Rewrite: pointer.String(path.Join("batch", api.Name)), diff --git a/pkg/operator/resources/job/taskapi/k8s_specs.go b/pkg/operator/resources/job/taskapi/k8s_specs.go index a3c58f9795..8ccd530c85 100644 --- a/pkg/operator/resources/job/taskapi/k8s_specs.go +++ b/pkg/operator/resources/job/taskapi/k8s_specs.go @@ -20,6 +20,7 @@ import ( "path" "github.com/cortexlabs/cortex/pkg/config" + "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/parallel" "github.com/cortexlabs/cortex/pkg/lib/pointer" @@ -42,7 +43,7 @@ func virtualServiceSpec(api *spec.API) *istioclientnetworking.VirtualService { Destinations: []k8s.Destination{{ ServiceName: _operatorService, Weight: 100, - Port: uint32(workloads.DefaultPortInt32), + Port: uint32(consts.ProxyListeningPortInt32), }}, PrefixPath: api.Networking.Endpoint, Rewrite: pointer.String(path.Join("tasks", api.Name)), diff --git a/pkg/operator/resources/realtimeapi/api.go b/pkg/operator/resources/realtimeapi/api.go index 7f01aa957c..0cc58c8dd1 100644 --- a/pkg/operator/resources/realtimeapi/api.go +++ b/pkg/operator/resources/realtimeapi/api.go @@ -116,7 +116,11 @@ func UpdateAPI(apiConfig *userconfig.API, projectID string, force bool) (*spec.A } func RefreshAPI(apiName string, force bool) (string, error) { - prevDeployment, err := config.K8s.GetDeployment(workloads.K8sName(apiName)) + prevDeployment, prevService, prevVirtualService, err := getK8sResources(&userconfig.API{ + Resource: userconfig.Resource{ + Name: apiName, + }, + }) if err != nil { return "", err } else if prevDeployment == nil { @@ -153,7 +157,7 @@ func RefreshAPI(apiName string, force bool) (string, error) { return "", errors.Wrap(err, "upload handler spec") } - if err := applyK8sDeployment(api, prevDeployment); err != nil { + if err := applyK8sResources(api, prevDeployment, prevService, prevVirtualService); err != nil { return "", err } diff --git a/pkg/operator/resources/realtimeapi/k8s_specs.go b/pkg/operator/resources/realtimeapi/k8s_specs.go index 831f43ae32..49368f7db2 100644 --- a/pkg/operator/resources/realtimeapi/k8s_specs.go +++ b/pkg/operator/resources/realtimeapi/k8s_specs.go @@ -17,6 +17,7 @@ limitations under the License. package realtimeapi import ( + "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/types/spec" @@ -30,7 +31,10 @@ var _terminationGracePeriodSeconds int64 = 60 // seconds func deploymentSpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Deployment { containers, volumes := workloads.UserPodContainers(*api) - // TODO add the proxy as well + proxyContainer, proxyVolume := workloads.RealtimeProxyContainer(*api) + + containers = append(containers, proxyContainer) + volumes = append(volumes, proxyVolume) return k8s.Deployment(&k8s.DeploymentSpec{ Name: workloads.K8sName(api.Name), @@ -80,8 +84,8 @@ func serviceSpec(api *spec.API) *kcore.Service { return k8s.Service(&k8s.ServiceSpec{ Name: workloads.K8sName(api.Name), PortName: "http", - Port: workloads.DefaultPortInt32, - TargetPort: workloads.DefaultPortInt32, + Port: consts.ProxyListeningPortInt32, + TargetPort: consts.ProxyListeningPortInt32, Annotations: api.ToK8sAnnotations(), Labels: map[string]string{ "apiName": api.Name, @@ -102,7 +106,7 @@ func virtualServiceSpec(api *spec.API) *istioclientnetworking.VirtualService { Destinations: []k8s.Destination{{ ServiceName: workloads.K8sName(api.Name), Weight: 100, - Port: uint32(workloads.DefaultPortInt32), + Port: uint32(consts.ProxyListeningPortInt32), }}, PrefixPath: api.Networking.Endpoint, Rewrite: pointer.String("/"), diff --git a/pkg/operator/resources/trafficsplitter/api.go b/pkg/operator/resources/trafficsplitter/api.go index 1ce3844b0d..e2b908d3fd 100644 --- a/pkg/operator/resources/trafficsplitter/api.go +++ b/pkg/operator/resources/trafficsplitter/api.go @@ -113,7 +113,7 @@ func getTrafficSplitterDestinations(trafficSplitter *spec.API) []k8s.Destination destinations[i] = k8s.Destination{ ServiceName: workloads.K8sName(api.Name), Weight: api.Weight, - Port: uint32(consts.ProxyListeningPort), + Port: uint32(consts.ProxyListeningPortInt32), Shadow: api.Shadow, } } diff --git a/pkg/types/spec/errors.go b/pkg/types/spec/errors.go index 9d628c5c2f..3b44ac5fa0 100644 --- a/pkg/types/spec/errors.go +++ b/pkg/types/spec/errors.go @@ -35,9 +35,9 @@ const ( ErrDuplicateEndpointInOneDeploy = "spec.duplicate_endpoint_in_one_deploy" ErrDuplicateEndpoint = "spec.duplicate_endpoint" ErrDuplicateContainerName = "spec.duplicate_container_name" - ErrConflictingFields = "spec.conflicting_fields" + ErrCantSpecifyBoth = "spec.cant_specify_both" ErrSpecifyOnlyOneField = "spec.specify_only_one_field" - ErrSpecifyOneOrTheOther = "spec.specify_one_or_the_other" + ErrNoneSpecified = "spec.none_specified" ErrSpecifyAllOrNone = "spec.specify_all_or_none" ErrOneOfPrerequisitesNotDefined = "spec.one_of_prerequisites_not_defined" ErrConfigGreaterThanOtherConfig = "spec.config_greater_than_other_config" @@ -45,17 +45,17 @@ const ( ErrMinReplicasGreaterThanMax = "spec.min_replicas_greater_than_max" ErrInitReplicasGreaterThanMax = "spec.init_replicas_greater_than_max" ErrInitReplicasLessThanMin = "spec.init_replicas_less_than_min" + ErrTargetInFlightLimitReached = "spec.target_in_flight_limit_reached" ErrInvalidSurgeOrUnavailable = "spec.invalid_surge_or_unavailable" ErrSurgeAndUnavailableBothZero = "spec.surge_and_unavailable_both_zero" ErrShmSizeCannotExceedMem = "spec.shm_size_cannot_exceed_mem" - ErrFieldCannotBeEmptyForKind = "spec.field_cannot_be_empty_for_kind" + ErrFieldMustBeSpecifiedForKind = "spec.field_must_be_specified_for_kind" + ErrFieldIsNotSupportedForKind = "spec.field_is_not_supported_for_kind" ErrCortexPrefixedEnvVarNotAllowed = "spec.cortex_prefixed_env_var_not_allowed" - ErrRegistryInDifferentRegion = "spec.registry_in_different_region" - ErrRegistryAccountIDMismatch = "spec.registry_account_id_mismatch" - ErrKeyIsNotSupportedForKind = "spec.key_is_not_supported_for_kind" + ErrDisallowedEnvVars = "spec.disallowed_env_vars" ErrComputeResourceConflict = "spec.compute_resource_conflict" ErrInvalidNumberOfInfs = "spec.invalid_number_of_infs" ErrIncorrectTrafficSplitterWeight = "spec.incorrect_traffic_splitter_weight" @@ -116,10 +116,10 @@ func ErrorDuplicateContainerName(containerName string) error { }) } -func ErrorConflictingFields(fieldKeyA, fieldKeyB string) error { +func ErrorCantSpecifyBoth(fieldKeyA, fieldKeyB string) error { return errors.WithStack(&errors.Error{ - Kind: ErrConflictingFields, - Message: fmt.Sprintf("please specify either the %s or %s field (both cannot be specified at the same time)", fieldKeyA, fieldKeyB), + Kind: ErrCantSpecifyBoth, + Message: fmt.Sprintf("please specify either %s or %s (both cannot be specified at the same time)", fieldKeyA, fieldKeyB), }) } @@ -130,10 +130,10 @@ func ErrorSpecifyOnlyOneField(fields ...string) error { }) } -func ErrorSpecifyOneOrTheOther(fieldKeyA, fieldKeyB string) error { +func ErrorNoneSpecified(fieldKeyA, fieldKeyB string) error { return errors.WithStack(&errors.Error{ - Kind: ErrSpecifyOneOrTheOther, - Message: fmt.Sprintf("please specify either the %s field or %s field (cannot be both empty at the same time)", fieldKeyA, fieldKeyB), + Kind: ErrNoneSpecified, + Message: fmt.Sprintf("please specify either %s or %s (cannot be both empty at the same time)", fieldKeyA, fieldKeyB), }) } @@ -188,6 +188,13 @@ func ErrorInitReplicasLessThanMin(init int32, min int32) error { }) } +func ErrorTargetInFlightLimitReached(targetInFlight float64, maxConcurrency, maxQueueLength int64) error { + return errors.WithStack(&errors.Error{ + Kind: ErrTargetInFlightLimitReached, + Message: fmt.Sprintf("%s cannot be greater than %s + %s (%f > %d + %d)", userconfig.TargetInFlightKey, userconfig.MaxConcurrencyKey, userconfig.MaxQueueLengthKey, targetInFlight, maxConcurrency, maxQueueLength), + }) +} + func ErrorInvalidSurgeOrUnavailable(val string) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidSurgeOrUnavailable, @@ -209,38 +216,31 @@ func ErrorShmSizeCannotExceedMem(shmSize k8s.Quantity, mem k8s.Quantity) error { }) } -func ErrorFieldCannotBeEmptyForKind(field string, kind userconfig.Kind) error { - return errors.WithStack(&errors.Error{ - Kind: ErrFieldCannotBeEmptyForKind, - Message: fmt.Sprintf("field %s cannot be empty for %s kind", field, kind.String()), - }) -} - -func ErrorCortexPrefixedEnvVarNotAllowed(prefixes ...string) error { +func ErrorFieldMustBeSpecifiedForKind(field string, kind userconfig.Kind) error { return errors.WithStack(&errors.Error{ - Kind: ErrCortexPrefixedEnvVarNotAllowed, - Message: fmt.Sprintf("environment variables starting with %s are reserved", s.StrsOr(prefixes)), + Kind: ErrFieldMustBeSpecifiedForKind, + Message: fmt.Sprintf("%s must be specified for %s kind", field, kind.String()), }) } -func ErrorRegistryInDifferentRegion(registryRegion string, awsClientRegion string) error { +func ErrorFieldIsNotSupportedForKind(field string, kind userconfig.Kind) error { return errors.WithStack(&errors.Error{ - Kind: ErrRegistryInDifferentRegion, - Message: fmt.Sprintf("registry region (%s) does not match cortex's region (%s); images can only be pulled from repositories in the same region as cortex", registryRegion, awsClientRegion), + Kind: ErrFieldIsNotSupportedForKind, + Message: fmt.Sprintf("%s is not supported for %s kind", field, kind.String()), }) } -func ErrorRegistryAccountIDMismatch(regID, opID string) error { +func ErrorCortexPrefixedEnvVarNotAllowed(prefixes ...string) error { return errors.WithStack(&errors.Error{ - Kind: ErrRegistryAccountIDMismatch, - Message: fmt.Sprintf("registry account ID (%s) doesn't match your AWS account ID (%s), and using an ECR registry in a different AWS account is not supported", regID, opID), + Kind: ErrCortexPrefixedEnvVarNotAllowed, + Message: fmt.Sprintf("environment variables starting with %s are reserved", s.StrsOr(prefixes)), }) } -func ErrorKeyIsNotSupportedForKind(key string, kind userconfig.Kind) error { +func ErrorDisallowedEnvVars(disallowedValues ...string) error { return errors.WithStack(&errors.Error{ - Kind: ErrKeyIsNotSupportedForKind, - Message: fmt.Sprintf("%s key is not supported for %s kind", key, kind.String()), + Kind: ErrDisallowedEnvVars, + Message: fmt.Sprintf("environment %s %s %s disallowed", s.PluralS("variables", len(disallowedValues)), s.StrsAnd(disallowedValues), s.PluralIs(len(disallowedValues))), }) } diff --git a/pkg/types/spec/validations.go b/pkg/types/spec/validations.go index fa687c487e..92d8ae00c7 100644 --- a/pkg/types/spec/validations.go +++ b/pkg/types/spec/validations.go @@ -148,6 +148,7 @@ func podValidation() *cr.StructFieldValidation { { StructField: "ShmSize", StringPtrValidation: &cr.StringPtrValidation{ + Required: false, Default: nil, AllowExplicitNull: true, }, @@ -165,6 +166,18 @@ func podValidation() *cr.StructFieldValidation { }, }, }, + { + StructField: "Port", + Int32PtrValidation: &cr.Int32PtrValidation{ + Required: false, + Default: nil, // it's a pointer because it's not required for the task API + AllowExplicitNull: true, + DisallowedValues: []int32{ + consts.ProxyListeningPortInt32, + consts.MetricsPortInt32, + }, + }, + }, containersValidation(), }, }, @@ -318,29 +331,29 @@ func autoscalingValidation() *cr.StructFieldValidation { }, }, { - StructField: "MaxReplicaQueueLength", + StructField: "MaxQueueLength", Int64Validation: &cr.Int64Validation{ - Default: consts.DefaultMaxReplicaQueueLength, + Default: consts.DefaultMaxQueueLength, GreaterThan: pointer.Int64(0), - // our configured nginx can theoretically accept up to 32768 connections, but during testing, + // the proxy can theoretically accept up to 32768 connections, but during testing, // it has been observed that the number is just slightly lower, so it has been offset by 2678 LessThanOrEqualTo: pointer.Int64(30000), }, }, { - StructField: "MaxReplicaConcurrency", + StructField: "MaxConcurrency", Int64Validation: &cr.Int64Validation{ - Default: consts.DefaultMaxReplicaConcurrency, + Default: consts.DefaultMaxConcurrency, GreaterThan: pointer.Int64(0), - // our configured nginx can theoretically accept up to 32768 connections, but during testing, + // the proxy can theoretically accept up to 32768 connections, but during testing, // it has been observed that the number is just slightly lower, so it has been offset by 2678 LessThanOrEqualTo: pointer.Int64(30000), }, }, { - StructField: "TargetReplicaConcurrency", - Float64Validation: &cr.Float64Validation{ - Default: consts.DefaultTargetReplicaConcurrency, + StructField: "TargetInFlight", + Float64PtrValidation: &cr.Float64PtrValidation{ + Default: nil, GreaterThan: pointer.Float64(0), }, }, @@ -555,6 +568,13 @@ func validatePod( } } + if api.Pod.Port != nil && api.Kind == userconfig.TaskAPIKind { + return ErrorFieldIsNotSupportedForKind(userconfig.PortKey, api.Kind) + } + if api.Pod.Port == nil && api.Kind != userconfig.TaskAPIKind { + api.Pod.Port = pointer.Int32(consts.DefaultUserPodPortInt32) + } + if err := validateCompute(totalCompute); err != nil { return errors.Wrap(err, userconfig.ComputeKey) } @@ -581,7 +601,7 @@ func validateContainers( containerNames = append(containerNames, container.Name) if container.Command == nil && (kind == userconfig.BatchAPIKind || kind == userconfig.TaskAPIKind) { - return errors.Wrap(ErrorFieldCannotBeEmptyForKind(userconfig.CommandKey, kind), strconv.FormatInt(int64(i), 10), userconfig.CommandKey) + return errors.Wrap(ErrorFieldMustBeSpecifiedForKind(userconfig.CommandKey, kind), strconv.FormatInt(int64(i), 10), userconfig.CommandKey) } if err := validateDockerImagePath(container.Image, awsClient, k8sClient); err != nil { @@ -601,8 +621,12 @@ func validateContainers( func validateAutoscaling(api *userconfig.API) error { autoscaling := api.Autoscaling - if autoscaling.TargetReplicaConcurrency > float64(autoscaling.MaxReplicaConcurrency) { - return ErrorConfigGreaterThanOtherConfig(userconfig.TargetReplicaConcurrencyKey, autoscaling.TargetReplicaConcurrency, userconfig.MaxReplicaConcurrencyKey, autoscaling.MaxReplicaConcurrency) + if autoscaling.TargetInFlight == nil { + autoscaling.TargetInFlight = pointer.Float64(float64(autoscaling.MaxConcurrency)) + } + + if *autoscaling.TargetInFlight > float64(autoscaling.MaxConcurrency)+float64(autoscaling.MaxQueueLength) { + return ErrorTargetInFlightLimitReached(*autoscaling.TargetInFlight, autoscaling.MaxConcurrency, autoscaling.MaxQueueLength) } if autoscaling.MinReplicas > autoscaling.MaxReplicas { diff --git a/pkg/types/userconfig/api.go b/pkg/types/userconfig/api.go index 156c0c8d03..8516aadfc7 100644 --- a/pkg/types/userconfig/api.go +++ b/pkg/types/userconfig/api.go @@ -22,6 +22,7 @@ import ( "time" "github.com/cortexlabs/cortex/pkg/lib/k8s" + "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/lib/urls" @@ -45,6 +46,7 @@ type API struct { type Pod struct { NodeGroups []string `json:"node_groups" yaml:"node_groups"` ShmSize *k8s.Quantity `json:"shm_size" yaml:"shm_size"` + Port *int32 `json:"port" yaml:"port"` Containers []*Container `json:"containers" yaml:"containers"` } @@ -80,9 +82,9 @@ type Autoscaling struct { MinReplicas int32 `json:"min_replicas" yaml:"min_replicas"` MaxReplicas int32 `json:"max_replicas" yaml:"max_replicas"` InitReplicas int32 `json:"init_replicas" yaml:"init_replicas"` - MaxReplicaQueueLength int64 `json:"max_replica_queue_length" yaml:"max_replica_queue_length"` - MaxReplicaConcurrency int64 `json:"max_replica_concurrency" yaml:"max_replica_concurrency"` - TargetReplicaConcurrency float64 `json:"target_replica_concurrency" yaml:"target_replica_concurrency"` + MaxQueueLength int64 `json:"max_queue_length" yaml:"max_queue_length"` + MaxConcurrency int64 `json:"max_concurrency" yaml:"max_concurrency"` + TargetInFlight *float64 `json:"target_in_flight" yaml:"target_in_flight"` Window time.Duration `json:"window" yaml:"window"` DownscaleStabilizationPeriod time.Duration `json:"downscale_stabilization_period" yaml:"downscale_stabilization_period"` UpscaleStabilizationPeriod time.Duration `json:"upscale_stabilization_period" yaml:"upscale_stabilization_period"` @@ -130,9 +132,9 @@ func (api *API) ToK8sAnnotations() map[string]string { if api.Autoscaling != nil { annotations[MinReplicasAnnotationKey] = s.Int32(api.Autoscaling.MinReplicas) annotations[MaxReplicasAnnotationKey] = s.Int32(api.Autoscaling.MaxReplicas) - annotations[MaxReplicaQueueLengthAnnotationKey] = s.Int64(api.Autoscaling.MaxReplicaQueueLength) - annotations[TargetReplicaConcurrencyAnnotationKey] = s.Float64(api.Autoscaling.TargetReplicaConcurrency) - annotations[MaxReplicaConcurrencyAnnotationKey] = s.Int64(api.Autoscaling.MaxReplicaConcurrency) + annotations[MaxQueueLengthAnnotationKey] = s.Int64(api.Autoscaling.MaxQueueLength) + annotations[TargetInFlightAnnotationKey] = s.Float64(*api.Autoscaling.TargetInFlight) + annotations[MaxConcurrencyAnnotationKey] = s.Int64(api.Autoscaling.MaxConcurrency) annotations[WindowAnnotationKey] = api.Autoscaling.Window.String() annotations[DownscaleStabilizationPeriodAnnotationKey] = api.Autoscaling.DownscaleStabilizationPeriod.String() annotations[UpscaleStabilizationPeriodAnnotationKey] = api.Autoscaling.UpscaleStabilizationPeriod.String() @@ -159,23 +161,23 @@ func AutoscalingFromAnnotations(k8sObj kmeta.Object) (*Autoscaling, error) { } a.MaxReplicas = maxReplicas - maxReplicaQueueLength, err := k8s.ParseInt64Annotation(k8sObj, MaxReplicaQueueLengthAnnotationKey) + maxQueueLength, err := k8s.ParseInt64Annotation(k8sObj, MaxQueueLengthAnnotationKey) if err != nil { return nil, err } - a.MaxReplicaQueueLength = maxReplicaQueueLength + a.MaxQueueLength = maxQueueLength - maxReplicaConcurrency, err := k8s.ParseInt64Annotation(k8sObj, MaxReplicaConcurrencyAnnotationKey) + maxConcurrency, err := k8s.ParseInt64Annotation(k8sObj, MaxConcurrencyAnnotationKey) if err != nil { return nil, err } - a.MaxReplicaConcurrency = maxReplicaConcurrency + a.MaxConcurrency = maxConcurrency - targetReplicaConcurrency, err := k8s.ParseFloat64Annotation(k8sObj, TargetReplicaConcurrencyAnnotationKey) + targetInFlight, err := k8s.ParseFloat64Annotation(k8sObj, TargetInFlightAnnotationKey) if err != nil { return nil, err } - a.TargetReplicaConcurrency = targetReplicaConcurrency + a.TargetInFlight = pointer.Float64(targetInFlight) window, err := k8s.ParseDurationAnnotation(k8sObj, WindowAnnotationKey) if err != nil { @@ -276,6 +278,9 @@ func (pod *Pod) UserStr() string { } else { sb.WriteString(fmt.Sprintf("%s: %s\n", NodeGroupsKey, s.ObjFlatNoQuotes(pod.NodeGroups))) } + if pod.Port != nil { + sb.WriteString(fmt.Sprintf("%s: %d\n", PortKey, *pod.Port)) + } sb.WriteString(fmt.Sprintf("%s:\n", ContainersKey)) for _, container := range pod.Containers { @@ -299,15 +304,11 @@ func (container *Container) UserStr() string { sb.WriteString(s.Indent(string(d), " ")) } - if container.Command == nil { - sb.WriteString(fmt.Sprintf("%s: null\n", CommandKey)) - } else { + if container.Command != nil { sb.WriteString(fmt.Sprintf("%s: %s\n", CommandKey, s.ObjFlatNoQuotes(container.Command))) } - if container.Args == nil { - sb.WriteString(fmt.Sprintf("%s: null\n", ArgsKey)) - } else { + if container.Args != nil { sb.WriteString(fmt.Sprintf("%s: %s\n", ArgsKey, s.ObjFlatNoQuotes(container.Args))) } @@ -381,9 +382,9 @@ func (autoscaling *Autoscaling) UserStr() string { sb.WriteString(fmt.Sprintf("%s: %s\n", MinReplicasKey, s.Int32(autoscaling.MinReplicas))) sb.WriteString(fmt.Sprintf("%s: %s\n", MaxReplicasKey, s.Int32(autoscaling.MaxReplicas))) sb.WriteString(fmt.Sprintf("%s: %s\n", InitReplicasKey, s.Int32(autoscaling.InitReplicas))) - sb.WriteString(fmt.Sprintf("%s: %s\n", MaxReplicaQueueLengthKey, s.Int64(autoscaling.MaxReplicaQueueLength))) - sb.WriteString(fmt.Sprintf("%s: %s\n", MaxReplicaConcurrencyKey, s.Int64(autoscaling.MaxReplicaConcurrency))) - sb.WriteString(fmt.Sprintf("%s: %s\n", TargetReplicaConcurrencyKey, s.Float64(autoscaling.TargetReplicaConcurrency))) + sb.WriteString(fmt.Sprintf("%s: %s\n", MaxQueueLengthKey, s.Int64(autoscaling.MaxQueueLength))) + sb.WriteString(fmt.Sprintf("%s: %s\n", MaxConcurrencyKey, s.Int64(autoscaling.MaxConcurrency))) + sb.WriteString(fmt.Sprintf("%s: %s\n", TargetInFlightKey, s.Float64(*autoscaling.TargetInFlight))) sb.WriteString(fmt.Sprintf("%s: %s\n", WindowKey, autoscaling.Window.String())) sb.WriteString(fmt.Sprintf("%s: %s\n", DownscaleStabilizationPeriodKey, autoscaling.DownscaleStabilizationPeriod.String())) sb.WriteString(fmt.Sprintf("%s: %s\n", UpscaleStabilizationPeriodKey, autoscaling.UpscaleStabilizationPeriod.String())) @@ -478,8 +479,11 @@ func (api *API) TelemetryEvent() map[string]interface{} { } event["pod.node_groups._is_defined"] = len(api.Pod.NodeGroups) > 0 event["pod.node_groups._len"] = len(api.Pod.NodeGroups) - event["pod.containers._len"] = len(api.Pod.Containers) + if api.Pod.Port != nil { + event["pod.port"] = *api.Pod.Port + } + event["pod.containers._len"] = len(api.Pod.Containers) totalCompute := GetTotalComputeFromContainers(api.Pod.Containers) event["pod.containers.compute._is_defined"] = true if totalCompute.CPU != nil { @@ -505,9 +509,9 @@ func (api *API) TelemetryEvent() map[string]interface{} { event["autoscaling.min_replicas"] = api.Autoscaling.MinReplicas event["autoscaling.max_replicas"] = api.Autoscaling.MaxReplicas event["autoscaling.init_replicas"] = api.Autoscaling.InitReplicas - event["autoscaling.max_replica_queue_length"] = api.Autoscaling.MaxReplicaQueueLength - event["autoscaling.max_replica_concurrency"] = api.Autoscaling.MaxReplicaConcurrency - event["autoscaling.target_replica_concurrency"] = api.Autoscaling.TargetReplicaConcurrency + event["autoscaling.max_queue_length"] = api.Autoscaling.MaxQueueLength + event["autoscaling.max_concurrency"] = api.Autoscaling.MaxConcurrency + event["autoscaling.target_in_flight"] = *api.Autoscaling.TargetInFlight event["autoscaling.window"] = api.Autoscaling.Window.Seconds() event["autoscaling.downscale_stabilization_period"] = api.Autoscaling.DownscaleStabilizationPeriod.Seconds() event["autoscaling.upscale_stabilization_period"] = api.Autoscaling.UpscaleStabilizationPeriod.Seconds() diff --git a/pkg/types/userconfig/config_key.go b/pkg/types/userconfig/config_key.go index 598db9f97f..4a303c9a46 100644 --- a/pkg/types/userconfig/config_key.go +++ b/pkg/types/userconfig/config_key.go @@ -34,6 +34,7 @@ const ( PodKey = "pod" NodeGroupsKey = "node_groups" ShmSizeKey = "shm_size" + PortKey = "port" ContainersKey = "containers" // Containers @@ -56,9 +57,9 @@ const ( MinReplicasKey = "min_replicas" MaxReplicasKey = "max_replicas" InitReplicasKey = "init_replicas" - MaxReplicaQueueLengthKey = "max_replica_queue_length" - MaxReplicaConcurrencyKey = "max_replica_concurrency" - TargetReplicaConcurrencyKey = "target_replica_concurrency" + MaxQueueLengthKey = "max_queue_length" + MaxConcurrencyKey = "max_concurrency" + TargetInFlightKey = "target_in_flight" WindowKey = "window" DownscaleStabilizationPeriodKey = "downscale_stabilization_period" UpscaleStabilizationPeriodKey = "upscale_stabilization_period" @@ -75,9 +76,9 @@ const ( EndpointAnnotationKey = "networking.cortex.dev/endpoint" MinReplicasAnnotationKey = "autoscaling.cortex.dev/min-replicas" MaxReplicasAnnotationKey = "autoscaling.cortex.dev/max-replicas" - MaxReplicaQueueLengthAnnotationKey = "autoscaling.cortex.dev/max-replica-queue-length" - MaxReplicaConcurrencyAnnotationKey = "autoscaling.cortex.dev/max-replica-concurrency" - TargetReplicaConcurrencyAnnotationKey = "autoscaling.cortex.dev/target-replica-concurrency" + MaxQueueLengthAnnotationKey = "autoscaling.cortex.dev/max-queue-length" + MaxConcurrencyAnnotationKey = "autoscaling.cortex.dev/max-concurrency" + TargetInFlightAnnotationKey = "autoscaling.cortex.dev/target-in-flight" WindowAnnotationKey = "autoscaling.cortex.dev/window" DownscaleStabilizationPeriodAnnotationKey = "autoscaling.cortex.dev/downscale-stabilization-period" UpscaleStabilizationPeriodAnnotationKey = "autoscaling.cortex.dev/upscale-stabilization-period" diff --git a/pkg/workloads/helpers.go b/pkg/workloads/helpers.go index bfb06dcf55..ca13be7f72 100644 --- a/pkg/workloads/helpers.go +++ b/pkg/workloads/helpers.go @@ -23,6 +23,7 @@ import ( "github.com/cortexlabs/cortex/pkg/lib/k8s" kcore "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" ) func K8sName(apiName string) string { @@ -129,39 +130,84 @@ func getKubexitEnvVars(containerName string, deathDeps []string, birthDeps []str return envVars } -func defaultVolumes(requiresKubexit bool) []kcore.Volume { - volumes := []kcore.Volume{ - k8s.EmptyDirVolume(_emptyDirVolumeName), - { - Name: "client-config", - VolumeSource: kcore.VolumeSource{ - ConfigMap: &kcore.ConfigMapVolumeSource{ - LocalObjectReference: kcore.LocalObjectReference{ - Name: "client-config", - }, +func MntVolume() kcore.Volume { + return k8s.EmptyDirVolume(_emptyDirVolumeName) +} + +func CortexVolume() kcore.Volume { + return k8s.EmptyDirVolume(_cortexDirVolumeName) +} + +func ClientConfigVolume() kcore.Volume { + return kcore.Volume{ + Name: _clientConfigDirVolume, + VolumeSource: kcore.VolumeSource{ + ConfigMap: &kcore.ConfigMapVolumeSource{ + LocalObjectReference: kcore.LocalObjectReference{ + Name: _clientConfigConfigMap, }, }, }, } +} - if requiresKubexit { - return append(volumes, k8s.EmptyDirVolume(_kubexitGraveyardName)) +func ClusterConfigVolume() kcore.Volume { + return kcore.Volume{ + Name: _clusterConfigDirVolume, + VolumeSource: kcore.VolumeSource{ + ConfigMap: &kcore.ConfigMapVolumeSource{ + LocalObjectReference: kcore.LocalObjectReference{ + Name: _clusterConfigConfigMap, + }, + }, + }, } - return volumes } -func defaultVolumeMounts(requiresKubexit bool) []kcore.VolumeMount { - volumeMounts := []kcore.VolumeMount{ - k8s.EmptyDirVolumeMount(_emptyDirVolumeName, _emptyDirMountPath), - { - Name: "client-config", - MountPath: path.Join(_clientConfigDir, "cli.yaml"), - SubPath: "cli.yaml", +func ShmVolume(q resource.Quantity) kcore.Volume { + return kcore.Volume{ + Name: _shmDirVolumeName, + VolumeSource: kcore.VolumeSource{ + EmptyDir: &kcore.EmptyDirVolumeSource{ + Medium: kcore.StorageMediumMemory, + SizeLimit: k8s.QuantityPtr(q), + }, }, } +} - if requiresKubexit { - return append(volumeMounts, k8s.EmptyDirVolumeMount(_kubexitGraveyardName, _kubexitGraveyardMountPath)) +func KubexitVolume() kcore.Volume { + return k8s.EmptyDirVolume(_kubexitGraveyardName) +} + +func MntMount() kcore.VolumeMount { + return k8s.EmptyDirVolumeMount(_emptyDirVolumeName, _emptyDirMountPath) +} + +func CortexMount() kcore.VolumeMount { + return k8s.EmptyDirVolumeMount(_cortexDirVolumeName, _cortexDirMountPath) +} + +func ClientConfigMount() kcore.VolumeMount { + return kcore.VolumeMount{ + Name: _clientConfigDirVolume, + MountPath: path.Join(_clientConfigDir, "cli.yaml"), + SubPath: "cli.yaml", } - return volumeMounts +} + +func ClusterConfigMount() kcore.VolumeMount { + return kcore.VolumeMount{ + Name: _clusterConfigDirVolume, + MountPath: path.Join(_clusterConfigDir, "cluster.yaml"), + SubPath: "cluster.yaml", + } +} + +func ShmMount() kcore.VolumeMount { + return k8s.EmptyDirVolumeMount(_shmDirVolumeName, _shmDirMountPath) +} + +func KubexitMount() kcore.VolumeMount { + return k8s.EmptyDirVolumeMount(_kubexitGraveyardName, _kubexitGraveyardMountPath) } diff --git a/pkg/workloads/init.go b/pkg/workloads/init.go index ef84d13aba..a5ea4ca0de 100644 --- a/pkg/workloads/init.go +++ b/pkg/workloads/init.go @@ -29,7 +29,7 @@ import ( ) const ( - JobSpecPath = "/mnt/job_spec.json" + JobSpecPath = "/cortex/job_spec.json" ) const ( @@ -43,8 +43,10 @@ func KubexitInitContainer() kcore.Container { Name: _kubexitInitContainerName, Image: config.ClusterConfig.ImageKubexit, ImagePullPolicy: kcore.PullAlways, - Command: []string{"cp", "/bin/kubexit", "/mnt/kubexit"}, - VolumeMounts: defaultVolumeMounts(true), + Command: []string{"cp", "/bin/kubexit", "/cortex/kubexit"}, + VolumeMounts: []kcore.VolumeMount{ + CortexMount(), + }, } } @@ -79,7 +81,9 @@ func TaskInitContainer(job *spec.TaskJob) kcore.Container { Value: strings.ToUpper(userconfig.InfoLogLevel.String()), }, }, - VolumeMounts: defaultVolumeMounts(true), + VolumeMounts: []kcore.VolumeMount{ + CortexMount(), + }, } } @@ -114,6 +118,8 @@ func BatchInitContainer(job *spec.BatchJob) kcore.Container { Value: strings.ToUpper(userconfig.InfoLogLevel.String()), }, }, - VolumeMounts: defaultVolumeMounts(true), + VolumeMounts: []kcore.VolumeMount{ + CortexMount(), + }, } } diff --git a/pkg/workloads/k8s.go b/pkg/workloads/k8s.go index b0f3d6fcf4..fe87d4cb94 100644 --- a/pkg/workloads/k8s.go +++ b/pkg/workloads/k8s.go @@ -33,23 +33,33 @@ import ( ) const ( - DefaultPortInt32 = int32(8888) - DefaultPortStr = "8888" ServiceAccountName = "default" ) const ( - _clientConfigDir = "/mnt/client" - _emptyDirMountPath = "/mnt" + _cortexDirVolumeName = "cortex" + _cortexDirMountPath = "/cortex" + _clientConfigDir = "/cortex/client" + _emptyDirVolumeName = "mnt" + _emptyDirMountPath = "/mnt" - _gatewayContainerName = "gateway" + _proxyContainerName = "proxy" - _neuronRTDContainerName = "neuron-rtd" - _neuronRTDSocket = "/sock/neuron.sock" + _gatewayContainerName = "gateway" _kubexitGraveyardName = "graveyard" _kubexitGraveyardMountPath = "/graveyard" + + _shmDirVolumeName = "dshm" + _shmDirMountPath = "/dev/shm" + + _clientConfigDirVolume = "client-config" + _clientConfigConfigMap = "client-config" + + _clusterConfigDirVolume = "cluster-config" + _clusterConfigConfigMap = "cluster-config" + _clusterConfigDir = "/configs/cluster" ) var ( @@ -69,13 +79,13 @@ func AsyncGatewayContainer(api spec.API, queueURL string, volumeMounts []kcore.V Image: config.ClusterConfig.ImageAsyncGateway, ImagePullPolicy: kcore.PullAlways, Args: []string{ - "-port", s.Int32(DefaultPortInt32), + "-port", s.Int32(consts.ProxyListeningPortInt32), "-queue", queueURL, "-cluster-config", consts.DefaultInClusterConfigPath, api.Name, }, Ports: []kcore.ContainerPort{ - {ContainerPort: DefaultPortInt32}, + {ContainerPort: consts.ProxyListeningPortInt32}, }, Env: []kcore.EnvVar{ { @@ -111,31 +121,35 @@ func AsyncGatewayContainer(api spec.API, queueURL string, volumeMounts []kcore.V func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { requiresKubexit := api.Kind == userconfig.BatchAPIKind || api.Kind == userconfig.TaskAPIKind - volumes := defaultVolumes(requiresKubexit) - containerMounts := []kcore.VolumeMount{} + volumes := []kcore.Volume{ + MntVolume(), + CortexVolume(), + ClientConfigVolume(), + } + containerMounts := []kcore.VolumeMount{ + MntMount(), + CortexMount(), + ClientConfigMount(), + } + + if requiresKubexit { + volumes = append(volumes, KubexitVolume()) + containerMounts = append(containerMounts, KubexitMount()) + } if api.Pod.ShmSize != nil { - volumes = append(volumes, kcore.Volume{ - Name: "dshm", - VolumeSource: kcore.VolumeSource{ - EmptyDir: &kcore.EmptyDirVolumeSource{ - Medium: kcore.StorageMediumMemory, - SizeLimit: k8s.QuantityPtr(api.Pod.ShmSize.Quantity), - }, - }, - }) - containerMounts = append(containerMounts, kcore.VolumeMount{ - Name: "dshm", - MountPath: "/dev/shm", - }) + volumes = append(volumes, ShmVolume(api.Pod.ShmSize.Quantity)) + containerMounts = append(containerMounts, ShmMount()) } var containers []kcore.Container - var podHasInf bool containerNames := userconfig.GetContainerNames(api.Pod.Containers) for _, container := range api.Pod.Containers { containerResourceList := kcore.ResourceList{} containerResourceLimitsList := kcore.ResourceList{} + securityContext := kcore.SecurityContext{ + Privileged: pointer.Bool(true), + } if container.Compute.CPU != nil { containerResourceList[kcore.ResourceCPU] = *k8s.QuantityPtr(container.Compute.CPU.Quantity.DeepCopy()) @@ -150,43 +164,35 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { containerResourceLimitsList["nvidia.com/gpu"] = *kresource.NewQuantity(container.Compute.GPU, kresource.DecimalSI) } - containerVolumeMounts := append(defaultVolumeMounts(requiresKubexit), containerMounts...) + containerVolumeMounts := containerMounts + if container.Compute.Inf > 0 { - volumes = append(volumes, kcore.Volume{ - Name: "neuron-sock", - }) - rtdVolumeMounts := []kcore.VolumeMount{ - { - Name: "neuron-sock", - MountPath: "/sock", - }, - } + totalHugePages := container.Compute.Inf * _hugePagesMemPerInf + containerResourceList["nvidia.com/gpu"] = *kresource.NewQuantity(container.Compute.Inf, kresource.DecimalSI) + containerResourceList["hugepages-2Mi"] = *kresource.NewQuantity(totalHugePages, kresource.BinarySI) + containerResourceLimitsList["nvidia.com/gpu"] = *kresource.NewQuantity(container.Compute.Inf, kresource.DecimalSI) + containerResourceLimitsList["hugepages-2Mi"] = *kresource.NewQuantity(totalHugePages, kresource.BinarySI) - containerVolumeMounts = append(containerVolumeMounts, rtdVolumeMounts...) - - if requiresKubexit { - rtdVolumeMounts = append(rtdVolumeMounts, - k8s.EmptyDirVolumeMount(_emptyDirVolumeName, _emptyDirMountPath), - kcore.VolumeMount{Name: _kubexitGraveyardName, MountPath: _kubexitGraveyardMountPath}, - ) - neuronRTDEnvVars := getKubexitEnvVars(_neuronRTDContainerName, containerNames.Slice(), nil) - containers = append(containers, neuronRuntimeDaemonContainer(container.Compute.Inf, rtdVolumeMounts, neuronRTDEnvVars)) - } else { - containers = append(containers, neuronRuntimeDaemonContainer(container.Compute.Inf, rtdVolumeMounts, nil)) + securityContext.Capabilities = &kcore.Capabilities{ + Add: []kcore.Capability{ + "SYS_ADMIN", + "IPC_LOCK", + }, } + } - podHasInf = true + containerEnvVars := []kcore.EnvVar{} + if api.Kind != userconfig.TaskAPIKind { + containerEnvVars = append(containerEnvVars, kcore.EnvVar{ + Name: "CORTEX_PORT", + Value: s.Int32(*api.Pod.Port), + }) } - var containerEnvVars []kcore.EnvVar if requiresKubexit { containerDeathDependencies := containerNames.Copy() containerDeathDependencies.Remove(container.Name) - if podHasInf { - containerEnvVars = getKubexitEnvVars(container.Name, containerDeathDependencies.Slice(), []string{"neuron-rtd"}) - } else { - containerEnvVars = getKubexitEnvVars(container.Name, containerDeathDependencies.Slice(), nil) - } + containerEnvVars = getKubexitEnvVars(container.Name, containerDeathDependencies.Slice(), nil) } for k, v := range container.Env { @@ -195,18 +201,10 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { Value: v, }) } - containerEnvVars = append(containerEnvVars, kcore.EnvVar{ - Name: "HOST_IP", - ValueFrom: &kcore.EnvVarSource{ - FieldRef: &kcore.ObjectFieldSelector{ - FieldPath: "status.hostIP", - }, - }, - }) var containerCmd []string - if requiresKubexit && container.Command[0] != "/mnt/kubexit" { - containerCmd = append([]string{"/mnt/kubexit"}, container.Command...) + if requiresKubexit && container.Command[0] != "/cortex/kubexit" { + containerCmd = append([]string{"/cortex/kubexit"}, container.Command...) } containers = append(containers, kcore.Container{ @@ -220,16 +218,9 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { Requests: containerResourceList, Limits: containerResourceLimitsList, }, - Ports: []kcore.ContainerPort{ - { - ContainerPort: int32(8888), - }, - }, ImagePullPolicy: kcore.PullAlways, - SecurityContext: &kcore.SecurityContext{ - Privileged: pointer.Bool(true), - }}, - ) + SecurityContext: &securityContext, + }) } return containers, volumes @@ -333,34 +324,40 @@ func GenerateNodeAffinities(apiNodeGroups []string) *kcore.Affinity { } } -func neuronRuntimeDaemonContainer(computeInf int64, volumeMounts []kcore.VolumeMount, envVars []kcore.EnvVar) kcore.Container { - totalHugePages := computeInf * _hugePagesMemPerInf +func RealtimeProxyContainer(api spec.API) (kcore.Container, kcore.Volume) { return kcore.Container{ - Name: _neuronRTDContainerName, - Image: config.ClusterConfig.ImageNeuronRTD, + Name: _proxyContainerName, + Image: config.ClusterConfig.ImageProxy, ImagePullPolicy: kcore.PullAlways, - Env: envVars, - SecurityContext: &kcore.SecurityContext{ - Capabilities: &kcore.Capabilities{ - Add: []kcore.Capability{ - "SYS_ADMIN", - "IPC_LOCK", - }, - }, + Args: []string{ + "-port", + consts.ProxyListeningPortStr, + "-metrics-port", + consts.MetricsPortStr, + "-user-port", + s.Int32(*api.Pod.Port), + "-max-concurrency", + s.Int32(int32(api.Autoscaling.MaxConcurrency)), + "-max-queue-length", + s.Int32(int32(api.Autoscaling.MaxQueueLength)), + "-cluster-config", + consts.DefaultInClusterConfigPath, }, - VolumeMounts: volumeMounts, - ReadinessProbe: SocketExistsProbe(_neuronRTDSocket), - Resources: kcore.ResourceRequirements{ - Requests: kcore.ResourceList{ - "hugepages-2Mi": *kresource.NewQuantity(totalHugePages, kresource.BinarySI), - "aws.amazon.com/neuron": *kresource.NewQuantity(computeInf, kresource.DecimalSI), - }, - Limits: kcore.ResourceList{ - "hugepages-2Mi": *kresource.NewQuantity(totalHugePages, kresource.BinarySI), - "aws.amazon.com/neuron": *kresource.NewQuantity(computeInf, kresource.DecimalSI), + Ports: []kcore.ContainerPort{ + {Name: "metrics", ContainerPort: consts.MetricsPortInt32}, + {ContainerPort: consts.ProxyListeningPortInt32}, + }, + Env: []kcore.EnvVar{ + { + Name: "CORTEX_LOG_LEVEL", + Value: strings.ToUpper(userconfig.InfoLogLevel.String()), }, }, - } + EnvFrom: baseClusterEnvVars(), + VolumeMounts: []kcore.VolumeMount{ + ClusterConfigMount(), + }, + }, ClusterConfigVolume() } // func getAsyncAPIEnvVars(api spec.API, queueURL string) []kcore.EnvVar { @@ -379,32 +376,3 @@ func neuronRuntimeDaemonContainer(computeInf int64, volumeMounts []kcore.VolumeM // return envVars // } - -// func RequestMonitorContainer(api *spec.API) kcore.Container { -// requests := kcore.ResourceList{} -// if api.Compute != nil { -// if api.Compute.CPU != nil { -// requests[kcore.ResourceCPU] = _requestMonitorCPURequest -// } -// if api.Compute.Mem != nil { -// requests[kcore.ResourceMemory] = _requestMonitorMemRequest -// } -// } - -// return kcore.Container{ -// Name: _requestMonitorContainerName, -// Image: config.ClusterConfig.ImageRequestMonitor, -// ImagePullPolicy: kcore.PullAlways, -// Args: []string{"-p", DefaultRequestMonitorPortStr}, -// Ports: []kcore.ContainerPort{ -// {Name: "metrics", ContainerPort: DefaultRequestMonitorPortInt32}, -// }, -// Env: requestMonitorEnvVars(api), -// EnvFrom: baseEnvVars(), -// VolumeMounts: defaultVolumeMounts(), -// ReadinessProbe: FileExistsProbe(_requestMonitorReadinessFile), -// Resources: kcore.ResourceRequirements{ -// Requests: requests, -// }, -// } -// } diff --git a/test/apis/realtime/Dockerfile b/test/apis/realtime/Dockerfile index 05845435a4..2955a09120 100644 --- a/test/apis/realtime/Dockerfile +++ b/test/apis/realtime/Dockerfile @@ -17,5 +17,4 @@ RUN pip install Flask gunicorn # webserver, with one worker process and 8 threads. # For environments with multiple CPU cores, increase the number of workers # to be equal to the cores available. -# Timeout is set to 0 to disable the timeouts of the workers to allow Cloud Run to handle instance scaling. -CMD exec gunicorn --bind :8888 --workers 1 --threads 8 --timeout 0 main:app +CMD exec gunicorn --bind :$CORTEX_PORT --workers 1 --threads $NUM_THREADS --timeout 0 main:app diff --git a/test/apis/realtime/cortex.yaml b/test/apis/realtime/cortex.yaml index 4770d44709..9f4e231cea 100644 --- a/test/apis/realtime/cortex.yaml +++ b/test/apis/realtime/cortex.yaml @@ -1,10 +1,15 @@ - name: realtime kind: RealtimeAPI pod: - node_groups: [cpu] containers: - name: api image: 499593605069.dkr.ecr.us-west-2.amazonaws.com/sample/realtime-caas:latest + env: + NUM_THREADS: "8" compute: cpu: 200m mem: 512Mi + autoscaling: + max_queue_length: 16 + max_concurrency: 8 + target_in_flight: 10 diff --git a/test/apis/realtime/main.py b/test/apis/realtime/main.py index 6ea9ba744f..f13fb6f436 100644 --- a/test/apis/realtime/main.py +++ b/test/apis/realtime/main.py @@ -1,4 +1,6 @@ import os +import time +import threading as td from flask import Flask @@ -7,9 +9,11 @@ @app.route("/") def hello_world(): - name = os.environ.get("NAME", "World") - return "Hello {}!".format(name) + time.sleep(1) + msg = f"Hello World! (TID={td.get_ident()}" + print(msg) + return msg if __name__ == "__main__": - app.run(debug=True, host="0.0.0.0", port=8888) + app.run(debug=True, host="0.0.0.0", port=int(os.getenv("CORTEX_PORT", "8080"))) diff --git a/test/apis/task/main.py b/test/apis/task/main.py index dcac680486..494e9bdfb0 100644 --- a/test/apis/task/main.py +++ b/test/apis/task/main.py @@ -2,7 +2,7 @@ def main(): - with open("/mnt/job_spec.json", "r") as f: + with open("/cortex/job_spec.json", "r") as f: job_spec = json.load(f) print(json.dumps(job_spec, indent=2)) diff --git a/test/apis/tensorflow/image-classifier-resnet50/cortex_inf.yaml b/test/apis/tensorflow/image-classifier-resnet50/cortex_inf.yaml index 77d9adde9f..5ba87a891d 100644 --- a/test/apis/tensorflow/image-classifier-resnet50/cortex_inf.yaml +++ b/test/apis/tensorflow/image-classifier-resnet50/cortex_inf.yaml @@ -17,4 +17,4 @@ cpu: 3 mem: 4G autoscaling: - max_replica_concurrency: 16384 + max_concurrency: 16384 diff --git a/test/apis/tensorflow/image-classifier-resnet50/cortex_inf_server_side_batching.yaml b/test/apis/tensorflow/image-classifier-resnet50/cortex_inf_server_side_batching.yaml index 615a2ab84f..9d587cecfb 100644 --- a/test/apis/tensorflow/image-classifier-resnet50/cortex_inf_server_side_batching.yaml +++ b/test/apis/tensorflow/image-classifier-resnet50/cortex_inf_server_side_batching.yaml @@ -20,4 +20,4 @@ cpu: 3 mem: 4G autoscaling: - max_replica_concurrency: 16384 + max_concurrency: 16384 From a123f5473470f61fbf44285d1e8ab1051045d2de Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Wed, 19 May 2021 18:35:09 +0100 Subject: [PATCH 21/82] Add readiness probe to realtime cortex proxy (#2176) --- cmd/proxy/main.go | 99 ++++++++++++--------- pkg/proxy/consts.go | 11 +-- pkg/proxy/handler.go | 3 +- pkg/proxy/probe/encoding.go | 46 ++++++++++ pkg/proxy/probe/encoding_test.go | 90 ++++++++++++++++++++ pkg/proxy/probe/handler.go | 33 +++++++ pkg/proxy/probe/handler_test.go | 117 +++++++++++++++++++++++++ pkg/proxy/probe/probe.go | 142 +++++++++++++++++++++++++++++++ pkg/proxy/probe/probe_test.go | 113 ++++++++++++++++++++++++ 9 files changed, 605 insertions(+), 49 deletions(-) create mode 100644 pkg/proxy/probe/encoding.go create mode 100644 pkg/proxy/probe/encoding_test.go create mode 100644 pkg/proxy/probe/handler.go create mode 100644 pkg/proxy/probe/handler_test.go create mode 100644 pkg/proxy/probe/probe.go create mode 100644 pkg/proxy/probe/probe_test.go diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index d7e4f0fa05..9bc7cff7c5 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -19,6 +19,7 @@ package main import ( "context" "flag" + "io/ioutil" "net/http" "os" "os/signal" @@ -30,6 +31,7 @@ import ( "github.com/cortexlabs/cortex/pkg/lib/logging" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/proxy" + "github.com/cortexlabs/cortex/pkg/proxy/probe" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" "github.com/cortexlabs/cortex/pkg/types/userconfig" "go.uber.org/zap" @@ -40,45 +42,24 @@ const ( _requestSampleInterval = 1 * time.Second ) -var ( - proxyLogger = logging.GetLogger() -) - -func Exit(err error, wrapStrs ...string) { - for _, str := range wrapStrs { - err = errors.Wrap(err, str) - } - - if err != nil && !errors.IsNoTelemetry(err) { - telemetry.Error(err) - } - - if err != nil && !errors.IsNoPrint(err) { - proxyLogger.Error(err) - } - - telemetry.Close() - - os.Exit(1) -} - func main() { var ( port int - metricsPort int + adminPort int userContainerPort int maxConcurrency int maxQueueLength int + probeDefPath string clusterConfigPath string ) - flag.IntVar(&port, "port", 8888, "port where the proxy is served") - flag.IntVar(&metricsPort, "metrics-port", 15000, "metrics port for prometheus") - flag.IntVar(&userContainerPort, "user-port", 8080, "port where the proxy redirects to the traffic to") + flag.IntVar(&port, "port", 8000, "port where the proxy server will be exposed") + flag.IntVar(&adminPort, "admin-port", 15000, "port where the admin server (for metrics and probes) will be exposed") + flag.IntVar(&userContainerPort, "user-port", 8080, "port where the proxy will redirect to the traffic to") flag.IntVar(&maxConcurrency, "max-concurrency", 0, "max concurrency allowed for user container") flag.IntVar(&maxQueueLength, "max-queue-length", 0, "max request queue length for user container") flag.StringVar(&clusterConfigPath, "cluster-config", "", "cluster config path") - + flag.StringVar(&probeDefPath, "probe", "", "path to the desired probe json definition") flag.Parse() log := logging.GetLogger() @@ -88,26 +69,26 @@ func main() { switch { case maxConcurrency == 0: - log.Fatal("-max-concurrency flag is required") + log.Fatal("--max-concurrency flag is required") case maxQueueLength == 0: - log.Fatal("-max-queue-length flag is required") + log.Fatal("--max-queue-length flag is required") case clusterConfigPath == "": - log.Fatal("-cluster-config flag is required") + log.Fatal("--cluster-config flag is required") } clusterConfig, err := clusterconfig.NewForFile(clusterConfigPath) if err != nil { - Exit(err) + exit(log, err) } awsClient, err := aws.NewForRegion(clusterConfig.Region) if err != nil { - Exit(err) + exit(log, err) } _, userID, err := awsClient.CheckCredentials() if err != nil { - Exit(err) + exit(log, err) } err = telemetry.Init(telemetry.Config{ @@ -122,7 +103,7 @@ func main() { BackoffMode: telemetry.BackoffDuplicateMessages, }) if err != nil { - Exit(err) + exit(log, err) } target := "http://127.0.0.1:" + strconv.Itoa(userContainerPort) @@ -139,6 +120,23 @@ func main() { promStats := proxy.NewPrometheusStatsReporter() + var readinessProbe *probe.Probe + if probeDefPath != "" { + jsonProbe, err := ioutil.ReadFile(probeDefPath) + if err != nil { + log.Fatal(err) + } + + probeDef, err := probe.DecodeJSON(string(jsonProbe)) + if err != nil { + log.Fatal(err) + } + + readinessProbe = probe.NewProbe(probeDef, log) + } else { + readinessProbe = probe.NewDefaultProbe(target, log) + } + go func() { reportTicker := time.NewTicker(_reportInterval) defer reportTicker.Stop() @@ -161,14 +159,18 @@ func main() { } }() + adminHandler := http.NewServeMux() + adminHandler.Handle("/metrics", promStats) + adminHandler.Handle("/healthz", probe.Handler(readinessProbe)) + servers := map[string]*http.Server{ "proxy": { - Addr: ":" + strconv.Itoa(port), + Addr: ":" + strconv.Itoa(userContainerPort), Handler: proxy.Handler(breaker, httpProxy), }, - "metrics": { - Addr: ":" + strconv.Itoa(metricsPort), - Handler: promStats, + "admin": { + Addr: ":" + strconv.Itoa(adminPort), + Handler: adminHandler, }, } @@ -184,8 +186,8 @@ func main() { signal.Notify(sigint, os.Interrupt) select { - case err := <-errCh: - Exit(errors.Wrap(err, "failed to start proxy server")) + case err = <-errCh: + exit(log, errors.Wrap(err, "failed to start proxy server")) case <-sigint: // We received an interrupt signal, shut down. log.Info("Received TERM signal, handling a graceful shutdown...") @@ -202,3 +204,20 @@ func main() { telemetry.Close() } } + +func exit(log *zap.SugaredLogger, err error, wrapStrs ...string) { + for _, str := range wrapStrs { + err = errors.Wrap(err, str) + } + + if err != nil && !errors.IsNoTelemetry(err) { + telemetry.Error(err) + } + + if err != nil && !errors.IsNoPrint(err) { + log.Error(err) + } + + telemetry.Close() + os.Exit(1) +} diff --git a/pkg/proxy/consts.go b/pkg/proxy/consts.go index 95c415378c..67bb86f7fc 100644 --- a/pkg/proxy/consts.go +++ b/pkg/proxy/consts.go @@ -17,14 +17,11 @@ limitations under the License. package proxy const ( - _userAgentKey = "User-Agent" + // UserAgentKey is the user agent header key + UserAgentKey = "User-Agent" + // KubeProbeUserAgentPrefix is the user agent header prefix used in k8s probes // Since K8s 1.8, prober requests have // User-Agent = "kube-probe/{major-version}.{minor-version}". - _kubeProbeUserAgentPrefix = "kube-probe/" - - // KubeletProbeHeaderName is the header name to augment the probes, because - // Istio with mTLS rewrites probes, but their probes pass a different - // user-agent. - _kubeletProbeHeaderName = "K-Kubelet-Probe" + KubeProbeUserAgentPrefix = "kube-probe/" ) diff --git a/pkg/proxy/handler.go b/pkg/proxy/handler.go index ed59616f99..39ba5f0b6f 100644 --- a/pkg/proxy/handler.go +++ b/pkg/proxy/handler.go @@ -43,6 +43,5 @@ func Handler(breaker *Breaker, next http.Handler) http.HandlerFunc { } func isKubeletProbe(r *http.Request) bool { - return strings.HasPrefix(r.Header.Get(_userAgentKey), _kubeProbeUserAgentPrefix) || - r.Header.Get(_kubeletProbeHeaderName) != "" + return strings.HasPrefix(r.Header.Get(UserAgentKey), KubeProbeUserAgentPrefix) } diff --git a/pkg/proxy/probe/encoding.go b/pkg/proxy/probe/encoding.go new file mode 100644 index 0000000000..38363d1dc4 --- /dev/null +++ b/pkg/proxy/probe/encoding.go @@ -0,0 +1,46 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package probe + +import ( + "encoding/json" + "errors" + + kcore "k8s.io/api/core/v1" +) + +// DecodeJSON takes a json serialised *kcore.Probe and returns a Probe or an error. +func DecodeJSON(jsonProbe string) (*kcore.Probe, error) { + pb := &kcore.Probe{} + if err := json.Unmarshal([]byte(jsonProbe), pb); err != nil { + return nil, err + } + return pb, nil +} + +// EncodeJSON takes *kcore.Probe object and returns marshalled Probe JSON string and an error. +func EncodeJSON(pb *kcore.Probe) (string, error) { + if pb == nil { + return "", errors.New("cannot encode nil probe") + } + + probeJSON, err := json.Marshal(pb) + if err != nil { + return "", err + } + return string(probeJSON), nil +} diff --git a/pkg/proxy/probe/encoding_test.go b/pkg/proxy/probe/encoding_test.go new file mode 100644 index 0000000000..e48aa62165 --- /dev/null +++ b/pkg/proxy/probe/encoding_test.go @@ -0,0 +1,90 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package probe_test + +import ( + "encoding/json" + "testing" + + "github.com/cortexlabs/cortex/pkg/proxy/probe" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + kcore "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func TestDecodeProbeSuccess(t *testing.T) { + t.Parallel() + + expectedProbe := &kcore.Probe{ + PeriodSeconds: 1, + TimeoutSeconds: 2, + SuccessThreshold: 1, + FailureThreshold: 1, + Handler: kcore.Handler{ + TCPSocket: &kcore.TCPSocketAction{ + Host: "127.0.0.1", + Port: intstr.FromString("8080"), + }, + }, + } + probeBytes, err := json.Marshal(expectedProbe) + require.NoError(t, err) + + gotProbe, err := probe.DecodeJSON(string(probeBytes)) + require.NoError(t, err) + + require.Equal(t, expectedProbe, gotProbe) +} + +func TestDecodeProbeFailure(t *testing.T) { + t.Parallel() + + probeBytes, err := json.Marshal("blah") + require.NoError(t, err) + + _, err = probe.DecodeJSON(string(probeBytes)) + require.Error(t, err) +} + +func TestEncodeProbe(t *testing.T) { + t.Parallel() + + pb := &kcore.Probe{ + SuccessThreshold: 1, + Handler: kcore.Handler{ + TCPSocket: &kcore.TCPSocketAction{ + Host: "127.0.0.1", + Port: intstr.FromString("8080"), + }, + }, + } + + jsonProbe, err := probe.EncodeJSON(pb) + require.NoError(t, err) + + wantProbe := `{"tcpSocket":{"port":"8080","host":"127.0.0.1"},"successThreshold":1}` + require.Equal(t, wantProbe, jsonProbe) +} + +func TestEncodeNilProbe(t *testing.T) { + t.Parallel() + + jsonProbe, err := probe.EncodeJSON(nil) + assert.Error(t, err) + assert.Empty(t, jsonProbe) +} diff --git a/pkg/proxy/probe/handler.go b/pkg/proxy/probe/handler.go new file mode 100644 index 0000000000..55c89b5c58 --- /dev/null +++ b/pkg/proxy/probe/handler.go @@ -0,0 +1,33 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package probe + +import "net/http" + +func Handler(pb *Probe) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + healthy := pb.ProbeContainer() + if !healthy { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte("unhealthy")) + return + } + + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("healthy")) + } +} diff --git a/pkg/proxy/probe/handler_test.go b/pkg/proxy/probe/handler_test.go new file mode 100644 index 0000000000..da2db383f9 --- /dev/null +++ b/pkg/proxy/probe/handler_test.go @@ -0,0 +1,117 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package probe_test + +import ( + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/cortexlabs/cortex/pkg/proxy" + "github.com/cortexlabs/cortex/pkg/proxy/probe" + "github.com/stretchr/testify/require" + kcore "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func TestHandlerFailure(t *testing.T) { + t.Parallel() + log := newLogger(t) + + pb := probe.NewDefaultProbe("http://127.0.0.1:12345", log) + handler := probe.Handler(pb) + + r := httptest.NewRequest(http.MethodGet, "http://fake.cortex.dev/healthz", nil) + w := httptest.NewRecorder() + + handler(w, r) + + require.Equal(t, http.StatusInternalServerError, w.Code) + require.Equal(t, "unhealthy", w.Body.String()) +} + +func TestHandlerSuccessTCP(t *testing.T) { + t.Parallel() + log := newLogger(t) + + var userHandler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + } + server := httptest.NewServer(userHandler) + + pb := probe.NewDefaultProbe(server.URL, log) + handler := probe.Handler(pb) + + r := httptest.NewRequest(http.MethodGet, "http://fake.cortex.dev/healthz", nil) + w := httptest.NewRecorder() + + handler(w, r) + + require.Equal(t, http.StatusOK, w.Code) + require.Equal(t, "healthy", w.Body.String()) +} + +func TestHandlerSuccessHTTP(t *testing.T) { + t.Parallel() + log := newLogger(t) + + headers := []kcore.HTTPHeader{ + { + Name: "X-Cortex-Blah", + Value: "Blah", + }, + } + + var userHandler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { + require.Contains(t, r.Header.Get(proxy.UserAgentKey), proxy.KubeProbeUserAgentPrefix) + for _, header := range headers { + require.Equal(t, header.Value, r.Header.Get(header.Name)) + } + + w.WriteHeader(http.StatusOK) + } + server := httptest.NewServer(userHandler) + targetURL, err := url.Parse(server.URL) + require.NoError(t, err) + + pb := probe.NewProbe( + &kcore.Probe{ + Handler: kcore.Handler{ + HTTPGet: &kcore.HTTPGetAction{ + Path: "/", + Port: intstr.FromString(targetURL.Port()), + Host: targetURL.Hostname(), + HTTPHeaders: headers, + }, + }, + TimeoutSeconds: 3, + PeriodSeconds: 1, + SuccessThreshold: 1, + FailureThreshold: 3, + }, log, + ) + handler := probe.Handler(pb) + + r := httptest.NewRequest(http.MethodGet, "http://fake.cortex.dev/healthz", nil) + w := httptest.NewRecorder() + + handler(w, r) + + require.Equal(t, http.StatusOK, w.Code) + require.Equal(t, "healthy", w.Body.String()) +} diff --git a/pkg/proxy/probe/probe.go b/pkg/proxy/probe/probe.go new file mode 100644 index 0000000000..3eeee7a9f9 --- /dev/null +++ b/pkg/proxy/probe/probe.go @@ -0,0 +1,142 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package probe + +import ( + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + "time" + + s "github.com/cortexlabs/cortex/pkg/lib/strings" + "github.com/cortexlabs/cortex/pkg/proxy" + "go.uber.org/zap" + kcore "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + _defaultTimeoutSeconds = 1 +) + +type Probe struct { + *kcore.Probe + logger *zap.SugaredLogger +} + +func NewProbe(probe *kcore.Probe, logger *zap.SugaredLogger) *Probe { + return &Probe{ + Probe: probe, + logger: logger, + } +} + +func NewDefaultProbe(target string, logger *zap.SugaredLogger) *Probe { + targetURL, err := url.Parse(target) + if err != nil { + panic(fmt.Sprintf("failed to parse target URL: %v", err)) + } + + return &Probe{ + Probe: &kcore.Probe{ + Handler: kcore.Handler{ + TCPSocket: &kcore.TCPSocketAction{ + Port: intstr.FromString(targetURL.Port()), + Host: targetURL.Hostname(), + }, + }, + TimeoutSeconds: _defaultTimeoutSeconds, + }, + logger: logger, + } +} + +func (p *Probe) ProbeContainer() bool { + var err error + + switch { + case p.HTTPGet != nil: + err = p.httpProbe() + case p.TCPSocket != nil: + err = p.tcpProbe() + case p.Exec != nil: + // Should never be reachable. + p.logger.Error("exec probe not supported") + return false + default: + p.logger.Warn("no probe found") + return false + } + + if err != nil { + p.logger.Warn(err) + return false + } + return true +} + +func (p *Probe) httpProbe() error { + targetURL := s.EnsurePrefix( + net.JoinHostPort(p.HTTPGet.Host, p.HTTPGet.Port.String())+s.EnsurePrefix(p.HTTPGet.Path, "/"), + "http://", + ) + + httpClient := &http.Client{} + req, err := http.NewRequest(http.MethodGet, targetURL, nil) + if err != nil { + return err + } + + req.Header.Add(proxy.UserAgentKey, proxy.KubeProbeUserAgentPrefix) + + for _, header := range p.HTTPGet.HTTPHeaders { + req.Header.Add(header.Name, header.Value) + } + + res, err := httpClient.Do(req) + if err != nil { + return err + } + + defer func() { + // Ensure body is both read _and_ closed so it can be reused for keep-alive. + // No point handling errors, connection just won't be reused. + _, _ = io.Copy(ioutil.Discard, res.Body) + _ = res.Body.Close() + }() + + // response status code between 200-399 indicates success + if !(res.StatusCode >= 200 && res.StatusCode < 400) { + return fmt.Errorf("HTTP probe did not respond Ready, got status code: %d", res.StatusCode) + } + + return nil +} + +func (p *Probe) tcpProbe() error { + timeout := time.Duration(p.TimeoutSeconds) * time.Second + address := net.JoinHostPort(p.TCPSocket.Host, p.TCPSocket.Port.String()) + conn, err := net.DialTimeout("tcp", address, timeout) + if err != nil { + return err + } + _ = conn.Close() + return nil +} diff --git a/pkg/proxy/probe/probe_test.go b/pkg/proxy/probe/probe_test.go new file mode 100644 index 0000000000..b0518396f3 --- /dev/null +++ b/pkg/proxy/probe/probe_test.go @@ -0,0 +1,113 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package probe_test + +import ( + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/cortexlabs/cortex/pkg/proxy/probe" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + kcore "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func newLogger(t *testing.T) *zap.SugaredLogger { + t.Helper() + + config := zap.NewDevelopmentConfig() + config.Level = zap.NewAtomicLevelAt(zap.FatalLevel) + logger, err := config.Build() + require.NoError(t, err) + + log := logger.Sugar() + + return log +} + +func TestDefaultProbeSuccess(t *testing.T) { + t.Parallel() + log := newLogger(t) + + var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + } + server := httptest.NewServer(handler) + pb := probe.NewDefaultProbe(server.URL, log) + + require.True(t, pb.ProbeContainer()) +} + +func TestDefaultProbeFailure(t *testing.T) { + t.Parallel() + log := newLogger(t) + + target := "http://127.0.0.1:12345" + pb := probe.NewDefaultProbe(target, log) + + require.False(t, pb.ProbeContainer()) +} + +func TestProbeHTTPFailure(t *testing.T) { + t.Parallel() + log := newLogger(t) + + pb := probe.NewProbe( + &kcore.Probe{ + Handler: kcore.Handler{ + HTTPGet: &kcore.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromString("12345"), + Host: "127.0.0.1", + }, + }, + TimeoutSeconds: 3, + }, log, + ) + + require.False(t, pb.ProbeContainer()) +} + +func TestProbeHTTPSuccess(t *testing.T) { + t.Parallel() + log := newLogger(t) + + var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + } + server := httptest.NewServer(handler) + targetURL, err := url.Parse(server.URL) + require.NoError(t, err) + + pb := probe.NewProbe( + &kcore.Probe{ + Handler: kcore.Handler{ + HTTPGet: &kcore.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromString(targetURL.Port()), + Host: targetURL.Hostname(), + }, + }, + TimeoutSeconds: 3, + }, log, + ) + + require.True(t, pb.ProbeContainer()) +} From fbbf1b511c92c39bef32e08b96851f2a544532ba Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 19 May 2021 10:44:23 -0700 Subject: [PATCH 22/82] Support binary data in configmap helper --- pkg/config/config.go | 2 +- pkg/lib/k8s/configmap.go | 14 ++++++++------ pkg/operator/operator/memory_capacity.go | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 2343c32ee5..dd88371b58 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -62,7 +62,7 @@ func InitConfigs(clusterConfig *clusterconfig.Config, operatorMetadata *clusterc } func getClusterConfigFromConfigMap() (clusterconfig.Config, error) { - configMapData, err := K8s.GetConfigMapData("cluster-config") + configMapData, _, err := K8s.GetConfigMapData("cluster-config") if err != nil { return clusterconfig.Config{}, err } diff --git a/pkg/lib/k8s/configmap.go b/pkg/lib/k8s/configmap.go index b5972e1049..04ecb67fb0 100644 --- a/pkg/lib/k8s/configmap.go +++ b/pkg/lib/k8s/configmap.go @@ -33,7 +33,8 @@ var _configMapTypeMeta = kmeta.TypeMeta{ type ConfigMapSpec struct { Name string - Data map[string]string + Data map[string]string // Data and BinaryData must not have overlapping keys + BinaryData map[string][]byte // Data and BinaryData must not have overlapping keys Labels map[string]string Annotations map[string]string } @@ -46,7 +47,8 @@ func ConfigMap(spec *ConfigMapSpec) *kcore.ConfigMap { Labels: spec.Labels, Annotations: spec.Annotations, }, - Data: spec.Data, + Data: spec.Data, + BinaryData: spec.BinaryData, } return configMap } @@ -92,15 +94,15 @@ func (c *Client) GetConfigMap(name string) (*kcore.ConfigMap, error) { return configMap, nil } -func (c *Client) GetConfigMapData(name string) (map[string]string, error) { +func (c *Client) GetConfigMapData(name string) (map[string]string, map[string][]byte, error) { configMap, err := c.GetConfigMap(name) if err != nil { - return nil, err + return nil, nil, err } if configMap == nil { - return nil, nil + return nil, nil, nil } - return configMap.Data, nil + return configMap.Data, configMap.BinaryData, nil } func (c *Client) DeleteConfigMap(name string) (bool, error) { diff --git a/pkg/operator/operator/memory_capacity.go b/pkg/operator/operator/memory_capacity.go index 5b2e041d96..18881f34f8 100644 --- a/pkg/operator/operator/memory_capacity.go +++ b/pkg/operator/operator/memory_capacity.go @@ -73,7 +73,7 @@ func getMemoryCapacityFromNodes(primaryInstances []string) (map[string]*kresourc } func getMemoryCapacityFromConfigMap() (map[string]*kresource.Quantity, error) { - configMapData, err := config.K8s.GetConfigMapData(_memConfigMapName) + configMapData, _, err := config.K8s.GetConfigMapData(_memConfigMapName) if err != nil { return nil, err } From 9722b5a9aff0ea3fae5e2c01c157f81bf5fef1bc Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Sat, 22 May 2021 15:41:30 -0700 Subject: [PATCH 23/82] Update CLI and Python Client (#2189) --- cli/cluster/patch.go | 51 --- cli/cmd/cluster.go | 44 +- cli/cmd/patch.go | 93 ----- cli/cmd/root.go | 2 - cmd/operator/main.go | 1 - dev/generate_cli_md.sh | 2 - dev/generate_python_client_md.sh | 6 +- docs/clients/cli.md | 31 +- docs/clients/python.md | 124 +----- docs/clusters/management/update.md | 2 +- docs/summary.md | 1 - docs/workloads/debugging.md | 25 -- .../realtime/traffic-splitter/example.md | 4 +- pkg/operator/endpoints/patch.go | 51 --- pkg/operator/resources/resources.go | 70 ---- python/client/cortex/client.py | 378 +----------------- python/client/cortex/consts.py | 2 - python/client/cortex/exceptions.py | 8 - python/client/cortex/util.py | 8 +- .../pytorch/text-generator/deploy_class.py | 40 -- 20 files changed, 48 insertions(+), 895 deletions(-) delete mode 100644 cli/cluster/patch.go delete mode 100644 cli/cmd/patch.go delete mode 100644 docs/workloads/debugging.md delete mode 100644 pkg/operator/endpoints/patch.go delete mode 100644 test/apis/pytorch/text-generator/deploy_class.py diff --git a/cli/cluster/patch.go b/cli/cluster/patch.go deleted file mode 100644 index 464a4cd6ad..0000000000 --- a/cli/cluster/patch.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2021 Cortex Labs, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cluster - -import ( - "path/filepath" - - "github.com/cortexlabs/cortex/pkg/lib/errors" - "github.com/cortexlabs/cortex/pkg/lib/files" - "github.com/cortexlabs/cortex/pkg/lib/json" - s "github.com/cortexlabs/cortex/pkg/lib/strings" - "github.com/cortexlabs/cortex/pkg/operator/schema" -) - -func Patch(operatorConfig OperatorConfig, configPath string, force bool) ([]schema.DeployResult, error) { - params := map[string]string{ - "force": s.Bool(force), - "configFileName": filepath.Base(configPath), - } - - configBytes, err := files.ReadFileBytes(configPath) - if err != nil { - return nil, err - } - - response, err := HTTPPostJSON(operatorConfig, "/patch", configBytes, params) - if err != nil { - return nil, err - } - - var deployResults []schema.DeployResult - if err := json.Unmarshal(response, &deployResults); err != nil { - return nil, errors.Wrap(err, "/patch", string(response)) - } - - return deployResults, nil -} diff --git a/cli/cmd/cluster.go b/cli/cmd/cluster.go index 1f2348bf11..7bb7f38387 100644 --- a/cli/cmd/cluster.go +++ b/cli/cmd/cluster.go @@ -19,7 +19,6 @@ package cmd import ( "fmt" "os" - "path" "path/filepath" "regexp" "strings" @@ -675,9 +674,9 @@ var _clusterDownCmd = &cobra.Command{ } var _clusterExportCmd = &cobra.Command{ - Use: "export [API_NAME] [API_ID]", - Short: "download the code and configuration for APIs", - Args: cobra.RangeArgs(0, 2), + Use: "export", + Short: "download the configurations for all APIs", + Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { telemetry.Event("cli.cluster.export") @@ -715,25 +714,13 @@ var _clusterExportCmd = &cobra.Command{ } var apisResponse []schema.APIResponse - if len(args) == 0 { - apisResponse, err = cluster.GetAPIs(operatorConfig) - if err != nil { - exit.Error(err) - } - if len(apisResponse) == 0 { - fmt.Println(fmt.Sprintf("no apis found in your cluster named %s in %s", accessConfig.ClusterName, accessConfig.Region)) - exit.Ok() - } - } else if len(args) == 1 { - apisResponse, err = cluster.GetAPI(operatorConfig, args[0]) - if err != nil { - exit.Error(err) - } - } else if len(args) == 2 { - apisResponse, err = cluster.GetAPIByID(operatorConfig, args[0], args[1]) - if err != nil { - exit.Error(err) - } + apisResponse, err = cluster.GetAPIs(operatorConfig) + if err != nil { + exit.Error(err) + } + if len(apisResponse) == 0 { + fmt.Println(fmt.Sprintf("no apis found in your cluster named %s in %s", accessConfig.ClusterName, accessConfig.Region)) + exit.Ok() } exportPath := fmt.Sprintf("export-%s-%s", accessConfig.Region, accessConfig.ClusterName) @@ -744,21 +731,16 @@ var _clusterExportCmd = &cobra.Command{ } for _, apiResponse := range apisResponse { - baseDir := filepath.Join(exportPath, apiResponse.Spec.Name, apiResponse.Spec.ID) + specFilePath := filepath.Join(exportPath, apiResponse.Spec.Name+".yaml") - fmt.Println(fmt.Sprintf("exporting %s to %s", apiResponse.Spec.Name, baseDir)) - - err = files.CreateDir(baseDir) - if err != nil { - exit.Error(err) - } + fmt.Println(fmt.Sprintf("exporting %s to %s", apiResponse.Spec.Name, specFilePath)) yamlBytes, err := yaml.Marshal(apiResponse.Spec.API.SubmittedAPISpec) if err != nil { exit.Error(err) } - err = files.WriteFile(yamlBytes, path.Join(baseDir, apiResponse.Spec.FileName)) + err = files.WriteFile(yamlBytes, specFilePath) if err != nil { exit.Error(err) } diff --git a/cli/cmd/patch.go b/cli/cmd/patch.go deleted file mode 100644 index 398ebe2d13..0000000000 --- a/cli/cmd/patch.go +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright 2021 Cortex Labs, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "fmt" - "strings" - - "github.com/cortexlabs/cortex/cli/cluster" - "github.com/cortexlabs/cortex/cli/types/flags" - "github.com/cortexlabs/cortex/pkg/lib/exit" - libjson "github.com/cortexlabs/cortex/pkg/lib/json" - "github.com/cortexlabs/cortex/pkg/lib/print" - "github.com/cortexlabs/cortex/pkg/lib/telemetry" - "github.com/spf13/cobra" -) - -var ( - _flagPatchEnv string - _flagPatchForce bool -) - -func patchInit() { - _patchCmd.Flags().SortFlags = false - _patchCmd.Flags().StringVarP(&_flagPatchEnv, "env", "e", "", "environment to use") - _patchCmd.Flags().BoolVarP(&_flagPatchForce, "force", "f", false, "override the in-progress api update") - _patchCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.UserOutputTypeStrings(), "|"))) -} - -var _patchCmd = &cobra.Command{ - Use: "patch [CONFIG_FILE]", - Short: "update API configuration for a deployed API", - Args: cobra.RangeArgs(0, 1), - Run: func(cmd *cobra.Command, args []string) { - envName, err := getEnvFromFlag(_flagPatchEnv) - if err != nil { - telemetry.Event("cli.patch") - exit.Error(err) - } - - env, err := ReadOrConfigureEnv(envName) - if err != nil { - telemetry.Event("cli.patch") - exit.Error(err) - } - telemetry.Event("cli.patch", map[string]interface{}{"env_name": env.Name}) - - err = printEnvIfNotSpecified(env.Name, cmd) - if err != nil { - exit.Error(err) - } - - configPath := getConfigPath(args) - - deployResults, err := cluster.Patch(MustGetOperatorConfig(env.Name), configPath, _flagPatchForce) - if err != nil { - exit.Error(err) - } - - switch _flagOutput { - case flags.JSONOutputType: - bytes, err := libjson.Marshal(deployResults) - if err != nil { - exit.Error(err) - } - fmt.Print(string(bytes)) - case flags.PrettyOutputType: - message, err := deployMessage(deployResults, env.Name) - if err != nil { - exit.Error(err) - } - if didAnyResultsError(deployResults) { - print.StderrBoldFirstBlock(message) - } else { - print.BoldFirstBlock(message) - } - } - }, -} diff --git a/cli/cmd/root.go b/cli/cmd/root.go index bee317fabf..2cd18267db 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -116,7 +116,6 @@ func init() { envInit() getInit() logsInit() - patchInit() refreshInit() versionInit() } @@ -155,7 +154,6 @@ func Execute() { _rootCmd.AddCommand(_deployCmd) _rootCmd.AddCommand(_getCmd) - _rootCmd.AddCommand(_patchCmd) _rootCmd.AddCommand(_logsCmd) _rootCmd.AddCommand(_refreshCmd) _rootCmd.AddCommand(_deleteCmd) diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 50a7b65110..0b40de677c 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -117,7 +117,6 @@ func main() { routerWithAuth.HandleFunc("/info", endpoints.Info).Methods("GET") routerWithAuth.HandleFunc("/deploy", endpoints.Deploy).Methods("POST") - routerWithAuth.HandleFunc("/patch", endpoints.Patch).Methods("POST") routerWithAuth.HandleFunc("/refresh/{apiName}", endpoints.Refresh).Methods("POST") routerWithAuth.HandleFunc("/delete/{apiName}", endpoints.Delete).Methods("DELETE") routerWithAuth.HandleFunc("/get", endpoints.GetAPIs).Methods("GET") diff --git a/dev/generate_cli_md.sh b/dev/generate_cli_md.sh index 2d49524e9d..63f9f40d0d 100755 --- a/dev/generate_cli_md.sh +++ b/dev/generate_cli_md.sh @@ -34,10 +34,8 @@ commands=( "deploy" "get" "logs" - "patch" "refresh" "delete" - "prepare-debug" "cluster up" "cluster info" "cluster scale" diff --git a/dev/generate_python_client_md.sh b/dev/generate_python_client_md.sh index 0b0215e76f..d6ec879de2 100755 --- a/dev/generate_python_client_md.sh +++ b/dev/generate_python_client_md.sh @@ -65,11 +65,7 @@ truncate -s -1 $docs_path # Cortex version comment sed -i "s/^## deploy$/## deploy\n\n/g" $docs_path -sed -i "s/^## deploy\\\_realtime\\\_api$/## deploy\\\_realtime\\\_api\n\n/g" $docs_path -sed -i "s/^## deploy\\\_async\\\_api$/## deploy\\\_async\\\_api\n\n/g" $docs_path -sed -i "s/^## deploy\\\_batch\\\_api$/## deploy\\\_batch\\\_api\n\n/g" $docs_path -sed -i "s/^## deploy\\\_task\\\_api$/## deploy\\\_task\\\_api\n\n/g" $docs_path -sed -i "s/^## deploy\\\_traffic\\\_splitter$/## deploy\\\_traffic\\\_splitter\n\n/g" $docs_path +sed -i "s/^## deploy\\\_from\\\_file$/## deploy\\\_from\\\_file\n\n/g" $docs_path pip3 uninstall -y cortex rm -rf $ROOT/python/client/cortex.egg-info diff --git a/docs/clients/cli.md b/docs/clients/cli.md index 7179115a3b..faae5c0be9 100644 --- a/docs/clients/cli.md +++ b/docs/clients/cli.md @@ -46,21 +46,6 @@ Flags: -h, --help help for logs ``` -## patch - -```text -update API configuration for a deployed API - -Usage: - cortex patch [CONFIG_FILE] [flags] - -Flags: - -e, --env string environment to use - -f, --force override the in-progress api update - -o, --output string output format: one of pretty|json (default "pretty") - -h, --help help for patch -``` - ## refresh ```text @@ -92,18 +77,6 @@ Flags: -h, --help help for delete ``` -## prepare-debug - -```text -prepare artifacts to debug containers - -Usage: - cortex prepare-debug CONFIG_FILE [API_NAME] [flags] - -Flags: - -h, --help help for prepare-debug -``` - ## cluster up ```text @@ -175,10 +148,10 @@ Flags: ## cluster export ```text -download the code and configuration for APIs +download the configurations for all APIs Usage: - cortex cluster export [API_NAME] [API_ID] [flags] + cortex cluster export [flags] Flags: -c, --config string path to a cluster configuration file diff --git a/docs/clients/python.md b/docs/clients/python.md index a12f1d4ba8..2e7cb704ad 100644 --- a/docs/clients/python.md +++ b/docs/clients/python.md @@ -7,16 +7,11 @@ * [env\_delete](#env_delete) * [cortex.client.Client](#cortex-client-client) * [deploy](#deploy) - * [deploy\_realtime\_api](#deploy_realtime_api) - * [deploy\_async\_api](#deploy_async_api) - * [deploy\_batch\_api](#deploy_batch_api) - * [deploy\_task\_api](#deploy_task_api) - * [deploy\_traffic\_splitter](#deploy_traffic_splitter) + * [deploy\_from\_file](#deploy_from_file) * [get\_api](#get_api) * [list\_apis](#list_apis) * [get\_job](#get_job) * [refresh](#refresh) - * [patch](#patch) * [delete](#delete) * [stop\_job](#stop_job) * [stream\_api\_logs](#stream_api_logs) @@ -86,129 +81,39 @@ Delete an environment configured on this machine. ```python - | deploy(api_spec: Dict[str, Any], project_dir: str, force: bool = True, wait: bool = False) + | deploy(api_spec: Dict[str, Any], force: bool = True, wait: bool = False) ``` -Deploy API(s) from a project directory. +Deploy or update an API. **Arguments**: - `api_spec` - A dictionary defining a single Cortex API. See https://docs.cortex.dev/v/master/ for schema. -- `project_dir` - Path to a python project. - `force` - Override any in-progress api updates. -- `wait` - Streams logs until the APIs are ready. +- `wait` - Streams logs until the API is ready. **Returns**: Deployment status, API specification, and endpoint for each API. -## deploy\_realtime\_api +## deploy\_from\_file ```python - | deploy_realtime_api(api_spec: Dict[str, Any], handler, requirements: Optional[List] = None, conda_packages: Optional[List] = None, force: bool = True, wait: bool = False) -> Dict + | deploy_from_file(config_file: str, force: bool = False, wait: bool = False) -> Dict ``` -Deploy a Realtime API. +Deploy or update APIs specified in a configuration file. **Arguments**: -- `api_spec` - A dictionary defining a single Cortex API. See https://docs.cortex.dev/v/master/workloads/realtime-apis/configuration for schema. -- `handler` - A Cortex Handler class implementation. -- `requirements` - A list of PyPI dependencies that will be installed before the handler class implementation is invoked. -- `conda_packages` - A list of Conda dependencies that will be installed before the handler class implementation is invoked. +- `config_file` - Local path to a yaml file defining Cortex API(s). See https://docs.cortex.dev/v/master/ for schema. - `force` - Override any in-progress api updates. - `wait` - Streams logs until the APIs are ready. -**Returns**: - - Deployment status, API specification, and endpoint for each API. - -## deploy\_async\_api - - - -```python - | deploy_async_api(api_spec: Dict[str, Any], handler, requirements: Optional[List] = None, conda_packages: Optional[List] = None, force: bool = True) -> Dict -``` - -Deploy an Async API. - -**Arguments**: - -- `api_spec` - A dictionary defining a single Cortex API. See https://docs.cortex.dev/v/master/workloads/async-apis/configuration for schema. -- `handler` - A Cortex Handler class implementation. -- `requirements` - A list of PyPI dependencies that will be installed before the handler class implementation is invoked. -- `conda_packages` - A list of Conda dependencies that will be installed before the handler class implementation is invoked. -- `force` - Override any in-progress api updates. - - -**Returns**: - - Deployment status, API specification, and endpoint for each API. - -## deploy\_batch\_api - - - -```python - | deploy_batch_api(api_spec: Dict[str, Any], handler, requirements: Optional[List] = None, conda_packages: Optional[List] = None) -> Dict -``` - -Deploy a Batch API. - -**Arguments**: - -- `api_spec` - A dictionary defining a single Cortex API. See https://docs.cortex.dev/v/master/workloads/batch-apis/configuration for schema. -- `handler` - A Cortex Handler class implementation. -- `requirements` - A list of PyPI dependencies that will be installed before the handler class implementation is invoked. -- `conda_packages` - A list of Conda dependencies that will be installed before the handler class implementation is invoked. - - -**Returns**: - - Deployment status, API specification, and endpoint for each API. - -## deploy\_task\_api - - - -```python - | deploy_task_api(api_spec: Dict[str, Any], task, requirements: Optional[List] = None, conda_packages: Optional[List] = None) -> Dict -``` - -Deploy a Task API. - -**Arguments**: - -- `api_spec` - A dictionary defining a single Cortex API. See https://docs.cortex.dev/v/master/workloads/task-apis/configuration for schema. -- `task` - A callable class implementation. -- `requirements` - A list of PyPI dependencies that will be installed before the handler class implementation is invoked. -- `conda_packages` - A list of Conda dependencies that will be installed before the handler class implementation is invoked. - - -**Returns**: - - Deployment status, API specification, and endpoint for each API. - -## deploy\_traffic\_splitter - - - -```python - | deploy_traffic_splitter(api_spec: Dict[str, Any]) -> Dict -``` - -Deploy a Task API. - -**Arguments**: - -- `api_spec` - A dictionary defining a single Cortex API. See https://docs.cortex.dev/v/master/workloads/realtime-apis/traffic-splitter/configuration for schema. - - **Returns**: Deployment status, API specification, and endpoint for each API. @@ -273,19 +178,6 @@ Restart all of the replicas for a Realtime API without downtime. - `api_name` - Name of the API to refresh. - `force` - Override an already in-progress API update. -## patch - -```python - | patch(api_spec: Dict, force: bool = False) -> Dict -``` - -Update the api specification for an API that has already been deployed. - -**Arguments**: - -- `api_spec` - The new api specification to apply -- `force` - Override an already in-progress API update. - ## delete ```python diff --git a/docs/clusters/management/update.md b/docs/clusters/management/update.md index 6b1f72afba..f881ae3ff5 100644 --- a/docs/clusters/management/update.md +++ b/docs/clusters/management/update.md @@ -27,7 +27,7 @@ cortex cluster up cluster.yaml In production environments, you can upgrade your cluster without downtime if you have a backend service or DNS in front of your Cortex cluster: 1. Spin up a new cluster. For example: `cortex cluster up new-cluster.yaml --configure-env cortex2` (this will create a CLI environment named `cortex2` for accessing the new cluster). -1. Re-deploy your APIs in your new cluster. For example, if the name of your CLI environment for your existing cluster is `cortex`, you can use `cortex get --env cortex` to list all running APIs in your cluster, and re-deploy them in the new cluster by changing directories to each API's project folder and running `cortex deploy --env cortex2`. Alternatively, you can run `cortex cluster export --name --region ` to export all of your APIs (including configuration and application code), change directories into each API/ID subfolder that was exported, and run `cortex deploy --env cortex2`. +1. Re-deploy your APIs in your new cluster. For example, if the name of your CLI environment for your existing cluster is `cortex`, you can use `cortex get --env cortex` to list all running APIs in your cluster, and re-deploy them in the new cluster by changing directories to each API's project folder and running `cortex deploy --env cortex2`. Alternatively, you can run `cortex cluster export --name --region ` to export all of your API specifications, change directories the folder that was exported, and run `cortex deploy --env cortex2 ` for each API that you want to deploy in the new cluster. 1. Route requests to your new cluster. * If you are using a custom domain: update the A record in your Route 53 hosted zone to point to your new cluster's API load balancer. * If you have a backend service which makes requests to Cortex: update your backend service to make requests to the new cluster's endpoints. diff --git a/docs/summary.md b/docs/summary.md index 0f58863d0c..f6f5d1db0c 100644 --- a/docs/summary.md +++ b/docs/summary.md @@ -75,7 +75,6 @@ * [Python packages](workloads/dependencies/python-packages.md) * [System packages](workloads/dependencies/system-packages.md) * [Custom images](workloads/dependencies/images.md) -* [Debugging](workloads/debugging.md) ## Clients diff --git a/docs/workloads/debugging.md b/docs/workloads/debugging.md deleted file mode 100644 index b03af8f971..0000000000 --- a/docs/workloads/debugging.md +++ /dev/null @@ -1,25 +0,0 @@ -# Debugging - -You can test and debug your handler implementation and image by running your API container locally. - -The `cortex prepare-debug` command will generate a debugging configuration file named `.debug.json` based on your api spec, and it will print out a corresponding `docker run` command that can be used to run the container locally. - -For example: - - - -```bash -cortex prepare-debug cortex.yaml iris-classifier - -> docker run -p 9000:8888 \ -> -e "CORTEX_VERSION=0.35.0" \ -> -e "CORTEX_API_SPEC=/mnt/project/iris-classifier.debug.json" \ -> -v /home/ubuntu/iris-classifier:/mnt/project \ -> quay.io/cortexlabs/python-handler-cpu:0.35.0 -``` - -Make a request to the api container: - -```bash -curl localhost:9000 -X POST -H "Content-Type: application/json" -d @sample.json -``` diff --git a/docs/workloads/realtime/traffic-splitter/example.md b/docs/workloads/realtime/traffic-splitter/example.md index baa9d72130..bb4fa6d28b 100644 --- a/docs/workloads/realtime/traffic-splitter/example.md +++ b/docs/workloads/realtime/traffic-splitter/example.md @@ -50,7 +50,7 @@ traffic_splitter_spec = { ], } -cx.deploy_traffic_splitter(traffic_splitter_spec) +cx.deploy(traffic_splitter_spec) ``` ## Update the weights of the traffic splitter @@ -62,5 +62,5 @@ traffic_splitter_spec = cx.get_api("text-generator")["spec"]["submitted_api_spec traffic_splitter_spec["apis"][0]["weight"] = 1 traffic_splitter_spec["apis"][1]["weight"] = 99 -cx.patch(traffic_splitter_spec) +cx.deploy(traffic_splitter_spec) ``` diff --git a/pkg/operator/endpoints/patch.go b/pkg/operator/endpoints/patch.go deleted file mode 100644 index 7f0dd32e0f..0000000000 --- a/pkg/operator/endpoints/patch.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2021 Cortex Labs, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package endpoints - -import ( - "io/ioutil" - "net/http" - - "github.com/cortexlabs/cortex/pkg/lib/errors" - "github.com/cortexlabs/cortex/pkg/operator/resources" -) - -func Patch(w http.ResponseWriter, r *http.Request) { - force := getOptionalBoolQParam("force", false, r) - - configFileName, err := getRequiredQueryParam("configFileName", r) - if err != nil { - respondError(w, r, errors.WithStack(err)) - return - } - - rw := http.MaxBytesReader(w, r.Body, 10<<20) - - bodyBytes, err := ioutil.ReadAll(rw) - if err != nil { - respondError(w, r, err) - return - } - - response, err := resources.Patch(bodyBytes, configFileName, force) - if err != nil { - respondError(w, r, err) - return - } - - respond(w, response) -} diff --git a/pkg/operator/resources/resources.go b/pkg/operator/resources/resources.go index ef6eb5482d..918174a1d6 100644 --- a/pkg/operator/resources/resources.go +++ b/pkg/operator/resources/resources.go @@ -169,76 +169,6 @@ func UpdateAPI(apiConfig *userconfig.API, projectID string, force bool) (*schema return nil, msg, err } -func Patch(configBytes []byte, configFileName string, force bool) ([]schema.DeployResult, error) { - apiConfigs, err := spec.ExtractAPIConfigs(configBytes, configFileName) - if err != nil { - return nil, err - } - - results := make([]schema.DeployResult, 0, len(apiConfigs)) - for i := range apiConfigs { - apiConfig := &apiConfigs[i] - result := schema.DeployResult{} - - apiSpec, msg, err := patchAPI(apiConfig, force) - if err == nil && apiSpec != nil { - apiEndpoint, _ := operator.APIEndpoint(apiSpec) - - result.API = &schema.APIResponse{ - Spec: *apiSpec, - Endpoint: apiEndpoint, - } - } - - result.Message = msg - if err != nil { - result.Error = errors.ErrorStr(err) - } - - results = append(results, result) - } - return results, nil -} - -func patchAPI(apiConfig *userconfig.API, force bool) (*spec.API, string, error) { - deployedResource, err := GetDeployedResourceByName(apiConfig.Name) - if err != nil { - return nil, "", err - } - - if deployedResource.Kind == userconfig.UnknownKind { - return nil, "", ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.RealtimeAPIKind, userconfig.AsyncAPIKind, userconfig.BatchAPIKind, userconfig.TaskAPIKind, userconfig.TrafficSplitterKind) // unexpected - } - - if deployedResource.Kind != apiConfig.Kind { - return nil, "", ErrorCannotChangeKindOfDeployedAPI(apiConfig.Name, apiConfig.Kind, deployedResource.Kind) - } - - prevAPISpec, err := operator.DownloadAPISpec(deployedResource.Name, deployedResource.ID()) - if err != nil { - return nil, "", err - } - - err = ValidateClusterAPIs([]userconfig.API{*apiConfig}) - if err != nil { - err = errors.Append(err, fmt.Sprintf("\n\napi configuration schema can be found at https://docs.cortex.dev/v/%s/", consts.CortexVersionMinor)) - return nil, "", err - } - - switch deployedResource.Kind { - case userconfig.RealtimeAPIKind: - return realtimeapi.UpdateAPI(apiConfig, prevAPISpec.ProjectID, force) - case userconfig.BatchAPIKind: - return batchapi.UpdateAPI(apiConfig, prevAPISpec.ProjectID) - case userconfig.TaskAPIKind: - return taskapi.UpdateAPI(apiConfig, prevAPISpec.ProjectID) - case userconfig.AsyncAPIKind: - return asyncapi.UpdateAPI(*apiConfig, prevAPISpec.ProjectID, force) - default: - return trafficsplitter.UpdateAPI(apiConfig) - } -} - func RefreshAPI(apiName string, force bool) (string, error) { deployedResource, err := GetDeployedResourceByName(apiName) if err != nil { diff --git a/python/client/cortex/client.py b/python/client/cortex/client.py index 22e5bcf333..17c3f4366b 100644 --- a/python/client/cortex/client.py +++ b/python/client/cortex/client.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import inspect import json import os import shutil @@ -20,18 +19,13 @@ import sys import threading import time -import uuid +import yaml from pathlib import Path from typing import Optional, List, Dict, Any -import dill -import yaml - from cortex import util from cortex.binary import run_cli, get_cli_path -from cortex.consts import EXPECTED_PYTHON_VERSION from cortex.telemetry import sentry_wrapper -from cortex.exceptions import InvalidKindForMethod class Client: @@ -48,368 +42,49 @@ def __init__(self, env: Dict): self.env_name = env["name"] # CORTEX_VERSION_MINOR + @sentry_wrapper def deploy( self, api_spec: Dict[str, Any], - project_dir: str, force: bool = True, wait: bool = False, ): """ - Deploy API(s) from a project directory. + Deploy or update an API. Args: api_spec: A dictionary defining a single Cortex API. See https://docs.cortex.dev/v/master/ for schema. - project_dir: Path to a python project. - force: Override any in-progress api updates. - wait: Streams logs until the APIs are ready. - - Returns: - Deployment status, API specification, and endpoint for each API. - """ - return self._create_api( - api_spec=api_spec, - project_dir=project_dir, - force=force, - wait=wait, - ) - - # CORTEX_VERSION_MINOR - def deploy_realtime_api( - self, - api_spec: Dict[str, Any], - handler, - requirements: Optional[List] = None, - conda_packages: Optional[List] = None, - force: bool = True, - wait: bool = False, - ) -> Dict: - """ - Deploy a Realtime API. - - Args: - api_spec: A dictionary defining a single Cortex API. See https://docs.cortex.dev/v/master/workloads/realtime-apis/configuration for schema. - handler: A Cortex Handler class implementation. - requirements: A list of PyPI dependencies that will be installed before the handler class implementation is invoked. - conda_packages: A list of Conda dependencies that will be installed before the handler class implementation is invoked. - force: Override any in-progress api updates. - wait: Streams logs until the APIs are ready. - - Returns: - Deployment status, API specification, and endpoint for each API. - """ - kind = api_spec.get("kind") - if kind != "RealtimeAPI": - raise InvalidKindForMethod( - f"expected an api_spec with kind 'RealtimeAPI', got kind '{kind}' instead" - ) - - return self._create_api( - api_spec=api_spec, - handler=handler, - requirements=requirements, - conda_packages=conda_packages, - force=force, - wait=wait, - ) - - # CORTEX_VERSION_MINOR - def deploy_async_api( - self, - api_spec: Dict[str, Any], - handler, - requirements: Optional[List] = None, - conda_packages: Optional[List] = None, - force: bool = True, - ) -> Dict: - """ - Deploy an Async API. - - Args: - api_spec: A dictionary defining a single Cortex API. See https://docs.cortex.dev/v/master/workloads/async-apis/configuration for schema. - handler: A Cortex Handler class implementation. - requirements: A list of PyPI dependencies that will be installed before the handler class implementation is invoked. - conda_packages: A list of Conda dependencies that will be installed before the handler class implementation is invoked. force: Override any in-progress api updates. + wait: Streams logs until the API is ready. Returns: Deployment status, API specification, and endpoint for each API. """ - kind = api_spec.get("kind") - if kind != "AsyncAPI": - raise InvalidKindForMethod( - f"expected an api_spec with kind 'AsyncAPI', got kind '{kind}' instead" - ) - - return self._create_api( - api_spec=api_spec, - handler=handler, - requirements=requirements, - conda_packages=conda_packages, - force=force, - ) - - # CORTEX_VERSION_MINOR - def deploy_batch_api( - self, - api_spec: Dict[str, Any], - handler, - requirements: Optional[List] = None, - conda_packages: Optional[List] = None, - ) -> Dict: - """ - Deploy a Batch API. - - Args: - api_spec: A dictionary defining a single Cortex API. See https://docs.cortex.dev/v/master/workloads/batch-apis/configuration for schema. - handler: A Cortex Handler class implementation. - requirements: A list of PyPI dependencies that will be installed before the handler class implementation is invoked. - conda_packages: A list of Conda dependencies that will be installed before the handler class implementation is invoked. - - Returns: - Deployment status, API specification, and endpoint for each API. - """ - - kind = api_spec.get("kind") - if kind != "BatchAPI": - raise InvalidKindForMethod( - f"expected an api_spec with kind 'BatchAPI', got kind '{kind}' instead" - ) - - return self._create_api( - api_spec=api_spec, - handler=handler, - requirements=requirements, - conda_packages=conda_packages, - ) - - # CORTEX_VERSION_MINOR - def deploy_task_api( - self, - api_spec: Dict[str, Any], - task, - requirements: Optional[List] = None, - conda_packages: Optional[List] = None, - ) -> Dict: - """ - Deploy a Task API. - - Args: - api_spec: A dictionary defining a single Cortex API. See https://docs.cortex.dev/v/master/workloads/task-apis/configuration for schema. - task: A callable class implementation. - requirements: A list of PyPI dependencies that will be installed before the handler class implementation is invoked. - conda_packages: A list of Conda dependencies that will be installed before the handler class implementation is invoked. - - Returns: - Deployment status, API specification, and endpoint for each API. - """ - kind = api_spec.get("kind") - if kind != "TaskAPI": - raise InvalidKindForMethod( - f"expected an api_spec with kind 'TaskAPI', got kind '{kind}' instead" - ) - - return self._create_api( - api_spec=api_spec, - task=task, - requirements=requirements, - conda_packages=conda_packages, - ) - # CORTEX_VERSION_MINOR - def deploy_traffic_splitter( - self, - api_spec: Dict[str, Any], - ) -> Dict: - """ - Deploy a Task API. + temp_deploy_dir = util.cli_config_dir() / "deployments" / api_spec["name"] + if temp_deploy_dir.exists(): + shutil.rmtree(str(temp_deploy_dir)) + temp_deploy_dir.mkdir(parents=True) - Args: - api_spec: A dictionary defining a single Cortex API. See https://docs.cortex.dev/v/master/workloads/realtime-apis/traffic-splitter/configuration for schema. + cortex_yaml_path = os.path.join(temp_deploy_dir, "cortex.yaml") - Returns: - Deployment status, API specification, and endpoint for each API. - """ - kind = api_spec.get("kind") - if kind != "TrafficSplitter": - raise InvalidKindForMethod( - f"expected an api_spec with kind 'TrafficSplitter', got kind '{kind}' instead" - ) - - return self._create_api( - api_spec=api_spec, - ) + with util.open_temporarily(cortex_yaml_path, "w", delete_parent_if_empty=True) as f: + yaml.dump([api_spec], f) # write a list + return self.deploy_from_file(cortex_yaml_path, force=force, wait=wait) # CORTEX_VERSION_MINOR @sentry_wrapper - def _create_api( - self, - api_spec: Dict, - handler=None, - task=None, - requirements: Optional[List] = None, - conda_packages: Optional[List] = None, - project_dir: Optional[str] = None, - force: bool = True, - wait: bool = False, - ) -> Dict: - """ - Deploy an API. - - Args: - api_spec: A dictionary defining a single Cortex API. See https://docs.cortex.dev/v/master/ for schema. - handler: A Cortex Handler class implementation. Not required for TaskAPI/TrafficSplitter kinds. - task: A callable class/function implementation. Not required for RealtimeAPI/BatchAPI/TrafficSplitter kinds. - requirements: A list of PyPI dependencies that will be installed before the handler class implementation is invoked. - conda_packages: A list of Conda dependencies that will be installed before the handler class implementation is invoked. - project_dir: Path to a python project. - force: Override any in-progress api updates. - wait: Streams logs until the APIs are ready. - - Returns: - Deployment status, API specification, and endpoint for each API. - """ - - requirements = requirements if requirements is not None else [] - conda_packages = conda_packages if conda_packages is not None else [] - - if project_dir is not None: - if handler is not None: - raise ValueError( - "`handler` and `project_dir` parameters cannot be specified at the same time, please choose one" - ) - if task is not None: - raise ValueError( - "`task` and `project_dir` parameters cannot be specified at the same time, please choose one" - ) - - if project_dir is not None: - cortex_yaml_path = os.path.join(project_dir, f".cortex-{uuid.uuid4()}.yaml") - - with util.open_temporarily(cortex_yaml_path, "w") as f: - yaml.dump([api_spec], f) # write a list - return self._deploy(cortex_yaml_path, force, wait) - - api_kind = api_spec.get("kind") - if api_kind == "TrafficSplitter": - if handler: - raise ValueError(f"`handler` parameter cannot be specified for {api_kind} kind") - if task: - raise ValueError(f"`task` parameter cannot be specified for {api_kind} kind") - elif api_kind == "TaskAPI": - if handler: - raise ValueError(f"`handler` parameter cannnot be specified for {api_kind} kind") - if task is None: - raise ValueError(f"`task` parameter must be specified for {api_kind} kind") - elif api_kind in ["BatchAPI", "RealtimeAPI"]: - if not handler: - raise ValueError(f"`handler` parameter must be specified for {api_kind}") - if task: - raise ValueError(f"`task` parameter cannot be specified for {api_kind}") - else: - raise ValueError( - f"invalid {api_kind} kind, `api_spec` must have the `kind` field set to one of the following kinds: " - f"{['TrafficSplitter', 'TaskAPI', 'BatchAPI', 'RealtimeAPI']}" - ) - - if api_spec.get("name") is None: - raise ValueError("`api_spec` must have the `name` key set") - - project_dir = util.cli_config_dir() / "deployments" / api_spec["name"] - - if project_dir.exists(): - shutil.rmtree(str(project_dir)) - - project_dir.mkdir(parents=True) - - cortex_yaml_path = os.path.join(project_dir, "cortex.yaml") - - if api_kind == "TrafficSplitter": - # for deploying a traffic splitter - with open(cortex_yaml_path, "w") as f: - yaml.dump([api_spec], f) # write a list - return self._deploy(cortex_yaml_path, force=force, wait=wait) - - actual_version = ( - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ) - - if actual_version != EXPECTED_PYTHON_VERSION: - is_python_set = any( - conda_dep.startswith("python=") or "::python=" in conda_dep - for conda_dep in conda_packages - ) - - if not is_python_set: - conda_packages = [f"python={actual_version}", "pip=19.*"] + conda_packages - - if len(requirements) > 0: - with open(project_dir / "requirements.txt", "w") as requirements_file: - requirements_file.write("\n".join(requirements)) - - if len(conda_packages) > 0: - with open(project_dir / "conda-packages.txt", "w") as conda_file: - conda_file.write("\n".join(conda_packages)) - - if api_kind in ["BatchAPI", "RealtimeAPI"]: - if not inspect.isclass(handler): - raise ValueError("`handler` parameter must be a class definition") - - if api_spec.get("handler") is None: - raise ValueError("`api_spec` must have the `handler` section defined") - - if api_spec["handler"].get("type") is None: - raise ValueError( - "the `type` field in the `handler` section of the `api_spec` must be set (tensorflow or python)" - ) - - impl_rel_path = self._save_impl(handler, project_dir, "handler") - api_spec["handler"]["path"] = impl_rel_path - - if api_kind == "TaskAPI": - if not callable(task): - raise ValueError( - "`task` parameter must be a callable (e.g. a function definition or a class definition called " - "`Task` with a `__call__` method implemented " - ) - - impl_rel_path = self._save_impl(task, project_dir, "task") - if api_spec.get("definition") is None: - api_spec["definition"] = {} - api_spec["definition"]["path"] = impl_rel_path - - with open(cortex_yaml_path, "w") as f: - yaml.dump([api_spec], f) # write a list - return self._deploy(cortex_yaml_path, force=force, wait=wait) - - def _save_impl(self, impl, project_dir: Path, filename: str) -> str: - import __main__ as main - - is_interactive = not hasattr(main, "__file__") - - if is_interactive and impl.__module__ == "__main__": - # class is defined in a REPL (e.g. jupyter) - filename += ".pickle" - with open(project_dir / filename, "wb") as pickle_file: - dill.dump(impl, pickle_file) - return filename - - filename += ".py" - with open(project_dir / filename, "w") as f: - f.write(dill.source.importable(impl, source=True)) - return filename - - def _deploy( + def deploy_from_file( self, config_file: str, force: bool = False, wait: bool = False, ) -> Dict: """ - Deploy or update APIs specified in the config_file. + Deploy or update APIs specified in a configuration file. Args: - config_file: Local path to a yaml file defining Cortex APIs. + config_file: Local path to a yaml file defining Cortex API(s). See https://docs.cortex.dev/v/master/ for schema. force: Override any in-progress api updates. wait: Streams logs until the APIs are ready. @@ -541,29 +216,6 @@ def refresh(self, api_name: str, force: bool = False): run_cli(args, hide_output=True) - @sentry_wrapper - def patch(self, api_spec: Dict, force: bool = False) -> Dict: - """ - Update the api specification for an API that has already been deployed. - - Args: - api_spec: The new api specification to apply - force: Override an already in-progress API update. - """ - - cortex_yaml_file = ( - util.cli_config_dir() / "deployments" / f"cortex-{str(uuid.uuid4())}.yaml" - ) - with util.open_temporarily(cortex_yaml_file, "w") as f: - yaml.dump([api_spec], f) - args = ["patch", cortex_yaml_file, "--env", self.env_name, "-o", "json"] - - if force: - args.append("--force") - - output = run_cli(args, hide_output=True) - return json.loads(output.strip()) - @sentry_wrapper def delete(self, api_name: str, keep_cache: bool = False): """ diff --git a/python/client/cortex/consts.py b/python/client/cortex/consts.py index 316ec3f5b3..42c5ec9546 100644 --- a/python/client/cortex/consts.py +++ b/python/client/cortex/consts.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Change if PYTHONVERSION changes -EXPECTED_PYTHON_VERSION = "3.6.9" CORTEX_VERSION = "master" # CORTEX_VERSION CORTEX_TELEMETRY_SENTRY_DSN = "https://5cea3d2d67194d028f7191fcc6ebca14@sentry.io/1825326" diff --git a/python/client/cortex/exceptions.py b/python/client/cortex/exceptions.py index 16acded963..5505156a65 100644 --- a/python/client/cortex/exceptions.py +++ b/python/client/cortex/exceptions.py @@ -35,11 +35,3 @@ class NotFound(CortexException): """ pass - - -class InvalidKindForMethod(CortexException): - """ - Raise when the specified resource kind is not supported by the used python client method. - """ - - pass diff --git a/python/client/cortex/util.py b/python/client/cortex/util.py index 1db3993ad0..af344ac39b 100644 --- a/python/client/cortex/util.py +++ b/python/client/cortex/util.py @@ -26,8 +26,10 @@ def cli_config_dir() -> Path: @contextmanager -def open_temporarily(path, mode): - Path(path).parent.mkdir(parents=True, exist_ok=True) +def open_temporarily(path, mode, delete_parent_if_empty: bool = False): + parentDir = Path(path).parent + parentDir.mkdir(parents=True, exist_ok=True) + file = open(path, mode) try: @@ -35,6 +37,8 @@ def open_temporarily(path, mode): finally: file.close() os.remove(path) + if delete_parent_if_empty and len(os.listdir(str(parentDir))) == 0: + shutil.rmtree(str(parentDir)) @contextmanager diff --git a/test/apis/pytorch/text-generator/deploy_class.py b/test/apis/pytorch/text-generator/deploy_class.py deleted file mode 100644 index 312d55d21e..0000000000 --- a/test/apis/pytorch/text-generator/deploy_class.py +++ /dev/null @@ -1,40 +0,0 @@ -import cortex -import os -import requests - -dir_path = os.path.dirname(os.path.realpath(__file__)) - -cx = cortex.client() - -api_spec = { - "name": "text-generator", - "kind": "RealtimeAPI", -} - - -class Handler: - def __init__(self, config): - from transformers import pipeline - - self.model = pipeline(task="text-generation") - - def handle_post(self, payload): - return self.model(payload["text"])[0] - - -api = cx.deploy_realtime_api( - api_spec, - handler=Handler, - requirements=["torch", "transformers"], - wait=True, -) - -response = requests.post( - api["endpoint"], - json={"text": "machine learning is great because"}, -) - -print(response.status_code) -print(response.text) - -cx.delete(api_spec["name"]) From 5d21a40b6a5a4f84b561b39460bb68d275df71f6 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Mon, 24 May 2021 18:17:15 +0300 Subject: [PATCH 24/82] Add readiness/liveness probes to k8s CaaS resources (#2187) --- build/images.sh | 1 - cmd/proxy/main.go | 23 +- docs/clusters/management/create.md | 1 - images/downloader/Dockerfile | 26 -- manager/install.sh | 63 ---- manager/manifests/image-downloader-cpu.yaml | 60 ---- manager/manifests/image-downloader-gpu.yaml | 49 --- manager/manifests/image-downloader-inf.yaml | 54 ---- .../manifests/prometheus-monitoring.yaml.j2 | 4 +- pkg/consts/consts.go | 4 +- .../apis/batch/v1alpha1/batchjob_types.go | 5 + .../batch/v1alpha1/zz_generated.deepcopy.go | 7 + .../crd/bases/batch.cortex.dev_batchjobs.yaml | 117 +++++++ .../controllers/batch/batchjob_controller.go | 18 ++ .../batch/batchjob_controller_helpers.go | 130 ++++++-- pkg/lib/configreader/string.go | 7 + pkg/lib/configreader/string_ptr.go | 2 + pkg/lib/urls/urls.go | 11 +- pkg/operator/resources/asyncapi/api.go | 38 ++- pkg/operator/resources/asyncapi/k8s_specs.go | 29 +- pkg/operator/resources/job/batchapi/job.go | 1 + pkg/operator/resources/job/taskapi/job.go | 27 +- .../resources/job/taskapi/k8s_specs.go | 28 +- .../resources/realtimeapi/k8s_specs.go | 2 +- pkg/proxy/probe/encoding.go | 46 --- pkg/proxy/probe/encoding_test.go | 90 ------ pkg/types/clusterconfig/cluster_config.go | 11 - pkg/types/spec/errors.go | 39 ++- pkg/types/spec/validations.go | 295 ++++++++++++++---- pkg/types/userconfig/api.go | 100 +++++- pkg/types/userconfig/config_key.go | 25 +- pkg/workloads/configmap.go | 65 ++++ pkg/workloads/helpers.go | 107 ++++--- pkg/workloads/init.go | 89 +----- pkg/workloads/k8s.go | 207 ++++++++---- python/downloader/download.py | 78 ----- 36 files changed, 1049 insertions(+), 810 deletions(-) delete mode 100644 images/downloader/Dockerfile delete mode 100644 manager/manifests/image-downloader-cpu.yaml delete mode 100644 manager/manifests/image-downloader-gpu.yaml delete mode 100644 manager/manifests/image-downloader-inf.yaml delete mode 100644 pkg/proxy/probe/encoding.go delete mode 100644 pkg/proxy/probe/encoding_test.go create mode 100644 pkg/workloads/configmap.go delete mode 100644 python/downloader/download.py diff --git a/build/images.sh b/build/images.sh index 96a7d75088..dcd3b8f49c 100644 --- a/build/images.sh +++ b/build/images.sh @@ -27,7 +27,6 @@ api_images=( ) dev_images=( - "downloader" "manager" "proxy" "async-gateway" diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 9bc7cff7c5..e241ba3d2e 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -19,7 +19,6 @@ package main import ( "context" "flag" - "io/ioutil" "net/http" "os" "os/signal" @@ -49,7 +48,6 @@ func main() { userContainerPort int maxConcurrency int maxQueueLength int - probeDefPath string clusterConfigPath string ) @@ -59,7 +57,6 @@ func main() { flag.IntVar(&maxConcurrency, "max-concurrency", 0, "max concurrency allowed for user container") flag.IntVar(&maxQueueLength, "max-queue-length", 0, "max request queue length for user container") flag.StringVar(&clusterConfigPath, "cluster-config", "", "cluster config path") - flag.StringVar(&probeDefPath, "probe", "", "path to the desired probe json definition") flag.Parse() log := logging.GetLogger() @@ -119,23 +116,7 @@ func main() { ) promStats := proxy.NewPrometheusStatsReporter() - - var readinessProbe *probe.Probe - if probeDefPath != "" { - jsonProbe, err := ioutil.ReadFile(probeDefPath) - if err != nil { - log.Fatal(err) - } - - probeDef, err := probe.DecodeJSON(string(jsonProbe)) - if err != nil { - log.Fatal(err) - } - - readinessProbe = probe.NewProbe(probeDef, log) - } else { - readinessProbe = probe.NewDefaultProbe(target, log) - } + readinessProbe := probe.NewDefaultProbe(target, log) go func() { reportTicker := time.NewTicker(_reportInterval) @@ -165,7 +146,7 @@ func main() { servers := map[string]*http.Server{ "proxy": { - Addr: ":" + strconv.Itoa(userContainerPort), + Addr: ":" + strconv.Itoa(port), Handler: proxy.Handler(breaker, httpProxy), }, "admin": { diff --git a/docs/clusters/management/create.md b/docs/clusters/management/create.md index 5a79091710..c2f75e3bad 100644 --- a/docs/clusters/management/create.md +++ b/docs/clusters/management/create.md @@ -99,7 +99,6 @@ The docker images used by the cluster can also be overridden. They can be config image_operator: quay.io/cortexlabs/operator:master image_controller_manager: quay.io/cortexlabs/controller-manager:master image_manager: quay.io/cortexlabs/manager:master -image_downloader: quay.io/cortexlabs/downloader:master image_proxy: quay.io/cortexlabs/proxy:master image_async_gateway: quay.io/cortexlabs/async-gateway:master image_cluster_autoscaler: quay.io/cortexlabs/cluster-autoscaler:master diff --git a/images/downloader/Dockerfile b/images/downloader/Dockerfile deleted file mode 100644 index c49815a408..0000000000 --- a/images/downloader/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -FROM ubuntu:18.04 - -RUN apt-get update -qq && apt-get install -y -q \ - curl \ - python3.6 \ - python3.6-distutils \ - && apt-get clean -qq && rm -rf /var/lib/apt/lists/* && \ - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ - python3.6 get-pip.py && \ - pip install --upgrade pip && \ - rm -rf /root/.cache/pip* - -COPY python/serve/cortex_internal.requirements.txt /src/cortex/serve/cortex_internal.requirements.txt - -RUN pip install --no-cache-dir \ - -r /src/cortex/serve/cortex_internal.requirements.txt && \ - rm -rf /root/.cache/pip* - -COPY python/downloader /src/cortex/downloader -COPY python/serve/ /src/cortex/serve -ENV CORTEX_LOG_CONFIG_FILE /src/cortex/serve/log_config.yaml - -RUN pip install --no-deps /src/cortex/serve/ && \ - rm -rf /root/.cache/pip* - -ENTRYPOINT ["/usr/bin/python3.6", "/src/cortex/downloader/download.py"] diff --git a/manager/install.sh b/manager/install.sh index 215baa6377..a84d572eb6 100755 --- a/manager/install.sh +++ b/manager/install.sh @@ -34,8 +34,6 @@ function main() { function cluster_up() { create_eks - start_pre_download_images - echo -n "○ updating cluster configuration " setup_configmap echo "✓" @@ -76,8 +74,6 @@ function cluster_up() { validate_cortex - await_pre_download_images - echo -e "\ncortex is ready!" if [ "$CORTEX_OPERATOR_LOAD_BALANCER_SCHEME" == "internal" ]; then echo -e "\nnote: you will need to configure VPC Peering to connect to your cluster: https://docs.cortex.dev/v/${CORTEX_VERSION_MINOR}/" @@ -324,65 +320,6 @@ function setup_istio() { output_if_error istio-${ISTIO_VERSION}/bin/istioctl install -f /workspace/istio.yaml } -function start_pre_download_images() { - registry="quay.io/cortexlabs" - if [ -n "$CORTEX_DEV_DEFAULT_IMAGE_REGISTRY" ]; then - registry="$CORTEX_DEV_DEFAULT_IMAGE_REGISTRY" - fi - export CORTEX_IMAGE_PYTHON_HANDLER_CPU="${registry}/python-handler-cpu:${CORTEX_VERSION}" - export CORTEX_IMAGE_PYTHON_HANDLER_GPU="${registry}/python-handler-gpu:${CORTEX_VERSION}-cuda10.2-cudnn8" - export CORTEX_IMAGE_PYTHON_HANDLER_INF="${registry}/python-handler-inf:${CORTEX_VERSION}" - export CORTEX_IMAGE_TENSORFLOW_SERVING_CPU="${registry}/tensorflow-serving-cpu:${CORTEX_VERSION}" - export CORTEX_IMAGE_TENSORFLOW_SERVING_GPU="${registry}/tensorflow-serving-gpu:${CORTEX_VERSION}" - export CORTEX_IMAGE_TENSORFLOW_SERVING_INF="${registry}/tensorflow-serving-inf:${CORTEX_VERSION}" - export CORTEX_IMAGE_TENSORFLOW_HANDLER="${registry}/tensorflow-handler:${CORTEX_VERSION}" - - envsubst < manifests/image-downloader-cpu.yaml | kubectl apply -f - &>/dev/null - - has_gpu="false" - has_inf="false" - - cluster_config_len=$(cat /in/cluster_${CORTEX_CLUSTER_NAME}_${CORTEX_REGION}.yaml | yq -r .node_groups | yq -r length) - for idx in $(seq 0 $(($cluster_config_len-1))); do - ng_instance_type=$(cat /in/cluster_${CORTEX_CLUSTER_NAME}_${CORTEX_REGION}.yaml | yq -r .node_groups[$idx].instance_type) - if [[ "$ng_instance_type" == p* || "$ng_instance_type" == g* ]]; then - has_gpu="true" - fi - if [[ "$ng_instance_type" == inf* ]]; then - has_inf="true" - fi - done - - if [ "$has_gpu" == "true" ]; then - envsubst < manifests/image-downloader-gpu.yaml | kubectl apply -f - &>/dev/null - fi - - if [ "$has_inf" == "true" ]; then - envsubst < manifests/image-downloader-inf.yaml | kubectl apply -f - &>/dev/null - fi -} - -function await_pre_download_images() { - echo -n "○ downloading docker images " - printed_dot="false" - for ds_name in image-downloader-cpu image-downloader-gpu image-downloader-inf; do - if ! kubectl get daemonset $ds_name > /dev/null 2>&1; then - continue - fi - i=0 - until [ "$(kubectl get daemonset $ds_name -n=default -o 'jsonpath={.status.numberReady}')" == "$(kubectl get daemonset $ds_name -n=default -o 'jsonpath={.status.desiredNumberScheduled}')" ]; do - if [ $i -eq 120 ]; then break; fi # give up after 6 minutes - echo -n "." - printed_dot="true" - ((i=i+1)) - sleep 3 - done - kubectl -n=default delete --ignore-not-found=true daemonset $ds_name &>/dev/null - done - - if [ "$printed_dot" == "true" ]; then echo " ✓"; else echo "✓"; fi -} - function validate_cortex() { set +e diff --git a/manager/manifests/image-downloader-cpu.yaml b/manager/manifests/image-downloader-cpu.yaml deleted file mode 100644 index d8e6b36479..0000000000 --- a/manager/manifests/image-downloader-cpu.yaml +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: image-downloader-cpu - namespace: default -spec: - selector: - matchLabels: - name: image-downloader-cpu - template: - metadata: - labels: - name: image-downloader-cpu - spec: - nodeSelector: - workload: "true" - tolerations: - - key: workload - value: "true" - operator: Equal - effect: NoSchedule - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule - - key: aws.amazon.com/neuron - value: "true" - operator: Equal - effect: NoSchedule - terminationGracePeriodSeconds: 0 - containers: - - name: python-handler-cpu - image: $CORTEX_IMAGE_PYTHON_HANDLER_CPU - command: ["/bin/sh"] - args: ["-c", "sleep 1000000"] - - name: tensorflow-serving-cpu - image: $CORTEX_IMAGE_TENSORFLOW_SERVING_CPU - command: ["/bin/sh"] - args: ["-c", "sleep 1000000"] - - name: tensorflow-handler - image: $CORTEX_IMAGE_TENSORFLOW_HANDLER - command: ["/bin/sh"] - args: ["-c", "sleep 1000000"] - - name: downloader - image: $CORTEX_IMAGE_DOWNLOADER - command: ["/bin/sh"] - args: ["-c", "sleep 1000000"] diff --git a/manager/manifests/image-downloader-gpu.yaml b/manager/manifests/image-downloader-gpu.yaml deleted file mode 100644 index 5c5394a0c0..0000000000 --- a/manager/manifests/image-downloader-gpu.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: image-downloader-gpu - namespace: default -spec: - selector: - matchLabels: - name: image-downloader-gpu - template: - metadata: - labels: - name: image-downloader-gpu - spec: - nodeSelector: - workload: "true" - nvidia.com/gpu: "true" - tolerations: - - key: workload - value: "true" - operator: Equal - effect: NoSchedule - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule - terminationGracePeriodSeconds: 0 - containers: - - name: python-handler-gpu - image: $CORTEX_IMAGE_PYTHON_HANDLER_GPU - command: ["/bin/sh"] - args: ["-c", "sleep 1000000"] - - name: tensorflow-serving-gpu - image: $CORTEX_IMAGE_TENSORFLOW_SERVING_GPU - command: ["/bin/sh"] - args: ["-c", "sleep 1000000"] diff --git a/manager/manifests/image-downloader-inf.yaml b/manager/manifests/image-downloader-inf.yaml deleted file mode 100644 index b3635bb1d2..0000000000 --- a/manager/manifests/image-downloader-inf.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: image-downloader-inf - namespace: default -spec: - selector: - matchLabels: - name: image-downloader-inf - template: - metadata: - labels: - name: image-downloader-inf - spec: - nodeSelector: - workload: "true" - aws.amazon.com/neuron: "true" - tolerations: - - key: workload - value: "true" - operator: Equal - effect: NoSchedule - - key: aws.amazon.com/neuron - value: "true" - operator: Equal - effect: NoSchedule - terminationGracePeriodSeconds: 0 - containers: - - name: python-handler-inf - image: $CORTEX_IMAGE_PYTHON_HANDLER_INF - command: ["/bin/sh"] - args: ["-c", "sleep 1000000"] - - name: tensorflow-serving-inf - image: $CORTEX_IMAGE_TENSORFLOW_SERVING_INF - command: ["/bin/sh"] - args: ["-c", "sleep 1000000"] - - name: neuron-rtd - image: $CORTEX_IMAGE_NEURON_RTD - command: ["/bin/sh"] - args: ["-c", "sleep 1000000"] diff --git a/manager/manifests/prometheus-monitoring.yaml.j2 b/manager/manifests/prometheus-monitoring.yaml.j2 index cee1eae033..d7868c1755 100644 --- a/manager/manifests/prometheus-monitoring.yaml.j2 +++ b/manager/manifests/prometheus-monitoring.yaml.j2 @@ -184,7 +184,7 @@ spec: - path: /metrics scheme: http interval: 10s - port: metrics + port: admin relabelings: - action: keep sourceLabels: [ __meta_kubernetes_pod_container_name ] @@ -221,7 +221,7 @@ metadata: spec: jobLabel: "statsd-exporter" podMetricsEndpoints: - - port: metrics + - port: admin scheme: http path: /metrics interval: 20s diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 50fca29b6c..66bb92b2e3 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -33,8 +33,8 @@ var ( ProxyListeningPortStr = "8888" ProxyListeningPortInt32 = int32(8888) - MetricsPortStr = "15000" - MetricsPortInt32 = int32(15000) + AdminPortStr = "15000" + AdminPortInt32 = int32(15000) AuthHeader = "X-Cortex-Authorization" diff --git a/pkg/crds/apis/batch/v1alpha1/batchjob_types.go b/pkg/crds/apis/batch/v1alpha1/batchjob_types.go index a3b7d366f8..99bd7116a4 100644 --- a/pkg/crds/apis/batch/v1alpha1/batchjob_types.go +++ b/pkg/crds/apis/batch/v1alpha1/batchjob_types.go @@ -58,6 +58,11 @@ type BatchJobSpec struct { // Node groups selector NodeGroups []string `json:"node_groups"` + // +kubebuilder:validation:Optional + // +nullable + // Readiness probes for the job (container name -> probe) + Probes map[string]kcore.Probe `json:"probes"` + // +kubebuilder:validation:Optional // Time to live for the resource. The controller will clean-up resources // that reached a final state when the TTL time is exceeded. diff --git a/pkg/crds/apis/batch/v1alpha1/zz_generated.deepcopy.go b/pkg/crds/apis/batch/v1alpha1/zz_generated.deepcopy.go index b71db11653..61f4bbb641 100644 --- a/pkg/crds/apis/batch/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/crds/apis/batch/v1alpha1/zz_generated.deepcopy.go @@ -114,6 +114,13 @@ func (in *BatchJobSpec) DeepCopyInto(out *BatchJobSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.Probes != nil { + in, out := &in.Probes, &out.Probes + *out = make(map[string]corev1.Probe, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } if in.TTL != nil { in, out := &in.TTL, &out.TTL *out = new(v1.Duration) diff --git a/pkg/crds/config/crd/bases/batch.cortex.dev_batchjobs.yaml b/pkg/crds/config/crd/bases/batch.cortex.dev_batchjobs.yaml index 63588aa6db..63b1987bd9 100644 --- a/pkg/crds/config/crd/bases/batch.cortex.dev_batchjobs.yaml +++ b/pkg/crds/config/crd/bases/batch.cortex.dev_batchjobs.yaml @@ -73,6 +73,123 @@ spec: type: string nullable: true type: array + probes: + additionalProperties: + description: Probe describes a health check to be performed against + a container to determine whether it is alive or ready to receive + traffic. + properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit status + of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be + considered failed after having succeeded. Defaults to 3. Minimum + value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod + IP. You probably want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to be + used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults + to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. Default + to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + description: Readiness probes for the job (container name -> probe) + nullable: true + type: object resources: description: Compute resource requirements properties: diff --git a/pkg/crds/controllers/batch/batchjob_controller.go b/pkg/crds/controllers/batch/batchjob_controller.go index adb643a3cd..edd656c523 100644 --- a/pkg/crds/controllers/batch/batchjob_controller.go +++ b/pkg/crds/controllers/batch/batchjob_controller.go @@ -57,6 +57,7 @@ type BatchJobReconciler struct { // +kubebuilder:rbac:groups=batch.cortex.dev,resources=batchjobs/finalizers,verbs=update // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=create;get;list;watch // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -137,6 +138,13 @@ func (r *BatchJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c return ctrl.Result{}, err } + log.V(1).Info("getting configmap") + configMap, err := r.getConfigMap(ctx, batchJob) + if err != nil && !kerrors.IsNotFound(err) { + log.Error(err, "failed to get configmap") + return ctrl.Result{}, err + } + log.V(1).Info("getting worker job") workerJob, err := r.getWorkerJob(ctx, batchJob) if err != nil && !kerrors.IsNotFound(err) { @@ -157,6 +165,7 @@ func (r *BatchJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c } workerJobExists := workerJob != nil + configMapExists := configMap != nil statusInfo := batchJobStatusInfo{ QueueExists: queueExists, EnqueuingStatus: enqueuingStatus, @@ -167,6 +176,7 @@ func (r *BatchJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c log.V(1).Info("status data successfully acquired", "queueExists", queueExists, + "configMapExists", configMapExists, "enqueuingStatus", enqueuingStatus, "workerJobExists", workerJobExists, ) @@ -230,6 +240,14 @@ func (r *BatchJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c case batch.EnqueuingFailed: log.Info("failed to enqueue payload") case batch.EnqueuingDone: + if !configMapExists { + log.V(1).Info("creating worker configmap") + if err = r.createWorkerConfigMap(ctx, batchJob, queueURL); err != nil { + log.Error(err, "failed to create worker configmap") + return ctrl.Result{}, err + } + + } if !workerJobExists { log.V(1).Info("creating worker job") if err = r.createWorkerJob(ctx, batchJob, queueURL); err != nil { diff --git a/pkg/crds/controllers/batch/batchjob_controller_helpers.go b/pkg/crds/controllers/batch/batchjob_controller_helpers.go index 0f5b01c88b..70815b8a15 100644 --- a/pkg/crds/controllers/batch/batchjob_controller_helpers.go +++ b/pkg/crds/controllers/batch/batchjob_controller_helpers.go @@ -56,10 +56,11 @@ const ( _cacheDuration = 60 * time.Second ) -var totalBatchCountCache *cache.Cache +var totalBatchCountCache, apiSpecCache *cache.Cache func init() { totalBatchCountCache = cache.New(_cacheDuration, _cacheDuration) + apiSpecCache = cache.New(_cacheDuration, _cacheDuration) } type batchJobStatusInfo struct { @@ -70,6 +71,22 @@ type batchJobStatusInfo struct { TotalBatchCount int } +func (r *BatchJobReconciler) getConfigMap(ctx context.Context, batchJob batch.BatchJob) (*kcore.ConfigMap, error) { + var configMap kcore.ConfigMap + err := r.Get(ctx, client.ObjectKey{ + Namespace: batchJob.Namespace, + Name: batchJob.Spec.APIName + "-" + batchJob.Name, + }, &configMap) + if err != nil { + if kerrors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + + return &configMap, nil +} + func (r *BatchJobReconciler) checkIfQueueExists(batchJob batch.BatchJob) (bool, error) { queueName := r.getQueueName(batchJob) input := &sqs.GetQueueUrlInput{ @@ -185,8 +202,41 @@ func (r *BatchJobReconciler) enqueuePayload(ctx context.Context, batchJob batch. return nil } +func (r *BatchJobReconciler) createWorkerConfigMap(ctx context.Context, batchJob batch.BatchJob, queueURL string) error { + apiSpec, err := r.getAPISpec(batchJob) + if err != nil { + return errors.Wrap(err, "failed to get API spec") + } + + jobSpec, err := r.ConvertControllerBatchToJobSpec(batchJob, *apiSpec, queueURL) + if err != nil { + return errors.Wrap(err, "failed to convert controller batch job to operator batch job") + } + + configMapConfig := workloads.ConfigMapConfig{ + BatchJob: &jobSpec, + Probes: batchJob.Spec.Probes, + } + + configMapData, err := configMapConfig.GenerateConfigMapData() + if err != nil { + return errors.Wrap(err, "failed to generate config map data") + } + + configMap, err := r.desiredConfigMap(batchJob, configMapData) + if err != nil { + return errors.Wrap(err, "failed to get desired configmap spec") + } + + if err := r.Create(ctx, configMap); err != nil { + return err + } + + return nil +} + func (r *BatchJobReconciler) createWorkerJob(ctx context.Context, batchJob batch.BatchJob, queueURL string) error { - apiSpec, err := r.getAPISpec(batchJob) // TODO: should be cached + apiSpec, err := r.getAPISpec(batchJob) if err != nil { return errors.Wrap(err, "failed to get API spec") } @@ -272,8 +322,11 @@ func (r *BatchJobReconciler) desiredWorkerJob(batchJob batch.BatchJob, apiSpec s var containers []kcore.Container var volumes []kcore.Volume - containers, volumes = workloads.UserPodContainers(apiSpec) + containers, volumes = workloads.BatchUserPodContainers(apiSpec, &jobSpec.JobKey) + // TODO add the proxy as well + // use workloads.APIConfigMount(batchJob.Spec.APIName + "-" + batchJob.Name) to mount the probes + // the probes will be made available at /cortex/spec/probes.json job := k8s.Job( &k8s.JobSpec{ @@ -308,7 +361,6 @@ func (r *BatchJobReconciler) desiredWorkerJob(batchJob batch.BatchJob, apiSpec s K8sPodSpec: kcore.PodSpec{ InitContainers: []kcore.Container{ workloads.KubexitInitContainer(), - workloads.BatchInitContainer(&jobSpec), }, Containers: containers, Volumes: volumes, @@ -333,20 +385,50 @@ func (r *BatchJobReconciler) desiredWorkerJob(batchJob batch.BatchJob, apiSpec s return job, nil } +func (r *BatchJobReconciler) desiredConfigMap(batchJob batch.BatchJob, data map[string]string) (*kcore.ConfigMap, error) { + configMap := k8s.ConfigMap(&k8s.ConfigMapSpec{ + Name: batchJob.Spec.APIName + "-" + batchJob.Name, + Data: data, + Labels: map[string]string{ + "apiKind": userconfig.BatchAPIKind.String(), + "apiName": batchJob.Spec.APIName, + "apiID": batchJob.Spec.APIID, + "jobID": batchJob.Name, + "cortex.dev/api": "true", + "cortex.dev/batch": "config", + }, + }) + configMap.Namespace = batchJob.Namespace + + if err := ctrl.SetControllerReference(&batchJob, configMap, r.Scheme); err != nil { + return nil, err + } + + return configMap, nil +} + func (r *BatchJobReconciler) getAPISpec(batchJob batch.BatchJob) (*spec.API, error) { apiSpecKey := spec.Key(batchJob.Spec.APIName, batchJob.Spec.APIID, r.ClusterConfig.ClusterUID) + cachedAPISpec, found := apiSpecCache.Get(apiSpecKey) + + var apiSpec spec.API + if found { + apiSpec = cachedAPISpec.(spec.API) + return &apiSpec, nil + } apiSpecBytes, err := r.AWS.ReadBytesFromS3(r.ClusterConfig.Bucket, apiSpecKey) if err != nil { return nil, err } - apiSpec := &spec.API{} - if err := json.Unmarshal(apiSpecBytes, apiSpec); err != nil { + if err := json.Unmarshal(apiSpecBytes, &apiSpec); err != nil { return nil, err } - return apiSpec, nil + apiSpecCache.Set(apiSpecKey, apiSpec, _cacheDuration) + + return &apiSpec, nil } func (r *BatchJobReconciler) getWorkerJob(ctx context.Context, batchJob batch.BatchJob) (*kbatch.Job, error) { @@ -472,6 +554,19 @@ func (r *BatchJobReconciler) deleteSQSQueue(batchJob batch.BatchJob) error { } func (r *BatchJobReconciler) uploadJobSpec(batchJob batch.BatchJob, api spec.API, queueURL string) (*spec.BatchJob, error) { + jobSpec, err := r.ConvertControllerBatchToJobSpec(batchJob, api, queueURL) + if err != nil { + return nil, err + } + + if err = r.AWS.UploadJSONToS3(&jobSpec, r.ClusterConfig.Bucket, r.jobSpecKey(batchJob)); err != nil { + return nil, err + } + + return &jobSpec, nil +} + +func (r *BatchJobReconciler) ConvertControllerBatchToJobSpec(batchJob batch.BatchJob, api spec.API, queueURL string) (spec.BatchJob, error) { var deadLetterQueue *spec.SQSDeadLetterQueue if batchJob.Spec.DeadLetterQueue != nil { deadLetterQueue = &spec.SQSDeadLetterQueue{ @@ -482,8 +577,9 @@ func (r *BatchJobReconciler) uploadJobSpec(batchJob batch.BatchJob, api spec.API var config map[string]interface{} if batchJob.Spec.Config != nil { - if err := yaml.Unmarshal([]byte(*batchJob.Spec.Config), &config); err != nil { - return nil, err + err := yaml.Unmarshal([]byte(*batchJob.Spec.Config), &config) + if err != nil { + return spec.BatchJob{}, errors.Wrap(err, "failed to unmarshal job spec config") } } @@ -494,32 +590,26 @@ func (r *BatchJobReconciler) uploadJobSpec(batchJob batch.BatchJob, api spec.API totalBatchCount, err := r.Config.GetTotalBatchCount(r, batchJob) if err != nil { - return nil, err + return spec.BatchJob{}, errors.Wrap(err, "failed to get total batch count") } - jobSpec := spec.BatchJob{ + return spec.BatchJob{ JobKey: spec.JobKey{ - ID: batchJob.Name, + ID: batchJob.Status.ID, APIName: batchJob.Spec.APIName, Kind: userconfig.BatchAPIKind, }, RuntimeBatchJobConfig: spec.RuntimeBatchJobConfig{ Workers: int(batchJob.Spec.Workers), SQSDeadLetterQueue: deadLetterQueue, - Timeout: timeout, Config: config, + Timeout: timeout, }, APIID: api.ID, SQSUrl: queueURL, StartTime: batchJob.CreationTimestamp.Time, TotalBatchCount: totalBatchCount, - } - - if err = r.AWS.UploadJSONToS3(&jobSpec, r.ClusterConfig.Bucket, r.jobSpecKey(batchJob)); err != nil { - return nil, err - } - - return &jobSpec, nil + }, nil } func (r *BatchJobReconciler) jobSpecKey(batchJob batch.BatchJob) string { diff --git a/pkg/lib/configreader/string.go b/pkg/lib/configreader/string.go index a178c1925a..63ab85a5dc 100644 --- a/pkg/lib/configreader/string.go +++ b/pkg/lib/configreader/string.go @@ -62,6 +62,7 @@ type StringValidation struct { CastScalar bool AllowCortexResources bool RequireCortexResources bool + DockerImage bool DockerImageOrEmpty bool Validator func(string) (string, error) } @@ -351,6 +352,12 @@ func ValidateStringVal(val string, v *StringValidation) error { } } + if v.DockerImage { + if !regex.IsValidDockerImage(val) { + return ErrorInvalidDockerImage(val) + } + } + if v.DockerImageOrEmpty { if !regex.IsValidDockerImage(val) && val != "" { return ErrorInvalidDockerImage(val) diff --git a/pkg/lib/configreader/string_ptr.go b/pkg/lib/configreader/string_ptr.go index b3670e1c23..7e35b689cc 100644 --- a/pkg/lib/configreader/string_ptr.go +++ b/pkg/lib/configreader/string_ptr.go @@ -54,6 +54,7 @@ type StringPtrValidation struct { CastScalar bool AllowCortexResources bool RequireCortexResources bool + DockerImage bool DockerImageOrEmpty bool Validator func(string) (string, error) } @@ -85,6 +86,7 @@ func makeStringValValidation(v *StringPtrValidation) *StringValidation { CastScalar: v.CastScalar, AllowCortexResources: v.AllowCortexResources, RequireCortexResources: v.RequireCortexResources, + DockerImage: v.DockerImage, DockerImageOrEmpty: v.DockerImageOrEmpty, } } diff --git a/pkg/lib/urls/urls.go b/pkg/lib/urls/urls.go index eeb387aac6..7a6b073338 100644 --- a/pkg/lib/urls/urls.go +++ b/pkg/lib/urls/urls.go @@ -62,7 +62,7 @@ func CheckDNS1123(str string) error { return nil } -func ValidateEndpoint(str string) (string, error) { +func ValidateEndpointAllowEmptyPath(str string) (string, error) { if !_endpointRegex.MatchString(str) { return "", ErrorEndpoint(str) } @@ -71,7 +71,14 @@ func ValidateEndpoint(str string) (string, error) { return "", ErrorEndpointDoubleSlash(str) } - path := CanonicalizeEndpoint(str) + return CanonicalizeEndpoint(str), nil +} + +func ValidateEndpoint(str string) (string, error) { + path, err := ValidateEndpointAllowEmptyPath(str) + if err != nil { + return "", err + } if path == "/" { return "", ErrorEndpointEmptyPath() diff --git a/pkg/operator/resources/asyncapi/api.go b/pkg/operator/resources/asyncapi/api.go index 8ce8abcae8..b71cd2387e 100644 --- a/pkg/operator/resources/asyncapi/api.go +++ b/pkg/operator/resources/asyncapi/api.go @@ -51,6 +51,7 @@ var ( type resources struct { apiDeployment *kapps.Deployment + apiConfigMap *kcore.ConfigMap gatewayDeployment *kapps.Deployment gatewayService *kcore.Service gatewayHPA *kautoscaling.HorizontalPodAutoscaler @@ -288,6 +289,7 @@ func UpdateAutoscalerCron(deployment *kapps.Deployment, apiSpec spec.API) error func getK8sResources(apiConfig userconfig.API) (resources, error) { var deployment *kapps.Deployment + var apiConfigMap *kcore.ConfigMap var gatewayDeployment *kapps.Deployment var gatewayService *kcore.Service var gatewayHPA *kautoscaling.HorizontalPodAutoscaler @@ -302,6 +304,11 @@ func getK8sResources(apiConfig userconfig.API) (resources, error) { deployment, err = config.K8s.GetDeployment(apiK8sName) return err }, + func() error { + var err error + apiConfigMap, err = config.K8s.GetConfigMap(apiK8sName) + return err + }, func() error { var err error gatewayDeployment, err = config.K8s.GetDeployment(gatewayK8sName) @@ -326,6 +333,7 @@ func getK8sResources(apiConfig userconfig.API) (resources, error) { return resources{ apiDeployment: deployment, + apiConfigMap: apiConfigMap, gatewayDeployment: gatewayDeployment, gatewayService: gatewayService, gatewayHPA: gatewayHPA, @@ -335,6 +343,10 @@ func getK8sResources(apiConfig userconfig.API) (resources, error) { func applyK8sResources(api spec.API, prevK8sResources resources, queueURL string) error { apiDeployment := deploymentSpec(api, prevK8sResources.apiDeployment, queueURL) + apiConfigMap, err := configMapSpec(api) + if err != nil { + return err + } gatewayDeployment := gatewayDeploymentSpec(api, prevK8sResources.gatewayDeployment, queueURL) gatewayHPA, err := gatewayHPASpec(api) if err != nil { @@ -345,8 +357,11 @@ func applyK8sResources(api spec.API, prevK8sResources resources, queueURL string return parallel.RunFirstErr( func() error { - err := applyK8sDeployment(prevK8sResources.apiDeployment, &apiDeployment) - if err != nil { + if err := applyK8sConfigMap(prevK8sResources.apiConfigMap, &apiConfigMap); err != nil { + return err + } + + if err := applyK8sDeployment(prevK8sResources.apiDeployment, &apiDeployment); err != nil { return err } @@ -375,6 +390,21 @@ func applyK8sResources(api spec.API, prevK8sResources resources, queueURL string ) } +func applyK8sConfigMap(prevConfigMap *kcore.ConfigMap, newConfigMap *kcore.ConfigMap) error { + if prevConfigMap == nil { + _, err := config.K8s.CreateConfigMap(newConfigMap) + if err != nil { + return err + } + } else { + _, err := config.K8s.UpdateConfigMap(newConfigMap) + if err != nil { + return err + } + } + return nil +} + func applyK8sDeployment(prevDeployment *kapps.Deployment, newDeployment *kapps.Deployment) error { if prevDeployment == nil { _, err := config.K8s.CreateDeployment(newDeployment) @@ -453,6 +483,10 @@ func deleteK8sResources(apiName string) error { _, err := config.K8s.DeleteDeployment(apiK8sName) return err }, + func() error { + _, err := config.K8s.DeleteConfigMap(apiK8sName) + return err + }, func() error { _, err := config.K8s.DeleteDeployment(gatewayK8sName) return err diff --git a/pkg/operator/resources/asyncapi/k8s_specs.go b/pkg/operator/resources/asyncapi/k8s_specs.go index fa27daf885..79fad9e7e6 100644 --- a/pkg/operator/resources/asyncapi/k8s_specs.go +++ b/pkg/operator/resources/asyncapi/k8s_specs.go @@ -171,14 +171,41 @@ func gatewayVirtualServiceSpec(api spec.API) v1beta1.VirtualService { }) } +func configMapSpec(api spec.API) (kcore.ConfigMap, error) { + configMapConfig := workloads.ConfigMapConfig{ + Probes: workloads.GetReadinessProbesFromContainers(api.Pod.Containers), + } + + configMapData, err := configMapConfig.GenerateConfigMapData() + if err != nil { + return kcore.ConfigMap{}, err + } + + return *k8s.ConfigMap(&k8s.ConfigMapSpec{ + Name: workloads.K8sName(api.Name), + Data: configMapData, + Labels: map[string]string{ + "apiName": api.Name, + "apiKind": api.Kind.String(), + "apiID": api.ID, + "specID": api.SpecID, + "deploymentID": api.DeploymentID, + "cortex.dev/api": "true", + }, + }), nil +} + func deploymentSpec(api spec.API, prevDeployment *kapps.Deployment, queueURL string) kapps.Deployment { var ( containers []kcore.Container volumes []kcore.Volume ) - containers, volumes = workloads.UserPodContainers(api) + containers, volumes = workloads.AsyncUserPodContainers(api) + // TODO add the proxy as well + // use workloads.APIConfigMount(workloads.K8sName(api.Name)) to mount the probes + // the probes will be made available at /cortex/spec/probes.json return *k8s.Deployment(&k8s.DeploymentSpec{ Name: workloads.K8sName(api.Name), diff --git a/pkg/operator/resources/job/batchapi/job.go b/pkg/operator/resources/job/batchapi/job.go index 481edd9562..dbf74deec0 100644 --- a/pkg/operator/resources/job/batchapi/job.go +++ b/pkg/operator/resources/job/batchapi/job.go @@ -146,6 +146,7 @@ func SubmitJob(apiName string, submission *schema.BatchJobSubmission) (*spec.Bat DeadLetterQueue: deadLetterQueue, TTL: &kmeta.Duration{Duration: _batchJobTTL}, NodeGroups: apiSpec.Pod.NodeGroups, + Probes: workloads.GetReadinessProbesFromContainers(apiSpec.Pod.Containers), }, } diff --git a/pkg/operator/resources/job/taskapi/job.go b/pkg/operator/resources/job/taskapi/job.go index 19260bc0ad..f95ce04b75 100644 --- a/pkg/operator/resources/job/taskapi/job.go +++ b/pkg/operator/resources/job/taskapi/job.go @@ -69,6 +69,10 @@ func SubmitJob(apiName string, submission *schema.TaskJobSubmission) (*spec.Task return nil, err } + if err := createJobConfigMap(*apiSpec, jobSpec); err != nil { + return nil, err + } + deployJob(apiSpec, &jobSpec) return &jobSpec, nil @@ -84,6 +88,19 @@ func uploadJobSpec(jobSpec *spec.TaskJob) error { return nil } +func createJobConfigMap(apiSpec spec.API, jobSpec spec.TaskJob) error { + configMapConfig := workloads.ConfigMapConfig{ + TaskJob: &jobSpec, + } + + configMapData, err := configMapConfig.GenerateConfigMapData() + if err != nil { + return err + } + + return createK8sConfigMap(k8sConfigMap(apiSpec, jobSpec, configMapData)) +} + func deployJob(apiSpec *spec.API, jobSpec *spec.TaskJob) { err := createK8sJob(apiSpec, jobSpec) if err != nil { @@ -115,12 +132,10 @@ func handleJobSubmissionError(jobKey spec.JobKey, jobErr error) { } func deleteJobRuntimeResources(jobKey spec.JobKey) error { - err := deleteK8sJob(jobKey) - if err != nil { - return err - } - - return nil + return errors.FirstError( + deleteK8sJob(jobKey), + deleteK8sConfigMap(jobKey), + ) } func StopJob(jobKey spec.JobKey) error { diff --git a/pkg/operator/resources/job/taskapi/k8s_specs.go b/pkg/operator/resources/job/taskapi/k8s_specs.go index 8ccd530c85..554506472e 100644 --- a/pkg/operator/resources/job/taskapi/k8s_specs.go +++ b/pkg/operator/resources/job/taskapi/k8s_specs.go @@ -60,7 +60,7 @@ func virtualServiceSpec(api *spec.API) *istioclientnetworking.VirtualService { } func k8sJobSpec(api *spec.API, job *spec.TaskJob) *kbatch.Job { - containers, volumes := workloads.UserPodContainers(*api) + containers, volumes := workloads.TaskUserPodContainers(*api, &job.JobKey) return k8s.Job(&k8s.JobSpec{ Name: job.JobKey.K8sName(), @@ -90,7 +90,6 @@ func k8sJobSpec(api *spec.API, job *spec.TaskJob) *kbatch.Job { RestartPolicy: "Never", InitContainers: []kcore.Container{ workloads.KubexitInitContainer(), - workloads.TaskInitContainer(job), }, Containers: containers, NodeSelector: workloads.NodeSelectors(), @@ -103,6 +102,21 @@ func k8sJobSpec(api *spec.API, job *spec.TaskJob) *kbatch.Job { }) } +func k8sConfigMap(api spec.API, job spec.TaskJob, configMapData map[string]string) kcore.ConfigMap { + return *k8s.ConfigMap(&k8s.ConfigMapSpec{ + Name: job.JobKey.K8sName(), + Data: configMapData, + Labels: map[string]string{ + "apiName": api.Name, + "apiID": api.ID, + "specID": api.SpecID, + "jobID": job.ID, + "apiKind": api.Kind.String(), + "cortex.dev/api": "true", + }, + }) +} + func applyK8sResources(api *spec.API, prevVirtualService *istioclientnetworking.VirtualService) error { newVirtualService := virtualServiceSpec(api) @@ -156,3 +170,13 @@ func createK8sJob(apiSpec *spec.API, jobSpec *spec.TaskJob) error { return nil } + +func deleteK8sConfigMap(jobKey spec.JobKey) error { + _, err := config.K8s.DeleteConfigMap(jobKey.K8sName()) + return err +} + +func createK8sConfigMap(configMap kcore.ConfigMap) error { + _, err := config.K8s.CreateConfigMap(&configMap) + return err +} diff --git a/pkg/operator/resources/realtimeapi/k8s_specs.go b/pkg/operator/resources/realtimeapi/k8s_specs.go index 49368f7db2..903268f9f1 100644 --- a/pkg/operator/resources/realtimeapi/k8s_specs.go +++ b/pkg/operator/resources/realtimeapi/k8s_specs.go @@ -30,7 +30,7 @@ import ( var _terminationGracePeriodSeconds int64 = 60 // seconds func deploymentSpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Deployment { - containers, volumes := workloads.UserPodContainers(*api) + containers, volumes := workloads.RealtimeUserPodContainers(*api) proxyContainer, proxyVolume := workloads.RealtimeProxyContainer(*api) containers = append(containers, proxyContainer) diff --git a/pkg/proxy/probe/encoding.go b/pkg/proxy/probe/encoding.go deleted file mode 100644 index 38363d1dc4..0000000000 --- a/pkg/proxy/probe/encoding.go +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright 2021 Cortex Labs, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package probe - -import ( - "encoding/json" - "errors" - - kcore "k8s.io/api/core/v1" -) - -// DecodeJSON takes a json serialised *kcore.Probe and returns a Probe or an error. -func DecodeJSON(jsonProbe string) (*kcore.Probe, error) { - pb := &kcore.Probe{} - if err := json.Unmarshal([]byte(jsonProbe), pb); err != nil { - return nil, err - } - return pb, nil -} - -// EncodeJSON takes *kcore.Probe object and returns marshalled Probe JSON string and an error. -func EncodeJSON(pb *kcore.Probe) (string, error) { - if pb == nil { - return "", errors.New("cannot encode nil probe") - } - - probeJSON, err := json.Marshal(pb) - if err != nil { - return "", err - } - return string(probeJSON), nil -} diff --git a/pkg/proxy/probe/encoding_test.go b/pkg/proxy/probe/encoding_test.go deleted file mode 100644 index e48aa62165..0000000000 --- a/pkg/proxy/probe/encoding_test.go +++ /dev/null @@ -1,90 +0,0 @@ -/* -Copyright 2021 Cortex Labs, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package probe_test - -import ( - "encoding/json" - "testing" - - "github.com/cortexlabs/cortex/pkg/proxy/probe" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - kcore "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -func TestDecodeProbeSuccess(t *testing.T) { - t.Parallel() - - expectedProbe := &kcore.Probe{ - PeriodSeconds: 1, - TimeoutSeconds: 2, - SuccessThreshold: 1, - FailureThreshold: 1, - Handler: kcore.Handler{ - TCPSocket: &kcore.TCPSocketAction{ - Host: "127.0.0.1", - Port: intstr.FromString("8080"), - }, - }, - } - probeBytes, err := json.Marshal(expectedProbe) - require.NoError(t, err) - - gotProbe, err := probe.DecodeJSON(string(probeBytes)) - require.NoError(t, err) - - require.Equal(t, expectedProbe, gotProbe) -} - -func TestDecodeProbeFailure(t *testing.T) { - t.Parallel() - - probeBytes, err := json.Marshal("blah") - require.NoError(t, err) - - _, err = probe.DecodeJSON(string(probeBytes)) - require.Error(t, err) -} - -func TestEncodeProbe(t *testing.T) { - t.Parallel() - - pb := &kcore.Probe{ - SuccessThreshold: 1, - Handler: kcore.Handler{ - TCPSocket: &kcore.TCPSocketAction{ - Host: "127.0.0.1", - Port: intstr.FromString("8080"), - }, - }, - } - - jsonProbe, err := probe.EncodeJSON(pb) - require.NoError(t, err) - - wantProbe := `{"tcpSocket":{"port":"8080","host":"127.0.0.1"},"successThreshold":1}` - require.Equal(t, wantProbe, jsonProbe) -} - -func TestEncodeNilProbe(t *testing.T) { - t.Parallel() - - jsonProbe, err := probe.EncodeJSON(nil) - assert.Error(t, err) - assert.Empty(t, jsonProbe) -} diff --git a/pkg/types/clusterconfig/cluster_config.go b/pkg/types/clusterconfig/cluster_config.go index ce36dd5eef..ab1e44259e 100644 --- a/pkg/types/clusterconfig/cluster_config.go +++ b/pkg/types/clusterconfig/cluster_config.go @@ -92,7 +92,6 @@ type CoreConfig struct { ImageOperator string `json:"image_operator" yaml:"image_operator"` ImageControllerManager string `json:"image_controller_manager" yaml:"image_controller_manager"` ImageManager string `json:"image_manager" yaml:"image_manager"` - ImageDownloader string `json:"image_downloader" yaml:"image_downloader"` ImageKubexit string `json:"image_kubexit" yaml:"image_kubexit"` ImageProxy string `json:"image_proxy" yaml:"image_proxy"` ImageAsyncGateway string `json:"image_async_gateway" yaml:"image_async_gateway"` @@ -339,13 +338,6 @@ var CoreConfigStructFieldValidations = []*cr.StructFieldValidation{ Validator: validateImageVersion, }, }, - { - StructField: "ImageDownloader", - StringValidation: &cr.StringValidation{ - Default: consts.DefaultRegistry() + "/downloader:" + consts.CortexVersion, - Validator: validateImageVersion, - }, - }, { StructField: "ImageKubexit", StringValidation: &cr.StringValidation{ @@ -1357,9 +1349,6 @@ func (cc *CoreConfig) TelemetryEvent() map[string]interface{} { if !strings.HasPrefix(cc.ImageManager, "cortexlabs/") { event["image_manager._is_custom"] = true } - if !strings.HasPrefix(cc.ImageDownloader, "cortexlabs/") { - event["image_downloader._is_custom"] = true - } if !strings.HasPrefix(cc.ImageKubexit, "cortexlabs/") { event["image_kubexit._is_custom"] = true } diff --git a/pkg/types/spec/errors.go b/pkg/types/spec/errors.go index 3b44ac5fa0..5a1f84012e 100644 --- a/pkg/types/spec/errors.go +++ b/pkg/types/spec/errors.go @@ -35,9 +35,7 @@ const ( ErrDuplicateEndpointInOneDeploy = "spec.duplicate_endpoint_in_one_deploy" ErrDuplicateEndpoint = "spec.duplicate_endpoint" ErrDuplicateContainerName = "spec.duplicate_container_name" - ErrCantSpecifyBoth = "spec.cant_specify_both" - ErrSpecifyOnlyOneField = "spec.specify_only_one_field" - ErrNoneSpecified = "spec.none_specified" + ErrSpecifyExactlyOneField = "spec.specify_exactly_one_field" ErrSpecifyAllOrNone = "spec.specify_all_or_none" ErrOneOfPrerequisitesNotDefined = "spec.one_of_prerequisites_not_defined" ErrConfigGreaterThanOtherConfig = "spec.config_greater_than_other_config" @@ -116,24 +114,25 @@ func ErrorDuplicateContainerName(containerName string) error { }) } -func ErrorCantSpecifyBoth(fieldKeyA, fieldKeyB string) error { - return errors.WithStack(&errors.Error{ - Kind: ErrCantSpecifyBoth, - Message: fmt.Sprintf("please specify either %s or %s (both cannot be specified at the same time)", fieldKeyA, fieldKeyB), - }) -} - -func ErrorSpecifyOnlyOneField(fields ...string) error { - return errors.WithStack(&errors.Error{ - Kind: ErrSpecifyOnlyOneField, - Message: fmt.Sprintf("please specify only one of the following fields %s", s.UserStrsOr(fields)), - }) -} - -func ErrorNoneSpecified(fieldKeyA, fieldKeyB string) error { +func ErrorSpecifyExactlyOneField(numSpecified int, fields ...string) error { + var msg string + + if len(fields) == 2 { + if numSpecified == 0 { + msg = fmt.Sprintf("please specify either %s", s.UserStrsOr(fields)) + } else { + msg = fmt.Sprintf("please specify either %s (both cannot be specified at the same time)", s.UserStrsOr(fields)) + } + } else { + if numSpecified == 0 { + msg = fmt.Sprintf("please specify one of the following fields: %s", s.UserStrsOr(fields)) + } else { + msg = fmt.Sprintf("please specify only one of the following fields: %s", s.UserStrsOr(fields)) + } + } return errors.WithStack(&errors.Error{ - Kind: ErrNoneSpecified, - Message: fmt.Sprintf("please specify either %s or %s (cannot be both empty at the same time)", fieldKeyA, fieldKeyB), + Kind: ErrSpecifyExactlyOneField, + Message: msg, }) } diff --git a/pkg/types/spec/validations.go b/pkg/types/spec/validations.go index 92d8ae00c7..b5b75be126 100644 --- a/pkg/types/spec/validations.go +++ b/pkg/types/spec/validations.go @@ -19,7 +19,6 @@ package spec import ( "context" "fmt" - "strconv" "strings" "time" @@ -34,6 +33,7 @@ import ( "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/regex" "github.com/cortexlabs/cortex/pkg/lib/slices" + s "github.com/cortexlabs/cortex/pkg/lib/strings" libtime "github.com/cortexlabs/cortex/pkg/lib/time" "github.com/cortexlabs/cortex/pkg/lib/urls" "github.com/cortexlabs/cortex/pkg/types/userconfig" @@ -52,26 +52,26 @@ func apiValidation(resource userconfig.Resource) *cr.StructValidation { switch resource.Kind { case userconfig.RealtimeAPIKind: structFieldValidations = append(resourceStructValidations, - podValidation(), + podValidation(userconfig.RealtimeAPIKind), networkingValidation(), autoscalingValidation(), updateStrategyValidation(), ) case userconfig.AsyncAPIKind: structFieldValidations = append(resourceStructValidations, - podValidation(), + podValidation(userconfig.AsyncAPIKind), networkingValidation(), autoscalingValidation(), updateStrategyValidation(), ) case userconfig.BatchAPIKind: structFieldValidations = append(resourceStructValidations, - podValidation(), + podValidation(userconfig.BatchAPIKind), networkingValidation(), ) case userconfig.TaskAPIKind: structFieldValidations = append(resourceStructValidations, - podValidation(), + podValidation(userconfig.TaskAPIKind), networkingValidation(), ) case userconfig.TrafficSplitterKind: @@ -140,7 +140,7 @@ func multiAPIsValidation() *cr.StructFieldValidation { } } -func podValidation() *cr.StructFieldValidation { +func podValidation(kind userconfig.Kind) *cr.StructFieldValidation { return &cr.StructFieldValidation{ StructField: "Pod", StructValidation: &cr.StructValidation{ @@ -172,19 +172,72 @@ func podValidation() *cr.StructFieldValidation { Required: false, Default: nil, // it's a pointer because it's not required for the task API AllowExplicitNull: true, + GreaterThan: pointer.Int32(0), + LessThanOrEqualTo: pointer.Int32(65535), DisallowedValues: []int32{ consts.ProxyListeningPortInt32, - consts.MetricsPortInt32, + consts.AdminPortInt32, }, }, }, - containersValidation(), + containersValidation(kind), }, }, } } -func containersValidation() *cr.StructFieldValidation { +func containersValidation(kind userconfig.Kind) *cr.StructFieldValidation { + validations := []*cr.StructFieldValidation{ + { + StructField: "Name", + StringValidation: &cr.StringValidation{ + Required: true, + AllowEmpty: false, + DNS1035: true, + MaxLength: 63, + DisallowedValues: consts.ReservedContainerNames, + }, + }, + { + StructField: "Image", + StringValidation: &cr.StringValidation{ + Required: true, + AllowEmpty: false, + DockerImage: true, + }, + }, + { + StructField: "Env", + StringMapValidation: &cr.StringMapValidation{ + Required: false, + Default: map[string]string{}, + AllowEmpty: true, + }, + }, + { + StructField: "Command", + StringListValidation: &cr.StringListValidation{ + Required: false, + AllowExplicitNull: true, + AllowEmpty: true, + }, + }, + { + StructField: "Args", + StringListValidation: &cr.StringListValidation{ + Required: false, + AllowExplicitNull: true, + AllowEmpty: true, + }, + }, + computeValidation(), + probeValidation("LivenessProbe", true), + } + + if kind != userconfig.TaskAPIKind { + validations = append(validations, probeValidation("ReadinessProbe", false)) + } + return &cr.StructFieldValidation{ StructField: "Containers", StructListValidation: &cr.StructListValidation{ @@ -192,66 +245,154 @@ func containersValidation() *cr.StructFieldValidation { TreatNullAsEmpty: true, MinLength: 1, StructValidation: &cr.StructValidation{ - StructFieldValidations: []*cr.StructFieldValidation{ - { - StructField: "Name", - StringValidation: &cr.StringValidation{ - Required: true, - AllowEmpty: false, - DNS1035: true, - MaxLength: 63, - DisallowedValues: consts.ReservedContainerNames, - }, - }, - { - StructField: "Image", - StringValidation: &cr.StringValidation{ - Required: true, - AllowEmpty: false, - DockerImageOrEmpty: true, - }, + StructFieldValidations: validations, + }, + }, + } +} + +func networkingValidation() *cr.StructFieldValidation { + return &cr.StructFieldValidation{ + StructField: "Networking", + StructValidation: &cr.StructValidation{ + StructFieldValidations: []*cr.StructFieldValidation{ + { + StructField: "Endpoint", + StringPtrValidation: &cr.StringPtrValidation{ + Validator: urls.ValidateEndpoint, + MaxLength: 1000, // no particular reason other than it works }, - { - StructField: "Env", - StringMapValidation: &cr.StringMapValidation{ - Required: false, - Default: map[string]string{}, - AllowEmpty: true, - }, + }, + }, + }, + } +} + +func probeValidation(structFieldName string, hasExecProbe bool) *cr.StructFieldValidation { + validations := []*cr.StructFieldValidation{ + httpGetProbeValidation(), + tcpSocketProbeValidation(), + { + StructField: "InitialDelaySeconds", + Int32Validation: &cr.Int32Validation{ + Default: 0, + GreaterThanOrEqualTo: pointer.Int32(0), + }, + }, + { + StructField: "TimeoutSeconds", + Int32Validation: &cr.Int32Validation{ + Default: 1, + GreaterThanOrEqualTo: pointer.Int32(0), + }, + }, + { + StructField: "PeriodSeconds", + Int32Validation: &cr.Int32Validation{ + Default: 10, + GreaterThanOrEqualTo: pointer.Int32(0), + }, + }, + { + StructField: "SuccessThreshold", + Int32Validation: &cr.Int32Validation{ + Default: 1, + GreaterThanOrEqualTo: pointer.Int32(0), + }, + }, + { + StructField: "FailureThreshold", + Int32Validation: &cr.Int32Validation{ + Default: 3, + GreaterThanOrEqualTo: pointer.Int32(0), + }, + }, + } + + if hasExecProbe { + validations = append(validations, execProbeValidation()) + } + + return &cr.StructFieldValidation{ + StructField: structFieldName, + StructValidation: &cr.StructValidation{ + Required: false, + AllowExplicitNull: true, + DefaultNil: true, + StructFieldValidations: validations, + }, + } +} + +func httpGetProbeValidation() *cr.StructFieldValidation { + return &cr.StructFieldValidation{ + StructField: "HTTPGet", + StructValidation: &cr.StructValidation{ + Required: false, + AllowExplicitNull: true, + DefaultNil: true, + StructFieldValidations: []*cr.StructFieldValidation{ + { + StructField: "Path", + StringValidation: &cr.StringValidation{ + Required: true, + Validator: urls.ValidateEndpointAllowEmptyPath, }, - { - StructField: "Command", - StringListValidation: &cr.StringListValidation{ - Required: false, - AllowExplicitNull: true, - AllowEmpty: true, + }, + { + StructField: "Port", + Int32Validation: &cr.Int32Validation{ + Required: true, + GreaterThan: pointer.Int32(0), + LessThanOrEqualTo: pointer.Int32(65535), + DisallowedValues: []int32{ + consts.ProxyListeningPortInt32, + consts.AdminPortInt32, }, }, - { - StructField: "Args", - StringListValidation: &cr.StringListValidation{ - Required: false, - AllowExplicitNull: true, - AllowEmpty: true, + }, + }, + }, + } +} + +func tcpSocketProbeValidation() *cr.StructFieldValidation { + return &cr.StructFieldValidation{ + StructField: "TCPSocket", + StructValidation: &cr.StructValidation{ + Required: false, + AllowExplicitNull: true, + DefaultNil: true, + StructFieldValidations: []*cr.StructFieldValidation{ + { + StructField: "Port", + Int32Validation: &cr.Int32Validation{ + Required: true, + GreaterThan: pointer.Int32(0), + LessThanOrEqualTo: pointer.Int32(65535), + DisallowedValues: []int32{ + consts.ProxyListeningPortInt32, + consts.AdminPortInt32, }, }, - computeValidation(), }, }, }, } } -func networkingValidation() *cr.StructFieldValidation { +func execProbeValidation() *cr.StructFieldValidation { return &cr.StructFieldValidation{ - StructField: "Networking", + StructField: "Exec", StructValidation: &cr.StructValidation{ + Required: false, + AllowExplicitNull: true, + DefaultNil: true, StructFieldValidations: []*cr.StructFieldValidation{ { - StructField: "Endpoint", - StringPtrValidation: &cr.StringPtrValidation{ - Validator: urls.ValidateEndpoint, - MaxLength: 1000, // no particular reason other than it works + StructField: "Command", + StringListValidation: &cr.StringListValidation{ + Required: true, }, }, }, @@ -596,23 +737,63 @@ func validateContainers( for i, container := range containers { if slices.HasString(containerNames, container.Name) { - return errors.Wrap(ErrorDuplicateContainerName(container.Name), strconv.FormatInt(int64(i), 10), userconfig.ImageKey) + return errors.Wrap(ErrorDuplicateContainerName(container.Name), s.Index(i), userconfig.ImageKey) } containerNames = append(containerNames, container.Name) if container.Command == nil && (kind == userconfig.BatchAPIKind || kind == userconfig.TaskAPIKind) { - return errors.Wrap(ErrorFieldMustBeSpecifiedForKind(userconfig.CommandKey, kind), strconv.FormatInt(int64(i), 10), userconfig.CommandKey) + return errors.Wrap(ErrorFieldMustBeSpecifiedForKind(userconfig.CommandKey, kind), s.Index(i), userconfig.CommandKey) } if err := validateDockerImagePath(container.Image, awsClient, k8sClient); err != nil { - return errors.Wrap(err, strconv.FormatInt(int64(i), 10), userconfig.ImageKey) + return errors.Wrap(err, s.Index(i), userconfig.ImageKey) } for key := range container.Env { if strings.HasPrefix(key, "CORTEX_") || strings.HasPrefix(key, "KUBEXIT_") { - return errors.Wrap(ErrorCortexPrefixedEnvVarNotAllowed("CORTEX_", "KUBEXIT_"), strconv.FormatInt(int64(i), 10), userconfig.EnvKey, key) + return errors.Wrap(ErrorCortexPrefixedEnvVarNotAllowed("CORTEX_", "KUBEXIT_"), s.Index(i), userconfig.EnvKey, key) + } + } + + if kind == userconfig.TaskAPIKind && container.ReadinessProbe != nil { + return errors.Wrap(ErrorFieldIsNotSupportedForKind(userconfig.ReadinessProbeKey, kind), s.Index(i), userconfig.ReadinessProbeKey) + } + + if container.ReadinessProbe != nil { + if err := validateProbe(*container.ReadinessProbe, true); err != nil { + return errors.Wrap(err, s.Index(i), userconfig.ReadinessProbeKey) } } + + if container.LivenessProbe != nil { + if err := validateProbe(*container.LivenessProbe, false); err != nil { + return errors.Wrap(err, s.Index(i), userconfig.LivenessProbeKey) + } + } + + } + + return nil +} + +func validateProbe(probe userconfig.Probe, isReadinessProbe bool) error { + numSpecifiedProbes := 0 + if probe.HTTPGet != nil { + numSpecifiedProbes++ + } + if probe.TCPSocket != nil { + numSpecifiedProbes++ + } + if probe.Exec != nil { + numSpecifiedProbes++ + } + + if numSpecifiedProbes != 1 { + validProbes := []string{userconfig.HTTPGetKey, userconfig.TCPSocketKey} + if !isReadinessProbe { + validProbes = append(validProbes, userconfig.ExecKey) + } + return ErrorSpecifyExactlyOneField(numSpecifiedProbes, validProbes...) } return nil diff --git a/pkg/types/userconfig/api.go b/pkg/types/userconfig/api.go index 8516aadfc7..e8e8e03644 100644 --- a/pkg/types/userconfig/api.go +++ b/pkg/types/userconfig/api.go @@ -58,6 +58,9 @@ type Container struct { Command []string `json:"command" yaml:"command"` Args []string `json:"args" yaml:"args"` + ReadinessProbe *Probe `json:"readiness_probe" yaml:"readiness_probe"` + LivenessProbe *Probe `json:"liveness_probe" yaml:"liveness_probe"` + Compute *Compute `json:"compute" yaml:"compute"` } @@ -71,6 +74,30 @@ type Networking struct { Endpoint *string `json:"endpoint" yaml:"endpoint"` } +type Probe struct { + HTTPGet *HTTPGetProbe `json:"http_get" yaml:"http_get"` + TCPSocket *TCPSocketProbe `json:"tcp_socket" yaml:"tcp_socket"` + Exec *ExecProbe `json:"exec" yaml:"exec"` + InitialDelaySeconds int32 `json:"initial_delay_seconds" yaml:"initial_delay_seconds"` + TimeoutSeconds int32 `json:"timeout_seconds" yaml:"timeout_seconds"` + PeriodSeconds int32 `json:"period_seconds" yaml:"period_seconds"` + SuccessThreshold int32 `json:"success_threshold" yaml:"success_threshold"` + FailureThreshold int32 `json:"failure_threshold" yaml:"failure_threshold"` +} + +type HTTPGetProbe struct { + Path string `json:"path" yaml:"path"` + Port int32 `json:"port" yaml:"port"` +} + +type TCPSocketProbe struct { + Port int32 `json:"port" yaml:"port"` +} + +type ExecProbe struct { + Command []string `json:"command" yaml:"command"` +} + type Compute struct { CPU *k8s.Quantity `json:"cpu" yaml:"cpu"` Mem *k8s.Quantity `json:"mem" yaml:"mem"` @@ -312,6 +339,16 @@ func (container *Container) UserStr() string { sb.WriteString(fmt.Sprintf("%s: %s\n", ArgsKey, s.ObjFlatNoQuotes(container.Args))) } + if container.ReadinessProbe != nil { + sb.WriteString(fmt.Sprintf("%s:\n", ReadinessProbeKey)) + sb.WriteString(s.Indent(container.ReadinessProbe.UserStr(), " ")) + } + + if container.LivenessProbe != nil { + sb.WriteString(fmt.Sprintf("%s:\n", LivenessProbeKey)) + sb.WriteString(s.Indent(container.LivenessProbe.UserStr(), " ")) + } + if container.Compute != nil { sb.WriteString(fmt.Sprintf("%s:\n", ComputeKey)) sb.WriteString(s.Indent(container.Compute.UserStr(), " ")) @@ -328,6 +365,52 @@ func (networking *Networking) UserStr() string { return sb.String() } +func (probe *Probe) UserStr() string { + var sb strings.Builder + + if probe.HTTPGet != nil { + sb.WriteString(fmt.Sprintf("%s:\n", HTTPGetKey)) + sb.WriteString(s.Indent(probe.HTTPGet.UserStr(), " ")) + } + if probe.TCPSocket != nil { + sb.WriteString(fmt.Sprintf("%s:\n", TCPSocketKey)) + sb.WriteString(s.Indent(probe.TCPSocket.UserStr(), " ")) + } + if probe.Exec != nil { + sb.WriteString(fmt.Sprintf("%s:\n", ExecKey)) + sb.WriteString(s.Indent(probe.Exec.UserStr(), " ")) + } + + sb.WriteString(fmt.Sprintf("%s: %d\n", InitialDelaySecondsKey, probe.InitialDelaySeconds)) + sb.WriteString(fmt.Sprintf("%s: %d\n", TimeoutSecondsKey, probe.TimeoutSeconds)) + sb.WriteString(fmt.Sprintf("%s: %d\n", PeriodSecondsKey, probe.PeriodSeconds)) + sb.WriteString(fmt.Sprintf("%s: %d\n", SuccessThresholdKey, probe.SuccessThreshold)) + sb.WriteString(fmt.Sprintf("%s: %d\n", FailureThresholdKey, probe.FailureThreshold)) + + return sb.String() +} + +func (httpProbe *HTTPGetProbe) UserStr() string { + var sb strings.Builder + + sb.WriteString(fmt.Sprintf("%s: %s\n", PathKey, httpProbe.Path)) + sb.WriteString(fmt.Sprintf("%s: %d\n", PortKey, httpProbe.Port)) + + return sb.String() +} + +func (tcpSocketProbe *TCPSocketProbe) UserStr() string { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("%s: %d\n", PortKey, tcpSocketProbe.Port)) + return sb.String() +} + +func (execProbe *ExecProbe) UserStr() string { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("%s: %s\n", CommandKey, s.ObjFlatNoQuotes(execProbe.Command))) + return sb.String() +} + func (compute *Compute) UserStr() string { var sb strings.Builder if compute.CPU == nil { @@ -484,8 +567,23 @@ func (api *API) TelemetryEvent() map[string]interface{} { } event["pod.containers._len"] = len(api.Pod.Containers) - totalCompute := GetTotalComputeFromContainers(api.Pod.Containers) + + var numReadinessProbes int + var numLivenessProbes int + for _, container := range api.Pod.Containers { + if container.ReadinessProbe != nil { + numReadinessProbes++ + } + if container.LivenessProbe != nil { + numLivenessProbes++ + } + } + + event["pod.containers._num_readiness_probes"] = numReadinessProbes + event["pod.containers._num_liveness_probes"] = numLivenessProbes + event["pod.containers.compute._is_defined"] = true + totalCompute := GetTotalComputeFromContainers(api.Pod.Containers) if totalCompute.CPU != nil { event["pod.containers.compute.cpu._is_defined"] = true event["pod.containers.compute.cpu"] = float64(totalCompute.CPU.MilliValue()) / 1000 diff --git a/pkg/types/userconfig/config_key.go b/pkg/types/userconfig/config_key.go index 4a303c9a46..b61784f249 100644 --- a/pkg/types/userconfig/config_key.go +++ b/pkg/types/userconfig/config_key.go @@ -38,11 +38,26 @@ const ( ContainersKey = "containers" // Containers - ContainerNameKey = "name" - ImageKey = "image" - EnvKey = "env" - CommandKey = "command" - ArgsKey = "args" + ContainerNameKey = "name" + ImageKey = "image" + EnvKey = "env" + CommandKey = "command" + ArgsKey = "args" + ReadinessProbeKey = "readiness_probe" + LivenessProbeKey = "liveness_probe" + + // Probe + HTTPGetKey = "http_get" + TCPSocketKey = "tcp_socket" + ExecKey = "exec" + InitialDelaySecondsKey = "initial_delay_seconds" + TimeoutSecondsKey = "timeout_seconds" + PeriodSecondsKey = "period_seconds" + SuccessThresholdKey = "success_threshold" + FailureThresholdKey = "failure_threshold" + + // Probe types + PathKey = "path" // Compute CPUKey = "cpu" diff --git a/pkg/workloads/configmap.go b/pkg/workloads/configmap.go new file mode 100644 index 0000000000..5b385c9f07 --- /dev/null +++ b/pkg/workloads/configmap.go @@ -0,0 +1,65 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workloads + +import ( + libjson "github.com/cortexlabs/cortex/pkg/lib/json" + "github.com/cortexlabs/cortex/pkg/types/spec" + kcore "k8s.io/api/core/v1" +) + +type ConfigMapConfig struct { + BatchJob *spec.BatchJob + TaskJob *spec.TaskJob + Probes map[string]kcore.Probe +} + +func (c *ConfigMapConfig) GenerateConfigMapData() (map[string]string, error) { + if c == nil { + return nil, nil + } + + data := map[string]string{} + if len(c.Probes) > 0 { + probesEncoded, err := libjson.MarshalIndent(c.Probes) + if err != nil { + return nil, err + } + + data["probes.json"] = string(probesEncoded) + } + + if c.TaskJob != nil { + jobSpecEncoded, err := libjson.MarshalIndent(*c.TaskJob) + if err != nil { + return nil, err + } + + data["job.json"] = string(jobSpecEncoded) + } + + if c.BatchJob != nil { + jobSpecEncoded, err := libjson.MarshalIndent(*c.BatchJob) + if err != nil { + return nil, err + } + + data["job.json"] = string(jobSpecEncoded) + } + + return data, nil +} diff --git a/pkg/workloads/helpers.go b/pkg/workloads/helpers.go index ca13be7f72..e6f4301e7a 100644 --- a/pkg/workloads/helpers.go +++ b/pkg/workloads/helpers.go @@ -17,64 +17,81 @@ limitations under the License. package workloads import ( - "fmt" "path" "strings" "github.com/cortexlabs/cortex/pkg/lib/k8s" + "github.com/cortexlabs/cortex/pkg/types/userconfig" kcore "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/intstr" ) func K8sName(apiName string) string { return "api-" + apiName } -type downloadContainerConfig struct { - DownloadArgs []downloadContainerArg `json:"download_args"` - LastLog string `json:"last_log"` // string to log at the conclusion of the downloader (if "" nothing will be logged) -} +func GetProbeSpec(probe *userconfig.Probe) *kcore.Probe { + if probe == nil { + return nil + } -type downloadContainerArg struct { - From string `json:"from"` - To string `json:"to"` - ToFile bool `json:"to_file"` // whether "To" path reflects the path to a file or just the directory in which "From" object is copied to - Unzip bool `json:"unzip"` - ItemName string `json:"item_name"` // name of the item being downloaded, just for logging (if "" nothing will be logged) - HideFromLog bool `json:"hide_from_log"` // if true, don't log where the file is being downloaded from - HideUnzippingLog bool `json:"hide_unzipping_log"` // if true, don't log when unzipping -} + var httpGetAction *kcore.HTTPGetAction + var tcpSocketAction *kcore.TCPSocketAction + var execAction *kcore.ExecAction -func FileExistsProbe(fileName string) *kcore.Probe { - return &kcore.Probe{ - InitialDelaySeconds: 3, - TimeoutSeconds: 5, - PeriodSeconds: 5, - SuccessThreshold: 1, - FailureThreshold: 1, - Handler: kcore.Handler{ - Exec: &kcore.ExecAction{ - Command: []string{"/bin/bash", "-c", fmt.Sprintf("test -f %s", fileName)}, + if probe.HTTPGet != nil { + httpGetAction = &kcore.HTTPGetAction{ + Path: probe.HTTPGet.Path, + Port: intstr.IntOrString{ + IntVal: probe.HTTPGet.Port, }, - }, + } + } + if probe.TCPSocket != nil { + tcpSocketAction = &kcore.TCPSocketAction{ + Port: intstr.IntOrString{ + IntVal: probe.TCPSocket.Port, + }, + } + } + if probe.Exec != nil { + execAction = &kcore.ExecAction{ + Command: probe.Exec.Command, + } } -} -func SocketExistsProbe(socketName string) *kcore.Probe { return &kcore.Probe{ - InitialDelaySeconds: 3, - TimeoutSeconds: 5, - PeriodSeconds: 5, - SuccessThreshold: 1, - FailureThreshold: 1, Handler: kcore.Handler{ - Exec: &kcore.ExecAction{ - Command: []string{"/bin/bash", "-c", fmt.Sprintf("test -S %s", socketName)}, - }, + HTTPGet: httpGetAction, + TCPSocket: tcpSocketAction, + Exec: execAction, }, + InitialDelaySeconds: probe.InitialDelaySeconds, + TimeoutSeconds: probe.TimeoutSeconds, + PeriodSeconds: probe.PeriodSeconds, + SuccessThreshold: probe.SuccessThreshold, + FailureThreshold: probe.FailureThreshold, } } +func GetReadinessProbesFromContainers(containers []*userconfig.Container) map[string]kcore.Probe { + probes := map[string]kcore.Probe{} + + for _, container := range containers { + // this should never happen, it's just a precaution + if container == nil { + continue + } + + if container.ReadinessProbe != nil { + probes[container.Name] = *GetProbeSpec(container.ReadinessProbe) + } + } + + return probes +} + func baseClusterEnvVars() []kcore.EnvFromSource { envVars := []kcore.EnvFromSource{ { @@ -138,6 +155,19 @@ func CortexVolume() kcore.Volume { return k8s.EmptyDirVolume(_cortexDirVolumeName) } +func APIConfigVolume(name string) kcore.Volume { + return kcore.Volume{ + Name: name, + VolumeSource: kcore.VolumeSource{ + ConfigMap: &kcore.ConfigMapVolumeSource{ + LocalObjectReference: kcore.LocalObjectReference{ + Name: name, + }, + }, + }, + } +} + func ClientConfigVolume() kcore.Volume { return kcore.Volume{ Name: _clientConfigDirVolume, @@ -188,6 +218,13 @@ func CortexMount() kcore.VolumeMount { return k8s.EmptyDirVolumeMount(_cortexDirVolumeName, _cortexDirMountPath) } +func APIConfigMount(name string) kcore.VolumeMount { + return kcore.VolumeMount{ + Name: name, + MountPath: path.Join(_cortexDirMountPath, "spec"), + } +} + func ClientConfigMount() kcore.VolumeMount { return kcore.VolumeMount{ Name: _clientConfigDirVolume, diff --git a/pkg/workloads/init.go b/pkg/workloads/init.go index a5ea4ca0de..4d452f8243 100644 --- a/pkg/workloads/init.go +++ b/pkg/workloads/init.go @@ -17,25 +17,12 @@ limitations under the License. package workloads import ( - "encoding/base64" - "encoding/json" - "strings" - "github.com/cortexlabs/cortex/pkg/config" - "github.com/cortexlabs/cortex/pkg/lib/aws" - "github.com/cortexlabs/cortex/pkg/types/spec" - "github.com/cortexlabs/cortex/pkg/types/userconfig" kcore "k8s.io/api/core/v1" ) const ( - JobSpecPath = "/cortex/job_spec.json" -) - -const ( - _downloaderInitContainerName = "downloader" - _downloaderLastLog = "downloading the serving image(s)" - _kubexitInitContainerName = "kubexit" + _kubexitInitContainerName = "kubexit" ) func KubexitInitContainer() kcore.Container { @@ -49,77 +36,3 @@ func KubexitInitContainer() kcore.Container { }, } } - -func TaskInitContainer(job *spec.TaskJob) kcore.Container { - downloadConfig := downloadContainerConfig{ - LastLog: _downloaderLastLog, - DownloadArgs: []downloadContainerArg{ - { - From: aws.S3Path(config.ClusterConfig.Bucket, job.SpecFilePath(config.ClusterConfig.ClusterUID)), - To: JobSpecPath, - Unzip: false, - ToFile: true, - ItemName: "the task spec", - HideFromLog: true, - HideUnzippingLog: true, - }, - }, - } - - downloadArgsBytes, _ := json.Marshal(downloadConfig) - downloadArgs := base64.URLEncoding.EncodeToString(downloadArgsBytes) - - return kcore.Container{ - Name: _downloaderInitContainerName, - Image: config.ClusterConfig.ImageDownloader, - ImagePullPolicy: kcore.PullAlways, - Args: []string{"--download=" + downloadArgs}, - EnvFrom: baseClusterEnvVars(), - Env: []kcore.EnvVar{ - { - Name: "CORTEX_LOG_LEVEL", - Value: strings.ToUpper(userconfig.InfoLogLevel.String()), - }, - }, - VolumeMounts: []kcore.VolumeMount{ - CortexMount(), - }, - } -} - -func BatchInitContainer(job *spec.BatchJob) kcore.Container { - downloadConfig := downloadContainerConfig{ - LastLog: _downloaderLastLog, - DownloadArgs: []downloadContainerArg{ - { - From: aws.S3Path(config.ClusterConfig.Bucket, job.SpecFilePath(config.ClusterConfig.ClusterUID)), - To: JobSpecPath, - Unzip: false, - ToFile: true, - ItemName: "the job spec", - HideFromLog: true, - HideUnzippingLog: true, - }, - }, - } - - downloadArgsBytes, _ := json.Marshal(downloadConfig) - downloadArgs := base64.URLEncoding.EncodeToString(downloadArgsBytes) - - return kcore.Container{ - Name: _downloaderInitContainerName, - Image: config.ClusterConfig.ImageDownloader, - ImagePullPolicy: kcore.PullAlways, - Args: []string{"--download=" + downloadArgs}, - EnvFrom: baseClusterEnvVars(), - Env: []kcore.EnvVar{ - { - Name: "CORTEX_LOG_LEVEL", - Value: strings.ToUpper(userconfig.InfoLogLevel.String()), - }, - }, - VolumeMounts: []kcore.VolumeMount{ - CortexMount(), - }, - } -} diff --git a/pkg/workloads/k8s.go b/pkg/workloads/k8s.go index fe87d4cb94..16396c4644 100644 --- a/pkg/workloads/k8s.go +++ b/pkg/workloads/k8s.go @@ -79,9 +79,9 @@ func AsyncGatewayContainer(api spec.API, queueURL string, volumeMounts []kcore.V Image: config.ClusterConfig.ImageAsyncGateway, ImagePullPolicy: kcore.PullAlways, Args: []string{ - "-port", s.Int32(consts.ProxyListeningPortInt32), - "-queue", queueURL, - "-cluster-config", consts.DefaultInClusterConfigPath, + "--port", s.Int32(consts.ProxyListeningPortInt32), + "--queue", queueURL, + "--cluster-config", consts.DefaultInClusterConfigPath, api.Name, }, Ports: []kcore.ContainerPort{ @@ -119,9 +119,132 @@ func AsyncGatewayContainer(api spec.API, queueURL string, volumeMounts []kcore.V } } -func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { - requiresKubexit := api.Kind == userconfig.BatchAPIKind || api.Kind == userconfig.TaskAPIKind +func RealtimeProxyContainer(api spec.API) (kcore.Container, kcore.Volume) { + return kcore.Container{ + Name: _proxyContainerName, + Image: config.ClusterConfig.ImageProxy, + ImagePullPolicy: kcore.PullAlways, + Args: []string{ + "--port", + consts.ProxyListeningPortStr, + "--admin-port", + consts.AdminPortStr, + "--user-port", + s.Int32(*api.Pod.Port), + "--max-concurrency", + s.Int32(int32(api.Autoscaling.MaxConcurrency)), + "--max-queue-length", + s.Int32(int32(api.Autoscaling.MaxQueueLength)), + "--cluster-config", + consts.DefaultInClusterConfigPath, + }, + Ports: []kcore.ContainerPort{ + {Name: "admin", ContainerPort: consts.AdminPortInt32}, + {ContainerPort: consts.ProxyListeningPortInt32}, + }, + Env: []kcore.EnvVar{ + { + Name: "CORTEX_LOG_LEVEL", + Value: strings.ToUpper(userconfig.InfoLogLevel.String()), + }, + }, + EnvFrom: baseClusterEnvVars(), + VolumeMounts: []kcore.VolumeMount{ + ClusterConfigMount(), + }, + ReadinessProbe: &kcore.Probe{ + Handler: kcore.Handler{ + HTTPGet: &kcore.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(int(consts.AdminPortInt32)), + }, + }, + InitialDelaySeconds: 1, + TimeoutSeconds: 1, + PeriodSeconds: 5, + SuccessThreshold: 1, + FailureThreshold: 1, + }, + }, ClusterConfigVolume() +} + +func RealtimeUserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { + return userPodContainers(api) +} + +func AsyncUserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { + containers, volumes := userPodContainers(api) + k8sName := K8sName(api.Name) + + for i := range containers { + containers[i].VolumeMounts = append(containers[i].VolumeMounts, + APIConfigMount(k8sName), + ) + } + volumes = append(volumes, APIConfigVolume(k8sName)) + + return containers, volumes +} + +func TaskUserPodContainers(api spec.API, job *spec.JobKey) ([]kcore.Container, []kcore.Volume) { + containers, volumes := userPodContainers(api) + k8sName := job.K8sName() + + volumes = append(volumes, + KubexitVolume(), + APIConfigVolume(k8sName), + ) + + containerNames := userconfig.GetContainerNames(api.Pod.Containers) + for i, c := range containers { + containers[i].VolumeMounts = append(containers[i].VolumeMounts, + KubexitMount(), + APIConfigMount(k8sName), + ) + + containerDeathDependencies := containerNames.Copy() + containerDeathDependencies.Remove(c.Name) + containerDeathEnvVars := getKubexitEnvVars(c.Name, containerDeathDependencies.Slice(), nil) + containers[i].Env = append(containers[i].Env, containerDeathEnvVars...) + + if c.Command[0] != "/cortex/kubexit" { + containers[i].Command = append([]string{"/cortex/kubexit"}, c.Command...) + } + } + + return containers, volumes +} + +func BatchUserPodContainers(api spec.API, job *spec.JobKey) ([]kcore.Container, []kcore.Volume) { + containers, volumes := userPodContainers(api) + k8sName := job.K8sName() + + volumes = append(volumes, + KubexitVolume(), + APIConfigVolume(k8sName), + ) + containerNames := userconfig.GetContainerNames(api.Pod.Containers) + for i, c := range containers { + containers[i].VolumeMounts = append(containers[i].VolumeMounts, + KubexitMount(), + APIConfigMount(k8sName), + ) + + containerDeathDependencies := containerNames.Copy() + containerDeathDependencies.Remove(c.Name) + containerDeathEnvVars := getKubexitEnvVars(c.Name, containerDeathDependencies.Slice(), nil) + containers[i].Env = append(containers[i].Env, containerDeathEnvVars...) + + if c.Command[0] != "/cortex/kubexit" { + containers[i].Command = append([]string{"/cortex/kubexit"}, c.Command...) + } + } + + return containers, volumes +} + +func userPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { volumes := []kcore.Volume{ MntVolume(), CortexVolume(), @@ -133,17 +256,12 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { ClientConfigMount(), } - if requiresKubexit { - volumes = append(volumes, KubexitVolume()) - containerMounts = append(containerMounts, KubexitMount()) - } if api.Pod.ShmSize != nil { volumes = append(volumes, ShmVolume(api.Pod.ShmSize.Quantity)) containerMounts = append(containerMounts, ShmMount()) } var containers []kcore.Container - containerNames := userconfig.GetContainerNames(api.Pod.Containers) for _, container := range api.Pod.Containers { containerResourceList := kcore.ResourceList{} containerResourceLimitsList := kcore.ResourceList{} @@ -151,6 +269,11 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { Privileged: pointer.Bool(true), } + var readinessProbe *kcore.Probe + if api.Kind == userconfig.RealtimeAPIKind { + readinessProbe = GetProbeSpec(container.ReadinessProbe) + } + if container.Compute.CPU != nil { containerResourceList[kcore.ResourceCPU] = *k8s.QuantityPtr(container.Compute.CPU.Quantity.DeepCopy()) } @@ -164,8 +287,6 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { containerResourceLimitsList["nvidia.com/gpu"] = *kresource.NewQuantity(container.Compute.GPU, kresource.DecimalSI) } - containerVolumeMounts := containerMounts - if container.Compute.Inf > 0 { totalHugePages := container.Compute.Inf * _hugePagesMemPerInf containerResourceList["nvidia.com/gpu"] = *kresource.NewQuantity(container.Compute.Inf, kresource.DecimalSI) @@ -188,13 +309,6 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { Value: s.Int32(*api.Pod.Port), }) } - - if requiresKubexit { - containerDeathDependencies := containerNames.Copy() - containerDeathDependencies.Remove(container.Name) - containerEnvVars = getKubexitEnvVars(container.Name, containerDeathDependencies.Slice(), nil) - } - for k, v := range container.Env { containerEnvVars = append(containerEnvVars, kcore.EnvVar{ Name: k, @@ -202,18 +316,15 @@ func UserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { }) } - var containerCmd []string - if requiresKubexit && container.Command[0] != "/cortex/kubexit" { - containerCmd = append([]string{"/cortex/kubexit"}, container.Command...) - } - containers = append(containers, kcore.Container{ - Name: container.Name, - Image: container.Image, - Command: containerCmd, - Args: container.Args, - Env: containerEnvVars, - VolumeMounts: containerVolumeMounts, + Name: container.Name, + Image: container.Image, + Command: container.Command, + Args: container.Args, + Env: containerEnvVars, + VolumeMounts: containerMounts, + LivenessProbe: GetProbeSpec(container.LivenessProbe), + ReadinessProbe: readinessProbe, Resources: kcore.ResourceRequirements{ Requests: containerResourceList, Limits: containerResourceLimitsList, @@ -324,42 +435,6 @@ func GenerateNodeAffinities(apiNodeGroups []string) *kcore.Affinity { } } -func RealtimeProxyContainer(api spec.API) (kcore.Container, kcore.Volume) { - return kcore.Container{ - Name: _proxyContainerName, - Image: config.ClusterConfig.ImageProxy, - ImagePullPolicy: kcore.PullAlways, - Args: []string{ - "-port", - consts.ProxyListeningPortStr, - "-metrics-port", - consts.MetricsPortStr, - "-user-port", - s.Int32(*api.Pod.Port), - "-max-concurrency", - s.Int32(int32(api.Autoscaling.MaxConcurrency)), - "-max-queue-length", - s.Int32(int32(api.Autoscaling.MaxQueueLength)), - "-cluster-config", - consts.DefaultInClusterConfigPath, - }, - Ports: []kcore.ContainerPort{ - {Name: "metrics", ContainerPort: consts.MetricsPortInt32}, - {ContainerPort: consts.ProxyListeningPortInt32}, - }, - Env: []kcore.EnvVar{ - { - Name: "CORTEX_LOG_LEVEL", - Value: strings.ToUpper(userconfig.InfoLogLevel.String()), - }, - }, - EnvFrom: baseClusterEnvVars(), - VolumeMounts: []kcore.VolumeMount{ - ClusterConfigMount(), - }, - }, ClusterConfigVolume() -} - // func getAsyncAPIEnvVars(api spec.API, queueURL string) []kcore.EnvVar { // envVars := apiContainerEnvVars(&api) diff --git a/python/downloader/download.py b/python/downloader/download.py deleted file mode 100644 index 8781a3de57..0000000000 --- a/python/downloader/download.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import base64 -import json -import os - -from cortex_internal.lib import util -from cortex_internal.lib.log import configure_logger -from cortex_internal.lib.storage import S3 - -util.expand_environment_vars_on_file(os.environ["CORTEX_LOG_CONFIG_FILE"]) -logger = configure_logger("cortex", os.environ["CORTEX_LOG_CONFIG_FILE"]) - - -def start(args): - download_config = json.loads(base64.urlsafe_b64decode(args.download)) - for download_arg in download_config["download_args"]: - from_path = download_arg["from"] - to_path = download_arg["to"] - item_name = download_arg.get("item_name", "") - - bucket_name, prefix = S3.deconstruct_s3_path(from_path) - client = S3(bucket_name) - - if item_name != "": - if download_arg.get("hide_from_log", False): - logger.info("downloading {}".format(item_name)) - else: - logger.info("downloading {} from {}".format(item_name, from_path)) - - if download_arg.get("to_file", False): - client.download_file(prefix, to_path) - else: - client.download(prefix, to_path) - - if download_arg.get("unzip", False): - if item_name != "" and not download_arg.get("hide_unzipping_log", False): - logger.info("unzipping {}".format(item_name)) - if download_arg.get("to_file", False): - util.extract_zip(to_path, delete_zip_file=True) - else: - util.extract_zip( - os.path.join(to_path, os.path.basename(from_path)), delete_zip_file=True - ) - - if download_config.get("last_log", "") != "": - logger.info(download_config["last_log"]) - - -def main(): - parser = argparse.ArgumentParser() - na = parser.add_argument_group("required named arguments") - na.add_argument( - "--download", - required=True, - help="a base64 encoded download_config (see k8s_specs.go for the structure)", - ) - parser.set_defaults(func=start) - - args = parser.parse_args() - args.func(args) - - -if __name__ == "__main__": - main() From 0cad67b5561ed15bfc695f1f1109e627da7b6828 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Mon, 24 May 2021 21:39:43 +0300 Subject: [PATCH 25/82] Support exec probe for realtime's readiness probe (#2190) --- pkg/types/spec/validations.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/types/spec/validations.go b/pkg/types/spec/validations.go index b5b75be126..10f2f761c7 100644 --- a/pkg/types/spec/validations.go +++ b/pkg/types/spec/validations.go @@ -234,7 +234,9 @@ func containersValidation(kind userconfig.Kind) *cr.StructFieldValidation { probeValidation("LivenessProbe", true), } - if kind != userconfig.TaskAPIKind { + if kind == userconfig.RealtimeAPIKind { + validations = append(validations, probeValidation("ReadinessProbe", true)) + } else if kind == userconfig.AsyncAPIKind || kind == userconfig.BatchAPIKind { validations = append(validations, probeValidation("ReadinessProbe", false)) } @@ -760,13 +762,14 @@ func validateContainers( } if container.ReadinessProbe != nil { - if err := validateProbe(*container.ReadinessProbe, true); err != nil { + supportsExecProbe := kind == userconfig.RealtimeAPIKind + if err := validateProbe(*container.ReadinessProbe, supportsExecProbe); err != nil { return errors.Wrap(err, s.Index(i), userconfig.ReadinessProbeKey) } } if container.LivenessProbe != nil { - if err := validateProbe(*container.LivenessProbe, false); err != nil { + if err := validateProbe(*container.LivenessProbe, true); err != nil { return errors.Wrap(err, s.Index(i), userconfig.LivenessProbeKey) } } @@ -776,7 +779,7 @@ func validateContainers( return nil } -func validateProbe(probe userconfig.Probe, isReadinessProbe bool) error { +func validateProbe(probe userconfig.Probe, supportsExecProbe bool) error { numSpecifiedProbes := 0 if probe.HTTPGet != nil { numSpecifiedProbes++ @@ -790,7 +793,7 @@ func validateProbe(probe userconfig.Probe, isReadinessProbe bool) error { if numSpecifiedProbes != 1 { validProbes := []string{userconfig.HTTPGetKey, userconfig.TCPSocketKey} - if !isReadinessProbe { + if supportsExecProbe { validProbes = append(validProbes, userconfig.ExecKey) } return ErrorSpecifyExactlyOneField(numSpecifiedProbes, validProbes...) From 079c20c25d644b1bf76570ca0c40865ec8ea9aad Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Tue, 25 May 2021 16:21:13 -0700 Subject: [PATCH 26/82] Initial docs pass (#2192) --- dev/export_images.sh | 19 +- dev/python_version_test.sh | 47 -- docs/clusters/advanced/self-hosted-images.md | 4 +- docs/clusters/instances/multi.md | 26 + docs/clusters/instances/spot.md | 2 + docs/clusters/management/create.md | 1 - docs/clusters/management/update.md | 2 +- docs/clusters/networking/custom-domain.md | 10 +- docs/clusters/networking/https.md | 8 +- docs/clusters/observability/logging.md | 12 - docs/clusters/observability/metrics.md | 24 +- docs/start.md | 2 +- docs/summary.md | 28 +- docs/workloads/async/async-apis.md | 6 +- docs/workloads/async/autoscaling.md | 89 +-- docs/workloads/async/configuration.md | 162 ++--- docs/workloads/async/handler.md | 240 -------- docs/workloads/async/models.md | 203 ------- docs/workloads/async/webhooks.md | 48 +- docs/workloads/batch/batch-apis.md | 3 + docs/workloads/batch/configuration.md | 118 ++-- docs/workloads/batch/handler.md | 125 ---- docs/workloads/batch/jobs.md | 58 +- docs/workloads/batch/models.md | 210 ------- docs/workloads/dependencies/example.md | 61 -- docs/workloads/dependencies/images.md | 106 ---- .../workloads/dependencies/python-packages.md | 140 ----- .../workloads/dependencies/system-packages.md | 62 -- docs/workloads/realtime/autoscaling.md | 38 +- docs/workloads/realtime/configuration.md | 186 ++---- docs/workloads/realtime/handler.md | 563 ------------------ docs/workloads/realtime/models.md | 439 -------------- .../workloads/realtime/multi-model/caching.md | 14 - .../realtime/multi-model/configuration.md | 118 ---- .../workloads/realtime/multi-model/example.md | 43 -- docs/workloads/realtime/parallelism.md | 9 - docs/workloads/realtime/realtime-apis.md | 3 + .../realtime/server-side-batching.md | 83 --- docs/workloads/realtime/statuses.md | 2 +- docs/workloads/realtime/traffic-splitter.md | 65 ++ .../traffic-splitter/configuration.md | 12 - .../realtime/traffic-splitter/example.md | 66 -- docs/workloads/realtime/troubleshooting.md | 21 +- docs/workloads/task/configuration.md | 54 +- docs/workloads/task/definitions.md | 91 --- docs/workloads/task/jobs.md | 8 +- docs/workloads/task/task-apis.md | 3 + pkg/consts/consts.go | 4 +- pkg/lib/aws/elb.go | 2 +- pkg/types/spec/validations.go | 3 +- 50 files changed, 457 insertions(+), 3186 deletions(-) delete mode 100755 dev/python_version_test.sh delete mode 100644 docs/workloads/async/handler.md delete mode 100644 docs/workloads/async/models.md create mode 100644 docs/workloads/batch/batch-apis.md delete mode 100644 docs/workloads/batch/handler.md delete mode 100644 docs/workloads/batch/models.md delete mode 100644 docs/workloads/dependencies/example.md delete mode 100644 docs/workloads/dependencies/images.md delete mode 100644 docs/workloads/dependencies/python-packages.md delete mode 100644 docs/workloads/dependencies/system-packages.md delete mode 100644 docs/workloads/realtime/handler.md delete mode 100644 docs/workloads/realtime/models.md delete mode 100644 docs/workloads/realtime/multi-model/caching.md delete mode 100644 docs/workloads/realtime/multi-model/configuration.md delete mode 100644 docs/workloads/realtime/multi-model/example.md delete mode 100644 docs/workloads/realtime/parallelism.md create mode 100644 docs/workloads/realtime/realtime-apis.md delete mode 100644 docs/workloads/realtime/server-side-batching.md create mode 100644 docs/workloads/realtime/traffic-splitter.md delete mode 100644 docs/workloads/realtime/traffic-splitter/configuration.md delete mode 100644 docs/workloads/realtime/traffic-splitter/example.md delete mode 100644 docs/workloads/task/definitions.md create mode 100644 docs/workloads/task/task-apis.md diff --git a/dev/export_images.sh b/dev/export_images.sh index 8db4acc676..0d040e777e 100755 --- a/dev/export_images.sh +++ b/dev/export_images.sh @@ -40,23 +40,10 @@ for image in "${all_images[@]}"; do done echo -cuda=("10.0" "10.1" "10.1" "10.2" "10.2" "11.0" "11.1") -cudnn=("7" "7" "8" "7" "8" "8" "8") - # pull the images from source registry and push them to ECR for image in "${all_images[@]}"; do - # copy the different cuda/cudnn variations of the python handler image - if [ "$image" = "python-handler-gpu" ]; then - for i in "${!cuda[@]}"; do - full_image="$image:$cortex_version-cuda${cuda[$i]}-cudnn${cudnn[$i]}" - echo "copying $full_image from $source_registry to $destination_registry" - skopeo copy --src-no-creds "docker://$source_registry/$full_image" "docker://$destination_registry/$full_image" - echo - done - else - echo "copying $image:$cortex_version from $source_registry to $destination_registry" - skopeo copy --src-no-creds "docker://$source_registry/$image:$cortex_version" "docker://$destination_registry/$image:$cortex_version" - echo - fi + echo "copying $image:$cortex_version from $source_registry to $destination_registry" + skopeo copy --src-no-creds "docker://$source_registry/$image:$cortex_version" "docker://$destination_registry/$image:$cortex_version" + echo done echo "done ✓" diff --git a/dev/python_version_test.sh b/dev/python_version_test.sh deleted file mode 100755 index ae94f689de..0000000000 --- a/dev/python_version_test.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# USAGE: ./dev/python_version_test.sh -# e.g.: ./dev/python_version_test.sh 3.6.9 aws - -set -e - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)" - -# create a new conda environment based on the supplied python version -conda create -n env -y -CONDA_BASE=$(conda info --base) -source $CONDA_BASE/etc/profile.d/conda.sh -conda activate env -conda config --append channels conda-forge -conda install python=$1 -y - -pip install requests - -export CORTEX_CLI_PATH=$ROOT/bin/cortex - -# install cortex -cd $ROOT/python/client -pip install -e . - -# run script.py -python $ROOT/dev/deploy_test.py $2 - -# clean up conda -conda deactivate -conda env remove -n env -rm -rf $ROOT/python/client/cortex.egg-info diff --git a/docs/clusters/advanced/self-hosted-images.md b/docs/clusters/advanced/self-hosted-images.md index 74b581c840..b88f5e55b5 100644 --- a/docs/clusters/advanced/self-hosted-images.md +++ b/docs/clusters/advanced/self-hosted-images.md @@ -1,6 +1,6 @@ # Self-hosted Docker images -Self-hosted Docker images can be useful for reducing the ingress costs, for accelerating image pulls, or for eliminating the dependency on Cortex's public container registry. +Self-hosting the Cortex cluster's internal Docker images can be useful for reducing the ingress costs, for accelerating image pulls, or for eliminating the dependency on Cortex's public container registry. In this guide, we'll use [ECR](https://aws.amazon.com/ecr/) as the destination container registry. When an ECR repository resides in the same region as your Cortex cluster, there are no costs incurred when pulling images. @@ -33,7 +33,7 @@ Feel free to modify the script if you would like to export the images to a diffe ./cortex/dev/export_images.sh ``` -You can now configure Cortex to use your images when creating a cluster (see [here](../management/create.md) for how to specify cluster images) and/or when deploying APIs (see the configuration docs corresponding to your API type for how to specify API images). +You can now configure Cortex to use your images when creating a cluster (see [here](../management/create.md) for instructions). ## Cleanup diff --git a/docs/clusters/instances/multi.md b/docs/clusters/instances/multi.md index b6fd6f7bb1..e34194ed88 100644 --- a/docs/clusters/instances/multi.md +++ b/docs/clusters/instances/multi.md @@ -20,11 +20,15 @@ Cortex can be configured to provision different instance types to improve worklo node_groups: - name: cpu-spot instance_type: m5.large + min_instances: 0 + max_instances: 5 spot: true spot_config: instance_distribution: [m5a.large, m5d.large, m5n.large, m5ad.large, m5dn.large, m4.large, t3.large, t3a.large, t2.large] - name: cpu-on-demand instance_type: m5.large + min_instances: 0 + max_instances: 5 ``` ### On-demand cluster supporting CPU, GPU, and Inferentia @@ -35,10 +39,16 @@ node_groups: node_groups: - name: cpu instance_type: m5.large + min_instances: 0 + max_instances: 5 - name: gpu instance_type: g4dn.xlarge + min_instances: 0 + max_instances: 5 - name: inf instance_type: inf.xlarge + min_instances: 0 + max_instances: 5 ``` ### Spot cluster supporting CPU and GPU (with on-demand backup) @@ -49,16 +59,24 @@ node_groups: node_groups: - name: cpu-spot instance_type: m5.large + min_instances: 0 + max_instances: 5 spot: true spot_config: instance_distribution: [m5a.large, m5d.large, m5n.large, m5ad.large, m5dn.large, m4.large, t3.large, t3a.large, t2.large] - name: cpu-on-demand instance_type: m5.large + min_instances: 0 + max_instances: 5 - name: gpu-spot instance_type: g4dn.xlarge + min_instances: 0 + max_instances: 5 spot: true - name: gpu-on-demand instance_type: g4dn.xlarge + min_instances: 0 + max_instances: 5 ``` ### CPU spot cluster with multiple instance types and on-demand backup @@ -69,13 +87,21 @@ node_groups: node_groups: - name: cpu-1 instance_type: t3.medium + min_instances: 0 + max_instances: 5 spot: true - name: cpu-2 instance_type: m5.2xlarge + min_instances: 0 + max_instances: 5 spot: true - name: cpu-3 instance_type: m5.8xlarge + min_instances: 0 + max_instances: 5 spot: true - name: cpu-4 instance_type: m5.24xlarge + min_instances: 0 + max_instances: 5 ``` diff --git a/docs/clusters/instances/spot.md b/docs/clusters/instances/spot.md index 4fc6b1d9f6..1a4c22f554 100644 --- a/docs/clusters/instances/spot.md +++ b/docs/clusters/instances/spot.md @@ -43,6 +43,8 @@ There is a spot instance limit associated with your AWS account for each instanc node_groups: - name: cpu-spot instance_type: m5.large + min_instances: 0 + max_instances: 5 spot: true spot_config: instance_distribution: [m5a.large, m5d.large, m5n.large, m5ad.large, m5dn.large, m4.large, t3.large, t3a.large, t2.large] diff --git a/docs/clusters/management/create.md b/docs/clusters/management/create.md index c2f75e3bad..5fe3a5d51e 100644 --- a/docs/clusters/management/create.md +++ b/docs/clusters/management/create.md @@ -104,7 +104,6 @@ image_async_gateway: quay.io/cortexlabs/async-gateway:master image_cluster_autoscaler: quay.io/cortexlabs/cluster-autoscaler:master image_metrics_server: quay.io/cortexlabs/metrics-server:master image_inferentia: quay.io/cortexlabs/inferentia:master -image_neuron_rtd: quay.io/cortexlabs/neuron-rtd:master image_nvidia: quay.io/cortexlabs/nvidia:master image_fluent_bit: quay.io/cortexlabs/fluent-bit:master image_istio_proxy: quay.io/cortexlabs/istio-proxy:master diff --git a/docs/clusters/management/update.md b/docs/clusters/management/update.md index f881ae3ff5..6e144602c3 100644 --- a/docs/clusters/management/update.md +++ b/docs/clusters/management/update.md @@ -27,7 +27,7 @@ cortex cluster up cluster.yaml In production environments, you can upgrade your cluster without downtime if you have a backend service or DNS in front of your Cortex cluster: 1. Spin up a new cluster. For example: `cortex cluster up new-cluster.yaml --configure-env cortex2` (this will create a CLI environment named `cortex2` for accessing the new cluster). -1. Re-deploy your APIs in your new cluster. For example, if the name of your CLI environment for your existing cluster is `cortex`, you can use `cortex get --env cortex` to list all running APIs in your cluster, and re-deploy them in the new cluster by changing directories to each API's project folder and running `cortex deploy --env cortex2`. Alternatively, you can run `cortex cluster export --name --region ` to export all of your API specifications, change directories the folder that was exported, and run `cortex deploy --env cortex2 ` for each API that you want to deploy in the new cluster. +1. Re-deploy your APIs in your new cluster. For example, if the name of your CLI environment for your existing cluster is `cortex`, you can use `cortex get --env cortex` to list all running APIs in your cluster, and re-deploy them in the new cluster by running `cortex deploy --env cortex2` for each API. Alternatively, you can run `cortex cluster export --name --region ` to export the API specifications for all of your running APIs, change directories the folder that was exported, and run `cortex deploy --env cortex2 ` for each API that you want to deploy in the new cluster. 1. Route requests to your new cluster. * If you are using a custom domain: update the A record in your Route 53 hosted zone to point to your new cluster's API load balancer. * If you have a backend service which makes requests to Cortex: update your backend service to make requests to the new cluster's endpoints. diff --git a/docs/clusters/networking/custom-domain.md b/docs/clusters/networking/custom-domain.md index 00d0d2f9d1..f0eb8162c7 100644 --- a/docs/clusters/networking/custom-domain.md +++ b/docs/clusters/networking/custom-domain.md @@ -115,13 +115,9 @@ You could run into connectivity issues if you make a request to your API without To test connectivity, try the following steps: -1. Deploy any api (e.g. examples/pytorch/iris-classifier). -1. Make a GET request to the your api (e.g. `curl https://api.cortexlabs.dev/iris-classifier` or paste the url into your browser). -1. If you run into an error such as `curl: (6) Could not resolve host: api.cortexlabs.dev` wait a few minutes and make the GET request from another device that hasn't made a request to that url in a while. A successful request looks like this: - -```text -{"message":"make a request by sending a POST to this endpoint with a json payload",...} -``` +1. Deploy an api. +1. Make a request to the your api (e.g. `curl https://api.cortexlabs.dev/my-api` or paste the url into your browser if your API supports GET requests). +1. If you run into an error such as `curl: (6) Could not resolve host: api.cortexlabs.dev` wait a few minutes and make the request from another device that hasn't made a request to that url in a while. ## Cleanup diff --git a/docs/clusters/networking/https.md b/docs/clusters/networking/https.md index 77df0ac276..38a25da691 100644 --- a/docs/clusters/networking/https.md +++ b/docs/clusters/networking/https.md @@ -56,13 +56,13 @@ Copy your "Invoke URL" You may now use the "Invoke URL" in place of your API load balancer endpoint in your client. For example, this curl request: ```bash -curl http://a9eaf69fd125947abb1065f62de59047-81cdebc0275f7d96.elb.us-west-2.amazonaws.com/iris-classifier -X POST -H "Content-Type: application/json" -d @sample.json +curl http://a9eaf69fd125947abb1065f62de59047-81cdebc0275f7d96.elb.us-west-2.amazonaws.com/my-api -X POST -H "Content-Type: application/json" -d @sample.json ``` Would become: ```bash -curl https://31qjv48rs6.execute-api.us-west-2.amazonaws.com/dev/iris-classifier -X POST -H "Content-Type: application/json" -d @sample.json +curl https://31qjv48rs6.execute-api.us-west-2.amazonaws.com/dev/my-api -X POST -H "Content-Type: application/json" -d @sample.json ``` ### Cleanup @@ -134,13 +134,13 @@ Copy your "Invoke URL" You may now use the "Invoke URL" in place of your API load balancer endpoint in your client. For example, this curl request: ```bash -curl http://a5044e34a352d44b0945adcd455c7fa3-32fa161d3e5bcbf9.elb.us-west-2.amazonaws.com/iris-classifier -X POST -H "Content-Type: application/json" -d @sample.json +curl http://a5044e34a352d44b0945adcd455c7fa3-32fa161d3e5bcbf9.elb.us-west-2.amazonaws.com/my-api -X POST -H "Content-Type: application/json" -d @sample.json ``` Would become: ```bash -curl https://lrivodooqh.execute-api.us-west-2.amazonaws.com/dev/iris-classifier -X POST -H "Content-Type: application/json" -d @sample.json +curl https://lrivodooqh.execute-api.us-west-2.amazonaws.com/dev/my-api -X POST -H "Content-Type: application/json" -d @sample.json ``` ### Cleanup diff --git a/docs/clusters/observability/logging.md b/docs/clusters/observability/logging.md index fee687c174..1e484649c4 100644 --- a/docs/clusters/observability/logging.md +++ b/docs/clusters/observability/logging.md @@ -64,15 +64,3 @@ fields @timestamp, message | sort @timestamp asc | limit 1000 ``` - -## Structured logging - -You can use Cortex's logger in your Python code to log in JSON, which will enrich your logs with Cortex's metadata, and -enable you to add custom metadata to the logs. - -See the structured logging docs for each API kind: - -- [RealtimeAPI](../../workloads/realtime/handler.md#structured-logging) -- [AsyncAPI](../../workloads/async/handler.md#structured-logging) -- [BatchAPI](../../workloads/batch/handler.md#structured-logging) -- [TaskAPI](../../workloads/task/definitions.md#structured-logging) diff --git a/docs/clusters/observability/metrics.md b/docs/clusters/observability/metrics.md index 72ab98ec27..55de3e85de 100644 --- a/docs/clusters/observability/metrics.md +++ b/docs/clusters/observability/metrics.md @@ -96,23 +96,23 @@ Currently, we only support 3 different metric types that will be converted to it ### Pushing metrics - - Counter +- Counter - ```python - metrics.increment('my_counter', value=1, tags={"tag": "tag_name"}) - ``` + ```python + metrics.increment('my_counter', value=1, tags={"tag": "tag_name"}) + ``` - - Gauge +- Gauge - ```python - metrics.gauge('active_connections', value=1001, tags={"tag": "tag_name"}) - ``` + ```python + metrics.gauge('active_connections', value=1001, tags={"tag": "tag_name"}) + ``` - - Histogram +- Histogram - ```python - metrics.histogram('inference_time_milliseconds', 120, tags={"tag": "tag_name"}) - ``` + ```python + metrics.histogram('inference_time_milliseconds', 120, tags={"tag": "tag_name"}) + ``` ### Metrics client class reference diff --git a/docs/start.md b/docs/start.md index e0f8e7f5a5..2fa0d570bd 100644 --- a/docs/start.md +++ b/docs/start.md @@ -21,7 +21,7 @@ cortex cluster up cluster.yaml cortex deploy apis.yaml ``` -* [RealtimeAPI](workloads/realtime/example.md) - create HTTP/gRPC APIs that respond to requests in real-time. +* [RealtimeAPI](workloads/realtime/example.md) - create APIs that respond to requests in real-time. * [AsyncAPI](workloads/async/example.md) - create APIs that respond to requests asynchronously. * [BatchAPI](workloads/batch/example.md) - create APIs that run distributed batch jobs. * [TaskAPI](workloads/task/example.md) - create APIs that run jobs on-demand. diff --git a/docs/summary.md b/docs/summary.md index f6f5d1db0c..cd40165657 100644 --- a/docs/summary.md +++ b/docs/summary.md @@ -29,52 +29,32 @@ ## Workloads -* Realtime APIs +* [Realtime APIs](workloads/realtime/realtime-apis.md) * [Example](workloads/realtime/example.md) - * [Handler](workloads/realtime/handler.md) * [Configuration](workloads/realtime/configuration.md) - * [Parallelism](workloads/realtime/parallelism.md) * [Autoscaling](workloads/realtime/autoscaling.md) - * [Models](workloads/realtime/models.md) - * Multi-model - * [Example](workloads/realtime/multi-model/example.md) - * [Configuration](workloads/realtime/multi-model/configuration.md) - * [Caching](workloads/realtime/multi-model/caching.md) - * [Server-side batching](workloads/realtime/server-side-batching.md) + * [Traffic Splitter](workloads/realtime/traffic-splitter.md) * [Metrics](workloads/realtime/metrics.md) * [Statuses](workloads/realtime/statuses.md) - * Traffic Splitter - * [Example](workloads/realtime/traffic-splitter/example.md) - * [Configuration](workloads/realtime/traffic-splitter/configuration.md) * [Troubleshooting](workloads/realtime/troubleshooting.md) * [Async APIs](workloads/async/async-apis.md) * [Example](workloads/async/example.md) - * [Handler](workloads/async/handler.md) * [Configuration](workloads/async/configuration.md) - * [TensorFlow Models](workloads/async/models.md) * [Metrics](workloads/async/metrics.md) * [Statuses](workloads/async/statuses.md) * [Webhooks](workloads/async/webhooks.md) -* Batch APIs +* [Batch APIs](workloads/batch/batch-apis.md) * [Example](workloads/batch/example.md) - * [Handler](workloads/batch/handler.md) * [Configuration](workloads/batch/configuration.md) * [Jobs](workloads/batch/jobs.md) - * [TensorFlow Models](workloads/batch/models.md) * [Metrics](workloads/batch/metrics.md) * [Statuses](workloads/batch/statuses.md) -* Task APIs +* [Task APIs](workloads/task/task-apis.md) * [Example](workloads/task/example.md) - * [Definition](workloads/task/definitions.md) * [Configuration](workloads/task/configuration.md) * [Jobs](workloads/task/jobs.md) * [Metrics](workloads/task/metrics.md) * [Statuses](workloads/task/statuses.md) -* Dependencies - * [Example](workloads/dependencies/example.md) - * [Python packages](workloads/dependencies/python-packages.md) - * [System packages](workloads/dependencies/system-packages.md) - * [Custom images](workloads/dependencies/images.md) ## Clients diff --git a/docs/workloads/async/async-apis.md b/docs/workloads/async/async-apis.md index f96e642a8d..724c76fc94 100644 --- a/docs/workloads/async/async-apis.md +++ b/docs/workloads/async/async-apis.md @@ -1,4 +1,4 @@ -# AsyncAPI +# Async APIs The AsyncAPI kind is designed for asynchronous workloads, in which the user submits a request to start the processing and retrieves the result later, either by polling or through a webhook. @@ -14,7 +14,3 @@ workload status and results. Cortex fully manages the Async Gateway and the queu AsyncAPI is a good fit for users who want to submit longer workloads (such as video, audio or document processing), and do not need the result immediately or synchronously. - -{% hint style="info" %} -AsyncAPI is still in a beta state. -{% endhint %} diff --git a/docs/workloads/async/autoscaling.md b/docs/workloads/async/autoscaling.md index 395d5d6e2e..c83c537154 100644 --- a/docs/workloads/async/autoscaling.md +++ b/docs/workloads/async/autoscaling.md @@ -4,6 +4,16 @@ Cortex auto-scales AsyncAPIs on a per-API basis based on your configuration. ## Autoscaling replicas +### Relevant pod configuration + +In addition to the autoscaling configuration options (described below), there is one field in the pod configuration which are relevant to replica autoscaling: + +**`max_concurrency`** (default: 1): The maximum number of requests that will be concurrently sent into the container by Cortex. If your web server is designed to handle multiple concurrent requests, increasing `max_concurrency` will increase the throughput of a replica (and result in fewer total replicas for a given load). + +
+ +### Autoscaling configuration + **`min_replicas`**: The lower bound on how many replicas can be running for an API.
@@ -12,97 +22,56 @@ Cortex auto-scales AsyncAPIs on a per-API basis based on your configuration.
-**`target_replica_concurrency`** (default: 1): This is the desired number of in-flight requests per replica, and is the -metric which the autoscaler uses to make scaling decisions. It is recommended to leave this parameter at its default -value. - -Replica concurrency is simply how many requests have been sent to the queue and have not yet finished being processed (also -referred to as in-flight requests). Therefore, it includes requests which are currently being processed and requests -which are waiting in the queue. +**`target_in_flight`** (default: `max_concurrency` in the pod configuration): This is the desired number of in-flight requests per replica, and is the metric which the autoscaler uses to make scaling decisions. The number of in-flight requests is simply how many requests have been submitted and are not yet finished being processed. Therefore, this number includes requests which are actively being processed as well as requests which are waiting in the queue. The autoscaler uses this formula to determine the number of desired replicas: -`desired replicas = sum(in-flight requests accross all replicas) / target_replica_concurrency` +`desired replicas = total in-flight requests / target_in_flight` -
- -**`max_replica_concurrency`** (default: 1024): This is the maximum number of in-queue messages before requests are -rejected with HTTP error code 503. `max_replica_concurrency` includes requests that are currently being processed as -well as requests that are waiting in the queue (a replica can actively process one request concurrently, and will hold -any additional requests in a local queue). Decreasing `max_replica_concurrency` and configuring the client to retry when -it receives 503 responses will improve queue fairness accross replicas by preventing requests from sitting in long -queues. +For example, setting `target_in_flight` to `max_concurrency` (the default) causes the cluster to adjust the number of replicas so that on average, there are no requests waiting in the queue.
-**`window`** (default: 60s): The time over which to average the API in-flight requests (which is the sum of in-flight -requests in each replica). The longer the window, the slower the autoscaler will react to changes in API wide in-flight -requests, since it is averaged over the `window`. API wide in-flight requests is calculated every 10 seconds, -so `window` must be a multiple of 10 seconds. +**`window`** (default: 60s): The time over which to average the API's in-flight requests. The longer the window, the slower the autoscaler will react to changes in in-flight requests, since it is averaged over the `window`. An API's in-flight requests is calculated every 10 seconds, so `window` must be a multiple of 10 seconds.
-**`downscale_stabilization_period`** (default: 5m): The API will not scale below the highest recommendation made during -this period. Every 10 seconds, the autoscaler makes a recommendation based on all of the other configuration parameters -described here. It will then take the max of the current recommendation and all recommendations made during -the `downscale_stabilization_period`, and use that to determine the final number of replicas to scale to. Increasing -this value will cause the cluster to react more slowly to decreased traffic, and will reduce thrashing. +**`downscale_stabilization_period`** (default: 5m): The API will not scale below the highest recommendation made during this period. Every 10 seconds, the autoscaler makes a recommendation based on all of the other configuration parameters described here. It will then take the max of the current recommendation and all recommendations made during the `downscale_stabilization_period`, and use that to determine the final number of replicas to scale to. Increasing this value will cause the cluster to react more slowly to decreased traffic, and will reduce thrashing.
-**`upscale_stabilization_period`** (default: 1m): The API will not scale above the lowest recommendation made during -this period. Every 10 seconds, the autoscaler makes a recommendation based on all of the other configuration parameters -described here. It will then take the min of the current recommendation and all recommendations made during -the `upscale_stabilization_period`, and use that to determine the final number of replicas to scale to. Increasing this -value will cause the cluster to react more slowly to increased traffic, and will reduce thrashing. +**`upscale_stabilization_period`** (default: 1m): The API will not scale above the lowest recommendation made during this period. Every 10 seconds, the autoscaler makes a recommendation based on all of the other configuration parameters described here. It will then take the min of the current recommendation and all recommendations made during the `upscale_stabilization_period`, and use that to determine the final number of replicas to scale to. Increasing this value will cause the cluster to react more slowly to increased traffic, and will reduce thrashing.
-**`max_downscale_factor`** (default: 0.75): The maximum factor by which to scale down the API on a single scaling event. -For example, if `max_downscale_factor` is 0.5 and there are 10 running replicas, the autoscaler will not recommend fewer -than 5 replicas. Increasing this number will allow the cluster to shrink more quickly in response to dramatic dips in -traffic. +**`max_downscale_factor`** (default: 0.75): The maximum factor by which to scale down the API on a single scaling event. For example, if `max_downscale_factor` is 0.5 and there are 10 running replicas, the autoscaler will not recommend fewer than 5 replicas. Increasing this number will allow the cluster to shrink more quickly in response to dramatic dips in traffic.
-**`max_upscale_factor`** (default: 1.5): The maximum factor by which to scale up the API on a single scaling event. For -example, if `max_upscale_factor` is 10 and there are 5 running replicas, the autoscaler will not recommend more than 50 -replicas. Increasing this number will allow the cluster to grow more quickly in response to dramatic spikes in traffic. +**`max_upscale_factor`** (default: 1.5): The maximum factor by which to scale up the API on a single scaling event. For example, if `max_upscale_factor` is 10 and there are 5 running replicas, the autoscaler will not recommend more than 50 replicas. Increasing this number will allow the cluster to grow more quickly in response to dramatic spikes in traffic.
-**`downscale_tolerance`** (default: 0.05): Any recommendation falling within this factor below the current number of -replicas will not trigger a scale down event. For example, if `downscale_tolerance` is 0.1 and there are 20 running -replicas, a recommendation of 18 or 19 replicas will not be acted on, and the API will remain at 20 replicas. Increasing -this value will prevent thrashing, but setting it too high will prevent the cluster from maintaining it's optimal size. +**`downscale_tolerance`** (default: 0.05): Any recommendation falling within this factor below the current number of replicas will not trigger a scale down event. For example, if `downscale_tolerance` is 0.1 and there are 20 running replicas, a recommendation of 18 or 19 replicas will not be acted on, and the API will remain at 20 replicas. Increasing this value will prevent thrashing, but setting it too high will prevent the cluster from maintaining it's optimal size.
-**`upscale_tolerance`** (default: 0.05): Any recommendation falling within this factor above the current number of -replicas will not trigger a scale up event. For example, if `upscale_tolerance` is 0.1 and there are 20 running -replicas, a recommendation of 21 or 22 replicas will not be acted on, and the API will remain at 20 replicas. Increasing -this value will prevent thrashing, but setting it too high will prevent the cluster from maintaining it's optimal size. +**`upscale_tolerance`** (default: 0.05): Any recommendation falling within this factor above the current number of replicas will not trigger a scale up event. For example, if `upscale_tolerance` is 0.1 and there are 20 running replicas, a recommendation of 21 or 22 replicas will not be acted on, and the API will remain at 20 replicas. Increasing this value will prevent thrashing, but setting it too high will prevent the cluster from maintaining it's optimal size.
## Autoscaling instances -Cortex spins up and down instances based on the aggregate resource requests of all APIs. The number of instances will be -at least `min_instances` and no more than `max_instances` for each node group (configured during installation and modifiable -via `cortex cluster scale`). +Cortex spins up and down instances based on the aggregate resource requests of all APIs. The number of instances will be at least `min_instances` and no more than `max_instances` for each node group (configured during installation and modifiable via `cortex cluster scale`). -## Autoscaling responsiveness +## Overprovisioning -Assuming that `window` and `upscale_stabilization_period` are set to their default values (1 minute), it could take up -to 2 minutes of increased traffic before an extra replica is requested. As soon as the additional replica is requested, -the replica request will be visible in the output of `cortex get`, but the replica won't yet be running. If an extra -instance is required to schedule the newly requested replica, it could take a few minutes for AWS to provision the -instance (depending on the instance type), plus a few minutes for the newly provisioned instance to download your api -image and for the api to initialize (via its `__init__()` method). +The default value for `target_in_flight` is `max_concurrency`, which behaves well in many situations (see above for an explanation of how `target_in_flight` affects autoscaling). However, if your application is sensitive to spikes in traffic or if creating new replicas takes too long (see below), you may find it helpful to maintain extra capacity to handle the increased traffic while new replicas are being created. This can be accomplished by setting `target_in_flight` to a lower value relative to the expected replica's concurrency. The smaller `target_in_flight` is, the more unused capacity your API will have, and the more room it will have to handle sudden increased load. The increased request rate will still trigger the autoscaler, and your API will stabilize again (maintaining the overprovisioned capacity). + +For example, if you've determined that each replica in your API can handle 2 concurrent requests, you would typically set `target_in_flight` to 2. In a scenario where your API is receiving 8 concurrent requests on average, the autoscaler would maintain 4 live replicas (8/2 = 4). If you wanted to overprovision by 25%, you could set `target_in_flight` to 1.6, causing the autoscaler maintain 5 live replicas (8/1.6 = 5). + +## Autoscaling responsiveness -If you want the autoscaler to react as quickly as possible, set `upscale_stabilization_period` and `window` to their -minimum values (0s and 10s respectively). +Assuming that `window` and `upscale_stabilization_period` are set to their default values (1 minute), it could take up to 2 minutes of increased traffic before an extra replica is requested. As soon as the additional replica is requested, the replica request will be visible in the output of `cortex get`, but the replica won't yet be running. If an extra instance is required to schedule the newly requested replica, it could take a few minutes for AWS to provision the instance (depending on the instance type), plus a few minutes for the newly provisioned instance to download your api image and for the api to initialize. -If it takes a long time to initialize your API replica (i.e. install dependencies and run your handler's `__init__()` -function), consider building your own API image to use instead of the default image. With this approach, you can -pre-download/build/install any custom dependencies and bake them into the image. +Keep these delays in mind when considering overprovisioning (see above) and when determining appropriate values for `window` and `upscale_stabilization_period`. If you want the autoscaler to react as quickly as possible, set `upscale_stabilization_period` and `window` to their minimum values (0s and 10s respectively). diff --git a/docs/workloads/async/configuration.md b/docs/workloads/async/configuration.md index 595f698a19..01641bc7c1 100644 --- a/docs/workloads/async/configuration.md +++ b/docs/workloads/async/configuration.md @@ -1,107 +1,63 @@ # Configuration ```yaml -- name: - kind: AsyncAPI - handler: # detailed configuration below - compute: # detailed configuration below - autoscaling: # detailed configuration below - update_strategy: # detailed configuration below - networking: # detailed configuration below -``` - -## Handler - -### Python Handler - - - -```yaml -handler: - type: python - path: # path to a python file with a Handler class definition, relative to the Cortex root (required) - dependencies: # (optional) - pip: # relative path to requirements.txt (default: requirements.txt) - conda: # relative path to conda-packages.txt (default: conda-packages.txt) - shell: # relative path to a shell script for system package installation (default: dependencies.sh) - config: # arbitrary dictionary passed to the constructor of the Handler class (optional) - python_path: # path to the root of your Python folder that will be appended to PYTHONPATH (default: folder containing cortex.yaml) - image: # docker image to use for the handler (default: quay.io/cortexlabs/python-handler-cpu:master, quay.io/cortexlabs/python-handler-gpu:master-cuda10.2-cudnn8, or quay.io/cortexlabs/python-handler-inf:master based on compute) - env: # dictionary of environment variables - log_level: # log level that can be "debug", "info", "warning" or "error" (default: "info") - shm_size: # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null) -``` - -### Tensorflow Handler - - - -```yaml -handler: - type: tensorflow - path: # path to a python file with a Handler class definition, relative to the Cortex root (required) - dependencies: # (optional) - pip: # relative path to requirements.txt (default: requirements.txt) - conda: # relative path to conda-packages.txt (default: conda-packages.txt) - shell: # relative path to a shell script for system package installation (default: dependencies.sh) - models: # (required) - path: # S3 path to an exported SavedModel directory (e.g. s3://my-bucket/exported_model/) (either this, 'dir', or 'paths' must be provided) - paths: # list of S3 paths to exported SavedModel directories (either this, 'dir', or 'path' must be provided) - - name: # unique name for the model (e.g. text-generator) (required) - path: # S3 path to an exported SavedModel directory (e.g. s3://my-bucket/exported_model/) (required) - signature_key: # name of the signature def to use for prediction (required if your model has more than one signature def) - ... - dir: # S3 path to a directory containing multiple SavedModel directories (e.g. s3://my-bucket/models/) (either this, 'path', or 'paths' must be provided) - signature_key: # name of the signature def to use for prediction (required if your model has more than one signature def) - config: # arbitrary dictionary passed to the constructor of the Handler class (optional) - python_path: # path to the root of your Python folder that will be appended to PYTHONPATH (default: folder containing cortex.yaml) - image: # docker image to use for the handler (default: quay.io/cortexlabs/tensorflow-handler:master) - tensorflow_serving_image: # docker image to use for the TensorFlow Serving container (default: quay.io/cortexlabs/tensorflow-serving-cpu:master, quay.io/cortexlabs/tensorflow-serving-gpu:master, or quay.io/cortexlabs/tensorflow-serving-inf:master based on compute) - env: # dictionary of environment variables - log_level: # log level that can be "debug", "info", "warning" or "error" (default: "info") - shm_size: # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null) -``` - -## Compute - -```yaml -compute: - cpu: # CPU request per replica. One unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the "m" suffix (default: 200m) - gpu: # GPU request per replica. One unit of GPU corresponds to one virtual GPU (default: 0) - inf: # Inferentia request per replica. One unit of Inf corresponds to one virtual Inferentia chip (default: 0) - mem: # memory request per replica. One unit of memory is one byte and can be expressed as an integer or by using one of these suffixes: K, M, G, T (or their power-of two counterparts: Ki, Mi, Gi, Ti) (default: Null) - node_groups: # to select specific node groups (optional) -``` - -## Autoscaling - -```yaml -autoscaling: - min_replicas: # minimum number of replicas (default: 1) - max_replicas: # maximum number of replicas (default: 100) - init_replicas: # initial number of replicas (default: ) - max_replica_concurrency: # the maximum number of in-flight requests per replica before requests are rejected with error code 503 (default: 1024) - target_replica_concurrency: # the desired number of in-flight requests per replica, which the autoscaler tries to maintain (default: processes_per_replica * threads_per_process) - window: # the time over which to average the API's concurrency (default: 60s) - downscale_stabilization_period: # the API will not scale below the highest recommendation made during this period (default: 5m) - upscale_stabilization_period: # the API will not scale above the lowest recommendation made during this period (default: 1m) - max_downscale_factor: # the maximum factor by which to scale down the API on a single scaling event (default: 0.75) - max_upscale_factor: # the maximum factor by which to scale up the API on a single scaling event (default: 1.5) - downscale_tolerance: # any recommendation falling within this factor below the current number of replicas will not trigger a scale down event (default: 0.05) - upscale_tolerance: # any recommendation falling within this factor above the current number of replicas will not trigger a scale up event (default: 0.05) -``` - -## Update strategy - -```yaml -update_strategy: - max_surge: # maximum number of replicas that can be scheduled above the desired number of replicas during an update; can be an absolute number, e.g. 5, or a percentage of desired replicas, e.g. 10% (default: 25%) (set to 0 to disable rolling updates) - max_unavailable: # maximum number of replicas that can be unavailable during an update; can be an absolute number, e.g. 5, or a percentage of desired replicas, e.g. 10% (default: 25%) -``` - -## Networking - -```yaml - networking: - endpoint: # the endpoint for the API (default: ) +- name: # name of the API (required) + kind: AsyncAPI # must be "AsyncAPI" for async APIs (required) + pod: # pod configuration (required) + port: # port to which requests will be sent (default: 8080; exported as $CORTEX_PORT) + max_concurrency: # maximum number of requests that will be concurrently sent into the container (default: 1) + containers: # configurations for the containers to run (at least one constainer must be provided) + - name: # name of the container (required) + image: # docker image to use for the container (required) + command: # entrypoint (default: the docker image's ENTRYPOINT) + args: # arguments to the entrypoint (default: the docker image's CMD) + env: # dictionary of environment variables to set in the container (optional) + compute: # compute resource requests (default: see below) + cpu: # CPU request for the container; one unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the "m" suffix (default: 200m) + gpu: # GPU request for the container; one unit of GPU corresponds to one virtual GPU (default: 0) + inf: # Inferentia request for the container; one unit of inf corresponds to one virtual Inferentia chip (default: 0) + mem: # memory request for the container; one unit of memory is one byte and can be expressed as an integer or by using one of these suffixes: K, M, G, T (or their power-of two counterparts: Ki, Mi, Gi, Ti) (default: Null) + shm: # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null) + readiness_probe: # periodic probe of container readiness; traffic will not be sent into the pod unless all containers' readiness probes are succeeding (optional) + http_get: # specifies an http endpoint which must respond with status code 200 (only one of http_get, tcp_socket, and exec may be specified) + port: # the port to access on the container (required) + path: # the path to access on the HTTP server (default: /) + tcp_socket: # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified) + port: # the port to access on the container (required) + initial_delay_seconds: # number of seconds after the container has started before the probe is initiated (default: 0) + timeout_seconds: # number of seconds until the probe times out (default: 1) + period_seconds: # how often (in seconds) to perform the probe (default: 10) + success_threshold: # minimum consecutive successes for the probe to be considered successful after having failed (default: 1) + failure_threshold: # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3) + liveness_probe: # periodic probe of container liveness; container will be restarted if the probe fails (optional) + http_get: # specifies an http endpoint which must respond with status code 200 (only one of http_get, tcp_socket, and exec may be specified) + port: # the port to access on the container (required) + path: # the path to access on the HTTP server (default: /) + tcp_socket: # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified) + port: # the port to access on the container (required) + exec: # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified) + command: [] # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) + initial_delay_seconds: # number of seconds after the container has started before the probe is initiated (default: 0) + timeout_seconds: # number of seconds until the probe times out (default: 1) + period_seconds: # how often (in seconds) to perform the probe (default: 10) + success_threshold: # minimum consecutive successes for the probe to be considered successful after having failed (default: 1) + failure_threshold: # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3) + autoscaling: # autoscaling configuration (default: see below) + min_replicas: # minimum number of replicas (default: 1) + max_replicas: # maximum number of replicas (default: 100) + init_replicas: # initial number of replicas (default: ) + target_in_flight: # desired number of in-flight requests per replica (including requests actively being processed as well as queued), which the autoscaler tries to maintain (default: ) + window: # duration over which to average the API's in-flight requests per replica (default: 60s) + downscale_stabilization_period: # the API will not scale below the highest recommendation made during this period (default: 5m) + upscale_stabilization_period: # the API will not scale above the lowest recommendation made during this period (default: 1m) + max_downscale_factor: # maximum factor by which to scale down the API on a single scaling event (default: 0.75) + max_upscale_factor: # maximum factor by which to scale up the API on a single scaling event (default: 1.5) + downscale_tolerance: # any recommendation falling within this factor below the current number of replicas will not trigger a scale down event (default: 0.05) + upscale_tolerance: # any recommendation falling within this factor above the current number of replicas will not trigger a scale up event (default: 0.05) + node_groups: # a list of node groups on which this API can run (default: all node groups are eligible) + update_strategy: # deployment strategy to use when replacing existing replicas with new ones (default: see below) + max_surge: # maximum number of replicas that can be scheduled above the desired number of replicas during an update; can be an absolute number, e.g. 5, or a percentage of desired replicas, e.g. 10% (default: 25%) (set to 0 to disable rolling updates) + max_unavailable: # maximum number of replicas that can be unavailable during an update; can be an absolute number, e.g. 5, or a percentage of desired replicas, e.g. 10% (default: 25%) + networking: # networking configuration (default: see below) + endpoint: # endpoint for the API (default: ) ``` diff --git a/docs/workloads/async/handler.md b/docs/workloads/async/handler.md deleted file mode 100644 index fe3f15d780..0000000000 --- a/docs/workloads/async/handler.md +++ /dev/null @@ -1,240 +0,0 @@ -# Handler implementation - -Your handler can be used to process any asynchronous workloads. It can also be used for running ML models using a variety of frameworks such as: PyTorch, ONNX, scikit-learn, XGBoost, TensorFlow (if not using `SavedModel`s), etc. - -If you plan on deploying models with TensorFlow in `SavedModel` format, you can also use the [TensorFlow Handler](models.md) that was specifically built for this purpose. - -## Project files - -Cortex makes all files in the project directory (i.e. the directory which contains `cortex.yaml`) available for use in -your handler implementation. Python bytecode files (`*.pyc`, `*.pyo`, `*.pyd`), files or folders that start with `.`, -and the api configuration file (e.g. `cortex.yaml`) are excluded. - -The following files can also be added at the root of the project's directory: - -* `.cortexignore` file, which follows the same syntax and behavior as a [.gitignore file](https://git-scm.com/docs/gitignore). This may be necessary if you are reaching the size limit for your project directory (32mb). -* `.env` file, which exports environment variables that can be used in the handler. Each line of this file must follow - the `VARIABLE=value` format. - -For example, if your directory looks like this: - -```text -./my-classifier/ -├── cortex.yaml -├── values.json -├── handler.py -├── ... -└── requirements.txt -``` - -You can access `values.json` in your Handler class like this: - -```python -# handler.py - -import json - -class Handler: - def __init__(self, config): - with open('values.json', 'r') as values_file: - values = json.load(values_file) - self.values = values -``` - -## Interface - -```python -# initialization code and variables can be declared here in global scope - -class Handler: - def __init__(self, config, metrics_client): - """(Required) Called once before the API becomes available. Performs - setup such as downloading/initializing the model or downloading a - vocabulary. - - Args: - config (required): Dictionary passed from API configuration (if - specified). This may contain information on where to download - the model and/or metadata. - metrics_client (optional): The cortex metrics client, which allows - you to push custom metrics in order to build custom dashboards - in grafana. - """ - pass - - def handle_async(self, payload, request_id): - """(Required) Called once per request. Preprocesses the request payload - (if necessary), runs the workload, and postprocesses the resulting output - (if necessary). - - Args: - payload (optional): The request payload (see below for the possible - payload types). - request_id (optional): The request id string that identifies a workload - - Returns: - JSON-serializeable result. - """ - pass -``` - -For proper separation of concerns, it is recommended to use the constructor's `config` parameter for information such as -from where to download the model and initialization files, or any configurable model parameters. You define `config` in -your API configuration, and it is passed through to your handler's constructor. - -Your API can accept requests with different types of payloads. Navigate to the [API requests](#api-requests) section to -learn about how headers can be used to change the type of `payload` that is passed into your `handle_async` method. - -At this moment, the AsyncAPI `handle_async` method can only return `JSON`-parseable objects. Navigate to -the [API responses](#api-responses) section to learn about how to configure it. - -## API requests - -The type of the `payload` parameter in `handle_async(self, payload)` can vary based on the content type of the request. -The `payload` parameter is parsed according to the `Content-Type` header in the request. Here are the parsing rules (see -below for examples): - -1. For `Content-Type: application/json`, `payload` will be the parsed JSON body. -1. For `Content-Type: text/plain`, `payload` will be a string. `utf-8` encoding is assumed, unless specified otherwise ( - e.g. via `Content-Type: text/plain; charset=us-ascii`) -1. For all other `Content-Type` values, `payload` will be the raw `bytes` of the request body. - -Here are some examples: - -### JSON data - -#### Making the request - -```bash -curl http://***.amazonaws.com/my-api \ - -X POST -H "Content-Type: application/json" \ - -d '{"key": "value"}' -``` - -#### Reading the payload - -When sending a JSON payload, the `payload` parameter will be a Python object: - -```python -class Handler: - def __init__(self, config): - pass - - def handle_async(self, payload): - print(payload["key"]) # prints "value" -``` - -### Binary data - -#### Making the request - -```bash -curl http://***.amazonaws.com/my-api \ - -X POST -H "Content-Type: application/octet-stream" \ - --data-binary @object.pkl -``` - -#### Reading the payload - -Since the `Content-Type: application/octet-stream` header is used, the `payload` parameter will be a `bytes` object: - -```python -import pickle - - -class Handler: - def __init__(self, config): - pass - - def handle_async(self, payload): - obj = pickle.loads(payload) - print(obj["key"]) # prints "value" -``` - -Here's an example if the binary data is an image: - -```python -from PIL import Image -import io - - -class Handler: - def __init__(self, config): - pass - - def handle_async(self, payload): - img = Image.open(io.BytesIO(payload)) # read the payload bytes as an image - print(img.size) -``` - -### Text data - -#### Making the request - -```bash -curl http://***.amazonaws.com/my-api \ - -X POST -H "Content-Type: text/plain" \ - -d "hello world" -``` - -#### Reading the payload - -Since the `Content-Type: text/plain` header is used, the `payload` parameter will be a `string` object: - -```python -class Handle: - def __init__(self, config): - pass - - def handle_async(self, payload): - print(payload) # prints "hello world" -``` - -## API responses - -The return value of your `handle_async()` method must be a JSON-serializable dictionary. The result for -each request will remain queryable for 7 days after the request was completed. - -## Chaining APIs - -It is possible to make requests from one API to another within a Cortex cluster. All running APIs are accessible from -within the handler at `http://api-:8888/`, where `` is the name of the API you are making a -request to. - -For example, if there is an api named `text-generator` running in the cluster, you could make a request to it from a -different API by using: - -```python -import requests - - -class Handler: - def handle_async(self, payload): - response = requests.post("http://api-text-generator:8888/", json={"text": "machine learning is"}) - # ... -``` - -## Structured logging - -You can use Cortex's logger in your handler implemention to log in JSON. This will enrich your logs with Cortex's -metadata, and you can add custom metadata to the logs by adding key value pairs to the `extra` key when using the -logger. For example: - -```python -... -from cortex_internal.lib.log import logger as log - - -class Handler: - def handle_async(self, payload): - log.info("received payload", extra={"payload": payload}) -``` - -The dictionary passed in via the `extra` will be flattened by one level. e.g. - -```text -{"asctime": "2021-01-19 15:14:05,291", "levelname": "INFO", "message": "received payload", "process": 235, "payload": "this movie is awesome"} -``` - -To avoid overriding essential Cortex metadata, please refrain from specifying the following extra keys: `asctime` -, `levelname`, `message`, `labels`, and `process`. Log lines greater than 5 MB in size will be ignored. diff --git a/docs/workloads/async/models.md b/docs/workloads/async/models.md deleted file mode 100644 index 6f1c4efff5..0000000000 --- a/docs/workloads/async/models.md +++ /dev/null @@ -1,203 +0,0 @@ -# TensorFlow Models - -In addition to the [standard Python Handler](handler.md), Cortex also supports another handler called the TensorFlow handler, which can be used to deploy TensorFlow models exported as `SavedModel` models. - -## Interface - -**Uses TensorFlow version 2.3.0 by default** - -```python -class Handler: - def __init__(self, config, tensorflow_client, metrics_client): - """(Required) Called once before the API becomes available. Performs - setup such as downloading/initializing a vocabulary. - - Args: - config (required): Dictionary passed from API configuration (if - specified). - tensorflow_client (required): TensorFlow client which is used to - make predictions. This should be saved for use in handle_async(). - metrics_client (optional): The cortex metrics client, which allows - you to push custom metrics in order to build custom dashboards - in grafana. - """ - self.client = tensorflow_client - # Additional initialization may be done here - - def handle_async(self, payload, request_id): - """(Required) Called once per request. Preprocesses the request payload - (if necessary), runs inference (e.g. by calling - self.client.predict(model_input)), and postprocesses the inference - output (if necessary). - - Args: - payload (optional): The request payload (see below for the possible - payload types). - request_id (optional): The request id string that identifies a workload - - Returns: - Prediction or a batch of predictions. - """ - pass -``` - - - -Cortex provides a `tensorflow_client` to your handler's constructor. `tensorflow_client` is an instance -of [TensorFlowClient](https://github.com/cortexlabs/cortex/tree/master/python/serve/cortex_internal/lib/client/tensorflow.py) -that manages a connection to a TensorFlow Serving container to make predictions using your model. It should be saved as -an instance variable in your handler class, and your `handle_async()` function should call `tensorflow_client.predict()` to make -an inference with your exported TensorFlow model. Preprocessing of the JSON payload and postprocessing of predictions -can be implemented in your `handle_async()` function as well. - -When multiple models are defined using the Handler's `models` field, the `tensorflow_client.predict()` method expects a second argument `model_name` which must hold the name of the model that you want to use for inference (for example: `self.client.predict(payload, "text-generator")`). There is also an optional third argument to specify the model version. - -If you need to share files between your handler implementation and the TensorFlow Serving container, you can create a new directory within `/mnt` (e.g. `/mnt/user`) and write files to it. The entire `/mnt` directory is shared between containers, but do not write to any of the directories in `/mnt` that already exist (they are used internally by Cortex). - -## `predict` method - -Inference is performed by using the `predict` method of the `tensorflow_client` that's passed to the handler's constructor: - -```python -def predict(model_input, model_name, model_version) -> dict: - """ - Run prediction. - - Args: - model_input: Input to the model. - model_name (optional): Name of the model to retrieve (when multiple models are deployed in an API). - When handler.models.paths is specified, model_name should be the name of one of the models listed in the API config. - When handler.models.dir is specified, model_name should be the name of a top-level directory in the models dir. - model_version (string, optional): Version of the model to retrieve. Can be omitted or set to "latest" to select the highest version. - - Returns: - dict: TensorFlow Serving response converted to a dictionary. - """ -``` - -## Specifying models - -Whenever a model path is specified in an API configuration file, it should be a path to an S3 prefix which contains your exported model. Directories may include a single model, or multiple folders each with a single model (note that a "single model" need not be a single file; there can be multiple files for a single model). When multiple folders are used, the folder names must be integer values, and will be interpreted as the model version. Model versions can be any integer, but are typically integer timestamps. It is always assumed that the highest version number is the latest version of your model. - -### API spec - -#### Single model - -The most common pattern is to serve a single model per API. The path to the model is specified in the `path` field in the `handler.models` configuration. For example: - -```yaml -# cortex.yaml - -- name: iris-classifier - kind: AsyncAPI - handler: - # ... - type: tensorflow - models: - path: s3://my-bucket/models/text-generator/ -``` - -#### Multiple models - -It is possible to serve multiple models from a single API. The paths to the models are specified in the api configuration, either via the `models.paths` or `models.dir` field in the `handler` configuration. For example: - -```yaml -# cortex.yaml - -- name: iris-classifier - kind: AsyncAPI - handler: - # ... - type: tensorflow - models: - paths: - - name: iris-classifier - path: s3://my-bucket/models/text-generator/ - # ... -``` - -or: - -```yaml -# cortex.yaml - -- name: iris-classifier - kind: AsyncAPI - handler: - # ... - type: tensorflow - models: - dir: s3://my-bucket/models/ -``` - -When using the `models.paths` field, each path must be a valid model directory (see above for valid model directory structures). - -When using the `models.dir` field, the directory provided may contain multiple subdirectories, each of which is a valid model directory. For example: - -```text - s3://my-bucket/models/ - ├── text-generator - | └── * (model files) - └── sentiment-analyzer - ├── 24753823/ - | └── * (model files) - └── 26234288/ - └── * (model files) -``` - -In this case, there are two models in the directory, one of which is named "text-generator", and the other is named "sentiment-analyzer". - -### Structure - -#### On CPU/GPU - -The model path must be a SavedModel export: - -```text - s3://my-bucket/models/text-generator/ - ├── saved_model.pb - └── variables/ - ├── variables.index - ├── variables.data-00000-of-00003 - ├── variables.data-00001-of-00003 - └── variables.data-00002-of-... -``` - -or for a versioned model: - -```text - s3://my-bucket/models/text-generator/ - ├── 1523423423/ (version number, usually a timestamp) - | ├── saved_model.pb - | └── variables/ - | ├── variables.index - | ├── variables.data-00000-of-00003 - | ├── variables.data-00001-of-00003 - | └── variables.data-00002-of-... - └── 2434389194/ (version number, usually a timestamp) - ├── saved_model.pb - └── variables/ - ├── variables.index - ├── variables.data-00000-of-00003 - ├── variables.data-00001-of-00003 - └── variables.data-00002-of-... -``` - -#### On Inferentia - -When Inferentia models are used, the directory structure is slightly different: - -```text - s3://my-bucket/models/text-generator/ - └── saved_model.pb -``` - -or for a versioned model: - -```text - s3://my-bucket/models/text-generator/ - ├── 1523423423/ (version number, usually a timestamp) - | └── saved_model.pb - └── 2434389194/ (version number, usually a timestamp) - └── saved_model.pb -``` diff --git a/docs/workloads/async/webhooks.md b/docs/workloads/async/webhooks.md index a9c085c2ad..f822725a7f 100644 --- a/docs/workloads/async/webhooks.md +++ b/docs/workloads/async/webhooks.md @@ -8,42 +8,44 @@ completion or failure, and the URL known in advance is some other service that w ## Example -Below is a guideline for implementing webhooks for an `AsyncAPI` workload. +Below is an example implementing webhooks for an `AsyncAPI` workload using FastAPI. ```python +import os import time -from datetime import datetime - import requests +from datetime import datetime +from fastapi import FastAPI, Header STATUS_COMPLETED = "completed" STATUS_FAILED = "failed" +webhook_url = os.getenv("WEBHOOK_URL") # the webhook url is set as an environment variable + +app = FastAPI() + -class Handler: - def __init__(self, config): - self.webhook_url = config["webhook_url"] # the webhook url is passed in the config +@app.post("/") +async def handle(x_cortex_request_id=Header(None)): + try: + time.sleep(60) # simulates a long workload + send_report(x_cortex_request_id, STATUS_COMPLETED, result={"data": "hello"}) + except Exception as err: + send_report(x_cortex_request_id, STATUS_FAILED) + raise err # the original exception should still be raised - def handle_async(self, payload, request_id): - try: - time.sleep(60) # simulates a long workload - self.send_report(request_id, STATUS_COMPLETED, result={"data": "hello"}) - except Exception as err: - self.send_report(request_id, STATUS_FAILED) - raise err # the original exception should still be raised! - # this is a utility method - def send_report(self, request_id, status, result=None): - response = {"id": request_id, "status": status} +def send_report(request_id, status, result=None): + response = {"id": request_id, "status": status} - if result is not None and status == STATUS_COMPLETED: - timestamp = datetime.utcnow().isoformat() - response.update({"result": result, "timestamp": timestamp}) + if result is not None and status == STATUS_COMPLETED: + timestamp = datetime.utcnow().isoformat() + response.update({"result": result, "timestamp": timestamp}) - try: - requests.post(url=self.webhook_url, json=response) - except Exception: - pass + try: + requests.post(url=webhook_url, json=response) + except Exception: + pass ``` ## Development diff --git a/docs/workloads/batch/batch-apis.md b/docs/workloads/batch/batch-apis.md new file mode 100644 index 0000000000..ab71f0be03 --- /dev/null +++ b/docs/workloads/batch/batch-apis.md @@ -0,0 +1,3 @@ +# Batch APIs + +Batch APIs run distributed and fault-tolerant batch processing jobs on demand. They can be used for batch inference or data processing workloads. diff --git a/docs/workloads/batch/configuration.md b/docs/workloads/batch/configuration.md index f7b6ecc9c4..31ef274089 100644 --- a/docs/workloads/batch/configuration.md +++ b/docs/workloads/batch/configuration.md @@ -1,79 +1,47 @@ # Configuration ```yaml -- name: - kind: BatchAPI - handler: # detailed configuration below - compute: # detailed configuration below - networking: # detailed configuration below -``` - -## Handler - -### Python Handler - - -```yaml -handler: - type: python - path: # path to a python file with a Handler class definition, relative to the Cortex root (required) - config: # arbitrary dictionary passed to the constructor of the Handler class (can be overridden by config passed in job submission) (optional) - python_path: # path to the root of your Python folder that will be appended to PYTHONPATH (default: folder containing cortex.yaml) - image: # docker image to use for the handler (default: quay.io/cortexlabs/python-handler-cpu:master or quay.io/cortexlabs/python-handler-gpu:master-cuda10.2-cudnn8 based on compute) - env: # dictionary of environment variables - log_level: # log level that can be "debug", "info", "warning" or "error" (default: "info") - shm_size: # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null) - dependencies: # (optional) - pip: # relative path to requirements.txt (default: requirements.txt) - conda: # relative path to conda-packages.txt (default: conda-packages.txt) - shell: # relative path to a shell script for system package installation (default: dependencies.sh) -``` - -### TensorFlow Handler - - -```yaml -handler: - type: tensorflow - path: # path to a python file with a Handler class definition, relative to the Cortex root (required) - models: # use this to serve a single model or multiple ones - path: # S3 path to an exported model (e.g. s3://my-bucket/exported_model) (either this or 'paths' field must be provided) - paths: # (either this or 'path' must be provided) - - name: # unique name for the model (e.g. text-generator) (required) - path: # S3 path to an exported model (e.g. s3://my-bucket/exported_model) (required) - signature_key: # name of the signature def to use for prediction (required if your model has more than one signature def) - ... - signature_key: # name of the signature def to use for prediction (required if your model has more than one signature def) - server_side_batching: # (optional) - max_batch_size: # the maximum number of requests to aggregate before running inference - batch_interval: # the maximum amount of time to spend waiting for additional requests before running inference on the batch of requests - config: # arbitrary dictionary passed to the constructor of the Handler class (can be overridden by config passed in job submission) (optional) - python_path: # path to the root of your Python folder that will be appended to PYTHONPATH (default: folder containing cortex.yaml) - image: # docker image to use for the handler (default: quay.io/cortexlabs/tensorflow-handler:master) - tensorflow_serving_image: # docker image to use for the TensorFlow Serving container (default: quay.io/cortexlabs/tensorflow-serving-cpu:master or quay.io/cortexlabs/tensorflow-serving-gpu:master based on compute) - env: # dictionary of environment variables - log_level: # log level that can be "debug", "info", "warning" or "error" (default: "info") - shm_size: # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null) - dependencies: # (optional) - pip: # relative path to requirements.txt (default: requirements.txt) - conda: # relative path to conda-packages.txt (default: conda-packages.txt) - shell: # relative path to a shell script for system package installation (default: dependencies.sh) -``` - -## Compute - -```yaml -compute: - cpu: # CPU request per worker. One unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the "m" suffix (default: 200m) - gpu: # GPU request per worker. One unit of GPU corresponds to one virtual GPU (default: 0) - inf: # Inferentia request per replica. One unit of Inf corresponds to one virtual Inferentia chip (default: 0) - mem: # memory request per worker. One unit of memory is one byte and can be expressed as an integer or by using one of these suffixes: K, M, G, T (or their power-of two counterparts: Ki, Mi, Gi, Ti) (default: Null) - node_groups: # to select specific node groups (optional) -``` - -## Networking - -```yaml -networking: - endpoint: # the endpoint for the API (default: ) +- name: # name of the API (required) + kind: BatchAPI # must be "BatchAPI" for batch APIs (required) + pod: # pod configuration (required) + port: # port to which requests will be sent (default: 8080; exported as $CORTEX_PORT) + containers: # configurations for the containers to run (at least one constainer must be provided) + - name: # name of the container (required) + image: # docker image to use for the container (required) + command: # entrypoint (required) + args: # arguments to the entrypoint (default: no args) + env: # dictionary of environment variables to set in the container (optional) + compute: # compute resource requests (default: see below) + cpu: # CPU request for the container; one unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the "m" suffix (default: 200m) + gpu: # GPU request for the container; one unit of GPU corresponds to one virtual GPU (default: 0) + inf: # Inferentia request for the container; one unit of inf corresponds to one virtual Inferentia chip (default: 0) + mem: # memory request for the container; one unit of memory is one byte and can be expressed as an integer or by using one of these suffixes: K, M, G, T (or their power-of two counterparts: Ki, Mi, Gi, Ti) (default: Null) + shm: # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null) + readiness_probe: # periodic probe of container readiness; traffic will not be sent into the pod unless all containers' readiness probes are succeeding (optional) + http_get: # specifies an http endpoint which must respond with status code 200 (only one of http_get, tcp_socket, and exec may be specified) + port: # the port to access on the container (required) + path: # the path to access on the HTTP server (default: /) + tcp_socket: # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified) + port: # the port to access on the container (required) + initial_delay_seconds: # number of seconds after the container has started before the probe is initiated (default: 0) + timeout_seconds: # number of seconds until the probe times out (default: 1) + period_seconds: # how often (in seconds) to perform the probe (default: 10) + success_threshold: # minimum consecutive successes for the probe to be considered successful after having failed (default: 1) + failure_threshold: # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3) + liveness_probe: # periodic probe of container liveness; container will be restarted if the probe fails (optional) + http_get: # specifies an http endpoint which must respond with status code 200 (only one of http_get, tcp_socket, and exec may be specified) + port: # the port to access on the container (required) + path: # the path to access on the HTTP server (default: /) + tcp_socket: # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified) + port: # the port to access on the container (required) + exec: # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified) + command: [] # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) + initial_delay_seconds: # number of seconds after the container has started before the probe is initiated (default: 0) + timeout_seconds: # number of seconds until the probe times out (default: 1) + period_seconds: # how often (in seconds) to perform the probe (default: 10) + success_threshold: # minimum consecutive successes for the probe to be considered successful after having failed (default: 1) + failure_threshold: # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3) + node_groups: # a list of node groups on which this API can run (default: all node groups are eligible) + networking: # networking configuration (default: see below) + endpoint: # endpoint for the API (default: ) ``` diff --git a/docs/workloads/batch/handler.md b/docs/workloads/batch/handler.md deleted file mode 100644 index 35eaeda7d5..0000000000 --- a/docs/workloads/batch/handler.md +++ /dev/null @@ -1,125 +0,0 @@ -# Handler implementation - -Batch APIs run distributed and fault-tolerant batch processing jobs on-demand. They can be used for batch inference or data processing workloads. It can also be used for running ML models using a variety of frameworks such as: PyTorch, ONNX, scikit-learn, XGBoost, TensorFlow (if not using `SavedModel`s), etc. - -If you plan on deploying models with TensorFlow in `SavedModel` format and run inferences in batches, you can also use the [TensorFlow Handler](models.md) that was specifically built for this purpose. - -## Project files - -Cortex makes all files in the project directory (i.e. the directory which contains `cortex.yaml`) available for use in your Handler class implementation. Python bytecode files (`*.pyc`, `*.pyo`, `*.pyd`), files or folders that start with `.`, and the api configuration file (e.g. `cortex.yaml`) are excluded. - -The following files can also be added at the root of the project's directory: - -* `.cortexignore` file, which follows the same syntax and behavior as a [.gitignore file](https://git-scm.com/docs/gitignore). This may be necessary if you are reaching the size limit for your project directory (32mb). -* `.env` file, which exports environment variables that can be used in the handler class. Each line of this file must follow the `VARIABLE=value` format. - -For example, if your directory looks like this: - -```text -./my-classifier/ -├── cortex.yaml -├── values.json -├── handler.py -├── ... -└── requirements.txt -``` - -You can access `values.json` in your Handler class like this: - -```python -# handler.py - -import json - -class Handler: - def __init__(self, config): - with open('values.json', 'r') as values_file: - values = json.load(values_file) - self.values = values -``` - -## Interface - -```python -# initialization code and variables can be declared here in global scope - -class Handler: - def __init__(self, config, job_spec): - """(Required) Called once during each worker initialization. Performs - setup such as downloading/initializing the model or downloading a - vocabulary. - - Args: - config (required): Dictionary passed from API configuration (if - specified) merged with configuration passed in with Job - Submission API. If there are conflicting keys, values in - configuration specified in Job submission takes precedence. - job_spec (optional): Dictionary containing the following fields: - "job_id": A unique ID for this job - "api_name": The name of this batch API - "config": The config that was provided in the job submission - "workers": The number of workers for this job - "total_batch_count": The total number of batches in this job - "start_time": The time that this job started - """ - pass - - def handle_batch(self, payload, batch_id): - """(Required) Called once per batch. Preprocesses the batch payload (if - necessary), runs inference, postprocesses the inference output (if - necessary), and writes the results to storage (i.e. S3 or a - database, if desired). - - Args: - payload (required): a batch (i.e. a list of one or more samples). - batch_id (optional): uuid assigned to this batch. - Returns: - Nothing - """ - pass - - def on_job_complete(self): - """(Optional) Called once after all batches in the job have been - processed. Performs post job completion tasks such as aggregating - results, executing web hooks, or triggering other jobs. - """ - pass -``` - -## Structured logging - -You can use Cortex's logger in your handler implemention to log in JSON. This will enrich your logs with Cortex's metadata, and you can add custom metadata to the logs by adding key value pairs to the `extra` key when using the logger. For example: - -```python -... -from cortex_internal.lib.log import logger as cortex_logger - -class Handler: - def handle_batch(self, payload, batch_id): - ... - cortex_logger.info("completed processing batch", extra={"batch_id": batch_id, "confidence": confidence}) -``` - -The dictionary passed in via the `extra` will be flattened by one level. e.g. - -```text -{"asctime": "2021-01-19 15:14:05,291", "levelname": "INFO", "message": "completed processing batch", "process": 235, "batch_id": "iuasyd8f7", "confidence": 0.97} -``` - -To avoid overriding essential Cortex metadata, please refrain from specifying the following extra keys: `asctime`, `levelname`, `message`, `labels`, and `process`. Log lines greater than 5 MB in size will be ignored. - -## Cortex Python client - -A default [Cortex Python client](../../clients/python.md#cortex.client.client) environment has been configured for your API. This can be used for deploying/deleting/updating or submitting jobs to your running cluster based on the execution flow of your batch handler. For example: - -```python -import cortex - -class Handler: - def on_job_complete(self): - ... - # get client pointing to the default environment - client = cortex.client() - # deploy API in the existing cluster using the artifacts in the previous step - client.deploy(...) -``` diff --git a/docs/workloads/batch/jobs.md b/docs/workloads/batch/jobs.md index 632da17870..aa2f34418a 100644 --- a/docs/workloads/batch/jobs.md +++ b/docs/workloads/batch/jobs.md @@ -1,6 +1,6 @@ # BatchAPI jobs -## Get the TaskAPI endpoint +## Get the Batch API's endpoint ```bash cortex get @@ -42,7 +42,7 @@ POST : ], "batch_size": , # the number of items per batch (the handle_batch() function is called once per batch) (required) } - "config": { # custom fields for this specific job (will override values in `config` specified in your api configuration) (optional) + "config": { # arbitrary input for this specific job (optional) "string": } } @@ -65,6 +65,8 @@ RESPONSE: } ``` +The entire job specification is written to `/cortex/spec/job.json` in the API containers. + ### S3 file paths If your input data is a list of files such as images/videos in an S3 directory, you can define `file_path_lister` in your submission request payload. You can use `file_path_lister.s3_paths` to specify a list of files or prefixes, and `file_path_lister.includes` and/or `file_path_lister.excludes` to remove unwanted files. The S3 file paths will be aggregated into batches of size `file_path_lister.batch_size`. To learn more about fine-grained S3 file filtering see [filtering files](#filtering-files). @@ -81,19 +83,19 @@ If a single S3 file contains a lot of samples/rows, try the next submission stra ```yaml POST : { - "workers": , # the number of workers to allocate for this job (required) - "timeout": , # duration in seconds since the submission of a job before it is terminated (optional) + "workers": , # the number of workers to allocate for this job (required) + "timeout": , # duration in seconds since the submission of a job before it is terminated (optional) "sqs_dead_letter_queue": { # specify a queue to redirect failed batches (optional) "arn": , # arn of dead letter queue e.g. arn:aws:sqs:us-west-2:123456789:failed.fifo "max_receive_count": # number of a times a batch is allowed to be handled by a worker before it is considered to be failed and transferred to the dead letter queue (must be >= 1) }, "file_path_lister": { - "s3_paths": [], # can be S3 prefixes or complete S3 paths (required) - "includes": [], # glob patterns (optional) - "excludes": [], # glob patterns (optional) - "batch_size": , # the number of S3 file paths per batch (the handle_batch() function is called once per batch) (required) + "s3_paths": [], # can be S3 prefixes or complete S3 paths (required) + "includes": [], # glob patterns (optional) + "excludes": [], # glob patterns (optional) + "batch_size": , # the number of S3 file paths per batch (the handle_batch() function is called once per batch) (required) } - "config": { # custom fields for this specific job (will override values in `config` specified in your api configuration) (optional) + "config": { # arbitrary input for this specific job (optional) "string": } } @@ -116,6 +118,8 @@ RESPONSE: } ``` +The entire job specification is written to `/cortex/spec/job.json` in the API containers. + ### Newline delimited JSON files in S3 If your input dataset is a newline delimited json file in an S3 directory (or a list of them), you can define `delimited_files` in your request payload to break up the contents of the file into batches of size `delimited_files.batch_size`. @@ -131,19 +135,19 @@ This submission pattern is useful in the following scenarios: ```yaml POST : { - "workers": , # the number of workers to allocate for this job (required) - "timeout": , # duration in seconds since the submission of a job before it is terminated (optional) + "workers": , # the number of workers to allocate for this job (required) + "timeout": , # duration in seconds since the submission of a job before it is terminated (optional) "sqs_dead_letter_queue": { # specify a queue to redirect failed batches (optional) "arn": , # arn of dead letter queue e.g. arn:aws:sqs:us-west-2:123456789:failed.fifo "max_receive_count": # number of a times a batch is allowed to be handled by a worker before it is considered to be failed and transferred to the dead letter queue (must be >= 1) }, "delimited_files": { - "s3_paths": [], # can be S3 prefixes or complete S3 paths (required) - "includes": [], # glob patterns (optional) - "excludes": [], # glob patterns (optional) - "batch_size": , # the number of json objects per batch (the handle_batch() function is called once per batch) (required) + "s3_paths": [], # can be S3 prefixes or complete S3 paths (required) + "includes": [], # glob patterns (optional) + "excludes": [], # glob patterns (optional) + "batch_size": , # the number of json objects per batch (the handle_batch() function is called once per batch) (required) } - "config": { # custom fields for this specific job (will override values in `config` specified in your api configuration) (optional) + "config": { # arbitrary input for this specific job (optional) "string": } } @@ -166,6 +170,8 @@ RESPONSE: } ``` +The entire job specification is written to `/cortex/spec/job.json` in the API containers. + ## Get a job's status ```bash @@ -188,19 +194,19 @@ RESPONSE: "api_id": , "sqs_url": , "status": , - "batches_in_queue": # number of batches remaining in the queue + "batches_in_queue": # number of batches remaining in the queue "batch_metrics": { - "succeeded": # number of succeeded batches - "failed": int # number of failed attempts + "succeeded": # number of succeeded batches + "failed": int # number of failed attempts "avg_time_per_batch": (optional) # average time spent working on a batch (only considers successful attempts) }, - "worker_counts": { # worker counts are only available while a job is running - "pending": , # number of workers that are waiting for compute resources to be provisioned - "initializing": , # number of workers that are initializing (downloading images or running your handler's init function) - "running": , # number of workers that are actively working on batches from the queue - "succeeded": , # number of workers that have completed after verifying that the queue is empty - "failed": , # number of workers that have failed - "stalled": , # number of workers that have been stuck in pending for more than 10 minutes + "worker_counts": { # worker counts are only available while a job is running + "pending": , # number of workers that are waiting for compute resources to be provisioned + "initializing": , # number of workers that are initializing (downloading images or running your handler's init function) + "running": , # number of workers that are actively working on batches from the queue + "succeeded": , # number of workers that have completed after verifying that the queue is empty + "failed": , # number of workers that have failed + "stalled": , # number of workers that have been stuck in pending for more than 10 minutes }, "created_time": "start_time": diff --git a/docs/workloads/batch/models.md b/docs/workloads/batch/models.md deleted file mode 100644 index 9f248482ed..0000000000 --- a/docs/workloads/batch/models.md +++ /dev/null @@ -1,210 +0,0 @@ -# TensorFlow Models - -In addition to the [standard Python Handler](handler.md), Cortex also supports another handler called the TensorFlow handler, which can be used to run TensorFlow models exported as `SavedModel` models. - -## Interface - -**Uses TensorFlow version 2.3.0 by default** - -```python -class Handler: - def __init__(self, tensorflow_client, config, job_spec): - """(Required) Called once during each worker initialization. Performs - setup such as downloading/initializing the model or downloading a - vocabulary. - - Args: - tensorflow_client (required): TensorFlow client which is used to - make predictions. This should be saved for use in handle_batch(). - config (required): Dictionary passed from API configuration (if - specified) merged with configuration passed in with Job - Submission API. If there are conflicting keys, values in - configuration specified in Job submission takes precedence. - job_spec (optional): Dictionary containing the following fields: - "job_id": A unique ID for this job - "api_name": The name of this batch API - "config": The config that was provided in the job submission - "workers": The number of workers for this job - "total_batch_count": The total number of batches in this job - "start_time": The time that this job started - """ - self.client = tensorflow_client - # Additional initialization may be done here - - def handle_batch(self, payload, batch_id): - """(Required) Called once per batch. Preprocesses the batch payload (if - necessary), runs inference (e.g. by calling - self.client.predict(model_input)), postprocesses the inference output - (if necessary), and writes the predictions to storage (i.e. S3 or a - database, if desired). - - Args: - payload (required): a batch (i.e. a list of one or more samples). - batch_id (optional): uuid assigned to this batch. - Returns: - Nothing - """ - pass - - def on_job_complete(self): - """(Optional) Called once after all batches in the job have been - processed. Performs post job completion tasks such as aggregating - results, executing web hooks, or triggering other jobs. - """ - pass -``` - - -Cortex provides a `tensorflow_client` to your Handler class' constructor. `tensorflow_client` is an instance of [TensorFlowClient](https://github.com/cortexlabs/cortex/tree/master/python/serve/cortex_internal/lib/client/tensorflow.py) that manages a connection to a TensorFlow Serving container to make predictions using your model. It should be saved as an instance variable in your Handler class, and your `handle_batch()` function should call `tensorflow_client.predict()` to make an inference with your exported TensorFlow model. Preprocessing of the JSON payload and postprocessing of predictions can be implemented in your `handle_batch()` function as well. - -When multiple models are defined using the Handler's `models` field, the `tensorflow_client.predict()` method expects a second argument `model_name` which must hold the name of the model that you want to use for inference (for example: `self.client.predict(payload, "text-generator")`). There is also an optional third argument to specify the model version. - -If you need to share files between your handler implementation and the TensorFlow Serving container, you can create a new directory within `/mnt` (e.g. `/mnt/user`) and write files to it. The entire `/mnt` directory is shared between containers, but do not write to any of the directories in `/mnt` that already exist (they are used internally by Cortex). - -## `predict` method - -Inference is performed by using the `predict` method of the `tensorflow_client` that's passed to the handler's constructor: - -```python -def predict(model_input, model_name, model_version) -> dict: - """ - Run prediction. - - Args: - model_input: Input to the model. - model_name (optional): Name of the model to retrieve (when multiple models are deployed in an API). - When handler.models.paths is specified, model_name should be the name of one of the models listed in the API config. - When handler.models.dir is specified, model_name should be the name of a top-level directory in the models dir. - model_version (string, optional): Version of the model to retrieve. Can be omitted or set to "latest" to select the highest version. - - Returns: - dict: TensorFlow Serving response converted to a dictionary. - """ -``` - -## Specifying models - -Whenever a model path is specified in an API configuration file, it should be a path to an S3 prefix which contains your exported model. Directories may include a single model, or multiple folders each with a single model (note that a "single model" need not be a single file; there can be multiple files for a single model). When multiple folders are used, the folder names must be integer values, and will be interpreted as the model version. Model versions can be any integer, but are typically integer timestamps. It is always assumed that the highest version number is the latest version of your model. - -### API spec - -#### Single model - -The most common pattern is to serve a single model per API. The path to the model is specified in the `path` field in the `handler.models` configuration. For example: - -```yaml -# cortex.yaml - -- name: iris-classifier - kind: BatchAPI - handler: - # ... - type: tensorflow - models: - path: s3://my-bucket/models/text-generator/ -``` - -#### Multiple models - -It is possible to serve multiple models from a single API. The paths to the models are specified in the api configuration, either via the `models.paths` or `models.dir` field in the `handler` configuration. For example: - -```yaml -# cortex.yaml - -- name: iris-classifier - kind: BatchAPI - handler: - # ... - type: tensorflow - models: - paths: - - name: iris-classifier - path: s3://my-bucket/models/text-generator/ - # ... -``` - -or: - -```yaml -# cortex.yaml - -- name: iris-classifier - kind: BatchAPI - handler: - # ... - type: tensorflow - models: - dir: s3://my-bucket/models/ -``` - -When using the `models.paths` field, each path must be a valid model directory (see above for valid model directory structures). - -When using the `models.dir` field, the directory provided may contain multiple subdirectories, each of which is a valid model directory. For example: - -```text - s3://my-bucket/models/ - ├── text-generator - | └── * (model files) - └── sentiment-analyzer - ├── 24753823/ - | └── * (model files) - └── 26234288/ - └── * (model files) -``` - -In this case, there are two models in the directory, one of which is named "text-generator", and the other is named "sentiment-analyzer". - -### Structure - -#### On CPU/GPU - -The model path must be a SavedModel export: - -```text - s3://my-bucket/models/text-generator/ - ├── saved_model.pb - └── variables/ - ├── variables.index - ├── variables.data-00000-of-00003 - ├── variables.data-00001-of-00003 - └── variables.data-00002-of-... -``` - -or for a versioned model: - -```text - s3://my-bucket/models/text-generator/ - ├── 1523423423/ (version number, usually a timestamp) - | ├── saved_model.pb - | └── variables/ - | ├── variables.index - | ├── variables.data-00000-of-00003 - | ├── variables.data-00001-of-00003 - | └── variables.data-00002-of-... - └── 2434389194/ (version number, usually a timestamp) - ├── saved_model.pb - └── variables/ - ├── variables.index - ├── variables.data-00000-of-00003 - ├── variables.data-00001-of-00003 - └── variables.data-00002-of-... -``` - -#### On Inferentia - -When Inferentia models are used, the directory structure is slightly different: - -```text - s3://my-bucket/models/text-generator/ - └── saved_model.pb -``` - -or for a versioned model: - -```text - s3://my-bucket/models/text-generator/ - ├── 1523423423/ (version number, usually a timestamp) - | └── saved_model.pb - └── 2434389194/ (version number, usually a timestamp) - └── saved_model.pb -``` diff --git a/docs/workloads/dependencies/example.md b/docs/workloads/dependencies/example.md deleted file mode 100644 index 2fcc1b330f..0000000000 --- a/docs/workloads/dependencies/example.md +++ /dev/null @@ -1,61 +0,0 @@ -# Deploy a project - -You can deploy an API by providing a project directory. Cortex will save the project directory and make it available during API initialization. - -```bash -project/ - ├── model.py - ├── util.py - ├── handler.py - ├── requirements.txt - └── ... -``` - -You can define your Handler class in a separate python file and import code from your project. - -```python -# handler.py - -from model import MyModel - -class Handler: - def __init__(self, config): - model = MyModel() - - def handle_post(payload): - return model(payload) -``` - -## Deploy using the Python Client - -```python -import cortex - -api_spec = { - "name": "text-generator", - "kind": "RealtimeAPI", - "handler": { - "type": "python", - "path": "handler.py" - } -} - -cx = cortex.client("cortex") -cx.deploy(api_spec, project_dir=".") -``` - -## Deploy using the CLI - -```yaml -# api.yaml - -- name: text-generator - kind: RealtimeAPI - handler: - type: python - path: handler.py -``` - -```bash -cortex deploy api.yaml -``` diff --git a/docs/workloads/dependencies/images.md b/docs/workloads/dependencies/images.md deleted file mode 100644 index fc9792e857..0000000000 --- a/docs/workloads/dependencies/images.md +++ /dev/null @@ -1,106 +0,0 @@ -# Docker images - -Cortex includes a default set of Docker images with pre-installed Python and system packages but you can build custom images for use in your APIs. Common reasons to do this are to avoid installing dependencies during replica initialization, to have smaller images, and/or to mirror images to your ECR registry (for speed and reliability). - -## Create a Dockerfile - -```bash -mkdir my-api && cd my-api && touch Dockerfile -``` - -Cortex's base Docker images are listed below. Depending on the Cortex Handler and compute type specified in your API configuration, choose one of these images to use as the base for your Docker image: - - -* Python Handler (CPU): `quay.io/cortexlabs/python-handler-cpu:master` -* Python Handler (GPU): choose one of the following: - * `quay.io/cortexlabs/python-handler-gpu:master-cuda10.0-cudnn7` - * `quay.io/cortexlabs/python-handler-gpu:master-cuda10.1-cudnn7` - * `quay.io/cortexlabs/python-handler-gpu:master-cuda10.1-cudnn8` - * `quay.io/cortexlabs/python-handler-gpu:master-cuda10.2-cudnn7` - * `quay.io/cortexlabs/python-handler-gpu:master-cuda10.2-cudnn8` - * `quay.io/cortexlabs/python-handler-gpu:master-cuda11.0-cudnn8` - * `quay.io/cortexlabs/python-handler-gpu:master-cuda11.1-cudnn8` -* Python Handler (Inferentia): `quay.io/cortexlabs/python-handler-inf:master` -* TensorFlow Handler (CPU, GPU, Inferentia): `quay.io/cortexlabs/tensorflow-handler:master` - -The sample `Dockerfile` below inherits from Cortex's Python CPU serving image, and installs 3 packages. `tree` is a system package and `pandas` and `rdkit` are Python packages. - - -```dockerfile -# Dockerfile - -FROM quay.io/cortexlabs/python-handler-cpu:master - -RUN apt-get update \ - && apt-get install -y tree \ - && apt-get clean && rm -rf /var/lib/apt/lists/* - -RUN pip install --no-cache-dir pandas \ - && conda install -y conda-forge::rdkit \ - && conda clean -a -``` - -If you need to upgrade the Python Runtime version on your image, you can follow this procedure: - - - -```Dockerfile -# Dockerfile - -FROM quay.io/cortexlabs/python-handler-cpu:master - -# upgrade python runtime version -RUN conda update -n base -c defaults conda -RUN conda install -n env python=3.8.5 - -# re-install cortex core dependencies -RUN /usr/local/cortex/install-core-dependencies.sh - -# ... -``` - -## Build your image - -```bash -docker build . -t org/my-api:latest -``` - -## Push your image to a container registry - -You can push your built Docker image to a public registry of your choice (e.g. Docker Hub), or to a private registry on ECR or Docker Hub. - -For example, to use ECR, first create a repository to store your image: - -```bash -# We create a repository in ECR - -export AWS_REGION="***" -export REGISTRY_URL="***" # this will be in the format ".dkr.ecr..amazonaws.com" - -aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $REGISTRY_URL - -aws ecr create-repository --repository-name=org/my-api --region=$AWS_REGION -# take note of repository url -``` - -Build and tag your image, and push it to your ECR repository: - -```bash -docker build . -t org/my-api:latest -t :latest - -docker push :latest -``` - -## Configure your API - -```yaml -# cortex.yaml - -- name: my-api - ... - handler: - image: :latest - ... -``` - -Note: for TensorFlow Handlers, two containers run together to serve requests: one runs your Handler code (`quay.io/cortexlabs/tensorflow-handler`), and the other is TensorFlow serving to load the SavedModel (`quay.io/cortexlabs/tensorflow-serving-gpu` or `quay.io/cortexlabs/tensorflow-serving-cpu`). There's a second available field `tensorflow_serving_image` that can be used to override the TensorFlow Serving image. Both of the default serving images (`quay.io/cortexlabs/tensorflow-serving-gpu` and `quay.io/cortexlabs/tensorflow-serving-cpu`) are based on the official TensorFlow Serving image (`tensorflow/serving`). Unless a different version of TensorFlow Serving is required, the TensorFlow Serving image shouldn't have to be overridden, since it's only used to load the SavedModel and does not run your Handler code. diff --git a/docs/workloads/dependencies/python-packages.md b/docs/workloads/dependencies/python-packages.md deleted file mode 100644 index 7178b92527..0000000000 --- a/docs/workloads/dependencies/python-packages.md +++ /dev/null @@ -1,140 +0,0 @@ -# Python packages - -## PyPI packages - -You can install your required PyPI packages and import them in your Python files using pip. Cortex looks for -a `requirements.txt` file in the top level Cortex project directory (i.e. the directory which contains `cortex.yaml`): - -```text -./my-classifier/ -├── cortex.yaml -├── handler.py -├── ... -└── requirements.txt -``` - -If you want to use `conda` to install your python packages, see the [Conda section](#conda-packages) below. - -Note that some packages are pre-installed by default (see "pre-installed packages" for your handler type in the -Realtime API Handler documentation and Batch API Handler documentation). - -## Private PyPI packages - -To install packages from a private PyPI index, create a `pip.conf` inside the same directory as `requirements.txt`, and -add the following contents: - -```text -[global] -extra-index-url = https://:@.com/pip -``` - -In same directory, create a [`dependencies.sh` script](system-packages.md) and add the following: - -```bash -cp pip.conf /etc/pip.conf -``` - -You may now add packages to `requirements.txt` which are found in the private index. - -## GitHub packages - -You can also install public/private packages from git registries (such as GitHub) by adding them to `requirements.txt`. -Here's an example for GitHub: - -```text -# requirements.txt - -# public access -git+https://github.com//.git@#egg= - -# private access -git+https://@github.com//.git@#egg= -``` - -On GitHub, you can generate a personal access token by -following [these steps](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line) -. - -## Installing with Setup - -Python packages can also be installed by providing a `setup.py` that describes your project's modules. Here's an example -directory structure: - -```text -./my-classifier/ -├── cortex.yaml -├── handler.py -├── ... -├── mypkg -│ └── __init__.py -├── requirements.txt -└── setup.py -``` - -In this case, `requirements.txt` will have this form: - -```text -# requirements.txt - -. -``` - -## Conda packages - -Cortex supports installing Conda packages. We recommend only using Conda when your required packages are not available -in PyPI. Cortex looks for a `conda-packages.txt` file in the top level Cortex project directory (i.e. the directory -which contains `cortex.yaml`): - -```text -./my-classifier/ -├── cortex.yaml -├── handler.py -├── ... -└── conda-packages.txt -``` - -The `conda-packages.txt` file follows the format of `conda list --export`. Each line of `conda-packages.txt` should -follow this pattern: `[channel::]package[=version[=buildid]]`. - -Here's an example of `conda-packages.txt`: - -```text -conda-forge::rdkit -conda-forge::pygpu -``` - -In situations where both `requirements.txt` and `conda-packages.txt` are provided, Cortex installs Conda packages -in `conda-packages.txt` followed by PyPI packages in `requirements.txt`. Conda and Pip package managers install packages -and dependencies independently. You may run into situations where Conda and pip package managers install different -versions of the same package because they install and resolve dependencies independently from one another. To resolve -package version conflicts, it may be in your best interest to specify their exact versions in `conda-packages.txt`. - -The current version of Python is `3.6.9`. Updating Python to a different version is possible with Conda, but there are -no guarantees that Cortex's web server will continue functioning correctly. If there's a change in Python's version, the -necessary core packages for the web server will be reinstalled. If you are using a custom base image, any other Python -packages that are built in to the image won't be accessible at runtime. - -Check the [best practices](https://www.anaconda.com/using-pip-in-a-conda-environment/) on using `pip` inside `conda`. - -## Customizing Dependency Paths - -Cortex allows you to specify different dependency paths other than the default ones. This can be useful when deploying -different versions of the same API (e.g. CPU vs GPU dependencies). - -To customize the path for your dependencies, you can specify `handler.dependencies` in your API's configuration file. You can set -one or more fields to specify the path for each dependency type. Each path should be a relative path with respect to the current file. - -For example: - -```yaml -# cortex.yaml - -- name: my-classifier - kind: RealtimeAPI - handler: - (...) - dependencies: - pip: requirement-gpu.txt - conda: conda-packages-gpu.txt - shell: dependencies-gpu.sh -``` diff --git a/docs/workloads/dependencies/system-packages.md b/docs/workloads/dependencies/system-packages.md deleted file mode 100644 index 4f2b60c76b..0000000000 --- a/docs/workloads/dependencies/system-packages.md +++ /dev/null @@ -1,62 +0,0 @@ -# System packages - -Cortex looks for a file named `dependencies.sh` in the top level Cortex project directory (i.e. the directory which contains `cortex.yaml`). For example: - -```text -./my-classifier/ -├── cortex.yaml -├── handler.py -├── ... -└── dependencies.sh -``` - -`dependencies.sh` is executed with `bash` shell during the initialization of each replica (before installing Python packages in `requirements.txt` or `conda-packages.txt`). Typical use cases include installing required system packages to be used in your Handler, building Python packages from source, etc. If initialization time is a concern, see [Docker images](images.md) for how to build and use custom Docker images. - -Here is an example `dependencies.sh`, which installs the `tree` utility: - -```bash -apt-get update && apt-get install -y tree -``` - -The `tree` utility can now be called inside your `handler.py`: - -```python -# handler.py - -import subprocess - -class Handler: - def __init__(self, config): - subprocess.run(["tree"]) - ... -``` - -If you need to upgrade the Python Runtime version on your image, you can do so in your `dependencies.sh` file: - -```bash -# upgrade python runtime version -conda update -n base -c defaults conda -conda install -n env python=3.8.5 - -# re-install cortex core dependencies -/usr/local/cortex/install-core-dependencies.sh -``` - -## Customizing Dependency Paths - -Cortex allows you to specify a path for this script other than `dependencies.sh`. This can be useful when deploying -different versions of the same API (e.g. CPU vs GPU dependencies). The path should be a relative path with respect -to the API configuration file, and is specified via `handler.dependencies.shell`. - -For example: - -```yaml -# cortex.yaml - -- name: my-classifier - kind: RealtimeAPI - handler: - (...) - dependencies: - shell: dependencies-gpu.sh -``` diff --git a/docs/workloads/realtime/autoscaling.md b/docs/workloads/realtime/autoscaling.md index ce46c218a4..c3a7cfbaca 100644 --- a/docs/workloads/realtime/autoscaling.md +++ b/docs/workloads/realtime/autoscaling.md @@ -1,34 +1,42 @@ # Autoscaling -Cortex autoscales your web services on a per-API basis based on your configuration. +Cortex autoscales each API independently based on its configuration. ## Autoscaling replicas -**`min_replicas`**: The lower bound on how many replicas can be running for an API. +### Relevant pod configuration -
+In addition to the autoscaling configuration options (described below), there are two fields in the pod configuration which are relevant to replica autoscaling: -**`max_replicas`**: The upper bound on how many replicas can be running for an API. +**`max_concurrency`** (default: 1): The maximum number of requests that will be concurrently sent into the container by Cortex. If your web server is designed to handle multiple concurrent requests, increasing `max_concurrency` will increase the throughput of a replica (and result in fewer total replicas for a given load).
-**`target_replica_concurrency`** (default: `processes_per_replica` * `threads_per_process`): This is the desired number of in-flight requests per replica, and is the metric which the autoscaler uses to make scaling decisions. +**`max_queue_length`** (default: 100): The maximum number of requests which will be queued by the replica (beyond `max_concurrency`) before requests are rejected with HTTP error code 503. For long-running APIs, decreasing `max_replica_concurrency` and configuring the client to retry when it receives 503 responses will improve queue fairness accross replicas by preventing requests from sitting in long queues. -Replica concurrency is simply how many requests have been sent to a replica and have not yet been responded to (also referred to as in-flight requests). Therefore, it includes requests which are currently being processed and requests which are waiting in the replica's queue. +
-The autoscaler uses this formula to determine the number of desired replicas: +### Autoscaling configuration + +**`min_replicas`**: The lower bound on how many replicas can be running for an API. -`desired replicas = sum(in-flight requests accross all replicas) / target_replica_concurrency` +
-For example, setting `target_replica_concurrency` to `processes_per_replica` * `threads_per_process` (the default) causes the cluster to adjust the number of replicas so that on average, requests are immediately processed without waiting in a queue, and processes/threads are never idle. +**`max_replicas`**: The upper bound on how many replicas can be running for an API.
-**`max_replica_concurrency`** (default: 1024): This is the maximum number of in-flight requests per replica before requests are rejected with HTTP error code 503. `max_replica_concurrency` includes requests that are currently being processed as well as requests that are waiting in the replica's queue (a replica can actively process `processes_per_replica` * `threads_per_process` requests concurrently, and will hold any additional requests in a local queue). Decreasing `max_replica_concurrency` and configuring the client to retry when it receives 503 responses will improve queue fairness accross replicas by preventing requests from sitting in long queues. +**`target_in_flight`** (default: `max_concurrency` in the pod configuration): This is the desired number of in-flight requests per replica, and is the metric which the autoscaler uses to make scaling decisions. The number of in-flight requests is simply how many requests have been sent to a replica and have not yet been responded to. Therefore, this number includes requests which are actively being processed as well as requests which are waiting in the replica's queue. + +The autoscaler uses this formula to determine the number of desired replicas: + +`desired replicas = sum(in-flight requests accross all replicas) / target_in_flight` + +For example, setting `target_in_flight` to `max_concurrency` (the default) causes the cluster to adjust the number of replicas so that on average, requests are immediately processed without waiting in a queue.
-**`window`** (default: 60s): The time over which to average the API wide in-flight requests (which is the sum of in-flight requests in each replica). The longer the window, the slower the autoscaler will react to changes in API wide in-flight requests, since it is averaged over the `window`. API wide in-flight requests is calculated every 10 seconds, so `window` must be a multiple of 10 seconds. +**`window`** (default: 60s): The time over which to average the API's in-flight requests (which is the sum of in-flight requests in each replica). The longer the window, the slower the autoscaler will react to changes in in-flight requests, since it is averaged over the `window`. An API's in-flight requests is calculated every 10 seconds, so `window` must be a multiple of 10 seconds.
@@ -62,14 +70,12 @@ Cortex spins up and down instances based on the aggregate resource requests of a ## Overprovisioning -The default value for `target_replica_concurrency` is `processes_per_replica` * `threads_per_process`, which behaves well in many situations (see above for an explanation of how `target_replica_concurrency` affects autoscaling). However, if your application is sensitive to spikes in traffic or if creating new replicas takes too long (see below), you may find it helpful to maintain extra capacity to handle the increased traffic while new replicas are being created. This can be accomplished by setting `target_replica_concurrency` to a lower value relative to the expected replica's concurrency. The smaller `target_replica_concurrency` is, the more unused capacity your API will have, and the more room it will have to handle sudden increased load. The increased request rate will still trigger the autoscaler, and your API will stabilize again (maintaining the overprovisioned capacity). +The default value for `target_in_flight` is `max_concurrency`, which behaves well in many situations (see above for an explanation of how `target_in_flight` affects autoscaling). However, if your application is sensitive to spikes in traffic or if creating new replicas takes too long (see below), you may find it helpful to maintain extra capacity to handle the increased traffic while new replicas are being created. This can be accomplished by setting `target_in_flight` to a lower value relative to the expected replica's concurrency. The smaller `target_in_flight` is, the more unused capacity your API will have, and the more room it will have to handle sudden increased load. The increased request rate will still trigger the autoscaler, and your API will stabilize again (maintaining the overprovisioned capacity). -For example, if you've determined that each replica in your API can handle 2 requests, you would set `target_replica_concurrency` to 2. In a scenario where your API is receiving 8 concurrent requests on average, the autoscaler would maintain 4 live replicas (8/2 = 4). If you wanted to overprovision by 25%, you can set `target_replica_concurrency` to 1.6 causing the autoscaler maintain 5 live replicas (8/1.6 = 5). +For example, if you've determined that each replica in your API can handle 2 concurrent requests, you would typically set `target_in_flight` to 2. In a scenario where your API is receiving 8 concurrent requests on average, the autoscaler would maintain 4 live replicas (8/2 = 4). If you wanted to overprovision by 25%, you could set `target_in_flight` to 1.6, causing the autoscaler maintain 5 live replicas (8/1.6 = 5). ## Autoscaling responsiveness -Assuming that `window` and `upscale_stabilization_period` are set to their default values (1 minute), it could take up to 2 minutes of increased traffic before an extra replica is requested. As soon as the additional replica is requested, the replica request will be visible in the output of `cortex get`, but the replica won't yet be running. If an extra instance is required to schedule the newly requested replica, it could take a few minutes for AWS to provision the instance (depending on the instance type), plus a few minutes for the newly provisioned instance to download your api image and for the api to initialize (via its `__init__()` method). +Assuming that `window` and `upscale_stabilization_period` are set to their default values (1 minute), it could take up to 2 minutes of increased traffic before an extra replica is requested. As soon as the additional replica is requested, the replica request will be visible in the output of `cortex get`, but the replica won't yet be running. If an extra instance is required to schedule the newly requested replica, it could take a few minutes for AWS to provision the instance (depending on the instance type), plus a few minutes for the newly provisioned instance to download your api image and for the api to initialize. Keep these delays in mind when considering overprovisioning (see above) and when determining appropriate values for `window` and `upscale_stabilization_period`. If you want the autoscaler to react as quickly as possible, set `upscale_stabilization_period` and `window` to their minimum values (0s and 10s respectively). - -If it takes a long time to initialize your API replica (i.e. install dependencies and run your handler's `__init__()` function), consider building your own API image to use instead of the default image. With this approach, you can pre-download/build/install any custom dependencies and bake them into the image. diff --git a/docs/workloads/realtime/configuration.md b/docs/workloads/realtime/configuration.md index 81f8e2a63f..9d2bf85bf6 100644 --- a/docs/workloads/realtime/configuration.md +++ b/docs/workloads/realtime/configuration.md @@ -1,128 +1,66 @@ # Configuration ```yaml -- name: - kind: RealtimeAPI - handler: # detailed configuration below - compute: # detailed configuration below - autoscaling: # detailed configuration below - update_strategy: # detailed configuration below - networking: # detailed configuration below -``` - -## Handler - -### Python Handler - - -```yaml -handler: - type: python - path: # path to a python file with a Handler class definition, relative to the Cortex root (required) - protobuf_path: # path to a protobuf file (required if using gRPC) - dependencies: # (optional) - pip: # relative path to requirements.txt (default: requirements.txt) - conda: # relative path to conda-packages.txt (default: conda-packages.txt) - shell: # relative path to a shell script for system package installation (default: dependencies.sh) - multi_model_reloading: # use this to serve one or more models with live reloading (optional) - path: # S3 path to an exported model directory (e.g. s3://my-bucket/exported_model/) (either this, 'dir', or 'paths' must be provided if 'multi_model_reloading' is specified) - paths: # list of S3 paths to exported model directories (either this, 'dir', or 'path' must be provided if 'multi_model_reloading' is specified) - - name: # unique name for the model (e.g. text-generator) (required) - path: # S3 path to an exported model directory (e.g. s3://my-bucket/exported_model/) (required) - ... - dir: # S3 path to a directory containing multiple models (e.g. s3://my-bucket/models/) (either this, 'path', or 'paths' must be provided if 'multi_model_reloading' is specified) - cache_size: # the number models to keep in memory (optional; all models are kept in memory by default) - disk_cache_size: # the number of models to keep on disk (optional; all models are kept on disk by default) - server_side_batching: # (optional) - max_batch_size: # the maximum number of requests to aggregate before running inference - batch_interval: # the maximum amount of time to spend waiting for additional requests before running inference on the batch of requests - processes_per_replica: # the number of parallel serving processes to run on each replica (default: 1) - threads_per_process: # the number of threads per process (default: 1) - config: # arbitrary dictionary passed to the constructor of the Handler class (optional) - python_path: # path to the root of your Python folder that will be appended to PYTHONPATH (default: folder containing cortex.yaml) - image: # docker image to use for the Handler class (default: quay.io/cortexlabs/python-handler-cpu:master, quay.io/cortexlabs/python-handler-gpu:master-cuda10.2-cudnn8, or quay.io/cortexlabs/python-handler-inf:master based on compute) - env: # dictionary of environment variables - log_level: # log level that can be "debug", "info", "warning" or "error" (default: "info") - shm_size: # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null) -``` - -### TensorFlow Handler - - -```yaml -handler: - type: tensorflow - path: # path to a python file with a Handler class definition, relative to the Cortex root (required) - protobuf_path: # path to a protobuf file (required if using gRPC) - dependencies: # (optional) - pip: # relative path to requirements.txt (default: requirements.txt) - conda: # relative path to conda-packages.txt (default: conda-packages.txt) - shell: # relative path to a shell script for system package installation (default: dependencies.sh) - models: # (required) - path: # S3 path to an exported SavedModel directory (e.g. s3://my-bucket/exported_model/) (either this, 'dir', or 'paths' must be provided) - paths: # list of S3 paths to exported SavedModel directories (either this, 'dir', or 'path' must be provided) - - name: # unique name for the model (e.g. text-generator) (required) - path: # S3 path to an exported SavedModel directory (e.g. s3://my-bucket/exported_model/) (required) - signature_key: # name of the signature def to use for prediction (required if your model has more than one signature def) - ... - dir: # S3 path to a directory containing multiple SavedModel directories (e.g. s3://my-bucket/models/) (either this, 'path', or 'paths' must be provided) - signature_key: # name of the signature def to use for prediction (required if your model has more than one signature def) - cache_size: # the number models to keep in memory (optional; all models are kept in memory by default) - disk_cache_size: # the number of models to keep on disk (optional; all models are kept on disk by default) - server_side_batching: # (optional) - max_batch_size: # the maximum number of requests to aggregate before running inference - batch_interval: # the maximum amount of time to spend waiting for additional requests before running inference on the batch of requests - processes_per_replica: # the number of parallel serving processes to run on each replica (default: 1) - threads_per_process: # the number of threads per process (default: 1) - config: # arbitrary dictionary passed to the constructor of the Handler class (optional) - python_path: # path to the root of your Python folder that will be appended to PYTHONPATH (default: folder containing cortex.yaml) - image: # docker image to use for the handler (default: quay.io/cortexlabs/tensorflow-handler:master) - tensorflow_serving_image: # docker image to use for the TensorFlow Serving container (default: quay.io/cortexlabs/tensorflow-serving-cpu:master, quay.io/cortexlabs/tensorflow-serving-gpu:master, or quay.io/cortexlabs/tensorflow-serving-inf:master based on compute) - env: # dictionary of environment variables - log_level: # log level that can be "debug", "info", "warning" or "error" (default: "info") - shm_size: # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null) -``` - -## Compute - -```yaml -compute: - cpu: # CPU request per replica. One unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the "m" suffix (default: 200m) - gpu: # GPU request per replica. One unit of GPU corresponds to one virtual GPU (default: 0) - inf: # Inferentia request per replica. One unit of Inf corresponds to one virtual Inferentia chip (default: 0) - mem: # memory request per replica. One unit of memory is one byte and can be expressed as an integer or by using one of these suffixes: K, M, G, T (or their power-of two counterparts: Ki, Mi, Gi, Ti) (default: Null) - node_groups: # to select specific node groups (optional) -``` - -## Autoscaling - -```yaml -autoscaling: - min_replicas: # minimum number of replicas (default: 1) - max_replicas: # maximum number of replicas (default: 100) - init_replicas: # initial number of replicas (default: ) - max_replica_concurrency: # the maximum number of in-flight requests per replica before requests are rejected with error code 503 (default: 1024) - target_replica_concurrency: # the desired number of in-flight requests per replica, which the autoscaler tries to maintain (default: processes_per_replica * threads_per_process) - window: # the time over which to average the API's concurrency (default: 60s) - downscale_stabilization_period: # the API will not scale below the highest recommendation made during this period (default: 5m) - upscale_stabilization_period: # the API will not scale above the lowest recommendation made during this period (default: 1m) - max_downscale_factor: # the maximum factor by which to scale down the API on a single scaling event (default: 0.75) - max_upscale_factor: # the maximum factor by which to scale up the API on a single scaling event (default: 1.5) - downscale_tolerance: # any recommendation falling within this factor below the current number of replicas will not trigger a scale down event (default: 0.05) - upscale_tolerance: # any recommendation falling within this factor above the current number of replicas will not trigger a scale up event (default: 0.05) -``` - -## Update strategy - -```yaml -update_strategy: - max_surge: # maximum number of replicas that can be scheduled above the desired number of replicas during an update; can be an absolute number, e.g. 5, or a percentage of desired replicas, e.g. 10% (default: 25%) (set to 0 to disable rolling updates) - max_unavailable: # maximum number of replicas that can be unavailable during an update; can be an absolute number, e.g. 5, or a percentage of desired replicas, e.g. 10% (default: 25%) -``` - -## Networking - -```yaml - networking: - endpoint: # the endpoint for the API (default: ) +- name: # name of the API (required) + kind: RealtimeAPI # must be "RealtimeAPI" for realtime APIs (required) + pod: # pod configuration (required) + port: # port to which requests will be sent (default: 8080; exported as $CORTEX_PORT) + max_concurrency: # maximum number of requests that will be concurrently sent into the container (default: 1) + max_queue_length: # maximum number of requests per replica which will be queued (beyond max_concurrency) before requests are rejected with error code 503 (default: 100) + containers: # configurations for the containers to run (at least one constainer must be provided) + - name: # name of the container (required) + image: # docker image to use for the container (required) + command: # entrypoint (default: the docker image's ENTRYPOINT) + args: # arguments to the entrypoint (default: the docker image's CMD) + env: # dictionary of environment variables to set in the container (optional) + compute: # compute resource requests (default: see below) + cpu: # CPU request for the container; one unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the "m" suffix (default: 200m) + gpu: # GPU request for the container; one unit of GPU corresponds to one virtual GPU (default: 0) + inf: # Inferentia request for the container; one unit of inf corresponds to one virtual Inferentia chip (default: 0) + mem: # memory request for the container; one unit of memory is one byte and can be expressed as an integer or by using one of these suffixes: K, M, G, T (or their power-of two counterparts: Ki, Mi, Gi, Ti) (default: Null) + shm: # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null) + readiness_probe: # periodic probe of container readiness; traffic will not be sent into the pod unless all containers' readiness probes are succeeding (optional) + http_get: # specifies an http endpoint which must respond with status code 200 (only one of http_get, tcp_socket, and exec may be specified) + port: # the port to access on the container (required) + path: # the path to access on the HTTP server (default: /) + tcp_socket: # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified) + port: # the port to access on the container (required) + exec: # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified) + command: [] # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) + initial_delay_seconds: # number of seconds after the container has started before the probe is initiated (default: 0) + timeout_seconds: # number of seconds until the probe times out (default: 1) + period_seconds: # how often (in seconds) to perform the probe (default: 10) + success_threshold: # minimum consecutive successes for the probe to be considered successful after having failed (default: 1) + failure_threshold: # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3) + liveness_probe: # periodic probe of container liveness; container will be restarted if the probe fails (optional) + http_get: # specifies an http endpoint which must respond with status code 200 (only one of http_get, tcp_socket, and exec may be specified) + port: # the port to access on the container (required) + path: # the path to access on the HTTP server (default: /) + tcp_socket: # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified) + port: # the port to access on the container (required) + exec: # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified) + command: [] # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) + initial_delay_seconds: # number of seconds after the container has started before the probe is initiated (default: 0) + timeout_seconds: # number of seconds until the probe times out (default: 1) + period_seconds: # how often (in seconds) to perform the probe (default: 10) + success_threshold: # minimum consecutive successes for the probe to be considered successful after having failed (default: 1) + failure_threshold: # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3) + autoscaling: # autoscaling configuration (default: see below) + min_replicas: # minimum number of replicas (default: 1) + max_replicas: # maximum number of replicas (default: 100) + init_replicas: # initial number of replicas (default: ) + target_in_flight: # desired number of in-flight requests per replica (including requests actively being processed as well as queued), which the autoscaler tries to maintain (default: ) + window: # duration over which to average the API's in-flight requests per replica (default: 60s) + downscale_stabilization_period: # the API will not scale below the highest recommendation made during this period (default: 5m) + upscale_stabilization_period: # the API will not scale above the lowest recommendation made during this period (default: 1m) + max_downscale_factor: # maximum factor by which to scale down the API on a single scaling event (default: 0.75) + max_upscale_factor: # maximum factor by which to scale up the API on a single scaling event (default: 1.5) + downscale_tolerance: # any recommendation falling within this factor below the current number of replicas will not trigger a scale down event (default: 0.05) + upscale_tolerance: # any recommendation falling within this factor above the current number of replicas will not trigger a scale up event (default: 0.05) + node_groups: # a list of node groups on which this API can run (default: all node groups are eligible) + update_strategy: # deployment strategy to use when replacing existing replicas with new ones (default: see below) + max_surge: # maximum number of replicas that can be scheduled above the desired number of replicas during an update; can be an absolute number, e.g. 5, or a percentage of desired replicas, e.g. 10% (default: 25%) (set to 0 to disable rolling updates) + max_unavailable: # maximum number of replicas that can be unavailable during an update; can be an absolute number, e.g. 5, or a percentage of desired replicas, e.g. 10% (default: 25%) + networking: # networking configuration (default: see below) + endpoint: # endpoint for the API (default: ) ``` diff --git a/docs/workloads/realtime/handler.md b/docs/workloads/realtime/handler.md deleted file mode 100644 index cfe3e4edfa..0000000000 --- a/docs/workloads/realtime/handler.md +++ /dev/null @@ -1,563 +0,0 @@ -# Handler implementation - -Realtime APIs respond to requests in real-time and autoscale based on in-flight request volumes. They can be used for realtime inference or data processing workloads. - -If you plan on deploying ML models and run realtime inferences, check out the [Models](models.md) page. Cortex provides out-of-the-box support for a variety of frameworks such as: PyTorch, ONNX, scikit-learn, XGBoost, TensorFlow, etc. - -The response type of the handler can vary depending on your requirements, see [HTTP API responses](#http-responses) and [gRPC API responses](#grpc-responses) below. - -## Project files - -Cortex makes all files in the project directory (i.e. the directory which contains `cortex.yaml`) available for use in your Handler class implementation. Python bytecode files (`*.pyc`, `*.pyo`, `*.pyd`), files or folders that start with `.`, and the api configuration file (e.g. `cortex.yaml`) are excluded. - -The following files can also be added at the root of the project's directory: - -* `.cortexignore` file, which follows the same syntax and behavior as a [.gitignore file](https://git-scm.com/docs/gitignore). This may be necessary if you are reaching the size limit for your project directory (32mb). -* `.env` file, which exports environment variables that can be used in the handler. Each line of this file must follow the `VARIABLE=value` format. - -For example, if your directory looks like this: - -```text -./my-classifier/ -├── cortex.yaml -├── values.json -├── handler.py -├── ... -└── requirements.txt -``` - -You can access `values.json` in your handler class like this: - -```python -# handler.py - -import json - -class Handler: - def __init__(self, config): - with open('values.json', 'r') as values_file: - values = json.load(values_file) - self.values = values -``` - -## HTTP - -### Handler - -```python -# initialization code and variables can be declared here in global scope - -class Handler: - def __init__(self, config): - """(Required) Called once before the API becomes available. Performs - setup such as downloading/initializing the model or downloading a - vocabulary. - - Args: - config (required): Dictionary passed from API configuration (if - specified). This may contain information on where to download - the model and/or metadata. - """ - pass - - def handle_(self, payload, query_params, headers): - """(Required) Called once per request. Preprocesses the request payload - (if necessary), runs workload, and postprocesses the workload output - (if necessary). - - Args: - payload (optional): The request payload (see below for the possible - payload types). - query_params (optional): A dictionary of the query parameters used - in the request. - headers (optional): A dictionary of the headers sent in the request. - - Returns: - Result or a batch of results. - """ - pass -``` - -Your `Handler` class can implement methods for each of the following HTTP methods: POST, GET, PUT, PATCH, DELETE. Therefore, the respective methods in the `Handler` definition can be `handle_post`, `handle_get`, `handle_put`, `handle_patch`, and `handle_delete`. - -For proper separation of concerns, it is recommended to use the constructor's `config` parameter for information such as from where to download the model and initialization files, or any configurable model parameters. You define `config` in your API configuration, and it is passed through to your Handler's constructor. - -Your API can accept requests with different types of payloads such as `JSON`-parseable, `bytes` or `starlette.datastructures.FormData` data. See [HTTP API requests](#http-requests) to learn about how headers can be used to change the type of `payload` that is passed into your handler method. - -Your handler method can return different types of objects such as `JSON`-parseable, `string`, and `bytes` objects. See [HTTP API responses](#http-responses) to learn about how to configure your handler method to respond with different response codes and content-types. - -### Callbacks - -A callback is a function that starts running in the background after the results have been sent back to the client. They are meant to be short-lived. - -Each handler method of your class can implement callbacks. To do this, when returning the result(s) from your handler method, also make sure to return a 2-element tuple in which the first element are your results that you want to return and the second element is a callable object that takes no arguments. - -You can implement a callback like in the following example: - -```python -def handle_post(self, payload): - def _callback(): - print("message that gets printed after the response is sent back to the user") - return "results", _callback -``` - -### HTTP requests - -The type of the `payload` parameter in `handle_(self, payload)` can vary based on the content type of the request. The `payload` parameter is parsed according to the `Content-Type` header in the request. Here are the parsing rules (see below for examples): - -1. For `Content-Type: application/json`, `payload` will be the parsed JSON body. -1. For `Content-Type: multipart/form-data` / `Content-Type: application/x-www-form-urlencoded`, `payload` will be `starlette.datastructures.FormData` (key-value pairs where the values are strings for text data, or `starlette.datastructures.UploadFile` for file uploads; see [Starlette's documentation](https://www.starlette.io/requests/#request-files)). -1. For `Content-Type: text/plain`, `payload` will be a string. `utf-8` encoding is assumed, unless specified otherwise (e.g. via `Content-Type: text/plain; charset=us-ascii`) -1. For all other `Content-Type` values, `payload` will be the raw `bytes` of the request body. - -Here are some examples: - -#### JSON data - -##### Making the request - -```bash -curl http://***.amazonaws.com/my-api \ - -X POST -H "Content-Type: application/json" \ - -d '{"key": "value"}' -``` - -##### Reading the payload - -When sending a JSON payload, the `payload` parameter will be a Python object: - -```python -class Handler: - def __init__(self, config): - pass - - def handle_post(self, payload): - print(payload["key"]) # prints "value" -``` - -#### Binary data - -##### Making the request - -```bash -curl http://***.amazonaws.com/my-api \ - -X POST -H "Content-Type: application/octet-stream" \ - --data-binary @object.pkl -``` - -##### Reading the payload - -Since the `Content-Type: application/octet-stream` header is used, the `payload` parameter will be a `bytes` object: - -```python -import pickle - -class Handler: - def __init__(self, config): - pass - - def handle_post(self, payload): - obj = pickle.loads(payload) - print(obj["key"]) # prints "value" -``` - -Here's an example if the binary data is an image: - -```python -from PIL import Image -import io - -class Handler: - def __init__(self, config): - pass - - def handle_post(self, payload, headers): - img = Image.open(io.BytesIO(payload)) # read the payload bytes as an image - print(img.size) -``` - -#### Form data (files) - -##### Making the request - -```bash -curl http://***.amazonaws.com/my-api \ - -X POST \ - -F "text=@text.txt" \ - -F "object=@object.pkl" \ - -F "image=@image.png" -``` - -##### Reading the payload - -When sending files via form data, the `payload` parameter will be `starlette.datastructures.FormData` (key-value pairs where the values are `starlette.datastructures.UploadFile`, see [Starlette's documentation](https://www.starlette.io/requests/#request-files)). Either `Content-Type: multipart/form-data` or `Content-Type: application/x-www-form-urlencoded` can be used (typically `Content-Type: multipart/form-data` is used for files, and is the default in the examples above). - -```python -from PIL import Image -import pickle - -class Handler: - def __init__(self, config): - pass - - def handle_post(self, payload): - text = payload["text"].file.read() - print(text.decode("utf-8")) # prints the contents of text.txt - - obj = pickle.load(payload["object"].file) - print(obj["key"]) # prints "value" assuming `object.pkl` is a pickled dictionary {"key": "value"} - - img = Image.open(payload["image"].file) - print(img.size) # prints the dimensions of image.png -``` - -#### Form data (text) - -##### Making the request - -```bash -curl http://***.amazonaws.com/my-api \ - -X POST \ - -d "key=value" -``` - -##### Reading the payload - -When sending text via form data, the `payload` parameter will be `starlette.datastructures.FormData` (key-value pairs where the values are strings, see [Starlette's documentation](https://www.starlette.io/requests/#request-files)). Either `Content-Type: multipart/form-data` or `Content-Type: application/x-www-form-urlencoded` can be used (typically `Content-Type: application/x-www-form-urlencoded` is used for text, and is the default in the examples above). - -```python -class Handler: - def __init__(self, config): - pass - - def handle_post(self, payload): - print(payload["key"]) # will print "value" -``` - -#### Text data - -##### Making the request - -```bash -curl http://***.amazonaws.com/my-api \ - -X POST -H "Content-Type: text/plain" \ - -d "hello world" -``` - -##### Reading the payload - -Since the `Content-Type: text/plain` header is used, the `payload` parameter will be a `string` object: - -```python -class Handler: - def __init__(self, config): - pass - - def handle_post(self, payload): - print(payload) # prints "hello world" -``` - -### HTTP responses - -The response of your `handle_()` method may be: - -1. A JSON-serializable object (*lists*, *dictionaries*, *numbers*, etc.) - -1. A `string` object (e.g. `"class 1"`) - -1. A `bytes` object (e.g. `bytes(4)` or `pickle.dumps(obj)`) - -1. An instance of [starlette.responses.Response](https://www.starlette.io/responses/#response) - -## gRPC - -To serve your API using the gRPC protocol, make sure the `handler.protobuf_path` field in your API configuration is pointing to a protobuf file. When the API gets deployed, Cortex will compile the protobuf file for its use when serving the API. - -### Python Handler - -#### Interface - -```python -# initialization code and variables can be declared here in global scope - -class Handler: - def __init__(self, config, module_proto_pb2): - """(Required) Called once before the API becomes available. Performs - setup such as downloading/initializing the model or downloading a - vocabulary. - - Args: - config (required): Dictionary passed from API configuration (if - specified). This may contain information on where to download - the model and/or metadata. - module_proto_pb2 (required): Loaded Python module containing the - class definitions of the messages defined in the protobuf - file (`handler.protobuf_path`). - """ - self.module_proto_pb2 = module_proto_pb2 - - def (self, payload, context): - """(Required) Called once per request. Preprocesses the request payload - (if necessary), runs workload, and postprocesses the workload output - (if necessary). - - Args: - payload (optional): The request payload (see below for the possible - payload types). - context (optional): gRPC context. - - Returns: - Result (when streaming is not used). - - Yield: - Result (when streaming is used). - """ - pass -``` - -Your `Handler` class must implement the RPC methods found in the protobuf. Your protobuf must have a single service defined, which can have any name. If your service has 2 RPC methods called `Info` and `Predict` methods, then your `Handler` class must also implement these methods like in the above `Handler` template. - -For proper separation of concerns, it is recommended to use the constructor's `config` parameter for information such as from where to download the model and initialization files, or any configurable model parameters. You define `config` in your API configuration, and it is passed through to your Handler class' constructor. - -Your API can only accept the type that has been specified in the protobuf definition of your service's method. See [gRPC API requests](#grpc-requests) for how to construct gRPC requests. - -Your handler method(s) can only return the type that has been specified in the protobuf definition of your service's method(s). See [gRPC API responses](#grpc-responses) for how to handle gRPC responses. - -### gRPC requests - -Assuming the following service: - -```protobuf -# handler.proto - -syntax = "proto3"; -package sample_service; - -service Handler { - rpc Predict (Sample) returns (Response); -} - -message Sample { - string a = 1; -} - -message Response { - string b = 1; -} -``` - -The handler implementation will also have a corresponding `Predict` method defined that represents the RPC method in the above protobuf service. The name(s) of the RPC method(s) is not enforced by Cortex. - -The type of the `payload` parameter passed into `Predict(self, payload)` will match that of the `Sample` message defined in the `handler.protobuf_path` file. For this example, we'll assume that the above protobuf file was specified for the API. - -#### Simple request - -The service method must look like this: - -```protobuf -... -rpc Predict (Sample) returns (Response); -... -``` - -##### Making the request - -```python -import grpc, handler_pb2, handler_pb2_grpc - -stub = handler_pb2_grpc.HandlerStub(grpc.insecure_channel("***.amazonaws.com:80")) -stub.Predict(handler_pb2.Sample(a="text")) -``` - -##### Reading the payload - -In the `Predict` method, you'll read the value like this: - -```python -... -def Predict(self, payload): - print(payload.a) -... -``` - -#### Streaming request - -The service method must look like this: - -```protobuf -... -rpc Predict (stream Sample) returns (Response); -... -``` - -##### Making the request - -```python -import grpc, handler_pb2, handler_pb2_grpc - -def generate_iterator(sample_list): - for sample in sample_list: - yield sample - -stub = handler_pb2_grpc.HandlerStub(grpc.insecure_channel("***.amazonaws.com:80")) -stub.Predict(handler_pb2.Sample(generate_iterator(["a", "b", "c", "d"]))) -``` - -##### Reading the payload - -In the `Predict` method, you'll read the streamed values like this: - -```python -... -def Predict(self, payload): - for item in payload: - print(item.a) -... -``` - -### gRPC responses - -Assuming the following service: - -```protobuf -# handler.proto - -syntax = "proto3"; -package sample_service; - -service Handler { - rpc Predict (Sample) returns (Response); -} - -message Sample { - string a = 1; -} - -message Response { - string b = 1; -} -``` - -The handler implementation will also have a corresponding `Predict` method defined that represents the RPC method in the above protobuf service. The name(s) of the RPC method(s) is not enforced by Cortex. - -The type of the value that you return in your `Predict()` method must match the `Response` message defined in the `handler.protobuf_path` file. For this example, we'll assume that the above protobuf file was specified for the API. - -#### Simple response - -The service method must look like this: - -```protobuf -... -rpc Predict (Sample) returns (Response); -... -``` - -##### Making the request - -```python -import grpc, handler_pb2, handler_pb2_grpc - -stub = handler_pb2_grpc.HandlerStub(grpc.insecure_channel("***.amazonaws.com:80")) -r = stub.Predict(handler_pb2.Sample()) -``` - -##### Returning the response - -In the `Predict` method, you'll return the value like this: - -```python -... -def Predict(self, payload): - return self.proto_module_pb2.Response(b="text") -... -``` - -#### Streaming response - -The service method must look like this: - -```protobuf -... -rpc Predict (Sample) returns (stream Response); -... -``` - -##### Making the request - -```python -import grpc, handler_pb2, handler_pb2_grpc - -def generate_iterator(sample_list): - for sample in sample_list: - yield sample - -stub = handler_pb2_grpc.HandlerStub(grpc.insecure_channel("***.amazonaws.com:80")) -for r in stub.Predict(handler_pb2.Sample())): - print(r.b) -``` - -##### Returning the response - -In the `Predict` method, you'll return the streamed values like this: - -```python -... -def Predict(self, payload): - for text in ["a", "b", "c", "d"]: - yield self.proto_module_pb2.Response(b=text) -... -``` - -## Chaining APIs - -It is possible to make requests from one API to another within a Cortex cluster. All running APIs are accessible from within the handler implementation at `http://api-:8888/`, where `` is the name of the API you are making a request to. - -For example, if there is an api named `text-generator` running in the cluster, you could make a request to it from a different API by using: - -```python -import requests - -class Handler: - def handle_post(self, payload): - response = requests.post("http://api-text-generator:8888/", json={"text": "machine learning is"}) - # ... -``` - -Note that the autoscaling configuration (i.e. `target_replica_concurrency`) for the API that is making the request should be modified with the understanding that requests will still be considered "in-flight" with the first API as the request is being fulfilled in the second API (during which it will also be considered "in-flight" with the second API). - -## Structured logging - -You can use Cortex's logger in your handler implemention to log in JSON. This will enrich your logs with Cortex's metadata, and you can add custom metadata to the logs by adding key value pairs to the `extra` key when using the logger. For example: - -```python -... -from cortex_internal.lib.log import logger as cortex_logger - -class Handler: - def handle_post(self, payload): - cortex_logger.info("received payload", extra={"payload": payload}) -``` - -The dictionary passed in via the `extra` will be flattened by one level. e.g. - -```text -{"asctime": "2021-01-19 15:14:05,291", "levelname": "INFO", "message": "received payload", "process": 235, "payload": "this movie is awesome"} -``` - -To avoid overriding essential Cortex metadata, please refrain from specifying the following extra keys: `asctime`, `levelname`, `message`, `labels`, and `process`. Log lines greater than 5 MB in size will be ignored. - -## Cortex Python client - -A default [Cortex Python client](../../clients/python.md#cortex.client.client) environment has been configured for your API. This can be used for deploying/deleting/updating or submitting jobs to your running cluster based on the execution flow of your handler. For example: - -```python -import cortex - -class Handler: - def __init__(self, config): - ... - # get client pointing to the default environment - client = cortex.client() - # get the existing apis in the cluster for something important to you - existing_apis = client.list_apis() -``` diff --git a/docs/workloads/realtime/models.md b/docs/workloads/realtime/models.md deleted file mode 100644 index 6f8685fa80..0000000000 --- a/docs/workloads/realtime/models.md +++ /dev/null @@ -1,439 +0,0 @@ -# Models - -Live model reloading is a mechanism that periodically checks for updated models in the model path(s) provided in `handler.models`. It is automatically enabled for all handler types, including the Python handler type (as long as model paths are specified via `multi_model_reloading` in the `handler` configuration). - -The following is a list of events that will trigger the API to update its model(s): - -* A new model is added to the model directory. -* A model is removed from the model directory. -* A model changes its directory structure. -* A file in the model directory is updated in-place. - -## Python Handler - -To use live model reloading with the Python handler, the model path(s) must be specified in the API's `handler` configuration, via the `multi_model_reloading` field. When models are specified in this manner, your `Handler` class must implement the `load_model()` function, and models can be retrieved by using the `get_model()` method of the `model_client` that's passed into your handler's constructor. - -### Example - -```python -class Handler: - def __init__(self, config, model_client): - self.client = model_client - - def load_model(self, model_path): - # model_path is a path to your model's directory on disk - return load_from_disk(model_path) - - def handle_post(self, payload): - model = self.client.get_model() - return model.predict(payload) -``` - -When multiple models are being served in an API, `model_client.get_model()` can accept a model name: - -```python -class Handler: - # ... - - def handle_post(self, payload, query_params): - model = self.client.get_model(query_params["model"]) - return model.predict(payload) -``` - -`model_client.get_model()` can also accept a model version if a version other than the highest is desired: - -```python -class Handler: - # ... - - def handle_post(self, payload, query_params): - model = self.client.get_model(query_params["model"], query_params["version"]) - return model.predict(payload) -``` - -### Interface - -```python -# initialization code and variables can be declared here in global scope - -class Handler: - def __init__(self, config, model_client): - """(Required) Called once before the API becomes available. Performs - setup such as downloading/initializing the model or downloading a - vocabulary. - - Args: - config (required): Dictionary passed from API configuration (if - specified). This may contain information on where to download - the model and/or metadata. - model_client (required): Python client which is used to retrieve - models for prediction. This should be saved for use in the handler method. - Required when `handler.multi_model_reloading` is specified in - the api configuration. - """ - self.client = model_client - - def load_model(self, model_path): - """Called by Cortex to load a model when necessary. - - This method is required when `handler.multi_model_reloading` - field is specified in the api configuration. - - Warning: this method must not make any modification to the model's - contents on disk. - - Args: - model_path: The path to the model on disk. - - Returns: - The loaded model from disk. The returned object is what - self.client.get_model() will return. - """ - pass - - # define any handler methods for HTTP/gRPC workloads here -``` - - -When explicit model paths are specified in the Python handler's API configuration, Cortex provides a `model_client` to your Handler's constructor. `model_client` is an instance of [ModelClient](https://github.com/cortexlabs/cortex/tree/master/python/serve/cortex_internal/lib/client/python.py) that is used to load model(s) (it calls the `load_model()` method of your handler, which must be defined when using explicit model paths). It should be saved as an instance variable in your handler class, and your handler method should call `model_client.get_model()` to load your model for inference. Preprocessing of the JSON/gRPC payload and postprocessing of predictions can be implemented in your handler method as well. - -When multiple models are defined using the Handler's `multi_model_reloading` field, the `model_client.get_model()` method expects an argument `model_name` which must hold the name of the model that you want to load (for example: `self.client.get_model("text-generator")`). There is also an optional second argument to specify the model version. - -### `load_model` method - -The `load_model()` method that you implement in your `Handler` can return anything that you need to make a prediction. There is one caveat: whatever the return value is, it must be unloadable from memory via the `del` keyword. The following frameworks have been tested to work: - -* PyTorch (CPU & GPU) -* ONNX (CPU & GPU) -* Sklearn/MLFlow (CPU) -* Numpy (CPU) -* Pandas (CPU) -* Caffe (not tested, but should work on CPU & GPU) - -Python data structures containing these types are also supported (e.g. lists and dicts). - -The `load_model()` method takes a single argument, which is a path (on disk) to the model to be loaded. Your `load_model()` method is called behind the scenes by Cortex when you call the `model_client`'s `get_model()` method. Cortex is responsible for downloading your model from S3 onto the local disk before calling `load_model()` with the local path. Whatever `load_model()` returns will be the exact return value of `model_client.get_model()`. Here is the schema for `model_client.get_model()`: - -```python -def get_model(model_name, model_version): - """ - Retrieve a model for inference. - - Args: - model_name (optional): Name of the model to retrieve (when multiple models are deployed in an API). - When handler.models.paths is specified, model_name should be the name of one of the models listed in the API config. - When handler.models.dir is specified, model_name should be the name of a top-level directory in the models dir. - model_version (string, optional): Version of the model to retrieve. Can be omitted or set to "latest" to select the highest version. - - Returns: - The value that's returned by your handler's load_model() method. - """ -``` - -### Specifying models - -Whenever a model path is specified in an API configuration file, it should be a path to an S3 prefix which contains your exported model. Directories may include a single model, or multiple folders each with a single model (note that a "single model" need not be a single file; there can be multiple files for a single model). When multiple folders are used, the folder names must be integer values, and will be interpreted as the model version. Model versions can be any integer, but are typically integer timestamps. It is always assumed that the highest version number is the latest version of your model. - -#### API spec - -##### Single model - -The most common pattern is to serve a single model per API. The path to the model is specified in the `path` field in the `handler.multi_model_reloading` configuration. For example: - -```yaml -# cortex.yaml - -- name: iris-classifier - kind: RealtimeAPI - handler: - # ... - type: python - multi_model_reloading: - path: s3://my-bucket/models/text-generator/ -``` - -##### Multiple models - -It is possible to serve multiple models from a single API. The paths to the models are specified in the api configuration, either via the `multi_model_reloading.paths` or `multi_model_reloading.dir` field in the `handler` configuration. For example: - -```yaml -# cortex.yaml - -- name: iris-classifier - kind: RealtimeAPI - handler: - # ... - type: python - multi_model_reloading: - paths: - - name: iris-classifier - path: s3://my-bucket/models/text-generator/ - # ... -``` - -or: - -```yaml -# cortex.yaml - -- name: iris-classifier - kind: RealtimeAPI - handler: - # ... - type: python - multi_model_reloading: - dir: s3://my-bucket/models/ -``` - -It is also not necessary to specify the `multi_model_reloading` section at all, since you can download and load the model in your handler's `__init__()` function. That said, it is necessary to use the `multi_model_reloading` field to take advantage of live model reloading or multi-model caching. - -When using the `multi_model_reloading.paths` field, each path must be a valid model directory (see above for valid model directory structures). - -When using the `multi_model_reloading.dir` field, the directory provided may contain multiple subdirectories, each of which is a valid model directory. For example: - -```text - s3://my-bucket/models/ - ├── text-generator - | └── * (model files) - └── sentiment-analyzer - ├── 24753823/ - | └── * (model files) - └── 26234288/ - └── * (model files) -``` - -In this case, there are two models in the directory, one of which is named "text-generator", and the other is named "sentiment-analyzer". - -#### Structure - -Any model structure is accepted. Here is an example: - -```text - s3://my-bucket/models/text-generator/ - ├── model.pkl - └── data.txt -``` - -or for a versioned model: - -```text - s3://my-bucket/models/text-generator/ - ├── 1523423423/ (version number, usually a timestamp) - | ├── model.pkl - | └── data.txt - └── 2434389194/ (version number, usually a timestamp) - ├── model.pkl - └── data.txt -``` - -## TensorFlow Handler - -In addition to the [standard Python Handler](handler.md), Cortex also supports another handler called the TensorFlow handler, which can be used to run TensorFlow models exported as `SavedModel` models. When using the TensorFlow handler, the model path(s) must be specified in the API's `handler` configuration, via the `models` field. - -### Example - -```python -class Handler: - def __init__(self, tensorflow_client, config): - self.client = tensorflow_client - - def handle_post(self, payload): - return self.client.predict(payload) -``` - -When multiple models are being served in an API, `tensorflow_client.predict()` can accept a model name: - -```python -class Handler: - # ... - - def handle_post(self, payload, query_params): - return self.client.predict(payload, query_params["model"]) -``` - -`tensorflow_client.predict()` can also accept a model version if a version other than the highest is desired: - -```python -class Handler: - # ... - - def handle_post(self, payload, query_params): - return self.client.predict(payload, query_params["model"], query_params["version"]) -``` - -Note: when using Inferentia models with the TensorFlow handler type, live model reloading is only supported if `handler.processes_per_replica` is set to 1 (the default value). - -### Interface - -```python -class Handler: - def __init__(self, tensorflow_client, config): - """(Required) Called once before the API becomes available. Performs - setup such as downloading/initializing a vocabulary. - - Args: - tensorflow_client (required): TensorFlow client which is used to - make predictions. This should be saved for use in the handler method. - config (required): Dictionary passed from API configuration (if - specified). - """ - self.client = tensorflow_client - # Additional initialization may be done here - - # define any handler methods for HTTP/gRPC workloads here -``` - - -Cortex provides a `tensorflow_client` to your Handler's constructor. `tensorflow_client` is an instance of [TensorFlowClient](https://github.com/cortexlabs/cortex/tree/master/python/serve/cortex_internal/lib/client/tensorflow.py) that manages a connection to a TensorFlow Serving container to make predictions using your model. It should be saved as an instance variable in your Handler class, and your handler method should call `tensorflow_client.predict()` to make an inference with your exported TensorFlow model. Preprocessing of the JSON payload and postprocessing of predictions can be implemented in your handler method as well. - -When multiple models are defined using the Handler's `models` field, the `tensorflow_client.predict()` method expects a second argument `model_name` which must hold the name of the model that you want to use for inference (for example: `self.client.predict(payload, "text-generator")`). There is also an optional third argument to specify the model version. - -If you need to share files between your handler implementation and the TensorFlow Serving container, you can create a new directory within `/mnt` (e.g. `/mnt/user`) and write files to it. The entire `/mnt` directory is shared between containers, but do not write to any of the directories in `/mnt` that already exist (they are used internally by Cortex). - -### `predict` method - -Inference is performed by using the `predict` method of the `tensorflow_client` that's passed to the handler's constructor: - -```python -def predict(model_input, model_name, model_version) -> dict: - """ - Run prediction. - - Args: - model_input: Input to the model. - model_name (optional): Name of the model to retrieve (when multiple models are deployed in an API). - When handler.models.paths is specified, model_name should be the name of one of the models listed in the API config. - When handler.models.dir is specified, model_name should be the name of a top-level directory in the models dir. - model_version (string, optional): Version of the model to retrieve. Can be omitted or set to "latest" to select the highest version. - - Returns: - dict: TensorFlow Serving response converted to a dictionary. - """ -``` - -### Specifying models - -Whenever a model path is specified in an API configuration file, it should be a path to an S3 prefix which contains your exported model. Directories may include a single model, or multiple folders each with a single model (note that a "single model" need not be a single file; there can be multiple files for a single model). When multiple folders are used, the folder names must be integer values, and will be interpreted as the model version. Model versions can be any integer, but are typically integer timestamps. It is always assumed that the highest version number is the latest version of your model. - -#### API spec - -##### Single model - -The most common pattern is to serve a single model per API. The path to the model is specified in the `path` field in the `handler.models` configuration. For example: - -```yaml -# cortex.yaml - -- name: iris-classifier - kind: RealtimeAPI - handler: - # ... - type: tensorflow - models: - path: s3://my-bucket/models/text-generator/ -``` - -##### Multiple models - -It is possible to serve multiple models from a single API. The paths to the models are specified in the api configuration, either via the `models.paths` or `models.dir` field in the `handler` configuration. For example: - -```yaml -# cortex.yaml - -- name: iris-classifier - kind: RealtimeAPI - handler: - # ... - type: tensorflow - models: - paths: - - name: iris-classifier - path: s3://my-bucket/models/text-generator/ - # ... -``` - -or: - -```yaml -# cortex.yaml - -- name: iris-classifier - kind: RealtimeAPI - handler: - # ... - type: tensorflow - models: - dir: s3://my-bucket/models/ -``` - -When using the `models.paths` field, each path must be a valid model directory (see above for valid model directory structures). - -When using the `models.dir` field, the directory provided may contain multiple subdirectories, each of which is a valid model directory. For example: - -```text - s3://my-bucket/models/ - ├── text-generator - | └── * (model files) - └── sentiment-analyzer - ├── 24753823/ - | └── * (model files) - └── 26234288/ - └── * (model files) -``` - -In this case, there are two models in the directory, one of which is named "text-generator", and the other is named "sentiment-analyzer". - -#### Structure - -##### On CPU/GPU - -The model path must be a SavedModel export: - -```text - s3://my-bucket/models/text-generator/ - ├── saved_model.pb - └── variables/ - ├── variables.index - ├── variables.data-00000-of-00003 - ├── variables.data-00001-of-00003 - └── variables.data-00002-of-... -``` - -or for a versioned model: - -```text - s3://my-bucket/models/text-generator/ - ├── 1523423423/ (version number, usually a timestamp) - | ├── saved_model.pb - | └── variables/ - | ├── variables.index - | ├── variables.data-00000-of-00003 - | ├── variables.data-00001-of-00003 - | └── variables.data-00002-of-... - └── 2434389194/ (version number, usually a timestamp) - ├── saved_model.pb - └── variables/ - ├── variables.index - ├── variables.data-00000-of-00003 - ├── variables.data-00001-of-00003 - └── variables.data-00002-of-... -``` - -##### On Inferentia - -When Inferentia models are used, the directory structure is slightly different: - -```text - s3://my-bucket/models/text-generator/ - └── saved_model.pb -``` - -or for a versioned model: - -```text - s3://my-bucket/models/text-generator/ - ├── 1523423423/ (version number, usually a timestamp) - | └── saved_model.pb - └── 2434389194/ (version number, usually a timestamp) - └── saved_model.pb -``` diff --git a/docs/workloads/realtime/multi-model/caching.md b/docs/workloads/realtime/multi-model/caching.md deleted file mode 100644 index 3177f68ca0..0000000000 --- a/docs/workloads/realtime/multi-model/caching.md +++ /dev/null @@ -1,14 +0,0 @@ -# Multi-model caching - -Multi-model caching allows each replica to serve more models than would fit into its memory by keeping a specified number of models in memory (and disk) at a time. When the in-memory model limit is reached, the least recently accessed model is evicted from the cache. This can be useful when you have many models, and some models are frequently accessed while a larger portion of them are rarely used, or when running on smaller instances to control costs. - -The model cache is a two-layer cache, configured by the following parameters in the `handler.models` configuration: - -* `cache_size` sets the number of models to keep in memory -* `disk_cache_size` sets the number of models to keep on disk (must be greater than or equal to `cache_size`) - -Both of these fields must be specified, in addition to either the `dir` or `paths` field (which specifies the model paths, see [models](../../realtime/models.md) for documentation). Multi-model caching is only supported if `handler.processes_per_replica` is set to 1 (the default value). - -## Out of memory errors - -Cortex runs a background process every 10 seconds that counts the number of models in memory and on disk, and evicts the least recently used models if the count exceeds `cache_size` / `disk_cache_size`. If many new models are requested between executions of the process, there may be more models in memory and/or on disk than the configured `cache_size` or `disk_cache_size` limits which could lead to out of memory errors. diff --git a/docs/workloads/realtime/multi-model/configuration.md b/docs/workloads/realtime/multi-model/configuration.md deleted file mode 100644 index c74d950f3d..0000000000 --- a/docs/workloads/realtime/multi-model/configuration.md +++ /dev/null @@ -1,118 +0,0 @@ -# Configuration - -## Python Handler - -### Specifying models in API configuration - -#### `cortex.yaml` - -The directory `s3://cortex-examples/sklearn/mpg-estimator/linreg/` contains 4 different versions of the model. - -```yaml -- name: mpg-estimator - kind: RealtimeAPI - handler: - type: python - path: handler.py - models: - path: s3://cortex-examples/sklearn/mpg-estimator/linreg/ -``` - -#### `handler.py` - -```python -import mlflow.sklearn - - -class Handler: - def __init__(self, config, python_client): - self.client = python_client - - def load_model(self, model_path): - return mlflow.sklearn.load_model(model_path) - - def handle_post(self, payload, query_params): - model_version = query_params.get("version") - - # model_input = ... - - model = self.client.get_model(model_version=model_version) - result = model.predict(model_input) - - return {"prediction": result, "model": {"version": model_version}} -``` - -### Without specifying models in API configuration - -#### `cortex.yaml` - -```yaml -- name: text-analyzer - kind: RealtimeAPI - handler: - type: python - path: handler.py - ... -``` - -#### `handler.py` - -```python -class Handler: - def __init__(self, config): - self.analyzer = initialize_model("sentiment-analysis") - self.summarizer = initialize_model("summarization") - - def handle_post(self, query_params, payload): - model_name = query_params.get("model") - model_input = payload["text"] - - # ... - - if model_name == "analyzer": - results = self.analyzer(model_input) - predicted_label = postprocess(results) - return {"label": predicted_label} - elif model_name == "summarizer": - results = self.summarizer(model_input) - predicted_label = postprocess(results) - return {"label": predicted_label} - else: - return JSONResponse({"error": f"unknown model: {model_name}"}, status_code=400) -``` - -## TensorFlow Handler - -### `cortex.yaml` - -```yaml -- name: multi-model-classifier - kind: RealtimeAPI - handler: - type: tensorflow - path: handler.py - models: - paths: - - name: inception - path: s3://cortex-examples/tensorflow/image-classifier/inception/ - - name: iris - path: s3://cortex-examples/tensorflow/iris-classifier/nn/ - - name: resnet50 - path: s3://cortex-examples/tensorflow/resnet50/ - ... -``` - -### `handler.py` - -```python -class Handler: - def __init__(self, tensorflow_client, config): - self.client = tensorflow_client - - def handle_post(self, payload, query_params): - model_name = query_params["model"] - model_input = preprocess(payload["url"]) - results = self.client.predict(model_input, model_name) - predicted_label = postprocess(results) - return {"label": predicted_label} -``` diff --git a/docs/workloads/realtime/multi-model/example.md b/docs/workloads/realtime/multi-model/example.md deleted file mode 100644 index d09fc790c9..0000000000 --- a/docs/workloads/realtime/multi-model/example.md +++ /dev/null @@ -1,43 +0,0 @@ -# Multi-model API - -Deploy several models in a single API to improve resource utilization efficiency. - -## Define a multi-model API - -```python -# multi_model.py - -import cortex - -class Handler: - def __init__(self, config): - from transformers import pipeline - self.analyzer = pipeline(task="sentiment-analysis") - - import wget - import fasttext - wget.download( - "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin", "/tmp/model" - ) - self.language_identifier = fasttext.load_model("/tmp/model") - - def handle_post(self, query_params, payload): - model = query_params.get("model") - if model == "sentiment": - return self.analyzer(payload["text"])[0] - elif model == "language": - return self.language_identifier.predict(payload["text"])[0][0][-2:] - -requirements = ["tensorflow", "transformers", "wget", "fasttext"] - -api_spec = {"name": "multi-model", "kind": "RealtimeAPI"} - -cx = cortex.client("cortex") -cx.deploy_realtime_api(api_spec, handler=Handler, requirements=requirements) -``` - -## Deploy - -```bash -python multi_model.py -``` diff --git a/docs/workloads/realtime/parallelism.md b/docs/workloads/realtime/parallelism.md deleted file mode 100644 index f271805f20..0000000000 --- a/docs/workloads/realtime/parallelism.md +++ /dev/null @@ -1,9 +0,0 @@ -# Replica parallelism - -Replica parallelism can be configured with the following fields in the `handler` configuration: - -* `processes_per_replica` (default: 1): Each replica runs a web server with `processes_per_replica` processes. For APIs running with multiple CPUs per replica, using 1-3 processes per unit of CPU generally leads to optimal throughput. For example, if `cpu` is 2, a value between 2 and 6 `processes_per_replica` is reasonable. The optimal number will vary based on the workload's characteristics and the CPU compute request for the API. - -* `threads_per_process` (default: 1): Each process uses a thread pool of size `threads_per_process` to process requests. For applications that are not CPU intensive such as high I/O (e.g. downloading files), GPU-based inference, or Inferentia-based inference, increasing the number of threads per process can increase throughput. For CPU-bound applications such as running your model inference on a CPU, using 1 thread per process is recommended to avoid unnecessary context switching. Some applications are not thread-safe, and therefore must be run with 1 thread per process. - -`processes_per_replica` * `threads_per_process` represents the total number of requests that your replica can work on concurrently. For example, if `processes_per_replica` is 2 and `threads_per_process` is 2, and the replica was hit with 5 concurrent requests, 4 would immediately begin to be processed, and 1 would be waiting for a thread to become available. If the replica were hit with 3 concurrent requests, all three would begin processing immediately. diff --git a/docs/workloads/realtime/realtime-apis.md b/docs/workloads/realtime/realtime-apis.md new file mode 100644 index 0000000000..ebfc7fc8d0 --- /dev/null +++ b/docs/workloads/realtime/realtime-apis.md @@ -0,0 +1,3 @@ +# Realtime APIs + +Realtime APIs respond to requests in real-time and autoscale based on in-flight request volumes. diff --git a/docs/workloads/realtime/server-side-batching.md b/docs/workloads/realtime/server-side-batching.md deleted file mode 100644 index f46646d0d7..0000000000 --- a/docs/workloads/realtime/server-side-batching.md +++ /dev/null @@ -1,83 +0,0 @@ -# Server-side batching - -Server-side batching is the process of aggregating multiple real-time requests into a single batch execution, which increases throughput at the expense of latency. Inference is triggered when either a maximum number of requests have been received, or when a certain amount of time has passed since receiving the first request, whichever comes first. Once a threshold is reached, the handling function is run on the received requests and responses are returned individually back to the clients. This process is transparent to the clients. - -The Python and TensorFlow handlers allow for the use of the following 2 fields in the `server_side_batching` configuration: - -* `max_batch_size`: The maximum number of requests to aggregate before running inference. This is an instrument for controlling throughput. The maximum size can be achieved if `batch_interval` is long enough to collect `max_batch_size` requests. - -* `batch_interval`: The maximum amount of time to spend waiting for additional requests before running inference on the batch of requests. If fewer than `max_batch_size` requests are received after waiting the full `batch_interval`, then inference will run on the requests that have been received. This is an instrument for controlling latency. - -{% hint style="note" %} -Server-side batching is not supported for APIs that use the gRPC protocol. -{% endhint %} - -{% hint style="note" %} -Server-side batching is only supported on the `handle_post` method. -{% endhint %} - -## Python Handler - -When using server-side batching with the Python handler, the arguments that are passed into your handler's function will be lists: `payload` will be a list of payloads, `query_params` will be a list of query parameter dictionaries, and `headers` will be a list of header dictionaries. The lists will all have the same length, where a particular index across all arguments corresponds to a single request (i.e. `payload[2]`, `query_params[2]`, and `headers[2]` correspond to a single request). Your handle function must return a list of responses in the same order that they were received (i.e. the 3rd element in returned list must be the response associated with `payload[2]`). - -## TensorFlow Handler - -In order to use server-side batching with the TensorFlow handler, the only requirement is that model's graph must be built such that batches can be accepted as input/output. No modifications to your handler implementation are required. - -The following is an example of how the input `x` and the output `y` of the graph could be shaped to be compatible with server-side batching: - -```python -batch_size = None -sample_shape = [340, 240, 3] # i.e. RGB image -output_shape = [1000] # i.e. image labels - -with graph.as_default(): - # ... - x = tf.placeholder(tf.float32, shape=[batch_size] + sample_shape, name="input") - y = tf.placeholder(tf.float32, shape=[batch_size] + output_shape, name="output") - # ... -``` - -### Troubleshooting - -Errors will be encountered if the model hasn't been built for batching. - -The following error is an example of what happens when the input shape doesn't accommodate batching - e.g. when its shape is `[height, width, 3]` instead of `[batch_size, height, width, 3]`: - -```text -Batching session Run() input tensors must have at least one dimension. -``` - -Here is another example of setting the output shape inappropriately for batching - e.g. when its shape is `[labels]` instead of `[batch_size, labels]`: - -```text -Batched output tensor has 0 dimensions. -``` - -The solution to these errors is to incorporate into the model's graph another dimension (a placeholder for batch size) placed on the first position for both its input and output. - -The following is an example of how the input `x` and the output `y` of the graph could be shaped to be compatible with server-side batching: - -```python -batch_size = None -sample_shape = [340, 240, 3] # i.e. RGB image -output_shape = [1000] # i.e. image labels - -with graph.as_default(): - # ... - x = tf.placeholder(tf.float32, shape=[batch_size] + sample_shape, name="input") - y = tf.placeholder(tf.float32, shape=[batch_size] + output_shape, name="output") - # ... -``` - -## Optimization - -When optimizing for both throughput and latency, you will likely want keep the `max_batch_size` to a relatively small value. Even though a higher `max_batch_size` with a low `batch_interval` (when there are many requests coming in) can offer a significantly higher throughput, the overall latency could be quite large. The reason is that for a request to get back a response, it has to wait until the entire batch is processed, which means that the added latency due to the `batch_interval` can pale in comparison. For instance, let's assume that a single request takes 50ms, and that when the batch size is set to 128, the processing time for a batch is 1280ms (i.e. 10ms per sample). So while the throughput is now 5 times higher, it takes 1280ms + `batch_interval` to get back a response (instead of 50ms). This is the trade-off with server-side batching. - -When optimizing for maximum throughput, a good rule of thumb is to follow these steps: - -1. Determine the maximum throughput of one API replica when `server_side_batching` is not enabled (same as if `max_batch_size` were set to 1). This can be done with a load test (make sure to set `max_replicas` to 1 to disable autoscaling). -1. Determine the highest `batch_interval` with which you are still comfortable for your application. Keep in mind that the batch interval is not the only component of the overall latency - the inference on the batch and the pre/post processing also have to occur. -1. Multiply the maximum throughput from step 1 by the `batch_interval` from step 2. The result is a number which you can assign to `max_batch_size`. -1. Run the load test again. If the inference fails with that batch size (e.g. due to running out of GPU or RAM memory), then reduce `max_batch_size` to a level that works (reduce `batch_interval` by the same factor). -1. Use the load test to determine the peak throughput of the API replica. Multiply the observed throughput by the `batch_interval` to calculate the average batch size. If the average batch size coincides with `max_batch_size`, then it might mean that the throughput could still be further increased by increasing `max_batch_size`. If it's lower, then it means that `batch_interval` is triggering the inference before `max_batch_size` requests have been aggregated. If modifying both `max_batch_size` and `batch_interval` doesn't improve the throughput, then the service may be bottlenecked by something else (e.g. CPU, network IO, `processes_per_replica`, `threads_per_process`, etc). diff --git a/docs/workloads/realtime/statuses.md b/docs/workloads/realtime/statuses.md index f61ea52d7b..2ee32aca40 100644 --- a/docs/workloads/realtime/statuses.md +++ b/docs/workloads/realtime/statuses.md @@ -5,6 +5,6 @@ | live | API is deployed and ready to serve requests (at least one replica is running) | | updating | API is updating | | error | API was not created due to an error; run `cortex logs ` to view the logs | -| error (image pull) | API was not created because one of the specified Docker images was inaccessible at runtime; check that your API's docker images exist and are accessible via your cluster operator's AWS credentials | +| error (image pull) | API was not created because one of the specified Docker images was inaccessible at runtime; check that your API's docker images exist and are accessible via your cluster's AWS credentials | | error (out of memory) | API was terminated due to excessive memory usage; try allocating more memory to the API and re-deploying | | compute unavailable | API could not start due to insufficient memory, CPU, GPU, or Inf in the cluster; some replicas may be ready | diff --git a/docs/workloads/realtime/traffic-splitter.md b/docs/workloads/realtime/traffic-splitter.md new file mode 100644 index 0000000000..07afe1726d --- /dev/null +++ b/docs/workloads/realtime/traffic-splitter.md @@ -0,0 +1,65 @@ +# Traffic Splitter + +Traffic Splitters can be used to expose multiple RealtimeAPIs as a single endpoint for A/B tests, multi-armed bandits, or canary deployments. + +## Configuration + +```yaml +- name: # name of the traffic splitter (required) + kind: TrafficSplitter # must be "TrafficSplitter" for traffic splitters (required) + networking: # networking configuration (default: see below) + endpoint: # the endpoint for the traffic splitter (default: ) + apis: # list of Realtime APIs to target (required) + - name: # name of a Realtime API that is already running or is included in the same configuration file (required) + weight: # percentage of traffic to route to the Realtime API (all non-shadow weights must sum to 100) (required) + shadow: # duplicate incoming traffic and send fire-and-forget to this api (only one shadow per traffic splitter) (default: false) +``` + +## Example + +This example showcases Cortex's Python client, but these steps can also be performed by using the Cortex CLI. + +### Deploy a traffic splitter + +```python +traffic_splitter_spec = { + "name": "sentiment-analyzer", + "kind": "TrafficSplitter", + "apis": [ + {"name": "sentiment-analyzer-a", "weight": 50}, + {"name": "sentiment-analyzer-b", "weight": 50}, + ], +} + +cx.deploy(traffic_splitter_spec) +``` + +### Update the weights + +```python +new_traffic_splitter_spec = { + "name": "sentiment-analyzer", + "kind": "TrafficSplitter", + "apis": [ + {"name": "sentiment-analyzer-a", "weight": 1}, + {"name": "sentiment-analyzer-b", "weight": 99}, + ], +} + +cx.deploy(new_traffic_splitter_spec) +``` + +### Update the target APIs + +```python +new_traffic_splitter_spec = { + "name": "sentiment-analyzer", + "kind": "TrafficSplitter", + "apis": [ + {"name": "sentiment-analyzer-b", "weight": 50}, + {"name": "sentiment-analyzer-c", "weight": 50}, + ], +} + +cx.deploy(new_traffic_splitter_spec) +``` diff --git a/docs/workloads/realtime/traffic-splitter/configuration.md b/docs/workloads/realtime/traffic-splitter/configuration.md deleted file mode 100644 index a498a569a4..0000000000 --- a/docs/workloads/realtime/traffic-splitter/configuration.md +++ /dev/null @@ -1,12 +0,0 @@ -# Configuration - -```yaml -- name: # Traffic Splitter name (required) - kind: TrafficSplitter - networking: - endpoint: # the endpoint for the Traffic Splitter (default: ) - apis: # list of Realtime APIs to target - - name: # name of a Realtime API that is already running or is included in the same configuration file (required) - weight: # percentage of traffic to route to the Realtime API (all non-shadow weights must sum to 100) (required) - shadow: # duplicate incoming traffic and send fire-and-forget to this api (only one shadow per traffic splitter) (default: false) -``` diff --git a/docs/workloads/realtime/traffic-splitter/example.md b/docs/workloads/realtime/traffic-splitter/example.md deleted file mode 100644 index bb4fa6d28b..0000000000 --- a/docs/workloads/realtime/traffic-splitter/example.md +++ /dev/null @@ -1,66 +0,0 @@ -# TrafficSplitter - -Expose multiple RealtimeAPIs as a single endpoint for A/B tests, multi-armed bandits, or canary deployments. - -## Deploy APIs - -```python -import cortex - -class Handler: - def __init__(self, config): - from transformers import pipeline - self.model = pipeline(task="text-generation") - - def handle_post(self, payload): - return self.model(payload["text"])[0] - -requirements = ["tensorflow", "transformers"] - -api_spec_cpu = { - "name": "text-generator-cpu", - "kind": "RealtimeAPI", - "compute": { - "cpu": 1, - }, -} - -api_spec_gpu = { - "name": "text-generator-gpu", - "kind": "RealtimeAPI", - "compute": { - "gpu": 1, - }, -} - -cx = cortex.client("cortex") -cx.deploy_realtime_api(api_spec_cpu, handler=Handler, requirements=requirements) -cx.deploy_realtime_api(api_spec_gpu, handler=Handler, requirements=requirements) -``` - -## Deploy a traffic splitter - -```python -traffic_splitter_spec = { - "name": "text-generator", - "kind": "TrafficSplitter", - "apis": [ - {"name": "text-generator-cpu", "weight": 50}, - {"name": "text-generator-gpu", "weight": 50}, - ], -} - -cx.deploy(traffic_splitter_spec) -``` - -## Update the weights of the traffic splitter - -```python -traffic_splitter_spec = cx.get_api("text-generator")["spec"]["submitted_api_spec"] - -# send 99% of the traffic to text-generator-gpu -traffic_splitter_spec["apis"][0]["weight"] = 1 -traffic_splitter_spec["apis"][1]["weight"] = 99 - -cx.deploy(traffic_splitter_spec) -``` diff --git a/docs/workloads/realtime/troubleshooting.md b/docs/workloads/realtime/troubleshooting.md index d2d441510f..7e01e6f524 100644 --- a/docs/workloads/realtime/troubleshooting.md +++ b/docs/workloads/realtime/troubleshooting.md @@ -1,13 +1,13 @@ # Troubleshooting -## 404 or 503 error responses from API requests +## 503 error responses from API requests -When making requests to your API, it's possible to get a `{"message":"Not Found"}` error message (with HTTP status code `404`), or a `no healthy upstream` error message (with HTTP status code `503`). This means that there are currently no live replicas running for your API. This could happen for a few reasons: +When making requests to your API, it's possible to get a `no healthy upstream` error message (with HTTP status code `503`). This means that there are currently no live replicas running for your API. This could happen for a few reasons: -1. It's possible that your API is simply not ready yet. You can check the status of your API with `cortex get API_NAME`, and stream the logs with `cortex logs API_NAME`. -1. Your API may have errored during initialization or while responding to a previous request. `cortex get API_NAME` will show the status of your API, and you can view the logs with `cortex logs API_NAME`. +1. It's possible that your API is simply not ready yet. You can check the status of your API with `cortex get API_NAME`, and stream the logs for a single replica (at random) with `cortex logs API_NAME`. +1. Your API may have errored during initialization or while responding to a previous request. `cortex get API_NAME` will show the status of your API, and you can view the logs for all replicas via Cloudwatch Logs Insights. -It is also possible to receive a `{"message":"Service Unavailable"}` error message (with HTTP status code `503`) if you are using API Gateway in front of your API endpoints and if your request exceeds API Gateway's 29 second timeout. If the request is exceeding the API Gateway timeout, your client should receive the `{"message":"Service Unavailable"}` response ~29 seconds after making the request. To confirm that this is the issue, you can modify your handle function to immediately return a response (e.g. `return "ok"`), re-deploy your API, wait for the update to complete, and try making a request. If your client successfully receives the "ok" response, it is likely that the API Gateway timeout is occurring. You can either modify your handler implementation to take less time, run on faster hardware (e.g. GPUs), or don't use API Gateway (there is no timeout when using the API's endpoint). +If you are using API Gateway in front of your API endpoints, it is also possible to receive a `{"message":"Service Unavailable"}` error message (with HTTP status code `503`) after 29 seconds if your request exceeds API Gateway's 29 second timeout. If this is the case, you can either modify your code to take less time, run on faster hardware (e.g. GPUs), or don't use API Gateway (there is no timeout when using the API's endpoint directly). ## API is stuck updating @@ -23,7 +23,7 @@ When you created your Cortex cluster, you configured `max_instances` for each no You can check the current value of `max_instances` for the selected node group by running `cortex cluster info --config cluster.yaml` (or `cortex cluster info --name --region ` if you have the name and region of the cluster). -Once you have the name and region of the cluster, you can update `max_instances` by specifying the desired number of `max_instances` for your node group with `cortex cluster scale --name --region --node-group --min-instances --max-instances `. +Once you have the name and region of the cluster, you can update `max_instances` by specifying the desired number of `max_instances` for your node group with `cortex cluster scale --name --region --node-group --max-instances `. ## Check your AWS auto scaling group activity history @@ -58,13 +58,12 @@ Here is an example: You set `max_instances` to 1, or your AWS account limits you If you're running in a development environment, this rolling update behavior can be undesirable. -You can disable rolling updates for your API in your API configuration (e.g. in `cortex.yaml`): set `max_surge` to 0 (in the `update_strategy` configuration). E.g.: +You can disable rolling updates for your API in your API configuration: set `max_surge` to 0 in the `update_strategy` section, E.g.: ```yaml -- name: text-generator - handler: - type: python - ... +- name: my-api + kind: RealtimeAPI + # ... update_strategy: max_surge: 0 ``` diff --git a/docs/workloads/task/configuration.md b/docs/workloads/task/configuration.md index 84c31312c6..6eabb1d7f7 100644 --- a/docs/workloads/task/configuration.md +++ b/docs/workloads/task/configuration.md @@ -1,27 +1,35 @@ # Configuration - ```yaml -- name: # API name (required) - kind: TaskAPI - definition: - path: # path to a python file with a Task class definition, relative to the Cortex root (required) - config: # arbitrary dictionary passed to the callable method of the Task class (can be overridden by config passed in job submission) (optional) - dependencies: # (optional) - pip: # relative path to requirements.txt (default: requirements.txt) - conda: # relative path to conda-packages.txt (default: conda-packages.txt) - shell: # relative path to a shell script for system package installation (default: dependencies.sh) - python_path: # path to the root of your Python folder that will be appended to PYTHONPATH (default: folder containing cortex.yaml) - shm_size: # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null) - image: # docker image to use for the Task (default: quay.io/cortexlabs/python-handler-cpu:master, quay.io/cortexlabs/python-handler-gpu:master-cuda10.2-cudnn8, or quay.io/cortexlabs/python-handler-inf:master based on compute) - env: # dictionary of environment variables - log_level: # log level that can be "debug", "info", "warning" or "error" (default: "info") - networking: - endpoint: # the endpoint for the API (default: ) - compute: - cpu: # CPU request per worker. One unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the "m" suffix (default: 200m) - gpu: # GPU request per worker. One unit of GPU corresponds to one virtual GPU (default: 0) - inf: # Inferentia request per worker. One unit corresponds to one Inferentia ASIC with 4 NeuronCores and 8GB of cache memory. Each process will have one NeuronCore Group with (4 * inf / processes_per_replica) NeuronCores, so your model should be compiled to run on (4 * inf / processes_per_replica) NeuronCores. (default: 0) - mem: # memory request per worker. One unit of memory is one byte and can be expressed as an integer or by using one of these suffixes: K, M, G, T (or their power-of two counterparts: Ki, Mi, Gi, Ti) (default: Null) - node_groups: # to select specific node groups (optional) +- name: # name of the API (required) + kind: TaskAPI # must be "TaskAPI" for task APIs (required) + pod: # pod configuration (required) + containers: # configurations for the containers to run (at least one constainer must be provided) + - name: # name of the container (required) + image: # docker image to use for the container (required) + command: # entrypoint (required) + args: # arguments to the entrypoint (default: no args) + env: # dictionary of environment variables to set in the container (optional) + compute: # compute resource requests (default: see below) + cpu: # CPU request for the container; one unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the "m" suffix (default: 200m) + gpu: # GPU request for the container; one unit of GPU corresponds to one virtual GPU (default: 0) + inf: # Inferentia request for the container; one unit of inf corresponds to one virtual Inferentia chip (default: 0) + mem: # memory request for the container; one unit of memory is one byte and can be expressed as an integer or by using one of these suffixes: K, M, G, T (or their power-of two counterparts: Ki, Mi, Gi, Ti) (default: Null) + shm: # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null) + liveness_probe: # periodic probe of container liveness; container will be restarted if the probe fails (optional) + http_get: # specifies an http endpoint which must respond with status code 200 (only one of http_get, tcp_socket, and exec may be specified) + port: # the port to access on the container (required) + path: # the path to access on the HTTP server (default: /) + tcp_socket: # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified) + port: # the port to access on the container (required) + exec: # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified) + command: [] # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) + initial_delay_seconds: # number of seconds after the container has started before the probe is initiated (default: 0) + timeout_seconds: # number of seconds until the probe times out (default: 1) + period_seconds: # how often (in seconds) to perform the probe (default: 10) + success_threshold: # minimum consecutive successes for the probe to be considered successful after having failed (default: 1) + failure_threshold: # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3) + node_groups: # a list of node groups on which this API can run (default: all node groups are eligible) + networking: # networking configuration (default: see below) + endpoint: # endpoint for the API (default: ) ``` diff --git a/docs/workloads/task/definitions.md b/docs/workloads/task/definitions.md deleted file mode 100644 index ed50b34918..0000000000 --- a/docs/workloads/task/definitions.md +++ /dev/null @@ -1,91 +0,0 @@ -# Implementation - -## Project files - -Cortex makes all files in the project directory (i.e. the directory which contains `cortex.yaml`) available for use in your Task implementation. Python bytecode files (`*.pyc`, `*.pyo`, `*.pyd`), files or folders that start with `.`, and the api configuration file (e.g. `cortex.yaml`) are excluded. - -The following files can also be added at the root of the project's directory: - -* `.cortexignore` file, which follows the same syntax and behavior as a [.gitignore file](https://git-scm.com/docs/gitignore). This may be necessary if you are reaching the size limit for your project directory (32mb). -* `.env` file, which exports environment variables that can be used in the task. Each line of this file must follow the `VARIABLE=value` format. - -For example, if your directory looks like this: - -```text -./my-classifier/ -├── cortex.yaml -├── values.json -├── task.py -├── ... -└── requirements.txt -``` - -You can access `values.json` in your Task like this: - -```python -import json - -class Task: - def __call__(self, config): - with open('values.json', 'r') as values_file: - values = json.load(values_file) - self.values = values -``` - -## Task - -### Interface - -```python -# initialization code and variables can be declared here in global scope - -class Task: - def __call__(self, config): - """(Required) Task runnable. - - Args: - config (required): Dictionary passed from API configuration (if - specified) merged with configuration passed in with Job - Submission API. If there are conflicting keys, values in - configuration specified in Job submission takes precedence. - """ - pass -``` - -## Structured logging - -You can use Cortex's logger in your handler implementation to log in JSON. This will enrich your logs with Cortex's metadata, and you can add custom metadata to the logs by adding key value pairs to the `extra` key when using the logger. For example: - -```python -... -from cortex_internal.lib.log import logger as cortex_logger - -class Task: - def __call__(self, config): - ... - cortex_logger.info("completed validations", extra={"accuracy": accuracy}) -``` - -The dictionary passed in via the `extra` will be flattened by one level. e.g. - -```text -{"asctime": "2021-01-19 15:14:05,291", "levelname": "INFO", "message": "completed validations", "process": 235, "accuracy": 0.97} -``` - -To avoid overriding essential Cortex metadata, please refrain from specifying the following extra keys: `asctime`, `levelname`, `message`, `labels`, and `process`. Log lines greater than 5 MB in size will be ignored. - -## Cortex Python client - -A default [Cortex Python client](../../clients/python.md#cortex.client.client) environment has been configured for your API. This can be used for deploying/deleting/updating or submitting jobs to your running cluster based on the execution flow of your task. For example: - -```python -import cortex - -class Task: - def __call__(self, config): - ... - # get client pointing to the default environment - client = cortex.client() - # deploy API in the existing cluster as part of your pipeline workflow - client.deploy(...) -``` diff --git a/docs/workloads/task/jobs.md b/docs/workloads/task/jobs.md index fd8701ecc6..bde647cfab 100644 --- a/docs/workloads/task/jobs.md +++ b/docs/workloads/task/jobs.md @@ -1,6 +1,6 @@ # TaskAPI jobs -## Get the TaskAPI endpoint +## Get the Task API's endpoint ```bash cortex get @@ -11,8 +11,8 @@ cortex get ```yaml POST : { - "timeout": , # duration in seconds since the submission of a job before it is terminated (optional) - "config": { # custom fields for this specific job (will override values in `config` specified in your api configuration) (optional) + "timeout": , # duration in seconds since the submission of a job before it is terminated (optional) + "config": { # arbitrary input for this specific job (optional) "string": } } @@ -30,6 +30,8 @@ RESPONSE: } ``` +The entire job specification is written to `/cortex/spec/job.json` in the API containers. + ## Get a job's status ```bash diff --git a/docs/workloads/task/task-apis.md b/docs/workloads/task/task-apis.md new file mode 100644 index 0000000000..f3d2a4abf0 --- /dev/null +++ b/docs/workloads/task/task-apis.md @@ -0,0 +1,3 @@ +# Task APIs + +Task APIs run multi-worker jobs on demand. diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 66bb92b2e3..1ba32c84c5 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -24,8 +24,8 @@ var ( CortexVersion = "master" // CORTEX_VERSION CortexVersionMinor = "master" // CORTEX_VERSION_MINOR - DefaultMaxQueueLength = int64(1024) - DefaultMaxConcurrency = int64(16) + DefaultMaxQueueLength = int64(100) + DefaultMaxConcurrency = int64(1) DefaultUserPodPortStr = "8080" DefaultUserPodPortInt32 = int32(8080) diff --git a/pkg/lib/aws/elb.go b/pkg/lib/aws/elb.go index f19caa8790..ccd8e624d7 100644 --- a/pkg/lib/aws/elb.go +++ b/pkg/lib/aws/elb.go @@ -42,7 +42,7 @@ func IsInstanceSupportedByNLB(instanceType string) (bool, error) { return true, nil } -// returns the the first load balancer which has all of the specified tags, or nil if no load balancers match +// returns the first load balancer which has all of the specified tags, or nil if no load balancers match func (c *Client) FindLoadBalancer(tags map[string]string) (*elbv2.LoadBalancer, error) { var loadBalancer *elbv2.LoadBalancer var fnErr error diff --git a/pkg/types/spec/validations.go b/pkg/types/spec/validations.go index 10f2f761c7..99ac629703 100644 --- a/pkg/types/spec/validations.go +++ b/pkg/types/spec/validations.go @@ -337,7 +337,8 @@ func httpGetProbeValidation() *cr.StructFieldValidation { { StructField: "Path", StringValidation: &cr.StringValidation{ - Required: true, + Required: false, + Default: "/", Validator: urls.ValidateEndpointAllowEmptyPath, }, }, From 0a1e2c21cff23cef83b6f6a6fab9db3450131f44 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Thu, 27 May 2021 18:02:13 +0300 Subject: [PATCH 27/82] CaaS - test APIs and E2E tests (#2191) --- Makefile | 16 +- build/build-image.sh | 14 +- build/images.sh | 8 - build/push-image.sh | 11 +- dev/registry.sh | 10 +- test/apis/README.md | 67 -- test/apis/async/iris-classifier/cortex.yaml | 11 - .../async/iris-classifier/expectations.yaml | 10 - test/apis/async/iris-classifier/handler.py | 26 - .../async/iris-classifier/requirements.txt | 2 - test/apis/async/iris-classifier/sample.json | 6 - test/apis/async/tensorflow/cortex.yaml | 7 - test/apis/async/tensorflow/handler.py | 11 - test/apis/async/tensorflow/sample.json | 6 - test/apis/async/text-generator/.dockerignore | 9 + .../apis/async/text-generator/cortex_cpu.yaml | 16 + .../apis/async/text-generator/cortex_gpu.yaml | 19 + .../expectations.yaml | 5 +- test/apis/async/text-generator/main.py | 42 + test/apis/async/text-generator/sample.json | 3 + .../text-generator-cpu.dockerfile | 23 + .../text-generator-gpu.dockerfile | 27 + .../image-classifier-alexnet/.dockerignore | 8 + .../image-classifier-alexnet/cortex_cpu.yaml | 20 + .../image-classifier-alexnet/cortex_gpu.yaml | 21 + .../image-classifier-alexnet-cpu.dockerfile | 25 + .../image-classifier-alexnet-gpu.dockerfile | 36 + .../batch/image-classifier-alexnet/main.py | 125 +++ .../image-classifier-alexnet/sample.json | 6 + .../batch/image-classifier-alexnet/submit.py | 41 + test/apis/batch/image-classifier/README.md | 568 ---------- test/apis/batch/image-classifier/cortex.yaml | 7 - test/apis/batch/image-classifier/handler.py | 79 -- .../batch/image-classifier/requirements.txt | 4 - test/apis/batch/image-classifier/sample.json | 3 - test/apis/batch/inferentia/cortex_inf.yaml | 16 - test/apis/batch/inferentia/dependencies.sh | 1 - test/apis/batch/inferentia/handler.py | 90 -- test/apis/batch/inferentia/requirements.txt | 5 - test/apis/batch/inferentia/sample.json | 3 - test/apis/batch/onnx/cortex.yaml | 9 - test/apis/batch/onnx/handler.py | 73 -- test/apis/batch/onnx/requirements.txt | 5 - test/apis/batch/onnx/sample.json | 3 - test/apis/batch/sum/.dockerignore | 10 + test/apis/batch/sum/cortex.yaml | 8 - test/apis/batch/sum/cortex_cpu.yaml | 22 + test/apis/batch/sum/handler.py | 24 - test/apis/batch/sum/main.py | 63 ++ test/apis/batch/sum/sample.json | 3 +- test/apis/batch/sum/submit.py | 41 + test/apis/batch/sum/sum-cpu.dockerfile | 22 + test/apis/batch/tensorflow/cortex.yaml | 9 - test/apis/batch/tensorflow/handler.py | 58 - test/apis/batch/tensorflow/requirements.txt | 1 - test/apis/batch/tensorflow/sample.json | 3 - .../grpc/iris-classifier-sklearn/README.md | 32 - .../grpc/iris-classifier-sklearn/cortex.yaml | 12 - .../iris-classifier-sklearn/expectations.yaml | 16 - .../grpc/iris-classifier-sklearn/handler.py | 26 - .../iris_classifier.proto | 18 - .../iris-classifier-sklearn/requirements.txt | 2 - .../test_proto/iris_classifier_pb2.py | 216 ---- .../test_proto/iris_classifier_pb2_grpc.py | 79 -- .../grpc/prime-number-generator/README.md | 25 - .../grpc/prime-number-generator/cortex.yaml | 9 - .../prime-number-generator/expectations.yaml | 17 - .../prime-number-generator/generator.proto | 15 - .../grpc/prime-number-generator/generator.py | 28 - .../test_proto/generator_pb2.py | 159 --- .../test_proto/generator_pb2_grpc.py | 79 -- test/apis/keras/document-denoiser/README.md | 44 - test/apis/keras/document-denoiser/cortex.yaml | 10 - .../keras/document-denoiser/dependencies.sh | 1 - test/apis/keras/document-denoiser/handler.py | 79 -- .../keras/document-denoiser/requirements.txt | 6 - test/apis/keras/document-denoiser/sample.json | 3 - .../keras/document-denoiser/trainer.ipynb | 617 ----------- .../python/mpg-estimator/cortex.yaml | 7 - .../python/mpg-estimator/handler.py | 24 - .../python/mpg-estimator/requirements.txt | 4 - .../python/mpg-estimator/sample.json | 7 - test/apis/live-reloading/tensorflow/README.md | 9 - .../onnx/multi-model-classifier/README.md | 75 -- .../onnx/multi-model-classifier/cortex.yaml | 20 - .../multi-model-classifier/dependencies.sh | 1 - .../onnx/multi-model-classifier/handler.py | 118 --- .../multi-model-classifier/requirements.txt | 4 - .../onnx/multi-model-classifier/sample.json | 3 - .../python/mpg-estimator/README.md | 73 -- .../python/mpg-estimator/cortex.yaml | 11 - .../python/mpg-estimator/handler.py | 25 - .../python/mpg-estimator/requirements.txt | 4 - .../python/mpg-estimator/sample.json | 7 - .../model-caching/python/translator/README.md | 123 --- .../python/translator/cluster.yaml | 17 - .../python/translator/cortex.yaml | 12 - .../python/translator/handler.py | 24 - .../python/translator/requirements.txt | 2 - .../python/translator/sample.json | 5 - .../python/translator/sample2.json | 5 - .../multi-model-classifier/README.md | 75 -- .../multi-model-classifier/cortex.yaml | 30 - .../multi-model-classifier/dependencies.sh | 1 - .../multi-model-classifier/handler.py | 61 -- .../multi-model-classifier/requirements.txt | 2 - .../multi-model-classifier/sample-image.json | 3 - .../multi-model-classifier/sample-iris.json | 8 - test/apis/onnx/iris-classifier/cortex.yaml | 7 - test/apis/onnx/iris-classifier/handler.py | 37 - .../onnx/iris-classifier/requirements.txt | 2 - test/apis/onnx/iris-classifier/sample.json | 6 - test/apis/onnx/iris-classifier/xgboost.ipynb | 231 ---- .../onnx/multi-model-classifier/README.md | 67 -- .../onnx/multi-model-classifier/cortex.yaml | 18 - .../multi-model-classifier/dependencies.sh | 2 - .../onnx/multi-model-classifier/handler.py | 118 --- .../multi-model-classifier/requirements.txt | 4 - .../onnx/multi-model-classifier/sample.json | 3 - test/apis/onnx/yolov5-youtube/README.md | 59 -- .../onnx/yolov5-youtube/conda-packages.txt | 3 - test/apis/onnx/yolov5-youtube/cortex.yaml | 12 - test/apis/onnx/yolov5-youtube/handler.py | 90 -- test/apis/onnx/yolov5-youtube/labels.json | 82 -- .../apis/onnx/yolov5-youtube/requirements.txt | 5 - test/apis/onnx/yolov5-youtube/sample.json | 3 - test/apis/onnx/yolov5-youtube/utils.py | 128 --- .../apis/pytorch/answer-generator/cortex.yaml | 9 - .../pytorch/answer-generator/generator.py | 42 - test/apis/pytorch/answer-generator/handler.py | 34 - .../pytorch/answer-generator/requirements.txt | 3 - .../apis/pytorch/answer-generator/sample.json | 3 - .../image-classifier-alexnet/cortex.yaml | 9 - .../image-classifier-alexnet/handler.py | 37 - .../image-classifier-alexnet/requirements.txt | 2 - .../image-classifier-alexnet/sample.json | 3 - .../image-classifier-resnet50/README.md | 57 - .../image-classifier-resnet50/cortex.yaml | 13 - .../image-classifier-resnet50/cortex_gpu.yaml | 14 - .../image-classifier-resnet50/cortex_inf.yaml | 14 - .../image-classifier-resnet50/dependencies.sh | 1 - .../expectations.yaml | 5 - .../generate_resnet50_models.ipynb | 119 --- .../image-classifier-resnet50/handler.py | 86 -- .../requirements.txt | 5 - .../image-classifier-resnet50/sample.json | 3 - test/apis/pytorch/iris-classifier/cortex.yaml | 7 - test/apis/pytorch/iris-classifier/deploy.py | 22 - .../pytorch/iris-classifier/expectations.yaml | 5 - test/apis/pytorch/iris-classifier/handler.py | 43 - test/apis/pytorch/iris-classifier/model.py | 58 - .../pytorch/iris-classifier/requirements.txt | 1 - test/apis/pytorch/iris-classifier/sample.json | 6 - .../pytorch/language-identifier/cortex.yaml | 5 - .../pytorch/language-identifier/handler.py | 16 - .../language-identifier/requirements.txt | 2 - .../pytorch/language-identifier/sample.json | 3 - .../multi-model-text-analyzer/README.md | 49 - .../multi-model-text-analyzer/cortex.yaml | 9 - .../multi-model-text-analyzer/handler.py | 23 - .../requirements.txt | 2 - .../sample-sentiment.json | 3 - .../sample-summarizer.json | 3 - .../pytorch/object-detector/coco_labels.txt | 91 -- test/apis/pytorch/object-detector/cortex.yaml | 9 - test/apis/pytorch/object-detector/handler.py | 47 - .../pytorch/object-detector/requirements.txt | 2 - test/apis/pytorch/object-detector/sample.json | 4 - .../pytorch/question-generator/cortex.yaml | 8 - .../question-generator/dependencies.sh | 2 - .../pytorch/question-generator/handler.py | 34 - .../question-generator/requirements.txt | 5 - .../pytorch/question-generator/sample.json | 4 - .../pytorch/reading-comprehender/cortex.yaml | 9 - .../pytorch/reading-comprehender/handler.py | 23 - .../reading-comprehender/requirements.txt | 1 - .../pytorch/reading-comprehender/sample.json | 4 - .../apis/pytorch/search-completer/cortex.yaml | 9 - test/apis/pytorch/search-completer/handler.py | 18 - .../pytorch/search-completer/requirements.txt | 5 - .../apis/pytorch/search-completer/sample.json | 3 - .../pytorch/sentiment-analyzer/cortex.yaml | 8 - .../pytorch/sentiment-analyzer/handler.py | 13 - .../sentiment-analyzer/requirements.txt | 2 - .../pytorch/sentiment-analyzer/sample.json | 3 - .../pytorch/server-side-batching/cortex.yaml | 11 - .../pytorch/server-side-batching/handler.py | 49 - .../pytorch/server-side-batching/model.py | 58 - .../server-side-batching/requirements.txt | 1 - .../pytorch/server-side-batching/sample.json | 6 - test/apis/pytorch/text-generator/cortex.yaml | 8 - test/apis/pytorch/text-generator/handler.py | 16 - .../pytorch/text-generator/requirements.txt | 2 - test/apis/pytorch/text-generator/sample.json | 3 - test/apis/pytorch/text-summarizer/README.md | 1 - test/apis/pytorch/text-summarizer/cortex.yaml | 9 - test/apis/pytorch/text-summarizer/handler.py | 16 - .../pytorch/text-summarizer/requirements.txt | 2 - test/apis/pytorch/text-summarizer/sample.json | 3 - test/apis/realtime/Dockerfile | 20 - test/apis/realtime/cortex.yaml | 15 - test/apis/realtime/hello-world/.dockerignore | 8 + .../apis/realtime/hello-world/cortex_cpu.yaml | 16 + .../hello-world/hello-world-cpu.dockerfile | 17 + test/apis/realtime/hello-world/main.py | 16 + .../hello-world}/sample.json | 0 .../.dockerignore | 3 +- .../image-classifier-resnet50/client.py | 69 ++ .../image-classifier-resnet50/cortex_cpu.yaml | 15 + .../image-classifier-resnet50/cortex_gpu.yaml | 16 + .../image-classifier-resnet50-cpu.dockerfile | 17 + .../image-classifier-resnet50-gpu.dockerfile | 17 + .../image-classifier-resnet50/sample.json | 1 + test/apis/realtime/main.py | 19 - .../realtime/prime-generator/.dockerignore | 8 + .../realtime/prime-generator/cortex_cpu.yaml | 16 + test/apis/realtime/prime-generator/main.py | 36 + .../prime-generator-cpu.dockerfile | 17 + .../apis/realtime/prime-generator/sample.json | 3 + test/apis/realtime/sleep/.dockerignore | 8 + test/apis/realtime/sleep/cortex_cpu.yaml | 16 + test/apis/realtime/sleep/main.py | 15 + test/apis/realtime/sleep/sample.json | 1 + test/apis/realtime/sleep/sleep-cpu.dockerfile | 17 + .../realtime/text-generator/.dockerignore | 8 + .../realtime/text-generator/cortex_cpu.yaml | 16 + .../realtime/text-generator/cortex_gpu.yaml | 19 + test/apis/realtime/text-generator/main.py | 42 + test/apis/realtime/text-generator/sample.json | 3 + .../text-generator-cpu.dockerfile | 23 + .../text-generator-gpu.dockerfile | 27 + test/apis/sklearn/iris-classifier/cortex.yaml | 11 - test/apis/sklearn/iris-classifier/handler.py | 25 - .../sklearn/iris-classifier/requirements.txt | 2 - test/apis/sklearn/iris-classifier/sample.json | 6 - test/apis/sklearn/iris-classifier/trainer.py | 23 - test/apis/sklearn/mpg-estimator/cortex.yaml | 7 - test/apis/sklearn/mpg-estimator/handler.py | 35 - .../sklearn/mpg-estimator/requirements.txt | 4 - test/apis/sklearn/mpg-estimator/sample.json | 7 - test/apis/sklearn/mpg-estimator/trainer.py | 23 - test/apis/sleep/cortex.yaml | 7 - test/apis/sleep/deploy.py | 19 - test/apis/sleep/handler.py | 10 - test/apis/spacy/entity-recognizer/cortex.yaml | 8 - test/apis/spacy/entity-recognizer/handler.py | 20 - .../spacy/entity-recognizer/requirements.txt | 1 - test/apis/spacy/entity-recognizer/sample.json | 3 - test/apis/task/cortex.yaml | 12 - .../.dockerignore | 3 +- .../iris-classifier-trainer/cortex_cpu.yaml | 12 + .../iris-classifier-trainer-cpu.dockerfile} | 10 +- .../apis/task/iris-classifier-trainer/main.py | 41 + .../task/iris-classifier-trainer/submit.py | 32 + test/apis/task/main.py | 21 - .../image-classifier-inception/cortex.yaml | 10 - .../cortex_server_side_batching.yaml | 14 - .../image-classifier-inception/handler.py | 19 - .../inception.ipynb | 198 ---- .../requirements.txt | 1 - .../image-classifier-inception/sample.json | 3 - .../image-classifier-resnet50/README.md | 88 -- .../image-classifier-resnet50/cortex.yaml | 17 - .../image-classifier-resnet50/cortex_gpu.yaml | 18 - .../cortex_gpu_server_side_batching.yaml | 21 - .../image-classifier-resnet50/cortex_inf.yaml | 20 - .../cortex_inf_server_side_batching.yaml | 23 - .../image-classifier-resnet50/dependencies.sh | 1 - .../generate_gpu_resnet50_model.ipynb | 129 --- .../generate_resnet50_models.ipynb | 176 ---- .../image-classifier-resnet50/handler.py | 61 -- .../requirements.txt | 2 - .../image-classifier-resnet50/sample.bin | Bin 8680 -> 0 bytes .../image-classifier-resnet50/sample.json | 3 - .../tensorflow/iris-classifier/cortex.yaml | 9 - .../tensorflow/iris-classifier/handler.py | 11 - .../tensorflow/iris-classifier/sample.json | 6 - .../iris-classifier/tensorflow.ipynb | 283 ----- .../tensorflow/license-plate-reader/README.md | 173 --- .../license-plate-reader/config.json | 8 - .../license-plate-reader/cortex_full.yaml | 34 - .../license-plate-reader/cortex_lite.yaml | 12 - .../license-plate-reader/dependencies.sh | 1 - .../license-plate-reader/handler_crnn.py | 42 - .../license-plate-reader/handler_lite.py | 113 -- .../license-plate-reader/handler_yolo.py | 44 - .../license-plate-reader/requirements.txt | 5 - .../license-plate-reader/sample_inference.py | 98 -- .../license-plate-reader/utils/__init__.py | 0 .../license-plate-reader/utils/bbox.py | 109 -- .../license-plate-reader/utils/colors.py | 97 -- .../license-plate-reader/utils/preprocess.py | 57 - .../license-plate-reader/utils/utils.py | 158 --- .../multi-model-classifier/README.md | 67 -- .../multi-model-classifier/cortex.yaml | 28 - .../multi-model-classifier/dependencies.sh | 1 - .../multi-model-classifier/handler.py | 60 -- .../multi-model-classifier/requirements.txt | 2 - .../multi-model-classifier/sample-image.json | 3 - .../multi-model-classifier/sample-iris.json | 8 - .../tensorflow/sentiment-analyzer/bert.ipynb | 993 ------------------ .../tensorflow/sentiment-analyzer/cortex.yaml | 10 - .../tensorflow/sentiment-analyzer/handler.py | 27 - .../sentiment-analyzer/requirements.txt | 5 - .../tensorflow/sentiment-analyzer/sample.json | 3 - .../tensorflow/sound-classifier/README.md | 15 - .../sound-classifier/class_names.csv | 522 --------- .../tensorflow/sound-classifier/cortex.yaml | 11 - .../tensorflow/sound-classifier/handler.py | 27 - .../sound-classifier/requirements.txt | 1 - .../tensorflow/sound-classifier/silence.wav | Bin 32058 -> 0 bytes .../tensorflow/text-generator/cortex.yaml | 10 - .../apis/tensorflow/text-generator/encoder.py | 116 -- .../tensorflow/text-generator/gpt-2.ipynb | 369 ------- .../apis/tensorflow/text-generator/handler.py | 17 - .../text-generator/requirements.txt | 2 - .../tensorflow/text-generator/sample.json | 3 - test/apis/traffic-splitter/README.md | 106 -- test/apis/traffic-splitter/cortex.yaml | 36 - test/apis/traffic-splitter/model.py | 58 - test/apis/traffic-splitter/onnx_handler.py | 37 - .../traffic-splitter/onnx_requirements.txt | 2 - test/apis/traffic-splitter/pytorch_handler.py | 43 - .../traffic-splitter/pytorch_requirements.txt | 1 - .../apis/traffic-splitter/request_recorder.py | 10 - test/apis/traffic-splitter/sample.json | 6 - .../trafficsplitter/hello-world/.dockerignore | 8 + .../hello-world/cortex_cpu.yaml | 67 ++ .../trafficsplitter/hello-world/sample.json | 1 + test/e2e/e2e/tests.py | 43 +- test/e2e/e2e/utils.py | 11 +- test/e2e/tests/aws/test_async.py | 23 +- test/e2e/tests/aws/test_autoscaling.py | 8 +- test/e2e/tests/aws/test_batch.py | 16 +- test/e2e/tests/aws/test_load.py | 6 +- test/e2e/tests/aws/test_long_running.py | 2 +- test/e2e/tests/aws/test_realtime.py | 61 +- test/e2e/tests/aws/test_task.py | 2 +- test/e2e/tests/conftest.py | 2 +- test/utils/build-and-push-images.sh | 60 ++ 340 files changed, 1391 insertions(+), 10544 deletions(-) delete mode 100644 test/apis/README.md delete mode 100644 test/apis/async/iris-classifier/cortex.yaml delete mode 100644 test/apis/async/iris-classifier/expectations.yaml delete mode 100644 test/apis/async/iris-classifier/handler.py delete mode 100644 test/apis/async/iris-classifier/requirements.txt delete mode 100644 test/apis/async/iris-classifier/sample.json delete mode 100644 test/apis/async/tensorflow/cortex.yaml delete mode 100644 test/apis/async/tensorflow/handler.py delete mode 100644 test/apis/async/tensorflow/sample.json create mode 100644 test/apis/async/text-generator/.dockerignore create mode 100644 test/apis/async/text-generator/cortex_cpu.yaml create mode 100644 test/apis/async/text-generator/cortex_gpu.yaml rename test/apis/async/{tensorflow => text-generator}/expectations.yaml (69%) create mode 100644 test/apis/async/text-generator/main.py create mode 100644 test/apis/async/text-generator/sample.json create mode 100644 test/apis/async/text-generator/text-generator-cpu.dockerfile create mode 100644 test/apis/async/text-generator/text-generator-gpu.dockerfile create mode 100644 test/apis/batch/image-classifier-alexnet/.dockerignore create mode 100644 test/apis/batch/image-classifier-alexnet/cortex_cpu.yaml create mode 100644 test/apis/batch/image-classifier-alexnet/cortex_gpu.yaml create mode 100644 test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-cpu.dockerfile create mode 100644 test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-gpu.dockerfile create mode 100644 test/apis/batch/image-classifier-alexnet/main.py create mode 100644 test/apis/batch/image-classifier-alexnet/sample.json create mode 100644 test/apis/batch/image-classifier-alexnet/submit.py delete mode 100644 test/apis/batch/image-classifier/README.md delete mode 100644 test/apis/batch/image-classifier/cortex.yaml delete mode 100644 test/apis/batch/image-classifier/handler.py delete mode 100644 test/apis/batch/image-classifier/requirements.txt delete mode 100644 test/apis/batch/image-classifier/sample.json delete mode 100644 test/apis/batch/inferentia/cortex_inf.yaml delete mode 100644 test/apis/batch/inferentia/dependencies.sh delete mode 100644 test/apis/batch/inferentia/handler.py delete mode 100644 test/apis/batch/inferentia/requirements.txt delete mode 100644 test/apis/batch/inferentia/sample.json delete mode 100644 test/apis/batch/onnx/cortex.yaml delete mode 100644 test/apis/batch/onnx/handler.py delete mode 100644 test/apis/batch/onnx/requirements.txt delete mode 100644 test/apis/batch/onnx/sample.json create mode 100644 test/apis/batch/sum/.dockerignore delete mode 100644 test/apis/batch/sum/cortex.yaml create mode 100644 test/apis/batch/sum/cortex_cpu.yaml delete mode 100644 test/apis/batch/sum/handler.py create mode 100644 test/apis/batch/sum/main.py create mode 100644 test/apis/batch/sum/submit.py create mode 100644 test/apis/batch/sum/sum-cpu.dockerfile delete mode 100644 test/apis/batch/tensorflow/cortex.yaml delete mode 100644 test/apis/batch/tensorflow/handler.py delete mode 100644 test/apis/batch/tensorflow/requirements.txt delete mode 100644 test/apis/batch/tensorflow/sample.json delete mode 100644 test/apis/grpc/iris-classifier-sklearn/README.md delete mode 100644 test/apis/grpc/iris-classifier-sklearn/cortex.yaml delete mode 100644 test/apis/grpc/iris-classifier-sklearn/expectations.yaml delete mode 100644 test/apis/grpc/iris-classifier-sklearn/handler.py delete mode 100644 test/apis/grpc/iris-classifier-sklearn/iris_classifier.proto delete mode 100644 test/apis/grpc/iris-classifier-sklearn/requirements.txt delete mode 100644 test/apis/grpc/iris-classifier-sklearn/test_proto/iris_classifier_pb2.py delete mode 100644 test/apis/grpc/iris-classifier-sklearn/test_proto/iris_classifier_pb2_grpc.py delete mode 100644 test/apis/grpc/prime-number-generator/README.md delete mode 100644 test/apis/grpc/prime-number-generator/cortex.yaml delete mode 100644 test/apis/grpc/prime-number-generator/expectations.yaml delete mode 100644 test/apis/grpc/prime-number-generator/generator.proto delete mode 100644 test/apis/grpc/prime-number-generator/generator.py delete mode 100644 test/apis/grpc/prime-number-generator/test_proto/generator_pb2.py delete mode 100644 test/apis/grpc/prime-number-generator/test_proto/generator_pb2_grpc.py delete mode 100644 test/apis/keras/document-denoiser/README.md delete mode 100644 test/apis/keras/document-denoiser/cortex.yaml delete mode 100644 test/apis/keras/document-denoiser/dependencies.sh delete mode 100644 test/apis/keras/document-denoiser/handler.py delete mode 100644 test/apis/keras/document-denoiser/requirements.txt delete mode 100644 test/apis/keras/document-denoiser/sample.json delete mode 100644 test/apis/keras/document-denoiser/trainer.ipynb delete mode 100644 test/apis/live-reloading/python/mpg-estimator/cortex.yaml delete mode 100644 test/apis/live-reloading/python/mpg-estimator/handler.py delete mode 100644 test/apis/live-reloading/python/mpg-estimator/requirements.txt delete mode 100644 test/apis/live-reloading/python/mpg-estimator/sample.json delete mode 100644 test/apis/live-reloading/tensorflow/README.md delete mode 100644 test/apis/model-caching/onnx/multi-model-classifier/README.md delete mode 100644 test/apis/model-caching/onnx/multi-model-classifier/cortex.yaml delete mode 100644 test/apis/model-caching/onnx/multi-model-classifier/dependencies.sh delete mode 100644 test/apis/model-caching/onnx/multi-model-classifier/handler.py delete mode 100644 test/apis/model-caching/onnx/multi-model-classifier/requirements.txt delete mode 100644 test/apis/model-caching/onnx/multi-model-classifier/sample.json delete mode 100644 test/apis/model-caching/python/mpg-estimator/README.md delete mode 100644 test/apis/model-caching/python/mpg-estimator/cortex.yaml delete mode 100644 test/apis/model-caching/python/mpg-estimator/handler.py delete mode 100644 test/apis/model-caching/python/mpg-estimator/requirements.txt delete mode 100644 test/apis/model-caching/python/mpg-estimator/sample.json delete mode 100644 test/apis/model-caching/python/translator/README.md delete mode 100644 test/apis/model-caching/python/translator/cluster.yaml delete mode 100644 test/apis/model-caching/python/translator/cortex.yaml delete mode 100644 test/apis/model-caching/python/translator/handler.py delete mode 100644 test/apis/model-caching/python/translator/requirements.txt delete mode 100644 test/apis/model-caching/python/translator/sample.json delete mode 100644 test/apis/model-caching/python/translator/sample2.json delete mode 100644 test/apis/model-caching/tensorflow/multi-model-classifier/README.md delete mode 100644 test/apis/model-caching/tensorflow/multi-model-classifier/cortex.yaml delete mode 100644 test/apis/model-caching/tensorflow/multi-model-classifier/dependencies.sh delete mode 100644 test/apis/model-caching/tensorflow/multi-model-classifier/handler.py delete mode 100644 test/apis/model-caching/tensorflow/multi-model-classifier/requirements.txt delete mode 100644 test/apis/model-caching/tensorflow/multi-model-classifier/sample-image.json delete mode 100644 test/apis/model-caching/tensorflow/multi-model-classifier/sample-iris.json delete mode 100644 test/apis/onnx/iris-classifier/cortex.yaml delete mode 100644 test/apis/onnx/iris-classifier/handler.py delete mode 100644 test/apis/onnx/iris-classifier/requirements.txt delete mode 100644 test/apis/onnx/iris-classifier/sample.json delete mode 100644 test/apis/onnx/iris-classifier/xgboost.ipynb delete mode 100644 test/apis/onnx/multi-model-classifier/README.md delete mode 100644 test/apis/onnx/multi-model-classifier/cortex.yaml delete mode 100644 test/apis/onnx/multi-model-classifier/dependencies.sh delete mode 100644 test/apis/onnx/multi-model-classifier/handler.py delete mode 100644 test/apis/onnx/multi-model-classifier/requirements.txt delete mode 100644 test/apis/onnx/multi-model-classifier/sample.json delete mode 100644 test/apis/onnx/yolov5-youtube/README.md delete mode 100644 test/apis/onnx/yolov5-youtube/conda-packages.txt delete mode 100644 test/apis/onnx/yolov5-youtube/cortex.yaml delete mode 100644 test/apis/onnx/yolov5-youtube/handler.py delete mode 100644 test/apis/onnx/yolov5-youtube/labels.json delete mode 100644 test/apis/onnx/yolov5-youtube/requirements.txt delete mode 100644 test/apis/onnx/yolov5-youtube/sample.json delete mode 100644 test/apis/onnx/yolov5-youtube/utils.py delete mode 100644 test/apis/pytorch/answer-generator/cortex.yaml delete mode 100644 test/apis/pytorch/answer-generator/generator.py delete mode 100644 test/apis/pytorch/answer-generator/handler.py delete mode 100644 test/apis/pytorch/answer-generator/requirements.txt delete mode 100644 test/apis/pytorch/answer-generator/sample.json delete mode 100644 test/apis/pytorch/image-classifier-alexnet/cortex.yaml delete mode 100644 test/apis/pytorch/image-classifier-alexnet/handler.py delete mode 100644 test/apis/pytorch/image-classifier-alexnet/requirements.txt delete mode 100644 test/apis/pytorch/image-classifier-alexnet/sample.json delete mode 100644 test/apis/pytorch/image-classifier-resnet50/README.md delete mode 100644 test/apis/pytorch/image-classifier-resnet50/cortex.yaml delete mode 100644 test/apis/pytorch/image-classifier-resnet50/cortex_gpu.yaml delete mode 100644 test/apis/pytorch/image-classifier-resnet50/cortex_inf.yaml delete mode 100644 test/apis/pytorch/image-classifier-resnet50/dependencies.sh delete mode 100644 test/apis/pytorch/image-classifier-resnet50/expectations.yaml delete mode 100644 test/apis/pytorch/image-classifier-resnet50/generate_resnet50_models.ipynb delete mode 100644 test/apis/pytorch/image-classifier-resnet50/handler.py delete mode 100644 test/apis/pytorch/image-classifier-resnet50/requirements.txt delete mode 100644 test/apis/pytorch/image-classifier-resnet50/sample.json delete mode 100644 test/apis/pytorch/iris-classifier/cortex.yaml delete mode 100644 test/apis/pytorch/iris-classifier/deploy.py delete mode 100644 test/apis/pytorch/iris-classifier/expectations.yaml delete mode 100644 test/apis/pytorch/iris-classifier/handler.py delete mode 100644 test/apis/pytorch/iris-classifier/model.py delete mode 100644 test/apis/pytorch/iris-classifier/requirements.txt delete mode 100644 test/apis/pytorch/iris-classifier/sample.json delete mode 100644 test/apis/pytorch/language-identifier/cortex.yaml delete mode 100644 test/apis/pytorch/language-identifier/handler.py delete mode 100644 test/apis/pytorch/language-identifier/requirements.txt delete mode 100644 test/apis/pytorch/language-identifier/sample.json delete mode 100644 test/apis/pytorch/multi-model-text-analyzer/README.md delete mode 100644 test/apis/pytorch/multi-model-text-analyzer/cortex.yaml delete mode 100644 test/apis/pytorch/multi-model-text-analyzer/handler.py delete mode 100644 test/apis/pytorch/multi-model-text-analyzer/requirements.txt delete mode 100644 test/apis/pytorch/multi-model-text-analyzer/sample-sentiment.json delete mode 100644 test/apis/pytorch/multi-model-text-analyzer/sample-summarizer.json delete mode 100644 test/apis/pytorch/object-detector/coco_labels.txt delete mode 100644 test/apis/pytorch/object-detector/cortex.yaml delete mode 100644 test/apis/pytorch/object-detector/handler.py delete mode 100644 test/apis/pytorch/object-detector/requirements.txt delete mode 100644 test/apis/pytorch/object-detector/sample.json delete mode 100644 test/apis/pytorch/question-generator/cortex.yaml delete mode 100644 test/apis/pytorch/question-generator/dependencies.sh delete mode 100644 test/apis/pytorch/question-generator/handler.py delete mode 100644 test/apis/pytorch/question-generator/requirements.txt delete mode 100644 test/apis/pytorch/question-generator/sample.json delete mode 100644 test/apis/pytorch/reading-comprehender/cortex.yaml delete mode 100644 test/apis/pytorch/reading-comprehender/handler.py delete mode 100644 test/apis/pytorch/reading-comprehender/requirements.txt delete mode 100644 test/apis/pytorch/reading-comprehender/sample.json delete mode 100644 test/apis/pytorch/search-completer/cortex.yaml delete mode 100644 test/apis/pytorch/search-completer/handler.py delete mode 100644 test/apis/pytorch/search-completer/requirements.txt delete mode 100644 test/apis/pytorch/search-completer/sample.json delete mode 100644 test/apis/pytorch/sentiment-analyzer/cortex.yaml delete mode 100644 test/apis/pytorch/sentiment-analyzer/handler.py delete mode 100644 test/apis/pytorch/sentiment-analyzer/requirements.txt delete mode 100644 test/apis/pytorch/sentiment-analyzer/sample.json delete mode 100644 test/apis/pytorch/server-side-batching/cortex.yaml delete mode 100644 test/apis/pytorch/server-side-batching/handler.py delete mode 100644 test/apis/pytorch/server-side-batching/model.py delete mode 100644 test/apis/pytorch/server-side-batching/requirements.txt delete mode 100644 test/apis/pytorch/server-side-batching/sample.json delete mode 100644 test/apis/pytorch/text-generator/cortex.yaml delete mode 100644 test/apis/pytorch/text-generator/handler.py delete mode 100644 test/apis/pytorch/text-generator/requirements.txt delete mode 100644 test/apis/pytorch/text-generator/sample.json delete mode 100644 test/apis/pytorch/text-summarizer/README.md delete mode 100644 test/apis/pytorch/text-summarizer/cortex.yaml delete mode 100644 test/apis/pytorch/text-summarizer/handler.py delete mode 100644 test/apis/pytorch/text-summarizer/requirements.txt delete mode 100644 test/apis/pytorch/text-summarizer/sample.json delete mode 100644 test/apis/realtime/Dockerfile delete mode 100644 test/apis/realtime/cortex.yaml create mode 100644 test/apis/realtime/hello-world/.dockerignore create mode 100644 test/apis/realtime/hello-world/cortex_cpu.yaml create mode 100644 test/apis/realtime/hello-world/hello-world-cpu.dockerfile create mode 100644 test/apis/realtime/hello-world/main.py rename test/apis/{sleep => realtime/hello-world}/sample.json (100%) rename test/apis/realtime/{ => image-classifier-resnet50}/.dockerignore (70%) create mode 100644 test/apis/realtime/image-classifier-resnet50/client.py create mode 100644 test/apis/realtime/image-classifier-resnet50/cortex_cpu.yaml create mode 100644 test/apis/realtime/image-classifier-resnet50/cortex_gpu.yaml create mode 100644 test/apis/realtime/image-classifier-resnet50/image-classifier-resnet50-cpu.dockerfile create mode 100644 test/apis/realtime/image-classifier-resnet50/image-classifier-resnet50-gpu.dockerfile create mode 100644 test/apis/realtime/image-classifier-resnet50/sample.json delete mode 100644 test/apis/realtime/main.py create mode 100644 test/apis/realtime/prime-generator/.dockerignore create mode 100644 test/apis/realtime/prime-generator/cortex_cpu.yaml create mode 100644 test/apis/realtime/prime-generator/main.py create mode 100644 test/apis/realtime/prime-generator/prime-generator-cpu.dockerfile create mode 100644 test/apis/realtime/prime-generator/sample.json create mode 100644 test/apis/realtime/sleep/.dockerignore create mode 100644 test/apis/realtime/sleep/cortex_cpu.yaml create mode 100644 test/apis/realtime/sleep/main.py create mode 100644 test/apis/realtime/sleep/sample.json create mode 100644 test/apis/realtime/sleep/sleep-cpu.dockerfile create mode 100644 test/apis/realtime/text-generator/.dockerignore create mode 100644 test/apis/realtime/text-generator/cortex_cpu.yaml create mode 100644 test/apis/realtime/text-generator/cortex_gpu.yaml create mode 100644 test/apis/realtime/text-generator/main.py create mode 100644 test/apis/realtime/text-generator/sample.json create mode 100644 test/apis/realtime/text-generator/text-generator-cpu.dockerfile create mode 100644 test/apis/realtime/text-generator/text-generator-gpu.dockerfile delete mode 100644 test/apis/sklearn/iris-classifier/cortex.yaml delete mode 100644 test/apis/sklearn/iris-classifier/handler.py delete mode 100644 test/apis/sklearn/iris-classifier/requirements.txt delete mode 100644 test/apis/sklearn/iris-classifier/sample.json delete mode 100644 test/apis/sklearn/iris-classifier/trainer.py delete mode 100644 test/apis/sklearn/mpg-estimator/cortex.yaml delete mode 100644 test/apis/sklearn/mpg-estimator/handler.py delete mode 100644 test/apis/sklearn/mpg-estimator/requirements.txt delete mode 100644 test/apis/sklearn/mpg-estimator/sample.json delete mode 100644 test/apis/sklearn/mpg-estimator/trainer.py delete mode 100644 test/apis/sleep/cortex.yaml delete mode 100644 test/apis/sleep/deploy.py delete mode 100644 test/apis/sleep/handler.py delete mode 100644 test/apis/spacy/entity-recognizer/cortex.yaml delete mode 100644 test/apis/spacy/entity-recognizer/handler.py delete mode 100644 test/apis/spacy/entity-recognizer/requirements.txt delete mode 100644 test/apis/spacy/entity-recognizer/sample.json delete mode 100644 test/apis/task/cortex.yaml rename test/apis/task/{ => iris-classifier-trainer}/.dockerignore (70%) create mode 100644 test/apis/task/iris-classifier-trainer/cortex_cpu.yaml rename test/apis/task/{Dockerfile => iris-classifier-trainer/iris-classifier-trainer-cpu.dockerfile} (78%) create mode 100644 test/apis/task/iris-classifier-trainer/main.py create mode 100644 test/apis/task/iris-classifier-trainer/submit.py delete mode 100644 test/apis/task/main.py delete mode 100644 test/apis/tensorflow/image-classifier-inception/cortex.yaml delete mode 100644 test/apis/tensorflow/image-classifier-inception/cortex_server_side_batching.yaml delete mode 100644 test/apis/tensorflow/image-classifier-inception/handler.py delete mode 100644 test/apis/tensorflow/image-classifier-inception/inception.ipynb delete mode 100644 test/apis/tensorflow/image-classifier-inception/requirements.txt delete mode 100644 test/apis/tensorflow/image-classifier-inception/sample.json delete mode 100644 test/apis/tensorflow/image-classifier-resnet50/README.md delete mode 100644 test/apis/tensorflow/image-classifier-resnet50/cortex.yaml delete mode 100644 test/apis/tensorflow/image-classifier-resnet50/cortex_gpu.yaml delete mode 100644 test/apis/tensorflow/image-classifier-resnet50/cortex_gpu_server_side_batching.yaml delete mode 100644 test/apis/tensorflow/image-classifier-resnet50/cortex_inf.yaml delete mode 100644 test/apis/tensorflow/image-classifier-resnet50/cortex_inf_server_side_batching.yaml delete mode 100644 test/apis/tensorflow/image-classifier-resnet50/dependencies.sh delete mode 100644 test/apis/tensorflow/image-classifier-resnet50/generate_gpu_resnet50_model.ipynb delete mode 100644 test/apis/tensorflow/image-classifier-resnet50/generate_resnet50_models.ipynb delete mode 100644 test/apis/tensorflow/image-classifier-resnet50/handler.py delete mode 100644 test/apis/tensorflow/image-classifier-resnet50/requirements.txt delete mode 100644 test/apis/tensorflow/image-classifier-resnet50/sample.bin delete mode 100644 test/apis/tensorflow/image-classifier-resnet50/sample.json delete mode 100644 test/apis/tensorflow/iris-classifier/cortex.yaml delete mode 100644 test/apis/tensorflow/iris-classifier/handler.py delete mode 100644 test/apis/tensorflow/iris-classifier/sample.json delete mode 100644 test/apis/tensorflow/iris-classifier/tensorflow.ipynb delete mode 100644 test/apis/tensorflow/license-plate-reader/README.md delete mode 100644 test/apis/tensorflow/license-plate-reader/config.json delete mode 100644 test/apis/tensorflow/license-plate-reader/cortex_full.yaml delete mode 100644 test/apis/tensorflow/license-plate-reader/cortex_lite.yaml delete mode 100644 test/apis/tensorflow/license-plate-reader/dependencies.sh delete mode 100644 test/apis/tensorflow/license-plate-reader/handler_crnn.py delete mode 100644 test/apis/tensorflow/license-plate-reader/handler_lite.py delete mode 100644 test/apis/tensorflow/license-plate-reader/handler_yolo.py delete mode 100644 test/apis/tensorflow/license-plate-reader/requirements.txt delete mode 100644 test/apis/tensorflow/license-plate-reader/sample_inference.py delete mode 100644 test/apis/tensorflow/license-plate-reader/utils/__init__.py delete mode 100644 test/apis/tensorflow/license-plate-reader/utils/bbox.py delete mode 100644 test/apis/tensorflow/license-plate-reader/utils/colors.py delete mode 100644 test/apis/tensorflow/license-plate-reader/utils/preprocess.py delete mode 100644 test/apis/tensorflow/license-plate-reader/utils/utils.py delete mode 100644 test/apis/tensorflow/multi-model-classifier/README.md delete mode 100644 test/apis/tensorflow/multi-model-classifier/cortex.yaml delete mode 100644 test/apis/tensorflow/multi-model-classifier/dependencies.sh delete mode 100644 test/apis/tensorflow/multi-model-classifier/handler.py delete mode 100644 test/apis/tensorflow/multi-model-classifier/requirements.txt delete mode 100644 test/apis/tensorflow/multi-model-classifier/sample-image.json delete mode 100644 test/apis/tensorflow/multi-model-classifier/sample-iris.json delete mode 100644 test/apis/tensorflow/sentiment-analyzer/bert.ipynb delete mode 100644 test/apis/tensorflow/sentiment-analyzer/cortex.yaml delete mode 100644 test/apis/tensorflow/sentiment-analyzer/handler.py delete mode 100644 test/apis/tensorflow/sentiment-analyzer/requirements.txt delete mode 100644 test/apis/tensorflow/sentiment-analyzer/sample.json delete mode 100644 test/apis/tensorflow/sound-classifier/README.md delete mode 100644 test/apis/tensorflow/sound-classifier/class_names.csv delete mode 100644 test/apis/tensorflow/sound-classifier/cortex.yaml delete mode 100644 test/apis/tensorflow/sound-classifier/handler.py delete mode 100644 test/apis/tensorflow/sound-classifier/requirements.txt delete mode 100644 test/apis/tensorflow/sound-classifier/silence.wav delete mode 100644 test/apis/tensorflow/text-generator/cortex.yaml delete mode 100644 test/apis/tensorflow/text-generator/encoder.py delete mode 100644 test/apis/tensorflow/text-generator/gpt-2.ipynb delete mode 100644 test/apis/tensorflow/text-generator/handler.py delete mode 100644 test/apis/tensorflow/text-generator/requirements.txt delete mode 100644 test/apis/tensorflow/text-generator/sample.json delete mode 100644 test/apis/traffic-splitter/README.md delete mode 100644 test/apis/traffic-splitter/cortex.yaml delete mode 100644 test/apis/traffic-splitter/model.py delete mode 100644 test/apis/traffic-splitter/onnx_handler.py delete mode 100644 test/apis/traffic-splitter/onnx_requirements.txt delete mode 100644 test/apis/traffic-splitter/pytorch_handler.py delete mode 100644 test/apis/traffic-splitter/pytorch_requirements.txt delete mode 100644 test/apis/traffic-splitter/request_recorder.py delete mode 100644 test/apis/traffic-splitter/sample.json create mode 100644 test/apis/trafficsplitter/hello-world/.dockerignore create mode 100644 test/apis/trafficsplitter/hello-world/cortex_cpu.yaml create mode 100644 test/apis/trafficsplitter/hello-world/sample.json create mode 100755 test/utils/build-and-push-images.sh diff --git a/Makefile b/Makefile index 3d82ad897c..e8d59f851e 100644 --- a/Makefile +++ b/Makefile @@ -124,8 +124,7 @@ async-gateway-update: @./dev/registry.sh update-single async-gateway @kubectl delete pods -l cortex.dev/async=gateway --namespace=default -# Docker images - +# docker images images-all: @./dev/registry.sh update all images-all-skip-push: @@ -136,15 +135,8 @@ images-dev: images-dev-skip-push: @./dev/registry.sh update dev --skip-push -images-api: - @./dev/registry.sh update api -images-api-skip-push: - @./dev/registry.sh update api --skip-push - images-manager-skip-push: @./dev/registry.sh update-single manager --skip-push -images-iris: - @./dev/registry.sh update-single python-handler-cpu registry-create: @./dev/registry.sh create @@ -179,6 +171,12 @@ test-go: test-python: @./build/test.sh python + +# build test api images +# the DOCKER_PASSWORD and DOCKER_USERNAME vars to the quay repo are required +build-and-push-test-images: + @./test/utils/build-and-push-images.sh quay.io + # run e2e tests on an existing cluster # read test/e2e/README.md for instructions first test-e2e: diff --git a/build/build-image.sh b/build/build-image.sh index e0cbd0ad14..42fcee6933 100755 --- a/build/build-image.sh +++ b/build/build-image.sh @@ -26,16 +26,4 @@ image=$1 if [ "$image" == "inferentia" ]; then aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 790709498068.dkr.ecr.us-west-2.amazonaws.com fi - -build_args="" - -if [ "${image}" == "python-handler-gpu" ]; then - cuda=("10.0" "10.1" "10.1" "10.2" "10.2" "11.0" "11.1") - cudnn=("7" "7" "8" "7" "8" "8" "8") - for i in ${!cudnn[@]}; do - build_args="${build_args} --build-arg CUDA_VERSION=${cuda[$i]} --build-arg CUDNN=${cudnn[$i]}" - docker build "$ROOT" -f $ROOT/images/$image/Dockerfile $build_args -t quay.io/cortexlabs/${image}:${CORTEX_VERSION}-cuda${cuda[$i]}-cudnn${cudnn[$i]} -t cortexlabs/${image}:${CORTEX_VERSION}-cuda${cuda[$i]}-cudnn${cudnn[$i]} - done -else - docker build "$ROOT" -f $ROOT/images/$image/Dockerfile $build_args -t quay.io/cortexlabs/${image}:${CORTEX_VERSION} -t cortexlabs/${image}:${CORTEX_VERSION} -fi +docker build "$ROOT" -f $ROOT/images/$image/Dockerfile -t quay.io/cortexlabs/${image}:${CORTEX_VERSION} -t cortexlabs/${image}:${CORTEX_VERSION} diff --git a/build/images.sh b/build/images.sh index dcd3b8f49c..b3c2d57551 100644 --- a/build/images.sh +++ b/build/images.sh @@ -19,13 +19,6 @@ set -euo pipefail -api_images=( - "python-handler-cpu" - "python-handler-gpu" - "tensorflow-handler" - "python-handler-inf" -) - dev_images=( "manager" "proxy" @@ -61,7 +54,6 @@ non_dev_images=( ) all_images=( - "${api_images[@]}" "${dev_images[@]}" "${non_dev_images[@]}" ) diff --git a/build/push-image.sh b/build/push-image.sh index 2b867a04f4..b87ccec98f 100755 --- a/build/push-image.sh +++ b/build/push-image.sh @@ -23,13 +23,4 @@ host=$1 image=$2 echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - -if [ "$image" == "python-handler-gpu" ]; then - cuda=("10.0" "10.1" "10.1" "10.2" "10.2" "11.0" "11.1") - cudnn=("7" "7" "8" "7" "8" "8" "8") - for i in ${!cudnn[@]}; do - docker push $host/cortexlabs/${image}:${CORTEX_VERSION}-cuda${cuda[$i]}-cudnn${cudnn[$i]} - done -else - docker push $host/cortexlabs/${image}:${CORTEX_VERSION} -fi +docker push $host/cortexlabs/${image}:${CORTEX_VERSION} diff --git a/dev/registry.sh b/dev/registry.sh index 3b05e85bb3..78baeb46cf 100755 --- a/dev/registry.sh +++ b/dev/registry.sh @@ -108,15 +108,13 @@ function build() { local tag=$2 local dir="${ROOT}/images/${image}" - build_args="" - tag_args="" if [ -n "$AWS_ACCOUNT_ID" ] && [ -n "$AWS_REGION" ]; then tag_args+=" -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/cortexlabs/$image:$tag" fi blue_echo "Building $image:$tag..." - docker build $ROOT -f $dir/Dockerfile -t cortexlabs/$image:$tag $tag_args $build_args + docker build $ROOT -f $dir/Dockerfile -t cortexlabs/$image:$tag $tag_args green_echo "Built $image:$tag\n" } @@ -150,10 +148,6 @@ function build_and_push() { set -euo pipefail # necessary since this is called in a new shell by parallel tag=$CORTEX_VERSION - if [ "${image}" == "python-handler-gpu" ]; then - tag="${CORTEX_VERSION}-cuda10.2-cudnn8" - fi - build $image $tag push $image $tag } @@ -240,8 +234,6 @@ elif [ "$cmd" = "update" ]; then images_to_build+=( "${dev_images[@]}" ) fi - images_to_build+=( "${api_images[@]}" ) - if [[ " ${images_to_build[@]} " =~ " operator " ]]; then cache_builder operator fi diff --git a/test/apis/README.md b/test/apis/README.md deleted file mode 100644 index 1eb711f57d..0000000000 --- a/test/apis/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# Examples - -## TensorFlow - -- [Iris classification](tensorflow/iris-classifier): deploy a model to classify iris flowers. - -- [Text generation](tensorflow/text-generator): deploy OpenAI's GPT-2 to generate text. - -- [Sentiment analysis](tensorflow/sentiment-analyzer): deploy a BERT model for sentiment analysis. - -- [Image classification](tensorflow/image-classifier-inception): deploy an Inception model to classify images. - -- [Image classification](tensorflow/image-classifier-resnet50): deploy a ResNet50 model to classify images. - -- [License plate reader](tensorflow/license-plate-reader): deploy a YOLOv3 model (and others) to identify license plates in real time. - -- [Multi-model classification](tensorflow/multi-model-classifier): deploy 3 models (ResNet50, Iris, Inception) in a single API. - -## Keras - -- [Denoisify text documents](keras/document-denoiser): deploy an Autoencoder model to clean text document images of noise. - -## PyTorch - -- [Iris classification](pytorch/iris-classifier): deploy a model to classify iris flowers. - -- [Text generation](pytorch/text-generator): deploy Hugging Face's GPT-2 model to generate text. - -- [Sentiment analysis](pytorch/sentiment-analyzer): deploy a Hugging Face transformers model for sentiment analysis. - -- [Search completion](pytorch/search-completer): deploy a Facebook's RoBERTa model to complete search terms. - -- [Answer generation](pytorch/answer-generator): deploy Microsoft's DialoGPT model to answer questions. - -- [Text summarization](pytorch/text-summarizer): deploy a BART model (from Hugging Face's transformers library) to summarize text. - -- [Reading comprehension](pytorch/reading-comprehender): deploy an AllenNLP model for reading comprehension. - -- [Language identification](pytorch/language-identifier): deploy a fastText model to identify languages. - -- [Multi-model text analysis](pytorch/multi-model-text-analyzer): deploy 2 models (Sentiment and Summarization analyzers) in a single API. - -- [Image classification](pytorch/image-classifier-alexnet): deploy an AlexNet model from TorchVision to classify images. - -- [Image classification](pytorch/image-classifier-resnet50): deploy a ResNet50 model from TorchVision to classify images. - -- [Object detection](pytorch/object-detector): deploy a Faster R-CNN model from TorchVision to detect objects in images. - -- [Question generator](pytorch/question-generator): deploy a transformers model to generate questions given text and the correct answer. - -## ONNX - -- [Iris classification](onnx/iris-classifier): deploy an XGBoost model (exported in ONNX) to classify iris flowers. - -- [YOLOv5 YouTube detection](onnx/yolov5-youtube): deploy a YOLOv5 model trained on COCO val2017 dataset. - -- [Multi-model classification](onnx/multi-model-classifier): deploy 3 models (ResNet50, MobileNet, ShuffleNet) in a single API. - -## scikit-learn - -- [Iris classification](sklearn/iris-classifier): deploy a model to classify iris flowers. - -- [MPG estimation](sklearn/mpg-estimator): deploy a linear regression model to estimate MPG. - -## spacy - -- [Entity recognizer](spacy/entity-recognizer): deploy a spacy model for named entity recognition. diff --git a/test/apis/async/iris-classifier/cortex.yaml b/test/apis/async/iris-classifier/cortex.yaml deleted file mode 100644 index 023f680ce5..0000000000 --- a/test/apis/async/iris-classifier/cortex.yaml +++ /dev/null @@ -1,11 +0,0 @@ -- name: async-iris-classifier - kind: AsyncAPI - handler: - type: python - path: handler.py - config: - bucket: cortex-examples - key: sklearn/iris-classifier/model.pkl - compute: - cpu: 0.2 - mem: 200M diff --git a/test/apis/async/iris-classifier/expectations.yaml b/test/apis/async/iris-classifier/expectations.yaml deleted file mode 100644 index bef65e0e0d..0000000000 --- a/test/apis/async/iris-classifier/expectations.yaml +++ /dev/null @@ -1,10 +0,0 @@ -response: - content_type: "json" - json_schema: - type: "object" - properties: - label: - type: "string" - const: "setosa" - required: - - "label" diff --git a/test/apis/async/iris-classifier/handler.py b/test/apis/async/iris-classifier/handler.py deleted file mode 100644 index 15529b144f..0000000000 --- a/test/apis/async/iris-classifier/handler.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -import pickle - -import boto3 -from botocore import UNSIGNED -from botocore.client import Config - -labels = ["setosa", "versicolor", "virginica"] - - -class Handler: - def __init__(self, config): - s3 = boto3.client("s3") - s3.download_file(config["bucket"], config["key"], "/tmp/model.pkl") - self.model = pickle.load(open("/tmp/model.pkl", "rb")) - - def handle_async(self, payload): - measurements = [ - payload["sepal_length"], - payload["sepal_width"], - payload["petal_length"], - payload["petal_width"], - ] - - label_id = self.model.predict([measurements])[0] - return {"label": labels[label_id]} diff --git a/test/apis/async/iris-classifier/requirements.txt b/test/apis/async/iris-classifier/requirements.txt deleted file mode 100644 index bbc213cf3e..0000000000 --- a/test/apis/async/iris-classifier/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -boto3 -scikit-learn==0.21.3 diff --git a/test/apis/async/iris-classifier/sample.json b/test/apis/async/iris-classifier/sample.json deleted file mode 100644 index 1e1eda2251..0000000000 --- a/test/apis/async/iris-classifier/sample.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "sepal_length": 5.2, - "sepal_width": 3.6, - "petal_length": 1.5, - "petal_width": 0.3 -} diff --git a/test/apis/async/tensorflow/cortex.yaml b/test/apis/async/tensorflow/cortex.yaml deleted file mode 100644 index f340f99741..0000000000 --- a/test/apis/async/tensorflow/cortex.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- name: async-iris-classifier - kind: AsyncAPI - handler: - type: tensorflow - path: handler.py - models: - path: s3://cortex-examples/tensorflow/iris-classifier/nn/ diff --git a/test/apis/async/tensorflow/handler.py b/test/apis/async/tensorflow/handler.py deleted file mode 100644 index 30900de58e..0000000000 --- a/test/apis/async/tensorflow/handler.py +++ /dev/null @@ -1,11 +0,0 @@ -labels = ["setosa", "versicolor", "virginica"] - - -class Handler: - def __init__(self, tensorflow_client, config): - self.client = tensorflow_client - - def handle_async(self, payload): - prediction = self.client.predict(payload) - predicted_class_id = int(prediction["class_ids"][0]) - return {"label": labels[predicted_class_id]} diff --git a/test/apis/async/tensorflow/sample.json b/test/apis/async/tensorflow/sample.json deleted file mode 100644 index 252c666b3a..0000000000 --- a/test/apis/async/tensorflow/sample.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "sepal_length": 5.2, - "sepal_width": 3.6, - "petal_length": 1.4, - "petal_width": 0.3 -} diff --git a/test/apis/async/text-generator/.dockerignore b/test/apis/async/text-generator/.dockerignore new file mode 100644 index 0000000000..7fa2250d13 --- /dev/null +++ b/test/apis/async/text-generator/.dockerignore @@ -0,0 +1,9 @@ +*.dockerfile +README.md +sample.json +expectations.yaml +*.pyc +*.pyo +*.pyd +__pycache__ +.pytest_cache diff --git a/test/apis/async/text-generator/cortex_cpu.yaml b/test/apis/async/text-generator/cortex_cpu.yaml new file mode 100644 index 0000000000..6d6332a910 --- /dev/null +++ b/test/apis/async/text-generator/cortex_cpu.yaml @@ -0,0 +1,16 @@ +- name: text-generator + kind: AsyncAPI + pod: + port: 9000 + containers: + - name: api + image: quay.io/cortexlabs-test/async-text-generator-cpu:latest + readiness_probe: + http_get: + path: "/healthz" + port: 9000 + compute: + cpu: 1 + mem: 2.5G + autoscaling: + max_concurrency: 1 diff --git a/test/apis/async/text-generator/cortex_gpu.yaml b/test/apis/async/text-generator/cortex_gpu.yaml new file mode 100644 index 0000000000..3b814daf93 --- /dev/null +++ b/test/apis/async/text-generator/cortex_gpu.yaml @@ -0,0 +1,19 @@ +- name: text-generator + kind: AsyncAPI + pod: + port: 9000 + containers: + - name: api + image: quay.io/cortexlabs-test/async-text-generator-gpu:latest + env: + TARGET_DEVICE: "cuda" + readiness_probe: + http_get: + path: "/healthz" + port: 9000 + compute: + cpu: 1 + gpu: 1 + mem: 512M + autoscaling: + max_concurrency: 1 diff --git a/test/apis/async/tensorflow/expectations.yaml b/test/apis/async/text-generator/expectations.yaml similarity index 69% rename from test/apis/async/tensorflow/expectations.yaml rename to test/apis/async/text-generator/expectations.yaml index bef65e0e0d..5797bcebae 100644 --- a/test/apis/async/tensorflow/expectations.yaml +++ b/test/apis/async/text-generator/expectations.yaml @@ -3,8 +3,7 @@ response: json_schema: type: "object" properties: - label: + prediction: type: "string" - const: "setosa" required: - - "label" + - "prediction" diff --git a/test/apis/async/text-generator/main.py b/test/apis/async/text-generator/main.py new file mode 100644 index 0000000000..020c7fc9e2 --- /dev/null +++ b/test/apis/async/text-generator/main.py @@ -0,0 +1,42 @@ +import os + +from fastapi import FastAPI, Response, status +from pydantic import BaseModel +from transformers import GPT2Tokenizer, GPT2LMHeadModel + + +class Request(BaseModel): + text: str + + +state = { + "model_ready": False, + "tokenizer": None, + "model": None, +} +device = os.getenv("TARGET_DEVICE", "cpu") +app = FastAPI() + + +@app.on_event("startup") +def startup(): + global state + state["tokenizer"] = GPT2Tokenizer.from_pretrained("gpt2") + state["model"] = GPT2LMHeadModel.from_pretrained("gpt2").to(device) + state["model_ready"] = True + + +@app.get("/healthz") +def healthz(response: Response): + if not state["model_ready"]: + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE + + +@app.post("/") +def text_generator(request: Request): + input_length = len(request.text.split()) + tokens = state["tokenizer"].encode(request.text, return_tensors="pt").to(device) + prediction = state["model"].generate(tokens, max_length=input_length + 20, do_sample=True) + return { + "prediction": state["tokenizer"].decode(prediction[0]), + } diff --git a/test/apis/async/text-generator/sample.json b/test/apis/async/text-generator/sample.json new file mode 100644 index 0000000000..36a3627568 --- /dev/null +++ b/test/apis/async/text-generator/sample.json @@ -0,0 +1,3 @@ +{ + "text": "machine learning is" +} diff --git a/test/apis/async/text-generator/text-generator-cpu.dockerfile b/test/apis/async/text-generator/text-generator-cpu.dockerfile new file mode 100644 index 0000000000..6b5367823f --- /dev/null +++ b/test/apis/async/text-generator/text-generator-cpu.dockerfile @@ -0,0 +1,23 @@ +FROM python:3.8-slim + +# Allow statements and log messages to immediately appear in the logs +ENV PYTHONUNBUFFERED True + +# Install production dependencies +RUN pip install --no-cache-dir \ + "uvicorn[standard]" \ + gunicorn \ + fastapi \ + pydantic \ + transformers==3.0.* \ + torch==1.7.1+cpu -f https://download.pytorch.org/whl/torch_stable.html + +# Copy local code to the container image. +COPY . /app +WORKDIR /app/ + +ENV PYTHONPATH=/app +ENV CORTEX_PORT=9000 + +# Run the web service on container startup +CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/async/text-generator/text-generator-gpu.dockerfile b/test/apis/async/text-generator/text-generator-gpu.dockerfile new file mode 100644 index 0000000000..2de181d169 --- /dev/null +++ b/test/apis/async/text-generator/text-generator-gpu.dockerfile @@ -0,0 +1,27 @@ +FROM nvidia/cuda:10.2-cudnn8-runtime-ubuntu18.04 + +RUN apt-get update \ + && apt-get install \ + python3 \ + python3-pip \ + pkg-config \ + git \ + build-essential \ + cmake -y \ + && apt-get clean -qq && rm -rf /var/lib/apt/lists/* + +# Allow statements and log messages to immediately appear in the logs +ENV PYTHONUNBUFFERED True + +# Install production dependencies +RUN pip3 install --no-cache-dir "uvicorn[standard]" gunicorn fastapi pydantic transformers==3.0.* torch==1.7.* + +# Copy local code to the container image. +COPY . /app +WORKDIR /app/ + +ENV PYTHONPATH=/app +ENV CORTEX_PORT=9000 + +# Run the web service on container startup. +CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/batch/image-classifier-alexnet/.dockerignore b/test/apis/batch/image-classifier-alexnet/.dockerignore new file mode 100644 index 0000000000..12657957ef --- /dev/null +++ b/test/apis/batch/image-classifier-alexnet/.dockerignore @@ -0,0 +1,8 @@ +*.dockerfile +README.md +sample.json +*.pyc +*.pyo +*.pyd +__pycache__ +.pytest_cache diff --git a/test/apis/batch/image-classifier-alexnet/cortex_cpu.yaml b/test/apis/batch/image-classifier-alexnet/cortex_cpu.yaml new file mode 100644 index 0000000000..1799203294 --- /dev/null +++ b/test/apis/batch/image-classifier-alexnet/cortex_cpu.yaml @@ -0,0 +1,20 @@ +- name: image-classifier-alexnet + kind: BatchAPI + pod: + containers: + - name: api + image: quay.io/cortexlabs-test/batch-image-classifier-alexnet-cpu:latest + command: + - "gunicorn" + - "-k" + - "uvicorn.workers.UvicornWorker" + - "--workers" + - "1" + - "--threads" + - "1" + - "--bind" + - ":$CORTEX_PORT" + - "main:app" + compute: + cpu: 1 + mem: 2G diff --git a/test/apis/batch/image-classifier-alexnet/cortex_gpu.yaml b/test/apis/batch/image-classifier-alexnet/cortex_gpu.yaml new file mode 100644 index 0000000000..11469453b2 --- /dev/null +++ b/test/apis/batch/image-classifier-alexnet/cortex_gpu.yaml @@ -0,0 +1,21 @@ +- name: image-classifier-alexnet + kind: BatchAPI + pod: + containers: + - name: api + image: quay.io/cortexlabs-test/batch-image-classifier-alexnet-gpu:latest + command: + - "gunicorn" + - "-k" + - "uvicorn.workers.UvicornWorker" + - "--workers" + - "1" + - "--threads" + - "1" + - "--bind" + - ":$CORTEX_PORT" + - "main:app" + compute: + cpu: 200m + gpu: 1 + mem: 512Mi diff --git a/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-cpu.dockerfile b/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-cpu.dockerfile new file mode 100644 index 0000000000..cee0dd0d34 --- /dev/null +++ b/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-cpu.dockerfile @@ -0,0 +1,25 @@ +FROM python:3.8-slim + +# Allow statements and log messages to immediately appear in the logs +ENV PYTHONUNBUFFERED True + +# Install production dependencies +RUN pip install --no-cache-dir \ + "uvicorn[standard]" \ + gunicorn \ + fastapi \ + pydantic \ + requests \ + torchvision \ + torch \ + boto3==1.17.72 + +# Copy local code to the container image. +COPY . /app +WORKDIR /app/ + +ENV PYTHONPATH=/app +ENV CORTEX_PORT=9000 + +# Run the web service on container startup. +CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-gpu.dockerfile b/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-gpu.dockerfile new file mode 100644 index 0000000000..442fdc22e3 --- /dev/null +++ b/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-gpu.dockerfile @@ -0,0 +1,36 @@ +FROM nvidia/cuda:10.2-cudnn8-runtime-ubuntu18.04 + +RUN apt-get update \ + && apt-get install \ + python3 \ + python3-pip \ + pkg-config \ + git \ + build-essential \ + cmake -y \ + && apt-get clean -qq && rm -rf /var/lib/apt/lists/* + +# allow statements and log messages to immediately appear in the logs +ENV PYTHONUNBUFFERED True + +# install production dependencies +RUN pip3 install --no-cache-dir \ + "uvicorn[standard]" \ + gunicorn \ + fastapi \ + pydantic \ + requests \ + torchvision \ + torch \ + boto3==1.17.72 + + +# copy local code to the container image. +COPY . /app +WORKDIR /app/ + +ENV PYTHONPATH=/app +ENV CORTEX_PORT=9000 + +# run the web service on container startup. +CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/batch/image-classifier-alexnet/main.py b/test/apis/batch/image-classifier-alexnet/main.py new file mode 100644 index 0000000000..45fa3ad2f1 --- /dev/null +++ b/test/apis/batch/image-classifier-alexnet/main.py @@ -0,0 +1,125 @@ +import os, json, re +from typing import Any, List + +from fastapi import FastAPI, Response, status +from pydantic import BaseModel + +import requests +import torch +import torchvision +from torchvision import transforms +from PIL import Image +from io import BytesIO +import boto3 + + +class Request(BaseModel): + payload: List[Any] + + +state = { + "ready": False, + "model": None, + "preprocess": None, + "job_id": None, + "labels": None, + "bucket": None, + "key": None, +} +device = os.getenv("TARGET_DEVICE", "cpu") +s3 = boto3.client("s3") +app = FastAPI() + + +@app.on_event("startup") +def startup(): + global state + + # read job spec + with open("/cortex/spec/job.json", "r") as f: + job_spec = json.load(f) + print(json.dumps(job_spec, indent=2)) + + # get metadata + config = job_spec["config"] + job_id = job_spec["job_id"] + state["job_id"] = job_spec["job_id"] + if len(config.get("dest_s3_dir", "")) == 0: + raise Exception("'dest_s3_dir' field was not provided in job submission") + + # s3 info + state["bucket"], state["key"] = re.match("s3://(.+?)/(.+)", config["dest_s3_dir"]).groups() + state["key"] = os.path.join(state["key"], job_id) + + # loading model + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + state["preprocess"] = transforms.Compose( + [transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize] + ) + state["labels"] = requests.get( + "https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt" + ).text.split("\n")[1:] + state["model"] = torchvision.models.alexnet(pretrained=True).eval().to(device) + + state["ready"] = True + + +@app.get("/healthz") +def healthz(response: Response): + if not state["ready"]: + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE + + +@app.post("/") +def handle_batch(request: Request): + payload = request.payload + job_id = state["job_id"] + tensor_list = [] + + # download and preprocess each image + for image_url in payload: + if image_url.startswith("s3://"): + bucket, image_key = re.match("s3://(.+?)/(.+)", image_url).groups() + image_bytes = s3.get_object(Bucket=bucket, Key=image_key)["Body"].read() + elif image_url.startswith("http"): + image_bytes = requests.get(image_url).content + else: + raise RuntimeError(f"{image_url}: invalid image url") + + img_pil = Image.open(BytesIO(image_bytes)) + tensor_list.append(state["preprocess"](img_pil)) + + # classify the batch of images + img_tensor = torch.stack(tensor_list) + with torch.no_grad(): + prediction = state["model"](img_tensor) + _, indices = prediction.max(1) + + # extract predicted classes + results = [ + {"url": payload[i], "class": state["labels"][class_idx]} + for i, class_idx in enumerate(indices) + ] + json_output = json.dumps(results) + + # save results + s3.put_object(Bucket=state["bucket"], Key=f"{state['key']}/{job_id}.json", Body=json_output) + + +@app.post("/on-job-complete") +def on_job_complete(): + all_results = [] + + # aggregate all classifications + paginator = s3.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=state["bucket"], Prefix=state["key"]): + for obj in page["Contents"]: + body = s3.get_object(Bucket=state["bucket"], Key=obj["Key"])["Body"] + all_results += json.loads(body.read().decode("utf8")) + + # save single file containing aggregated classifications + s3.put_object( + Bucket=state["bucket"], + Key=os.path.join(state["key"], "aggregated_results.json"), + Body=json.dumps(all_results), + ) diff --git a/test/apis/batch/image-classifier-alexnet/sample.json b/test/apis/batch/image-classifier-alexnet/sample.json new file mode 100644 index 0000000000..5efc42f10f --- /dev/null +++ b/test/apis/batch/image-classifier-alexnet/sample.json @@ -0,0 +1,6 @@ +[ + "https://i.imgur.com/PzXprwl.jpg", + "https://i.imgur.com/E4cOSLw.jpg", + "https://i.imgur.com/jDimNTZ.jpg", + "https://i.imgur.com/WqeovVj.jpg" +] diff --git a/test/apis/batch/image-classifier-alexnet/submit.py b/test/apis/batch/image-classifier-alexnet/submit.py new file mode 100644 index 0000000000..8e9be6e80b --- /dev/null +++ b/test/apis/batch/image-classifier-alexnet/submit.py @@ -0,0 +1,41 @@ +""" +Typical usage example: + + python submit.py +""" + +from typing import List + +import sys +import json +import requests +import cortex + + +def main(): + # parse args + if len(sys.argv) != 3: + print("usage: python submit.py ") + sys.exit(1) + env_name = sys.argv[1] + dest_s3_dir = sys.argv[2] + + # read sample file + with open("sample.json") as f: + sample_items: List[str] = json.load(f) + + # get batch endpoint + cx = cortex.client(env_name) + batch_endpoint = cx.get_api("image-classifier-alexnet")["endpoint"] + + # submit job + job_spec = { + "item_list": {"items": sample_items, "batch_size": 1}, + "config": {"dest_s3_dir": dest_s3_dir}, + } + response = requests.post(batch_endpoint, json=job_spec) + print(json.dumps(response.json(), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/test/apis/batch/image-classifier/README.md b/test/apis/batch/image-classifier/README.md deleted file mode 100644 index 2e893c8fbe..0000000000 --- a/test/apis/batch/image-classifier/README.md +++ /dev/null @@ -1,568 +0,0 @@ -# Deploy models as Batch APIs - -This example shows how to deploy a batch image classification api that accepts a list of image urls as input, downloads the images, classifies them, and writes the results to S3. - -## Pre-requisites - -* [Install](../../../docs/aws/install.md) Cortex and create a cluster -* Create an S3 bucket/directory to store the results of the batch job -* AWS CLI (optional) - -
- -## Implement your handler - -1. Create a Python file named `handler.py`. -1. Define a Handler class with a constructor that loads and initializes an image-classifier from `torchvision`. -1. Add a `handle_post()` function that will accept a list of images urls (http:// or s3://), downloads them, performs inference, and writes the predictions to S3. -1. Specify an `on_job_complete()` function that aggregates the results and writes them to a single file named `aggregated_results.json` in S3. - -```python -# handler.py - -import os -import requests -import torch -import torchvision -from torchvision import transforms -from PIL import Image -from io import BytesIO -import boto3 -import json -import re - - -class Handler: - def __init__(self, config, job_spec): - self.model = torchvision.models.alexnet(pretrained=True).eval() - - normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) - self.preprocess = transforms.Compose( - [transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize] - ) - - self.labels = requests.get( - "https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt" - ).text.split("\n")[1:] - - if len(config.get("dest_s3_dir", "")) == 0: - raise Exception("'dest_s3_dir' field was not provided in job submission") - - self.s3 = boto3.client("s3") - - self.bucket, self.key = re.match("s3://(.+?)/(.+)", config["dest_s3_dir"]).groups() - self.key = os.path.join(self.key, job_spec["job_id"]) - - def handle_batch(self, payload, batch_id): - tensor_list = [] - - # download and preprocess each image - for image_url in payload: - if image_url.startswith("s3://"): - bucket, image_key = re.match("s3://(.+?)/(.+)", image_url).groups() - image_bytes = self.s3.get_object(Bucket=bucket, Key=image_key)["Body"].read() - else: - image_bytes = requests.get(image_url).content - - img_pil = Image.open(BytesIO(image_bytes)) - tensor_list.append(self.preprocess(img_pil)) - - # classify the batch of images - img_tensor = torch.stack(tensor_list) - with torch.no_grad(): - prediction = self.model(img_tensor) - _, indices = prediction.max(1) - - # extract predicted classes - results = [ - {"url": payload[i], "class": self.labels[class_idx]} - for i, class_idx in enumerate(indices) - ] - json_output = json.dumps(results) - - # save results - self.s3.put_object(Bucket=self.bucket, Key=f"{self.key}/{batch_id}.json", Body=json_output) - - def on_job_complete(self): - all_results = [] - - # aggregate all classifications - paginator = self.s3.get_paginator("list_objects_v2") - for page in paginator.paginate(Bucket=self.bucket, Prefix=self.key): - for obj in page["Contents"]: - body = self.s3.get_object(Bucket=self.bucket, Key=obj["Key"])["Body"] - all_results += json.loads(body.read().decode("utf8")) - - # save single file containing aggregated classifications - self.s3.put_object( - Bucket=self.bucket, - Key=os.path.join(self.key, "aggregated_results.json"), - Body=json.dumps(all_results), - ) -``` - -Here are the complete [Handler docs](../../../docs/workloads/batch/handler.md). - -
- -## Specify your Python dependencies - -Create a `requirements.txt` file to specify the dependencies needed by `handler.py`. Cortex will automatically install them into your runtime once you deploy: - -```python -# requirements.txt - -boto3 -torch -torchvision -pillow -``` - -
- -## Configure your API - -Create a `cortex.yaml` file and add the configuration below. An `api` with `kind: BatchAPI` will expose your model as an endpoint that will orchestrate offline batch inference across multiple workers upon receiving job requests. The configuration below defines how much `compute` each worker requires and your `handler.py` determines how each batch should be processed. - -```yaml -# cortex.yaml - -- name: image-classifier - kind: BatchAPI - handler: - type: python - path: handler.py - compute: - cpu: 1 -``` - -Here are the complete [API configuration docs](../../../docs/workloads/batch/configuration.md). - -
- -## Deploy your Batch API - -`cortex deploy` takes your model, your `handler.py` implementation, and your configuration from `cortex.yaml` and creates an endpoint that can receive job submissions and manage running jobs. - -```bash -$ cortex deploy - -created image-classifier (BatchAPI) -``` - -Get the endpoint for your Batch API with `cortex get image-classifier`: - -```bash -$ cortex get image-classifier - -no submitted jobs - -endpoint: http://***.elb.us-west-2.amazonaws.com/image-classifier -``` - -
- -## Setup destination S3 directory - -Our `handler.py` implementation writes results to an S3 directory. Before submitting a job, we need to create an S3 directory to store the output of the batch job. The S3 directory should be accessible by the credentials used to create your Cortex cluster. - -Export the S3 directory to an environment variable: - -```bash -$ export CORTEX_DEST_S3_DIR= # e.g. export CORTEX_DEST_S3_DIR=s3://my-bucket/dir -``` - -
- -## Submit a job - -Now that you've deployed a Batch API, you are ready to submit jobs. You can provide image urls directly in the request by specifying the urls in `item_list`. The curl command below showcases how to submit image urls in the request. - -```bash -$ export BATCH_API_ENDPOINT= # e.g. export BATCH_API_ENDPOINT=https://***.elb.us-west-2.amazonaws.com/image-classifier -$ export CORTEX_DEST_S3_DIR= # e.g. export CORTEX_DEST_S3_DIR=s3://my-bucket/dir -$ curl $BATCH_API_ENDPOINT \ - -X POST -H "Content-Type: application/json" \ - -d @- <` then type `EOF`. - -After submitting the job, you should get a response like this: - -```json -{"job_id":"69d6faf82e4660d3","api_name":"image-classifier", "config":{"dest_s3_dir": "YOUR_S3_BUCKET_HERE"}} -``` - -Take note of the job id in the response. - -### List the jobs for your Batch API - -```bash -$ cortex get image-classifier - -job id status progress start time duration -69d6faf82e4660d3 running 0/3 20 Jul 2020 01:07:44 UTC 3m26s - -endpoint: http://***.elb.us-west-2.amazonaws.com/image-classifier -``` - -### Get the job status with an HTTP request - -You can make a GET request to your `?JOB_ID` to get the status of your job. - -```bash -$ curl http://***.elb.us-west-2.amazonaws.com/image-classifier?jobID=69d6faf82e4660d3 - -{ - "job_status":{ - "job_id":"69d6faf82e4660d3", - "api_name":"image-classifier", - ... - }, - "endpoint":"https://***.elb.us-west-2.amazonaws.com/image-classifier" -} -``` - -### Get job status using Cortex CLI - -You can also use the Cortex CLI to get the status of your job using `cortex get `. - -```bash -$ cortex get image-classifier 69d6faf82e4660d3 - -job id: 69d6faf82e4660d3 -status: running - -start time: 27 Jul 2020 15:02:25 UTC -end time: - -duration: 42s - -batch stats -total succeeded failed avg time per batch -3 0 0 - - -worker stats -requested initializing running failed succeeded -1 1 0 0 0 - -job endpoint: http://***.elb.us-west-2.amazonaws.com/image-classifier/69d6faf82e4660d3 -``` - -### Stream logs - -You can stream logs realtime for debugging and monitoring purposes with `cortex logs ` - -```bash -$ cortex logs image-classifier 69d6fdeb2d8e6647 - -started enqueuing batches to queue -partitioning 5 items found in job submission into 3 batches of size 2 -completed enqueuing a total of 3 batches -spinning up workers... -... -2020-08-07 14:44:05.557598:cortex:pid-25:INFO:processing batch c9136381-6dcc-45bd-bd97-cc9c66ccc6d6 -2020-08-07 14:44:26.037276:cortex:pid-25:INFO:executing on_job_complete -2020-08-07 14:44:26.208972:cortex:pid-25:INFO:no batches left in queue, job has been completed -``` - -### Find your results - -Wait for the job to complete by streaming the logs with `cortex logs ` or watching for the job status to change with `cortex get --watch`. - -The status of your job, which you can get from `cortex get `, should change from `running` to `succeeded` once the job has completed. If it changes to a different status, you may be able to find the stacktrace using `cortex logs `. If your job has completed successfully, you can view the results of the image classification in the S3 directory you specified in the job submission. - -Using the AWS CLI: - -```bash -$ aws s3 ls $CORTEX_DEST_S3_DIR// - 161f9fda-fd08-44f3-b983-4529f950e40b.json - 40100ffb-6824-4560-8ca4-7c0d14273e05.json - c9136381-6dcc-45bd-bd97-cc9c66ccc6d6.json - aggregated_results.json -``` - -You can download the aggregated results file with `aws s3 cp $CORTEX_DEST_S3_DIR//aggregated_results.json .` and confirm that there are 16 classifications. - -
- -## Alternative job submission: image URLs in files - -In addition to providing the image URLs directly in the job submission request, it is possible to use image urls stored in newline delimited json files in S3. A newline delimited JSON file has one complete JSON object per line. - -Two newline delimited json files containing image urls for this tutorial have already been created for you and can be found at `s3://cortex-examples/image-classifier/`. If you have AWS CLI, you can list the directory and you should be able to find the files (`urls_0.json` and `urls_1.json`). - -```text -$ aws s3 ls s3://cortex-examples/image-classifier/ - PRE inception/ -... -2020-07-27 14:19:30 506 urls_0.json -2020-07-27 14:19:30 473 urls_1.json -``` - -To use JSON files as input data for the job, we will specify `delimited_files` in the job request. The Batch API will break up the JSON files into batches of desired size and push them onto a queue that is consumed by the pool of workers. - -### Dry run - -Before we submit the job, let's perform a dry run to ensure that only the desired files will be read. You can perform a dry run by appending `dryRun=true` query parameter to your job request. - -Get the endpoint from `cortex get image-classifier` if you haven't done so already. - -```bash -$ export BATCH_API_ENDPOINT= # e.g. export BATCH_API_ENDPOINT=https://***.elb.us-west-2.amazonaws.com/image-classifier -$ export CORTEX_DEST_S3_DIR= # e.g. export CORTEX_DEST_S3_DIR=s3://my-bucket/dir -$ curl $BATCH_API_ENDPOINT?dryRun=true \ --X POST -H "Content-Type: application/json" \ --d @- <` then type `EOF`. - -You should expect a response like this: - -```text -s3://cortex-examples/image-classifier/urls_0.json -s3://cortex-examples/image-classifier/urls_1.json -validations passed -``` - -This shows that the correct files will be used as input for the job. - -### Classify image urls stored in S3 files - -When you submit a job specifying `delimited_files`, your Batch API will get all of the input S3 files based on `s3_paths` and will apply the filters specified in `includes` and `excludes`. Then your Batch API will read each file, split on the newline characters, and parse each item as a JSON object. Each item in the file is treated as a single sample and will be grouped together into batches and then placed onto a queue that is consumed by the pool of workers. - -In this example `urls_0.json` and `urls_1.json` each contain 8 urls. Let's classify the images from the URLs listed in those 2 files. - -```bash -$ export BATCH_API_ENDPOINT= # e.g. export BATCH_API_ENDPOINT=https://***.elb.us-west-2.amazonaws.com/image-classifier -$ export CORTEX_DEST_S3_DIR= # e.g. export CORTEX_DEST_S3_DIR=s3://my-bucket/dir -$ curl $BATCH_API_ENDPOINT \ --X POST -H "Content-Type: application/json" \ --d @- <` then type `EOF`. - -After submitting this job, you should get a response like this: - -```json -{"job_id":"69d6faf82e4660d3","api_name":"image-classifier", "config":{"dest_s3_dir": "YOUR_S3_BUCKET_HERE"}} -``` - -### Find results - -Wait for the job to complete by streaming the logs with `cortex logs ` or watching for the job status to change with `cortex get --watch`. - -```bash -$ cortex logs image-classifier 69d6faf82e4660d3 - -started enqueuing batches to queue -enqueuing contents from file s3://cortex-examples/image-classifier/urls_0.json -enqueuing contents from file s3://cortex-examples/image-classifier/urls_1.json -completed enqueuing a total of 8 batches -spinning up workers... -2020-08-07 15:11:21.364179:cortex:pid-25:INFO:processing batch 1de0bc65-04ea-4b9e-9e96-5a0bb52fcc37 -... -2020-08-07 15:11:45.461032:cortex:pid-25:INFO:no batches left in queue, job has been completed -``` - -The status of your job, which you can get from `cortex get `, should change from `running` to `succeeded` once the job has completed. If it changes to a different status, you may be able to find the stacktrace using `cortex logs `. If your job has completed successfully, you can view the results of the image classification in the S3 directory you specified in the job submission. - -Using the AWS CLI: - -```bash -$ aws s3 ls $CORTEX_DEST_S3_DIR// - 161f9fda-fd08-44f3-b983-4529f950e40b.json - 40100ffb-6824-4560-8ca4-7c0d14273e05.json - 6d1c933c-0ddf-4316-9956-046cd731c5ab.json - ... - aggregated_results.json -``` - -You can download the aggregated results file with `aws s3 cp $CORTEX_DEST_S3_DIR//aggregated_results.json .` and confirm that there are 16 classifications. - -
- -## Alternative job submission: images in S3 - -Let's assume that rather downloading urls on the internet, you have an S3 directory containing the images. We can specify `file_path_lister` in the job request to get the list of S3 urls for the images, partition the list of S3 urls into batches, and place them on a queue that will be consumed by the workers. - -We'll classify the 16 images that can be found here `s3://cortex-examples/image-classifier/samples`. You can use AWS CLI to verify that there are 16 images `aws s3 ls s3://cortex-examples/image-classifier/samples/`. - -### Dry run - -Let's do a dry run to make sure the correct list of images will be submitted to the job. - -```bash -$ export BATCH_API_ENDPOINT= # e.g. export BATCH_API_ENDPOINT=https://***.elb.us-west-2.amazonaws.com/image-classifier -$ export CORTEX_DEST_S3_DIR= # e.g. export CORTEX_DEST_S3_DIR=s3://my-bucket/dir -$ curl $BATCH_API_ENDPOINT?dryRun=true \ --X POST -H "Content-Type: application/json" \ --d @- <` then type `EOF`. - -You should expect a response like this: - -```text -s3://cortex-examples/image-classifier/samples/img_0.jpg -s3://cortex-examples/image-classifier/samples/img_1.jpg -... -s3://cortex-examples/image-classifier/samples/img_8.jpg -s3://cortex-examples/image-classifier/samples/img_9.jpg -validations passed -``` - -### Classify images in S3 - -Let's actually submit the job now. Your Batch API will get all of the input S3 files based on `s3_paths` and will apply the filters specified in `includes` and `excludes`. - -```bash -$ export BATCH_API_ENDPOINT= # e.g. export BATCH_API_ENDPOINT=https://***.elb.us-west-2.amazonaws.com/image-classifier -$ export CORTEX_DEST_S3_DIR= # e.g. export CORTEX_DEST_S3_DIR=s3://my-bucket/dir -$ curl $BATCH_API_ENDPOINT \ --X POST -H "Content-Type: application/json" \ --d @- <` then type `EOF`. - -You should get a response like this: - -```json -{"job_id":"69d6f8a472f0e1e5","api_name":"image-classifier", "config":{"dest_s3_dir": "YOUR_S3_BUCKET_HERE"}} -``` - -### Verify results - -Wait for the job to complete by streaming the logs with `cortex logs ` or watching for the job status to change with `cortex get --watch`. - -```bash -$ cortex logs image-classifier 69d6f8a472f0e1e5 - -started enqueuing batches to queue -completed enqueuing a total of 8 batches -spinning up workers... -2020-07-18 21:35:34.186348:cortex:pid-1:INFO:downloading the project code -... -2020-08-07 15:49:10.889839:cortex:pid-25:INFO:processing batch d0e695bc-a975-4115-a60f-0a55c743fc57 -2020-08-07 15:49:31.188943:cortex:pid-25:INFO:executing on_job_complete -2020-08-07 15:49:31.362053:cortex:pid-25:INFO:no batches left in queue, job has been completed -``` - -The status of your job, which you can get from `cortex get `, should change from `running` to `succeeded` once the job has completed. If it changes to a different status, you may be able to find the stacktrace using `cortex logs `. If your job has completed successfully, you can view the results of the image classification in the S3 directory you specified in the job submission. - -Using the AWS CLI: - -```bash -$ aws s3 ls $CORTEX_DEST_S3_DIR// - 6bee7412-4c16-4d9f-ab3e-e88669cf7a89.json - 3c45b4b3-953e-4226-865b-75f3961dcf95.json - d0e695bc-a975-4115-a60f-0a55c743fc57.json - ... - aggregated_results.json -``` - -You can download the aggregated results file with `aws s3 cp $CORTEX_DEST_S3_DIR//aggregated_results.json .` and confirm that there are 16 classifications. - -
- -## Stopping a Job - -You can stop a running job by sending a DELETE request to `/`. - -```bash -$ export BATCH_API_ENDPOINT= # e.g. export BATCH_API_ENDPOINT=https://***.elb.us-west-2.amazonaws.com/image-classifier -$ curl -X DELETE $BATCH_API_ENDPOINT?jobID=69d96a01ea55da8c - -stopped job 69d96a01ea55da8c -``` - -You can also use the Cortex CLI `cortex delete `. - -```bash -$ cortex delete image-classifier 69d96a01ea55da8c - -stopped job 69d96a01ea55da8c -``` - -
- -## Cleanup - -Run `cortex delete` to delete the API: - -```bash -$ cortex delete image-classifier - -deleting image-classifier -``` - -Running `cortex delete` will stop all in progress jobs for the API and will delete job history for that API. It will not spin down your cluster. diff --git a/test/apis/batch/image-classifier/cortex.yaml b/test/apis/batch/image-classifier/cortex.yaml deleted file mode 100644 index e4e181f88a..0000000000 --- a/test/apis/batch/image-classifier/cortex.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- name: image-classifier - kind: BatchAPI - handler: - type: python - path: handler.py - compute: - cpu: 1 diff --git a/test/apis/batch/image-classifier/handler.py b/test/apis/batch/image-classifier/handler.py deleted file mode 100644 index a815912571..0000000000 --- a/test/apis/batch/image-classifier/handler.py +++ /dev/null @@ -1,79 +0,0 @@ -import os -import requests -import torch -import torchvision -from torchvision import transforms -from PIL import Image -from io import BytesIO -import boto3 -import json -import re - - -class Handler: - def __init__(self, config, job_spec): - self.model = torchvision.models.alexnet(pretrained=True).eval() - - normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) - self.preprocess = transforms.Compose( - [transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize] - ) - - self.labels = requests.get( - "https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt" - ).text.split("\n")[1:] - - if len(config.get("dest_s3_dir", "")) == 0: - raise Exception("'dest_s3_dir' field was not provided in job submission") - - self.s3 = boto3.client("s3") - - self.bucket, self.key = re.match("s3://(.+?)/(.+)", config["dest_s3_dir"]).groups() - self.key = os.path.join(self.key, job_spec["job_id"]) - - def handle_batch(self, payload, batch_id): - tensor_list = [] - - # download and preprocess each image - for image_url in payload: - if image_url.startswith("s3://"): - bucket, image_key = re.match("s3://(.+?)/(.+)", image_url).groups() - image_bytes = self.s3.get_object(Bucket=bucket, Key=image_key)["Body"].read() - else: - image_bytes = requests.get(image_url).content - - img_pil = Image.open(BytesIO(image_bytes)) - tensor_list.append(self.preprocess(img_pil)) - - # classify the batch of images - img_tensor = torch.stack(tensor_list) - with torch.no_grad(): - prediction = self.model(img_tensor) - _, indices = prediction.max(1) - - # extract predicted classes - results = [ - {"url": payload[i], "class": self.labels[class_idx]} - for i, class_idx in enumerate(indices) - ] - json_output = json.dumps(results) - - # save results - self.s3.put_object(Bucket=self.bucket, Key=f"{self.key}/{batch_id}.json", Body=json_output) - - def on_job_complete(self): - all_results = [] - - # aggregate all classifications - paginator = self.s3.get_paginator("list_objects_v2") - for page in paginator.paginate(Bucket=self.bucket, Prefix=self.key): - for obj in page["Contents"]: - body = self.s3.get_object(Bucket=self.bucket, Key=obj["Key"])["Body"] - all_results += json.loads(body.read().decode("utf8")) - - # save single file containing aggregated classifications - self.s3.put_object( - Bucket=self.bucket, - Key=os.path.join(self.key, "aggregated_results.json"), - Body=json.dumps(all_results), - ) diff --git a/test/apis/batch/image-classifier/requirements.txt b/test/apis/batch/image-classifier/requirements.txt deleted file mode 100644 index 2c0ef31b51..0000000000 --- a/test/apis/batch/image-classifier/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -torch -torchvision -boto3 -pillow diff --git a/test/apis/batch/image-classifier/sample.json b/test/apis/batch/image-classifier/sample.json deleted file mode 100644 index eb45c463fd..0000000000 --- a/test/apis/batch/image-classifier/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -[ - "https://i.imgur.com/PzXprwl.jpg" -] diff --git a/test/apis/batch/inferentia/cortex_inf.yaml b/test/apis/batch/inferentia/cortex_inf.yaml deleted file mode 100644 index a1d4bcf8c5..0000000000 --- a/test/apis/batch/inferentia/cortex_inf.yaml +++ /dev/null @@ -1,16 +0,0 @@ -- name: image-classifier-resnet50 - kind: BatchAPI - handler: - type: python - path: handler.py - config: - model_path: s3://cortex-examples/pytorch/image-classifier-resnet50 - # this model only supports batch size of 1, given the way it was saved. - # It is intended for testing only. - model_name: resnet50_neuron.pt - device: inf - classes: https://s3.amazonaws.com/deep-learning-models/image-models/imagenet_class_index.json - input_shape: [224, 224] - compute: - inf: 1 - cpu: 1 diff --git a/test/apis/batch/inferentia/dependencies.sh b/test/apis/batch/inferentia/dependencies.sh deleted file mode 100644 index 057530cb85..0000000000 --- a/test/apis/batch/inferentia/dependencies.sh +++ /dev/null @@ -1 +0,0 @@ -apt-get update && apt-get install -y libgl1-mesa-glx libegl1-mesa diff --git a/test/apis/batch/inferentia/handler.py b/test/apis/batch/inferentia/handler.py deleted file mode 100644 index 73f257d79a..0000000000 --- a/test/apis/batch/inferentia/handler.py +++ /dev/null @@ -1,90 +0,0 @@ -import os -import re - -import boto3 -import cv2 -import numpy as np -import requests -import torch -from torchvision import models, transforms - - -def get_url_image(url_image): - """ - Get numpy image from URL image. - """ - resp = requests.get(url_image, stream=True).raw - image = np.asarray(bytearray(resp.read()), dtype="uint8") - image = cv2.imdecode(image, cv2.IMREAD_COLOR) - image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) - return image - - -class Handler: - def __init__(self, config, job_spec): - # load classes - classes = requests.get(config["classes"]).json() - self.idx2label = [classes[str(k)][1] for k in range(len(classes))] - - # download the model - model_path = config["model_path"] - model_name = config["model_name"] - bucket, key = re.match("s3://(.+?)/(.+)", model_path).groups() - s3 = boto3.client("s3") - s3.download_file(bucket, os.path.join(key, model_name), model_name) - - # load the model - self.device = None - if config["device"] == "gpu": - self.device = torch.device("cuda") - self.model = models.resnet50() - self.model.load_state_dict(torch.load(model_name, map_location="cuda:0")) - self.model.eval() - self.model = self.model.to(self.device) - elif config["device"] == "cpu": - self.model = models.resnet50() - self.model.load_state_dict(torch.load(model_name)) - self.model.eval() - elif config["device"] == "inf": - import torch_neuron - - self.model = torch.jit.load(model_name) - else: - raise RuntimeError("invalid handler: config: must be cpu, gpu, or inf") - - # save normalization transform for later use - normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) - self.transform = transforms.Compose( - [ - transforms.ToPILImage(), - transforms.Resize(config["input_shape"]), - transforms.ToTensor(), - normalize, - ] - ) - - def handle_batch(self, payload): - # preprocess images - tensor_list = [] - for image_url in payload: - image = get_url_image(image_url) - image = self.transform(image) - image = torch.from_numpy(image.numpy()) - tensor_list.append(image) - - # predict - img_tensor = torch.stack(tensor_list) - with torch.no_grad(): - if self.device: - results = self.model(img_tensor.to(self.device)) - else: - results = self.model(img_tensor) - - # Get the top 5 results - top5_idx = results[0].sort()[1][-5:] - - # Lookup and print the top 5 labels - top5_labels = [self.idx2label[idx] for idx in top5_idx] - top5_labels = top5_labels[::-1] - - print(top5_labels) diff --git a/test/apis/batch/inferentia/requirements.txt b/test/apis/batch/inferentia/requirements.txt deleted file mode 100644 index df61209f31..0000000000 --- a/test/apis/batch/inferentia/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -torch==1.7.1 -torchvision==0.8.2 -opencv-python==4.4.0.42 ---extra-index-url https://pip.repos.neuron.amazonaws.com -torch-neuron==1.7.1.1.2.3.0 diff --git a/test/apis/batch/inferentia/sample.json b/test/apis/batch/inferentia/sample.json deleted file mode 100644 index 31c110e720..0000000000 --- a/test/apis/batch/inferentia/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -[ - "https://i.imgur.com/213xcvs.jpg" -] diff --git a/test/apis/batch/onnx/cortex.yaml b/test/apis/batch/onnx/cortex.yaml deleted file mode 100644 index 7d68bfffbc..0000000000 --- a/test/apis/batch/onnx/cortex.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- name: image-classifier - kind: BatchAPI - handler: - type: python - path: handler.py - config: - path: s3://cortex-examples/image-classifier/alexnet_batch/alexnet_batch.onnx - compute: - cpu: 1 diff --git a/test/apis/batch/onnx/handler.py b/test/apis/batch/onnx/handler.py deleted file mode 100644 index 29f95a03bb..0000000000 --- a/test/apis/batch/onnx/handler.py +++ /dev/null @@ -1,73 +0,0 @@ -import json -import re -import os -from io import BytesIO - -import requests -import numpy as np -import onnxruntime as rt -from PIL import Image -from torchvision import transforms -import boto3 - - -class Handler: - def __init__(self, config, job_spec): - self.labels = requests.get( - "https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt" - ).text.split("\n")[1:] - - # https://github.com/pytorch/examples/blob/447974f6337543d4de6b888e244a964d3c9b71f6/imagenet/main.py#L198-L199 - normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) - self.preprocess = transforms.Compose( - [transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize] - ) - - if len(config.get("dest_s3_dir", "")) == 0: - raise Exception("'dest_s3_dir' field was not provided in job submission") - - self.s3 = boto3.client("s3") - - # download and load ONNX model - model_bucket, model_key = re.match("s3://(.+?)/(.+)", config["path"]).groups() - self.s3.download_file(model_bucket, model_key, "model.onnx") - self.session = rt.InferenceSession("model.onnx") - self.input_name = self.session.get_inputs()[0].name - self.output_name = self.session.get_outputs()[0].name - - # store destination bucket/key - self.bucket, self.key = re.match("s3://(.+?)/(.+)", config["dest_s3_dir"]).groups() - self.key = os.path.join(self.key, job_spec["job_id"]) - - def handle_batch(self, payload, batch_id): - arr_list = [] - - # download and preprocess each image - for image_url in payload: - if image_url.startswith("s3://"): - bucket, image_key = re.match("s3://(.+?)/(.+)", image_url).groups() - image_bytes = self.s3.get_object(Bucket=bucket, Key=image_key)["Body"].read() - else: - image_bytes = requests.get(image_url).content - - img_pil = Image.open(BytesIO(image_bytes)) - arr_list.append(self.preprocess(img_pil).numpy()) - - # classify the batch of images - result = self.session.run( - [self.output_name], - { - self.input_name: np.stack(arr_list, axis=0), - }, - ) - - # extract predicted classes - predicted_classes = np.argmax(result[0], axis=1) - results = [ - {"url": payload[i], "class": self.labels[class_idx]} - for i, class_idx in enumerate(predicted_classes) - ] - - # save results - json_output = json.dumps(results) - self.s3.put_object(Bucket=self.bucket, Key=f"{self.key}/{batch_id}.json", Body=json_output) diff --git a/test/apis/batch/onnx/requirements.txt b/test/apis/batch/onnx/requirements.txt deleted file mode 100644 index 0cd9b2f35f..0000000000 --- a/test/apis/batch/onnx/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -torchvision -boto3 -pillow -onnxruntime==1.6.0 -numpy==1.19.1 diff --git a/test/apis/batch/onnx/sample.json b/test/apis/batch/onnx/sample.json deleted file mode 100644 index eb45c463fd..0000000000 --- a/test/apis/batch/onnx/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -[ - "https://i.imgur.com/PzXprwl.jpg" -] diff --git a/test/apis/batch/sum/.dockerignore b/test/apis/batch/sum/.dockerignore new file mode 100644 index 0000000000..30fea70be8 --- /dev/null +++ b/test/apis/batch/sum/.dockerignore @@ -0,0 +1,10 @@ +*.dockerfile +README.md +sample.json +submit.py +sample_generator.py +*.pyc +*.pyo +*.pyd +__pycache__ +.pytest_cache diff --git a/test/apis/batch/sum/cortex.yaml b/test/apis/batch/sum/cortex.yaml deleted file mode 100644 index 7a80918a2a..0000000000 --- a/test/apis/batch/sum/cortex.yaml +++ /dev/null @@ -1,8 +0,0 @@ -- name: summing-api - kind: BatchAPI - handler: - type: python - path: handler.py - compute: - cpu: 100m - mem: 200Mi diff --git a/test/apis/batch/sum/cortex_cpu.yaml b/test/apis/batch/sum/cortex_cpu.yaml new file mode 100644 index 0000000000..22e59ef857 --- /dev/null +++ b/test/apis/batch/sum/cortex_cpu.yaml @@ -0,0 +1,22 @@ +# this API is only meant to run with 1-worker jobs + +- name: sum + kind: BatchAPI + pod: + containers: + - name: api + image: quay.io/cortexlabs-test/batch-sum-cpu:latest + command: + - "gunicorn" + - "-k" + - "uvicorn.workers.UvicornWorker" + - "--workers" + - "1" + - "--threads" + - "1" + - "--bind" + - ":$CORTEX_PORT" + - "main:app" + compute: + cpu: 200m + mem: 256Mi diff --git a/test/apis/batch/sum/handler.py b/test/apis/batch/sum/handler.py deleted file mode 100644 index c628f158ca..0000000000 --- a/test/apis/batch/sum/handler.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import boto3 -import json -import re - - -class Handler: - def __init__(self, config, job_spec): - if len(config.get("dest_s3_dir", "")) == 0: - raise Exception("'dest_s3_dir' field was not provided in job submission") - - self.s3 = boto3.client("s3") - - self.bucket, self.key = re.match("s3://(.+?)/(.+)", config["dest_s3_dir"]).groups() - self.key = os.path.join(self.key, job_spec["job_id"]) - self.list = [] - - def handle_batch(self, payload, batch_id): - for numbers_list in payload: - self.list.append(sum(numbers_list)) - - def on_job_complete(self): - json_output = json.dumps(self.list) - self.s3.put_object(Bucket=self.bucket, Key=f"{self.key}.json", Body=json_output) diff --git a/test/apis/batch/sum/main.py b/test/apis/batch/sum/main.py new file mode 100644 index 0000000000..7c82267d88 --- /dev/null +++ b/test/apis/batch/sum/main.py @@ -0,0 +1,63 @@ +import os +import boto3 +import json +import re + +from typing import List +from pydantic import BaseModel +from fastapi import FastAPI, Response, status + + +class Request(BaseModel): + payload: List[List[int]] + + +state = { + "ready": False, + "bucket": None, + "key": None, + "numbers_list": [], +} +app = FastAPI() +s3 = boto3.client("s3") + + +@app.on_event("startup") +def startup(): + global state + + # read job spec + with open("/cortex/spec/job.json", "r") as f: + job_spec = json.load(f) + print(json.dumps(job_spec, indent=2)) + + # get metadata + config = job_spec["config"] + job_id = job_spec["job_id"] + if len(config.get("dest_s3_dir", "")) == 0: + raise Exception("'dest_s3_dir' field was not provided in job submission") + + # s3 info + state["bucket"], state["key"] = re.match("s3://(.+?)/(.+)", config["dest_s3_dir"]).groups() + state["key"] = os.path.join(state["key"], job_id) + + state["ready"] = True + + +@app.get("/healthz") +def healthz(response: Response): + if not state["ready"]: + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE + + +@app.post("/") +def handle_batch(request: Request): + global state + for numbers_list in request.payload: + state["numbers_list"].append(sum(numbers_list)) + + +@app.post("/on-job-complete") +def on_job_complete(): + json_output = json.dumps(state["numbers_list"]) + s3.put_object(Bucket=state["bucket"], Key=f"{state['key']}.json", Body=json_output) diff --git a/test/apis/batch/sum/sample.json b/test/apis/batch/sum/sample.json index e7f2f5ed0b..1ba3480b44 100644 --- a/test/apis/batch/sum/sample.json +++ b/test/apis/batch/sum/sample.json @@ -1,3 +1,4 @@ [ - [1, 2, 67, -2, 43] + [1, 2, 67, -2, 43], + [9, 0, 1, 0, 0, -1] ] diff --git a/test/apis/batch/sum/submit.py b/test/apis/batch/sum/submit.py new file mode 100644 index 0000000000..b6baffdab9 --- /dev/null +++ b/test/apis/batch/sum/submit.py @@ -0,0 +1,41 @@ +""" +Typical usage example: + + python submit.py +""" + +from typing import List + +import sys +import json +import requests +import cortex + + +def main(): + # parse args + if len(sys.argv) != 3: + print("usage: python submit.py ") + sys.exit(1) + env_name = sys.argv[1] + dest_s3_dir = sys.argv[2] + + # read sample file + with open("sample.json") as f: + sample_items: List[str] = json.load(f) + + # get batch endpoint + cx = cortex.client(env_name) + batch_endpoint = cx.get_api("sum")["endpoint"] + + # submit job + job_spec = { + "item_list": {"items": sample_items, "batch_size": 1}, + "config": {"dest_s3_dir": dest_s3_dir}, + } + response = requests.post(batch_endpoint, json=job_spec) + print(json.dumps(response.json(), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/test/apis/batch/sum/sum-cpu.dockerfile b/test/apis/batch/sum/sum-cpu.dockerfile new file mode 100644 index 0000000000..264ae82b32 --- /dev/null +++ b/test/apis/batch/sum/sum-cpu.dockerfile @@ -0,0 +1,22 @@ +FROM python:3.8-slim + +# Allow statements and log messages to immediately appear in the logs +ENV PYTHONUNBUFFERED True + +# Install production dependencies +RUN pip install --no-cache-dir \ + "uvicorn[standard]" \ + gunicorn \ + fastapi \ + pydantic \ + boto3==1.17.72 + +# Copy local code to the container image. +COPY ./main.py /app/ +WORKDIR /app/ + +ENV PYTHONPATH=/app +ENV CORTEX_PORT=9000 + +# Run the web service on container startup. +CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/batch/tensorflow/cortex.yaml b/test/apis/batch/tensorflow/cortex.yaml deleted file mode 100644 index 16b3f78387..0000000000 --- a/test/apis/batch/tensorflow/cortex.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- name: image-classifier - kind: BatchAPI - handler: - type: tensorflow - path: handler.py - models: - path: s3://cortex-examples/tensorflow/image-classifier/inception/ - compute: - cpu: 1 diff --git a/test/apis/batch/tensorflow/handler.py b/test/apis/batch/tensorflow/handler.py deleted file mode 100644 index 70f1c0649f..0000000000 --- a/test/apis/batch/tensorflow/handler.py +++ /dev/null @@ -1,58 +0,0 @@ -import requests -import numpy as np -from PIL import Image -from io import BytesIO -import json -import os -import re -import boto3 -import tensorflow as tf - - -class Handler: - def __init__(self, tensorflow_client, config, job_spec): - self.client = tensorflow_client - self.labels = requests.get( - "https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt" - ).text.split("\n")[1:] - - if len(config.get("dest_s3_dir", "")) == 0: - raise Exception("'dest_s3_dir' field was not provided in job submission") - - self.s3 = boto3.client("s3") - - self.bucket, self.key = re.match("s3://(.+?)/(.+)", config["dest_s3_dir"]).groups() - self.key = os.path.join(self.key, job_spec["job_id"]) - - def handle_batch(self, payload, batch_id): - arr_list = [] - - # download and preprocess each image - for image_url in payload: - if image_url.startswith("s3://"): - bucket, image_key = re.match("s3://(.+?)/(.+)", image_url).groups() - image_bytes = self.s3.get_object(Bucket=bucket, Key=image_key)["Body"].read() - else: - image_bytes = requests.get(image_url).content - - decoded_image = np.asarray(Image.open(BytesIO(image_bytes)), dtype=np.float32) / 255 - resized_image = tf.image.resize( - decoded_image, [224, 224], method=tf.image.ResizeMethod.BILINEAR - ) - arr_list.append(resized_image) - - # classify the batch of images - model_input = {"images": np.stack(arr_list, axis=0)} - predictions = self.client.predict(model_input) - - # extract predicted classes - reshaped_predictions = np.reshape(np.array(predictions["classes"]), [-1, len(self.labels)]) - predicted_classes = np.argmax(reshaped_predictions, axis=1) - results = [ - {"url": payload[i], "class": self.labels[class_idx]} - for i, class_idx in enumerate(predicted_classes) - ] - - # save results - json_output = json.dumps(results) - self.s3.put_object(Bucket=self.bucket, Key=f"{self.key}/{batch_id}.json", Body=json_output) diff --git a/test/apis/batch/tensorflow/requirements.txt b/test/apis/batch/tensorflow/requirements.txt deleted file mode 100644 index 7e2fba5e6c..0000000000 --- a/test/apis/batch/tensorflow/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -Pillow diff --git a/test/apis/batch/tensorflow/sample.json b/test/apis/batch/tensorflow/sample.json deleted file mode 100644 index eb45c463fd..0000000000 --- a/test/apis/batch/tensorflow/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -[ - "https://i.imgur.com/PzXprwl.jpg" -] diff --git a/test/apis/grpc/iris-classifier-sklearn/README.md b/test/apis/grpc/iris-classifier-sklearn/README.md deleted file mode 100644 index 50aa786b5b..0000000000 --- a/test/apis/grpc/iris-classifier-sklearn/README.md +++ /dev/null @@ -1,32 +0,0 @@ -## gRPC client - -#### Step 1 - -```bash -pip install grpcio -python -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. iris_classifier.proto -``` - -#### Step 2 - -```python -import cortex -import iris_classifier_pb2 -import iris_classifier_pb2_grpc - -sample = iris_classifier_pb2.Sample( - sepal_length=5.2, - sepal_width=3.6, - petal_length=1.4, - petal_width=0.3 -) - -cx = cortex.client("cortex") -api = cx.get_api("iris-classifier") -grpc_endpoint = api["endpoint"] + ":" + str(api["grpc_ports"]["insecure"]) -channel = grpc.insecure_channel(grpc_endpoint) -stub = iris_classifier_pb2_grpc.HandlerStub(channel) - -response = stub.Predict(sample) -print("prediction:", response.classification) -``` diff --git a/test/apis/grpc/iris-classifier-sklearn/cortex.yaml b/test/apis/grpc/iris-classifier-sklearn/cortex.yaml deleted file mode 100644 index ebb7235cf2..0000000000 --- a/test/apis/grpc/iris-classifier-sklearn/cortex.yaml +++ /dev/null @@ -1,12 +0,0 @@ -- name: iris-classifier - kind: RealtimeAPI - handler: - type: python - path: handler.py - protobuf_path: iris_classifier.proto - config: - bucket: cortex-examples - key: sklearn/iris-classifier/model.pkl - compute: - cpu: 0.2 - mem: 200M diff --git a/test/apis/grpc/iris-classifier-sklearn/expectations.yaml b/test/apis/grpc/iris-classifier-sklearn/expectations.yaml deleted file mode 100644 index 79e7987ac4..0000000000 --- a/test/apis/grpc/iris-classifier-sklearn/expectations.yaml +++ /dev/null @@ -1,16 +0,0 @@ -grpc: - proto_module_pb2: "test_proto/iris_classifier_pb2.py" - proto_module_pb2_grpc: "test_proto/iris_classifier_pb2_grpc.py" - stub_service_name: "Handler" - input_spec: - class_name: "Sample" - input: - sepal_length: 5.2 - sepal_width: 3.6 - petal_length: 1.4 - petal_width: 0.3 - output_spec: - class_name: "Response" - stream: false - output: - classification: "setosa" diff --git a/test/apis/grpc/iris-classifier-sklearn/handler.py b/test/apis/grpc/iris-classifier-sklearn/handler.py deleted file mode 100644 index 6a61c21563..0000000000 --- a/test/apis/grpc/iris-classifier-sklearn/handler.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -import boto3 -from botocore import UNSIGNED -from botocore.client import Config -import pickle - -labels = ["setosa", "versicolor", "virginica"] - - -class Handler: - def __init__(self, config, proto_module_pb2): - s3 = boto3.client("s3") - s3.download_file(config["bucket"], config["key"], "/tmp/model.pkl") - self.model = pickle.load(open("/tmp/model.pkl", "rb")) - self.proto_module_pb2 = proto_module_pb2 - - def Predict(self, payload): - measurements = [ - payload.sepal_length, - payload.sepal_width, - payload.petal_length, - payload.petal_width, - ] - - label_id = self.model.predict([measurements])[0] - return self.proto_module_pb2.Response(classification=labels[label_id]) diff --git a/test/apis/grpc/iris-classifier-sklearn/iris_classifier.proto b/test/apis/grpc/iris-classifier-sklearn/iris_classifier.proto deleted file mode 100644 index 923d6e5f75..0000000000 --- a/test/apis/grpc/iris-classifier-sklearn/iris_classifier.proto +++ /dev/null @@ -1,18 +0,0 @@ -syntax = "proto3"; - -package iris_classifier; - -service Handler { - rpc Predict (Sample) returns (Response); -} - -message Sample { - float sepal_length = 1; - float sepal_width = 2; - float petal_length = 3; - float petal_width = 4; -} - -message Response { - string classification = 1; -} diff --git a/test/apis/grpc/iris-classifier-sklearn/requirements.txt b/test/apis/grpc/iris-classifier-sklearn/requirements.txt deleted file mode 100644 index bbc213cf3e..0000000000 --- a/test/apis/grpc/iris-classifier-sklearn/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -boto3 -scikit-learn==0.21.3 diff --git a/test/apis/grpc/iris-classifier-sklearn/test_proto/iris_classifier_pb2.py b/test/apis/grpc/iris-classifier-sklearn/test_proto/iris_classifier_pb2.py deleted file mode 100644 index 41bb0fccca..0000000000 --- a/test/apis/grpc/iris-classifier-sklearn/test_proto/iris_classifier_pb2.py +++ /dev/null @@ -1,216 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: iris_classifier.proto -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database - -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -DESCRIPTOR = _descriptor.FileDescriptor( - name="iris_classifier.proto", - package="iris_classifier", - syntax="proto3", - serialized_options=None, - create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\x15iris_classifier.proto\x12\x0firis_classifier"^\n\x06Sample\x12\x14\n\x0csepal_length\x18\x01 \x01(\x02\x12\x13\n\x0bsepal_width\x18\x02 \x01(\x02\x12\x14\n\x0cpetal_length\x18\x03 \x01(\x02\x12\x13\n\x0bpetal_width\x18\x04 \x01(\x02""\n\x08Response\x12\x16\n\x0e\x63lassification\x18\x01 \x01(\t2H\n\x07Handler\x12=\n\x07Predict\x12\x17.iris_classifier.Sample\x1a\x19.iris_classifier.Responseb\x06proto3', -) - - -_SAMPLE = _descriptor.Descriptor( - name="Sample", - full_name="iris_classifier.Sample", - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name="sepal_length", - full_name="iris_classifier.Sample.sepal_length", - index=0, - number=1, - type=2, - cpp_type=6, - label=1, - has_default_value=False, - default_value=float(0), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="sepal_width", - full_name="iris_classifier.Sample.sepal_width", - index=1, - number=2, - type=2, - cpp_type=6, - label=1, - has_default_value=False, - default_value=float(0), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="petal_length", - full_name="iris_classifier.Sample.petal_length", - index=2, - number=3, - type=2, - cpp_type=6, - label=1, - has_default_value=False, - default_value=float(0), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - _descriptor.FieldDescriptor( - name="petal_width", - full_name="iris_classifier.Sample.petal_width", - index=3, - number=4, - type=2, - cpp_type=6, - label=1, - has_default_value=False, - default_value=float(0), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - serialized_options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=42, - serialized_end=136, -) - - -_RESPONSE = _descriptor.Descriptor( - name="Response", - full_name="iris_classifier.Response", - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name="classification", - full_name="iris_classifier.Response.classification", - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=b"".decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - serialized_options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=138, - serialized_end=172, -) - -DESCRIPTOR.message_types_by_name["Sample"] = _SAMPLE -DESCRIPTOR.message_types_by_name["Response"] = _RESPONSE -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -Sample = _reflection.GeneratedProtocolMessageType( - "Sample", - (_message.Message,), - { - "DESCRIPTOR": _SAMPLE, - "__module__": "iris_classifier_pb2" - # @@protoc_insertion_point(class_scope:iris_classifier.Sample) - }, -) -_sym_db.RegisterMessage(Sample) - -Response = _reflection.GeneratedProtocolMessageType( - "Response", - (_message.Message,), - { - "DESCRIPTOR": _RESPONSE, - "__module__": "iris_classifier_pb2" - # @@protoc_insertion_point(class_scope:iris_classifier.Response) - }, -) -_sym_db.RegisterMessage(Response) - - -_HANDLER = _descriptor.ServiceDescriptor( - name="Handler", - full_name="iris_classifier.Handler", - file=DESCRIPTOR, - index=0, - serialized_options=None, - create_key=_descriptor._internal_create_key, - serialized_start=174, - serialized_end=246, - methods=[ - _descriptor.MethodDescriptor( - name="Predict", - full_name="iris_classifier.Handler.Predict", - index=0, - containing_service=None, - input_type=_SAMPLE, - output_type=_RESPONSE, - serialized_options=None, - create_key=_descriptor._internal_create_key, - ), - ], -) -_sym_db.RegisterServiceDescriptor(_HANDLER) - -DESCRIPTOR.services_by_name["Handler"] = _HANDLER - -# @@protoc_insertion_point(module_scope) diff --git a/test/apis/grpc/iris-classifier-sklearn/test_proto/iris_classifier_pb2_grpc.py b/test/apis/grpc/iris-classifier-sklearn/test_proto/iris_classifier_pb2_grpc.py deleted file mode 100644 index d321782257..0000000000 --- a/test/apis/grpc/iris-classifier-sklearn/test_proto/iris_classifier_pb2_grpc.py +++ /dev/null @@ -1,79 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -"""Client and server classes corresponding to protobuf-defined services.""" -import grpc - -import iris_classifier_pb2 as iris__classifier__pb2 - - -class HandlerStub(object): - """Missing associated documentation comment in .proto file.""" - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.Predict = channel.unary_unary( - "/iris_classifier.Handler/Predict", - request_serializer=iris__classifier__pb2.Sample.SerializeToString, - response_deserializer=iris__classifier__pb2.Response.FromString, - ) - - -class HandlerServicer(object): - """Missing associated documentation comment in .proto file.""" - - def Predict(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details("Method not implemented!") - raise NotImplementedError("Method not implemented!") - - -def add_HandlerServicer_to_server(servicer, server): - rpc_method_handlers = { - "Predict": grpc.unary_unary_rpc_method_handler( - servicer.Predict, - request_deserializer=iris__classifier__pb2.Sample.FromString, - response_serializer=iris__classifier__pb2.Response.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - "iris_classifier.Handler", rpc_method_handlers - ) - server.add_generic_rpc_handlers((generic_handler,)) - - -# This class is part of an EXPERIMENTAL API. -class Handler(object): - """Missing associated documentation comment in .proto file.""" - - @staticmethod - def Predict( - request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None, - ): - return grpc.experimental.unary_unary( - request, - target, - "/iris_classifier.Handler/Predict", - iris__classifier__pb2.Sample.SerializeToString, - iris__classifier__pb2.Response.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - ) diff --git a/test/apis/grpc/prime-number-generator/README.md b/test/apis/grpc/prime-number-generator/README.md deleted file mode 100644 index 53bcf7da72..0000000000 --- a/test/apis/grpc/prime-number-generator/README.md +++ /dev/null @@ -1,25 +0,0 @@ -## Prime number generator - -#### Step 1 - -```bash -pip install grpcio -python -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. generator.proto -``` - -#### Step 2 - -```python -import cortex -import generator_pb2 -import generator_pb2_grpc - -cx = cortex.client("cortex") -api = cx.get_api("prime-generator") -grpc_endpoint = api["endpoint"] + ":" + str(api["grpc_ports"]["insecure"]) - -channel = grpc.insecure_channel(grpc_endpoint) -stub = generator_pb2_grpc.GeneratorServicer(channel) -for r in stub.Predict(generator_pb2.Input(prime_numbers_to_generate=5)): - print(r) -``` diff --git a/test/apis/grpc/prime-number-generator/cortex.yaml b/test/apis/grpc/prime-number-generator/cortex.yaml deleted file mode 100644 index 38cba123c3..0000000000 --- a/test/apis/grpc/prime-number-generator/cortex.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- name: prime-generator - kind: RealtimeAPI - handler: - type: python - path: generator.py - protobuf_path: generator.proto - compute: - cpu: 200m - mem: 200Mi diff --git a/test/apis/grpc/prime-number-generator/expectations.yaml b/test/apis/grpc/prime-number-generator/expectations.yaml deleted file mode 100644 index 15265c06a2..0000000000 --- a/test/apis/grpc/prime-number-generator/expectations.yaml +++ /dev/null @@ -1,17 +0,0 @@ -grpc: - proto_module_pb2: "test_proto/generator_pb2.py" - proto_module_pb2_grpc: "test_proto/generator_pb2_grpc.py" - stub_service_name: "Generator" - input_spec: - class_name: "Input" - input: - prime_numbers_to_generate: 5 - output_spec: - class_name: "Output" - stream: true - output: - - prime_number: 2 - - prime_number: 3 - - prime_number: 5 - - prime_number: 7 - - prime_number: 11 diff --git a/test/apis/grpc/prime-number-generator/generator.proto b/test/apis/grpc/prime-number-generator/generator.proto deleted file mode 100644 index 6be4f808ac..0000000000 --- a/test/apis/grpc/prime-number-generator/generator.proto +++ /dev/null @@ -1,15 +0,0 @@ -syntax = "proto3"; - -package prime_generator; - -service Generator { - rpc Predict (Input) returns (stream Output); -} - -message Input { - int64 prime_numbers_to_generate = 1; -} - -message Output { - int64 prime_number = 1; -} diff --git a/test/apis/grpc/prime-number-generator/generator.py b/test/apis/grpc/prime-number-generator/generator.py deleted file mode 100644 index 9cd7146c0c..0000000000 --- a/test/apis/grpc/prime-number-generator/generator.py +++ /dev/null @@ -1,28 +0,0 @@ -from collections import defaultdict - - -class Handler: - def __init__(self, config, proto_module_pb2): - self.proto_module_pb2 = proto_module_pb2 - - def Predict(self, payload): - prime_numbers_to_generate: int = payload.prime_numbers_to_generate - for prime_number in self.gen_primes(): - if prime_numbers_to_generate == 0: - break - prime_numbers_to_generate -= 1 - yield self.proto_module_pb2.Output(prime_number=prime_number) - - def gen_primes(self, limit=None): - """Sieve of Eratosthenes""" - not_prime = defaultdict(list) - num = 2 - while limit is None or num <= limit: - if num in not_prime: - for prime in not_prime[num]: - not_prime[prime + num].append(prime) - del not_prime[num] - else: - yield num - not_prime[num * num] = [num] - num += 1 diff --git a/test/apis/grpc/prime-number-generator/test_proto/generator_pb2.py b/test/apis/grpc/prime-number-generator/test_proto/generator_pb2.py deleted file mode 100644 index bf458f4c80..0000000000 --- a/test/apis/grpc/prime-number-generator/test_proto/generator_pb2.py +++ /dev/null @@ -1,159 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: generator.proto -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database - -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -DESCRIPTOR = _descriptor.FileDescriptor( - name="generator.proto", - package="prime_generator", - syntax="proto3", - serialized_options=None, - create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\x0fgenerator.proto\x12\x0fprime_generator"*\n\x05Input\x12!\n\x19prime_numbers_to_generate\x18\x01 \x01(\x03"\x1e\n\x06Output\x12\x14\n\x0cprime_number\x18\x01 \x01(\x03\x32I\n\tGenerator\x12<\n\x07Predict\x12\x16.prime_generator.Input\x1a\x17.prime_generator.Output0\x01\x62\x06proto3', -) - - -_INPUT = _descriptor.Descriptor( - name="Input", - full_name="prime_generator.Input", - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name="prime_numbers_to_generate", - full_name="prime_generator.Input.prime_numbers_to_generate", - index=0, - number=1, - type=3, - cpp_type=2, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - serialized_options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=36, - serialized_end=78, -) - - -_OUTPUT = _descriptor.Descriptor( - name="Output", - full_name="prime_generator.Output", - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name="prime_number", - full_name="prime_generator.Output.prime_number", - index=0, - number=1, - type=3, - cpp_type=2, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - serialized_options=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - serialized_options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=80, - serialized_end=110, -) - -DESCRIPTOR.message_types_by_name["Input"] = _INPUT -DESCRIPTOR.message_types_by_name["Output"] = _OUTPUT -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -Input = _reflection.GeneratedProtocolMessageType( - "Input", - (_message.Message,), - { - "DESCRIPTOR": _INPUT, - "__module__": "generator_pb2" - # @@protoc_insertion_point(class_scope:prime_generator.Input) - }, -) -_sym_db.RegisterMessage(Input) - -Output = _reflection.GeneratedProtocolMessageType( - "Output", - (_message.Message,), - { - "DESCRIPTOR": _OUTPUT, - "__module__": "generator_pb2" - # @@protoc_insertion_point(class_scope:prime_generator.Output) - }, -) -_sym_db.RegisterMessage(Output) - - -_GENERATOR = _descriptor.ServiceDescriptor( - name="Generator", - full_name="prime_generator.Generator", - file=DESCRIPTOR, - index=0, - serialized_options=None, - create_key=_descriptor._internal_create_key, - serialized_start=112, - serialized_end=185, - methods=[ - _descriptor.MethodDescriptor( - name="Predict", - full_name="prime_generator.Generator.Predict", - index=0, - containing_service=None, - input_type=_INPUT, - output_type=_OUTPUT, - serialized_options=None, - create_key=_descriptor._internal_create_key, - ), - ], -) -_sym_db.RegisterServiceDescriptor(_GENERATOR) - -DESCRIPTOR.services_by_name["Generator"] = _GENERATOR - -# @@protoc_insertion_point(module_scope) diff --git a/test/apis/grpc/prime-number-generator/test_proto/generator_pb2_grpc.py b/test/apis/grpc/prime-number-generator/test_proto/generator_pb2_grpc.py deleted file mode 100644 index 1eb11b6083..0000000000 --- a/test/apis/grpc/prime-number-generator/test_proto/generator_pb2_grpc.py +++ /dev/null @@ -1,79 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -"""Client and server classes corresponding to protobuf-defined services.""" -import grpc - -import generator_pb2 as generator__pb2 - - -class GeneratorStub(object): - """Missing associated documentation comment in .proto file.""" - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.Predict = channel.unary_stream( - "/prime_generator.Generator/Predict", - request_serializer=generator__pb2.Input.SerializeToString, - response_deserializer=generator__pb2.Output.FromString, - ) - - -class GeneratorServicer(object): - """Missing associated documentation comment in .proto file.""" - - def Predict(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details("Method not implemented!") - raise NotImplementedError("Method not implemented!") - - -def add_GeneratorServicer_to_server(servicer, server): - rpc_method_handlers = { - "Predict": grpc.unary_stream_rpc_method_handler( - servicer.Predict, - request_deserializer=generator__pb2.Input.FromString, - response_serializer=generator__pb2.Output.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - "prime_generator.Generator", rpc_method_handlers - ) - server.add_generic_rpc_handlers((generic_handler,)) - - -# This class is part of an EXPERIMENTAL API. -class Generator(object): - """Missing associated documentation comment in .proto file.""" - - @staticmethod - def Predict( - request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None, - ): - return grpc.experimental.unary_stream( - request, - target, - "/prime_generator.Generator/Predict", - generator__pb2.Input.SerializeToString, - generator__pb2.Output.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - ) diff --git a/test/apis/keras/document-denoiser/README.md b/test/apis/keras/document-denoiser/README.md deleted file mode 100644 index 3a46d3ea16..0000000000 --- a/test/apis/keras/document-denoiser/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# Clean Dirty Documents w/ Autoencoders - -This example model cleans text documents of anything that isn't text (aka noise): coffee stains, old wear artifacts, etc. You can inspect the notebook that has been used to train the model [here](trainer.ipynb). - -Here's a collage of input texts and predictions. - -![Imgur](https://i.imgur.com/M4Mjz2l.jpg) - -*Figure 1 - The dirty documents are on the left side and the cleaned ones are on the right* - -## Sample Prediction - -Once this model is deployed, get the API endpoint by running `cortex get document-denoiser`. - -Now let's take a sample image like this one. - -![Imgur](https://i.imgur.com/JJLfFxB.png) - -Export the endpoint & the image's URL by running -```bash -export ENDPOINT= -export IMAGE_URL=https://i.imgur.com/JJLfFxB.png -``` - -Then run the following piped commands -```bash -curl "${ENDPOINT}" -X POST -H "Content-Type: application/json" -d '{"url":"'${IMAGE_URL}'"}' | -sed 's/"//g' | -base64 -d > prediction.png -``` - -Once this has run, we'll see a `prediction.png` file saved to the disk. This is the result. - -![Imgur](https://i.imgur.com/PRB2oS8.png) - -As it can be seen, the text document has been cleaned of any noise. Success! - ---- - -Here's a short list of URLs of other text documents in image format that can be cleaned using this model. Export these links to `IMAGE_URL` variable: - -* https://i.imgur.com/6COQ46f.png -* https://i.imgur.com/alLI83b.png -* https://i.imgur.com/QVoSTuu.png diff --git a/test/apis/keras/document-denoiser/cortex.yaml b/test/apis/keras/document-denoiser/cortex.yaml deleted file mode 100644 index 3b61ed1375..0000000000 --- a/test/apis/keras/document-denoiser/cortex.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- name: document-denoiser - kind: RealtimeAPI - handler: - type: python - path: handler.py - config: - model: s3://cortex-examples/keras/document-denoiser/model.h5 - resize_shape: [540, 260] - compute: - cpu: 1 diff --git a/test/apis/keras/document-denoiser/dependencies.sh b/test/apis/keras/document-denoiser/dependencies.sh deleted file mode 100644 index 0577b68228..0000000000 --- a/test/apis/keras/document-denoiser/dependencies.sh +++ /dev/null @@ -1 +0,0 @@ -apt-get update && apt-get install -y libsm6 libxext6 libxrender-dev diff --git a/test/apis/keras/document-denoiser/handler.py b/test/apis/keras/document-denoiser/handler.py deleted file mode 100644 index 10f9e4cac6..0000000000 --- a/test/apis/keras/document-denoiser/handler.py +++ /dev/null @@ -1,79 +0,0 @@ -import boto3, base64, cv2, re, os, requests -from botocore import UNSIGNED -from botocore.client import Config -import numpy as np -from tensorflow.keras.models import load_model - - -def get_url_image(url_image): - """ - Get numpy image from URL image. - """ - resp = requests.get(url_image, stream=True).raw - image = np.asarray(bytearray(resp.read()), dtype="uint8") - image = cv2.imdecode(image, cv2.IMREAD_GRAYSCALE) - return image - - -def image_to_png_nparray(image): - """ - Convert numpy image to jpeg numpy vector. - """ - is_success, im_buf_arr = cv2.imencode(".png", image) - return im_buf_arr - - -def image_to_png_bytes(image): - """ - Convert numpy image to bytes-encoded png image. - """ - buf = image_to_png_nparray(image) - byte_im = buf.tobytes() - return byte_im - - -class Handler: - def __init__(self, config): - # download the model - bucket, key = re.match("s3://(.+?)/(.+)", config["model"]).groups() - model_path = os.path.join("/tmp/model.h5") - s3 = boto3.client("s3") - s3.download_file(bucket, key, model_path) - - # load the model - self.model = load_model(model_path) - - # resize shape (width, height) - self.resize_shape = tuple(config["resize_shape"]) - - def handle_post(self, payload): - # download image - img_url = payload["url"] - image = get_url_image(img_url) - resized = cv2.resize(image, self.resize_shape) - - # prediction - pred = self.make_prediction(resized) - - # image represented in bytes - byte_im = image_to_png_bytes(pred) - - # encode image - image_enc = base64.b64encode(byte_im).decode("utf-8") - - return image_enc - - def make_prediction(self, img): - """ - Make prediction on image. - """ - processed = img / 255.0 - processed = np.expand_dims(processed, 0) - processed = np.expand_dims(processed, 3) - pred = self.model.predict(processed) - pred = np.squeeze(pred, 3) - pred = np.squeeze(pred, 0) - out_img = pred * 255 - out_img[out_img > 255.0] = 255.0 - out_img = out_img.astype(np.uint8) - return out_img diff --git a/test/apis/keras/document-denoiser/requirements.txt b/test/apis/keras/document-denoiser/requirements.txt deleted file mode 100644 index bd5f105631..0000000000 --- a/test/apis/keras/document-denoiser/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -numpy==1.18.0 -requests==2.22.0 -opencv-python==4.1.2.30 -keras==2.3.1 -h5py==2.10.0 -tensorflow-cpu==2.3.0 diff --git a/test/apis/keras/document-denoiser/sample.json b/test/apis/keras/document-denoiser/sample.json deleted file mode 100644 index 651595f4fb..0000000000 --- a/test/apis/keras/document-denoiser/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "url": "https://i.imgur.com/JJLfFxB.png" -} diff --git a/test/apis/keras/document-denoiser/trainer.ipynb b/test/apis/keras/document-denoiser/trainer.ipynb deleted file mode 100644 index d557fdcde4..0000000000 --- a/test/apis/keras/document-denoiser/trainer.ipynb +++ /dev/null @@ -1,617 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training a Document Denoiser Model with AutoEncoders\n" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "metadata": {}, - "outputs": [], - "source": [ - "import keras\n", - "import cv2\n", - "import numpy as np\n", - "import pandas as pd\n", - "import seaborn as sns\n", - "import os\n", - "import ntpath\n", - "from glob import glob\n", - "from matplotlib.pyplot import imshow\n", - "from sklearn.model_selection import train_test_split\n", - "from keras.preprocessing.image import ImageDataGenerator\n", - "from keras.models import Sequential, Model, load_model\n", - "from keras.layers import Activation, Flatten, Dropout, SpatialDropout2D, Conv2D, UpSampling2D, MaxPooling2D, add, concatenate, Input, BatchNormalization\n", - "from keras.backend import set_image_data_format\n", - "from keras.utils import plot_model" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Download Dataset\n", - "\n", - "Download the dataset from [kaggle (denoising dirty documents)](https://www.kaggle.com/c/denoising-dirty-documents/data). You will need to be logged in to be able to download the data.\n", - "\n", - "Once downloaded run the following commands" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!unzip denoising-dirty-documents.zip && rm denoising-dirty-documents.zip\n", - "!mv denoising-dirty-documents/*.zip . && rm -rf denoising-dirty-documents\n", - "!unzip '*.zip' > /dev/null && rm *.zip" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Define the Data Generator\n", - "\n", - "Include data augmentation because the dataset is rather small." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "x_dirty = sorted(glob(\"train/*.png\"))\n", - "x_cleaned = sorted(glob(\"train_cleaned/*.png\"))\n", - "x_test = sorted(glob(\"test/*.png\"))\n", - "input_shape = (260, 540)\n", - "height = input_shape[0]\n", - "width = input_shape[1]" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "x_train, x_valid, y_train, y_valid = train_test_split(x_dirty, x_cleaned, test_size=0.20)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "set_image_data_format(\"channels_last\")" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "def model_train_generator(x_train, y_train, epochs, batch_size, resize_shape):\n", - " white_fill = 1.0\n", - " datagen = ImageDataGenerator(\n", - " rotation_range=180,\n", - " width_shift_range=0.2,\n", - " height_shift_range=0.2,\n", - " zoom_range=0.3,\n", - " fill_mode=\"constant\",\n", - " cval=white_fill,\n", - " horizontal_flip=True,\n", - " vertical_flip=True,\n", - " )\n", - " \n", - " for _ in range(epochs):\n", - " for x_file, y_file in zip(x_train, y_train):\n", - " x_img = cv2.imread(x_file, cv2.IMREAD_GRAYSCALE) / 255.0\n", - " y_img = cv2.imread(y_file, cv2.IMREAD_GRAYSCALE) / 255.0\n", - " \n", - " xs = []\n", - " ys = []\n", - " for i in range(batch_size):\n", - " if i == 0:\n", - " x = x_img\n", - " y = y_img\n", - " else:\n", - " params = datagen.get_random_transform(img_shape=x_img.shape)\n", - " x = datagen.apply_transform(np.expand_dims(x_img, 2), params)\n", - " y = datagen.apply_transform(np.expand_dims(y_img, 2), params)\n", - " x = cv2.resize(x, resize_shape[::-1], interpolation=cv2.INTER_AREA)\n", - " y = cv2.resize(y, resize_shape[::-1], interpolation=cv2.INTER_AREA)\n", - " x = np.expand_dims(x, 2)\n", - " y = np.expand_dims(y, 2)\n", - " xs.append(x)\n", - " ys.append(y)\n", - " xs_imgs = np.array(xs)\n", - " ys_imgs = np.array(ys)\n", - " yield (xs_imgs, ys_imgs)\n", - "\n", - "def model_valid_generator(x_valid, y_valid, epochs, resize_shape):\n", - " xs = []\n", - " ys = []\n", - " for x_file, y_file in zip(x_valid, y_valid):\n", - " x_img = cv2.imread(x_file, cv2.IMREAD_GRAYSCALE) / 255.0\n", - " y_img = cv2.imread(y_file, cv2.IMREAD_GRAYSCALE) / 255.0\n", - " x = cv2.resize(x_img, resize_shape[::-1], interpolation=cv2.INTER_AREA)\n", - " y = cv2.resize(y_img, resize_shape[::-1], interpolation=cv2.INTER_AREA)\n", - " x = np.expand_dims(x, 2)\n", - " x = np.expand_dims(x, 0)\n", - " y = np.expand_dims(y, 2)\n", - " y = np.expand_dims(y, 0)\n", - " xs.append(x)\n", - " ys.append(y)\n", - " \n", - " for _ in range(epochs):\n", - " for xs_img, ys_img in zip(xs, ys):\n", - " yield (xs_img, ys_img)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create the Model" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "def create_encoder(input_shape):\n", - " inp = Input(shape=input_shape)\n", - " x = Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), \n", - " input_shape=input_shape, activation=\"relu\", padding=\"same\")(inp)\n", - " x = BatchNormalization()(x)\n", - " x = MaxPooling2D(pool_size=(2,2))(x)\n", - " \n", - " x = Conv2D(filters=32, kernel_size=(3,3), strides=(1,1), \n", - " activation=\"relu\", padding=\"same\")(x)\n", - " x = BatchNormalization()(x)\n", - "\n", - " return inp, x" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "def create_decoder(inp):\n", - " x = Conv2D(filters=32, kernel_size=(3,3), strides=(1,1), activation=\"relu\",\n", - " padding=\"same\")(inp)\n", - " x = BatchNormalization()(x)\n", - " x = UpSampling2D(size=(2,2))(x)\n", - " \n", - " x = Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), \n", - " activation=\"relu\", padding=\"same\")(x)\n", - " x = BatchNormalization()(x)\n", - " \n", - " x = Conv2D(filters=1, kernel_size=(1,1), strides=(1,1), \n", - " activation=\"sigmoid\", padding=\"same\")(x)\n", - " x = BatchNormalization()(x)\n", - " \n", - " return inp, x" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "def create_autoencoder(input_shape):\n", - " enc_inp, encoder = create_encoder(input_shape)\n", - " dec_inp, autoencoder = create_decoder(encoder)\n", - " model = Model(inputs=[enc_inp], outputs=[autoencoder], name='AutoEncoder')\n", - " \n", - " return model" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "WARNING:tensorflow:From C:\\Users\\OboTh\\Anaconda3\\envs\\lightweight-gpu-python\\lib\\site-packages\\tensorflow_core\\python\\ops\\resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "If using Keras pass *_constraint arguments to layers.\n", - "WARNING:tensorflow:From C:\\Users\\OboTh\\Anaconda3\\envs\\lightweight-gpu-python\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:4070: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n", - "\n", - "Model: \"AutoEncoder\"\n", - "_________________________________________________________________\n", - "Layer (type) Output Shape Param # \n", - "=================================================================\n", - "input_1 (InputLayer) (None, 260, 540, 1) 0 \n", - "_________________________________________________________________\n", - "conv2d_1 (Conv2D) (None, 260, 540, 64) 640 \n", - "_________________________________________________________________\n", - "batch_normalization_1 (Batch (None, 260, 540, 64) 256 \n", - "_________________________________________________________________\n", - "max_pooling2d_1 (MaxPooling2 (None, 130, 270, 64) 0 \n", - "_________________________________________________________________\n", - "conv2d_2 (Conv2D) (None, 130, 270, 32) 18464 \n", - "_________________________________________________________________\n", - "batch_normalization_2 (Batch (None, 130, 270, 32) 128 \n", - "_________________________________________________________________\n", - "conv2d_3 (Conv2D) (None, 130, 270, 32) 9248 \n", - "_________________________________________________________________\n", - "batch_normalization_3 (Batch (None, 130, 270, 32) 128 \n", - "_________________________________________________________________\n", - "up_sampling2d_1 (UpSampling2 (None, 260, 540, 32) 0 \n", - "_________________________________________________________________\n", - "conv2d_4 (Conv2D) (None, 260, 540, 64) 18496 \n", - "_________________________________________________________________\n", - "batch_normalization_4 (Batch (None, 260, 540, 64) 256 \n", - "_________________________________________________________________\n", - "conv2d_5 (Conv2D) (None, 260, 540, 1) 65 \n", - "_________________________________________________________________\n", - "batch_normalization_5 (Batch (None, 260, 540, 1) 4 \n", - "=================================================================\n", - "Total params: 47,685\n", - "Trainable params: 47,299\n", - "Non-trainable params: 386\n", - "_________________________________________________________________\n" - ] - } - ], - "source": [ - "model = create_autoencoder((height, width, 1))\n", - "model.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "model.compile(optimizer='adam', loss='mse')\n", - "epochs = 20\n", - "batch_size = 8\n", - "samples = len(x_train)\n", - "validation_samples = len(x_valid)\n", - "train_generator = model_train_generator(x_train, y_train, epochs=epochs, batch_size=batch_size, resize_shape=(height, width))\n", - "valid_generator = model_valid_generator(x_valid, y_valid, epochs=epochs, resize_shape=(height, width))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Train the AutoEncoder Model" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "WARNING:tensorflow:From C:\\Users\\OboTh\\Anaconda3\\envs\\lightweight-gpu-python\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n", - "\n", - "Epoch 1/20\n", - "115/115 [==============================] - 49s 429ms/step - loss: 1.2062 - val_loss: 0.1817\n", - "Epoch 2/20\n", - "115/115 [==============================] - 43s 373ms/step - loss: 0.5792 - val_loss: 0.1720\n", - "Epoch 3/20\n", - "115/115 [==============================] - 43s 373ms/step - loss: 0.4297 - val_loss: 0.1399\n", - "Epoch 4/20\n", - "115/115 [==============================] - 43s 375ms/step - loss: 0.3160 - val_loss: 0.1023\n", - "Epoch 5/20\n", - "115/115 [==============================] - 44s 385ms/step - loss: 0.2276 - val_loss: 0.0609\n", - "Epoch 6/20\n", - "115/115 [==============================] - 44s 379ms/step - loss: 0.1599 - val_loss: 0.0292\n", - "Epoch 7/20\n", - "115/115 [==============================] - 43s 376ms/step - loss: 0.1091 - val_loss: 0.0112\n", - "Epoch 8/20\n", - "115/115 [==============================] - 43s 376ms/step - loss: 0.0730 - val_loss: 0.0074\n", - "Epoch 9/20\n", - "115/115 [==============================] - 44s 381ms/step - loss: 0.0473 - val_loss: 0.0055\n", - "Epoch 10/20\n", - "115/115 [==============================] - 45s 393ms/step - loss: 0.0301 - val_loss: 0.0047\n", - "Epoch 11/20\n", - "115/115 [==============================] - 45s 387ms/step - loss: 0.0189 - val_loss: 0.0041\n", - "Epoch 12/20\n", - "115/115 [==============================] - 43s 376ms/step - loss: 0.0118 - val_loss: 0.0042\n", - "Epoch 13/20\n", - "115/115 [==============================] - 44s 380ms/step - loss: 0.0075 - val_loss: 0.0061\n", - "Epoch 14/20\n", - "115/115 [==============================] - 43s 377ms/step - loss: 0.0051 - val_loss: 0.0048\n", - "Epoch 15/20\n", - "115/115 [==============================] - 43s 378ms/step - loss: 0.0037 - val_loss: 0.0045\n", - "Epoch 16/20\n", - "115/115 [==============================] - 43s 373ms/step - loss: 0.0029 - val_loss: 0.0045\n", - "Epoch 17/20\n", - "115/115 [==============================] - 44s 378ms/step - loss: 0.0025 - val_loss: 0.0048\n", - "Epoch 18/20\n", - "115/115 [==============================] - 43s 375ms/step - loss: 0.0023 - val_loss: 0.0047\n", - "Epoch 19/20\n", - "115/115 [==============================] - 43s 376ms/step - loss: 0.0022 - val_loss: 0.0043\n", - "Epoch 20/20\n", - "115/115 [==============================] - 44s 380ms/step - loss: 0.0021 - val_loss: 0.0042\n" - ] - } - ], - "source": [ - "hist_obj = model.fit_generator(train_generator, validation_data=valid_generator, validation_steps=validation_samples, steps_per_epoch=samples, epochs=epochs, shuffle=True) " - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEGCAYAAAB1iW6ZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU5d338c9vJpMESMIWlrAvgrIpanCtqLRlq2u1iqJWa+WhLtW+Krf6tFXv2t59bG9tbWu11lK1dQGXupRNa1txl4DsICKyhDWsYQtJZq7njzOBELJMYCYnM/N9v17zmplzrpn5zWH45sw117mOOecQEZHkF/C7ABERiQ8FuohIilCgi4ikCAW6iEiKUKCLiKSIDL9eOD8/3/Xq1cuvlxcRSUpz587d6pzrUNs63wK9V69eFBUV+fXyIiJJyczW1LVOXS4iIilCgS4ikiIU6CIiKcK3PnQRSU8VFRUUFxdTVlbmdynNWnZ2Nt26dSMUCsX8GAW6iDSp4uJicnNz6dWrF2bmdznNknOObdu2UVxcTO/evWN+nLpcRKRJlZWV0b59e4V5PcyM9u3bN/pbTIOBbmaTzWyLmS2uY/14M1sYvXxgZic1qgIRSTsK84YdzTaKZQ/9KWB0Peu/BM51zp0IPAA80egqRETkmDUY6M652cD2etZ/4JzbEb37EdAtTrXVbuNC+PNI2PBpQl9GRCTZxLsP/UZgRl0rzWyCmRWZWVFJScnRvUKoJaz7GLYsO8oSRURil5OTU+e61atXM3jw4Caspn5xC3QzOx8v0O+qq41z7gnnXKFzrrBDh1qnImhY254QCMHWFUf3eBGRFBWXYYtmdiLwJDDGObctHs9Zp2AI2vWBrZ8n9GVEJPH++40lLN1QGtfnHNglj/suHFTn+rvuuouePXty8803A3D//fdjZsyePZsdO3ZQUVHBz372My6++OJGvW5ZWRnf+973KCoqIiMjg4cffpjzzz+fJUuWcMMNN1BeXk4kEuHll1+mS5cuXHHFFRQXFxMOh/nJT37ClVdeeUzvG+IQ6GbWA3gFuNY51zS7zfn9tIcuIkdl3Lhx3HHHHQcDferUqcycOZMf/OAH5OXlsXXrVs444wwuuuiiRo00efTRRwFYtGgRy5cvZ+TIkaxYsYLHH3+c22+/nfHjx1NeXk44HGb69Ol06dKFadOmAbBr1664vLcGA93MngfOA/LNrBi4DwgBOOceB+4F2gN/iL75SudcYVyqq0t+f1gxC8IV3h67iCSl+vakE+Xkk09my5YtbNiwgZKSEtq2bUtBQQE/+MEPmD17NoFAgPXr17N582Y6d+4c8/O+99573HbbbQCccMIJ9OzZkxUrVnDmmWfy85//nOLiYr75zW/Sr18/hgwZwp133sldd93FBRdcwDnnnBOX99ZgoDvnrmpg/XeB78almlgNuxGGjgcLNunLikhquPzyy3nppZfYtGkT48aN49lnn6WkpIS5c+cSCoXo1atXow/qcc7Vuvzqq6/m9NNPZ9q0aYwaNYonn3ySESNGMHfuXKZPn84999zDyJEjuffee4/5fSXnof+tEzsyUkRS27hx47jpppvYunUr77zzDlOnTqVjx46EQiH+/e9/s2ZNnVOO12n48OE8++yzjBgxghUrVrB27VqOP/54Vq1aRZ8+ffj+97/PqlWrWLhwISeccALt2rXjmmuuIScnh6eeeiou7ys5Az0Shhl3Qc+zYPA3/a5GRJLMoEGD2L17N127dqWgoIDx48dz4YUXUlhYyNChQznhhBMa/Zw333wzEydOZMiQIWRkZPDUU0+RlZXFlClT+Nvf/kYoFKJz587ce++9zJkzh0mTJhEIBAiFQjz22GNxeV9W19eERCssLHTHdMai/+0Px30dLnk0fkWJSMItW7aMAQMG+F1GUqhtW5nZ3Lp+p0zeybny+2uki4hINcnZ5QLQ/jhY8ndwDjTRj4gk0KJFi7j22msPW5aVlcXHH3/sU0W1S95Az+8PZTth71bIOcqjTkVEYjBkyBDmz5/vdxkNSu4uF4BtOmJURASSOdC7ngKXTz4U7CIiaS55u1xatoPBl/ldhYhIs5G8e+gAy6fDvL/6XYWIJJn6psRNZskd6ItehHcf8rsKEZFmIbkDPb8/7FwDFY2bc0FEBLz5VyZNmsTgwYMZMmQIU6ZMAWDjxo0MHz6coUOHMnjwYN59913C4TDXX3/9wba//vWvfa7+SMnbhw7eNLouAttXQaeBflcjIkfjL9+offkN3tSyzLgbNi06cv3oX0DBifDpszD/uSMfF4NXXnmF+fPns2DBArZu3cqwYcMYPnw4zz33HKNGjeJHP/oR4XCYffv2MX/+fNavX8/ixYsB2LlzZ8yv01SSfA+9n3etI0ZF5Ci89957XHXVVQSDQTp16sS5557LnDlzGDZsGH/5y1+4//77WbRoEbm5ufTp04dVq1Zx2223MXPmTPLy8vwu/wjJvYfe/jjvWmcvEkleDe1Rj/l/9a8/ebx3OQp1zWU1fPhwZs+ezbRp07j22muZNGkS1113HQsWLGDWrFk8+uijTJ06lcmTJx/V6yZKcu+hZ7aCET/2Zl0UEWmk4cOHM2XKFMLhMCUlJcyePZvTTjuNNWvW0LFjR2666SZuvPFG5s2bx9atW4lEIlx22WU88MADzJs3z+/yj5Dce+gAwyf5XYGIJKlLL72UDz/8kJNOOgkz45e//CWdO3fm6aef5le/+hWhUIicnByeeeYZ1q9fzw033EAkEgHgF7/4hc/VHyl5p8+tsnMdrPvYO8hIk3SJNHuaPjd26TN9bpXPZ8HLN8LujX5XIiLiq+QP9PYa6SIiAqkQ6FWTc2mki0jS8KurN5kczTZK/kDP7QyZudpDF0kS2dnZbNu2TaFeD+cc27ZtIzs7u1GPS/5RLmbeAUbaQxdJCt26daO4uJiSkhK/S2nWsrOz6datW6Mek/yBDjDoEqjY73cVIhKDUChE7969/S4jJTUY6GY2GbgA2OKcG1zLegMeAcYC+4DrnXNNO+L+7Nub9OVERJqjWPrQnwJG17N+DNAvepkAPHbsZTVSuNLrctm/o8lfWkSkuWgw0J1zs4Ht9TS5GHjGeT4C2phZQbwKjMm2z+H3hfD5P5v0ZUVEmpN4jHLpCqyrdr84uuwIZjbBzIrMrCiuP4i06wMW0EgXEUlr8Qj02o63r3U8knPuCedcoXOusEOHDnF46aiMLGjbS4EuImktHoFeDHSvdr8bsCEOz9s4+f1h28omf1kRkeYiHoH+OnCdec4Adjnnmn5ilfx+XqBHwk3+0iIizUEswxafB84D8s2sGLgPCAE45x4HpuMNWVyJN2zxhkQVW6+Cod6lbBe0bOdLCSIifkr+6XNFRNJIak+fW10kAuX7/K5CRMQXqRXoj5wIs+7xuwoREV+kVqDnddEkXSKStlIr0PP7aSy6iKStFAv0/rC3RHO6iEhaSr1AB9iqA4xEJP2kXqCHWsLeLX5XIiLS5FLjBBdV2vWBe9ZDILX+TomIxCK1At3Mu4iIpKHU25V9+6fw55F+VyEi0uRSL9Cdg/VzIVzhdyUiIk0q9QI9vz9EKmHHar8rERFpUqkZ6KADjEQk7aRgoB/nXSvQRSTNpF6gZ7eGnM6a00VE0k5qDVusctPbkNPJ7ypERJpUagZ6625+VyAi0uRSr8sFYPV78MwlsKfE70pERJpMagZ6ZRms+rd+GBWRtJKagV41dHGbfhgVkfSRmoGe1w0yWmiki4ikldQM9EDAG4+uLhcRSSOpGejgdbso0EUkjaTmsEWAc+705nQREUkTMe2hm9loM/vMzFaa2d21rG9tZm+Y2QIzW2JmN8S/1EbqNBAKTvS7ChGRJtNgoJtZEHgUGAMMBK4ys4E1mt0CLHXOnQScBzxkZplxrrVx9u+Et+6DtR/5WoaISFOJZQ/9NGClc26Vc64ceAG4uEYbB+SamQE5wHbA3/6OYCa8/xv4cravZYiINJVYAr0rsK7a/eLosup+DwwANgCLgNudc5GaT2RmE8ysyMyKSkoSfBRnZkto3UNDF0UkbcQS6LWdpNPVuD8KmA90AYYCvzezvCMe5NwTzrlC51xhhw4dGl1so+X300gXEUkbsQR6MdC92v1ueHvi1d0AvOI8K4EvgRPiU+IxyO/v7aG7mn9/RERSTyyBPgfoZ2a9oz90jgNer9FmLfBVADPrBBwPrIpnoUcl/zio2AulNf/+iIikngbHoTvnKs3sVmAWEAQmO+eWmNnE6PrHgQeAp8xsEV4XzV3Oua0JrDs2fc6Hi34Hma38rkREJOHM+dQdUVhY6IqKinx5bRGRZGVmc51zhbWtS91D/6ssnwYr3vS7ChGRhEvdQ/+rzP5fyMqF/iP9rkREJKFSfw89vz9sW+l3FSIiCZcGgd4PStfDgd1+VyIiklBpEOhVZy/SXrqIpLb0CXRNASAiKS71A71dHxj2XWjb2+9KREQSKvVHuWRkwjce8rsKEZGES/09dIBd6zWNroikvPQI9I8fg79dDpGw35WIiCRMegR6fn8IH4Cda/2uREQkYdIn0EEjXUQkpaVZoOtkFyKSutIj0Fu2gxbtFOgiktJSf9hilUGXQJseflchIpIw6RPoF/za7wpERBIqPbpcwDuvaOkGqDzgdyUiIgmRPoH++Zvw8ADYuNDvSkREEiJ9Ar39cd61fhgVkRSVPoHepicEMxXoIpKy0ifQgxnQrq8OLhKRlJU+gQ6Qf5z20EUkZaVXoHc+CbJyIBLxuxIRkbhLr0A/dxJM+A8E0utti0h6iCnZzGy0mX1mZivN7O462pxnZvPNbImZvRPfMuNM0+iKSApqMNDNLAg8CowBBgJXmdnAGm3aAH8ALnLODQK+lYBaj13FfvhlX3j/Eb8rERGJu1j20E8DVjrnVjnnyoEXgItrtLkaeMU5txbAObclvmXGSagFBEOwbaXflYiIxF0sgd4VWFftfnF0WXX9gbZm9h8zm2tm19X2RGY2wcyKzKyopKTk6Co+Vu010kVEUlMsgW61LHM17mcApwLfAEYBPzGz/kc8yLknnHOFzrnCDh06NLrYuMjv7wW6q/kWRESSWyyBXgx0r3a/G7ChljYznXN7nXNbgdnASfEpMc7y+0PZLtjTPHuFRESOViyBPgfoZ2a9zSwTGAe8XqPNa8A5ZpZhZi2B04Fl8S01TnqcDhnZULLc70pEROKqwfnQnXOVZnYrMAsIApOdc0vMbGJ0/ePOuWVmNhNYCESAJ51zixNZ+FHrcjLcsQhyOvpdiYhIXJnzqS+5sLDQFRUV+fLaAFSWw9y/wKnXQ0aWf3WIiDSCmc11zhXWti59D5lc+wHM+C949yG/KxERiYv0DfQ+58GQK7xA39Q8e4dERBojfQMdYMyD0KItvHYLhCv9rkZE5Jikd6C3bAdjfwUb58OHv/e7GhGRY5LegQ4w8BI44QJYX6SDjUQkqTU4bDHlmcE3/+TN82K1HRQrIpIctIcOkNnSC/NV78DCF/2uRkTkqGgPvbr3H4F1H3tHk7bp4Xc1IiKNoj306i78jXf9xu3qTxeRpKNAr65ND/ja/fDFv2D+c35XIyLSKAr0mgpvhB5nwax7YPcmv6sREYmZAr2mQAAu+h20aAc71zXcXkSkmdCPorXJPw5umwuBoN+ViIjETHvodQkEvZNgvHYr7N3mdzUiIg1SoNdnbwkseAFm3u13JSIiDVKg16fTIDjnh7BoKqyY5Xc1IiL1UqA35JwfQseB8MYd3rlIRUSaKQV6QzIy4eLfw55N8Na9flcjIlInjXKJRddT4ezbwUW8I0g1iZeINEMK9Fh99T4FuYg0a+pyiVVVmH/yJ3j7AX9rERGphQK9sbYs885DuvKfflciInIYBXpjff2n0GkwTL0eNi/xuxoRkYNiCnQzG21mn5nZSjOr8ygbMxtmZmEzuzx+JTYzWTlw9RTv+rkrNYGXiDQbDQa6mQWBR4ExwEDgKjMbWEe7B4HUPwKndVe46gXYtx1eu8XvakREgNhGuZwGrHTOrQIwsxeAi4GlNdrdBrwMDItrhc1Vl6Ew7m/QpqfflYiIALF1uXQFqs8jWxxddpCZdQUuBR6PX2lJoO8IaN8XyvfqhBgi4rtYAr22wdc1z8/2G+Au51y43icym2BmRWZWVFJSEmuNzd+cP8Or34M5T/pdiYiksVi6XIqB7tXudwM21GhTCLxg3ljtfGCsmVU6516t3sg59wTwBEBhYWHqnLTzzFtgzfswfZLXBdPv635XJCJpKJY99DlAPzPrbWaZwDjg9eoNnHO9nXO9nHO9gJeAm2uGeUoLBOGyP3uzM754PWxa5HdFIpKGGgx051wlcCve6JVlwFTn3BIzm2hmExNdYNLIyoGrp0JWXnQ442a/KxKRNBPTXC7OuenA9BrLav0B1Dl3/bGXlaTyunhj1Oc8CS3a+F2NiKQZHSkabwUnwkW/hYws2P4lROr9nVhEJG4U6Imyqxj+OBze/LHflYhImlCgJ0rrbjB0PHz0B2+GRhGRBNN86Ik06uewYzXM+C9vOGP/kX5XJCIpTHvoiRQIwmVPerMzvnQDbFzod0UiksIU6IlWNZyxVT5sW+l3NSKSwtTl0hTyCuCWT7yRLwDhSghq04tIfGkPvalUhfnbD8CU8V6oi4jEkQK9qeUVwIqZMPVaKN/ndzUikkIU6E1t2Hdh7P/CZzPgr5fC/h1+VyQiKUKB7ofTboJvPQUb5sHkMbBrvd8ViUgKUKD7ZdAlcM3L3o+jAf1AKiLHToHup97DYcJsyO3knZ90/Ty/KxKRJKZA91sg+k8wfRI89Q34/C1/6xGRpKVAby5G/wLy+3lzqc9/3u9qRCQJKdCbi5yO8O1/QK+vwKsT4f1H/K5IRJKMAr05yc6D8S/CoEvhrXth+TS/KxKRJKLhFc1NRhZcNhn6jYL+Y/yuRkSSiPbQm6NAAIZe5V1/+S48fzUc2ON3VSLSzCnQm7tdxbBiBjx9Iezd6nc1ItKMKdCbu6FXwZXPwpalMHkU7Fjjd0Ui0kwp0JPBCWPh2ldhbwn8eSRsWux3RSLSDCnQk0XPM+GGmZCRCXs2+V2NiDRDCvRk0mkg3FoEx30NnIMPfgdlu/yuSkSaiZgC3cxGm9lnZrbSzO6uZf14M1sYvXxgZifFv1QBDp0oY/08eOs+eOxsbySMiKS9BgPdzILAo8AYYCBwlZkNrNHsS+Bc59yJwAPAE/EuVGrodip8ZxYEQ94ImFk/gooyv6sSER/Fsod+GrDSObfKOVcOvABcXL2Bc+4D51zVmRo+ArrFt0ypVfdhMPE9KPwOfPh7+NP5mltdJI3FEuhdgXXV7hdHl9XlRmBGbSvMbIKZFZlZUUlJSexVSt0yW8EFD8P4l6BtL29OGBFJS7EEutWyzNXa0Ox8vEC/q7b1zrknnHOFzrnCDh06xF6lNKzf1+Gq570umE2LvW6Y7V/6XZWINKFYAr0Y6F7tfjdgQ81GZnYi8CRwsXNuW3zKO9KmXWXMWrKJ+et2smHnfirCkUS9VPLatQ42zIfHvwLznvFGxIhIyotlcq45QD8z6w2sB8YBV1dvYGY9gFeAa51zK+JeZTWfrN7O95//9LBl7Vpl0jE3i4552XTMzaJTXhYdc7Ojy7zbHXKzyA4FE1la83H8GPjeB/Dq9+D122D5dLjot+qOEUlx5mLYezOzscBvgCAw2Tn3czObCOCce9zMngQuA6qOS690zhXW95yFhYWuqKio0QXvLqtg9dZ9bNldxpbdB9hc6l1vKT1Aye4yNpceoGTPAcKRI99X6xYhOuVl0aVNC84/viOjB3emU152o2tIGpEIfPwY/PO/Ia+LN4Y9qAk2RZKZmc2tK19jCvREONpAj0Uk4ti+r/xg2JeUVgv+3WWs3LKHL0r2YgaFPdsyZnABowd3pkubFgmpx3dblnn96SeMhfJ94MKQlet3VSJyFNIu0GOxcstupi/axPRFG1m+aTcAJ/dow9jBBYwZ0plubVv6VltCTbsTPp8FX/8pDLj40DlNRSQpKNAbsKpkDzMWe+G+ZEMpACd1a82YIQWMHVxAj/YpFO5rP4Y3vg8ly6HziTDiJ94IGattMJOINDcK9EZYs20vMxZvYsaijSwo9uZJGdQlj7FDChg7pIDe+a18rjAOImFY9CL8+39g5xrocRZc95o38ZeINGsK9KO0bvs+Zi7exPTFG/l07U4ATuicy9ghBVx6cle6t0vyPffKcvj0r7B9FYz6ufcj6ubFUHCi35WJSB0U6HGwYed+Zi7exIzFGyla481ycHbffK4Y1p2RAzulxpDIxS/DS9+B478BI34EnQb5XZGI1KBAj7PiHft4aW4xLxYVs37nflq3CHHJ0C5cMaw7g7q09ru8o3dgN3z0mDct74HdMORyOO8eaN/X78pEJEqBniCRiOODL7YxpWgdsxZvojwcYXDXPK4s7M5FJ3WldcuQ3yUenX3b4f1H4OM/Qrgcrp/mnWBDRHynQG8CO/eV89r8DUyZs46lG0vJyggwenBnrizszhl92hMIJOEokt2boWgyDJ/kHZC0+GXodY6OOBXxkQK9iS1ev4upRet49dP1lJZV0r1dC751ancuP7Vb8h68tHcbPDwAAkE4fSIM+y60rm/STRFJBAW6T8oqwsxasompRet4f+U2zOCcfh24srA7Iwd1IhRMsoN6tq6E//yPt6eOQZ9zofBGGHiR35WJpA0FejOwbvs+Xixax4tzi9m4q4yOuVlcfXoPrj69Bx1zk2w+me2rYMEUWPA89B8FY3/lndt040LoebaOPhVJIAV6MxKOON5ZsYWnP1jDOytKCAWNsUMKuO7MXpzSow2WTEdsRiJQud87ycbcp+CN26F1DzhpnHfR6BiRuFOgN1Nfbt3LMx+u5qWiYnYfqGRI19Zcd2ZPLjypS/KNay/fB8unwYLnYNV/wEWg++kw4sfQe7jf1YmkDAV6M7f3QCWvfLqeZz5Yzedb9tC2ZYhxp/XgmjN60jUZf0Qt3QALp8D852HsL6HPed4cMgdKoc/5msJX5Bgo0JOEc44Pv9jGUx+s5p/LNgMwcmBnrjurJ2f2aZ9c3TFw6ExJZjD127D0VcjpBEO+5fW9dzsNQkn2+4GIzxToSah4xz7+9tFaXpizlp37KujfKYfrzuzFpSd3pVVWEu7hVpbD5296P6SumAWRCsjIhhvfhIKTvIOZslt7wyJFpE4K9CRWVhHm9QUbePqD1SzZUEpudgaXn9qNS4Z25cRurZNvrx2grBTWvA9fzoav3uftpb8w3lvW6xxvOGSf86FdH03rK1KDAj0FOOeYt3YHT32whhmLNlIZcXRt04LRgzszZnBnTunRNjmPRq2y7A34bKb3g2ppsbcsrxtc+wp0OB7CFRBM0qkUROJIgZ5idu4r562lm5m5eBPvfr6V8nCEjrlZjB7cmdGDO3Nar3ZkJNtBS1Wc88a5r/qPtwd/6eMQagFTrvEObOpzLvQ+F7oMhdwC7cFL2lGgp7DdZRX8a/kWZizaxH9WbKGsIkK7VpmMGtSJ0YMLOKtv++Q7IrU2c570hkWu+QAqy7xl2W3gOzOh4wDY8KnXT99xAGTn+VurSAIp0NPEvvJK3vmshOmLN/GvZZvZWx4mLzuDrw3sxNjBBXylX37yjW+vqaIMNsyDTYthyxIY+TPvhNdTrvG6bcA7uKnTQOg4EIaOh/zj/K1ZJI4U6GmorCLMe59vZfrijfxz6WZKyypplRlkxIBOjB7UmZN7tKGgdXZy/qham13rYdMiL+Q3L4UtS2HrCrjudeh1Nrz/W2+ETceBXti36Qmtu0N+P2jZzu/qRWKmQE9z5ZURPly1jRmLNvLm0s1s31sOQOsWIQYU5DKgII8BBXkMLMijX6ccsjKSfC++SmU5WMA7kGnRS97BTpuXHvrRFWD0g3DGRPjiX/CfB70ZJFt3836Qbd3N+0FWUxhIM6JAl4MqwxEWFO9i6YZdLN24m2UbS/ls0272V4QByAgYfTvkHBb0Awry6JCb5XPlcVRWCruKoXS9t4fetpcX6O8+fGh52Pujx9Br4JJHvW8Az1wEedHAb5UPLdpCmx4w+DKv7daVkJXjLc9Ioe0lzcoxB7qZjQYeAYLAk865/1djvUXXjwX2Adc75+bV95wK9OYjHHGs2baXpRtLWbaxlGXRoN+4q+xgm/ycLAYU5DKwII/jO+fSukWIFqEg2ZlBWoSil8wg2dHboaAlb3dOJAL7tsKudZCZ4+2l71gDb/3EC/bS9bBvmxf6nYbA997zHveLHnBgl3c71MrrymnRxjvjU3ZrbwKznWuhRTtvQrPMHMhs6R0xm9PBO7iqbJe3LtTSu2jmSqnhmALdzILACuDrQDEwB7jKObe0WpuxwG14gX468Ihz7vT6nleB3vzt2FvOsk2HAn7phlJWbtlDeTjS4GODAfMCPxSkRWbgYOhnRy+hYIBQ0MgIBggFjIzDbgfICBqhQPQ6GCAjujwz2i5gYGYEzAgYBMyw6HXVMqtlXfU2Zt6oR+Pw9la9PRzWtuo2zmGV+wlU7iPcIh+A3C/+QbBsB4GyHQTLdhA8sJNg2U6KR/0JAhl0fXMCuavfxFz4sG21dMRkSjoPp+vixzhu4UOHrasMtmBJr+uZ2/v/kFv6Gecse4CKQAvCgUwiwRCRQCa7WvVh4XETCQaMwhUPgwUhmIkLhiCYCcFM1vUdTyAjRH7JR2RW7MQCGTgL4iwDAgH2tB9COKsdmfs2krV/M1gQZ0EIBHGBEJVZbQm3aIeFDxA6sNM7otcCQAAX8J4nEmoJgIUPAIazQLSN1Tq81DmHO3g7eo2Dg7drLK92vyEN7UsY0X9TDv17N3ibav/+Mbye94ja27RvlUnHvKOb9qK+QI/lGPLTgJXOuVXRJ3sBuBhYWq3NxcAzzvvr8JGZtTGzAufcxqOqWJqFtq0yOatvPmf1zT+4rCIcYe32few9UMn+8jD7K8KUVXjX+8sjh+5H1+2vCFNW7fb+8jA795VTHnZUhiNURhwV4QiVYUdlJEJFdHlFxLuO+NMjeJRaRy+9Dl+84sPojWuBa+3qQlkAAAmhSURBVMhlPy0po6UdoCVlrJ0eYTef0N86MsQm0tLKaMkBb31lGR8ua8W/lizleFtLpwyjpe0iRCWZVJJJBRtdKf+9zPvvuDDrRbKoIMsqDivhwk8GESHAc6EHOSW4lJquLv+/fBAZzG3BV/hh6KUj1v+u8hIeqryCYbacF7N+esT6TyLHc0X5fQB8kTWeoB3+DxdxRr8DzxAmyF9CD3JmYCkOw0VDzwE3Vkzio8hAJgTf4NaM17xAP7je+FPlN/hD+GJOts95IrPqD9+h55gf6cuEih8C8F7W9wkQOezxAOce+DVhgvw69Cin2oqD9VWt/0HFzcxz/bkuOIvvBGdWW+/5a3gkk8NjGGKreCT0+yMev8j15o6KWwF4M3MSAY78AI8s/yUTzu3H3WNOOGLdsYol0LsC66rdL8bbC2+oTVfgsEA3swnABIAePXo0tlZpBkLBAH075DTZ60UijorqQR8Nfucg4tzB68jB+4duRyKHt3FULY/ejhxahuPQcxy27PDHVz1/Q51JDe0hBgM1v4UYGYGzD/tGEop+U7kkcOjbTEZgAhkBw+F1lYUjjg7OMT/iqIw4yiJfsifiCIcjhCsriITLcRVlTMtsQzjiCOyezJKyUsxVQiSMRSrBhbmzdT/CWXlk7urC8tKxWCQMB9uEOad1X05tN4DQ3t6sLA5hOHBhzEXAOVq36MCzvbxYWLf4TowIuAjmnHdNhGdOPAMsQMcV32LrnjXeOgBzmIO7+p9HWV4f2mwoZ9/6LKpi1KLRfmnXcxnR/UyydxfA4hXeencosgfm9eTlIWfiHGS9P+LQ80fbAUz5ylk4C9B54Xyyd0bPjVvtm8KPB57O3jb9aLd2Jy3W7Yg+7NBXhku6FnJGt1NpWdqGFksLDz1/1ODc3vxx0Kk4B7kfnXTYuip/OP0UendIzLESsXS5fAsY5Zz7bvT+tcBpzrnbqrWZBvzCOfde9P7bwH855+bW9bzqchERabz6ulxi+cWlGOhe7X43YMNRtBERkQSKJdDnAP3MrLeZZQLjgNdrtHkduM48ZwC71H8uItK0GuxDd85VmtmtwCy8YYuTnXNLzGxidP3jwHS8ES4r8YYt3pC4kkVEpDYxnSnBOTcdL7SrL3u82m0H3BLf0kREpDF01IKISIpQoIuIpAgFuohIilCgi4ikCN9mWzSzEmCNLy/esHxgq99F1KO51wfNv0bVd2xU37E5lvp6Ouc61LbCt0BvzsysqK4jsZqD5l4fNP8aVd+xUX3HJlH1qctFRCRFKNBFRFKEAr12T/hdQAOae33Q/GtUfcdG9R2bhNSnPnQRkRShPXQRkRShQBcRSRFpG+hm1t3M/m1my8xsiZndXkub88xsl5nNj17ubeIaV5vZouhrH3E2kOh0xb81s5VmttDMTmnC2o6vtl3mm1mpmd1Ro02Tbz8zm2xmW8xscbVl7czsLTP7PHrdto7Hjjazz6Lb8+4mrO9XZrY8+m/4dzNrU8dj6/08JLC++81sfbV/x7F1PNav7TelWm2rzWx+HY9N6ParK1Oa9PPnoqftSrcLUACcEr2di3ci7IE12pwH/MPHGlcD+fWsHwvMwDsj2hnAxz7VGQQ24R3w4Ov2A4YDpwCLqy37JXB39PbdwIN1vIcvgD5AJrCg5uchgfWNBDKitx+srb5YPg8JrO9+4M4YPgO+bL8a6x8C7vVj+9WVKU35+UvbPXTn3Ebn3Lzo7d3AMrzzoCaTgyfnds59BLQxswIf6vgq8IVzzvcjf51zs4HtNRZfDDwdvf00cEktDz14MnTnXDlQdTL0hNfnnHvTOVcZvfsR3hm/fFHH9ouFb9uvipkZcAXwfLxfNxb1ZEqTff7SNtCrM7NewMnAx7WsPtPMFpjZDDMb1KSFeWeYfdPM5kZPsF1TXSfnbmrjqPs/kZ/br0onFz2DVvS6Yy1tmsu2/A7et67aNPR5SKRbo11Ck+voMmgO2+8cYLNz7vM61jfZ9quRKU32+Uv7QDezHOBl4A7nXGmN1fPwuhFOAn4HvNrE5Z3tnDsFGAPcYmbDa6yv7dzyTToO1bzTEl4EvFjLar+3X2M0h235I6ASeLaOJg19HhLlMaAvMBTYiNetUZPv2w+4ivr3zptk+zWQKXU+rJZljd5+aR3oZhbC2/DPOudeqbneOVfqnNsTvT0dCJlZflPV55zbEL3eAvwd72tZdc3h5NxjgHnOuc01V/i9/arZXNUVFb3eUksbX7elmX0buAAY76KdqjXF8HlICOfcZudc2DkXAf5Ux+v6vf0ygG8CU+pq0xTbr45MabLPX9oGerS/7c/AMufcw3W06Rxth5mdhre9tjVRfa3MLLfqNt4PZ4trNGsOJ+euc6/Iz+1Xw+vAt6O3vw28VkubWE6GnhBmNhq4C7jIObevjjaxfB4SVV/132UureN1fdt+UV8Dljvnimtb2RTbr55MabrPX6J+8W3uF+AreF9pFgLzo5exwERgYrTNrcASvF+cPwLOasL6+kRfd0G0hh9Fl1evz4BH8X4dXwQUNvE2bIkX0K2rLfN1++H9cdkIVODt9dwItAfeBj6PXreLtu0CTK/22LF4IxO+qNreTVTfSrz+06rP4eM166vr89BE9f01+vlaiBcyBc1p+0WXP1X1uavWtkm3Xz2Z0mSfPx36LyKSItK2y0VEJNUo0EVEUoQCXUQkRSjQRURShAJdRCRFKNBFjoJ5M0n+w+86RKpToIuIpAgFuqQ0M7vGzD6JzoH9RzMLmtkeM3vIzOaZ2dtm1iHadqiZfWSH5iVvG11+nJn9MzrJ2Dwz6xt9+hwze8m8ucyfrToqVsQvCnRJWWY2ALgSb1KmoUAYGA+0wpt/5hTgHeC+6EOeAe5yzp2Id2Rk1fJngUedN8nYWXhHKoI3m94deHNe9wHOTvibEqlHht8FiCTQV4FTgTnRnecWeBMjRTg0idPfgFfMrDXQxjn3TnT508CL0fk/ujrn/g7gnCsDiD7fJy46d0j0LDm9gPcS/7ZEaqdAl1RmwNPOuXsOW2j2kxrt6pv/or5ulAPVbofR/yfxmbpcJJW9DVxuZh3h4Lkde+J97i+PtrkaeM85twvYYWbnRJdfC7zjvPmsi83skuhzZJlZyyZ9FyIx0h6FpCzn3FIz+zHeWWoCeDP03QLsBQaZ2VxgF14/O3hTmz4eDexVwA3R5dcCfzSzn0af41tN+DZEYqbZFiXtmNke51yO33WIxJu6XEREUoT20EVEUoT20EVEUoQCXUQkRSjQRURShAJdRCRFKNBFRFLE/wfQbJrYpKBGxgAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "hist_pd = pd.DataFrame(hist_obj.history, index=np.arange(1, len(hist_obj.history['loss'])+1))\n", - "hist_pd.index.name = 'epoch'\n", - "sns.lineplot(data=hist_pd)" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [], - "source": [ - "model_name = \"model.h5\"" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [], - "source": [ - "model.save(model_name)" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [], - "source": [ - "# model = load_model(model_name)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Testing Accuracy" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [], - "source": [ - "def test_generator(x_test, resize_shape):\n", - " for sample in x_test:\n", - " img = cv2.imread(sample, cv2.IMREAD_GRAYSCALE) / 255.0\n", - " res_img = cv2.resize(img, resize_shape[::-1], interpolation=cv2.INTER_AREA)\n", - " res_img = np.expand_dims(res_img, 0)\n", - " res_img = np.expand_dims(res_img, 3)\n", - " np_img = np.array(res_img)\n", - " yield (np_img, np_img)" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "MSE Loss: 0.07084273546934128\n" - ] - } - ], - "source": [ - "steps = len(x_test)\n", - "test_gen = test_generator(x_test, input_shape)\n", - "loss = model.evaluate_generator(test_gen, steps=steps)\n", - "print(\"MSE Loss:\", loss)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Sample Prediction" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [], - "source": [ - "img = cv2.imread(x_test[0], cv2.IMREAD_GRAYSCALE)\n", - "img = cv2.resize(img, input_shape[::-1], interpolation=cv2.INTER_AREA)" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 36, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAADECAYAAABk6WGRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOx9eXSUVbbvr+aqpCqpzANhSAgkkDCEQEBEBkHupZkntVEc2qG7tbu1u7Ubve29IKJt215akBlU2kYBEZlRoEEBAYFAICSBEDJVUpVUKjXP03l/5O7DqTR631vv+p6rV85aWZm++r5z9tnDbw9nfxLGGHpGz+gZPaNn/HMN6f/vCfSMntEzekbP+J8fPcq9Z/SMntEz/glHj3LvGT2jZ/SMf8LRo9x7Rs/oGT3jn3D0KPee0TN6Rs/4Jxw9yr1n9Iye0TP+Ccf3ptwlEsm/SiSSGxKJpE4ikSz5vp7TM3pGz+gZPeMfh+T7qHOXSCQyALUA7gPQAuACgB8zxqr/xx/WM3pGz+gZPeMfxveF3MsA1DHG6hljQQDbAcz+np7VM3pGz+gZPaPb+L6Uey8ABuH3lv/6W8/oGT2jZ/SM/wdD/j3dV3KHv8XEfyQSydMAngaA+Pj40gEDBkAqlUIikSAajd7+EGOQSCT8ezQa5T/TF11Pfxc/CwAymYxfQz/T58Tv/zUv/rw7hawYY5BKpfx/9DlxTuKzu3+W7i0+606D5kvPutN9u9OFrhc/Lz5PnCP9TVzLtz3nTv8T70N/F/dNpHn3ffk2GtM6xDndadAau6+Tfqf/S6VSRCKRmOvF/bvT3tP1d+KlO/Em3U98Xvc9FXm1O63pHvQzPVtcf3d63Gmt3elP13wXr9LP3e9D14rP7L4v3flYHOKe3ImG4prp793XKtL122Sx+1q779l3XSd+0XPEPerOV3Qd8QDxDf3tTrQQ6Squwe12IxwOx9xLKpUiGo1CLpcjEolAo9FArVaDMQa5XH7HvWWM4cqVKxbGWNqd1vt9KfcWAL2F33MAGMULGGMbAWwEgJKSEnbq1Cn4fD4oFArEx8fD7/cjEonwxUajUSgUCoRCIUilUiiVSvh8PkQiEchkMiiVSoRCIQSDQSgUCqjVakSjUXi9Xmg0Gv75QCDACUmMpFQq4fV6oVar+bNCoRDi4uLAGINMJoNMJgNjDOFwmOaPYDAImUzG5xQOhyGVSqHRaBAOh7mCUyqV8Pv90Gg0CIVCAIBwOMyZSKVS8WsDgQDi4uLg9/v5fGn+4XAY4XAYKpUK4XAYCoUCCoUCQBfD0HW0PtG40VxpfcRAtB66XyAQgFwu598ZY9BqtXyetMZgMMifFQqF+N+JXnQNACiVSsjlcni9XsjlcigUCgSDQahUKng8Hsjlcv55uVyOuLg4uN1uzsQi7UVhpDkrlUp4PB4AXcpBKpVCpVIBAH8u7VMkEkF8fDy/H63L7/dDJpNBpVLxdZGyBwCfz8fXp1KpYtYvCjrNIRKJxAi8XC7nfBSJRKBSqRAIBDj/0npofzQaDQAgEAhApVJxugLgvCCTyfj+KRQKyGQyKBQKOJ1OPh+aE32WPk/8pVAouOIIBoP8c0Rzmm8gEOAKinhQNI6MMYRCISiVSkilUs6XwWAQUqkUfr+fK19SWsT/3UGR+Hka4XCYy45UKkVcXByX9Wg0imAwCKVSyb+TUiQ6KRQKTt9QKASfzxdzD5J9mjPJYPf5SSQSTneJRAK/3/8PPOfz+QAA8fHxiEQi/FqZTAa73Y49e/bA4XCgb9++CAQCcDqdSE5ORjgchlwuh9vtxuDBgzFixAiEw2FOQ9oX4rtwOIzMzMwmfMv4vpT7BQADJBJJLoBWAA8CWPRtFxPh5HI5Z2RS1kqlEpFIBD6fD4wxqNVqyOVyhEIhaDQazjD0f6VSye9HhKdNJ4VHm+92u/n/RcEkwkUiEX4tCQRdQ0aCrhEFnZjQ7XZzxSEyRygU4sxLwkn/J+Yk5UjXk0IRGYWUJ93H7/dDrVZzRibGJgVCzyDDR/cgmhFtIpEI1Go1p0koFOL3IeVBCpqUIN2LDF1ycjJCoRAkEglXogqFAnK5nD+Lng10KSdaK92ju3GnQfMlY0C8Q8qGhEyhUHABBQCVSsXvSTSLi4vj641EIhx5E32IjqRQ/H4/fD4fv4dIHxrdkSbxq2iobTZbjPInOtLvpExJAdHfid/p+aQIRV5QKpV8PqS8iGcZY1xuCPzEx8dzQ0m8QvtJfED7IK6R1k3rpeeSQiOlKpFIOL/TNSQjdD3RgvZI/DzJlGgY/X5/zJxkMhk3hEqlkssvyUsgEIDH4+FGkPiOaEl8QWBMlEviYaK5yI9yuZzLPfEU/c3pdEKtVvM1yWQyOBwORKNR6PV6zndk+NRqNdRqNZRKJeLj4wF06TDaH5qLRqPh+uy7xvei3BljYYlE8gsAXwCQAXiPMVb1HddzZEkM5fF4oNFoEAgEEA6HuUKKRCIx1xJDkhImYWSMcYYiZNldKZJSjkQi0Ol0HEkBt11/v98fgzoIZRPCA8CFXyKRIC4ujjOZVquNEUS6nhQKAK50icG1Wi1n/lAohPj4eK4YJBIJEhISuCEhKx6JRBAXF8f/HggEANxmTELiNAi90M+ErERDRcJLghmJRLhAkdKkudNn1Wo1tFotIpEI7HZ7DCokoxkKhfjeiCEQMqgi4lapVPD5fFz5uVwufn8SFgIA5NKq1WoEg8EYD4IUHX1G9HpISEQh745O4+Pj4fV64Xa7uaDR/UVjQnxJKC4UCnFaktGg5xG/it4ozYf2gQw6zYsUBxlZuVzODay4b3Qvuo/oUXUPqYXDYfh8PkSjUa5YRENvt9s5AiXPjp5JaxTpRWsjQ6JQKPiz5XI55HJ5jFdOIIa8RK/Xy2UM6PJc6LMAuEdDMkNKX+Qvl8sFjUbD10tzSUxM5DJAQI2QskQigVarjZERmq9MJoPL5eL8TushIy+GwORyOTweD5ddkj3igbq6Ok4rrVYLp9OJ1NRUHmHw+XxISUnhtI2Pj48BWRqNBjabjXs/3zW+L+QOxtghAIf+d64VLbvP5+Pok/5OTCCXy+Hz+fhmd7f2tBkiwgDA7wGAKyhSFiQgTU1NyMnJ4Z+nEA+hmHA4zAUyEAhwhRAfHw+Px8OZjTZORMXA7TAMXSfmDmgQIxBioefabDZ4PB5kZ2cDAAwGA7lkXNmKFp2UGAkOKV5SyiJaF0MTJKgulwtKpRJxcXGc7iSohKJEZSoaRKIfMSVdzxiD1WqFXq/nSk6tVkMikcDr9fLPk3EmpC+VSnnoRtx3ETUSD4nhBlIO5JqTYae/0f5S6I0AhshXpIhsNhvfUworkKDRfAkwEL8QkhQVslQqhcPhQCQSQVZWFucBMQRBtCQPoXs8mBBtMBiE3++PQak0h4SEBLS2tkKj0SA5ORkA+H4R0CHa0jOIZqLRpPVEIhF4vV5Eo1HExcXFKD1an2gUiEbi3Cl0YjKZkJWVBbfbDa1Wy/kdAJcv4n8Kb4h7JoZoSQ6JB8kYkkEn/eH1eqFQKNDW1galUomkpKQY74z23OfzcVpqtVruDRPSJzknIEfzEUO7pMNo0P1obzs6OjhNgC7jRaFFop/NZkPv3r1jwIioC8k76a7nuo8fxAlVcpuI6Ww2G6LRKHw+HwKBAAKBAHw+H1dKxKDAbeFVqVT8Z3HRxPSBQAChUAharZZvPinerVu3Ys+ePRyB0WYpFArOzBqNBiqVigs5ITFiKlJ0pPSJaX0+H0feLpeLu4e0BvpZdEVJ8YTDYcyZMwdnzpzBqVOncOnSJSxYsAAZGRmIj4/HoEGDOIIyGo2YPHkyVxaisJKystlsHOkCXeEZQkJkDNRqNRYvXowNGzZwZqL50NrEMBaFp8jVN5vNHKnQd/rM3/72N5jNZgDg6I48ElKSXq83RhgcDgc0Gg3//IwZM7gHdeDAATidTi7ky5Yt48JN+ygiVUJZxDsksOSxkcAplUpoNBpOjw8++ACBQADx8fExcXIKbcjlcpw6dQorVqzgvEChNzHspFQq8dhjj+Fvf/sb5zEy+nQvMYYvIlKpVMqRIM1TDO0Rsk5ISMBLL72Ezz//HE899RTfO+JJMtAejwfBYBA6nY7fg7w7Wt+SJUu4cvF4PFi7di33NETFRvwlImfaB3rWr3/9a1y6dAlarRbTpk3Diy++CJVKxXlHzCuEQiEeyiTwRUqVDD/Jfnx8PPdypk2bxg0N8TSBFKVSiR//+MfYsGEDB4JkDESjSeskj5jm6PV6oVQq+bOIx2hPCDyKnhuFimm/yJhnZGRAoVCgs7MTPp8PDocDLS0t6OjogMvlQnx8PNRqNd8X4gcyst3l+tvGD0K502bRIogBiclUKhU0Gg3faACccUgRkKIgqw+AKzoSImI0QpOE0Pv06YOmpibYbDZurQnJud3umHgqbWRCQgJkMhn8fn9MnBgATwyLyRiK2YqJMQrjiNeSsGs0Gmg0Guj1ekyZMgULFizA2rVrkZeXBwDIyspCUlIS/2xOTg6OHTvG6SfmAygMRAqYnk0InNAQISCz2YxZs2bFIE5SKsTEooKnuCK5miKiI4QHAHv37kWvXr1iYrvk0iuVSmi1Wuh0Ov5MqVSKxMREjsJ+8YtfYO/evRy5btmyhf+/vb0dCQkJ0Ov13DCK+0W/A4jJDdD+kJcGdCk3Mrrt7e347LPPoFQq4Xa7AdwOw9F9wuEw1q5dy/eXeJkAi8iLZrMZc+bMAWMMHo+HGzWiKfErGQmZTMYrJ8SkY1xcHAcjRLu4uDhIJBLs27cPixcvxrZt2/ieiaGEUCgEnU7H0R95snQ/oEtJnTp1CvHx8VAoFPjyyy8xb948zrMiwKEvUshiriISiWD79u345S9/iWHDhkGr1WLChAmYP38+bDYbLzYg8NVdbml9FB4l2RdzYySvn3zyCaelmAMiT8JisWDOnDl8f8Q8EHmpx48fx+LFi/keEC+SQqd4v1qthk6ni/EICdxpNBpotVoOskg2yBCKCdjMzEzIZDL07dsXiYmJSEhIQO/evTk6Jw+F1imGDUUP4U7jB6HciWlFRE6bQgsEEIMQAMQIATGSmOCjcAUNERlQqKK6uhrDhg3joYhwOMwNiUqlgkKhgMVi4YizqakJbrebK0NSdg6HA1arlSvllpaWGE9DLpfDarXCbrdzxiNDRdUIjY2NcDgckMvlsNvtqK6uRnJyMux2O9ra2nD9+nUkJyejtbUVdXV1XPmSu0txwaamJjgcDjgcDphMJu5aArcTTyaTiSM1EkaKp+bm5kKv18NkMsXEt2UyGcxmM1paWmJCHCaTCRKJBG1tbTxJLVZbtLW1wWKxwOFwAAAPNdHnyUCbTKYYhUmuNKEfp9MJpVIJp9OJq1evorm5GWazGVarFbW1tUhOTsbNmze5wQoEAqitreVKpKWlBQ0NDQiFQjAYDLzChniO1kqK3mQyob6+Hna7nSs0uVwOo9HIBUulUqG1tRVNTU3Izs7m3opUKoXZbIbdbuc8GwgE0K9fPx7nJdSpUCjg8Xg4fUgmZDIZmpubuXDTfnV2dqKzs5PPkxKjHo8Hra2tiEQiMXRjjOHWrVsxRrOxsRHRaBTt7e0wm83cuwQAs9mM2tpaJCUlwWq1AgAuX76MxMREtLS08PADDbVajba2NpjNZg4siAZKpRJmsxlVVVWcvqNHj0ZxcTG8Xi+MRiM3omTkSZmbTCbOn1KpFBaLBRqNBgqFAvX19TyMRvxP3qparYbb7UZbWxu/JhQKITc3FxqNBkajkc9FTIZHo1GcO3cuBoQ5HA60trZyGYlEIrBarfD5fGhpaYHL5YrhIalUCoPBgKamJoRCIbjdbi5rot5ISEjgoIgMAf2s1+uhVqvR2trK10z3djqdMBgMHLh91/hBKHcqW6KSOTFhSCgKuF0ZIKJPt9vNXXuqqvH5fLzsjhASZempfNLv9+PAgQNob29HSkoKmpqauGtL7hPFn00mE958802cOnUKLpcLI0aM4K7j5cuXcejQIajVarz55pvw+/344IMPsH79eowbNw4AMG3aNBiNRnz66aeYOXMmTxBTcrC9vR27du1CcnIyzpw5gwsXLkCn08FkMmHx4sVISkpCVlYWGGNYsGAB0tLSIJfLMWvWLITDYXz44YdYs2YN7r77bpw/fx5NTU0oLS2F1+vFxo0bMWrUKO4yr1q1CiaTCcnJyVi4cCHa29s53Uno09PT4fV68c4772D06NGIi4uDx+PBxo0b4Xa7kZSUhKKiIgQCAaxatQorVqzAuHHjoNVqcdddd2H16tWIRqNobm7GT3/6U6hUKnR2dmLMmDG4evUqNm/ejPHjx2PdunXo7OzEZ599BrVajUWLFuHdd9/Frl27UFZWhmXLlsHv92PNmjXYvHkzRo0ahdWrV0MikWDbtm0oKChASkoK0tLSsH//fkyePBnJycncaO/fvx8ZGRl44403uPFZuXIlLl++jOTkZAwbNgw6nQ4ajSbGk2lsbMQvfvEL6HQ6PPfccxgzZgwHDS+88AKysrKwZcsWXLlyBTKZDKmpqSgoKMD06dM5//7qV7/iCbLf//73kMvlqK+vR1FREdRqNV577TW4XC5Eo1H89re/RWtrK959911cunQJkUgE1dXVmDNnDjIzM/Gzn/0MVqsVly9fxp49e6BUKrF8+XKsWbOGKynKvWg0GkycOBFJSUkAgF//+td47733kJycjClTpqCiogJr1qzBG2+8gXHjxiEpKQn79+/nAIBKdjs6OvDss89yr2b37t1gjCEtLQ133303Ojo6IJPJ8O677+KFF15AWloabDYb2traeIgS6DKYzz//PDZv3owRI0ZgwoQJmDRpEtLT09HW1oY33ngDtbW1aGlpwb333otoNIra2locPHgQbrcb99xzD/e27XY77rnnHpw5cwaRSARr1qwBAGzbtg1GoxHvvfceAODChQs4f/480tLS8Oyzz4Ixhhs3biAnJwcSiQRr1qyBzWbj8yT9cenSJRw7dgwpKSmIRqPYvHkzHA4HUlNTUVRUBABYtWoV3nzzTZw5cwZ6vR6lpaVYs2YNpFIpKisr8cwzzyAhIQEulwuvv/46/va3v8FgMGD9+vUAgE8++QQejwcdHR1wOp04f/48vF4v/vCHP2DLli1gjMFoNOKXv/wldDod7HY7Xn75ZXg8HmzYsAEulwspKSl46KGHcPTo0e/Uqz8Y5U4JDlLsxLQUoiArSzFOQgfkVpHrTvcRE6OE9Cl2LpPJEB8fj6tXryIrKwsWiwV2ux1erzemsiAajcLlciEtLQ319fUYNmwYvF4vr9KRy+X405/+hEGDBqG9vZ2jJ3K/CwsLEQqFkJeXh8TERIwfPx5lZWW8XpxQ6rvvvotp06YhISEBhYWF+Prrr6FSqdDY2Ig+ffpAqVQiISEB2dnZSE1NhUajgdPpRGlpKU8Kt7a2oqSkBHa7Ha2trZDL5cjIyIDdbufhHolEgq1bt0KhUODGjRu45557eEkWudHV1dW4//770atXL47QotEoqqur8eGHHyIvLw9xcXFITk4GYwwpKSm4desWSktLkZCQwOO6MpkM69atg8/ng16v58rUbDZj6NChkMvlKC4uxq1bt7jg3HvvvcjPz0dJSQkUCgVmz56N7Oxs5OXloaSkBEqlEiUlJZBIJLhy5QqGDBnCE1FXrlxBcnIylEol2tvb8cYbb6CoqAjHjh2Dz+eD2WxGRkYG6uvrMXTo0JhKLJFPotEo1q1bB7/fj8TERMjlcpSWlgIAKioqMHXqVOj1eigUCmRnZyMUCsHr9WL48OFITU3ltPZ4PEhISIBOp8OgQYMgl8tRXV2NxYsXIycnh8d3L1++jEmTJmHgwIFQqVTIzs6GTCbDiRMn8M0338But+N3v/sdUlJSsGLFChQVFcFqtcLv9+Puu+/mCVHyVEOhEPr378/LYRsaGjB58mTEx8ejqakJp06dQkpKCurr6zF27FgolUrMmDEDcXFx3KPVaDSoqKjA4MGDAYAj8NTUVC6zBIDef/99lJSUoLq6GocOHYJer48JY1IIadWqVVi3bh2USiUqKyvh9XqRkZGBhoYGDB8+HElJSTAajQgGg9iwYQOmTJnCq1e0Wi1CoRAyMjJQVlaGkSNH8tAIAIwcORLRaBRDhgyBSqXC2bNn0draCofDgRdffBFyuRzXrl3DT37yE2RnZ/P4OXnn5GVeuHCB/3zp0iVs2bIFOTk5UKvVSE5OhsvlQnJyMurr6zFq1KiYhLBEIsHatWsRCASQnJwMnU6HgoICDB8+nM+NFLfP54NOp8PBgwdRUFAAoAuoZmVlITs7G2vXruX5EKVSifz8fBw9ehTbtm0DANy6dYvT4L9VrP+/v4YNG8YsFguzWCzMarUyi8XCbDYbM5lMrKOjg7W3tzOfz8ecTifz+/3M5XIxp9PJPB4P83q9zOl0MpvNxpxOJzOZTKy9vZ11dnYyv9/PAoEA8/l8zOfzMZfLxQKBAAsEAuzSpUusubmZmc1m1traypKTk1l5eTmzWq3M6XQyr9fLXC4Xc7lcbO3atay4uJi5XC7285//nI0ZM4ZZLBbmcrmYTqdjBoOBtba28s+6XC6WlpbG9u3bxw4fPsz279/PbDYbe//999nhw4eZzWZjZrOZmc1m9sc//pHpdDpmtVqZy+Viy5cvZ0eOHGFGo5FNmzaNr6+hoYGtWLGCeTwe1tbWxn7+85+zuro6Po/S0lJ26dIl5nK52MSJE9krr7zCXC4X69u3L9u0aROzWq2spqaGPfroo8xkMjGj0ci8Xi+z2+382S6Xi02aNInZ7XZmNptZ37592dq1a5nFYmGFhYVs5syZrLOzk9XX17M333yTmc1mZrFYWFFREbty5Qrr7OxkY8aMYc3NzczlcjGtVsv27t3LfD4fW716Nbt+/Tprbm5mtbW1bMaMGayhoYE9+OCDrL29nTmdTvbWW2+x9vZ21tDQwJYsWcKsViuz2Wysra2N1dbWsuXLlzOz2cw6OztZv3792JUrV1hzczPr6Ohgffv2ZR0dHaypqYktWbKE6XQ61tnZyZqbm5nBYGAWi4Vt2bKFFRUVMY/Hw2w2GxszZgxra2tjVquV81lnZyefdyAQYCNGjGDXr19nN27cYIWFhay1tZUZDAZWVlbGrFYrM5vN7NVXX2WVlZXM7/czp9PJ/vSnP7EdO3Yws9nMVq5cyWpra1ldXR0bN24ca2trY2azmfXr149VVlaywsJCZjabmcfjYWVlZcxisTCTycRqa2vZ4cOH2fTp09nGjRuZ1+tlOp2OtbW1MaPRyKxWK/N4PMzlcjGfz8f5dc+ePezixYvM7/ezP/3pT+zgwYPM7/ezUCjEkpKS2MGDB5nT6WRFRUWssrKSOZ1O5nQ6WWdnJ+fLyspKlp+fz+x2O2tra2MNDQ3sd7/7HXO5XOz8+fNs9erVzGazsRs3brBHHnmEr8lms3Feor2zWCysvLycdXZ2ss7OTlZZWcn+/Oc/s87OTr4fdrudLV26lOn1erZy5Uqm1WqZ1Wplb7/9NhsxYgTXB5s2bWKXL19mLpeLPfvss1x+m5qa2Lhx45jVamV2u53V19ezt99+m82aNYtt2bKFGQwGNmnSJOb1epnH42F9+/ZlbW1trL29ndntdi5D/fr1Y0uWLInhd6vVyurq6tgf//hHZjAYmN1uZ8XFxcxisTCn08lGjx7N6urqmMvlYvHx8Wzfvn3MZrOxN998k9XV1bGmpiY2ZcoUVl9fz5qamlhRURFbu3Yt27RpE9Nqteyzzz5jn332GSsuLmbbt29nbW1tLD4+nn366aesvb2dLV++nFVUVLCsrCz28MMPs5s3b7KamhrW2trK9u/fzwBc/Da9+oNA7oSWKT4uWn6xYoPK+ERkTe4yXaPVapGcnAytVsuz1WLSKhgMwufz4eDBg0hLS+OJKMYY6uvreWyUPuPz+bBjxw7MnDkTSqUSe/bswYoVK3DoUFeVZ3p6OvR6PTQaDXbs2METmowxZGdn4+OPP0ZOTg6PJY4ePZrXAiuVSgwaNAhpaWlQKBRYuXIl6urqMH78eNy8eRO1tbUcmX7zzTeYNm0aotEoHA4HDh8+jLNnz0Imk2H37t34j//4D7S3t0OhUKCjowOLFi2C0+nEhAkTMGHCBNjtdqSkpPA4bSAQwMcffxyTUKM4M4Wmxo4diylTpsBisSAlJQV9+/aFxWLBW2+9hfvuuw9GoxGBQAAzZ85E7969sX37drz++uu4ePEiACAlJQW9evXC0aNH8c4776C5uZm7/v369UN6ejoOHz4MqVQKk8mE++67DyqVCseOHcOiRYs4k9rtdjz99NOYP38+3n//fbS0tGDGjBnIycnBhQsXYDKZMG3aNBw8eBDnzp3DqFGjkJGRAZ1OB5/Ph0OHDiEajeLDDz/E9OnT4Xa7sX37dqxYsQJHjx7l+0WJ+5SUFOTm5uLIkSOYM2cOWlpaoFKpkJKSAr1ej9dffx1OpxNr1qxBW1sb3nvvPfTt2xe7d+/mpXZ5eXk4ffo01qxZw9cNdFVXmEwmPP3009Dr9dwTM5lMcDqdWLduHWw2G15++WUUFRVh8eLFyM/Ph0QiQUZGBveCPvnkE5SXl/P8BFWU7dq1C3l5eZBKpSgsLORJv7feegvHjh3DvffeC4/Hw+lHXhuFRuVyOT777DPYbDYYDAZ4vV6cOnUKCxcuRGdnJ37zm9/gvvvuw7p165CSksJzVxKJBOXl5dwLInkuLy/H+fPneVh09erVeOyxxyCTybB161bk5+cjHA7j4sWLOHjwIBISEpCcnAyJRIJ169Zh/vz5nE+3bduGgQMHAgB2796Nixcv4tixY3jssceQmZmJzZs345133sGSJUvw9NNP49FHH0W/fv3w5ZdfwmQy8XLcsWPHYsuWLTwfQnrG7XZj3rx5qKmpQWpqKvLy8tDZ2Ym3334bU6dOhd1uRyAQwOzZs6FSqfDRRx/htddew5UrV3jIqnfv3jhw4ABWrVoFvV4Pv9+P2tpapKam4urVq+jTpw8qKip4uaVMJsO1a9dwzz33cNlMS0tDZmYmjvkj1G8AACAASURBVBw5gvXr1yMrKwtjxoxBOBxGYmIiFAoF9u3bh6ysrO/Uq7KlS5f+z2jo/4uxfv36pY899hgPqwDgdadiwo0YCbhdt0vKmzHGFZVYW01JB/p/Y2MjPvroI1y9ehWzZ89GNBrF559/jiNHjkCn0+Huu+8GgJgDLrt27cIDDzyA7OxsHDp0CMnJyZg6dSp0Oh2CwSBaW1vR2NiI7OxsZGdnIxqN4tq1a3C73cjKysLVq1dhNBoxZswYJCQk8GQZYwyZmZmQSCRobW0FACxatAgpKSn46quvUFFRgSeeeAKRSAQHDx5EWVkZNwqU5Bk2bBja2trQ2NiIYDCIoqIiVFdXY8GCBYhGozhy5AgA4K677oJMJsPZs2fh9/vR3NyMsWPHIjExkde5SyQSXLx4EdOnT0cwGMSxY8cQiURQUlKCpKQk3Lp1i7udlZWVGD9+PDo7O5GWloa+ffvCZDLhxo0biEajKCoqgsFg4Im9+Ph4uN1uFBcXQ61W49ChQwgEAkhLS0N7ezsMBgPGjBkDuVyOEydOYMKECTHH1CsqKrjBLCwsxIEDBxCJRHDXXXdBp9PhwIEDGDZsGIYPH85DG5To7N27N/r06YOdO3diwYIFyMvLg9VqxY0bNzBhwgS+dgprmEwmGI1GyGQy3LhxAy6XC5MmTeLhitTUVLhcLgwdOhSlpaU8aR0fH4+8vDykp6dj9+7dMecgJk6cCL1ej/LyclRVVWHhwoVQq9XQ6/W4du0a6uvr+T3HjBmDxsZG2Gw2XL9+HfPmzYs5j9HQ0IDMzEwMHTqUV0JR6enWrVvx+OOPIxqNIiUlBTt27EBbWxui0SgmTZqEcDgMh8OBuLg49OnTJ6a2XTz5W1tbi6ysLJSWluLo0aMYO3YsFAoFKisrEQqFMHjwYPTv3x9ff/01TCYTN07FxcVcscvlcpw8eRK1tbVob2/HzZs3MWLECG5UVq5ciYKCAthsNsydOxcDBw5EamoqbDYbHA4HtFotPB4PJk+ejIyMDOzcuRMPPvgg5HI59u/fj6KiItx9992oqqoCYwwTJ05Ebm4uAoEAmpqaUFdXh7lz5+LEiRNQKBSYOnUqGGM4duwYpk6diqKiIl7rzhjD8ePHoVKpMH78eKSkpKCuro7ze01NDSZNmgSr1Yrk5GTk5OTAbDajpqYGEokExcXFcDqdaGlpwfXr11FdXY1nnnkGarUa4XAYra2tuHXrFsrLyzFhwgQkJSXxkB7JczQaxciRI+F0OnnRRHV1NX7+858jPT2dJ7NbWlowePBgmM1mfPzxx6alS5duvJNe/V76uf+fjhEjRrCTJ0/ySgMaKpUKXq+X1zWLiL77EX3qVUI9KCj2DoCfWiRGFo82UxkgfSdhobg/tTAQEQqAGKagZ5GhoQoFuj9wu3yLeraIdbvkfdDBiUOHDuGTTz7B+PHjUVJSgsLCQsyZMwcHDx7kSeNoNIqEhISYfhtU7SPWdJP3QhVF5N2QQNO6nE5nzElAWgvlK+h6MqoOh4MbGrVaDZ/PB7/fzxUl0Qy4fTyfkpZksMUDIpRQF49909oIWTPG+MEdojudBqY8CFXZiLXidDiMaujJoNFzaK+A2/1VxPwNgBglKManaYiHYWjQXovtJjweD6/tFmOmtH/kmZI3ReV6lIOi+VKfIspDnT17FsFgEFu3bsXOnTtjehhFIhEkJSXxQzhEK5o3VZRQYpHo0v1wGwEkAk1ivxaxbJDkg67tnkNTq9WQyWRYtGgRNm7cyOvc6WAWVaZQaSbRIRwO89Oy3avgxHMnpCfI66d1EV3pepJZytlR/oIOrxEd6DtV2pEc0MElpVIJh8OB+Ph4SCQSPPvss0hOTsarr74aU7bY0tKCiooKaDQaxMXF8SqYSCSCgQMHcrmkMsjf/OY3SEhIwB/+8IeYMxBA15mVffv2YenSpeWMsZF3UKs/jLAMgJg2A6JgUb2ryCi0OWJdrPg7cPtEHylO8f908IAEgxQWMTqddKTNpwMt5BGINd70fPIYxBOjxEBURgiAl0/SNfSz2LBq9erVyMrKwujRo3H69Gm0trZiypQpMYJEjEFrJQVIwit6LsSwYpKZvsilJ+VLtBQTdCIzA13KmtZB+0TKiX4nb+pOR/tJmcbHx/Pae1KANE+iJf0snncQj8qTMaHkJ9GVAAEZUKCrhPDKlStcEZCSEeuyxQMptK+kxLoLPtE4EAjwE6Vk5IiGZNTF+1Hlkrg+Wnv3k57iSUuaC4EK8bj7n//8Z1RVVXGPjUoStVotN2biIS2xAIGSwDRIcYutIGjv4+Li+GlnGuKBNvHMCq1BVEpEr5s3byItLY2X79LpUJEeRF8qayWkS3tBte8isAJiw7oi+CODQ7JJ8yOax8XF8fWIPayI9yl0KQJNMrynTp1CZWUlOjs7ER8fj0ceeSTGgyFeJANAvEDykpiYiLi4OFy4cAG1tbW8rPqBBx74BwNJe9K9JLX7+EEg95KSEnb8+HGOLmlDCcmQsJEiEI90K5VK3pKABIasPm0keQBkJIgxRNQiHvoBwBEwEZNKxILBIEcP5DmIB3XIa6D50fFnmh9VD4gHZggV0mdcLhcPP1E9NVW1yOVy3qJBPClIjYZoHcRAdJ2IykRjRHMmRSSeFxCZT/QwxLYGpERIQYhnC4iGYiMwUnZ0TzGsQHMjV5bWEwwGOdIl40773N24iDX7lJuh4fF4YLfbeQUSCQu1ZSB6UMktrYkUIV0rVtaQohaNGN2DaCYaOpG3SVkQremgHT2H/kdCTPXf9BxaK1XeqFQqfuxf9CxJOdD6qJ6d5iWeI6H9Jd6kv1G8l55PMkhroCP+tFckM6JXS96G3+/nB8IUCgV0Oh2nKx0Son0WgQJwu0MoGTvy3sLhMHQ6HTe8dCiJ0DjxJlUDuVyuGI+c5JaeIcojzYFkSpQfsROk2K6EjKZ4+ryyspIjfIVCAZfLhXA4DK1Wi3HjxnFecjgcCIVC0Ov1nP5iopTOfPxXjuFbkfv31lvm/3SQ8iS0SUQhpiTERBsqWjMAMUd+6cg1oVn6n+jaEBoj4yA2ERNPx5KBIfRKKJMScN3dQ1JWpPSJAcQTnuJBHWIOeq5UKuXxuHA4jKSkpJij/4QqRGREaEhMiqlUKrjdbn7MnMoURUUqKuzuioy+iL4isic0RApIDH/QXtKgPSK0Td4TKScxZENeG5VTimE3KjGluZCwiaEcUubk3dA+UtmpWq3mZYg+ny/mcAgpQxoiUqcENPElXU/KRzyoRvQSjSutAbgdqhEVKd0fuN0niU5dJiQk8DXSmmn+crmct2MmL0REd+IeiD2GyLARv9E6RC+G6CkCAuovIwIIAHztZOTocB7dNxKJxIQ76RAPyZgYrrtT+wiSM5H2Pp8PCQkJnOepoZvoXZIRFb0vohHQZVjoVLZotMkzJRqJ4EQMxRLQpLYP1C6FDB/tNR3yIr0gAgeFQsHDMQSsdDpdDHih0KMYtu4eArzT+MGEZUg5kFDTJlIDKlG5kIIQQx+0YbSZ4okyEhTgdixQjKdSLLq7+0PMICpSEXEBtwWHKnncbjd3y0mREsNTSEScj4iMRAQMgMdfgdshAFGBkQIWE8si8iJFLyI4UXmLVRLiXGhdovISUTtdL66RjBExtqgUgNuNsOgzhIaIzhQKo//TaVmaOxlkEZFGo9GYDphi2wExpk77R7wBgAs7DRHhEt2IjmKIT+QZkYeIPqKRI8VANBLbTouGhDpfikChO/ITjSh9p/mRMiVPjuYnzkPkfaI7hVvE/afvBCZormTI6Rg9yazIU2KYSQyDEPigQeBLBCViTkHcEzLSBAJoj0TZD4VC3MMl3UHzobmIgEPcd9EY0h6L8k+0o3vR88m7697qhOhEwJCMhVjdR4csifZi2IvuRyEumhvNhXQktWz4rvGDQe4U7wJuJytFFAHcPjpPzEibFolEkJCQAOB2AoSIDIC7tsScYjglGo3y5knkvhFSoPgpWV8SUHI9iXFI2ZMCp42n5li0HjIWoocC3EY+pLxobYSwCFWQQu6eaCSGIXRKDC+iB0rq6HQ6LlzEfGJ8FwB/YQhwuz2A6PWI+0NKiO5B15GgdFfMFDYgxS02LhO9MLG9L/EHKTExFEZotKOjg3tu1DaC4pLUeKw7Mg+FQjxU0D0RSMqP6Cuiayo7pEFtmGlNlPClPY9GozFhQTEhTIJP3gXNQ2wjTNeTF0j3IHRJXicNUjJkQKm1hohOaS4ul4t7YQQ+RPBAayW+of0hz1CMVxNoIb6k/aJ2E6FQCL169eI0pQN45GmJRkqn03FeokQwAL5eOthEckHGWrwPrZfoRvcjT4pki0b3l2t0D5VRrJ32mpQs7W1CQkJMuJOMEQDeq91isSA1NZUDK4/HA61Wyw9Q+v1+Hq4kwEa8QvJP+yr2ArrT+EEgdzEJSEIFdBHHZDLh+PHjHGESgYlwbrcby5Yt4y6PmLwU69vJQooJFPpO1xOTUAKVlJPI7KRYSBgJdYixdrL+Ho+HIwZSuMRYhHZp3vQ7ISYx1i26g6I7L8b/Ghsb8e6778YgV5o7fYaUgYhI6Xli/J3mLyJYsRqEjJeIBEUG7O6VdJ8/eSlieMjr9XJams1mvPbaa2hsbOTKQvTexNJFmvuJEyfQ1NTEBYCMKikP8XmEJsVkq6iUxGeJ+QMxbgsAb7zxBj7//POYsI7YrlqMn5NxERU+0ZYxBpPJhH//93/nz6V5AuCggTwa8kZFJdUdTRNPhkIh3Lp1C/v37wdw28MQ+wqRoRSTnsS3brcbR44cQV1dXQySp8/S50npisibntfa2oqzZ8/i3LlzOHnyJO+xIiJXki8x10ND3AsxD0Q8KIb4iBZED1L0otGm30mu6Vqv18tzL8TX5G3SftD9RI+wvr4eBw4c4MZDzF+R/BIoIlAIgCtwp9PJ9U19fT0voyW+EaMZRBexM+W3jR+EcgfArSvFF2kjLBZLjNIQrbtEIkF6ejpu3rzJUTlZVNEAUOkgue3EuGKHSfoCbpePkSEh5pFKpXj22Wd5x0CxxI0UmFqt/ofujyTMlCSlg1Ri4oY2H7hd/UPuMAmqGPYgYSDrrdVqceDAAZ6MDIfDHPGICICUl6iAxUoMADHPJCTXvZqDytKoPamIcojp6f5kLMWXlxBdaR/feustOBwOhMNh9OrVC/X19UhPT+cCRcqa4q2ESIlPKisrYzpC0j6Lryuk0A/NjdYkhlhIeIiO5FaLSkihUECv1+Po0aPo3bt3TAiAKmZo30lRisqFeJNCbQqFAn379uXNvYg3SLgJtYuVZJS/oPWKfEixcQC8ZXVGRgbfO+IdmovIv0qlEm+88Qaee+45SCQSpKamoqamBhkZGTHhJfJeiYfFtsniOtvb2/H1119j7ty5uP/++/Hpp58iLS2NK09aAw2xuiUYDMLhcMQoTOL3aDSKW7duYcaMGdz7EtvkklckJs7F0C4BGCo2IMMgfu8OIEivEOgSE72ZmZkxQIGeR3rFbrfjlVdeQTgc5joiGo3yxC/xiE6nQ1JSEjcOJEvEdzQnrVb73x5i+sEodwonALdDAX6/H9nZ2ZgwYQJfYDAYhMFg4AzR1NSEhx9+GO3t7ejs7OShDlIcYkmX0+mEx+PhhCJvoa2tDUAXwzidTq5kOjs74fV64fP54Ha7ce7cOYwbNy6mf0ogEOAuJ2OMd5CkUAltGiF3uq9Y6hWN3n7tGjFEJNLVfU58NZroVYhCDwA7duxAZmYmzGZzTOyXnuH3+2MqUyhkQd0ZSSi1Wi2Uyq7XfFmtVkQiEX4IhpJFVqsVbrebr4/CY4S+iZYdHR38FCCt3ePxoL29nc/L5XLhwoUL2LNnDw+xXb16FY888ggSExP/4UQygJjwFtAVs96/fz8YY7yWnZSgzWbjxp/4zGq1or29nX+WfvZ4PHwvKcRFyh24HQOmtQ8bNgxlZWUxLaKp0sHr9cLlcvGKEZvNxg86AV1lmW63m6OympoaPProo1ypMNZ1mtLpdEKr1UKtVvMmb9RmmpKj1AWxs7MTjHUdGKMKFo/Hg61bt2L06NFcIRHooetEg2uz2XDgwAFMmDABANDc3Iy9e/fyroYUTiTeN5vN/J2tdG+SF5lMhp07dyIvL48Dgzlz5iAYDHKAxBhDW1sbV/TEuwaDgaNyiUSCzs5ORKNRGI1GTpcPPvgAarUanZ2dHNXabDbO06QzqD2yxWLhshcOh2GxWLiipfnR+lwuF3+bGNFJLIoQ38O7fft2jB49mv+fOsTStXS4jMpWyRNwOBz8lDK9F2Hnzp0oKSlBe3t7jOdL9yTeYoxh+PDh36lTfxAxd9FdJoulVCpx8eJF7NixA4sWLUJJSQnkcjn27dsHAEhMTMTMmTNx8+ZNuFwuHDlyBI2NjVi+fDmA2+EFoMuF2b9/P1dCI0eOxODBg9He3o6KigoAQEFBAYqKinDo0CG0tbXhrrvuQn19PW7duoWXXnoJFosFe/fuRVZWFsrLyzFt2jQA4EfbjUYjnnzySRw/fhwtLS0YO3Ys3G43zpw5g9/+9rccfdTW1qK5uRkSiQQTJ04E0NVO1ePxwOfzYebMmVAoFNi/fz+Pe997770x4QBiWrGU8NKlS1Cr1bh8+TIcDgfmz58PubyrPe2FCxe4YZw7dy4Uiq4Ws0ePHoVUKsWQIUPQv39/LugKhQI1NTXYvXs3CgsLIZF0naB96KGHcPLkSUSjUbS0tOCJJ56AVCrFF198gUgkgs7OTjz66KOIRCL44osveGvhhx56CGfOnEFzczPy8/O50pk9ezZXzF6vF1euXMGoUaNQUVGBkSNHIhzuOtBVUFCAkSNH8oZQM2fO5IiR9tlqteLSpUtobW3FwoULER8fD4PBgGvXrvGXpBQXF6O1tRUVFRUIBAIoLCxEfX09jEYjhg8fDofDgYqKCjzzzDMAwOO9YnXI6dOn0dbWBqfTieHDh0Oj0cDr9eLChQtob2/nr6WbMWMGjhw5gsrKSrz11ltYsWIFZs2ahX/5l39BS0sLqqurYTAYMHHiRBQXF8c06lIoFDh69Cj8fj9aWlrw5JNPIhKJ4NChQxg0aBDcbjdu3ryJiRMnYtCgQTh//jza2tr4ydP58+fjwoULaGpqgt/vx8WLF3m1xUcffYR77rkHvXr14gaXwIfdbsf58+djqj7q6urgcDhw+fJlfPPNN1i4cCEKCwshlUpx4cIFbqznzZsXU35MwKOgoAArV66EXC7HoEGDUFZWBo/Hg6+++go3b97EyJEj0dbWhmHDhqFfv35gjOH8+fNoaWlBKBRCWVkZAoEAysvLeXWM2WxGfn4+vvrqK/Tq1QvNzc0YMmQIfD4fvvjiCw7c7rrrLuTk5CAajaKyshINDQ3w+XyYMWMGTp48ifLycvz6178GYwy7d+/Gww8/DKlUioqKCrS0tMDv92PWrFkx3i/xwjfffIPW1la43W5cuHCBe07hcBjHjx/nVXiLFi3CrVu3eMM2q9WK9PR0KBQK1NXVobGxEQAwb948/mxay+DBg1FQUACTyYTy8nJe7kldXisrK79Tr/4gkLtYFUDumFQqhdvtxqFDh2AymaDVavHXv/4VSqUSQ4YMQUJCAjweD7Zs2YJp06Zh+PDh2Lx5M38LDFnaQCAAo9GIbdu2oaSkBGlpaZDJut6J+PLLL/Pj4m1tbbBarRg3bhxOnjzJ3z6+adMmRKNRZGZm4uLFi/jZz36GqVOnwmKx4Pnnn0d2djaSk5Nx/fp12Gw2jBo1CqdPn0Z+fj6mTJmCLVu28JDRo48+ioqKCsyaNQu3bt2CVCrFzJkzcf36dRQWFuLSpUvw+XzYunUrAGDs2LHcFaeErRhrpZhjNNrVtfGpp57C9OnT8frrr8NqtSIajeKll17C8OHDMX78eLz77rtQqVTo6OjAjBkz+FF3QlIUUmhra8Pp06dx7tw5HDp0CFOnTkVhYSFefPFFpKWlobi4GDU1NVAoFFizZg0KCwsxdepUmM1muFwuPP/888jJyUFSUhJqa2tRUVEBnU6H9evXY8KECZg2bRreeustdHZ2IiMjA9988w3mzp2LyZMnIykpCZs2bUJBQQHOnj2LwsJCGI1GAMCyZctgsVh4WScdvgoEAvjRj36EWbNmYcKECdi6dSuCwSD+8Ic/YPjw4cjKyoLJZMJzzz2HF154AaWlpejVqxcsFgvvcGmxWDBq1Ch+jJ54ErhdtVRRUQGDwYCZM2fC7Xbjxz/+MSKRCL788kvU1NSgtLQUwWAQVVVVKC8vR2FhIfbv389DWEajEc8++yxeeeUVjBgxAtevX4fRaERzczPWrl2LQYMGIRqN4i9/+QuysrLwox/9CBMmTMCHH36Ic+fOISkpCYwxTJgwAbdu3cKDDz6IY8eOobq6GmPGjIHX68Xly5dRVVWFpqYmzJo1Cw6Hg58clsvlyM/P572OKJxDcd20tDSMGzcO8+bNwzPPPAO5XI733nsP48ePx7Rp07B582YcPnwYoVAIf//731FbW4v+/fvzU9AU/gLAD5VNnz4d//Zv/4ajR49iwYIFaG9vx6VLl5Camopt27ZBo9HgwQcfxAMPPIDOzk48+OCDaGpqwoIFC7Bq1SpoNBqcP38e48ePx5EjRzB9+nRMnDgREyZMgNPpxCuvvIKSkhLIZDLcf//9mDdvHu6//3688847PIyxcOFCnD9/HtOnT0dqairsdjuKiorQ1taGQCCAbdu2Yf369QgEAlCr1Vi4cCFGjRqFAQMG8PAeyaBMJsP169fR2NiIWbNmcY9VLpejqqoKc+fOxaxZszB37lysXr2a95MifTN06FCYzWa8/vrrKCoq4rrJarXC4XCgoaEB9913H2bPno1HH30UJpMJy5cvx+zZszFmzBhUVVVxT+2TTz75Tr36g1DuQOwr0QilDhs2DH369MGIESMQDAaRmpqKNWvW4PTp0yguLkZ7ezvvP97Y2MiVKMU3xVh5ZWUlfv/73/Na5/Pnz0Ov18NgMHDkRgeEDAYDUlJSUFtbG4NmW1paeOy1pqYGR44cQUtLC8xmM2bOnMkrCgwGA++nTQzh8/nQ2NiI8ePHIxqNYv78+YhEbr9I+tq1a/jxj38MlUqFnJwcrFu3Di+++CKGDRvGXU4SQko8Ed0AIDc3FyUlJTCbzRg8eDC0Wi1u3LiBwsJC9OrVK+blHGq1Gnl5eViyZAn+8z//E7m5uTzkQ0I6ceJEGAwGTJkyhcfojx8/jqamJly+fBnTp0/HjRs38PHHHyM3NxdSqRSPPPIILl26hCNHjvAXDfzoRz9Cbm4uioqK0L9/f16FNHjwYOj1eoRCIU5/MlRmsxmRSAT5+fm4ePEiBg0axMNUdB3FTYPBIDo6OjBmzBhEIhG0trZCIulqAZyUlIRLly7BYrGgtLQUn3/+OZKSklBeXg6j0Yji4mIUFRXh6tWrGDZsGORyOerq6rjLLZYdRiJd/X1GjRqFaDSKhoYGHhv97LPPwBjjfeIfeugh5OXl4eLFi7xSJykpCUOGDMHRo0eRmJiI8vJyzJo1C2VlZairq0NHRwc0Gg0aGxvxySefID8/H9FoFG63G06nE4MGDUJFRQVvj2w2mxGNRvHpp59yFJ2SkoKHH34Ye/fuxejRo6FWq9HY2IiBAwfy6op77rknprJLPLwlkUhw8+ZNlJaW8sRmY2MjpkyZwgHT4MGD4XQ6sWvXLh5OSk1N5WEgMSFqtVoRDocxcuRIPPPMM3jiiSdw7tw5DBw4EIWFhejTpw8GDhzIcxS0f6NHj0YwGMTAgQORlJSESZMmwe/3c0+3T58+iEQi6NOnD7Kzs3kxgtFo5OdSBg4ciLS0NITDYTQ2NmLcuHGIRqMoKSnh7/El0HH16lXuuQJd+auXXnoJZ8+ejUmyU2h17969GDVqFGQyGerr6zFo0CAwxrBnzx4YjUae+xs4cCCi0a624b1790ZKSgqkUim++eYbHvI0m83o3bs3nE4nbDYbMjIyeM6B+Iyaq1mtVhQVFfHcXlNT03fq1B9EWEZMZIlJnsuXL2Pp0qXo6OjAzZs3cfHiRXz00UcoKyvjREtPT0c0GsWmTZvw4IMP4vjx47jvvvt4kocxhk2bNuH8+fPo6OjA3LlzsW/fPpw/fx4vvPACBgwYwHu0JyYmYuXKlbyx0aZNm/Dwww/jzJkzGD58OPLy8tDe3o6qqipcvHgRoVAICxcuRCgU4jHGHTt2oFevXpDJut5atGjRIhw/fhzZ2dmYPHkyBgwYAMYYdDodrFYrpk2bhsWLF0MikfAY7JUrV/D+++/j+vXr2LlzJ5577jnY7XZotVrOyKKraDAY8MgjjyAtLQ3bt2/H0qVLcf36dXzxxReYP38+1Go1tm7diuLiYnz11Vc4f/48Fi5ciPz8fLz22mtwu93Q6/XcYMTFxUGn0yEnJ4e/l5V6l8ybNw9AF5pdu3Ytb2BGyd1Tp04hGAxi4sSJUKvVPNbf0NCAxYsXIxqN4vDhw3j55Zdx48YN5OfnIzc3l3tMQ4cOxcCBA3H69GmUlZVh7dq1eOKJJxCNRjF8+HAUFxfzCgaKR3766aeYPXs2bDYbNmzYgLVr1+L999/HkiVLeIdB2uPf/OY36N+/Pw9DmEwmHD58GK+++irC4TBMJhOqq6uh1+tRWFjI8z8dHR3461//ildeeQVXrlzB8ePHUVVVhbi4OHz55ZdYv3493w+73Y6MjAysW7cOU6ZMgd/vR0FBAYYNG4ZQKITnn38eeXl5PJm3ceNGFBQU4OTJk7wRF+3v22+/jVWrViEhIQGff/45XnvtNUSjUVy6dAnLly/HSy+9hA0bNvCT0x0dHXjvvffwyiuvAACOHz+O1atXo66ujisXKjGl98IC4P2FVq9ejXXr1sHhcEClUiE9PR1TpkyBzWbD/fffz98ZcOLECWzcJpIWGAAAIABJREFUuJEbWvJuCAhIpVK89NJLePnll9G3b19kZ2ejT58+SExMRHp6OgwGA37yk58gMTERe/fuxauvvspBRVZWFq5du4bHHnsM1dXVKC0txauvvopf/OIXHCydOHECjz/+OFJSUnD9+nXEx8fj3nvvRTgcRkVFBZ566ilUVVUhIyMDkydPRl5eHpe7UCiE3bt3o6mpCTKZDF9//TX+/Oc/w2AwoLW1FUeOHMG5c+fwy1/+Ek899RSA2+dM5HI5PvjgA7zwwgsIh8M4duwYVq5ciaamJrz//vuYN28eZDIZqqqq8OSTT+LGjRtobm7GokWLeE7j8OHDKCsrg0QiweXLl/H444/zt60NHjwY0WgUf//73/Hyyy/jq6++wsyZM+H1evHhhx/i+eefx+nTp3HlyhUcOXIEQ4cO/Va9+oPoCrlx48alTz75JC8To8qN1157Dbm5ubDZbAgGg9wyFxUVYdGiRfjLX/6Cp556CkOGDMG6deswbdo0FBYWIjExkSecJBIJTpw4AQCora3Fww8/jMLCQuTn5+PTTz+Fw+FAc3Mzb//75ptv4le/+hX69++PdevW4V//9V8xePBgyOVyXL16FYmJiRgyZAiGDBnC4951dXVobm5Gbm4uVq5ciZ/97GfIz8/HiRMnuJLo168fTpw4AbvdDovFgmCw63V2x48fRzAY5Gg1JSWFv7uytrYW999/PzQaDcrKyvDTn/6Uh6wIbSkUCmzfvh0zZ86EVCrFO++8g4yMDN6J8PTp0zCbzbh69SpycnIwePBg3Lx5E3Fxcejo6EA0GuWthKlkSyLpemtR7969MWTIEABdOQmK+be0tKCxsRH33XcfT0obDAYYDAb+Nh25XI6GhgaYTCbk5eVh27ZteOCBB/jboXr16oVgMMi7Zur1ev6SiatXryI3Nxf9+/dHU1MTVCoVtm/fjunTpyM3NxcA+LkDiUSCnJwcnDt3DhcuXMCkSZNQUFCAgQMHYufOnQC6Xm6QkZGBUCjEX1tHryw8fPgwGGOYMWMGrFYr6urq0Lt3b47kqeQ1ISGBv3bu8uXLcLvd0Gq1KCsrw8CBA3Ht2jWYzWbcuHEDMpkMSUlJqKmp4cmvadOmQaFQ8M6BhMp69eqFEydOICsrC3l5eZg6dSqv5qqqqsLkyZO5x7Nx40YMGjQIp06dwgsvvIARI0ZgwIABuH79Ojo6OnD16lXI5XLk5eWho6MD165dg9FoRFpaGkaPHo2JEyeiT58+yM3NjSk7JI/O6/Xi+PHj/OU0Op0OmZmZGDJkCL766ivodDo0Njbi7rvvRn5+PmpqamAymVBTU8PvSZVFEokEu3bt4l1BL168iHA4jEmTJkEmk2H9+vVc2TU0NHCvlTozVlRUIBgMor29HRkZGdiwYQMef/xxbjxqamoQiURgNBoxdOhQKBQKnDhxAuFwGDU1Nfw1e2PGjOHdTY1GI65du4aCggJ4vV40NTUhISEBBoMBGRkZGDVqFBoaGjgf3HvvvRgyZAhHymTAEhIS4Ha7UVtbC6PRiKysLAwdOhQpKSno7OyE3+9HVVUV/H4/SktLee6jvr6eG0w6e9DR0QG9Xo/MzExotVpcuXKFJ8cXLlyI9PR0VFRUwGg0oq6uDj6fD9nZ2WhoaIBCocCuXbu+tSvk/5Vyl0gkjcuWLXti2bJlP122bNmTS5cu3SiRSJKXLVu2b9myZf++bNmyWcuWLTuwdOnS73xlyIYNG5Y+9thjMfWyjDGkp6cjMTERAwYMQEZGBi/9ufvuu/lbbkaPHg2dTge9Xo+cnBwUFBTwagCqy87KykI4HIZer+fxOXq7k0aj4W9KArp6uIwdOxbx8fHIyspCeno6CgsLeW11aWkp0tPToVQqeXvRzMxM/oYivV6PUaNG8f7fjDHk5+dDKu1qKyCXy5GWlob+/ftDpVIhIyMDarUaGRkZGDx4MKRSKfR6PeRyOXJzc9G7d2+eSS8pKeGVPhTGotxC7969eRlhcnIyfwVdIBBAQkICxo4di6SkJIwYMQL9+/fn/S3Kysp4X27xngB44oe++vXrx98wlJ+fj9TUVOTm5vKSrgEDBiAxMRH5+fkIhULIyspCQUEBj/f26tULcrkcffv25W8ooiZuycnJvKpCJpNhzJgxUKvVSE9Ph1wuR3l5OW+zLJYlymRdr7mjdrPDhw+HRCLhrZXj4uKQmZmJrKws9OvXDyqVComJiUhNTUVmZiYUCgWGDh3KS/2USiVGjhyJpKSkmJI82iOfz4cRI0YgNzcXQ4YMQUZGBn/jlUKhwODBg5GRkQGZTIacnBxkZGQgNzcXaWlp3EhSNdKQIUOg1+vxv5h78/Ao63N9/J6ZJCQzWWYmK9lISFjDTkAExIoCVkVpqSgoLse1bm3VVnvq17bWU63VWq16XAABFxCr7CDIJmENi4EsbCH7HpJJMjMJITPz/v6I95Nncqz9fk/P+V2+18XFZJb3/SzPej/LJz09XXp2O51OZGRkIBAIwOl0YtKkSQJllZWVSdwoKysLQC9EwWKgzMxMZGdnIz09XYTAiBEjYLfbMWLECAnGORwOyfMmLRG2SU5ORkREBDIyMmC325GQkCCHQft8PmRnZyMyMhLx8fHwer1ISEhAenq6rDfQB7FmZGQgMzMThmFg8ODBGDVqlPRUWbJkCRYsWADDMDB+/HjZV9La5MmTYRgGxowZg7CwMMTHx2Po0KGSKeZ0OuHz+ZCUlASHwwG/3y9ewahRo2Cx9LbDttvtMrbExESkpqbKeQ+ZmZmIi4vD6NGj4XA4kJ2dLXuZnp4uJ1UxiMorIyMDFy9eRFxcnJzHQF6NjY1FQkICxo8fDwDIzs5GQkKCZPRwz5gs4XQ6ReADkBTH3NxcOVPV4/Fg2LBhGD58OBwOB4YOHYrBgwfj4sWLWL169f9Oy1+TyVQBINcwjAvqvZcAtBqG8aLJZHoagMMwjKe+6z4TJkwwdu/eLVYj81aB4OZG/TMkdE61rlrVxURsRgT0FS7w0lWatF50QQefx+ZD+nOgr8Md15BHt9F1NJlMQf1wmHmh2yxwXHoOvB8DVZcuXUJZWRlGjx4t82MvEXo6XA9Wy+mxMruGUIYutuAcmc5Hl5qQxzd7Kmum8375bN0mQFe8cu31WupMChay6Lxg3aOlvr4eX375JUJCQtDU1ITHH3/8v+Dg+l563GazGVarNaiVA4uvWJxiNpulKlFXX5IumAFBemPAjbTI/i5MNeU6WSy9xzi63e6gYhpCV7pWQUNauikX404UvBs3bkRKSgomT54sAk4XnAF9bbBJT8zu4LroQjvSm87A0pWPOveflZjcY73WpG2d481/zP3XFbGXLvWex/rcc88hPz9fUlY5Tj6f89J4tx476y/4TMYRdAq0xWKR3kpMdeTakPbofeuOpnocTLnms/g5x8R5W61Wac6nK1pJ+xUVFaitrZUamp6eHkm7Ju2FhYWJoRka2nuMo05rpmdtNpul+dngwYP/f235exOAFd+8XgFg3v/Nj8h4ZHYSpxbGFAwARADrQCwXiYsLIMjKYy4tF5dEQsansKWg1xWqWplQUBJ+4IJTyOmiK53/q4ta9Lw0zMK58f6EltLS0oKUnBauuhCFgpJpaSwA6l/MBCCogIOMCwR3B9SMp6v7mKlDWIbrwt+SKblvep46SM0gFZ+rxxEdHY0BAwYgLi4ON998s8yNz+OlBQrXj+vB/zkmHbSnMtW5/3qu3DudH87ncg9JV7otBAApJCJtcJwsrKKg5fz7Vy7rmo/S0lLU19ejqalJ5sC5aZpjWqNuy6EVnl4n0oYukOPacM/4+/7rTGu/P/1yLcgf/FvTBgA0NDRg2rRpUttB/tRVs/ytpiGtTDgG7jXHrXvlcGxUVqQ9beTxHhw376ML+rSC0QWHgUBv4aVOAaWBAgT3h+F+k3/Ii9yHzs5O6Tvl9XrR3d2N1tZWyYtnlhwv7u93Xf+q5V4OwAXAAPCOYRjvmkymNsMw7Oo7LsMwHN91n/Hjxxv79+8XjcuN7V8WDQR3T+QGscMfiZOH55KwKby1xU/BwvnTJSfT8vfcBFa7sWqTRKatCloDTM/j3wDEAu+vEDheamMSDRmdhMMcZj6TRVAAxAug16AtarYc0JWYGrfXRP7Nfsm66DkBCLJUWGmn94P7pwUt/2eOLisZabVqQa8FKteJ3Ri15QlAGJiCmKl9HCutNT0GMh7XiLnfISEh8Hg8AuNQ4FIoaMWmrWadZcJDRFghrddRW8jE8NmAS3shpAv+ljRmGH19iLQF3X+vtJDnWHlP0jHphffgeLQ1r61U/oZ0CfR5CIZhoL29HRaLRQ6zZisJfodj4D5wPvQGtAKhQuFYaOxpz1lXhZMnWS1LBUI+Z+Un6cXn8yEiIgIej0fGReFOXtB8wvGQVpixpvvv9PdyuQbcG4vFIhi8x+OB3+9HR0eHZAaRvqKioqQ/FqEY8nNGRoacmsVgOGn6uyz3fzVbZpphGHUmkykBwJcmk+n0/+0PTSbT/QDuB4C0tDQRVBRKuocD0NdfAoAIZjImG++QwMmceuNopZBJqE11cyFtvWnXjkJBu+v8nBY7CZf51xSeJAYA8mygr9cMUxDdbjeioqJE6JHI6dqTuGhBkrlIzACCCJtwEf+nBaFhEApMrfgYiNaeCtdSu9pWq1WqQbk+VET9LUmmqOqyeN0YSzMmoQ/tedFN1jAL6YSChBBWIBCQk31Yhct10W65hlKoiLm29PaI//J9MjN/x7lqRaIFLwUSGVgLR9ImhZNWFECw16irbnlPrXC0xc/X2jPkGHSLagpPLVA5HzbZ094s15z7yq6drO7UkFx/j49z4x7QwPg2T4H7zT3i2vBvQo8U/BpG0TxJSIU0QKHt8XiktQTXj2uloT7Suf5ce936RDV+ppMSNB8TggkEAvB6vQD6LG/SAscWGhqK+vp6WS/Oqba2VmKQUVFRcDqdcq9/dP1LsIxhGHXf/N8EYC2AyQAaTSbTwG82bCCApn/w23cNw8g1DCOXKVrcLG0VaxePzM8F54J+8ywAvUKTbi8A0bzcMFqWZGDt6vM1hRIvjkULT46HSkO7jDqPWFsRtAa0m6/deQ1LaMue9wwNDQ3q+qgVHz0Z3oMEyTnrNeoPN5GxaDVqHFbPXTOEFiz8Tn/cVVt9at+DxqS9J86T99dQBX/L+fJiewmt/MlYnCP3hUqcAkTfk/un14F7o2lOCy3Sg4ZGuDcaztEWNpWI/i73hJf2qEhnhOe0Naz3mftPr470onFzbV1qgdkfrtH35JpoL5VrSiMA6M271z33ubZ6vvr3jCvQg9HwlYbRNA1pr1VDTXrPaJDwnlQ2VBz0KLQXo/enPyxHOuJztMLT4yUNfFvzOd3vipW/9MZIs0Cfxa+blfH95uZmNDY2oqGhAXV1dWhubobb7cZ3Xf9t4W4ymWwmkymKrwHMBlAEYAOAO7/52p0A1v+ze2nmJQbMDTh+/Dhee+01+ZxnOpLhOjo6sHDhQrGoA4GAnCpOwUVLLhAI4OGHH8bNN9+MgwcPYtmyZXjjjTeEYEgAJKKuri60tbWJBUDLJxAIoKysTCwYCjgKMTbS+uSTT8QLoSKgEiHxlpWVwWTq69hI2IIEpgM4S5cuxZ133ilrw8+112M29xZs3Xfffd+qHPz+vtOTSHxUdCaTSdZOQ0PEFgk5caxvvfUWOjo6ZD7aK+CeElrhGE0mE2pqarBixQqcOHECL774IpYvX46TJ0/ilVdewUsvvRRkeeoSeSpMdu/zeDxYvHgx9u7dKwKQwsXtdksDLjKUtrJ4vf322/j444+xZ8+eoKMDdTC9rKwMGzZswJ49e7Bz507s3r0bb775JsLCeo/5Y7tb7h37s1OYaeXOdacQ8Hg82L17twgrrWQ4VnpGpC+uMb+vG5Vt2bIFeXl5WLt2Ld5++220tLQEeQ4cIy1FjptKr7OzE3v37pU9poLtDwXRGiVdd3d3Y+nSpSK0qPx0nIZj1F7l+++/j/z8fGzatAkLFy7Evn37cP78eTz88MPw+/0oLy/H3/72t6AYgI7LEXahQNWQK2koIiJC6JM8Qm+StEAhr2FHYt1A3zGgpEV6m9pCJ69ow6y7u1vSb+ndGoYh1jnHzjoMegSMmwBAbW0tzGaz9ESqqalBXl4etm7d+p1y9V+x3BMB7DOZTCcA5APYbBjGFwBeBDDLZDKdAzDrm7+/exAKc9MuEBeBjYU0gVD42O12nD9/XrrjaZye39FWbF1dHQKB3p4XY8aMwddffy0brBlPa2jikrSWzWYzVq1aFRSo7O9CdnZ24vz580GuNJ9BgRsWFoZVq1aJJqelqK023t9kMqG+vh51dXVBLiovDQ+w6k5b9Hx2fwyUvyVD6n3Q1oMWNiRS5jAzBsH5a6VgMvUdbk7BeeLECUycOBFZWVlobGzE2LFjJd31woULMi/tcWjsmOOPjo5GTU1NUB9tjRPTQjQpTB/oa6EbGhqKLVu2IDU1NUgAa8jkwIED2LRpE4YMGYKcnByMHDkSf/3rX1FTUxNkRVJ5kX7omfWHabTVyWwNfdCEtji1Z6YVUn+a4vpu3rwZhYWFGDNmDIYNG4a8vLxv3VeuEQUihSEVJytNNY3p/Tebe89HKCkpEePFbrfDZrMFeT2a3mk4UXEZRm/TuJKSEskRP378uNSE6KwwbSnTM+d4NHRG/tTepzbyuAZa5ujvAH3ZMDS0KAt0nIVrQg9Ue4pms1lqMIBe+IXH6QUCAXl95MiRoAQQ7g0VMemEcoH/WCfT3t4uDdL+0fXfxtwNwygDMPZb3m8BcPX/472C8FkKIFaA3njjjQCC8Uz+6+jowLx584IWg7gxXWbii2azGcXFxfjDH/6AyMhI5OTkwGq1wu12y3FmJDxuWn+cNiQkBBcuXIDT6ZTqPgoLCp2enh44HA688MILQRk/GscOCQlBS0sLnE6n9MnhpvbHPGkt7dixA3feeWdQwEd7MiTQLVu2SEk4LTXCOYFAQAJKVIY8xouMR0FJQUDC1hhzaGioNGnTcJYOdBKD1xh6c3Mz6uvrcdNNNyEkJAT79+/Hu+++C7/fj0GDBklgTgtB7dJSYPp8Ptn7sWPHikAh3XAv6Z1oS03HF9LS0jBt2jSxAjUmXFdXhz179uAXv/iFHGcXERGB2NhYzJs3LygATMFGy5UYL5WkVhoa1omJicGcOXNknPTsOBcNKTG2wDXlXvCeTz/9NNauXYvIyEgMGzYMU6ZMkTiFjiHpgGdXV5fwhmEYcDgcmDdvnuwBaZbCmrSwfft2/OAHPxD+sFgsmDlzJurq6pCUlCSxKD6Hni/H6/P5kJeXh7lz5yIiIgI7duxAbGys1IaweGjPnj246aabpBkY+ZC8RyHM/dbrroWjVlZcLw3B6kCphmxotPj9vTUx/Js0xWdog1DzgdfrFU+jqqoKDQ0NqK+vR3l5Oa688sogRIDzcTqd0taavMWAf//g9Hdd34v2AwCEYAGIBXj48GGBImbNmgWPx4PPP/8cQK/bfffdd2PLli1ITEzExo0bsWnTJrz77rtBViyzKCwWCzweD1JSUjBp0iS0tLRg9erVUtRw6dIl7Nu3DwAk62LSpEnYunUrwsPDUVdXh5tuugmHDx/Gpk2bpEthINBbKhwTE4OxY8fiq6++wrx587B+/Xq43W786U9/gmEYUsEIQA7IXr9+PcaNG4evvvoK06dPx549e1BfX4+EhASYzWZMnToVR48eRVVVFaKjoxEREYHrr79eBLF29SiIEhISsHTpUjzyyCMwDAMulwsHDhxAXFwcWltbkZKSIn0s6uvrMXbsWLS1tcHn8yE2Nha1tbXIysqSHiomkwkrV65EdHQ0ioqK8MQTT0gnR5vNhquuugrr1q1DIBCA3W5HeHg48vLy8Pjjj0tfe7qXLNa49957RSHpgPbkyZMBAOvWrYPdbkdFRQWmTJmC7OxsVFZWory8HCEhIYiJicH48eOxZcsW3HTTTSgtLcWGDRtw3333wW63BzFnT0/vgcMHDhxAIBDAmTNn8Otf/xp+vx+rV69GRkYGjh07hunTpwuz0xq8/fbbsXnzZmFACuff//73yMjIwObNm3H06FE8/PDDuHTpEt5++2089dRTOHz4MPLy8pCTkwOn04k9e/YI5JCSkoLTp0/j0UcfxbZt25Cfn4+HHnoIMTEx2L17N3bu3Inf/OY3WL16tdD//v37pYoSAKZMmSICl0Kvp6cHSUlJeOCBB3DLLbdg4cKFuP/++2UP169fD4fDgaNHj+Kpp57C2rVrUVxcjBEjRiAuLg47d+7E5ZdfjiNHjuBXv/oVbDYb1q9fj8rKSowfPx5lZWWYNWsW4uPjsWXLFrz++utISUmRIjMAcLlcIph0wBMIPqaOSkYrtfXr12PhwoXigd9zzz1obW3FRx99hEGDBqGiogIHDhzA448/jrCwMOzfvz8I9rnyyiuxfv16VFdXIz09HXV1dWhoaMA111wDoA//nzZtmtAeIdsvv/wShw8fxmOPPYaoqCg8//zz0nfpjjvugMPhwObNmxEXFyf9ktavXy905nA48MUXX+DJJ59Efn4+nE4nGhoacNVVV6Gurg5ms1mquuvr67F3714MGzYMDQ0NQUZkeno6CgoKMGnSJHg8Hpw6dQrDhg1DU1MTbDYbvF4vOjo6pJJbn173bdf3onEYmUYHTi0WC+rq6pCfn4/a2loAwLFjx+BwOGCz2XDhwgX4/X4UFBRg8ODByMzMxM6dO8Vi0Dg+mZbtNs+fP49t27YhLi4Od911F3w+H/bt24fPP/8cI0aMQFZWFsrKylBQUIB3330X48ePl6PUhg8fjoaGBkydOhXDhw9HQUEBqqqqsGPHDgwfPhyBQABHjx5FR0cHDh8+LALlk08+wahRozB06FBUVFQgMzMTDQ0NuOKKKzBkyBAp7nA6ndKKFgA+/fRTOBwODB48GEOGDJG2p0BfwAsAVq9ejdLSUkyYMAEejwdjxoyBxWLB2rVrpeFUbm4uVqxYgeLiYmRmZmLbtm0YMGAAcnJysGPHDqkuPXz4sGQrHTt2DImJiRgyZAi+/PJLmEwmlJWVYeDAgdJXpru7G7GxsaisrMSoUaPg8XhEYeixAgjysDweD4YPHy6xA8PoPXbs3LlzGDt2LNxuN3p6eqRT5ogRI5CZmQm3241AIICTJ0/C4/GgtrYWycnJch8NM1y8eBHr1q1Deno6cnNzUVlZKdZjdHQ0Zs6ciVGjRgXFTegZnT9/XjwJXhcvXkRmZib8/t5WEV6vFz6fD8XFxfjyyy/h9XpRWFiI6upqqWAcMGAAli9fjhMnTiArK0tO6HI4HNIj3efzITExUXr4xMTEoLKyEoFAb3Mwu92OzMxMVFVVCVxFa5Tr+Ytf/AKjR4/G+vXrsWrVKvEQT58+jdDQUGRnZ8uapaWlSdO9CRMmAABiY2Oli6TH40FmZiYOHDgAu92OQYMG4euvv4bVaoXNZkNXVxcmTZoEAOJd1dbWynkDOpZDa1kHNzl+ejWdnZ0YPXq0BGqtVivq6upQW1uLtLQ0jBo1CmvXrgXQa/ytWbMGOTk5GDFiBKqqqtDZ2YnU1FTs3LkTsbGxSEpKQnh4ONauXYsRI0Zg8ODBqKyshGEYQZlVnZ2dcrg1jcqvv/4agUAAcXFx6OnpQX19PcrKyjBs2DDpl9TV1YX4+Hi0tLRg2LBhiIiIwAcffIBz585h+PDhsFqtcuIUAMljZ4HmsGHDEBkZKRXkTU1N6OnpwcGDB4NoIhAI4PTp04iMjERKSgoKCgrQ0dER5GH8o+t7Idx50QLhBs+dOxfjxo0LchNfe+01HDp0CHfffTfKy8uxa9cuzJgxQ/pqAH2FSz5f72k9xCQ3bNiAZ555BrNnz8bChQvx6aefShDt/vvvR2JiIk6ePImSkhLcfvvt8sx77rkHR44cQXR0NAoKCnDo0CEkJiYiOjoa8+fPR0FBAR599FHExMTg6aefxo033ojDhw/jiSeegGEYuPnmm5Geng673Y7ExET5zYEDB6R8ffPmzUhMTERISAgKCwtx6623Ij8/H8OHD8esWbPgcrmwePFihIeH480338Rf/vIXvPrqq2KZfvLJJ7jvvvsQFhaG2bNnY+LEiejq6sIHH3yAe+65R8qwDx48iAULFsDn8+FnP/sZxo8fj4qKCtx5550YP348Tpw4gSlTpgAAjh49irvvvhtXX301mpub4fV6YbFYMHDgQGzduhU33ngjAoEAfvSjH2HHjh24/fbb0d3djRMnTiAmJiYIy2QATWOheXl5uOuuuwRDNplMeO6553D33XcjIiICp0+fxrBhwyRd9KGHHsJvfvMb/OAHP8C5c+ewbds27Ny5E4ZhYP78+bDZbOjs7AQAcbGffvppLF++HDk5OYiPj0dHRwdaW1tx6dIl7N69G2PGjBFYjKc26bRA0hNdcQo/KrRdu3YhISEBH3zwASZMmACz2YwFCxagoKAAV1xxBRISEvDUU0/BarWipKQEzz77LIYMGSIpvEeOHBFPLSoqCgsXLoTD4UBWVhZuvfVWbN26FbGxsRgwYACKi4uxcOFCUURAH45eVlaGuXPn4t1338XWrVvxyiuvYNmyZdizZw8WLFiAH/7wh4iLi8N7770n+1JQUICrrroKUVFR+O1vf4sBAwbg8OHD0ibDZDLhgQcekP7i9fX1CAQCeP/996WfC5/vdruxadMmaVnRP5DOAh0qcR3kbWpqwsSJE3HZZZcF1UF89NFHGDt2LJxOJ/x+P6ZMmQKLxYIvvvgC8fHxKCgowLFjx7Bw4ULh40ceeQSTJ0/GzTffjNGjRyMuLg5Hjx7FkSNHMHv2bDQ2NsLr9cqBMzwMZffu3aJQ5s6dK3E5AHjiiScwZMgQnDhxAgAQFRWFG264AVu2bMH2liLOAAAgAElEQVQtt9yCuLg4PPzww/jiiy+kGygVANMuA4EAYmJikJqaisGDByMjIwM2mw0jRoxAYWGhtJMAIE3bxo8fj+7ubmRmZsJmsyEyMhLNzc0Cqzmdzu+Up98L4U6sWWe8XLp0CX//+9/x3HPPobCwEJs2bcJnn32GzZs3Y/Xq1di0aZNYcxaLBUuWLMGjjz6KDRs2iAU3YMAAOJ1OmM1mVFVV4cMPP8SYMWNgGIbg5uwSFxoaioULF+L666/HnDlz0NDQgLVr12LFihV49tlnsXr1aoSEhOC1117DpEmTUF1dje3bt8PpdCInJwfTpk0TpvF6vbj22msFy+3q6sLVV18tn7e0tOCNN97AxIkTUVNTgy1btqCmpga33XYbrrzySsyaNQvt7e04ePAg5syZA7PZjHfeeQexsbFYt24dFi1ahDvuuEMyZ9rb23HllVfC6XQiLy8Pd9xxB3bu3AmXy4UrrrgCycnJciLPK6+8goSEBHzyySeYNGkSfD4fVqxYgbFjx6K7uxvvvfce7HY79u3bh3379klO95IlS3D55Zdj48aNiI6ORl5eHk6fPo3y8nKUlZUJc3z88ccAgPz8fAAQ6ICZJRQIPJ1m6tSpYuE3Nzdj+/btiIuLw969e3HkyBEcPHgQv/vd7zBp0iSsWbMGdrsdFy5cwEcffYRRo0ZhwYIFePLJJ+FyuYLKv5k/vnfvXvzgBz+AyWTCsmXLUF9fD7vdDr/fj/3790vWSlhYmOC2DEKztSxx9c7OTqxfvx5mc29rg88//xzNzc0AgKKiItxxxx1oampCdHQ0cnJycOnSJcm4ys3NxebNmxEbG4vXXnsNERER+N3vfoc5c+Zgy5Yt4mVdd911CA3t7Qy6YMEC1NTU4Oabb8a0adMwZ84c1NbWStUiBXxDQ4Ocv8qYAr9/4sQJyU6xWCxobm5GSEgIPv74Y4wePToo2Pr73/8ec+bMwebNmxEIBLBy5UpMnToVhmHg7bffxqJFi9DV1YXDhw/jrrvuwldffSW8u27dOhGwhmHg9OnTCAsLEwiS2UzMQmG8KDw8HPn5+bj77rvhcDjE8u/p6cHu3btx1113ITQ0FB9//DEeeeQRadG9YMECTJs2DVdffTUaGxsRCATw8ccfY/r06ZJBxN7ow4cPx7hx41BaWoq2tja0tbVJNlVnZyc2bNggDb8+/fRTXHbZZaisrERhYSE6OjpQXl6OadOm4bLLLsO4cePgcrlQUlKCvLw8uFwu1NXVobGxERMnTkRubi5iY2Olb/yFCxdQV1eHtrY2ybKbMmWKrJPX68WBAweQnJwMn88nDcTq6uqQkJCAqqoqZGRkICYmBl1dXZg5cyYuu+wyWdvvlKv/WwL7//ViIEzn1H7xxRfiHtfU1KC5uRmtra2YMGECJk+ejJKSEowd2xvTraurk8MO+hciXLx4ESdPnkRdXR26urokaMUGYM3NzUhPT0dRUREaGxtRWloKr9crR4jV1NRg3Lhx8Pt7TxtKSkrCqVOnYLVa0d7ejvHjx0s/jJCQEBQVFWHMmDEoLi6GYRhIT09HRUUFXC4Xzp49i66uLrS0tGDgwIEoKSmB1WrFuHHjcPbsWbjdbpw9exYejwcjR44UAqmqqsL58+fh9Xqlmi0mJkbSF3t6euByuXD48GEAQElJCaKiotDR0YHGxkbs27cP5eXlmDBhAlwuF4qLi4WR9OHHLS0tOHHihDTBSkhIQFNTE6qrq5GQkCDl82lpaSgrK4PVakVhYaFUzh08eBCDBw+Wvuq0fumecj+Ki4ulsx8DvQMGDBDmOX78uFjvZWVlYjkSTuEaM0WzqKhIglrMjgGAq666Cm1tbWhpaUFZWRkefPBBCXzpfjAa4uJ78+bNQ35+PlpbW9HS0oKzZ89K8yqTqbc5WXh4OKqrqyVgHRoaCpfLJa2Je3p620F3dnbK0WvXXXcdgN4upaNGjQLQ6yFGRkbC5XKhoqICw4cPh81mw4QJE3DmzBm4XC6UlZXJmHVWGPmjuroaLS0tOHr0KBYvXoz09HSMGzcOqamp8Hq9KC0tRUFBAUwmE0pKSqQ3PvnlzJkzYvyEhoaisLAQLpcLNTU1yMzMFIjKYult1saj70JCQnD06FFcc8016OnpwcmTJ/HSSy+JYKdC57gJ1fn9vdWa+/fvR2xsrJyJy+eHh4djyJAh8Hq9yMvLEyNswoQJOHv2LNrb23H+/Hl4PB4AkANkuru70dXVhaFDh+LMmTOoqanB+fPn5SxlBuYJFzmdTlitVrS0tAhWX1ZWhuTkZERFReGKK65AQ0MDmpqaUFJSAq/Xi5KSEukUSXnS1taG8vJynDhxAocOHRLDp6KiQjwSZiJ1dHTIXtrtdnR3d6OtrU08XPJLZmam9KCpqKiQdufMPvuu619qP/A/dU2YMMHYtWuXEBoJgC7UoEGDYDKZUFdXh46ODumsd/jwYWGCoqIiBAIB5ObmikvOBXK5XDh9+jS6urowZMgQDBo0SI7xS09PR2xsLNra2lBRUYGwsDDExcUhOjoatbW1cLlciIqKwsCBA+UcxvPnz2PUqFFwOBxyhBuzJZjtcurUKdG4PT09cu/Y2FjExMTA5XKhtLQUEydOlDzcgoICEZzsU19YWIiQkBAkJyejra0NWVlZQQVEFIrFxcWyNuXl5Rg8eLC0M3W5XIiPj0dsbKzENs6cOYOcnBxhYvYaLygowKBBgxAbG4tAoPf4wJaWFsHUmZlSWFiIoUOHIioqSgg9LS0NDQ0NqK2tlaPAdD4+g3/V1dVyHqTNZpMufjyftKioCCNHjkRVVZUE7HgaU3JyMiIjI3Hq1ClkZWUhIiJChH58fLzAcTrTqLq6Gm1tbRg3bpzsD9B7tNkrr7wiqZC0JnUxVkVFBTwej3SSZK6/yWSSY/CYzujxeDB27NggeIdK5ty5c7h48SIyMjKk42RRURF8Ph+GDh0KoNfLOXv2LKxWqxz0EBLSezCM1+vFkCFDYLfbha45hsbGRpjNZjQ3N0s3Uh5iAfQmH5w+fVp+bxgGiouLMXz48KA+MSdPnoTZbMbll18OAJg8eTL+9re/wWKxYMiQIQB6BfvJkyfR1dWF3NxcRERESEvp6dOnS3C/rKxMmpwxNsDfM+ZBgeVyuaQ7KD1xoNdgS01NlQNZOjo6RLkWFxcjEAhItlloaCi+/vprjBw5UjKKvF4vKioqxACyWq0iG1jqTyXZ1taG9vZ2xMbGoqKiAoMHDxZ+8fl62wjHxsbC6XTC7XZL6wAeyt7e3o7Kyko0NTUJ1m6326XlBtDbVmDAgAGor69He3s7zp49iyuuuAJ2ux0tLS2C/dODoRKqra2VDpHE8mm5P/zww/+w/cD3Rrhv3749KKLOVDpd+s58Uwo3Xfmm04L4G7rX7N6nO0BSw+ucYRaL0KpjVJ+pULSQWXlKa53FMox662frtCzm0DKuwHvwuUAvw7rdbhH4upovEOhtJKY7UernMDMC6MsUISRCWIFCVOcCs8cK79c/v17nRDPgyHno/is6L5p7yXGx/J1z5Hro1Lz+OeFAX7Bd50dTCNMaYs2ATrPTfUP4TOLoq1evxowZM/Dss8/i+uuvx5gxYzBw4EARyLoARqecMTDPdeq/twzQ0Vrl9/p7MBwX98vr9Qo0xJxtrjWtM74mL/A7pGWm6ul0Sr2OPT094l31HzehHK4reWzx4sVYuXKlpIZyDbTl2N3djY0bN+L6668X6I20RDogb+nAOd/TvE1hRlqKiIiQ+IbeA/6W6YLMbiMEyvtwL3R+Oj16jod1IYyr6L2h59G/SLC1tRWdnZ2SpFFdXS3FWUAvTEbDxW63S7ZYeHi49I8qLi5GVVUVpk2bhqioKGlNwoxBZroRYtQ58KR3k8mExx577H+tt8z/yKWZEegrYOJmkCj43f7FHSRibbHTC2CEmkEa9oYg8/H+Whj4/X5JfSMGaxhGUDthjs9kMknObf/WCHQv+T2dJaBzmLUAITapx8f/KTz0mHR75LCwMHR2dgoWx/xyWqocN5UDc6VZFde/6lUXXHGPNLxCAtOFKrTUmOPM+3IPgOAOfhRaFID8js5R1wVs/C0LfyIiIgD0BVApJIG++gN6DSEhIYKB5+fnY+HChThx4oSc0EUrjJitzWYLqh/QQlA/h3vNNeVrXhTGWvHqKl6tIGmIAAjKUeczSUs6n5t7yz3ozx+kOe47vxcRESGKhcqks7MT7e3tOHLkCKZPny50QLoh/MQg6r59+zBx4sSg5m7aa9YGB8fC+evWC9zjixcvIiIiAhaLRTKRtMIiDdDI4n2IX1NhMQ1Xt4AgP1PA67RMGlBU3gkJCSIPWP/AcSYkJEiF7+HDh3HhwgVERkbCZOptPEdvIyQkRAQ8m/XROMvJyUF2drbkzlPe0KA1mUxB8UB+pnmCPPCPru+FcKdgBhCEJeniBF0erC14bWXReiMBc2EoaHiRySms+F0Wo9B150IDwb04SOz9rWcSVGhoqDANGYwKRTMSBURPTw/cbrccgEwBqS1fLcB1ChwFqq6oY1aCruzj/CmsaNmTGb1er1gm/RmJDMT70/pnUy6+p6trOVcSJfdWF2ZQ0epCrYsXL4olZRiGPIOFVhQc2mvidyn0eCA0GUZXABuGIemvFotFYKuvvvoKubm5cmITgKAGdrT2qJip1GhIUDnRqKC1RWHL+dFy1oyuLXubzSZjozDV3iR5g0JQ0x8Fan9+ogLlONn9lN6pFn4WS29l7owZM+T8XADSfIvz9/v9iI6Oxpw5c4QuuQderzeoJzxpib8nz+pgNc8ToHdBuuacSIvaetVGAhUflQyrWNktlsc96p5T9FJsNpvwBNdSrymL8TgH0iXrX3p6eqRKmggBlQLfo1wjHfCid6AVc1tbG8LCwgSa1YqQCjg8PFwOgPlH1/dCuFMIcFE4eQ1rkKi5yHqB+B5fk3h0EyNtkfNzXZ3GDaXQ1daGhlooJIA+YuNzqXA4BgpdegaEeHQeMH9Li68/xKItdy18Obf+1hkQDEvxPc5dW+caeuD4gD5PShOiZn666Vqg6XXgd/V66vn0t/Y5Fj6La8x91x4VP+dFJtQCQa8N15tCgXPkvbOzs1FUVITKykpERUUhPj5eaAHo60CqPcn+cwH60iV5aWuTypWwEe+r2yRwTTgPQmnEj7X3YrFYxH3XdKrpSQtRLQj7f86907zCvSEdcPykfx181lXBXF+9//TiSJcUlP09UhoDhFB05pIeN2WF9k6oRDQsRYNQH9LC52lITMMc+lCf/nCcNu4sFgvOnz8vQpj8oJWC7n/DveF3OWd6KPQ0+sObGsLj2AnpaqjsH13fC+GuNbS2LLQ240bQAiURaPgE6GsuRkvSZDJ9a4tPjWtyDFqo0GrXxNPd3S2WgMb7eQ/2Zec49HsaeyahkfnJyDqFk/PWEXHdVE1b5lrAcIwcLz/nPAAEQSSayYA+JUBLj9+jVQP0Hdah//b5fGKJci3JnGSwiIgI8ax0uTitV2ZI8Hf0prg/WuhSwfWHr8g83EdtPWrLkp7gxIkTkZGRgQ0bNqC4uBgOh0MOHabAYIYDrWltCQ8YMCCo9apWULy018D70mvifbifXBvWFGjYSu8l70uviXvF1/y/v+InPZGO9HgoVDkOHa/g87iWXq9XYD2uDT0sQkZasPUXyOSN/l4QW0CzxQWVmI6zAX2HlFCQkldpwNATbWpqEpiSFm94eLicdhQeHi5zplIgf3BNSKc9PT3wer04ffo09u7dK0qO68h0WpvNhvDwcEmy4H35fa6J7v2vDb6BAwfCbDbD7XbLsZY+X2/La9Z8cD7fdX0vhDsAwZ3oUmnrSwfSGhoaEBUVJQKQBEKmotDQQqZ/wJRET8bQwR4dAKJ7ry1xrQy4UQyokmi1xawxUSosEiIFVH+clMxOhtEQh2EYkj3hdDqDmlRlZWWJtUrYQlv/2tXUlqcW7Byrtka0ZcY1BRAkAJqammAymTBw4MAgRtbQEe9FC7W1tRWBQADx8fHw+XyorKxEREREUH45BQ7Hp+EN7omel25epi1tQglagHGMdrsdaWlpOH/+PEpKSqRSMhAICE5NAc0eLyaTCa2trWhvb0dWVlYQPKNxcaAPCuOa6pgLA/vd3d2orKxEWlqarBv3TNMw11YLXM6bwkUfGQn0nUzW35MivRlGbzyJ56SWlpYiMjISSUlJQQJeKyntbei9pjLnmDlWrgv3iTygaUpDghR6HD8/6++N0hugMUM+pMHAA24Mw4DdbofVakVYWJhYzRqL1/TKSxsTbrcbVVVVknLJS8eX2D6htbVVxhwVFSVzIbzIsesgOeeklS4NXSo18oLVav2nlvv3Ks+9u7tbGuSw/eWePXvw+uuvi4DfsGEDqqurZcIk0k2bNuGxxx4D0NebpqurCxcvXkRbW5sIfAZYAEjJMYmWQoeLxh7aWtPyn8lkkuAmLV0SMbFpMrB28Ql/kEGYJ6shA46RiojjZKD11KlTuPnmm+FwOIRYn3nmGbhcLjQ1NUn/HVq2GsriunINiOW9+uqroqRY8g/0MbVmMOYR854vvPAC3njjDfz5z38OIlSOXR/MoYUBAKxatUoE4cMPP4xVq1aJx6XdVO3FcT49PT1B+0Fa0MU03AOuOyENndMeFhaGa665BjfccAN27tz5X5Q0GYxeIb2Ljo4O3H777fJ9zpkWLNf5r3/9axCsxx4h3AugN6j605/+NEhwkRZJC/yuFpy6CpQwDgWPhtL6Bzh1IFHDX4FAALfddhs2bdokz6flSiUfEREh3iatUs6VhkVERIT8zX1jLEt3cPV6vSgoKJD9pBLm+GnA6cwYHQfRh3LQINPescPhQEpKCrKzs7Fu3ToJcpLfeE96n36/Hz/96U9lLblHxcXF2LVrF/Lz8+XkNypni8Ui7VDKy8tRWVkpKYtbtmwJgmPoVWuoSK8xeZQZQFarFWazWRSEw+FAUVGRQELfdX0vhDuxJLqVTCsKBAJ4/fXXJUDj9/uxadMmyXsnQwcCAVx77bXS950LRwIiUdJSpuCg5qYw4fN1WbXW7ux1TtyeykBbKGazOegAAlontFSI7fW3OunaWywWREREyKZSw5ORfD4ftm/fjmHDhmHgwIGIjo5GVlYWZsyYgcjISCQkJGDhwoVB1izdUs6PgSQAklvOoBAFEIlWB8No9bKDJueydu1azJ07F3/9619lTwnb6OfQEibkcODAAdxyyy0yZ6/Xi1tuuUWIXge4KFQ4FsIXVBS6n5AWANotpuVOmiMuymdlZmaitLQ0SDFTcPE1GdFsNmPv3r2Ijo4W4U63OSwsDJGRkUILmzZtEouUe8Cx6fVi/jLXjbABYUb2FeLc+CyNq3PPqSQoaLWhwedrXJjHC1qtVkRFRUnLDyoHehtRUVFSnEVcW1vMWrGbzWaBbqjwdJomAOzatQsxMTGiiK1Wq6ydhvDIO/yfXggFLHmJe8Dv2Ww2GIYhFamxsbFBZzDQw+WYm5ubMW/ePOTl5eHAgQMoKipCfn4+Tp48KbnwnZ2dqKurE6XjdrsxcOBAHD16FB988IFkx6SlpWHSpEmyboSEiL8TYmEqpPawOFc2CAsJCUFERAQuXLiAkSNHCtT8Xdf3CpahUCHUUl9fj4qKClxzzTWy+dXV1WhsbERXVxfS0tLE4q+rq0NycrJgbjxc1uFwICoqCoFAQCwuBqxo1ZaXlyMuLk7y2H0+H2pqamC1WhEbGwu/34+qqiqkpKRI5WX/wgOfz4fU1FQEAgG0tLQAgMANZAIAUsQTFRUFu90uwpTpVgAk3YwES+XCDJfi4mLp/1JSUoLc3FyMHTsW4eHhOH36NLKzs2EYveX87JLY2tqKyMhI6fzY09OD1NRU6QXjdDpRU1ODxMTE/wIJUBh4PB40NzfDZrNJtSp77euUM3okGk5ra2uDy+VCdHQ0oqKiAACHDh3CDTfcIPPPzMyUFDQd19B4K+9XVVWFQCCAxMREtLe3w+fzweFwoL29XYrF9DwqKiokLY0C6Pz587BYLEhISEBbW5vgmy6XCzExMfI8DUWZzb3FQh6PBwUFBRgyZIhYXV1dXaitrUVkZCTi4+PR2tqKrq4uREdHw+12IyYmRui1oaEBcXFxck/SS0VFBUJCQpCYmAjD6K16vHjxolTD8iorK0MgEEBycrLsT3t7u3hUycnJAPpOSHI4HJKuR8+Wz6YBUVFRgZiYGAwZMkSqn/1+P5qbm9He3o6MjAwAfTn3/G1oaCgqKipgNpths9kEXuMYDKO3SZzL5UJPT4/00mlsbMT+/fuRkZEh9E5FQiVosfQWO3V0dMBkMklxltfrlaKt+Ph4oXen04lAoLf4LiQkBJmZmaiurkZtbS2cTieqqqokI6qtrQ0ejweJiYkIDQ1FY2MjDhw4gAsXLkjxIau3W1paEBkZia6uLkmTpXFJhcR+NW63GxaLRfjE5/OhpaUFVqtVAuQsvKKCIM4fFhYmBX6EXqh8Acih4klJSUHw0bdd3wvLXVu9QF+yv9VqxeDBgzFv3rwgqCIsLAxutxvLly9HaGgoPvvsM8ydOxerVq2CYRh48MEHsWfPHhQVFeHll18OslRJMD6fD5s3b8bzzz8Ph8OB+++/H83Nzdi2bRv+9Kc/ITQ0FE8++ST+8Ic/YPny5WhoaMD8+fMRERGBXbt24f3330djYyMeeOABWK1WnDlzRtoBFxcX48c//jFaW1tx/PhxFBQUIDQ0FAUFBbj77rsRFRWFl19+WSCoQCAgxAL05cvr4gsGAg2jt3I3JycH69atQ0REBEJDQ5Gbm4vPP/8cb731FpqamnDp0iWUlZVh/vz52LNnD3bt2oXnnnsOr732Gjo7O9Hd3Y1NmzbBMAx8+OGH0l9dY6wad3/55Zexdu1aREdHY/fu3Th58qR4P9OmTZM+JTpnnQr0+eefx7Zt2+B2u/HSSy8B6O09s2PHDrFMv/76ayxevDgo44jEz1iFFvZ1dXV49dVXcfLkSURFReGxxx5DWVkZIiMjcdVVV+HDDz8EAHz55Zf44x//iMjISPz617/Gn//8Z3R3d+O1115DT08P3nvvPWzfvh0RERGoqKjAvffeGxQv0QFbk8mEdevWoa6uDjabDdu3b8fixYvh9/e2D37hhRdgs9nw/vvvo6CgAOHh4SgvL8c999wj67l27Vq8+uqriIyMxHXXXYdTp04hJCQEZWVlGDVqFAYMGIAXX3wRPp9Puo1euHAB8+bNk7149913RcDu2bMHFosFhYWF2LFjBwDgpZdeQmRkpPT8iYiIwAsvvCAGEmmKB08bhiFrtHz5ctx1113CIy+88AJOnTqFnp4e/PGPf5TAOT0EBi9ramrw4osv4uTJk/B6vZg9e7bQQkVFBT755BN0dXVh//79OH2696jl8vJybN++HYmJiSLQCfnRMq+oqMCGDRvQ09ODZ555Rg6J+fzzzxETE4MDBw7g7NmzMJt72+ree++9KCgoQGdnJ+bPny9B8U8//RSzZs2Swz9eeOEF/P3vf8e5c+fw05/+FAcPHsTHH3+Mc+fOYcmSJaiqqsLRo0fx6quv4vjx47BYLHjnnXcERo2MjIRhGNJHhrnrM2bMwLvvvou3334bhw4dQnR0NBobG+FyufDb3/4WALB27VqsXLkSHR0dKCwsRGhoKP7jP/5DPLzm5mZs3rwZTU1N8Pv9eP7558XD7ujowMsvvwwA/zTP/Xsh3IE+ods/3Wn8+PFIS0uD2dyb08xGWLrbGosBhg4dCpPJhL1798rpMNdee20Qvgr0pV7+53/+JzIyMpCQkIBJkybB4XDIe0lJSUhOTsaGDRsQFRWFwYMHIysrC6mpqZg9ezaSkpIwZMgQDB8+HHFxcRg8eDC+/vprXLx4ETk5OUhLS0NkZCSOHTuGjIwM9PT04Ouvv8bevXuFWWlJARA3l+6ZhhmAvtPnu7u7kZCQgISEBDQ2NiItLQ0Wi0WOnuvq6pIDSDIyMjBkyBDMmjULM2fOxEMPPYTPPvsMXq8XXq9Xys/PnTsnfe2Bvlx6rtnFixdx7NgxzJkzBykpKZg6dSry8/Pl+MKcnBykp6d/a3CNgdJrr70WKSkpyMzMhNVqRUFBQRAsUlBQgGHDhgUF3nw+n2DcOkvE4/EgOzsbTU1NyM3NRVxcnJT+JyYmircUFhaGt956C+np6UhKSkJSUhLWr18vzaI6Ojowffp0jBw5EjabDVVVVRg/frz0UOmfFWWxWLBs2TLk5uYiNTUVAwcORHZ2NgBIp8Tk5GTExMQgLS0NdrsdNTU1GDt2rEB6K1aswLhx45CSkgKbzYaUlBSYTCacOXMGt912G1JTU6UF7Mcff4zrr78eSUlJgvd7PB6sXbsWXq8X7e3tyMnJgdvtxnvvvYecnBwAEOv10KFDuHTpElpbW3HdddcFBWUJ5YSEhEifGY598ODBMJl6+88MGjQIU6ZMwdChQ2XNqHy5Ln6/HxkZGWhqasKECRMERgN6ocBVq1Zh5syZSEtLQ25uLo4cOYKwsDApq2c+t4ZAufZr1qzB7NmzkZGRIXP96KOPMHPmTMTFxWHSpEk4ePAgurq6kJ6ejpSUFIwbN06weCrV8+fPIzU1FXa7HV1dXSgsLERycrLEPg4ePIja2lrpttje3o6BAwfCZOotUqT3T2ucSiMyMlJkkdPpRG5uLqZOnYqUlBRs2rRJYnOEZZ1OpzRL2717N0aPHo2YmBgAEG8tNTUVnZ2dSEhIgM1mE6iG8CBbE1RWVn6nTP1eCHdaAMTZiY2uWrUKCxYsEKz8q6++ws9//nPZ9J/85CdoaWlBVlYW7rzzTuTm5sLlcuH111/H2bNnce+996KhoQFut1uUBXF49tOeP38+vF4vnn76aenrsGjRIoSFhWHXrl0YN24cFi5ciP3798uzk5OTceutt8IKM04AACAASURBVOLAgQO477774Pf7sX79eowaNQqLFi3C8ePHcc8998DhcOCTTz6Bw+FAINB7GMVrr72GX/7ylxJNB/qi7RR0tMa1krPZbNKk67777sOPf/xj/OhHP5I2t3a7HQsWLEBpaangswcPHsRjjz2GsLAwZGRkIDMzE2+++SbefvttLF68WLIhLly4AJ+v92QjCmS6xREREVi5ciUeeughgXE2btwo/VCOHz+Oq6++OigQrtP+1qxZI21sN2/ejHnz5qGpqQlLlizB/Pnz5TjCpUuXwul0SgZNd3e3pJHSAmRswOl04vDhw2hoaAhK6/T7/dJD5kc/+hGqqqrgcDhw6623wufzYc+ePZg4cSKA3gDy8uXLsXLlSiQkJKC+vh7vvPMOMjMzxbNhEJW47oULFxAfH49AIIALFy7g1ltvhd1ul949s2bNQn19PdatWyeW81tvvYVBgwahu7sb1dXViI+Px5w5c9DS0oIFCxYgNDQUbW1tWLp0KYYOHYry8nLs3LkT27Ztw5IlS+BwOMDWHK2treju7sYrr7yCd955B4sXL4bT6cTKlSuxdetWpKamIjw8HM8++yyamppw+eWXo7S0FL/61a/Q0dEBoC99kkqzpaUFjz/+uDRY27BhgxT7PPLII1i4cCEAiOJjphrxfsIHR44ckUy2zz77DEOGDIHL5YLH48GSJUsQGRmJjo4ObN26FSNHjoTJZMJbb72Fm266SQKg9CoI+XzyySd48803ER4ejs7OTvz85z/HihUr8OabbyIjIwN+vx9bt27F0KFDMWDAABw7dgw/+9nPYLVasWbNGmRnZ8PlcmHNmjVoamqCy+XC+fPn8dZbbyEtLQ3Hjx/HunXrBEqKjo7GsWPHMHz4cGkKNnLkSMTFxeHUqVOYPn060tLSEBsbi9DQUDQ1NaGurg7t7e2C+yckJGDUqFG4+uqr4XK5YDabkZ2djR07duDKK69EaGgojh49ivHjx2PTpk2iHENCQjBmzBjx9lwulyjRpKQk6cG/efNmzJgxQ+Ct77q+F8KdwRMGXhh4+vDDD5GWloavvvoKFy5cwLJly5CZmYlNmzbh73//O0pLS7Fnzx4888wzmDt3LtasWYMVK1YgNzcXP//5z6UpGAM5zNYggTudTnR2dsLtdmPjxo3weDzSGKigoACLFi3C3/72NzQ2NuLDDz9EXFycFJQ0Nzfj/fffR2xsLDZu3IiioiJcfXXv6YIlJSXIzs4WZly6dClqamrwf/7P/8Fll12GX/7yl9Io6de//jWOHDkCoK8sm0FABuSYmdDS0oKlS5di1qxZoogo/ABg586d+NnPfobNmzcjLCwMH3zwATIzM4Vh6uvrcebMGTz//POYPXs2LBYLmpqacM011yAvLw/79u0TggL6slGSkpJgs9nQ1taGjz76CMXFxZgxYwYslt5zOwcNGiTFPmRM9n0JCQmRlsPLly9HXV0dtm7dCrfbjeuuuw6lpaXYvn072tvbsXr1alRWVmLlypXSC5vrTUVDWOajjz7C3LlzYbFY0NjYiEWLFiEqKgp79+7Fo48+iuPHjyM2NhYJCQno6enBqVOnsGjRIrz55pu49dZbUVpaij/+8Y9ihW3cuBEXLlxAdXW1YLkMrOsgV1paGurq6rBkyRJMmzYNS5cuhcViQWZmpljmbW1teP/997Ft2za0tLSgtrYWra2tsFgsSEtLE5qYMWMGPv30UwB9aaivvvoq5s+fj5iYGCQkJAj8OHv2bGzYsAG33347SkpK8O///u9Cb0OHDkV6erocpLFjxw4sX74czzzzDJ555hk8+eSTcDgcAPrad7DfSWhoKKKjo2EymfDee++htbUVH3zwAWpqaiTF0+124y9/+Qv+8pe/BAVwyVeBQAAffPABbrjhBgQCAaxbtw6/+MUvsHnzZnR2diIxMVHaGpw7dw5XXnklBgwYgJaWFlx//fVYs2ZNUDok1zs1NRUpKSkC1+zatQuZmZlISUlBW1sbVq1ahcLCQkyfPh1Ab+YVT6vauHEjnnjiCRw4cABr1qzBmDFjsGzZMrz33nvS6M4wDJw8eVLibkCvsQJAMPysrCx0dXXh6NGjyM7OxqFDh8QSZ599i8WC8vJynD17Fq2trQgJCYHb7cZPfvIT8cDLysowYcIE1NfXY/r06bjrrrukeVl5eTnGjh0rSRVfffUVxo8fj5iYGJSVleHGG29EZWUl3G43QkNDMXnyZJhMJnz00UffKVe/VwFVYssk9KlTp6KwsFACQBRiZrMZWVlZaGpqwqRJk7Br1y5UVlYiJSUFKSkpctr47bffLgcoMGALQAKiVqsVJ0+eRGhoKJKTk+X0lkOHDqGpqQn/9m//hvDwcBE0zNi5dOmSnEJeVFSE6upqPPjgg+KO5ubm4ty5c6iursaIESMkHevqq69GVVUViouLsWjRIgDA/v37MWXKFPT09Ii1oHPqiauREJubm+WEnuzs7KDqOZar8xQmXRDk8/kkiMtj9xik4ck6kydPDnKnGae4/PLL8dlnn8Hv98PlcuGBBx6A2WyGx+ORtC+uKzOTGBibNm0ajh07BrfbjXHjxuHUqVMYOXIkxo4di9raWkyZMgXNzc1itWRnZ+ONN97AD3/4Q+m7TpiKsA+x8ClTpsDn6z2QZfLkySIcWltbBcagRVdVVYXbbrsNoaGhmDt3Lux2O/Ly8nDvvffCarUiKysLo0ePxunTpzFz5kyhSXp6Pp9PuvrxRKLTp0/DZrNJ8PHEiROwWq0YO3YsYmJiYLfbMWrUKOkHzlS/oqIiREdH4+zZs4iJiYHD4cCdd94pJ2YtWLAA7e3tmDlzJk6cOIGxY8ciJCQEo0ePxrXXXgu73S4nZEVGRmLSpEm48cYbcfz4ceGRMWPGICIiAvn5+SguLsb9998ve8Q5+f1+WK1WLFq0CCdOnIDNZpOxJycn4+6778bBgwcxYMAATJkyRXqdEDoF+tJtw8PDMXXqVPT09CA9PR0dHR3S7fOqq65CWVkZ2tvb8eCDD4p3PmrUKDQ0NEirbhp2hGg4Lx40bxgGJk+ejBtuuAElJSVoa2vD/fffL3Sn2wRkZmaiq6sLI0aMQGpqKlpbW1FTUwO73Q6n04mWlhZERUUhPT0dTU1NSExMRFtbmxQgmc1mtLW1ISEhQboxNjY2CgTjdrsRFxeHzs5OhIaGStA3IiIC0dHR6OzsxMiRIyVOyINAAIjlPWHCBDQ0NMDlcolnmJaWBp/PJ0kRNJQI7cbHx6O9vR0pKSm4/PLLcfLkyX8oU78XXSHHjx9v5OXlSeMfnaBP64mFIAyo6spGAGLdAn2l2DqFEugjbF0gQaWiK/RYgcnn89m8n2EY+N3vfofS0lJ8+OGHUuZM4c4UK2a3MFWM49DpaBaLJSgwYhiG9GHXMQJm0BCi0Cl1QF/FJi1deifMASbDAH3ZOPwNhT8vWhucq85Lp7A9cuSIWOPLli0TjBPoy6Vmto8uO9cFLKwdCAQC8nu/3w+32y15vVqQ6PFzHSkQtDLjfXRhmc6Z1xdjGbyf7kPEsWuaYsCX1YVayHHtSUPsjcQEAN5T90UilMX10c3IdGVnZ2en5IpzbroYTmeb6bkT+iOvGIYhcRnyA9eWipl0SiVK3tBeL6FUph0z6H3p0iU51UrTdExMjGTT6PHr9fL7/dI1kXTC3jpcZ36XY6d3yHuRBsPCwtDe3o7CwkI5dIbQJoO2PKWNtQ4ul0tSKdkjPj09HSZT7wlcumc/eTYjIwMNDQ2IiYnBmTNnEBcXh4EDB0qfqJ6eHnR2dqK5uVmOZ4yMjBQPqv/h2HrPtFHDe3EPyaePPvroP+wK+b2AZXjR3aNw0XmfQLAg5oaTMfm3FmbM7aX25DNInDoopIs9uJBAX1EKszfIlEVFRUhJSZFncty6uIQpkhSeOnee2SQ6uEWsHejLzqAiYICJc+f4KVBI7LqghUqFn7GZkg7Okil0ERfQJ9g0RKOx9DfffBPnzp3DD3/4w6C8Y+4XhQ0FDy9CLFxT4uoUbBaLBTU1NTJ3ZtzoAjCuI4Nl/B2JX++9tjT7F4mRfngfzo1z1Xnofr9fsHi9/kBfEQrhg/b2dlE6fL5uZEYBpLOKuOcUUFwfjp00SoHG/QL6+t9wLfgdnW2lA5bMade0zu9z7zX/6dx1rbg5B9Iq78UzSonL0xLW1d+a5/m/fi6fwzFR0ZN+GQ/RtQt87ff7UVFRgVOnTqGwsFA+ozI2mXpbktjtdrS3t0tKKguWXC6X0FRHR4fMq6WlRc4G5pnKLJT0+fp6u3s8HrS0tEg8pbS0FD09PWhsbJRiL86NqZXMCqMM0YYI58l5cJ//2fVPLXeTybQMwA0AmgzDGPXNe04AnwDIAFABYIFhGK5vPvs1gHsA+AE8ZhjGtn82iHHjxhl79uwRa5yTY38XEiEtF1q1FLY2m02Euk6p0lV4tDz4PVoMtJD4HC00SGR0YSmoGHwko7C4ROdj08LWp75oTJr/aHkwM0SXjlMYG4YRlINPBcDsCRICBaruZkkcTzea4ue6So7C3mw2Swc9bTlzPagovF6vFDHxOxS+WiCQiHWBC61BEirXnczPjpxaqFAIcA+5R7rQTCtDzoVrTq+Ll7ZoOW4qB46bUBeVjO4tRCiAljBphEpetwvWRgEFDK000p4umKInw73mHPQpTFSM7CLJ9wgbkP6oeBh4Jh/Qq3C73bh06RIcDocIFhac0bOiBU0low0q8hdxfPbtIc3x2VxT7pU2rrRhxf81PEs+0gJN0w5ft7a24uDBg6ipqUFDQ4MoF6vVKqemcc/5fWaWcY8vXrwoMFtbW5sUH3V0dCAmJkbkQGdnJxwOh2RyXbp0CcnJyYKLky5Zqaz3hllmujiOgWrueX86Z/CaMiAiIgLh4eG44447/qV+7ssBvAFgpXrvaQA7DcN40WQyPf3N30+ZTKaRAG4FkAMgGcAOk8k01DCM78y25waTsLgoLGYiY9P9o2vHwqRAICDNhrQw4e80YVNRsIJVBxBJ0CzC0VWK1PxAr+C12+0iCDo6OoI+o+dAi1OXrms3VEMFhmFIgyoyD79Pa1EfuBsa2ttWmKXULJWm4OZakFG5fmRCvibMpT0UKgK67Lq0m68JV/Fz7XFx/zR0RIteW65k9P7Vm/QEaKFSgei/tbVI5tDxAv0sWs68Jy035lNT2VBY695GHJsueuNRh4S1uCYaNjQMQzr9kbapUKhY+7vhpDEqWGZ0UejzexSmQF8HR96LipV7oZvmkc6APniDNSXcDwoVrjPXpLOzU84OoKKmIvX5fHC5XKKsCDXoWAzpi/MnBMr1oQHGveN4tUFExcQ14H1qa2tRXl6OM2fOoKWlRQwQs9ksypi4N3Hr0tJSUZAejwdxcXFwu91wOp0wmUyIiYmR7CwqNyqFqKgoNDU1yYlaXBumQ3ItebYCaSEQCMDj8QhPkze5vpy79u65PoQvWW3Mdtnfdf1T4W4Yxl6TyZTR7+2bAPzgm9crAOwB8NQ37682DKMbQLnJZCoFMBnAwX/yDNk8EiA3SLvLJAadB01rjQxD60bjWCQoakCNY2nGIzRAItLErq1AoA+/JyFzoWl1UaDS0tLFQRTw/L0uzqEFrKECPptCTQsK4r9AX9MxCiy+5hppC5jEz/nxM+3pkCmJQ3PNKCA5Ts6b68A5UNloz4vP5qWtZhIuBZX2pDj+/u2XeVFZ8jU7K3LNuba0JMm0vKg8NGzHFEjOhe0OaJFqxcR11tYnIaj+86cQJo3x2RqWIEynlQv3hftK7F7TFp8B9EFrHJN27TlP0pFeB9KPHoumI/JsSEgIOjo6ROBz/nw26VobSfyc/2vviM/gPmnPQK8jhXJ9fT06Ojpw6tQp1NTUBNEoPSD2eGGwFABaWlpgsfS24bDb7YiPjw+CRFhJnJSUJCc7mUwmdHR0wG63i3DlEZukYd0lU3sWnBM9Oi1fNP9oo5PeilZoNBx8Pp94bN91/XezZRINw6j/ZnPqTSYTEy5TABxS36v55r1/etGKogXOyVJwuVwudHV1YeDAgUFWL4Wthj36C3agL+Clg020bnWPDS3AaTH19PSIu0Ym1VghX9MCoVbl4nM8ZAhuLF19nbnAv9nOk56Mxl7JMGRuWg9cBxKayWRCU1OTpG31twwoHPVJM7pqkUyp11L35dDMyzVgfxR6KtxHrgcVKItHEhMTg3BnjYHrQCSfQyGvFYVeLx1/oLXFSwfnNNMAfQe00xoHIBXBFOz0HDQUxHn2x8a1wNMemlaCZrNZxq0FM9dLrzMtONIfISL+lvMFgk820vCYTi4gH7S3t6O2thYjRowIUlQUUAyGEzrUcRke8sxx2Ww2gb4YsGRBFXmDa2Cx9PZTIqzK93kv3kd7uRcvXkR1dTWqqqrQ1NSE2tpa8doouPmbpKQkmM1mOVc1ISEBDQ0NuHTpUhDsyGpTVgkzf7yyshKDBw/GwYMH4fF4kJmZKWeqhoSEIC0tTbxBKqrw8HB4vV4EAgGBUQcMGCCWPpEJ7k9nZ6f0lWloaAhqD0FFQdokrk+l/s/a/QL/8wFV07e8962gvslkut9kMh01mUxHW1pa/ot7zEwPv9+P6upqHDx4EAcOHEB+fr7gzNx04u+6KRgJlVah7iKpha7P55NeK1xQ/paMfPbsWaxYsULcVWbG0LqhAuju7kZhYaGcr0gB4vV6RSgGAn2nKmlFo91sClbOCehr20pG43tAr4IpLi5GXV2dtDLlWHfu3Illy5YhIiIiyNqnINOBZm29E96h8tAWGxUp3UuuY/9e7jrwR6XLz3fs2IGVK1cGzZljACCwju6wyTlr4cffM4OKSurMmTNiaXMOvKfGfrl/xGcZ4DWbzcjLy0Nzc7N04+Q8qIx1FTHnyrUgnKENDO4916k/ZEOFS9rjxftq4aeNH9KLxne5pzSY9N5x7n6/HwcOHMAbb7wBoFeI0HvSjau0EuyvOHQzL3rNhmGgsrISX375ZVAcSkNQpGdtqJFfNETHvz0eD44cOYJdu3ahoKAA586dkzkSw7ZarUhOTobNZkNiYiJiYmKksIueodnc28ysvb0dgUAAUVFRspcs6mtpaZFgaVpaGtLS0qQuhZf2cjn2mpoa4Tv2k6eSo8FFHqYFD/Sd+aBje/p/oK+Cnfvt8Xj+11r+NppMpoEA8M3/Td+8XwMgTX0vFUDdt93AMIx3DcPINf4/5t48PMr6Xh++ZyaTPTPZ95CQkJAEAkkIO4ICsguKoggiCGhdW21ta3uqB2ttPedq5ViXumNRoSIgmyCi7CD7lhAIIetkz2Qyk8lMtpl53j/i/eEbfqf2vO95r9/lc11cCcnkeb7Pd/ms9+f+aFpRdHT0gJBCZ2enxKR+/vOf46uvvsKCBQtw99134+OPPx6QOORBoFXK5JYai2QclQeGWp5VesTF85+avKML9Pnnnw/YwNyYVEb8V1FRIQvIz6ubGrjRIUiNddMyZSyboQGSQXGzUzjwZ11dXejs7MTVq1fR0dEh70B3/PTp08LloYae+EwVVsoDx43FkmtVCVDhMDZPWBctIAohCgqGrbg+QP9GPXv2LK5evYqenh4pCqHQVhUYPQ51o6tWoBqe4f7p7u4WIis1JKEKWl60pKxWq1hanJtLly6Jt8aEs5r/4R77/hyIcaIm1Pg9BSaVHNedY1SFHO/Pd1VbuJFVUP0ZhTXnjfuIwoPGCAWaurdfffVV4aThvFI4UZDwvRmeYpxahR02NTVh4cKFEhIzm80Sz+fe4ecZflB7BavCnrKAyKMDBw5g+/btWLduHVpaWtDZ2SnvScZEJj5JwaFpmnj67HugaZrUeoSGhkqZP6vYzWaztNnMysqCw+FAbm4uenp6EB0dDY/HIzF07ku1LaDZbEZwcDBiYmJgMpkQEREhBWJfffWVvDuVotfrlfNNgd3X188bTxZLeoHcJ4zNR0REiNL+Z9f/V+G+A8Dy779fDmC78vPFOp0uQKfTDQaQCeDU/+SG3JgUjHTzdu3ahYKCAomHTZ8+XYQT8bR2u10OYFBQEDo6OkRbAjdirgyD2Gw2sbR8Ph8KCgqkqMnn88Fms8HpdEp8ddOmTQgLCxNmOCZxmZBiSMHtdmPRokUSw3O73ejs7ByQbANuWKUU/EQsUMAxCcbEKQUgcMOK07T+Cjq+97x585CbmysMc5rW3z39m2++wcqVK+FyudDX14f29nZYrVYRwBScHR0dsNvtAPrd7c7OTimw4PuSpoBrQaFOxakillwul1DD0jJlGbnX68X+/fuxYsUKcVVVN5lxXH62p6dHwjiqt6NpmswdBRHncM6cOQO8MdWq9Hq9QklRV1eHjo4OUUJMOPf19WHz5s0Ck2PYpr29XUI9Pp9P4q7d3d2wWq2ilCi4Oe/853K5ZG+qiTTej0yFFBis+WBXHu4T3oucQpx/CklVkHLsDJtQQG/evBmrV68WgUMlydL37u5uOSuqgm1tbUVbW5sI4+7ubnz22WcwmUyCb9+wYYMUqDF0w73f1dUFq9UqY6PSoWDjZ86dO4ctW7bg1KlTqKysxJAhQyRsoVIiq0ZDZGQk4uPjUVlZCZvNBovFAqfTKV6c0WhEZWUlQkJChJIhMDAQCQkJcDgcgqKxWq0wGAw4ffo0Ll++jM7OTin2c7lcaG9vF5gkY/R6fT8KiS34uru70dHRgba2NmRmZgoKzGAwyFmjN8sQInNCPDtdXV2yL1nkSOOAIct/dv3LmLtOp9uI/uRptE6nqwPw7wBeAbBJp9OtAlALYNH3G+myTqfbBKAUgAfAE9q/QMrwBYEbiSZ+TwH0+uuv41e/+hVSUlIwdepUuFwulJaWoqKiAvn5+bh+/TomTJiA1NRUtLe3Y//+/dA0DUOGDEF2djaam5uxe/dujBkzBs3NzWhtbcWsWbMQHx+P69evY/v27fjVr36Fnp4eVFZWory8HE6nE3feeSd0Oh2Ki4sRFxeHI0eOICQkBNOmTRNkA8fY29uLvXv3YuTIkcjKyoLL5cLly5dRWVmJyZMnIz4+fgA+monU48ePS7XcrFmzBghDo9EIp9OJ4uJiWCwW3HvvvWhraxNe6HXr1uHhhx/G5cuXUVtbi/nz58Ng6C/Hv3r1Ktra2hASEiLkVh6PBxcuXEBTUxMSEhJw6623wuv14siRI2hpaUFUVBSys7OxY8cODBs2TDbxkiVL0NDQgJMnT0qnoKqqKtx1111wOBzCwldYWChwsKNHj8JkMsFsNiM/Px86nQ7Nzc04cuQIDIZ+OtTMzMwB1mtvby9OnjyJ+vp6ZGdnw2KxSOFJXV0dIiMjMWHCBEmW1tTUoLi4GOHh4RgxYgTCw8NRW1uLffv2CWEacca1tbXIzMzE0KFD0dDQgDNnziAuLg41NTXo7u6WtTYY+ovKwsLCBsTm29racOHCBTgcDgwZMgR5eXmor6/HhQsXMGjQINjtdjQ0NGD69OnSjYrVy3q9HoWFhTCbzdi3bx+uX7+O5557DteuXUNtba3QCFRVVeH8+fMwmUzIz88XxVJaWor6+noAwKhRoxAREQG3243q6mpUVVUhNDQUU6ZMGQCRVFFb9EhoXGiahqtXrwpyhJY0Pcrdu3fDaDQiLS1NuFamTZuG3t5etLW14fLly3A4HBg6dCiys7NRWlqKw4cPIzo6GpWVlcjIyEBxcTGqq6vR1NSEwMBAzJ49WzzR06dPw+l0YuzYsUhKSsKhQ4eEyresrAyhoaFISUlBcXGxeOJUbklJSfD398eVK1eQn5+PpqYmdHd3Iz09HTabDW1tbUhKSoLZbEZbWxuqqqoQFhaG69evo6OjA0lJSUhLSxOmRk3TEBERge7ublRVVYnlTiF65swZOJ1O1NTUIC0tDeHh4WL0MPnqcDjQ0NAg7RnLysqQlZUl81FdXQ2TyYS6ujokJCSId20ymVBWVgaHw4Hk5GTY7Xa0tbUJEV5VVRWCgoKQm5sLAKJ0Ojo6kJGRIYVW/+z6l5a7pmn3a5qWoGmaUdO0ZE3TPtA0rU3TtGmapmV+/9WmfP5lTdMyNE0bqmnann91fxmIfmCVJK24kpISPPnkk3jttddwyy23yObweDx4/fXXoWkalixZgjvuuAN1dXV4+OGHcffdd+O2227DgQMHEBgYiD179uDgwYNYu3Yt5s6di/Hjx2P9+vU4cOAArl27hg8//FCKC+bMmYOwsDBERUWJ9VNSUoLFixdj7ty5+OUvf4nGxkbxCpg1P3z4MNLT01FaWgoAWLhwIdavX49JkyaJ9c64LsMx77zzDlJSUnDnnXfi97//vVS9MW5nMBiwefNm5Obm4rPPPoPb7caf/vQnKXEuKSnB1q1bkZ2djc8//1ySeE899RRGjBiBW265Bffccw/S0tJw6tQpzJ49G6NHj0ZYWBh27NgBna6fn8JgMAg/+969e3H8+HHs3LkTd9xxB8aMGYO//e1v2Lt3L8aPH4/3338fubm5GDVqFJ5//nk88cQTmDBhAgoKCnD58mW8/vrrePzxx7FgwQK89NJLOHTokHg5jz/+OO644w6MGzcOd999t7B9MiR08OBBeL1evPzyy0L+9atf/QpFRUVYvHgxnn32WVitVnR2duK+++7D+fPnsXTpUuHY37x5MwICAnDw4EHs3LkTBoMBy5cvh9lsxoIFC7Bo0SL84x//wM6dOzF27Fi8/fbbWLx4MQoKCiSBx2RYe3s7Zs6cKWG9J598Erm5uRg/fjz2798Pn8+Hffv2YcKECXj99dcxZcoUjBo1SjpJXb58GcXFxZg9ezZqa2vR3d2NI0eOYOjQofjkk0/g8/nwi1/8AlevXkVwcDBqa2tx6tQpLF26FI2NjZIgvXDhAi5cuIBZs2YJ7URISAjmz5+PU6dOYcqUKdi1a5dw7avhNuYXaO2qMeLi4mLceeedAt3j7w8cASY0TgAAIABJREFUOIDk5GS8/PLLGD9+POrr6/GLX/wCvb29+Oijj/DEE09gzpw5WLx4MZYvX46wsDBMmjQJHR0dePnllzF8+HCcOnUKJSUlGDt2LBYsWIDf/va3gjZ58sknMWXKFBQVFeHy5ctob2/H+PHjsW7dOuTl5cFgMODChQv46quvpDEGm5eXlZUhPDwcdrtdoIhDhgzB6dOncejQIWRlZWHr1q2orq6WhKXT6URwcLCQbl24cAHr16+Hn19/R6bq6moRrAkJCTh16pRQFzidToSGhmLUqFESuz937hxMJhNiYmJQVlaG4OBgdHd3IzIyEs3NzQgJCUFbWxveffdd2Gw2jBw5Eu3t7Zg2bRpGjhyJ2tpa6HQ3CskyMjJgsVgQGxuLkSNHIjo6Gl9//bVQYjQ2NiIyMhJbtmxBbGwsMjMzsXPnTgnJ/q+E+/+NS43T0n2ie2QymTBx4kSsXbsWzz33HI4fP47MzEyMHDkSqampyM3NFYu4qqoKhYWF8Hq9aG5uxsiRI+Hz+TBjxgxUVVUJ0VRFRQVGjhyJzMxMTJo0CQkJCRI60ev1+MMf/iAYVYPBgNTUVIwaNUo4myMjIwcgF/R6vZAK5efno7u7G1FRUTh69Ci2b9+OoUOHDkicAUB1dTXWr1+P+Ph4Cd/QqqcF5fV6ceutt6Knpwdjx46Fw+FASUmJhHIKCgowZcoUeDwe4WKprq5Gfn4+IiMj0dDQgPz8fGiahg0bNqC5uRl79uxBVVUVfvKTn8BgMCAsLAyvvPIKOjs7cdttt2H27NmorKzE1KlTodfr0dLSAk3TMHv2bHR1dWHu3LmIiIhAbm4utm3bhujoaHz55ZfYs2cPpk6dio0bN6KgoEDCKoWFhTAajbBYLCgoKICmaWhsbMTw4cMlrMOvGRkZGDVqFLKysiR2Tn4Svi+xxbW1tRg3bhzcbrcc+nHjxolLfvvtt6OyshIFBQVI+74ZhMvlQktLixCvzZkzB729vcjOzh4AodPr9fK33d3dqKmpEa/EarWKMpgxY4ZY/UajETU1NcjJyYFer8fWrVsxdepU+Pv749q1a4iNjUVKSgpOnDghsWar1Yr8/Hz09PRg06ZNuOWWW+ByuVBeXo6oqCj4fD5s2rQJkyZNgs/nw5UrV6QhSXNzM7xeL3bs2IGHH34YZrMZNTU1gjdXYaPqpWmaxK337duHkydPDgj5ZWRk4PTp08jMzISfnx+qq6uRl5eH3t5erFu3Drm5uZIvUI2RxMREREVFAQBOnz6NwYMHS1hq+PDhACDj//LLL7F3717ceuut0oFr5syZ0Ol0aG1tFWOA1nNUVBT0+n4isc7OTgQFBaG1tVVQJDabDUVFRdKUOicnB3l5eRgxYgQaGxsHdEZjOEvT+rHnLpcLDQ0NYimbzWZJpg4ePBgtLS0Sr6+rq8OlS5dgMpkkhETFSX4ahosHDRoEo9GIpqYmNDQ0iLWu1h1Q7lmtVoE4MkTT0NCAlpYWDB06FO3t7UhOTpb8ltp28IeuH4VwZxKFriQF529+8xsAkMRFeno6hg0bJkRAy5cvh8FgwNdff43nn38eBw4cwL333gufz4d169ahoKAAZ86cQWpqKhISEjBnzhy0tLTg7bffRmFhIZKSknDx4kX88Y9/RF1dHS5cuIBz585hx44deOyxxxAcHIzq6mo8+OCDiImJwVdffYXf/OY3qK6uFjeWiigtLQ0fffQR+vr6cPXqVfzud79DcXExiouLBXGhZts3bNgg8Uiv14ucnBwUFxdLMpTJkrS0NGzcuBELFy7E/v37UVdXJ/HaRYsWIS4uDi+88ALuvvtunDp1Cl988QXuueceaJqGTz/9FFlZWThx4gS++eYbLFmyBIsWLcITTzyBxMREnD9/Hk6nE7t27cJTTz2FdevWCU/5xIkT4XA48P777+OBBx5AWloaNm/eLIKR8dbnnnsOCxcuxMqVK9Hb24vW1lYsXLhQ3ikvLw8ejwcbN27EXXfdBa+3n9ExNTUVR48eFZSIn58fUlNT0draKqRqZWVlWLlyJXQ6HTZs2IA1a9YIHerMmTMxePBgXL58Gfv27ZOwS2BgIFJSUjBlyhRs2rQJDzzwAHQ6HZxOJ4qKirBy5Uqkp6dj48aNuP322yVJyfwGC97efvttTJo0Cf7+/tiyZQvuueceGI1GbNy4EcOHD8fJkycRFxeHjz/+GNOmTYPNZsN7772HoqIiXLhwQVhEe3r6+wDr9XoMGjQIb731FiZPnixCLycnB01NTfjggw+QlpaGq1ev4ptvvkFlZSUsFgs2bNiAIUOGwOfz4ciRI8KBcu+99+Kxxx7D0qVLhTmxr6+/0QOFBoWFCtf1+XzYs2cPnnzySdx3331Yvnw5MjIypN4hLS0N27Ztw8qVK6Fp/WyGv/3tb2Gz2dDU1ISlS5fCYOgvUhs1ahR8Ph/OnDmD5cuXIyoqCrW1tdixYweWLVsGn8+HQ4cO4bnnnoPFYsGxY8fwwgsvYNGiRXj44YfFgt26dSu8Xi/eeustQa6FhYXJmFwulygMJsDJyBgSEoLk5GTExMTA6XRixowZiI2NFXQKwRp9fX1IT0/H+PHjMXjwYMnVtba24tq1a4iOjkZNTQ3y8vJQW1sLr7e/R25UVJTw+zN+39jYiOvXryMxMRGtra1ISUnBlStXxANtbGzE5MmTERQUhAMHDggzKVFUPp9P9tq5c+ckWWu323Hu3DlMnz4dhYWFyM/PF7rgsWPHwufz4dixY0hKSkJfX5/E7f/ZZVizZs3/37L6//X1zjvvrFmxYsUAWF5AQABef/11CY989dVXsFqtklB97733xGo+fPgwfvrTnyI2NhZlZWU4fvw4Tp8+Da/Xi/DwcAQFBSEhIQE5OTnYuXMn9u3bh8jISAwaNAi//OUvMXr0aIwcORKnTp1CQEAAmpubER0djQkTJmD9+vXST/KFF14Qa4CWCQBZmPr6ehgMBgwdOhRVVVXweDwoKSnBggULBiSsfD4fEhISYLFYkJGRgX379sFkMiExMRGpqamyAQjx8/l8sFgsKCkpQUdHB0aMGIEjR45g0qRJ2LRpE7Zt24bIyEgkJSWhsLAQJSUl0pyXMcVJkybh9OnTyM3NxZ49e6DT6VBXV4f6+npJKK1evRoNDQ04ceIEMjIy8NFHH2HmzJlCVfrqq69ixYoVoqQqKyulfdj58+eRnJyMK1euIC4uDidPnpQ44+jRo5GYmIiysjKcOnUKZ86cgZ9ffys5NvmgxfLxxx9LH9U33ngD9913H9xuN/7jP/4DaWlpcDqdyM7Oxr59+5CUlITNmzeLhTd69Ghcv34dSUlJKCgowLBhw7Br1y6YTCa89957eP/992Ew9NMc/+Uvf8GqVaskeUoIpZ+fH9ra2rB9+3aEhoYiLy8PSUlJKC4uhsPhwJ49ewRbbTQa8eabb2L16tXYunUrvvnmG0RGRiIzM1ME5vHjx9HZ2Yl58+YJAickJETaFTY3N+OWW24RQbdlyxYZz+TJkxEeHo6+vj5899130uEoJCQE//jHP2AymWCxWGC325GUlISpU6ciIyMDhYWFAxAqLN2vqKjAxx9/jJEjRyIxMRFGoxEHDx7E0aNHMWrUKCQkJAgE9PHHH0dHRwcaGxsRGxuLjIwM1NXVSbu8zz//HG+88QZ0Oh3Onz+PsLAwXLlyBaNHj8aBAwfwyCOPoKOjAy+99BKGDRuGrKws5ObmYsuWLbJfYmJiEBYWhueeew4BAQEDqi7Z3ESn6+fxv3z5Msxm8wBhbTKZ0NHRgczMTAQFBeHChQsICgqCzWZDVFQU7HY7rl69Cj8/PxQXF2PmzJnQNA1NTU1IS0tDa2srhg8fjoyMDISFhaGiokJaDFJBNDc3Iz09HQEBAYiOjpb90t7eLu0Z6UFERkbCarVCr+/ndU9OTsa5c+ekGUlERITQDjDk++2332L8+PGIjo5GREQE4uPjUVNTg9DQUMHyjxo1ShpzlJaWIiEhATExMWhtbcWlS5ca16xZ8+5/J1d/FML97bffXkMrnOERo9GInJwc1NTUwOFwYPTo0ULF2tvbiw8++AAzZ86ExWLB8uXL4e/vj+joaJw5cwbTpk3DxIkTERAQgMLCQuh0OgwePFj4uAcNGoTCwkLExcVh+PDhcLvdyMjIQEpKCo4dOwabzYalS5fC398fXV1dyMjIgMfjwciRI9HV1YWJEydKA2vCL4nSmTFjBsLDw1FZWYna2lrce++9UqTBQhBN0xAZGYkRI0bAYrFg7Nix6OjowIQJEwSPTcWh1+sRFxeHs2fP4oEHHsCYMWPkb0jNmpaWhuzsbOTk5CA2NhYXL16E1+vFokWLYDQaMXr0aGRmZiI9PR3Xr1/HbbfdhtTUVAwZMgShoaGorq4WPvTz588jJycHHR0dmDp1KsaPHy/InaioKAwbNkw8rcLCQkFLZGVlIS4uDnl5ebBYLLjttttgtVqRnJyM5ORkRERE4PTp05g6dapYxEVFRYJkUsNRREUwJmkymZCbmwuDwSAJVbrJ9913n3SHCgwMxJEjRzB8+HBERUUhJCQETU1NsFgsWLhwofxdb28vIiMjkZOTMwB7T0RTcHAw8vLyoGkaUlNTYTabcenSJfT29uLuu++GwWCQxscxMTEYMWIE0tLSkJSUhNGjR4sgO3z4sIQqioqK4PV6JWzI8F1qaipSUlKQmZmJU6dOYcmSJUhKSsKYMWMQExODjIwMnD9/Xrr1FBQUQKfTISMjA7W1tYiJiRELOjY2FrfddpvAVdUwYFBQEM6cOQOr1YrQ0FCkpqYC6KeczszMRF9fH4YPHy6GR0ZGBoKCgtDY2CiGx+jRo6UA6I477hAakNTUVJSUlGDo0KGIjo5GfHw8hgwZAoPBgJycHHg8HhHAdrsdbrcbmZmZiIqKQnV1Nc6ePSvQSafTCZ/PJ31Og4KCkJSUhIiICHR1dQkjY29vrxQ0Em4ZFRWFzs5ODB8+XEAXiYmJ8Hg8GDx4sPTPjY+Px5UrVwTSy3vrdP20A6xKpeFCL8FutyM8PBxmsxmRkZHwer1ITExEUFAQYmJiBPHV1dWF8PBw6QDm9XqRlJQ0oFiPqBmfz4eUlBRBdRH51dTUhIyMDMTExCA0NBSNjY3o6+unE3e5XBgyZAiCg4Nx4MCBfyrcfxSUv4WFhdq+ffuEr8Xlcol2Dg0NHcC6p2ka9uzZg9/97nf47rvvBlSeqnErwsyCg4NhNpvhdDoH8JIQCqlW7QE3+GW6u7sRFhYGj8cjiRVaFjeX1tPSpoDifdQKM8L0yPvCYiMmv7ioVGzEIzNWSDgb4Zd8Z7WqEcCAKt+enh6YzWZJFjNHQOQE8fR8Zm9vLx544AF89NFHgtEnZhq4AcNkSEZt5K3iwBnPdLvd8kyOTVWIHDfnlJ8hhp4wSyp9leeHz6P7vmbNGsyfPx+vvvoq3njjDan8Y0EZC9y4J9SiOV5qHoXCnjFV4AaklqEc4q1VBcGaih07dmDhwoV455138NBDD8mh5lyocD5y2agFVswRfPnll5g/fz7WrVuHpUuXCtcMn0Vhd3NtB9eVewXAAAVKCKpKXcH7cowUcvQg1QI7tT6DF5FGer1eeM6BGwRiXFvGpXfv3o3y8nIEBgZKTUVHRwdcLpfw8vj7+6OlpUXCQEB/DcD169elmTqbobe1tUni0+Fw4L333sO0adOQnJwsbf0cDgfMZjNKS0sFnkj4JhUo559oGoZKExMTBXba3d0t8G1y1pCXpqurS5Qtcyf0Ljk36l7zevu7LdGip4FLEAafx2peQl6NRiMeffTR/xVx2P+Viy8N3OAeUcm9ePn5+eHAgQOIiYlBc3OzJCQDAgJEs7vdbvj7+0unE5XgyuPxDGjNxkOsfs8CHP6MiobkQMANXL5qaVNRkJOaFytqKcz5XhQE5H5XebKJbKCC4KGgEgFuVCCqDIhMUGnajfJ/9T7qoWRrvPDwcAAQ7O65c+cwbtw42eQUFmplJ59P74ZzR5SGWnFJYcmvHJu64YnvJ00qN7TKs8N15fypBVINDQ04cuQIVq1aJUKEh4DKkTFRjqOzs1PuwfGwUERdK1r0VMKkO1B5vUNCQiSE1tHRgfPnzyM8PFzCH3wHriP3D4Ut95xKquV2u3Hu3DlERkYKR7jqZXAPsUqU863uEZ4ftdqT+5o/V6uL+TtCcjnXnAO1aEs1QPi39GJ5jqigaczQWCouLkZlZaVUcjocDjnDNJ5IPMZ6BavVKsItNjZWmmLQG6GC8ng8OHfuHHp7eyU/FhISIh603W4XeCFpevkuNFgYJmpvb5f909raiq6uLqSnp8s8cr2Mxv7ObmrXN3qKVHIcG9DvSVFuUcGqRiyNmZ6eHtkvapEdPdAfun4UljubdajJVB4GWoacFLXEmxuB1hf5VNQNqfKb0DIiZJDWam9vr/AzAxDaUpU5kUJCrTrkAeAB4oZkQQKtMVKK8r3IC0JLRw1HcWFVAiEqDL6/SrTEv7nZuubPVa4UtdiDB4f/p9ADbtAaEIvf29sr92CmnvzkFPAUdBTs/L9er0dHR4esDalLu7u7JcnMNablybmmYuUa8DMUtHw/Vvdy7/DdTSbTAKoKtRJULavnAeReUNedBWFUwirEkIeaFAyqQuBaM4nGIiti9LkfKMjpIfDwUpHQ6lWpIG6ucOWaqZa0mkS9eV05h/SE1HsAGOAlcs55/tRqX1qZKucPgAGKgnuX73v9+nVs3boVbrdbvCh/f38kJCQICoaUAGFhYXA6nVKkyGpx7iPmxnQ6HaxWq1RMs7G0x+OBw+FAZ2cnOjs7Zb7ZTL2zsxNnzpyB2WyGy+VCaGioxM05f6Ghobh48SKKiopkz8bFxUlxWltbG0wmExoaGpCamioWNnMlvAdRPlTa9NiIzElISBDjk3uJ600DhEqO62cwGLBs2bIft+WubiAeNGbKVWwuhSI1GDeTak2oiRlOws0bWbV4KRCcTqegJijUurq6BnR+ASBUrKzG4wEwGAzSlZwVcbTSVD4SCl7ySdBSMJlMIiQoROim0QKgUOM88WAxSUMLnYefn1ddbgBSsamik26GAnJcFJwUbnxPxghpQdC15aYjTIwhFlXoEclAOB09HlpxDAHodDfoiQGIu07hrc4nQzWsxGQpOoUvk/RqKE4NU1GoU4lwjFS2FCpcT747lQvng6EMtZuY2oWJio6GAr0Xrgu9Kd6X+5e0sjcLZioqCnLVg1L5jWiMqEpNDSmq3oDT6ZRCO+4fKhWurcFgkLWg4KWSYsiOysnn66/6Li0txb59+wD0e+VxcXFS8TpkyBD09vYiLCxM9lh7e7vkFVgEWF9fD7fbLbkyhkJiYmKk6M1qtUq/geDgYAQEBMBkMuHq1asYPHgwzGazVCZHRkYiMjJS5j4sLExChKQqSEtLE0/MYrFg+vTpQlsQERGBtrY2mM1mtLS0yPP8/PzQ2dmJuLg4dHV1SbgpKSlJPAiPx4OIiAghC+Rzudc6Ojpk/QAMqGqnLPqh60ch3LmxaAkBEMuXFguLDngYaN1wA1PQqZYrIVvqxFFYq1a3Stur0hIwLMGDzL8HblAIM5RCxaBpmpAvUdEAkHvw/nwHChkmLfkcCi5aunyuim0lMRHnTH1/xvk4Rwyf0FJU76N6SjzI3FAGg0Goc2n58HkA5P+cDwo/fk+Fw/WkwAduxNepIEimRAWhJgQZTuN7UinfbM2q5f+0mKlo+Xf8qj6L+4lry7GrcXm1cpXvxMPHEnuG/KhsqCg5VnLC8L3J5Mj5UGmdmVfgPuNeVmtB1FwK9446x/wdFbVqcHDe1dAYBT0VuXp/ho84xyq0j3uWXpvqfQDAyZMncfXqVRHIYWFhEg6JiooSfn2z2QyDwYD6+noxWOglmEwmURZ2u11i3aQvYFiPXiWVYE9PD5qamgbMCYuACH5gopjngrkqem96vR7R0dFITk5GdHS0KB2n0ynnHein7khMTJRnkUaAeQHOE88hzw+fq8oHnhc1Ka6GZNTmM//d9aPAuQMDtRAXlIfvww8/xKOPPopt27YBuNFTNDQ0VEiKuKEYFuHhJLEQDzeFNSef4REKRmAg5zoLmbiRabXx9zwkDBExfs7n8zCqMWC206PFpWkayIwJ3BCoPT09Ysn99a9/FU+Dmpshh5sFGq1Ujl0V+rxURsuAgAAZExVaY2OjHGhicmnFU2C9//77cDgcwndy8+8ZJ2fMU3XdObdEGISFhWHt2rVYuHDh/3FAmWBvamqC3W4Xha8mYhkGYjhBpXqw2+1YvXq1wNb4O5XYDLhxEIEbyUcqj9DQUHH1NU1DfX09fv/73+Obb77Bo48+in//93/HwYMH8dRTT+HXv/41Tp06hRdffFF6n/JiolRFTlCpUqHxUq1xehJqUpl70OFwyLj5LO4VCtibk3gMKcybNw9HjhwRlIrKxsnzoCb61PyNavWrhpBerx9ALHbmzBkUFxfDbrdLyIOFT8nJyWK9+3w+1NTU4Ny5c+KdJyUlISwsTPhoCGFmEREL4LgHOjs74XQ6xUvhXiZCJywsDMAN44wKwGAwIDc3F83NzXC5XCguLsbp06fR0tICp9OJoUOHDjBODh8+jDNnzsBiscBsNksLRnq5DQ0NSExMlIp0GqdcM7Wwil8ZGqLRw8Q9140yg57BH/7whx+UqT8a4a4KIQpFoB/XuX79erhcLnR0dMjveMAZkgBuxAqBG+gRusT8ucfjkZgnDxI3NAU1Dx7j44x1qVY1v+eCUiEQpaIKU1rgKsmRqhgYM+fXjo4OHDp0aIDQ3bZtm8SMaSWqsX+GBvhM1StQ49mcY4YqVE+JeOje3l6sWbNmwMFR7wP0k6R98cUX0tScAhW4IcDa2trw/PPPizDiXFJIMH4PQFx4t9s9wLJmCMhoNOJPf/oTTp48KUqMio17hUlJ0jwwPBEaGor6+nph8DMab1AIc314aPh3XG/1nbxerwjrEydOYOXKlZg2bRrCwsKwfPlyTJ8+HRMmTIDb7UZOTo4geRhfZshGtUZpBHB9OMdcGypoNUENQBo5A8Dzzz+Pw4cPi3HBc8F9zzOjvpPRaER8fDw6OjqQnp4usEYiaUJDQ0WYqNBKCiYaLRROKq6eHgyF0rlz59DV1SVQQjUkSOOMqCyvt79gkbmR7u5utLS0oKqqCp2dnWhpaZGWiTExMTIW1ftm6IvKjUqotrYWHR0d6OrqgslkknAn2SBramoQExOD8PBw+Pv7Y/DgwTJ3DN8ZjUa0tLSgo6MDXq8XsbGxAvHkenm9XunOxjg9G2YzXEUvR6fTCZ1IdHS0jJfFWmpITM0V0qP/oetHI9wZX6Vmo3D68MMPkZOTg08//RQLFy4UjQf0E+kQ8fDfCUsAA4Qm3e3AwEBx8QAIjaeakLrZhaN7rFrvPIz8yrAAK+cAiPCl0mLHF1V4mc1mcfe7urqwc+dOFBYWimusaRr2798vh5zCjvFP3lNlkKPAYjKOQuNmS0tVRrSSv/jiCxQXF8sB45pQKHKjf/nll0hOThZ0gYqk8fPzw/79+1FcXCzCi+Ph+vl8PrGk9Ho9tmzZghUrVgxAflDp+vv748KFCxgxYoSEQ6hMAUhpt8p6SKXCWLcqgPjeXFvOARUGFSkFntvtFte/sbERV69eRXR0NPr6+rBnzx7BxZtMJkyePBnbt2/H/PnzodPpYDKZxOLX62/wq/Owcq+oqDCGfbimnFsaHWyq7PP5UFxcjPz8fLknz4hqADFExT3d1dWFuro63HXXXUhLS5PzRyXAdaRFT2ppCiYaPrwfzxu9HYOhvwBo27ZtcDqdMJlMiI+PR3l5OSoqKlBRUSFFOklJSUKGl5KSgsGDByMuLg7JyclCuTx69GhERETAYDDAarWKl6Vpmuwht9uNwMBAZGRkwO12IzQ0VAwBwiFVtFN4eDh8Ph+Sk5MRHx+Pc+fOwWKxoKKiAomJiYJtZ3FRa2srqqurxVuaMGECoqOj0djYiEGDBgliymAwIC4uDq2trXA4HMJSSzlH74vrROoRvsPN+5DGJtewq6tLqrR/6PpRxNyBgdSk7CV68eJF7Nq1C/PmzUNxcTGOHTuGkpISZGVlwd+/v9vKnDlz8Prrr0vS4qWXXsI777yDpqYmsaIOHDiAu+++Gw0NDdixYwcKCwvx6KOPinBXe4q+++670Ol0cDgceOGFF2A0GnHt2jXs378f/v7+yMnJwZgxY/Daa6+hqqoKd955J8rKynDp0iUsXLgQFy5cQGlpKTZu3ChhjzfffFOQALNnz0ZycjLeffddtLa2YtKkSejp6YHFYsEzzzyDo0eP4qWXXoLdbseoUaPgcDhw5swZjBo1SoqcvvvuO2H1e+SRRyRWSIFw4cIFHDt2TBjuKisr8fOf/xzNzc04c+YMLl68iHvuuQejRo2C2+3Gpk2b4HA4MGXKFLhcLvzlL3/B6NGjceHCBfh8Pvz973/HiBEj4O/vj5KSEvz+97/Hq6++ivz8fEyfPh2BgYFoamrCjh070NTUhNtvvx1WqxWvvPIKioqKcO3aNRQWFsJut+PTTz+VMvWVK1eitbUV3377rSiQmTNnyvpTSX377be4cuUKhg4diq+++grLli2D3W7HN998I2iKmTNnYsiQIXC5XHjrrbcQGxuL3t5eCccsWLAAZ8+exeXLlxEaGorFixfjnXfekbE/++yzkn+hV8ODSOHK2GhcXBx+/etfy95Vcztz586FTqfDrFmzYDAYcPz4cTQ3N+PnP/85/Pz629J98803cLvdaGlpwaOPPorGxkZs2bIFISEhGDZsGAoLC6HX61FVVYWTJ0/i8uXLuOeeezB+/HiBz3GMb731FjIyMrBz50489NBDMBgM2Lp1K6xWK9rb2zF37lxkZ2fDZrMJsdrw4cMxcuRIHDp0CPfffz+am5vx6qumb2i/AAAgAElEQVSv4mc/+xlCQ0PxwQcfyBn46U9/KslcGi6Eir799tuoqKhAX18f1q5dC5PJhFdeeUVCcPv374fT6cTo0aPh8XjQ2NiIadOmiXKxWq2CiCFXitvtRltbG2JiYiSs0dPTg/379wsPEOG6EyZMwOnTpxESEoK0tDTU1NSgqqoKRUVFCAkJwdGjR1FXV4fc3FxUVVVJeKmurg5GoxHZ2dmS1G1paUFoaChaW1tRVFSErq4utLa2SkHR4cOHkZOTA5PJhIsXLwqjZWlpKaqqqqRTU3l5OVJTU0XIp6WlwePxoK6uDna7HdXV1Zg1a5bQOAcHB6OtrQ319fXi3Q8fPhzd3d04c+YMgoODYbFYMHfuXGiahpMnT8JoNErl+w9dPxrLndatWlQzbtw4mEwmrF69Gm1tbdA0Dbt27cIDDzyA+fPnw9/fH1u3bsWMGTPwxBNPoL6+Xiy2efPm4fDhw5g0aRJuueUWlJaWYtasWcLvwQbHKjrC7XbjypUrWLVqFSorK6Fp/RC4NWvWIDk5GY888ghOnDgBr9eLGTNmiBu2atUqtLS0IC0tDcuXL8fRo0fF4n377bexZMkSPPHEE3jvvffg9XpRWlqKO+64A7t27cKtt96Ke++9F+vWrQMA3HbbbdDr9fjJT36CwsJCXLt2DbNmzcKlS5cA9Fu4n332GaZOnQqTyQSbrZ+Qk1aq0WjEoUOHMG/ePLzzzjtYtWoVMjIycObMGaxZswb33nsvDAYDDh8+DK/Xi1dffRUPPvggKisrUVVVhRkzZkCv1+PBBx/EiBEjcPjwYRiNRgwfPhyrVq2Cx+PBxYsXMXfuXFy5cgUBAQFobGzEmjVrsGzZMhw6dAhHjx7F7bffTqgWcnJy0NXVhf/6r//CXXfdhWXLlmH//v1wOBz4wx/+gNWrV2P+/PlCysaYLq2ZqVOnYvLkyXjooYeE8Gzjxo04fvw4Vq9eLdZ6YGCgsBEuXboUx48fFwsyPz8fPp8Pc+fOFQ+mpKRECMaAG4lauryqV0YvgRYtscx2ux0jRowYEOP3eDwoLy/HlClTsGLFCllbTdOwdu1aHDx4EPfffz/27t2LtrY2rF27FqWlpZgxYwZKS0vh8/U3WXnxxReFy+Xo0aPSlo6eV1BQECZPnowVK1bgkUcekUQ5y/+JGPL5fPjzn/+MVatWYfr06SgvL4de39+Crq+vDydOnEBmZqZ4c1evXsWKFStw/fp1ORvcYwy5MKFoMplw6NAhQYfMnTsXb731lpB4aZqG7777DllZWZg4cSIuXbokEEiGwhwOB8rLy/Hll18iIyMDmZmZqKioQFtbG3bt2oWUlBQUFRXh7NmzEtq6ePEiwsLCUFBQgP3798NqtWLMmDHYvXs3vF6vnIvjx48jKioKEydOxOHDh9Hd3Y28vDzs3r1bmvTs3r0bTqcTOTk5EjayWq2IiIjAqVOnBJ1TVVWF5ORkREVFITMzUzzE5ORk4bevq6uDyWRCeHi4UDYz7xMfH4+zZ89KkRWVZE9PD2JjY1FYWIizZ8+it7cXFRUV0DQNmZmZ6OjogJ+fH2w2G/z9/TFs2DBUV1f/S5n6oxDuKtaWuFiGMHJycpCYmIhhw4bhwQcfxIQJExAWFobo6Ghs374da9eulaIClgyvWLEC69evx+9+9zuEh4dj27ZteOCBB0TwzZ07d0BohNaBXq/H9u3bkZWVhcWLF8Pf3x979uxBXFwcrFYrNm3ahGeeeUYOzKFDhzBu3DgcP34cVVVVGDp0KL7++mux9vR6PdavX4/IyEg4HA7MmzcPBQUFKCoqwgcffIAJEyYgNDQUNpsN06ZNE7cxLy8PdrsdXV1dWLlypYwfAJ555hm89NJLyMnJwdKlSxEZGYmPP/4Yr7zyCl588UX09PTgqaeeQldXF9auXQuPx4PVq1djxYoVSEpKwsaNG7F69Wo89thjkhfIy8vDwoULcd9998HlciEvLw/jxo2Dz+fDY489Ju/Z29uL//zP/0R+fj4+++wzPPjgg/B4PJg4caIw/+3atQs/+9nP5EAUFhZKvHX79u3YsWMHPvzwQ3z++edobm7G2LFjhWvlwQcflLATY74Upq+99hrGjx8vCJRPPvkE//Zv/4aenh5cuXIFI0eOlDj6kiVLUFBQgLVr16K3txd///vfsXPnTuTl5SEuLg7Lli2DpmnYsWMHVqxYgcWLFwO4UWHJcA1DG2q+RC0yCQgIwKFDh/DAAw9IqIlu89ixYxETEwOHw4Hp06dD0zTYbDbs2rULmZmZ+Pjjj7F582YkJyfDYDBg586dWLZsGe6//34AwIIFCxAXF4cNGzZgxYoVePTRRwfUabBW4NVXX5W1cbvdePrpp/Hss8+ip6cHly9flk5CBkM/59FDDz2EZcuWoaysDLt27cInn3yCxMRE3H///TCZTDAajdixYwdyc3Nxzz33ALjRxJror7Nnz2LWrFlYtGgRCgsLJYzKfNatt96K9vZ2FBUVYdSoUQgICMDhw4dRVVWFOXPmIC4uDhEREYiJiUFPTw9CQkKwe/dupKWlCR49KysLb7zxhljBfX19Utw2ZMgQabDj7++PoUOHIi8vD263GwUFBRKzz8/Pl31ZVlaGzMxM5Ofnw2AwoKioCFFRUXC5XEj7nr7D7XYjKioKpaWlEgZZtGgRRo4ciaamJgQHB6Ojo0PoPdrb25Geni6kdT6fD+Xl5RJ9YM9Wr9eLgoICBAYGYu7cufD398eRI0cEo+92u/Hmm2/i3/7t33DbbbehuLgYf/3rX9HU1IS9e/eKkffyyy8jMzNTmrnQ+Pln149CuBOVwZgk0F+Q1N7ejnnz5iE6Ohpmsxnl5eVYvXo1gP44+bfffovbb78dAQEB2LBhAy5duoS4uDhUVVVh//79UiBx9uxZ+d7Pzw+/+c1v0NjYKHFrJkAeeeQRNDY2oq6uDs888wzcbjdKS0vx3HPPYdmyZbjrrrvQ2NiIkJAQfPLJJ8Kz8umnn2LKlCkwGvuJpNasWYMvv/wSjY2NmDlzJgICAnD+/Hk8/PDDOHDgAEJDQ3HkyBGsWLECRqMRW7Zswa9//WuUl5fj22+/xUMPPSShlx07dmDXrl24cuUK7HY7vv32W9mUQH+C+L777sNvf/tbPP/88xKr++STTzBmzBgAELf68ccfx5IlSxAdHQ2n04nHH38cOTk5qKiowHPPPQer1Yqvv/5ahOyVK1cQFBSEESNGiEXK+N9XX32FS5cuyUafMWOGeD82mw0HDx7E8uXLodPpUFlZCY/Hg3nz5gn3fltbG3bu3In58+ejubkZr7/+OhISErB582b09fVJIwKDwYDdu3cLg+SXX36Juro6zJ49WwjKvvvuO+zfvx+bN2/Gm2++iYaGBmzbtg0ffPABrl27hn379uHvf/87Fi9ejI6ODuzfvx+PPPIILBYLxo0bh6effloUKxU3E3PExjNko1bI2mw2fPjhh5gwYcKAOoqKigqsXLkSPp8PGzZswLPPPourV6+it7cXCxYswFNPPYUVK1aIOz5s2DBUVVVh7NixaGtrg7+/P4KCgvD0008L5QC7BrH+AujPJx09ehT+/v7Yu3evNKWJj4/HiRMncOLECRw5cgRnzpxBdnY2iouLMXr0aFRUVGD9+vUoLCzE8uXL8fTTT0sjiCeffBLXr19HVVWVhJ78/Pwkfm00GrFr1y7JeezcuRO33nortmzZgtbWVrzyyitISUmR3E55eTni4uKQn5+PzMxMGI1GxMbGClggLi5OajxGjx4tFj+LgDIzM2GxWFBaWoqJEyciKioKVqsV48ePR2dnJ2pqalBUVASfz4dTp05h1qxZkg9pa2vDpEmTEBQUhKCgIEyaNAnNzc347rvvMG3aNPT09KC8vBzh4eGizA8ePIiEhARER0cPoNe+fPmyQCCvX7+O6OhoQbacPHkSgYGBOHTokODok5KSxOsJCQmRMCIbBJWXl6Ourk54/1988UWsWrUKxcXFqKmpgcFgwJw5c0QZ7Nu3T2DPAQEBGDJkCBoa/tsOpnL9KIQ73WTC3+gG19bWIjs7Ww7VxYsXkZaWJpnwyZMno6WlBV1dXfj666+x5nsStPPnzyMwMFAKB6Kjo6WLTFFREex2u7jytLhcLhcsFgsAwGq14s4774Tb7caYMWNQVlYm2fampiYYDP0NqYcPHw4/Pz+UlZUhLy8Per0e7e3tGDZsmEAy6aJv2LABYWFhKC4uBtAPwUxPT0d9fT02btwIo9EIs9mMsrIyDBo0SDq//O1vf0NhYSGKi4sRGBiICRMmSKXekSNHAGAANYDP199+q6SkRDDVzNbX1NRA0zRs3rwZLS0tqKurQ3x8PJqamnDXXXchMjISV65cwZAhQ1BZWYmEhATYbDaMGDFCQhReb3/fx6KiIpSUlCA2NhYRERFoaWmBTqfDP/7xDzQ0NKCqqgqpqamora1FRESExA4BSB5k2LBhcDqdOH78uDQ9ZpEYk8ddXV3SA7a6uhr19fWIjo4Wy4Wx6uLiYly7dk3QCKWlpbj99ttRXFws0FK32y2YaIvFAo/HA4vFIh2szp49ix07doh1ziQ3D76aQGS45/LlywOKeQwGA0pKSpCeno62tjZ89tln0Ol0gsiwWCyw2Wyw2Ww4cOAA2tvbBQPd0NCAyMhI9Pb2Cj97b28vvvjiCzQ0NIjXwvARk/wWi0UgeSNGjIBO19+4JDAwEKWlpdDr9UhKSkJLS4uwPHL/sv6hqqoKfX19sFgsCAgIgNVqxV133SUoNrIhapqGgoICmEwm6PV6nDt3DllZWZL7aGhokOS60WhEfn4+oqKi4OfnJzz0gYGBCA4OlpyQz+eD2WxGZ2enKM22tjZERUWhr68PERERaG9vxx133IGoqCjB4QNAU1MTBg0aBJ/Ph7Nnzw4ouGtpaZEw37Vr1+RMXrhwQeCSiYmJ4tk0NTUhPT0dwcHBKC0thc1mg9vtxtWrV4VBlcCD8vJyOBwOSYy6XC5UV1cjJycHNpsNHR0dgvwiAig6OhrBwcEoLy9HUFAQGhoaJP7f3d2N2tpa5OTkyJwx2d/Q0IChQ4cKn7vFYkFUVBSampp+UK7+KOgHRo4cqe3bt08y7qwMfP755/HCCy+IK+pwOBARESGFKUxGWCwW4fI2GPqbcRgMBiQkJECn00kFmb+/P65evYrAwEDptsLEh16vFwFPJjjC7QjDIj0tyfSTkpIQHByMiooKsUYsFgt8Ph8GDx4Ml8slFmhISAjq6+sRFxeH8PBwtLe3IzIyEh6PBy0t/f3Fmb2vrKxETk6O4Hbr6uqQkZEhKBEmX9LS0iQ5Rdc4MDAQLpcLjY2NwmYJ9MeIm5qaoGkaBg0aJHBIdu9JS0uTg15WVoaUlBRRgB6PR5pfc66uXbuGtLQ0qRhuaGiAx+NB2veNMTwej0DLKCTtdjscDgdCQkKEUInzyP6gUVFR/weFAiscQ0NDYTab0dfXh5aWFnR3d2PQoEFob28X7pD29nYRmGazGXa7HT6fD5GRkairq4Om9TM99vb2Sks4CgQSxDHnQ0SICitk/LmhoQHt7e2CYElPTwcAEfrx8fHo7e0VhAnnoLGxEXa7HYmJiQgLC4Pb7RZe/+zsbAA3oJnkZo+JiRGIJlEiDFfZbDaEhIQIyR0TlxkZGbBarQINZQyapfAejweRkZHw8+unOA4KCkJoaCh6enpQX1+PoKAgwaCz1oFeLguAiIJpbW1FRUUFqqur0dDQgMGDB6Ovr0/47FlIRMoLJlGJ/Glvb5dkM2GYRC81NTXJWnKszG+xIQ2rdtmMWq/XIyQkBA6HA8HBwbhy5QrcbjeysrKEH95qtWLQoEEyhw6HA4mJiYLBb2lpQXR0NBITE6HX6/HCCy+Id5qfny9jcTqdck4PHDiA5cuXo6amBk1NTcjJyRG0Dznbm5ubRTCfPn0at99+u9AksOWhz+cTQ8Rms2HYsGFSsc/3bGhoQHp6Oh577LF/Sj/woxDuBQUF2sGDByVhVVdXh+LiYmzfvh0ff/yxFP/wsAMYgEvn4rIYRi2FZwJIrVYjtpSwKFpcKib45upFYGArQBWmyLgaf86LG4UxQ45HJZLiYgI3ClA6OzsFk833oIWjJtMYKvF4POLaqnE4teCJ41YLmtS1p6Vlt9slfMU5JYTOaLzRkJjl3VwTbj7i/YnoUDHlfK5aoef1egdUhqrJVLXilBWW/HpzwpOWs0rPwHfk90w4c605V/yMWjXI9aai4lg5TxRwfD8KQMJ5ST/AdVbDaCounDhozgHXiMqaJfH8Oc8CjRKOne/BdSCcUSW142f5HjrdDfI4lZZALdpTi8L4O6KzVKjsp59+KrBkk8kkSiAwMFAscO5fUvtyLYiOoiJgM3oVS851aWpqEow65QDHy/PP+WGxGLllOCf0Mshh4/X2sy1GR0ejo6MDLS0tUtzIOTlx4gSGDh2KlJQU4Z8BgC+++AKJiYlISUmR6lvyvOt0OkRHR8Pn66cXUWs6goODxajjOKjg2IybvV157vg+5HoPDw/H008//ePnlqEA48Zva2vDyy+/LBuWVgMPqpq07OrqQkhIiLhAPFQszCAKh0KHwoh4Vx4Oeg4sViHTnMouSAHs8/mkUIHCkFWStPjZBJdEZHw2FQ0FJw+pmm9Qq/0o5FmqTHeNv6fFTXysWpCiFuEQ4UEWSFUAUjHw0FPxUfhS0FJxMoFGZUVFwq/8p1IAqNYHu9uoY/N6vVKlR2gsv6o4bJUNj0KZ78G5VqkNuLYqdl5Njvr7+0vYgVh2NUHKsTPhSqFPBUtrmjF7f39/dHZ2oqenB+3t7QPoGijMuKc4ZioN7h3SCbtcLtTW1opLT4XGAh56VKqhwr3HcQMQ747njfOlKluOk+dHNUhUbh2+o8fjEXgfACmzd7lcCAwMRFJSkiRhaQSxqIsxdxYIsvyfCq2npweJiYlyHlmTYTab5bMmk0nml0YAwRGqAlR5fohcSU9PR1dXl1SGBwQESK/b4OBglJWVScOPsLAwJCcnIzU1FQ0NDejr68P169cRGhqKsWPHoqGhAd3d3cjNzRWkDZPfzc3NMJlMUuRENsju7m6kpaWJdx8WFibt+Ww2m8ggla+H79be3o6YmJh/2SD7R2O5Hz9+XEqmaf2x5Fq1YGmlqZY7NwsVAHCjWrWnp0eQFzyAAESzk4yMwp+KALhRkKGyI/LeKpKC4+Jc3my9d3V1CaaeykYthKLQIZTOaDSKN0Dhrx5C9f48cCriiDwt/J5C2mAYSBjGSklWgbKkW/UsVE4SWtUcK6vxqEBU0jbghuXLcdKyVb0GWl0UrOq88B70OjjHqnLmuG5WmlxjHjLgRkGbKtS5FvwZ309Fb9HS5//5t1RkXKfOzk4R1BRiFIbq8yjUmbxVq20pAKk4XS6XCDauAzmWDAaDFMAxcafX93cB4n6nglUNI64fqzq5r/lMcrbU19dj3bp16Ovrw9y5c1FUVCReDM9DaWkprl27Jk2sKXTDw8PFU1b3NC1RxqnVPcR55HwQdcIYPYUgsfY+nw/Xr19HZmamxKN5r6CgIJl3FvmxapYeF+8fGRkJk8mEa9euyb5uamqSFnp6vR4xMTFob2+Xd7Hb7VJMGB4ejtbWVqkuz8jIgNPpREREBMxms1S20mil0A4JCUFlZaWEe/neLFLjGaXx4/F4pCEJ5dIvfvGLH7flDmCA66tW89FK58GmoOXfAJCDplpwAKTqTWWnCwkJEUZGWku0UijUeR8KQX42LCxMNiufy2pNPl+18JjgpHdBocdNz3sAEEuLB5JWNA+RWmlLYUiiLeAGBlmv18NkMsl4aPnSUqSQpQDle1OZUGDRM+A8q8qJSpcWEi1YNZzBi4qa1rAquCk06YGR6ZPCmxa1amlS6avVtnwGLUqGsLi2PCCq96V6AOpaqWsL3FCOvPhsde44Bs4bx841JmUBDRPOEd9NFbp6vR5Op1NCFFwbh8Mhljhj1Nx75BhyuVyIjY0FgAHsimplqhq64Xh43rjeKobf5XLB6XSisrISo0aNEo8M6Ach1NfXC6NqaGioKFiGSfhehG7SWKN31d3dLTF2FanEuDz3Gs8Nc0U833wmlTS9c84tLX232y3JXuYaSIdAZRUTEyMd1eLj4xEYGIjk5GRZv8bGRqSkpKC4uBhjx46F0+mEw+GA3W6H1+tFVlaWePQqvYfBYBC+eo/Hg87OTqEZjo2NFQgpK6wZSlKVOOeTNCxUnj90/UvhrtPpPgQwD0CLpmnDv//ZGgAPA2j9/mO/1TRt9/e/+w2AVQC8AH6qadre/8EzxO3m4bw5HkcCHgoUus+0BGhxk0KW1j0nhovNMmAeftUVVePMqtak0FWFEoW2GvvlZPNZFPQqmx/Hz/cDblTnUnBR0TEJRauRCoqxZT5PVYAsFaelSAXE+6hl+0QP0Nqm56KWsPv7+4sFSguQgo3PZXIMgFggN1vjtEbUmDoFgVrur84j15T7g54ElTcFNK0Y3ofzrgprxp+p7FWvhAKU68i5UAt41K5SHJPP55NiFIZnGMbh3mFVJxUX31lVqlQQDodjgJJjtyfir5kEJAqDc2Sz2SQ0QSROX18f4uPjkZWVBaPRKCyM9Po431wjKi16keRWIUlXVlbWAJI1TdNQXV2N9vZ2QbRwb9BIYzKeRhGJs9QQIBUJ54ANOVR+GIZ1OD6dTid0u/Hx8dJL1d/fX5RafX09AgIChHpa0zRpnuLz+VBRUSEeQW9vLzo6OsSCJtae4U4m/l0uF1pbWxEbGwudTofs7Gz4fD4JlVCoR0RESCcwm80mnEl1dXVyPhwOh3APmUwm2TcMuajQcMoYtgmkHPhX3DL/E8v9IwBvAFh/08/Xapr2Z/UHOp0uF8BiAMMAJAL4RqfTZWma5sUPXIzbcrAMe9CaULlmeKjUrki8B90xupoUZGpjCgpAVRBTObAQxO12Izk5WTYyD7rNZhvAtqha2hwD3WAKVLWbEIW3GrcnIkW1CHn4Ge/X6XSSUOHh50V3VhUonEM+Q21qQmuYnBcBAQGShKKloSb3uLHUuDEFO4Uj34mbkd4Xf8cNTuibao3Tqqe1wkbDnDPGamkx8rkMy3FctIj5ngxn8N3Uz/Ar9x33kRri4R7iu5B5MTIyUt6bcXoKKP49Le2uri7h96bVyr0SFBQkVdcMH6hNGRg+4NrTOGAcmbkPVi2qIQmGcRoaGqDT6aTXLueQAkP1zKhQVMPDaDRi8uTJAIDk5GRZUwBobm5GcXGxdE8zGo3yPc8U35koJJ4xvf4Gvw73P40JPttgMAzwiLkHGP5RFQPpgtVuT35+/SyMKispw7MulwshISEDWkEyjMZxOxwOADe8OrvdjrFjxyIoKEhyDKrnEx0dLeFhdoBTPaWQkBDxOLh2hGhzzYODg4VPiEYhWVOp/FRFrMq1/+76lzh3TdMOA7D9q899fy0A8A9N03o0TasCcB3AmP/JHxJNoJI40friwaGw9vl80lCAE0z6VjVurQoC3oPan64qP8Nk1R//+Ed8++23sNlsAxJwvb292L59u8Td3nnnHcyePVvieKrQpgtOIRgcHCwbj4eGv2MyjpauXq/HHXfcIRuXnB78nF7f3y/02LFjQnm6bt06WCwWsQq+X7cBcVZatNwce/fuFQghN6CKHFIRGGqiWk2WsrMMC8Q4B+o4/Pz88NBDD+Fvf/ubhKu4phRILMMOCwvDp59+KnPsdDpl3Yh/p+BliEYl0GI8nAnid999VwQlv/LwMGbrcrkkJsuELt1ku90uoQk1QWa322Gz2eTZqoVPK3Pfvn0ICgoShRYaGioC2s/PDxs2bJCDe+rUKbzwwgvitbKuIy4uDnFxcTLPbBBTW1srMdvQ0FC8//772L17txT+tLW1Cdzv2LFj2L17N/bt2weLxQKr1SrGUXt7OzZu3CjUAQAk4U0vZ+jQocjOzpb+xnq9Hm1tbTh//rwkl2mB6nT9BWs8kzS2XnvtNRFsAKT+w+VyobKycgCqh8gRh8MhWHQqWJ/Ph7179+Ls2bOIiIiQ+o3g4GDExMSgsbERf/7znwUCGRgYiOjoaJELpDrwer3SCPvSpUuw2WzQNA1VVVVwu92orKzErl275LmBgYFITU0VAyU3NxcxMTGoq6tDSUkJqqqq5AwNGTJkAFiCIAqbzQaz2Yzw8HCBmTocDjlbVKrZ2dkICQlBQkIC0tPTkZiYKPuXiVubzSay5oeu/00R05M6ne6STqf7UKfTRXz/syQAFuUzdd//7AcvutzMoFMrq3Fw/p+ChTFfTgxjl9xURI+w8lBN1vj59Te15d8xllleXo7Zs2dj6dKlUkTAQxsQEIBNmzYhKSkJvb29WLFiBb744gsRGgxtUAGRpIjCS435EtpEi4ljZ8Jp06ZNADBg7Nyg/v7++Prrr5GSkiKFJCtWrJBu9pqmSUUeXUsePFrSLpcL27dvx/Dhw4Xjg2OXjfG9wmRYh3NLgUzhSsGrQk15kClsrVYrlixZItYYrQ5/f3/xQoiv3rt3r7i9DE2pDVfU0AuTmEzEc4w+nw/Hjh3DfffdJ40WentvsEACEAVD95cC3+v1SmKUFjGVLy0uktQRUqcyN/r59dMOHzt2DCaTSThH2tvbJdnKvchKzOPHj6OgoAAABqCu7HY7uru70dPTA5fLJVC62NhYEYyc36KiIlk3o9EoeHoqCovFgosXL+Ls2bOor69HV1cX4uLicP/998u541kMDAwUD5h7iPPGojPGw6OjoxEUFCSWM0MMxcXF0ks4Ly9P3qG9vV3WW9M0JCUliSHidDolKcw8gNpgxs/PT4oEzWYzTCaTePD0rh9++GE5B21tbQOS9LTGqcgiIiIwZMgQmM1mVFRU/D/UvXl0m+W1Pvp8kix5tuRJkuc5HmJncoAQEkIgDdAEKBCmNlAK/fV7wAgAACAASURBVME5paWHrp5CRzrRcy7ltJz2MCRQUtIGAiEhQEISQhwyx3ESx7HjRJ5t2bIsyZbl2bKs+4d5dl7za+m55/7WulytlZXElvR93/vud4/Pfja6urrQ3d0Nm82G4uJiFBYWSi2A0N729nZ88sknaGpqwujoKHJzc2G329HX14eYmBg5B/Hx8YiPjxeGWL1+BsJIqKzX64Wmaejv7xeQAVNc3Ee/3y8puPr6elgsFtGJ/J7Pe/1PlfuLAPIBzAfgAvDcpz//W+bkb8JxNE37X5qm1WiaVuP1emflSYHLSAduECvTat6blKEAZnlebrdbDiTDQL7oCYyOjsLtdosn63Q6ceLECWmMUvO6zGerXYL03MLhMPr6+gQW5vF4xPOlMRodHRVvi+EfUyzAZXQGu2pVSCa7/qjEurq6kJycLB5Ib28vAoGAKFg+m9PpFCWi5qHD4TA8Hg+6urowMTEzEpDQK/7d19eHrq4uacDh+gMzHYGkJaU3wehATQnw8E5MTCA7O3tWt+fQ0JBA0OgNEl/MZhtGbmywYQGOqZyuri709fUJhpmeGZ+zrq5Ovhe4nI7zeDxSx6EM0RjRg+QzBwIBWUd+x8DAACYnJ+H1emW+wPT0tBiG1tZWoXpluoxKmU05TU1NMu2ehbqioiL4fD7huOdZ4IEPhUIytIJG0mAwICEhQRQsIZbM/1NBsk5Bme/u7obL5RJyPKZzmH7q7++XexsfH5du3qGhIbS1tcmIOq/Xi+HhYfHgIyIikJ2dLTDi0dFRtLW1SfMQEVler1dQMMyXU37ogPHc8F64FuyIpg7gcwEQT7++vh4+n0+Uo9frhcfjEWeEkSzrTrxPnmGLxQKDwYCuri6YzWbZC6bXiHZhQ9jQ0JAYUeogNsUFgzPNlwQ5qMVwNszpdDNTnlgPooypAILo6Gi0t7dLao+Qy897/Y+UezgcdofD4VA4HJ4GsBGXUy9OAJnKWzMA/E0ChHA4vCEcDleGw+FK8jSrA5aNRiM2bdqEZcuW4ZVXXsHo6Ci+/e1vi2V79tlnpfK+fft2BAIB4RXp6urCgw8+KJOa6FUx3/zhhx8K+953vvMdwVYfOHAAJSUl0kAUDM5wnPDeCgoK0NLSgr/85S9YuXIl/vrXv6KhoQHNzc1Yvnw5fD6fEFzFxcUhLi4OL7/8skxnr6ioQG1tLY4dO4YVn5IrnTx5Etdddx2MRiM2b96MtrY2vPDCC9DpZrhdyH73gx/8ADExMXA6nXjkkUeQk5ODhoYGdHZ2YsWKFRLSf/TRR0J8tG7dulkoFkYNb731FjIzM/Hxxx+ju7sbjz76KF566SW0trZi3bp1giR48cUXJS3z5JNP4i9/+QuSkpJw/PhxNDQ0YPPmzejq6sKLL76IyclJrF27FnFxcfB6vdiwYQPi4+PR0NCA9evXIzo6GmfPnsXu3bsRERGB3/72t3jxxRfR3t6O999/H+FwGE888QQWL16McDgMh8OBf/mXf0FUVBTWrl2Ln/zkJxgaGsLGjRvR09ODhoYGPPHEE7NQLUzbtLW14d1330U4HMbPf/5zfPvb34bD4cDHH38MTdPw+9//Hps2bcLOnTuxceNG3HLLLdDpdKivrxe+7hMnTsDlcsFkMuE3v/kNgsEgdu/eja1bt6KlpQVJSUn45je/iT179kjB7YUXXhBOk7KyMkxOTsqhplMSGxuLvXv3YtWqVbJno6OjSElJgd/vx/bt2yXsPnjwIAKBADo7O/HKK69IeodGNCkpCW63G7fccovQFtTX12NsbAzJyck4ePCgGKv+/n7U1NTA6/Vi165d2LZtG37wgx+gqqoKhw8fxu7du/Ef//EfOHz4MB566CFs3LgRu3fvRmtrK4xGI5588kmcPHkShw8fRiAQEJrtffv24Xe/+92swj2pbA0GAw4dOoS8vDzpkuU0rH379kGv1+PcuXPYs2ePIECee+45jI+Po729XRAjO3fulNw9h6PQyDGVQwfn+PHjyMnJgd/vR1VVFVpbW9Hc3Izf/va3MBqNonDJK9PX1wefzweHw4GysjJkZGTg2LFjMJvN0Ol0ePHFF3H48GGpn505cwYTExOoqalBTU0NrFYr8vPzAcx0oft8PrjdbvziF7/A6OgotmzZgqamJnzwwQc4ffo0kpOTceHCBaSlpWHv3r0wGAyoq6uTYvw777wjnDnV1dWwWCxob29HSkoKkpOT8fvf/14cstdf/2wZ9P+Actc0za789ysA6j/993sA7tE0zaRpWi6AQgDV/53vZK6SqQsAKC0tBQAUFBTA7/ejrq5OEAHbtm2TOYrLly/HgQMH8Nprr8FkMuH48eOYP3++5MyAy8NlGxsb8fzzz6OoqAg2m00aFYCZlnKz2TwLFUBvOxAIoKKiAk6nE3PnzoVeP8OyNzAwIF4B2+hVRMDmzZuRm5sr5GeRkZEoKytDfn4+4uLicObMGQnVSURWWlqK6elpHDp0SDilv/GNbyAuLg6NjY0oKyuTIQEAZkFFn3/+eZSVlcFsNmPt2rWzCp5MGdXW1sJsNiM/Px+ZmZl46qmnkJCQgMzMTMyZMwdWq1W8IobGDocDy5cvR1xcHEpKSnDy5EmUlpZKnpCpgMnJmfFp2dnZ0DQNp0+fRmFhITRNEwbPgYEBjI6OYt68efjTn/6EFStWwGq1AgAqKioQDofx2muvYXh4WDr7cnJycPDgQfz5z3+G0WhEQ0MDysvLBQkCXG5CIxWxzWaD0WjEkSNH8F//9V8oKiqSPHl5ebnk3PPy8qDX63H+/HmYzWY0Njbi1VdfRVpamox5IwVsV1cXsrOzZyEcIiMj8f7772NoaEiMen5+vnjRbF6hIWptbRX46NDQEBYsWIDMzEzBcMfGxsLhcGDv3r0wmUzo7u5GYWGh7DWjAcoz+VOamppQVVWFYDAIl8uFlJQUBINBZGRkSL8HURcs3g0NDcHpdGL37t1ISEjA0NAQMjMzodfr8cc//hHFxcWIi4tDZGSkRHNE9XAABqPbyclJSa0wddDR0YHIyEi43W588MEHSElJgdfrFbgqoyWmgbKzs0X5M/1XUFAgUZfNZoPJZEIgEEBvb6/MWuX4RZvNBp1OB4vFgqGhIRmAzUgsOjpaoiF2rXZ1dSEiIgIpKSnS/VlQUIDMzExJiZJUr7m5WfaRqWQWTNX5rnQmb7jhBhQUFGBgYEBoAwoLC9Hf34/CwkIYDAakpKTI/WZmZiI6OhoJCQk4fvw43G43Pv74Y6HXIGKGg7s/7/UPlbumaW8AOA5gjqZpTk3THgLwf2madl7TtDoA1wH4FwAIh8MNAN4CcAHAHgDf+kdIGQCzwnMWIIaHh7FgwQKUlZVh6dKleO+99wT5ERkZiQ0bNmDjxo246aaboNPp8OMf/xiVlZXCfviTn/xEwikai4iICHznO9+BzWaTKen79+9HbGwsTp8+LSkB4lHVosWf//xn3HnnnVi2bBmKiorw0EMP4eqrr8aqVauwefNm3HXXXYiMjERVVRUee+wxBINBuN1u4XHp7u7G1772NZSXl6O6uhrf+MY3EB8fjzfffBN33303xsbGkJeXh1//+te44ooroNPpcPfdd8PtduPxxx9HV1cXOjs7sWHDBmRkZIgX8+yzz+JrX/saxsfHBZ/LHOUjjzwi+VPWDzRtht7hvvvuw69+9St0dHSgvLwc999/P6qrq/HP//zPmJiYwBtvvIH169cDAF577TU8/PDDyMjIAAB8+OGHKCkpQWlpKZ599llce+21mJ6exrx586DX6zEwMICVK1fi4sWL2Lhxo/C3fPzxx7Db7YiNjcWzzz6L6667Dq+99hpSU1PFC7v11lsRERGBt99+W7jMo6KicMcdd+BnP/sZFi5ciIiICDz88MP47ne/K40zhHSywLxo0SIYjUbs378fy5YtQ1VVFRISEhAdHY1f/epXuPLKK3HLLbfg4MGDuPPOO+FyubBjxw5YrVY89dRTsNlsSEhIQH9/P44fP47MzEysXr0abrcbycnJiIiIQE5ODr785S/DYDDggw8+kMjl3LlzWL58OWw2G5KTk5Gfn4+MjAxYLBbs2bMHPp8PExMTGB0dxZkzZ3DPPffAZDJh7969uOaaaxAKhbBx40YUFRUhMTERK1askNkGKlbeYDBg165dwl+ydetWZGRkiJK79tprYTab0dfXJ2ykREe1trZieHhYuikPHToETZsZ0FFaWgq/3w+dTocTJ06gvr4eVVVVUuStra1FZWUlLBYLzpw5g6uuugp9fX2S/iMGm1FEIBBAU1MTzp49C7PZjIyMDNx6662YmppCXl6eDJ6ZmJjAokWLhHuH9bVFixZhenoa+/fvh9lsFu+f+oJe//Hjx0Ueent70d/fD7PZjEuXLsm0JZU3qrCwEGlpaWhoaEBFRYUUyvv7+zE1NYX6+nokJSUhLS0N3d3dOHfuHB588EGUlpaipKQEGRkZcDgccDgcklfPycnBJ598gsWLFws54JkzZ7B48WKYTCaBl27cuBFz585FfHw80tLSMDg4iLfeegulpaUip3RGkpOTkZycDL1+hqo4Ly8Pvb29/++HdYTD4XvD4bA9HA5HhMPhjHA4/Go4HF4fDofLw+FwRTgcviUcDruU9/86HA7nh8PhOeFw+MN/9P18sXBJRIbRODPKqqKiAhaLBa+//jpuu+02bN++HXfddRdcLhd+9KMf4YorrkBcXBwWLlwITZuhHKivr8fWrVtnwfh4IHJycmCxWDAxMYEf//jHeOSRRxAfH48dO3Zg/vz54v2pxbTu7m5s2bJFrPKjjz6Ke++9F5s2bUIoFILL5cJXv/pV+P1+LFmyBDfeeCP8fj9iY2ORkZGB/v5+/PSnP8Xtt98u6ZaioiIMDg5iYmICN910E1wuF/7pn/4JZrMZmzdvxh/+8Af84Ac/wGOPPYZHHnkEdrsdb7/9Nnw+Hzo6OrB582YhUnvggQfQ3d0t8C9iz99++23JEbIgODAwgJtuugmrVq3C2bNnceTIEbz99tvweDz405/+hKysLOzYsQPbt29HQ0MDdu/ejaysLOHzeOGFF3Du3DksW7YMo6Ojghc+f/48bDYbvF4v9u7di6ioKOzatQuDg4PYtGkTzp8/L/lShtoNDQ3isRw8eBBr165Fd3c3pqenhbTpo48+wrp165CQkCBNNGlpaWhqaoJKNsdCFpFQiYmJaG9vxz333INnnnkGKSkpSEhIQFRUlFybab2ioiJhb9TpdFIjGB8fx/PPP49169ZJnnPlypVS1P7hD38Ip9MpXZmFhYW4dOkS3nnnHblPi8Uiw7U5g+BLX/oSHA4H4uLisH//fsyZMweJiYn45JNP4HK50NbWhrKyMmn8cTqdaG9vh8/nkyIwi4M0PgMDAygvL0coFBKWxKamJhw8eFDSGADEk66qqsLNN98Mr9eLkZERwaKPjo6iu7tbjIjH48GZM2ewdOlSyW37fD5kZ2ejv78fycnJmDdvnqBo7Ha7FGg/+eQTlJeXo66uDoWFhaKgRkZGUF9fLyAJFg27u7uRn58v1ArqbFODYYZ35dChQ+LgEXI5NDSEnp4e6PV6dHV1YWxsTIrU4XAYVVVVWLVqFRwOh0SiRHSR+XTRokW4dOkS/H4/srKypKC9ZMkSKTaTJ95kMmFgYAAJCQmIiIiA1WpFQ0ODsEFqmoYrr7xSZr5mZGQgOTlZisculwtWq1Vqgiy6JycnCzKOE+iSkpIQExODoaEhfPDBB1I4P3PmjEAu/65OffpTmtz/L18vv/zy0w8//LCEacyrG41G7Ny5E5OTM0xuwWAQN954o3yusbERRqMRS5cuhdVqRWdnpxQhOVVF7UCdnJyE1WpFR0cHxsbGUFRUhNWrV8NoNGLDhg1YuHAhli1bJjAmFmODwSDa29sRCoVQUVGBEydOQK/XIyUlBUVFRTh37pzQo3700UcwmUyYN28eTCYTDhw4gNHRUezduxcPPvigeBxELfT09CAUCmHp0qU4duwYTCYTli5dCrvdLrA2h8OBW265BcFgUMZ4zZ07F4mJiTh8+DAiIyNRWVkJk8mEY8eOAQC6uroEwsYCDtMQkZGRKC4uxoULF+DxeLB27VqEQiF8+OGHWLduHbxeL+rq6lBSUoKKigrk5+dj586dwsT4la98RVI309PTcLvd8Pl8aGhogMFgQF5eHqxWK4LBIBwOB0pLS3HVVVchGAyiv78fLpcLZrMZRUVFElYHAgG0tLRgZGQElZWVcLvd8Hq9aG5uxpVXXgmbzSZpkXA4jKGhIcydO1eY9Iiw0LQZ8qiOjg4MDw/j5ptvluk6AwMDcLvdSExMRGFhIaKionDixAkYjUa8++67KCsrw+rVq2G329HT04PJyUnk5uZi5cqVMJlM0hWckZEBt9sNl8slxoFFbzoTU1NTWLBggRTxgJkItaOjA+Pj41iwYAFSUlKwZ88eqXMcOHAARUVFKCkpEaXDRi5GEuzqJA3BpUuXkJubi4yMDOTn5wsAgfj2S5cuzeqUjI6ORnZ2ttRyiAhJSkrC4OCgtLjHxsbC7/fLjF6mEXW6Gb76srIyxMXF4cKFC0hMTJRuaavVKkVv0nuUlJSgsLBQDACRWETTtLW1ISEhAXq9HomJiYiMjERHRwcASAoiPj4ezc3NyM3NRWpqqiBMEhISpIDf2dmJuXPnIioqCqdPn8Y111wDo9GIuro6Mb5MX2VlZQl+vKGhQQwJHUt2mHo8HhiNRqSkpCApKUkKpXq9HklJSVJbYIqGEE4+LxsCOzs74fV6MTExgcTERCQnJ8PpdELTZoZ15OTkICYmBi6XC8PDw7Db7YIGIsz5woULuOaaa6DX63HixAl86UtfwtatW11PP/30hr+lV78w3DIff/yxCODIyIhAsOh9s42agqPCJ9klptPpZDI6BZo/Zysvc+LMCxJd88Mf/hDPP/880tPTJWfOdJFaYWeBkfUBwsWYdyTcLTIyUmBVhw8fxlNPPYWTJ08iFAoJjp95x/HxcSFm4vWIfyVLI71Mrgc7H4kRVwnRCAtTce48fERecK1JKEXFzwIVlZIKf5yamhLvhfdK2B7XhzQJaj8BoyDmitXmL51ONwsdQiQS6WqfffZZ/PznP0dERIQ0bbBJhWvF52KXJqMUUsCq/CLMWQeDQVRXVyMtLQ35+fl45JFH8NOf/hRWq1Xul5h/nU43i8ud1+DvyTao8tADl7tw2WRD+C4A2QPmdFkkBS7PZOX3RUdHy8xRAAgEArNmbvI72bDHKG5sbAwjIyMYGRlBf38/SkpKBHvOvDGLuoTdjo+Pw+12Q6fTwel0IjExcVZ3LPdncnISNptNGoIAyBQr9owQPsi5AmqXLwDJ3QcCAVitVpGZiIgIQYexkYmfJ9yU9SNi1r1er6RzAKCurg7j4+MoKCiQulkwGMTJkydRUVEhg90pyzwfQ0NDaG9vh043wyfDgRhMO3k8HhQUFGBqakrmObAr3mq1IhAIYGBgQM4r1yIQCMjZ4RkbHBwUyO/o6CjGxsZgs9kkhcrIPikpCe3t7Xj77bfx5JNPyvOEQiF8/etf/7vcMl+IYR1UFCoWm/lhNvEQr0psNKln+XMqXCJkqCz4PcQoq116wAx8at++fbjtttuEnpPKkxtPpUqFAmAWNI+pAXqQPAD//u//Dq/XixMnTuCuu+6ahVwhtJAFQbU2wGImC3FqxyBRCQCk+UrlMWG08VmjTaNCHLXa/EXIFTHDKk0sDY1ef5mYinvEdeF6sztUbZjiOrN3gY07REaFQiGBudJIHzp0CA0NDcIRPzg4KF6xCmtVG6b4bwCiJGgg+fyxsbGzmskcDgdGRkZw5MgRrF+/Xoq6VKo8RHypz8nWdV6bh1SlHVDXkFhs7iELkcTh01Hhc0VGRsJsNiMhIUFQKFarFXa7fVahl4aFskLHhntrMpmkmM+WeKPRiOTkZMFNq0My+IwkxGLBkPBDQjJTUlIkSqLcMfXD9n21H4GKH7hMG0FFT2ZL9osQoUbZ5t7yO3k9Gkg6F3QAx8fHcfbsWdTV1c1CJJlMJmRkZODChQsSXXm9XrS3t2NsbAy9vb0IBoMwm82IiYmRRid2eHNdvF6vpJNU/n/m+S0WizBe8lzz3KnnhI4N751RCL35wcFBHDhwABMTE+jo6MAVV1wxqznzH9EPfCE89/nz54cPHDgwq4BJr4geH5U3KV254Z/9zPj4uBxgbjwwe5QfC47EhA8NDcFqtcr30rtUuV6mpqaQkJAgConKS238IUKAiz84OCiHTi1qqp+jIlKfgdEJr81DQRQPC868H7Wxh7hlVYDoXapt/yp3BT1h4tJVegOuiUrToBpOlWaAWGYaB3q3qtIFLjNlqnQMmqbB4XBgYGAAfr8fPp8PiYmJMJlMctgyMjKE+oFrqHbU8hCqUQTzuVxj/s00HZkJ1U5jRgFGo3EWXbOaYuH7iDtmlESDR1mgHNK54LN+tveCe8dnYXTAtWX0wGdnQ8zIyAja2toAQLx0l8slfQVDQ0MYHh6G0WiUcXJDQ0PIzs6WYRv0Pru7u5GXlyfFeSLAONGMOH2SajFKnZ6elhRDfn6+KDaeI07qYpTtdrsFox8TEyPdslNTU8IFD2BWtMo6Etde9eopb+ytGBsbk45QOg40UmrvAAdV09GoqKiAXq+Hy+WSKJUduxzzabfbceLECXR3d2P58uUoKiqadY4ISQUgiDkaVTYtApejFpX+OC4uDj09PUhLS0MgEBC6EyKdEhISZO/ZGPnggw9+sVkhuej0xllo0TRNDiaVC4sbwGXrTY+T3haLsiqBlgq1pEdHi0naUV6P30dFS8vMqe1s+AAuswfSyKjPYLPZJA3CMJKCQAoC8uao6R9CAEmoRIXFZwiHw7M4aVRlC1zmWqenzOvSO2bKi0pdTXMxjULFzfSJSqlM40hlwzVS0y2MKniPND7cEzUK8fv9MsKQ7d42m008sUAgIHtts9mkFZ5GkN/NtRoaGhJoXVRUlHSmqt269Gw5/YpywX9TBqgUuN80ggCEeI4GhDLIBjbuBZX330pbqUaSBlJNL6qpQCJKSDxFp6GoqEhCfzaiqc1eXG++hoeHxfN0Op2Ynp5GRkYGcnJy4PP5xHCNj4/j/Pnz6O7uljF4TI0NDAxAr9cLrQJhherAGCKd6O0zPcS1Vam6uV+USVIIqGvKpiF6wDSQbIgiSZhOp0NJScmsSHlsbAwWi0XkkogbUhSMjY3B5XKJjLOhUKfTicGJioqC1+tFTk4O5syZg9TUVHR2dsp9FxcXw2AwwOv1Ij4+HrGxsejq6pJnY5cxZYqeP50Gko6p8sUUMmccUD8wRfW5evV/pI3/D79UxaQKIj0wNVWiphwYplPBAJdTGvSkgMser4rJVT1/HtzPhvwqyuaz4SDvhfentg2reVUqH4ZbDD9VegIeNIauPNxqrpswPzUU47NTsdFwqHlNrgWbJLhG/J0aOdBLZOqLv+P6UmnwGmQ7VBU8150KS41QaPS4L3xvT0+P4JRJNgVg1ntp3EZHR6V7kp2kdAb4edYQmA5g2oGKmevD97BGAVxmoqRM8t98BjUKIH8OHREqa5XMjPtNY8bojUqEa6I+Lw0c15YGgntKY6Hup6ZpiI+PFz4aRhT8HjWNODw8jN7eXskxk1BrYGAAXq9XFBBrATTYJDBjBKgONeE0KTUl9VnqCjUVyGvSS+d7udZsIuJ60CGhY8YzrdPpRNlR1ik/bEakHNC583q9sodZWVkIh8OIj4+XYij3nZBJn88Hg8Eg/Srs7SBahj0VNFoxMTHicXPUIIBZVB90RBhZhMNhMbp8hujoaHE+1BoXZYzP+fdeXwjPXX1R8aoHnIJNj5kCxA3/LPe4ahn5e4apVLLqIWO+zG63z8p3c2IK74VGgIpFDZfZXEVvQ60TfNZghcNhKeDQG+e9BAIBSRHQMyS3Bb1P3iMwY3TUAiw3Xk1XcU1VRckDonqGXBM1T82/VYMKQOBkVGhUbmwxpyfH3gT+nzlilTvE6XRicnISFosFIyMjcDgc4qkwbcDIjfdBildGMFQmanRE2BqjJN6nypUyPT3D8a+m79QIRq3bUAYYbamFXKZguB7qunHvVSeGB57rTwVPmaKnx+cCLtNS8xpcC16XjktZWRnGx8fh9/thtVol+qEizMrKQmpqKiYnJwWlwjQVZZOKmLKsUjSQS8bv9wu7oqqA1M+pvCykLmb9Q80dq4M1WJSljDGy5Pg75uXj4uIkPTk8PAyr1SpKkpEMi5cARF/MnTsXg4OD4kDRay8uLhbjTjoJAEhMTJRpb3q9HuXl5XJvRUVFMkh7ZGREoo6hoSHYbDb4fD6RP9WB5XWo2+gsGQwGeDwe6HQ6qQGGw2FBKtHJ4yzhz3t9ITz3z+Yg1Tw2BYAbTIsHYFbKgl4tBUn9DiosHmwqdB4On8+HEydOSD6dRRlen9dSC2G8XxqLiYkJvPTSS7OKieqzUKFSAU9MTMzy9jVNw549e3D8+HHs2rULW7duFVSCqjypVOjV836ZG+QzqflbChUFhX/Ue1M/p+btuTdqGAxc9rTo9fIzRKWoh3JqagqdnZ3Yv3+/rCe/m0RS9IojIiKE6U9NIdHLIyvi+++/PyviYF1ErYlQ2alFTzoJ9AhVI6zWIrgmVD7q2nFP1EIp/63WsBhRce1oJHp7e4UFVZUx9bNqyovPp6YpqcxHR0fx3HPPwefzzYpGY2JikJKSgtzcXBQVFcFut0sxlh2bXCfuJVlIuc+9vb3o7e2VSV1xcXECL6bnTu719vb2WWumpkfZCavmn1XngzJMFFlDQwPOnj07K5XE9aOnHAqF0NPTg08++UTOa29vr3jo/BmNAQ1wKDTD/0OWVdZ+MjIy4HK5hLedUWJ8fPz/5hyw+cvv90s9g8g88tazE5byp57FAM43QQAAIABJREFU6OhoQcepPQvcc8oio1+mtviHUYuq4/7W6wuj3Omd8AHoTaoQPypX1QMCLg93Zs6Si8I0ATATAZDjm9fkgXz55ZdRX18vHgxweZQcP89UAO/PYDCgp6cHR44ckfvcsWMH4uLi5IAQBsXPswBEr4iKBAAee+wxLF++HKtWrcLNN9+M1tZWQUqoxTSGtXFxcVKppyeqGhyVMK2zsxMff/yxFAVVNkQAwmbHSIHPqSIUGDGpxT71WfgZALOUKQBZD3ZS0tgODw/PGmDg9Xrxy1/+EgUFBdDr9bDZbLBarVIo42FkAZQGhMpGRSXwbypwNR/MCJAesEorzMiCB5IKmsbzsy/KIhUxybvUNaBc8hlYKFPz8FQC09PT6OrqwvHjx8VbVmGSdDKIHEtMTERnZyeSkpJmOT98ZqYI8vLykJ2djbKyMlxxxRWIjY1FUlLSrLkIweDMmEWv1yv7wtTQ0NCQeKdEs5BYToXTMqohyRYdmejoaGkMoxdPJcg6l8lkwjvvvIPx8XFB8lC+SDVCIxAZGYmmpia0t7cjHA4LIILnn8aTEakamfE88nkPHz4sUY7FYoHb7ZbxeEzpkaqXQ7T9fr+8x2g0SvQwPDwszWvkxhkYGEBnZ6dEfYyQIyIipOmQUQSjKdanVOeVRorZCNXZ+luvL0RahqE7w3jVylJx0hozzKWCUPOWLKLS0vGlpiDUoqLBYMCFCxdw33334Te/+Y2EePwMDyqVND9HJMyrr76K+++/H8CMECUnJ0sBhYVXGhh6ErTYanGNhuEPf/iDKKX169fDaDTC6/XOGnFH0v+IiAh4PB7JyVEReTweqabHx8ejr68Pr7/+Ou6++24hMCOviJrqoGEj3zmhb9wXek4qyoZ1A6KTqChYtGKIrNfrsXXrVjz99NOSU3Q6nQId4/N9+OGHMBgMcLvdSEpKEv4Qi8UihdeUlBS88847uPfee2fh49k1SKNKpa7T6dDX1ycsfmp+3ePxCDSThXrKXWxsrDgDKiKGVL9se+fMWUYCiYmJszxttVhKGY2PjxfmUuZ0gRnFHQgE8N577+HGG29Ec3OzpOi47qSXUJXrnXfeKe/h+jLVwWHoExMTolg0TUNlZaVwl3NwS3JyMoxGo7Bs8jkvXrwIs9kMm80m59VoNCI7Oxt+vx/V1dX40pe+JAqeTIiBQEAcAxppvV4vMwTYIMSCfSg00+19yy23yOxSDpY2m83Cq09DcPbsWVx//fVCcWAwzPC00Ei63W6YzWZ0d3eLzMbExCA+Pl5G5/n9fqSmpmJ6ehqtra0SPTidTilkulwuDA4OorKyUhxC4tRTU1MRFxeH7u5uIXUjvTFTyrW1tZgzZ44wlEZFRclgmqSkJOnSDoVmGFnZn6FGlVT+LLgyAv281xdGuVNRU/GpaQLmwD6LLmC+lUVN4HJOWFXk/H4uCL9nbGwMZ86cwR133IFAICAHmE0Uhw4dgk6nw5w5c4Qvnd7gwYMHceTIESxatAh5eXmCJKirq0NzczO+/vWvS3PHqVOn4PP5hLho7dq1s4ptjFxef/11lJSUYPHixcjMzMTIyIgw5pWXl8PhcGDlypXIzs6Gy+XCyZMnMT09jcTERCxbtgwtLS3Yu3cv8vPz5aBYLBYcPHgQ5eXlKCwsRGFhIU6cOCHIFE3TcNttt4lyOnTokGB9y8vLUVtbi56eHqxfvx5jY2Po6OhAUVER9Ho99u3bJ2HytddeKykK7mFNTQ1cLhfGx8dRU1MjkUEwGMTx48cxPDyMQCCApUuXorW1FXV1dbDZbAJji4mJwYULF8SwVlZWIjY2FufOncP8+fMxMTGBvLw85OTkSJv89PQ04uPjsXjxYuj1ehw4cEDyqiRS83g8uHjxotAR5+fni5dNZ2FwcBCnT5+G3+/HvHnz5L6qq6vR29uLlJQUZGRkoL6+Hn19fcjLy8Pg4KDkZAOBgODzw+EwcnNzAQBNTU1oaWnBLbfcIrBF8rgsXLgQJ06cQE1NDTIyMpCRkQG73Q6Xy4XR0VH4fD6sWLECwEx3djAYREdHB1auXCkDI+jUXLhwAW63G0uXLhW2yGXLlsHn86GmpgYPP/ywDIXw+Xzo7u4WVAvx1qdPn8aCBQvkPJBgj9DLm266CZqmyUi7cHhm4IXBYIDVakVXVxf0ej1KSkok0mNuv6+vT5oVy8rKMDY2Jp/t6emRDlK1qKsS0wEQdExHRwf6+vowf/58uY7P58PY2BjOnTuHzMxM9Pf3y6AVs9ks09ZaWlpgNBrh8XhEyRKllZCQgM7OTqESUeHEbrcb4XAYc+bMQU9PDy5cuICCggJMTk5i//79KC0txdjYGBobG9HY2AiLxSJNZv39/UJ7nZqaCo/Hg9raWtjtdtm7iooKxMbGigNDWgXSOLS2toqx+HuvL0RahggN4HJahoqAP2OxQS04qSOyeDDV6rrqITPPOjIyItC1V199FYWFhfD7/ejo6BDIXDAYxG233YaSkhLMnz9/VthOnpZweIab+dZbb4Ver8fRo0fxwAMPYM2aNejs7MTw8DAMBgOqqqpw/vx5LFy4UDgvmAogUmNkZATHjh3D8PAw3njjDdx2222Ij4/HqVOnYDabsWHDBixZsgQOhwPr1q3DH/7wBzz11FO4/fbb8eyzz6K6uhqBQACHDx/GwYMHsWDBAqxduxalpaXQtJmBALfffjtKSkrQ0dEBh8OBG2+8EW63G8888ww0TUNLSwsefvhhLF26FHFxcTh58iT27t2L8vJybN++HZGRkdi2bZukgrZs2QIAQiugpnlIO3Dp0iXccMMNGBgYwPDwMKKionDhwgWsX78e1113Ha699lps27YNoVAIeXl5GBkZwX333QebzYbOzk48++yzqKyshMvlwpYtW6TxxuFw4Nprr8Xq1avx4IMPCnfPDTfcgIULF+Ls2bOIiopCX18f3nzzTWmVB2YikZ/85CdISkqC1WqF2+2WFAIV/BtvvIHvf//7WLJkCY4dO4Zjx47hueeew3e/+11EREQgISEBv/jFL3Dx4kXo9Xps374dSUlJKCsrw0svvSRTvJ5++mlJJwBAe3s7LBYL9u3bB7/fj2eeeQY7d+5EZmamyBSjT7vdjlAohFOnTuHkyZMyK6Cvrw9vvfWWeJ779u1DQ0MDqqursWfPHnz00UfYu3cvLly4gO7ubuzYsQMOhwO9vb04d+6cKCx2XL799tt48803UVpail27duH8+fPQ6XTwer0ScRQVFSEtLQ3nz58HMDNyb8+ePcJKyQHRbOCh53/06FHxvPkyGo3YunUr4uPjMW/ePKSkpODUqVOIj4/HwoULcdVVV2Hp0qUS8ZLf/oUXXkBGRgbC4TD8fj/C4TBycnKQnZ2N9PR0nDx5UuoBJ06cwPT0NLKzs6HT6YRSNzIyEmfOnBFlbbFYcPHiRdxwww1ChJadnY2cnBykpKSgubkZHo8H6enpiIyMhN/vh8PhQGdnp+yZw+FAT08P2traYLPZYLfb8e6776K6ulrGAQ4PD6OyshLZ2dnSNMVu1UAggK6uLly6dAnHjh1DWVkZ3nvvPdEHmqZhy5YtmDNnDsrLy+H3+9Hc3Ayn0ylUw3/v9YVQ7gxfPwvJY9jK8EQt9gGXc7tqwZGeteoVA5fhYmrePhAIwOVy4fTp0xgYGJC8HHN4P//5z/HSSy/JVHPm/6anp3HkyBGkp18eMnXu3DlcffXVGBoaQnNzs+SXt2/fDqPRiNraWkRFReHee++d1ekaFRWFwcFB2Gw2fP3rX8d3v/td1NXVIRwOIzs7G3V1dcjKypI0zOTkJLZu3So0usFgEAsWLIDJZMI111yDrq4upKenQ6fTYdGiRTh8+LB4OtPT09i5cyeWLFkiVKfFxcXQ6XTYuXMnLl26hKioKCxatAj3338/lixZglAohEWLFmFsbAxnz55FUlKSNHW88sor+OUvf4nKykqJUmhsd+3aJdfp6OjAnDlzAAB79uxBW1ubhKN5eXmCArJarVLAOn36NJxOJwyGmek7eXl5sFgs0Ol0gopgMayzsxP5+fnQ6XRCoMVUTUNDA371q1/JXNbGxkZJi3R3d2P+/PmzeiDC4Rl2xdTUVPT19eGKK65ASkoKPvroI6SkpIicMqQuLCyE3W6X8XC5ublCqWs0GrF37140NTXBYDDAbDajpaVFeHlSU1PR3t6Ojz76SNAjDocDFosFg4ODGBwcRE1NjfCbR0dHw+Px4MiRI8KbTjgkvVuen5SUFPT29srs0+bmZkRERKCjowNNTU2iVN5//33J109OTiIrK0vSk0zjpaenQ9M0lJaWwmw2o6enR1rm29vbBdURExODvr4+WafJyUkUFBQIYonYcXbY0lGjBz48PCwRjlo4np6ehtlsxltvvQWj0Sh0CBUVFYLcYa1hZGQEZ8+eFYQPYcXkRY+LixO6b9J3jI6Owu/3IzExEVarFbGxsdDr9bBYLKipqcGePXukmYtdr/39/cKpk5mZib6+PmiaJnlys9kMo9Eo3P+ssdXW1iIcDqO1tVXqGSkpKRgYGMDcuXMl5ZqXlycc8pQXk8mE9PR0HD16FACEf+fvvb4Qyh2AKHA2mqgIDHqEfA/zpvzDAoV6QFUMOnC56MqC4bZt2/C9730Pa9aswZo1azAxMSHzJf1+Px5++GH88Ic/RHt7u3TU0YBER0dj9+7deOCBB3Du3Dn09PRg586dSE5OFuV16dIldHR04KOPPsIDDzyAm2++GXfddRcyMzMFNhcOz0x7/9nPfgaj0Sg0n8Te5uXl4f3338cDDzwAg8GA6upq/OhHP0JXVxduv/12TE5Oori4GBUVFYiIiMCcOXOkcYrIIX7+7Nmz6OnpwcaNG5Geno5gMIi9e/fioYceQkdHBzZt2oQVK1YgFAqJ0JaUlOC9997Drbfeivr6ehw8eFBC4NbWVvzxj3/EsmXLsH37dgCQ/LTJZMKGDRsk3XHgwAHcf//9cDgceOWVV7By5UpRVN/85jdF6a5Zs0YY8LZt24YFCxbAarXizJkzeOihh2RMG1kIydlz5MgR3HTTTQiHw9iyZQuKiopw/PhxbN68Ge+//z6+973v4fHHH8fk5CSOHz+Oxx9/HKtXrxYSOhr+kZERdHV1obm5GXl5eWhra5N6gMvlwsqVK0XJVFRUSE2Egzfq6+vx1a9+VVAmv/vd7/C9730PmzZtkk7b3bt346677sLAwABWrVqFp556Ct3d3Xj11Veh1+tx/PhxXH311UJQVVtbi0WLFmHhwoUoLi7G0aNH4XQ6hbPFZDKJcmMnLx2GCxcuiNw2NDRgenpaphtVV1ejvr4eXV1dYryzs7ORlZUFAIK2CYfDwleTkpKCQCCA2tpazJ07F16vFw0NDViyZImgpN577z2kpqYiMjISmZmZSE5OFqdsYmICx44dEwrfwcFB7N27F0uXLgUAnD59WhqyiJKy2Wy4dOkS1qxZg9WrV+PNN9+EpmloamqSjtITJ07gqquuwsjICKqrq+F2uxEXF4ehoSHU19djyZIliI6ORktLC66++mpYLBZR0DabTZw+m80mUUZPTw88Hg/Wr1+PZcuW4dy5c5iensa5c+dgs9mQk5OD4uJiGYAeHx8Pj8cDAFiyZAnKy8uRkpKCmpoaXHvttejs7ERUVBROnTol6VHSM9MY5OTkoLe3F0uWLEF2djaMxpmhMQsWLJB5yb29vTh27BiWLVuGuXPnfq5O/UKwQm7YsOHphx9+eBaZFENC5tBVDDZ/zoVRUR2sXtNzVw3B5OTMRPj9+/fj4MGDuPnmmxEKhXDmzBkJrRcuXIjR0VE0NjbKiK/Vq1dL6ojRxY4dO3DttdciIiIChw4dwtDQEG699VY899xzEtZVVFQgNzcXbW1t8Hq9QnPK3Jqmafjwww9RXV0tOPlTp07h8ccfl+LySy+9hIULF+LixYt44okncOWVV8LhcMBkMqG5uRnj4+Pw+XxYvHgxurq6YLPZBG0CANu3b8c111yD6OhoYcWbnp5Gc3MzWltbkZmZKURK3d3dwvQXCoWQmZkp0c2ZM2dkEMOiRYtQVVUl3tC6detmoT+4VhMTE3A4HGhpaUFOTg4qKiqkEKnT6WRQydVXXw2XyyW46LKyMiQlJWFsbEyoe/Py8lBRUYGYmBhUVVWJIv7a176G1NRU1NXVoaurS9bYbDYLra7T6cSaNWuQnZ2NgoIC7Nu3DxMTE2hra5vlMXs8HjidTkGPhMNhdHd3S76URcuCggLceOONiImJwalTp1BZWYnJyUns2LEDubm5s7pqh4eHRRFwPfLz85GQkIC+vj4ppK1evRpJSUk4cuQIUlNTJf2Tnp4Or9cr2O6CggKMj4/DYDDg0qVLMnSZSjQpKQnT09M4ffo0pqenhVfc4/GgtLQUExMTcLvdyM/PF2505tQHBweFJqC+vh5WqxWJiYmYmJjA0aNHsWzZMqSkpODkyZNYsGCB8MKnpKQIyog01PX19SgrK5MOVqZY7Ha7pC1bWlpQUFAg0duePXtw3XXXST2N9Qru4+TkJMrLy5GZmYn3338fc+fOhdFoRFVVFa688koZlMIemKmpKdjtdmRmZsJsNqO9vR2rV68WNsrh4WEZoqHT6YSyd3R0VGpSJpNJhmjEx8cjOTlZ9ElHR4c0ReXl5cFsNqOjo0NQQQkJCairq0POp5OhMjMzkZaWNovojUg1DlN3u92w2+1y1tLS0tDe3i4DQxISEpCbmwu320398XdZIb8Qyv2ll156+sEHHxQ4HvPe9KjogdPbZXGUaBoiBJgzpXdPoSLmNhQKiQCnp6fLRCWXy4WsrCzk5uaisLBQ0CQGgwHz5s2TsIgGg5Y2Li4OixcvhqZpKC8vR3Z2NpKSkmC327FkyRIkJSUhPT0dbrcbERERyM3NRc6n01PUZpmKigpo2gw5lM1mQ2FhocD+WlpacP311yMmJgbl5eXQNA1paWnSQMHJ7KQIJlUo1y8hIQEpKSlYvHgxjEYjMjIy4PV6YTabUVJSgpiYGJnIRCVWUFAg4TE798rLy5GXl4f58+fDarUKMobGhAqRxpdFLLPZjLy8PAmH7XY7LBYLkpKSUFFRgfHxcaEmDgaDMsUpOzsbCQkJsFgsMg2ooKBA0llJSUlYsGCB5E7dbjeKi4tleHJxcTGKiooQCARgMpmwYMECwWZz6LLFYpHOzO7ubqmD5OTkSFqA8LjMzExpwiGdKyO5rKwshEIz1Ah6/QyPu81mE5wz942yazabYbfbZbh2Tk6OdDiS6TE7O1uQHSzEpaamwmq1Ijo6WlgGExMThaiKKBKiR7KyspCVlQW9Xi/rRMRMSUkJAEjxNDU1VepSvA+DwSCY+OjoaBmgQWQR0TUkt9M0TdgWe3p6kJ+fPytVOjw8LE1MAJCZmYnU1FSJNI4cOYJly5YJTJEwYIvFImc9OTkZ0dHRYujoSMTExCApKQlDQ0NIS0sTZBkNJeGy6enpQm1MVFZ6erqkUMbGxiSNSTy6Xj9D7xsZGYnExESMjIzIQA0i4ZKSkmQoNteTBG2JiYmyfuQE0uv1sn6EN0dFRcnw88TEROmAJYzaZDKJTPT09HD4yxeb8nf+/Pnhw4cPC6SMXrja0aVCEal4qcgJo1Jx6fTmWW1XjYKKkQ8GgwIdpLKl5VXz9ioKh9/P7+F7WFRUibcAiNfHtVa7VdkAQ+w4MEOXMDIygg8//BDJycnilai8GnxeFUvNa6mdqvRi+Dfzsvw9P8PGMHbescDNyEeNlvh5tYCq1jRIysT3EnPP9WTOmkKtdiBzbbnfPDxqlyevzWIV15voKLVDkgV64txDoRniLA54Jp6YKAleh3BKdb+ZZ1ebbgg17OvrE9wzcJmagN/PPWZqxGQySeFVTfkRD01uEq6J2ojDXCxz2Ix4yIjK7spQaGYAxPDwsDg5lGPKEOGC5DMiFUR8fDzsdrukKtkQFB8fL3NIed9cs/HxcbS2tsJsNqO3txfLly8XueM1CZGljE1OTuKvf/0rcnNzEQwG8eUvf3lWU5hKacy0B1EtgUAAoVAIKSkpIhdEwxDS6Pf7Jf1BGfX5fNIDQGAGIYxerxfFxcVwOp2IiIhAcnIyRkZG4PP5YLFYZvWKsHkpGAwKfJGyz/PIISgDAwMSJaqOF/d3cHBQGjApN+p+8VwR7srO2/vvv///H8RhPLCqMlR/zg0kllZts6d3wHCM36u+KIgUHl6D76ci4zWpjJjXZyGWSpWoHrW1GLjM5wLMLvryc1QYaseqiiE3mUxoaWlBZ2cnAAjZEotnvFcqbT4L00YUIHrRqtFTm4h4f+rrs0r/sy9+jt/L9eIaqURpKpyVa0kDo6bbVENFI8DfszjOe+Wh5+9VpU8DqSp4vo/e7MTEhKTbqCg1TZPmM76Xa6b2XJCE7LNGRH1+7i2f57MySU+TTUZETlHOuDdqByedHt4Ta1M0GJQ9ngsWoakYjUYjBgYG5LyQf4cItMnJScH1qw1TTLMQTUTjTodKpVBgkxL3Yd68eRJBq/0rKoUH5dNiscBsNsugD71eLw1nlA8AYrzo/LCXhJE2nYvh4WG5LzI7jo2NSVMgodU8a5zpyj90cBITE0WhUk5ICcEicFpamuwzDRmNL1M6Ho9HjDiNP+VSlW0WgLmHJDzki3qGOpIsmX/v9YVQ7lQ49H7VLkm1pZsLoTIpElVDoafAEh9PAaTl48FRvVOVFlbdKAofFYzaBcrvUA8a74UVcypt1XuksKhQL+L11caf0tJSFBUVyXVVxU4vDphNlDYyMjKr41BVCPy32t1G70/1xicnJwX5QsNENAnJoaampkTJUSnxHtR9VENrGmO+l140MGO8enp6MG/ePHlG3guxzlxHGiNePzY2FsDlQRGMMKanZ4aP9Pb2CoUtgFleFpuvLBaLdFlSPthVyRF6xFhTTtThMeFwWMYQ0gkYGhqSepAqB9x7FrypiNQ1UuUQuIznZuGOskDK2qmpy0NU6FEODAxI4xDRH/SU6aXzfHEdBgYGJK3Bbk+mg6gAR0dHRYlPTEwIKoz9Dtddd90s3nYahZiYGPF4qYS51mvXrgVw2WFhdMMURjgclhSfTjfDGUNZYpMRh3szF11RUSF0x5qmSYTFwi+bojRNk27YiYkJpKamIiEhAampqZiamkJ7ezv0+pnZqPz+3NxcoRcYHx9HSkqKQG/ZcUolznWn1860GZ0j6hoafDocNKg8C3ypMqUq/r/1+kIodyoA1VNkaA5A2sXVVAgFBZjtvbEdXa/Xo7+/Xzo+qeRUnDz/zz/01KgM2eo7MjKCyclJaZlWvWF2ELJjk92cVOgUbnojasqB16CwqwRYajcu10eNDngQaMy4TlSan61X0OtUh3qo6Qx6MioElZ4gPRGVvoHrxYiBxSE+L4t0wAzyggZZ9aZphBMTE8XQUh64Pvwcm9XoAXFP1evRs+ru7kYgEBClTmVLRUTDRqWocuCoESHXjvtO71Dl6edzcL/9fr8ofnqrIyMjQoRGR4UdoeRFJ8kdX9xrhvmMNomq4hAJQkc5aIMeZm5urhgssjeqcN9QKCT7wzUxmUwCgQyFQrBarVKnUqMVonNodJkDHx0dlQ5vyhL3nilFne7ycBU1nUhZUg0gjXdfX58o53A4LHTASUlJQj3MTufIyEhhcwRmajPEtQ8ODiIUCkkum+c0MTEROt3MbOOenh709PQgMTFR7nloaEjy/oRlkg6A9Q+fzyfDU1gPodPIuiBpFRgZkLYZmHEwSa6nZiHU8wBA0sXquf97ry8MFFK1TsyRU3lRYao5ZBWzzv+reWV6MPRs6dnxEPI9akel6n3Su1SjAt6nisAJh8MStlNBsDBHD5BKg8pXTS8Bsxu3VIGnIeJ1aLXVWgD/VlMhNJQMQdkYw+9QU0VUtFxf/s1nVfOlvI6azlLrIWrdQx1aoq4tv5fUvVQCDN3V9acsMPWhRin8Tu5PKBSCx+OBy+WSoc9U4PS4+R3kPFG9IOByf0VERISw8NHAqKkiXp8duXyph5FKVv0c/03FT/kjtwqNgZrqIpKD8s6/AYhCJ6qDMs38MKcl8SzweSkDLOLRcSBnEfeZSpR8MKTV4J5S5rkH4+PjkrZRMfeqMlLTLmp6EMCsWQ2MrvgdlDmj0SiRAx0KNaK32+1Sq2AUy9+zUzkUCgkqhqmTvr4+9PT04OTJkwIb5efsdrsAMlj4p3PEebM0vJStUOhy4+VnnQVGSmraknJP3cFzxb2nzlE5kVRn4G+9vjDKXRU+HmZ6NPRgAcjv6DWonhyAWYqEgkTFTz7l6elpoTVVU0K8NhcXgNASqCkgek0TExMC66Ol/dd//Ve88MILIsAqWRg9KDV6CAaDkjKiceLGOZ1OuY/29na8/PLLorB5SAHMUrhcGzLHsVikKlYaEtUwqfzYNEKvvvqqNMoMDg6ivb191qg9pmkAiHdNAWVKhnUKtXYxMTGB7u5udHZ2Cs8814R5c+bluZc0tNz/sbExkY1gMIjW1lY0Nzejs7MTTqdT9onXJ848Pj5e9k/TNHi9XrS1tcm1uL7j4+P4/e9/L4pcrQUwbKb3S/4cnU6H2tpavPTSS+JRAxD4HMNoeqRqrYFRFH9GznM6Ccw3c/CD2WxGZmamKIzh4WEZLh0TE4O4uDhJSzL1wo5JRgtsjmpra4PP50NfXx+am5vFA6bsMxqmx8uUDJUsydx0Op10p5LXiB2m9EZVB0eN4igfk5OTcLlcOHTokIya4/lWQQeJiYkYGhoSyCD1h9/vlzF4ACQ6YVdrfHw8qqurZ+XbSQGRmZmJkpISpKenC2otLy9PBlVz8pTf7xeZIM2EyibKdeeQd2DGeUpOTpb8/+DgIILBoMCOaegYlXF2MteFhphEaz09PWhpaflcnfoPlbumaZmaplVpmtaoaVqDpmmPf/rzRE07QPHTAAAgAElEQVTTPtI0renTvy3KZ57SNK1Z07RLmqat/kfXACBejFpc5IFRW8PVQhvzZ/S8OIJNLVLQQ6EAAZfTOFwofl4tCtLiGwwGGaTB/1NhqLlDehWffPIJCgoKxJLzHqh46aGqBVF6sPxbp5uZdH7vvfeKog6FQpgzZ44cLlp+/pset5r/56EgZSuVrGr8KEBcF/U7582bJ2v6b//2b3jmmWfEK+P7RJB0lwenkKSNB0A9wDTITqcTfX19knvmGoVCIeERomIBIN4O14upkM7OTly8eBHd3d2zFAsVOTHgHEk4NTUlk+wJPcvNzRWjRqQDr0evVTXKqufO++S+nT9/XtJqXFve144dO2AwGEQZ0dtmhMF7IBKH5GIs4BF1QWXCa4+MjKCvrw8TExPo7e2F0WicNSqO1+BoNnU/srKykJGRgfHxmYHiZWVlAGaUJzH2NARnz56Fy+USxkXuMesQPFeU+02bNgnahs+RkJAgMgpcbi7kegaDQWzZsgXz58+Xe+S5o8EmZLe/v1/kkNS+9Ny5PpQri8WC0dFRdHd34+jRo9Dr9ZLTJ4ePTqdDcXEx5s6dC5/PB4/Hg5GREXHSSBLGnD+RMHq9XlIyer1e9ocvOmEejwcejweHDh0SSu+pqalZNQR1SE8gEBCdR/55RoXvvPMOMjMzP1en/nc89ykA3wuHwyUArgLwLU3TSgE8CeDjcDhcCODjT/+PT393D4AyADcCeEHTtM+fB8WbUdAtVHzA5bQFAMmxApiVxlAPGZUcPQF6XfQy+H1qSMhFVAtmvA9W0dXCJA+6WlCk1ea4LYaTanjFv5l64e9U6Fw4HMb58+fFm5uYmEB9fb0QgtHQfRYpw+9gyM8CpJpWYkhHZcWX2gzGP4sXLxaPqbGxEWVlZXKY+QxMIfD/NJZ8Tn5eVfKhUAgFBQWCyadxUfddTZeoz8zX+Pi4TA7y+/2iJCIiZvjgGamxhsFIj2vA71XfxzUlNDI3N1f2jLLCvWLUw5/xHru7u5Gfny9KmpSxbNVX033qulGeKcdU+AMDA7P2ioMhWNSm4SHnz2fXkQaeThDXkWk1pm1Y8COdLYeTM7JjmoNKjcqMKVNOaVJTUxERM9OMmB7lmaJR4M+5FtPT03J9Foi5X/Ra6XHTI+f9cwQmf8Z1MRqNgpLiXsXGxsLv9yMQCMjMWJXdMzo6Gl1dXcKQyuhHBSdQ3hgVq3qBSCI+L+WDe+l0OqUBkUgfyml8fDyio6NlfixlXk1TEz//j17/sKAaDoddAFyf/ntI07RGAOkAbgWw4tO3/RnAQQA/+PTnb4bD4QkAbZqmNQO4AsDxf3QtHggWZd59910cO3YM119/PTRNw9mzZ/HEE0/A4/Hg6NGjiImJgdPpxDXXXIOBgQFpGS4oKEBjYyPuvfdeNDU1wefzweFwIDU1FbfeeitCoRBOnz6NQCAgpP1ZWVniTaanp6OzsxP33Xcfmpub0d/fLymiG264AVVVVTh9+jSWL18uYe7y5csxMDCAnJwcOJ1ObNmyBUajEY899hgmJiZw6tQpaNoMI+Ftt90mYSo3l2kMg8GAhoYG/PGPf8SCBQtw8eJF5Ofn45VXXsGTTz4Jp9OJM2fO4NFHH4VOp8OpU6fgdruRmJiIuro6fP/735fwcXx8HC6XC5cuXcLRo0fxxBNPoKurSxqEvvWtb+H73/8+jh8/Lm3r+/btw5o1a+D1epGamooVK1Zg//79cDqdSEtLE47quro6oZyNjY1FSUkJPvjgA2mAOXToEH784x9j//79UvBas2aNhPTENZ86dQp//vOfcf/996OyshKbN2/GI488gvXr1+Puu+/GypUrsWvXLlitVpw9exYLFy5EMBjEBx98gIiICNhsNvT29mL+/Pno6+uDxWJBY2MjSktLUVhYKEqOxdaGhgbExsaiv79fGCEnJydx6tQpmcN5ww034JNPPsE111yD7u5uNDY2YunSpYiKioLD4RBlEhsbi4KCArS3twtjosFgwBVXXCGHnoXl7du3w+PxoKamBsXFxdDr9WhsbBRoZn5+vnDnqEo/FArB4XBA02YGm5SVlcHj8aCjo0PSHYsXL4bH48Hu3bvFQ+3s7MSCBQsQDofR29uLiIgIWK1WtLa2ShctG2I0TYPL5cKFCxcAzAzaZtonOztbFDUww2qp081QS5eXl2Nqagrd3d3o7+8XNk8yWBYVFcHj8QgOnYoqMjISPp9PmBtDoZBMR3r33XcRGRmJvr4+KYyysHnp0iVMTU2htbUVBQUFgpRqb2+HTqfDuXPnUFBQIAY/Pz9fokM6NR6PRwavAzNIpNbWVgAzqZuysjJcvHgRTqcTkZGRcLlcaG1txcqVK/HBBx8gJSUFiYmJ4kheffXVGB0dlT0ZHh7GwoULERERgZaWFnEACgoKkJ+fLzWW6Oho7NmzB7feeiump6dx9uxZ9Pb2orKyEj09PWhqasLVV18twz9o+Pv6+nDhwgXExsaitrb2c/Xp/6Ocu6ZpOQAWADgJwPqp4qcBSP30bekAupSPOT/92ed9r3idzClysT/88EO4XC4sWrQInZ2dCAaD2LFjB+rr61FZWYmBgQGMjo7i9OnTyMjIwHvvvYd58+bBYrGgrq4O27ZtQ3l5Od555x3U1tbKdf7617+ivLwcWVlZqK2txalTp5CWlob33nsPUVFRSEhIQGNjI7Zu3Yq5c+ciJycH586dQ0tLCwYHB7Fnzx709fWhsrISmzZtwujoqHTl2e12DA4OYuvWrTAYDKitrcULL7yA0tJSREdHCy+1WjhTm0IIu1q1apWQhnV1dSElJQXz58/Hli1bpIHj3XffhcFgQHFxMZqammZZe51Oh5qaGsTHx6Oqqgrj4+M4cuSIpE9IR2q1WtHR0YGCggIEg0E4nU4htTKZTCgrK8OcOXNw/fXXC7viG2+8IXBNh8OBpqYmWCwWtLa2isJsbGzEyy+/jPnz50uzklpv0Ov1aG9vx5EjR9Da2gqfz4dt27ZJLryrqwvHjh1DVFQU5syZg6amJjgcDjkI1dXVsNvtsj56/Ux3aG9vr/AB0eOdnJzEiRMn4HQ6hQmUstfV1QWTyYSSkhI4nU4AwKVLlxAREYH4+PhZY9qOHj2K9PR0ZGdno6OjQ35WVFSE5ORkpKenIyoqSuTMZDIJooJd0JGRkaipqUFiYiKys7PR0tKCqqoqKZxRNgwGA1pbWxEfH4+8vDzU1tYK0Z3NZkNubi4OHDgAj8cDt9uN/v5+NDY2wm6349ChQ+jt7RV+cL/fD03T0NjYKOk4cuBzLJzBYBDWTbbfM61nMBjEqcrOzhbPksYvNjYWaWlp8Pl8SE5Ohtlsxvz585Gfny9eOfPgXq8XJ06cgMViQX5+Pnbt2gXgcuNYbm6ucMerReQ9e/agpKRE6g+apuH06dOIiopCWloampqaZFwmGSzT0tKEmZNY+vz8fMTGxiIuLg4XL16EwWBAdnY22traEAwGZS4Du3o5yKejo0MQeHa7HW1tbZiamsKZM2ekwcnhcIjBI9KMeXdGKHReh4eHxbEjFUQoFEJ6ejpqamoQGRkpETidhdjYWCQkJKCsrAx5eXmfq6//28pd07RYAO8A+G44HP48ImHtb/zsf2uD1TTtf2maVqNpWg07z2gNWf2/8847UVFRgbvuukuIpvx+P1577TU88sgjiI6ORnNzM4qKinDPPffAZDLh8ccfR2xsLO677z48+uijSE9Ph91ux9jYGL7yla/AZDLh+PHjKC4uRnp6OoaHh/GVr3xF+FGeeOIJrFixAnfddRceeugh4ZQYHh7G7bffjvz8fNxxxx0oKyvDbbfdhsjISDQ2NmJkZASbN2/Gk08+iby8PNTV1WHx4sVS8JuYmMB3vvMd1NXVCe6YRVfmiVVE0Ny5c4WQKhgMory8HEVFRZiYmMDChQsRGRkp05VIZXr//feLN8ewcdWqVUhPT8dXv/pVZGRkYOvWrdKB+9BDD2HJkiXYu3cvHn30UWRnZ+PXv/411qxZg4MHD+KOO+7AyMgI6uvr8eCDD0pr+d69e2E0GnH+/HnU1tZi3bp1yMrKwsGDB/HNb34T+fn5+M///E/JoX7rW99CXV2d4JTViUxr165FSkoKbrjhhv+bujcPj7I+14DvWTLZk8m+LxASSEjYl0R2lF1xqwWtKKKtbT3dzmk9PbWnxdrj0XPsqd20ilJLLQhIVVAJEMKWAIEECCH7vk6WmUwmk5kkM5N5vz/G+8kbTvWc7zrfH35zXVzGLDPv+3t/v2e5n/u5Hxw7dkye/5YtW7BhwwY8/fTTMBgMuHr1Kh566CGsWLECM2fORH19PR555BGEhITgzjvvxOuvv46AgAA0NTXhvvvuQ35+vqTsTMmPHj2KLVu2wN/fHx0dHYiPjxfHevDgQbz88svYsGEDuru7UVJSgpKSEgQGBmLp0qUICwvDzZs34fV6UVNTg7a2NsybNw+tra0ypcfpdGLdunVTYBrAB//U1NTgzjvvRExMDLRaLY4ePSoCbU6nEzdu3IBG42umCgsLE0rcuXPnkJycjICAADz88MN4++23ERoaivj4eISFhYm+SX5+Pjo6OrBmzRqEhYVhcHAQNpsNc+bMEZEts9mMjo4OUeDkmRsaGsK7776L9PR06blYunSpsGYcDod0hHo8Hhw4cACtra0wGAx44403oCiK0E/z8/MRGBiI8+fPY/r06UIaYKQcGBiIl156CZGRkaI709zcLE6/q6sLBQUFUyi2Op1OPvt3v/sdKioqEBISgsOHD+P06dMiKbJp0ybMmDEDiqLgrrvuQmhoKNrb25H+meRHbW0tjh8/jrlz56KqqgptbW0oLS2FwWBAb28vtm3bhvDwcDgcDmzcuFEyOUJRy5cvR3NzM5YuXYrIyEjce++9ePfdd1FaWoqOjg5MTExIJO7xeFBSUoKOjg6sWrVqSvZTUFCARx55BBqNRiDEkJAQtLS0IDk5Weom8fHxUl8gZBoSEiL6RIShPu/1vzLuGo3GDz7D/ldFUf722bf7NBpNwmc/TwDQ/9n3uwCokf5kAD23v6eiKG8qirJIUZRFUVFRgjsx4vF4fFKuu3btQmRkJA4ePIjOzk4YDAasXLkSMTExuHDhAi5fvowrV64gMjIS7777LpYtWyYR2+joKO6++26MjY1h6dKlSEhIgF6vR0VFBTZs2ACr1Yo33ngDubm5iI6Oxv79+5Gfny9G1uFwSDv0nj17kJaWJmPNtm/fjtDQUBw+fBgvvvgiYmNjUVRUJBvS4XDg2WefRVdXF86ePYv9+/fjRz/6Efbu3QuDwYC6ujoZ5KvRaKbwj0+fPo3HHntMjFBjYyN27twpcr8/+tGPcPXqVbS0tODxxx/H2rVrcffddyM1NfW/sS4iIyNx5MgRbNjgq2v39PRgYmICFosFd911F0JCQnDmzJkpnXOFhYUoKipCeXk5mpub8eqrr2LJkiUyOKO9vR07d+7E8uXLsWHDBtjtdsTExODMmTPiuBRFQWFhIfbt24fvf//72Lt3r1yXGq+32+0iSLV3714sW7YMExMTmDVrFlJSUqDT6bBixQoUFBTAbrejtrYWVqsVaWlpyMzMFDhJo9Fg/vz5mDdvnhRP3W43goODRWmSWjQ3btxAdXU1uru7UV5ejrKyMrz88svYtm0bysvLcezYMWRmZqKgoADXr1+XYnp3dzdWr16NBQsWYOHChdBoNLh69apIDB8/fhwRERGor6/nuZH7nTVrFrKzs8UxzJw5U6Kyvr4+PPXUU1OKyoCvrpCRkSETr8haogCb2+3GihUrJIpOT09HVlYWhoaGUFBQgIULF8LlcuHkyZNISkrC+fPnMXv2bKHskXtPrXKXyyWFVIqTkdVhMBjQ1taGJ554Aps3b8aZM2d4jnHXXXchNzcXs2fPFnE0Qjzt7e0SdVKLxeFwIDc3V7DvlStXYnR0FGNjY4iOjsa0adOkoE0efUNDA771rW/hrrvuwtmzZ+HxeNDa2orc3FxkZ2cjNjZW6KelpaUiznfhwgXodDrU19cjICAAPT09cDqdmDNnDsLCwpD+mchfZmamNGjV1NQI/HL27FksX74cH330ERoaGqYU871eL9ra2jBt2jSkpqaK2F1jYyPq6+vx9NNPo6ioCBUVFVOokKxbzZo1C1arFcHBwbh48SJSUlIwMjKC06dPY82aNeju7obRaBT9H46lrKurg5+fnwxN+bzX/4YtowHwNoBaRVH+S/WjowAe/+zrxwF8pPr+do1G46/RaKYByARw5Ys+g1Hc7YXO6upqeDwe2Gw2VFZW4vHHH5eNbLPZcPnyZQQEBODWrVsYGRlBdXW1FEUmJiZEorS8vBxJSUloa2uDx+NBdnY2hoaGUFxcjBs3bsBgMGBoaEgkUlnZj4mJgd1ux7Vr19De3o7a2looioK6ujoAPtjoxo0bWLx4MQBIgcVms0nKRGYIdSvy8vJgMBjwq1/9CpcuXZIClJo7bzKZEBcXJ9TDmpoazJo1Cy6XC+fOncPEhG/I74IFC9DU1ASz2YyGhgZUVFTIe9GwAJCGDGpOW61W3Lp1SzSn1XNDvV4vPv74Y6SkpKC5uRl6vR4mkwmjo6NoaGiQ1vLm5mYMDQ0JnMLag5qn3dbWJjK9c+bMEeaLmuNPuMZut2NsbAxjY2MCEXm9XiQlJaG/vx/d3d0yXYg1EjpHPz8/JCUliRBXU1OT8PpZ5CWDxW63y+Fob29HZ2en6OQ7nU7k5OSgvb0dGRkZCAwMhMViQW9vr6hBkmY3ODgoMrljY2Nobm5Gf38/Ojs7YbPZhLMOQBpl2KBD1s7o6Cja2tqwbt06pKenT+Hvcz+QSjg6Ogqz2Yy4uDjBq2/evIkVK1ZIDYHMrrq6OhQUFEjmEBISIjWptLQ0kRwgOyg8PByRkZHweHySshxDB0BqQaT98nymp6fD5XIhJiZGHADXn8O61R2i6mwmJiYGIyMjGBgYQHV1NVauXCm2IDExUXoMqN2i1/sUU8n4IRstNzd3CjOMhVMyh+icXC6XYOIs2gYGBgqV1Gw2Y3x8HAMDA1LQBXxQntVqRXx8vIyvZFbE7uc5c+ZMKZ7b7XaBisfHx5GRkSGZAwvcBoMB3d3dIr1NaDAtLU3qAvyZuoDPgJU4Pgdrf97rfxQO02g0ywFcAFAFgNy3n8CHux8CkAqgA8BDiqIMfvY3zwHYBR/T5vuKohz/os9YuHChcu7cOTkM3JTr16/Hq6++isHBQRnWQCaJ3W5Hbm4u2trakJqaCoPBgJs3b4oUKQA0Nzejvb0dOTk5Io+ak5MDALhx4wYKCwtx7tw5nDp1CoODg6iursby5cvF0bS0tEj6zcOVkJCAJ598Er/85S8xNDSERYsWCV2xqqpKhlKUlJTI5zU3N8NutyM0NBQJCQkwGAw4evQopk+fLhocwCRF0+PxoKysTGhqzc3NmD17Nmw2m7AYUlJS4Ofnh6qqKhk3FxkZOaURxOl0CkWM0278/f1hs9mQkpIinaFtbW3IzMwUI8KBIwsXLhSBK8qaer1eGSc4Pj4u4+YmJibQ3NyM7OxsuZ+amhppg+f1kqmh5ttfuXIFExO+QRdtbW1CT5yY8Il8lZeXT5mNywiLTSVkuHR1dSE8PFxGooWFhclh9vf3R2trK5xOp+ybuLg4hIeHo7e3F2azWQaINDU1YcaMGQgKCsLZs2eRmJgoHGXS11jMCw0NRX19vYyFGxkZQUJCwhQD7fV6UVdXJ89/dHQU3d3dIkRmNBqFRUMuN/dgf38/nE6nKDK6XC7BzwMCAgR/vn79OlJSUoSCl5SUBH9/f/ldi8WCjz/+GDt37hQGDzn7iuLr+hwaGkJUVBQ6OzuFPkpnxP3rdDphNBqFfUSuOADRk/d6vejv70dAQIA0EkVFRQnjY2BgQDB+Ki+OjY2ht7cXISEhmD59uvRnMJgjjTEmJgahoaFSo2IwMDo6ivj4eAQHB08Zt8ieBafTiczMTHR1dcHr9Uqtxmw2Q6PRyHNhljB79mz09/djdHQUIyMjiI+PF9kGm80mOjiBgYEYGhqC3W6XoRyhoaGyH9PT0wVPJwRIx0UpA5fLherqamRmZsJgMKCpqQkajUZqV5QscLvdKCkpwfj4OB5++GFMTEzgiSee+FzhsC+FKuTChQuVoqIi2TCkD86ZMwdVVVVwu90YGRkRutbtTUqMJsj5ZaONWmWQDsNsNqOurg6rV6/Grl27kJSUhJ/+9KdTuuVYhCOfm3gXi6NPPfUUysvLBcsFIC3i/BzSzwAI55mR2/j4OLq6upCSkiKUL4/HI3RGNu1oPmtuUad0agokI2FmDGzOIr1MLSOgjp7UDUXk5jI6o6YKZUkZGfPnVM2kQVCLLbEgTIohG3DUNMPbo3d19yZ1dlhnYEfxwMCAGEMaODXHmAaAz4H3RmkI4qbcW8SVb6dEsoGN1EgaNZ4RFjoVRREYTe1MuW6kxDkcDrkXYHK6kMPhEOVF9XtzD6qdPY0qI1P12gIQ2t2JEyewbds2YcewDlBYWIjU1FRER0ejq6sLCxYskKyD/Gyv1yvDxdWdk9Qh4vlLTk6WngtGkmoxNTprPmu1gBfv1e12Y3h4WGiUvHd2Fs+dO1cMMjtkqXSp7odQNwh6PB709/fD4/HI6MSWlhYkJibK/zOj5P5Wa9dERUVJVOz1ekWL3Ww2IygoCDabTYIJSv6SMslzSFonJUh4jQxAeBbtdju8Xq9IhjscDuj1eiFZ8LkTniarqbGxEWazGS0tLcjKysKSJUvgcrnw9NNPf7lVIXngmN7odDrU1NRg3bp1MiyBBk6NSd6O36oxezULhWmbx+NBc3MziouL4Xa78YMf/ABZWVn/7YDRkKsV/XQ6HZxOp1AzOVAXwJQ2dDV3nhAPx4xRBImMGBofRVEkwlEbdV4XNwYNKY2Gej1ooFnEZFTFa1M3FtG4c8CGw+EQI0/HQiPHz+ffMfXX6XTisIjJ3t7qTUPIZ6zu9qQxYPRFWiijehqP7u5u3Lx5U96bQw2Y4tL4qQXP/Pz8RBCN7elkLvA50tjzcKq7JRlBqznbNOy8X75I7VTz1RVFkYiXh50GkU6QcBQpsfx7OnreExkk/HvubzrS4eFhmfV569Yt0RXnZ1Fjxu12Y8mSJQAgTBn2cNBx8F4ZHPFz+Gxo7NXYOdeRv8s1YaABQGAE1gwINzECd7lc6OjoQFRUlGj48HPUglvsluY+4PPX6XSIjIyUZsfAwEDk5OTIfYaFhQHwdZtTJ/3mzZtwOp2YOXMmrFarQDU8J6wRkN3CzI1ibdHR0SL5zGKv2+2G0+kUp8W9wXoiMz6eee5z9R5hwxnhZe5HMmfuuOMOxMfHi5jcF72+FJH7ggULlDNnzsiB5GJw49BrsylJvTiMXNjlRmhDfV9qo8IX2+bVBhGAFHH4d4wUaTC50Ri182saMn4+nZTaKKizA34mDQ4NML/P++PP+Z58v9sbjrhRAEjkwGibn88Uj0bi9oiUkYe6IYlGlvfPteVm5Zozeqeh4nswleV68DmonRGzHHWzEvU+Wlpa5MDwHpOSksSZEq5hFM89wyxQ3T/A5+V0OuV6KNhFA65W5aNTAyARLNdAfYgZxXPP0tlRa53Pi79P58jITh2NMtKnmBu7i91uN6xWq2jQBAcHSzZrsVjkffhcUlNTRdaYzk5NveM+YHZCZ0mGkcvlkoCDwYXRaBQ2Fp0SgwIqkrL1Xp3t8nMURZHMWl2P4H1zfzBoUDtbrVYr926322VuK3X5qVBJ7XrCjxydxzNCR9fd3S0sGEb1wGRjJLtfY2JipEGKWDy19j0eD+rq6pCSkgK32y38d9oLisjxfAcEBCAiIkLOWHh4uEA6tAts+mK2zeAyMDBQakwUHNPr9fjWt771uZH7l0Jbhp5efcDVhVEeDv7s9iYPGjguxu3pLQ02NzAwafD5d2o4iO+nNvJqA63uTCXkQviBURY/V+1Q1NAA75vvr3ZG/D122TKD4D8aHd4XDSXxXRoNGlF+n+umLlzze2oBLB5eHkBgEtq6HRZjtsTmDH4mAKFuqT+b98174f+zmYXfGx4eRm9vr2xo/qMh535RHwQ6Gjoytegbr1ndGcqioRriUGeI6kyQ981nT3iCn0UeN7FwGngebt6nWn6Az5bOh3+vprLyetSGjwdb7cxYqFNrgtNJcI/R8Kn/0SEzewIgOjQAhGbIgIOfe/tzVmd96v4COgx1QMJASg17co+q4UxmB+rzy31IR0rnwjUi7EF4j9xxAKJjRCfNWodOpxPdekbu1JOhUBmzKWYtlCgh9KTWzlHvS55Dfs2OeT57Xqefn59kOtwPtCtq3Syuu1p25fNeXwrjzgfFaEodVQOTg7H5UFgwUzcBcUMRN2XkRS/OKI4PgOmvWjeED4ROxul0ymAD9SGmlKcau1ZDOTzANAg8gDzIxBtDQkKmRIQ8ZMCkoeEh5KGlwiFfNHSMDHmoCEWojRHxYW4WGnDCRmpjyHWgsaWTovNSO9LbMx9uaDWsxbXhhuRnAZBZmWqox2w2ixYKBcrCw8Mlde7p6cGhQ4fEQHPgg9rhqzF+XgejJq1WK3xyDrVQZ3fEUdW1De4jqpGqawlcR0bsGo1G2Ehs3fd6vbBYLBLREzYgo4QZA88D9wsZHCwQ+vn5RgMaDAZhg5BWGhsbi8jISNFDIgPJ7XZL447RaJTzFBISgtjYWPmZWiSL60eohwVY1nzUZ4Lnj9kaMDlDQVEUDA8Pi+HVan0SAOzRmJjwteYDkOI9z6haG169Z+x2u2QsgYGBwpsnjEMHx2zY6/WKDC/HK2o0vo5xo9GImTNnitHk3ggODsbExKTMMhU0NZpJhc6cnBwEBwfLrNnBwUHZB3QM/v7+CAkJESloRVGmCPpxzQnZhIWFyf1y73Z3d0tNgOv6/x6YBQIAACAASURBVAvjToxSDVvQaPb09KC3txcABNflQaVxpUEDJg0XDTwdg9rI8vfpAJqbm9HV1SUqg/wcblhuUOpcA1OnFHV0dGBwcHAK9KKO4JkdMIqhQ6EBUhtrwktqY6mmNVLDw+12o7W1dQrUAUx6dzVerIYGGMF6PL4ZkuQhq/E7Rhk02mrpY0YSNODqyJKpr7qIyuhSHQGrGTNer1d0QwBftNbY2AiTyQSv14vQ0FApgvf09Ex5tjSOfMaEVW5/7urrolNXF5bdbjdMJhNaW1vlcPO58W/V66LOCGlsg4ODRUqCk3cYJTLaZvqvLpZxH9MRM3ggVEGnTsx7aGgIbrdbmBk87DwvlITQaDTiDHg/3Cu8ZtZPeI3ErNVdqyx0q2tJvG71PqMBVT9bNTzHa2WwxCh7cHBQ/oZGj2eLRdOhoSE5m2rI0OVyiS4MnaL6LNEhBQQEwOl0SkFanbHHxsYKRZYRNNeG8BFpjizIMhgiFMYzS7tFYgQnOanPLZ+1OotVF/FZm+O549minfB4PAIPqVGBv/f6Uhj32zFeRgajo6P42te+hj/96U9yGL/5zW9KAUyv18ui8X3UqbM6quTPaJy0Wi2OHDmC5557DkajEceOHcPDDz88JdJUb16tVotvfOMbU9gqvKaHHnoIR44ckQOqrnqr4RwayO9///uSAqpTSjUOrjYqvJdXX30VNTU1ckC//vWvixFVsxV4vbfXH3joud7/+Z//iZKSEilKqddfzXqgY7i96KeGkggDqJ3v7caUz4iHixuf6bKi+JgepaWleOWVV0QBMC4uDpcvX5bCnlarRVxcHL761a9KKzcHqqiVOAEfHfbkyZOynsx+tFqfXOuFCxfwzjvvICIiAqGhofj1r389ZV/p9Xp5TlwDYNKJcu15L8TB9Xq9RJ0UORsfHxfq4u1MKzq64eFhVFZWCmWPES4DDHLAaWBpeCcmfEMoGLgQW6bj4dqQ78/Axev1SuZA1ozZbBYDxr+n8VVDUDS4ariQcAeNVkBAgEAdHHri9XrR1dWFlpYWvP/++1AU38CW0dFR1NTUwO12o7i4GFarFR6PB3v37hUSBJvSWFj3er1ITEzE2NiYzE+NiIiQoKG3txd79uxBdXU1KisrERUVBY/Hg7y8PAQGBsoUJtYVuH79/f3o6OiQZ8x6AxlR7CugIeYaqsdhhoSECIxDQ826B4vYfH6sGdHBsY5Ip8ih8jabDW+//bZMivqi15fCuAOT9ENGXB6PRwb3PvTQQ7J5XnvtNYFFqNp2+/vQwHAT00iqMWK9Xo/nn38e9957L+Li4rBz507s27dPIhB6SeLRTHt1Op1wVoFJbe577rlHDgswWYBjlE3DrigK3nrrLdhsNhm2wIeqZmEAk7UIFmOOHDkiWKiaGgdMzv3kzzhfktEjDwPhg8bGRmzduhU7d+6UVPb2CFWd6aijP94Prw+YVLcknEQK4d+Lkvk9r9cr8rFut0+TvaOjA52dPmmiY8eOyTSlpKQk0b7h+/GzSf9UQxHDw8NwuVz429/+JlQ4ZlXkxQcHB2Pfvn1YtGiRGLGf/OQnU+onVOrj/dIwezw+pUer1SoOk5h2UFAQBgcHhXVCo8oIm9fKfUh2Eo1mRUUFUlNTBbIgbn97HUSn06Gvr09E7ZjlGQwGMcgApFjLf+wXULPTWJjs7OycwioKDg6Wwd/cIxw9qMaLuee4Xxm5q58Fxxbq9XqcOHECc+fOxde//nWJwAMCArBoka82SPneiIgI2Gw2rFixYspwbc57ZfGTfQcBAQEIDQ0VaJfONC8vD7NmzRJ7QUepjrz7+vrE+EZGRiIzMxMejwcREREYGRkRlpr63qkuyQ5vr9crYyPNZrM0CLJOEhkZCQCSzRGeJVefTYWEgGivOKawsbFR3pO24PNeXwrjro4AGV1rtT66VGpqKhISEuDx+MR4iLFaLBYMDg7C6/XCZDIJK4BGaWxsTMSU1OkLF4zCSDqdDp2dnTIRnZGaxWIREX9GrykpKXKQR0dH0dvbi8HBQWRmZiIqKgqArwDV19c3haWiLugNDw9jYGBANkN7e7vocFssFgCThVZSJScmJtDa2ipNJexapCKiyWQCMCmNPDIygs7OTvT398s1qDONvr4+3LhxAzqdT2WOPNu+vj4ZW+bxeGAymWCz2WA2m9HV1QWXy4Wuri7YbDYMDw+jp6cHdrsdNpsNJpNJBvbyfnt6etDf3z9l3f38/ASvVGcGDodDjOP4+Djmz58vE3F4eOjAyP9mZEdn1N3dLV2FQ0NDsFgsaG9vF2YFjbWap8/op7u7G21tbbKOXq9XiomsCXi9XvT29k5hWPH5UImQtLjx8XHpzuVAF/4d/9FYqtPs0dFR9Pf3y/1ZrVbZ04qiyJxT9RAMBh2EEycmJiQAYYRvt9sl0uYzCg4OFu3ziYkJCTjoQMjZd7l8052YQaqhGHU3J1/MVKln393dLY6ezUq9vb2w2WwyuJvYONeLXawjIyOIjo6Gv7//FFiFgmx+fj7VTTpJ2gidTjdlqI7JZEJQUBDCw8OFccJrdjgcYvS53/i3Ho9H9gmhQ6fTKbAko3c6c8JDrHPRoaphYnVww+ZIOhgGiKxR0AmzK9lkMiEhIeG/Qal/7/WlMe6kbXGj+fn5oby8HDt37oTRaMR7772HlpYWvPfee2hoaMChQ4ewadMmvP766zAYDKLFoNFo8LOf/QxHjhzB2NgYfvazn0nxFYBE+4qiID8/XzTSf/GLXwjccuDAATQ3N6O+vh7PPPMM9Ho9Kisr8cQTTwgt6aWXXkJbWxt+//vf47HHHoNOp0NDQwOOHz8OjUaD3bt3S1s7o7bCwkJcuXIFmzdvxsTEhOjSPProo/B4PFi5ciX27dsn2DwNgL+/P9rb2/HEE0+I+lxnZyeSkpIQHByMP/zhD0JFq6+vx5EjRzA8PIwXX3xRIlYaRkbdRUVFUkTq7+/Hu+++C6fTicLCQjQ1NaGxsRGVlZW488470dfXh5/85Cf4zW9+g4qKCmzZsgVjY2M4fvw47rvvPtmcDzzwgDRjPPvsswCAsrIyDAwMyLOlsSGjh86mp6dHDlVcXBzWr1+P+Ph4KIqC2tpazJs3Twpb6kJqUVERvF4vqqurMTAwgJ/97GdwOp146623YDAYkJiYiHnz5snhUhfMtFotZs+ejeTkZBGiYiTa0tKCsrIyDA0N4Xvf+x7sdjsuX76Mf/qnf0JERASCgoLwt7/9DTExMSgtLcXhw4fxxhtvwGg0oqysDOPj47h48SI0Gg0KCwv/m6xzcHAwCgsLcejQIWi1Wpw9exZ6vR6XLl0SeqfdbkdNTY1g4UePHoXH49MtKioqkoagN998E9XV1XC73di7dy/MZjN++9vf4sSJEzh//jwOHz6MgIAAkbZVFEUMVXBwMEZHR1FSUgJFUQRmICb+/vvvY3BwUGQ8SMG8cuWK0FXfeOMN4eMTDtHpdKiursbx48dhNBpx9OhRlJSUwOv1oqOjQ2o9NIjkzAcEBODixYtSqKT8cFdXF/76178KT350dBQffPABenp60NfXh9deew0RERGy/1iYHR8fx1133YWYmBgEBQWhq6sL3d3dsFgsOHz4MPR6n7RBUVERTCYTNBoNLly4gI6ODvzlL39Bf3+//C4nJL344osSAHV1deH8+fNwOByorKwUEgAj+/b2dpjNZvzqV78SpzYx4WseLC0tlSCAk6JMJhPKy8sRGBiIEydOoKKiApWVlRgaGoK/vz8qKiqwatUqyUy+6PWlMO7AJHeaHt7r9eLatWsiF5CRkSFCP319fZg3bx4mJiYwd+5cYQcQ07p58ybmzZuHwcHBKSk1AOnM0+t9wj2JiYkAIJNfLly4gN/+9rdISUmBxWLBzJkz4fV6UVZWJmJPNTU1yM3NxdKlSxEdHY2srCzodDrRBHe73di0aZOkfYAvDRsdHcXixYuRkZEBm80Gg8EAi8WCuLg4pH+m1cFiKzAZwSuKIjII6mG+X/3qVxEZGSlRqcfjweuvv4758+fDz89P2u+BScVNYsTUrwkODsb+/fuxfv16pKenY+nSpTh79qx0MXq9XuTl5eGZZ54RffSJiQmkpqZK6j5jxgxERUUJxKIoCs6fPy+dfmQBMGIht53XxAxrZGQENptNKGpz5sxBb28vdDqddBHabDbU19dLZhcTEwOHw4FZs2ZJxpaUlITt27cDADIzM5GcnCx7S1EU2QOBgYFIS0tDcnIygoKCJIsZHx/HyZMnsXjxYhiNRqHkRUdHy/McGxtDfHw8NBrfCDyHw4GkpCRERkZi0aJFKCoqQkZGBgCImBodEmG/kpISgXWSk5Ph8XjQ09ODvLw8hIWFSWOOVqsVrfrg4GDEx8fj0qVLcDqdwqyg/klfXx/i4uKwaNEiJCQkoLS0VKJuykczYmU/xI0bN0R8Kz09He3t7QgODkZmZibCwsIQGxuLwMBAwfFNJhNOnTolezo5ORnAJOzJCPTjjz8W+V8/Pz9UVFRIP0RwcDBiY2MFaiLkQj53TEwM3G43Ojs7sXjxYhEFozxGc3MzIiIiZA8lJCQIE41wYUBAAMbGxgT+6OvrQ2FhIUJDQwW7t9lswmWn8N7q1asRFRUFl8uF5ORk0ahiPUdtT8rKymAymYSZQ5E5Zq/j4+OIi4tDdHS0zJ8gy+jy5csSdKSmpiIoKAhFRUVITU0VBxAbG4tz584hNjYW4eHhiIqKEnkLdbb0915fGuPOBg81hLJnzx7Ex8fDZrNh1qxZePHFF7F69WpRoaMEbF1dHb773e8iLCwMe/fuxXe/+10kJiYiPT0du3fvnsKr5UG7ceOGKD7+9a9/xdatWzE2NoYf/vCHopz3wAMP4IUXXoDZbMbevXuRmpoKm82G73//+9iyZQsCAwNx9OhRSfPvuusulJWV4amnnoLb7ZZhw4ria4Hfvn07Tpw4gZ/+9KcICwvDjh070N3djWeffRYulwvp6em455575HpJxVMUBW+88YYIJlksFhw4cABr1qxBY2MjTp06JVzcjz/+WPRrdu/eLRtN3SVXVVWFwcFBKbj9+te/loNfWlqKjIwMrFq1CgcOHMC2bdswMTGBBQsWYOvWrfjzn/+MrVu3QlEUHDx4EI8/7tOOq66uxve+9z2Jql566SV8+9vfxne+8x1p2KEDByahG+q1UHxqYGAA4eHhsFqtWLFiBfbt24eMjAyhpZ04cQIPPPAA/Px8OuMZGRnCfz958iRWrlwJf39/JCcno6ioCBs2bJjSGxESEiIFxvr6eixevBhWqxVFRUUAfBS748eP46OPPkJISAiuXLkCg8GApKQkZGZmYsaMGdDpdCgvL0d+fj7Gx8cxb9489PX14d577xVphDNnzggP+sEHHxRYyul0wmw2Y3R0FNu3b0d2djZefvlllJaWSvr9wAMPwOPx4Ny5c1i7di1cLhdef/11kY7t6ekRTD46OhqJiYnIyMhAcXGxTIFKSEhAWloadu3ahenTp+OFF14Q50eao9vtxqVLl3DmzBmkpaWJ/g4dYW9vL+644w5MTEyI+uXo6Cj+9Kc/SRFz9uzZ2LRpk9RbgMnpSOHh4SgoKBD9l2nTpolwW3Z2NkZGRoStQ+y7vLwcZrNZ4MBz586JBn51dbXUjd58801kZ2fD7XZj5cqV2LZtG/z8fANJ/Px8E6BSUlJEi31sbAz79u1DWFiYqIY2NDTIfTKoYBdpf3+/nBFme4SOMjIyBEZ58MEHMX/+fLzzzjswGo0YHx9HWFiYQGSzZs3CwMAA1q5dC71ej9LSUsTFxcFut2PDhg04cuQIXnrpJWFTnT9/XqiumzdvFllo2oLly5fL6ER1Uf7vvb4Uxp2FNhaCAIhA0nvvvYf6+nr8+Mc/hlarxYEDB+Dv74/i4mI8/PDD0Gg0eOGFF7Bp0ya8++67yMnJEfU2ADh//vwU7jAr0B9++CHSP1Nro5zwvn37kJubK9FmU1MTjh07hg8++AA2mw379+9HaGioRAI9PT2wWCx455130Nraiu985zt44YUX8Pzzzwv9CpiUVyguLsZ//Md/yGQZu92OTZs2IT4+HqdOncK//uu/ygAIsoUYYfb29kpqevnyZbS1tcHr9eL5559HXl4e3nnnHZmWRFre6dOn5bM1Go3IDBw7dgxz584VzDguLk5wwytXrmDNmjWYmJhAZ2cnHnroIXEwrHts27ZNsPwHHngAw8PDeOGFF1BQUIDR0VH85S9/weLFi/GTn/wEiYmJwq//4x//CGCSgz86Oor6+nopjg0PD+Py5cvSMs7vZWdnS+E4OTkZiqKIBIHFYkFLS4vIoS5ZsgTV1dW4ePEijh8/jqCgIFRXV0On003RJVcUBWfOnEFsbCy0Wp/M8dy5c1FSUoLQ0FBERUVhbGwMR44cwerVq4XGFxMTg46ODtERJ4snPz8fSUlJ0oMRExODtLQ0xMfHo76+HkNDQ8L/Zo3DarVi9erVyM/Pl6Bj0aJFMBqNqKqqwtmzZ2GxWGTWq9frRWdnJwoLC3HPPfdIFkJY49q1a9LklZ2dLfj2+vXrkZ+fL4XG4OBgmM1mtLW1we12IywsDMPDw7h58yaqqqqkIF5cXCy67+Xl5bBYLGhqahItcaPRCKvVioaGBjidTtnv6oKq1WpFb28vFi5ciEcffRQxMTFob29HXl6ekB3UNL9z584hOzsbFotFlFSJt2dkZKC8vBxDQ0Mi7EbN9sbGRtkvNptNOOf19fXo6+sTKKqrqwu1tbUoLi7G/PnzERERgcuXL4uMMsXRysrKkJeXJ70jy5Ytg1arRUtLC5YvX47a2lqhaS9btgzz5s1DcHCwZJ0MpCYmJtDU1ASj0QibzYbR0VFcvnxZhmw/+uijghp4PL75vvzMrq4uWCwWUew8f/485s+fj4sXL8rw8i+0q/+fWOf/44scdGAql3jWrFnweDxIS0uDy+VCZmamjOzq6uqSr7Va30DptLQ0LFy4EFevXkVNTQ2uXr0qei6MGBkVt7W1ISwsDDqdDgsXLkRdXR2WLFmCHTt2wOv1ora2VrQ6UlNTkZWVJRX/r3zlK7h58yauXbuGWbNmISgoCImJiVi7di0qKipQVlaG5cuXS8MNXw6HA7GxsWhpaZGJTBxuwFoDR6Kp2TgAMHPmTNTU1GDevHno6urC3LlzZd2Sk5ORlJSEuLg4bNy4EXV1dWhqahLjTayXLeednZ3Izs4W5samTZtQX1+PK1eu4KmnnhKYZ+7cuVPU/DweD+bPny90siVLlohSZ0hICGpra6UhhHKu27dvx/j4OJxOJy5fvixsFzaJ0TgFBASgsbERnZ2dUxqPli5dKs5Jp9MhKysLnZ2dGBwcRFdXFyIjI9HX14eOjg5kZWWhr68PdrsdixYtQnZ2Ntrb2wUuYsGcmGh/f79w7HNycuDxeJCTk4P58+ejoKAALS0tsNvtmDlzpuDk3DuRkZEwmUwi00o1UDKrCgoKUFtbK3LBVD+k4/b394fRaER3dzdCQ0OxevVqWCwWpKenQ1F8Q7WNRiP6+/uh0+mwcuVKNDc3o7OzEykpKcJ7J5RhMpnESJjNZslQoqKi0NXVJTUGPlvABy1NmzYNixYtwvDwMLxer0zNIuWPsAGVDlNSUrB8+XIEBATAarViYGBgynsDECZHaGgoOjs70dXVhbVr18pe5PNm5E74wuPxYMaMGcJh7+npQWJiojjz0NBQREREICwsDMuWLRNtdqvVKoM1AAiEwqYsGsHp06fDbrejqqoKsbGxyM/Pl+xx+vTpQsdlBD9jxgxxUJmZmVIUd7lcohdvNBrR2toq2Tv3ABlmWq0WWVlZsNlscDqdSEhIEK2amJgYmEwmxMbGSmS+ZMkS1NbWYmhoCA6HQ/oZ2NPS19eHkJCQKdz6z3t9abRlOOiCQwKIDZMiRgobC400gOqKMTcYq/o0bmSdaDS+4QpGoxHPPvssDh06BGCyUYielu/LIg8A2ZQ8HKyEs3GFxocPibQ7fs0oS439chPwIQUFBUkBlp6f18WJPSxC0amphzQTg2NNQW0kSUuzWCx47rnn8Ktf/Up0NYBJ0ScKKKnvmdASACmEktJGlhKzLwDyfMi0ACadNgBxYlarVcbaabVaST9Zq1DTEdU4PfVf+F7cG2yAIU+d2CuNK9e2pqYGSUlJ+N3vfodf/vKXwmigOBPnd7pcLhw4cACPPvqorCnvj9kHG8xYBKVh1uv1U9QEST/s7e0VRUu9Xi/fp+Ilu2YdDscUfjjZMi0tLdDpfNo5UVFRogBJyJHrxMyGe5pzSf39/TE2Ngaz2SxNYNHR0RgdHcXg4KCM5fPz80Nqaqo4bBY3qcfCvcVC6vj4uBg47inWbLg2lKtubW3F8uXLYTabpWdBrR/jdrthNpsF3uBzIbWZCqxcb9Zi2EjocDgwNjaGtLQ0nDt3DosWLRJYkFCGw+FATEwM/Pz8BHdXa9zwOZPCyGyGn8UOYvLk2TzHYIWjFQMDA0Wjh8Vh9puQN28wGGAymaTTlgw9SlzweZOdQ9YRgC+U/P1SRO7qJhw+SGJd/JpGnMaT0T152OyeBHyQjnpYg5q//corr+DkyZPYuHEjgKlt6Tz8LGwCkwJfZDuQQ0/IhJ/Lz+ImYiROrrK6o4/RmxoL5vcpgUDjfXs08Pc8Nt+DbfrqLkE6LtLoioqKsH79ehiNRll7bnquGaMTrg3fS+3EaDRpsNnApZYwYIMG740GnkU5wiOUlFB3MtIoMepXUwfVn817ZBBA48J0nw6XmZDH48GHH36Ic+fOYdmyZVIkZCDg9Xrx4YcfyhzWO+64Q96DDkctV3t7tybTfz8/P1ljh8MhzoBTo8LDw6cwHnhwXS6XMD7UHb7qTk021QAQCI5OmEwXaphz/9LBABAjSGiQ+1oNZ/IZELYiL5s/43PiWSF7BIBooJPa63Q6odVq0fbZ8BZq/nOdaKjIiVfbAXVfCvcHnbX6DPH58m/PnDmDzs5O5OXlyf2HhoYKFTI6Olr45+pGQlJ0aRu4Z9Q9M+rrodMn95wZGrNkNQ06JCREAjKKkamDPIraKYoi+0utN0ObODIyMqXT/nPt6pclcj937twUY0FvSeiARocvVqx1Op1U/bk49HY0TOpOTafTCbvdjsTERDkITJPJtwZ8cAhTRXpaemB19sC/pSfnS611QwyS2QQ3rtqIkf/NCJIPTr1xeT98Zoyc1REl75P3rdFo5PBptVqYzeYpuCDf3+VyyWHntbNww5ea8UL2AjMCYFIyQt3BC2CKge/s7ERnZ6e0cxPzVTf6cG0JafE5hISECB9Z3TDDtXK5JmV/2XBEo63uFGZDSmJiolwb6YZkMjidTmHHcJ+o71dRlCkNSOo14l6yWq0S5fKZ03EywuQ98hnycNNI0TGZzWZ5H64D1zQ8PFwMBg89IUH+Dp2P0+lEW1ubNMDExMRIhzCLiiMjIwgKCkJlZaVAPYQY+DX3vNlsFnYL9zvH5xGus1gs0l1psVgkC2VxVK2tHhsbC2BStZUZA88I+xwYTDCgcjgciIiIQGNjozhCOicaUj7HiIgIDAwMID4+XpwzHRA1YrhfOKKRNoYUVe5t3jOL9PweHbpO5xsTaLVa5VnTHtG+8Z/b7UZERIQ0ADIb4hkBIE163H+7du368kfujH7YxcZUX81RV3sqYqBMU8iTV0fyPGT0rnq9HqGhoTLVBoBAJcPDw5J2csPRwfB9AMjXjCjosRnpqw2NOpvgf2kseM28DzXGzkifm47RpToiV0fSxDJ5feromu9JwxweHi4/J/xEbjKvm9EiG0D4e/wMOjlgMlKi8VIrZLKITQdisVjQ3NwMh8OBkJAQGI1GhIWFISkpSdLx0dFRhIeHT9EI4RqoYTJGpTS6/Gy1FgvvSd0zoNPpEBwcLPrcNPiER3Q6HcLDw5GUlCRRF5tP+GzJhacjoCPVarUC8TAjURQFERERkoEyeOCwcLXxZaMMU3nuiZGREYFoXC6XNEUxmlQ3ZwUFBUnLPz+T0abNZoPNZhP8mu9lMplQV1cHnU4nFEOPx4NPPvlERk8yS+I6A5AisTqi1uv1UixkhkshNU4Ko5YLW+yNRiPi4+MRHR2N/v5+dHV1ScbB9aWj4/uSykjjSaYKYZGEhAQkJSXh/fffl7OsVnV0Op3o7e2Fw+GQ/aIODmlQ7Xa7XIsagmNAwEyMcCCNtFrKm818AwMDElioBcQ4nnBwcBBWq1Uyfzo8RVGkCxnw1Y5ot77o9aUw7jxAjFhooGmkaVy4oPwZb04NdVCrmxGZmk7IqJIGlsaHHpfaKswS1F8Dk7K3fIBM92j01FIHvC7eA6NtQjx0GsAkrMLfYdSg1oJhizeLaDSY/Ll6Pfg9XjujDH4WISZg0hnx0Py9iVIApLjGiIKQCfFiHmxemzorojKhyWSS6Gd8fFwq/pw9GRwcLOkon5vNZhOFPkaxbDWnwaEDp5EjnsmoXa09dPsa8fPUNRP1M1RrqHC91XuUho7QBZtXWFTkXlPvIf6XGQ/3uLr4zuyBn8n6DtUPgckZmoxm+Sx5jQyYiAkzmuUeIrbOgIdrS5x8+vTpcDgcAqdxf3KvMUqPiIgQw8aOWzpZvV4vBUqXyyXt9wy+uC4DAwNiUJl9UQqA+09dh+L98n6YCcXFxcn7ajQa6VVQkxsYlGm1WqHgMpCjwwMgHaoMMml0uR4M0G4/B2pbwmwrNDRUuoEJV6qDIAACEWm1WllTddDAgI8yBWopir/3+lJMYgIgBo9FDKb11PlmZMC0nBuL3u92nJWbgjAHH5LNZpNIjdGnuqDKxebGodMhHMEUkcUjPjx1AZdwCLFgbhxmFxS/4s94sHmYWcjki99jRMj3t9vtUthksZVrwohcDRkw0lU7Id4z783pdArOyu+rIQB1tMAon4aN6T35wDwIXq9PRmWSEgAAIABJREFUkKqjowMAptQsGGlyziSfIdeZa8sIlc+BjUMsIqqjZ3X7OZuH3G63NCRpNJMKnYyA1M6Oe0d98DhajVLO3Jfj4+NS0GQwwb+j41EbAf6O3W5HUlISFMXXA6EegkFsXq/Xy7xTrqPNZpM6TkREhET56sHJlDBmNsSsicVCZnVkwUREREjkSHjB5XLhkUcekX3mdruFnz0wMCAGkgMp2KJPCimFsiwWi+xrFhPpLCnrMDw8jISEBKE8ApMCbIQd2z6b88tmNj5jsp7YU0IjzFd6erpowNTX16OhoQE7d+6cYkgZMPJvCTkyMqc9IHTJZ8yf8exSo4kOnUV9nU435brp7CmMp3ZQPNvU2lHvHwBCi3S73XKePu/1pYjc6YkZndLrarVaVFdXY//+/bh27RrOnDmD9957D9euXUNxcTEaGhpgtVrxxz/+Uf6ebAAukNpoM9WhCA9fjLDVER69OTApAqbVavHxxx/j0KFDUwqm9ORqOIWGnJAJI1l2zamLjOqCDTcWH6Bac0INLwGQgtPEhG849f79++V31feljupu17CgEaIz4Ibi+hNbpYqlupgETGYzzA6YXQGThSg2KVGTndg6B0Q0NjbKz2lc1XojzLBuz5J4D8zKqEXCA0joxmw2o6ysTLIhOlQAclCZPVLfRh1VMYICIHzuW7du4fr165LyqxtN1MWu22sufAY0gBy7x2ujEVfr6IyNjQn8wazAZrOhpqZGjDIdK+UD+vv7pahNKIABh5rFNTw8LPo/AASuIVWRbe/h4eFyJnhGHQ6HNPow2h4eHhbMnk4/NjZWIlm9Xi8dluHh4VAUBYmJiRL86PV6hIeHizIijRvhTzVUA0CgPK4vzxUHpPv5+cZSqqdVDQwMyB4kdHd79smsOSAgYEqwRkhV3QTI6yGczOekntZ1u7ibuoYwMTEh+5AZN8/l4OAgqqqqJGMm1DYyMiJ6Vp/3+lIYd2ByeIN6Af39/XHo0CFs3boVGzduRExMDFJTU7Fy5Ups3LgRQ0NDCAkJQXFxsUSjLB4SC6PRofcldMMDqGa6cHMwZeQ1UQnu6NGjmDt3rgxhoGFlVAZMzpzkZ9PB0DCziYSGlUMcuGnJhqDD4yZT1xv4tyyKMcV+7733JAoljs5oUY3Ds6bAA0QMUO04uC40rupiLj+fEAY3Ig+c1+vTDykuLoai+Oh7/f39ssHJFomPj0d4eDhaW1slO+MBZLpOOQcWt8hV7+vrg8VimUJ3jYqKEliE2LpaJZDRtsfjkaIVBaKY4dTX1wOYFIfyeDwyCo0G/7/+67/Q2dkpzUqs17DzkwdWbRC4VjTWBoMBJ06cwNtvvy1ZIot1hHQATKn/AJMDQ9555x1Mnz59ioMgC8Zms6GpqUm03VnUJeOHBU3+PmEbarvs2bMHXV1dYqiYEVmtVoEEdDodYmNjpcmJAYLH45FGIJvNBo1Gg4iICBw+fFhYUhUVFejr64NWqxWZ4s7OThliwoI99xrgG8bCAjkDM3LzObpOXZD3eDwYHh4WcbmAgACkpKRg48aNqK2txcSEr9N7ZGREajB08upzwLmtXCNG5OwoZT9CaGiodP/SKHPAtp+fH2JiYmRv6vV6RERESJBHFhSfXU1NjawlzwX1jEZGRmTAuJpg8vdeXwrjTq/Hoga98MjICL71rW8hLS0NVqsVb731FtavXy+RMTvZHnroISnQMLrk4VJH04ySyRNl5KvRaESNTo2zcxMTdz106BDS0tJw//33y4MhFkjHwQIMoyNyVZmGqTFEVuoZSTK1ZgRG2INpHgCBHajeSIO4b98+GdjNar3dbhfjQCM2MDAghk0NqdChqPHZkZER0cqgQST2R/61x+ORDTc6Ogqn0wmHw4EjR44gJycHAwMD0Gg0wo5gNhAQECCMl9WrVyMwMFAUBuls+KyYxTCCprGJjIwUXFbd+EUjyqipsLAQM2fOlBSfaTEjXmqdAMDs2bOn1GqYzRD+cDgc0kSWk5MzBdLjf1kwJDWOzyQoKEjYWiaTCaWlpVi4cKFAFmNjY5LaOxwOkSkgTENmi9VqRVJSkkz9YVaTmpqKmJgYREZGYtmyZQLXsMmG+06v92nRGI1GkVc2GAyS+YSEhGDGjBkStZL5EhUVJcwcrVYrWjw9PT2yjlwnPz8/REZGwmAwoKWlBemfaUL5+/sjKysLS5YskT3E5h4aUWLdAKQASqeoZtRRt0Y9SNtsNiMiImJK9h8ZGSkGWK/XY9myZdDpdIiOjkZoaCiGh4fR2dk5pUZG407YhgEhP5+wDo01bQZh0dLSUqSlpUm2zqh7dHRUsv3g4GBxUMwST548ibi4OHHY165dQ2pqqnTsBgQESN3i/4y5azSaFAD7AMQD8AJ4U1GU32g0mt0Avg5g4LNf/YmiKJ9+9jf/AuBJABMAvqsoyon/6XPUXHQurE6nQ3p6OsbHx2Gz2VBZWSkHlg+qoqICYWFhOHToEBRFwTe+8Q1hh1y9elU6Fh977DFotVpcvXoVbW1t0Ol0uO++++DxeNDe3o7r169jZGQEO3bsEL4xo2W9Xo/r16+jpqYGn3zyCRYuXIiYmBg0NTWhtrYWeXl5mDZtGoaGhnDs2DEsXLgQnZ2dcDqdWLlyJT799FPccccd6Ovrg8lkwp133omKigqYTCbce++9iIyMhMvlQkNDA3p6eqAoCu65554phk0NrZw9exZDQ0PIzs4W41BeXo45c+ZgbGwMDQ0NaG1thb+/PzZu3ChSCJ2dnbh27Rry8/NFaIkZwmfPDRMTE7h27RpMJhNSU1ORm5sLrVaL69evo7e3FwkJCcjLy4NGo8HHH38sQwTa29sRExODpUuX4tKlSzh79ixCQ0MRGxsrVMeWlhZJk6dPnw5/f3+cOXNGBNFqampgs9mQn5+P69evAwBWr14tUa/aabJ57OrVqzAYDMjKykJMTIxkLmporaWlBdnZ2TCbzZg9ezYiIyPR2tqKnp4eBAUFwW63Y86cOQgICMDp06dFm9/tdqOhoQF9fX1ISkrC9OnTcePGDbhcLty4cQN33XWXGH610+vv70dzczMWLlwoxre1tRV9fX3CDOG9ms1m2WuEA5jhaTQa9Pb2oq6uDunp6YiMjERPTw9qa2sxNjaGuro6WX91Q9mNGzeQmZkJPz8/tLe3w2QyYdmyZbh8+TJ0Oh1WrFghbJre3l50d3cjJycHPT090oB18+ZNzJo1C7W1tUhPT0dVVRWio6ORmpqKsrIyJCYmIjExEX19fbIvuc84UGbFihWw2Wy4ePGiGM3o6Gi0tLRg2rRpiI+PR3BwMKxWq0TRer0e0dHRgsfn5eWhsrISDocDycnJyMrKwvj4uOzv1NRUqXk4nU5cvHgRmzdvlppVf3+/ZBN0zAsWLEBAQIDIXaxbtw5msxlmsxkJCQnwen3a7t3d3YiJiUFsbCzq6uoQGxuLgIAAVFZWIjc3F2FhYZiYmBD2EuCDSi9evChdsMuXL4fXOznIe2xsDLm5uVOeN+ALaLq7u6esc3h4uDR+Xb16FTk5OVJjc7vdaG9v/0Kb+r+J3D0A/klRlGwA+QCe0Wg0OZ/97NeKosz77B8New6A7QBmA9gI4DWNRvOFbHtGPAAkLVJDJW63GwcOHMDw8LBgeTqdDoODg9izZw+2bNmC/Px8/PznP5do+l/+5V8QHx+PO++8E1euXBFscs+ePVi3bp1MQSkpKcGlS5ewdu1ahIWFif6HOopVFEUO6le+8hWkpqaKrs3XvvY1PPjggzh8+DA++OADnD9/Hr/5zW+wbds2LFq0CB9++CEuXLiAF198ERs2bMCWLVvw0ksvYcOGDVi5ciX+8pe/SHSwb98+rF27FjabTbBXpojj4+Nob2/HV77yFdx///3YuXMnnnnmGcHnOjo6sHPnTrhcLuzduxdr1qyR6N7r9WLHjh3w9/fHQw89JNfLSJgRqqIo+MEPfoBly5Zh27ZteO+99zA+Po7HH38cc+bMwd13340TJ07g/vvvx4ULF5CamornnnsOS5cuxfbt2/GP//iP0Ol8OhxDQ0MoKChAUFAQBgYG8PbbbyM3Nxf9/f343e9+J3IDeXl5uHXrFqxWK7KyslBTU4OJiQnMnj0be/bsEfog4SnWMoqKivDKK69g1apVyMrKwtGjR6W+oXZWVVVVqKurw/z589HY2Ihnn30WN2/elHXKyclBbm4uWltb0djYiPfff19gpp/+9KfIy8vDqlWrcPLkSdjtdixbtgzLli3D3XffPUWvm1nNL37xC3zyySeYPXs2uru7MTg4iJ///OeIi4tDTk4OLl68CL1ejzvuuAN33HEHli9fLvADoYKenh709PRgz549MBqNmDt3LioqKkRBNCYmBg899BAeeOABBAYGoqmpCYODgwgICEBbWxsSEhJkQEl0dDS6uroQEhKCzMxMHDp0CKOjo+jp6cErr7yC/Px8LF26FL29vcjOzkZYWJjs3dbWVrS0tODo0aNITEzE6OgoqqqqMDw8jKKiIng8Hhw4cAAajUYYTxcuXMCCBQsQHByMgQFf3Ge1WrFy5UrMmjULBoMBJSUlaG5uBuCby0BJ5zVr1uDIkSOC2R88eBC1tbXIyclBR0cH/vCHP8Bms+Hll1/GjBkz0Nvbi6GhIYHNEhIS8Mgjj8BgMODMmTN46623sGDBAhw6dAi3bt2Cx+NBZmYmOjs7MTAwAIfDgbNnz8JgMCAkJAQmkwlarRa//vWvER8fj+zsbFy9ehV2ux3Tp0/HJ598Aq/Xi7S0NJSUlEg9gvNRAV8GFB8fj/HxcSxfvlwCIz8/P8ydOxdJSUkoKSmRTJx71t/fHxkZGRgfH8eWLVuQmJiI5uZm1NbWIiwsDENDQ/jtb38rzuS1115DUlLS/824K4piUhTl2mdf2wHUAviid70XwHuKoowritIKoAnAkv/hM8SYElOjUSWWd+PGDaSkpAgGpSgKBgcH0dDQgLCwMLS1tQm8UVlZif3796OkpATvv/8+Fi9eLAXBU6dO4Yc//CFWrlwJr9eLQ4cOYWRkBEeOHEFzc7Pg6WSWkEvLVFaj8anyzZ8/X/RmHA4HTCYTNm7ciObmZtx9992YmJhAbm4uNm/ejKamJskSuru7sXXrVmi1PtnemTNnilzxqVOn8Oyzz+LOO++cwkxhevn++++jtbVVinFZWVliFJKSkhAVFQWNRoPTp0/jueeew5o1awAAnZ2dyM3NRUpKCgBflECteTJqiMneunVLjNuPfvQjhIWFobq6Wpwmh0lkZWWhrKxMZICdTidyc3PFWcTFxcHl8ikDVlZWoqOjQ2Ch9PR06PV6xMTEoKqqStQux8bG0NraKpor7Clg4YusBoPBgCNHjmDatGnSbLZkyRIZwKzunKyoqEBsbKxcu9PpRFxcHDIyMpCamgpF8U3kycjIwMyZMxEXFyd7sLu7G4WFhSgtLcXdd98tHZQzZ84Udg4jMDVEUl5ejitXriAjIwOK4htacvXqVVRXV4sWt8lkwowZM6QGwboGC8mVlZXo7OzExMQEIiMjBabU6XRobGwU5oXBYEBUVBSio6OF1tfc3CwNWoAveqV8MPnXGo2vKe/w4cO4fPmy6AU1NjZK63x6ejq6urpQUFAgCpzTpk1DZ2cnMjMzYTAYEB0dLc+8qqpKipiLFy9Gb2+vsDoIuaampiIiIgLR0dFoa2vDtWvXMHv2bGFHKYpP2yUyMhJJSUlITk6WqDwrK0sojfv27UNWVhYSExOF18/6GACcOXNG9GBGR0eRk5ODnJwcUacMCgpCSkrKlKHxVHvs7+9HaWkpampqsGrVKmkkosCby+XTuSKriww+BmK1tbWIioqCn58fBgcH8emnnyIyMlKKzf39/VKL497mGkZFRUmwVVVVhaSkJFitVoyMjMj8hZ6eHvj7++PmzZtfaLv/X2HuGo0mHcB8AGWffesfNBrNTY1Gs1ej0UR89r0kAJ2qP+vCFzsDAJN6KOTgcjOw8l1UVISnn35aOJ9erxdHjhxBbGwsNBoN3n77bdx///24fPkyioqKMDExgYcffhjbt2/H/fffj9HRUXz88cdobW3Fv//7v2Pz5s0YGhpCYWEh/uEf/gFPPvkkfvCDHwiuR5oTcfOBgQFs3rwZiqLg3Xffxa5du6R7bvbs2di1axeysrIQHx+PrVu3Stt8WloaYmNjsWnTJsHGCwoKMDQ0hD179mDJkiVoa2vDyZMncfPmTfz4xz/Ghg0bxFCwaOLv7489e/Zg06ZN8Hp9wmY7duzAwMAArl27hieffBJOpxMnT57ErVu3sHv3bvndgwcPCiylKAqys7OxY8cO4coSvujt7cXatWtFmyMqKgrd3d3YvHmzrMvJkyfxz//8z0hJScGxY8ewY8cOaLVanD59Gj/84Q9hsVhQU1ODDRs2oLu7Gz09Pfjb3/6GefPmITQ0FBUVFSgoKEB/fz9iYmJw5MgRWeuysjLRvT5x4gTWrl0Ls9ksuKzX65UoiTKqTqcTn376KTIyMuSQDw0NoaOjA/7+/igsLMT06dOhKApKSkrw5JNPStp/3333ISIiAuHh4YiMjERHRwe++c1vwuFwwGazYc2aNVi1ahVWrVqFxMREREZGoqysDPPnzxdcnE6YMy137NiBN954A01NTbBYLOjt7UVBQQGWLVuGvLw8aZ46fvw48vLyRI+e9QY6jMLCQsybNw8OhwO3bt1CRUWFFMAHBgaEAkqIisPDQ0NDcerUKRH2amhoQExMDFwuFz766CPMnj0bfX196Ovrw65du7BlyxY0NDSgt7cXxcXFKCkpQVtbG6xWK6KiomA0GjFr1qwpGDbVNxVFwaJFi4StcurUKSxdulRqMSzaJicnC3R1+PBhrF+/HgkJCTAajSguLhYaZENDA+6++26kpqYCANasWYPExETYbDYMDAzgscceQ21tLR555BGR4+bEKRbWPR4PIiMjYTabsWDBAvj5+cm5bGtrw6lTp6DT6WSwR2pqqmguBQcHw2KxYOnSpVi2bBkKCgpE+fLSpUvIzMyEv78/zp49K4JxasdMu1RSUoJVq1ZhYGAAn376qUyKc7t9s2HXrFkjDlav18vYPpvNhtWrVwt76dKlSygoKBDYc8uWLfB6fXMuHnnkEaxcufILber/2rhrNJoQAEcAfF9RlGEArwPIADAPgAnAr/irf+fP/5vGgUaj+YZGoynXaDTlZrPZdzHayfmSDodDoqeysjIEBgYiPj5eOvx0Oh0+/PBDPPnkkxgdHUVdXR02bdqEiIgI7Ny5E+vWrUNtbS1qa2tx8OBB2O12lJWVobGxEaWlpXjzzTcRFRWFF198ESUlJSgvL8ef//xnwesZqZKad/PmTWzatAn+/v74+te/jpMnT6KmpgYvvviiYM8mkwlPPvmk6FdoNBp0d3dj165dMBqNqK+vx4ULF2AwGPDhhx+ip6cHR48ehdFolOi2qqoKb7zxhtAEGX15vV58+9vfxujoKE6ePIn9+/cLE6K9vR0hISE4ePAgrl+/jra2NlRUVOD3v/89FEXBzp07cfr0aTQ3N+OFF17Ahx9+KPxaOrKxsTGEhoZicHAQ7e3tqKysxJUrVxAfH4+enh40NTXh3XffRXFxMR577DEp4OXn58NkMuH111+HxWKBn58fqqqqoNfr0dXVBavViq1bt0Kj0aC1tVWUPA0GgwymaGhoQHBwMEpKSrBu3TqMj4+jqalJFCEpcMYozO12Y/78+ejs7ERRURG6urpQUlIiBigiIgKxsbHQ6XR48MEH4Xa7UVZWhldffVUGP5w5cwYLFiyQteXgYUbPsbGxcDgc6O/vl9mu/v7+qKqqEjiGGR7Tehbc2trahCAQHR0t0rUDAwPo7u6GXu+bf2uxWFBZWYnAwECEhYVJk5LRaMQDDzwAm82G3t5emM1m/Nu//RtycnLQ0tIirBIAMj4vNTUVISEhsNvtSElJwbVr1xAeHo6ysjKsXbsWbrdboBfK7nK0IhuXSB/UaHwNdna7HYsXL5bOVhb/udeLi4sxMeGThvZ6vdiyZYt0o1ZWViIrKwuhoaEiYUzYKSoqStgy69evR3FxMc6ePYuGhgYsWrQIQ0NDuHjxIhYvXgyNRoMPPvgAc+bMkfVj5vntb39b6IAsUrIQnp+fj+7ubrz//vtYt24dgoODcfDgQWRkZKC+vh7+/v7S4U6+uNFoRHJyMsbGxjAwMID+/n60tbXB398ftbW1ACBYd319vRTJiYMzMCUde2hoCPfccw+WLl0qU6vWrl2LqKgoCazYg2Oz2dDc3CznhoJuJALMnTtX+mvuv/9+1NXViXbP5710u3fv/t8Ydj8AHwH4QFGUvQCwe/dux+7du5Xdu3crzz//fAuAH+3evfu1559/fi6A8N27d5cAwPPPP/8MgGO7d+/uUr/n7t27K3bv3v3m7t3/T3vXFhvXVUXX9tjjx4xnxvXY4/EjdqZ2qhYaP6RA0qYkRREhAYX0r2lB+YgEUvng8YFaKqHwCVIjviuoRAUBIYWqbb6oCih9KCqJmvcDp5aVErt+zziT8Thu5vBx79o+DkkV2qQzvT5Lsjy+Y3vuPvecffY5Z6+197/48ssv79+7d+8KtiSj5sOHD2N0dBT9/f1YXFzUpQmXYNu2bUNDQ4NqSGzevBk1NTXYunUr3nrrLZRKJezatQuJRAJ9fX04evQoqqurVTiM+6BNTU3YuHGjNjaX/1zmvf7663jssceUhTg+Po7R0VHs2bNHI89isYjOzk6lSnOwsxCGMQatra3o7+9HT08P2tvbsWXLFqTTaTz44IM4evQoampqsGXLFgDLeiNMSxwYGNDtjCeeeAInT57UMnGnT5/Gpk2bMDg4iGPHjiEc9koPcj9vYmICIyMjeOqpp1YoPTKbiOmH7e3tGB4eRiQSUUnUVCqFS5cuYXBwUKNgwMtW4MFob2+vEmdCoRDOnDmDDRs2oLGxUSs+xWIxZDIZfPzxx1pUIZ/PY9OmTRrFcLmfSCRQKBTw8MMPa4oZDy8LhQIymQwmJibQ39+PwcFBhMNeST0evnLLoq2tTSO1VCqln0PdEZs4kslkMD09jUwmowzN0dFRJJNJdHV1YWlpCa+++iq2bdumAUAkElEORH19va5W2JcAoLu7G6O+TDBzvNetW4eZmRn09fVpJsv8/LwWJWlublYNlMHBQa2fe/z4cVRVVeHxxx9HJBJBsVjE3NwcmpqaUCgUlAzW0dGhUg7sN0yJbGlpQVdXF8bGxlBV5ZUaBIC1a9cik8kgHo8jnU4jFPIEyHj2k0wm1ZnNz8+jt7dXa/lGo1E0NDSoZDNz18lHyGQyCIfDGBgY0MmIWSf5fF4LciQSCSQSCZ2kAWifGRgYQFdXl9YM7uvr03RTRtA8m3nggQfw0UcfAfBqACQSCdx///0olUoYGhrS8T0/P4+uri6t1sWUVGqwp9NpPZtas2aNrgx6e3u1WhTPtegrmpubsbi4iA0bNqiPOXv2rJ578G8ol8Esv/b2dly4cAHr169XeYVUKoW6ujrkcjlEo1G0trau4I68++674/v373/xln7bZnPdxrELgN8DmDXG/Ni6njbGjPuvfwLgq8aYJ0XkSwAOwttnbwfwJoA+Y8yN//3vHoaGhsyRI0d0C4SHqIwmuBdnn0jzQbJBbUIAsCz/a0e+zA2lQ7NzvpnGxOUd070OHTqETCaDl156CS+88MIKVqOtOcJsH5KBKA96c+44nQnv0ZYYYG4/t2Lse6cwGXP56aAAqCYJI31OLHaapS1BYKcM8vnz1J6sPLYtV1AAdAnJzBXaysFVLBZx/vx5LXxsqydy8qivr0djY6NqbzNjgAdTpVIJ165d0/zqc+fOoaenR5+PTQhin7Dbjcw+m3DD/U2SSshcZapidXW1MjgpDVwqlRCPx1Uk6sCBA7qdxkIZ5GXYjFl7i4B55lVVVSqgFYvFtD3JkWDiwMjICFKpFOLxuKZCUvLWGIODBw8iFothx44dOjFxYhkbG8O6deswPj6OQqGA9vZ21NTUYGxsDDdu3FAnRf7DlStXVN+GhdFbWlpQX1+/QguGVYri8bhqs1AqgExy3iNZ43Rk3CZjGuzExAS6u7vVqV2+fBn5fB6xWAzFYlGrJF28eFFrFLDfk/zHn0n4s/sAxxJL2XE1u2/fPu33/OyGhgY9gymVSpienkYqldLgYHp6Wsch0yyZnsjJGFhWQ+UY5lYNt2xJwOP4Z+owuRrkYFDpkmObAQlTV5nowb6dTCYxNTWFffv23VY47E7kBx4F8D0Ap0XkhH/t5wD2iMgAvC2XUQA/8AfrWRH5C4Bz8DJtfvhJjp2gM+FS146w7CiTTC3+roisUFjksox/y5x3Omz+H+4/09naOiR01PX19QiHw3jnnXewe/duJWcwP5xEJ87EdDikRfOh84FTE4RsQd4HH77tiGmzLQPANmG0mMvlUFtbqzRzm+5uE0DoTDhJMOokA5AkIX4eJwKmFJKBRyYil59M4aMtZEJevXpVl51sZ67KqN3BQ3SmdXG/lKQp2sRBDCxLVNCZ2GQ06nUwN7u62hOwolY5ByYHCydkinJxtVUsFjUiJi+Bz+7DDz/E9u3btX9x245MUDoXewKnU1paWkJbW5vupzKKZv/IZrPo7u5GNpvVA1Ruk/A51tXV4ZFHHkE6nUZNTY1ug9Gp2oxOPqPm5mZ11uwHCwsLiEajuHHjhlb+4sRHe8m5qK2t1bxxe/VE5u7k5KQ6bH4G99e5kmOBDQZcTJjgRFIqlfQZU6ee/ZSTJyNjrqqz2aw6OhIK2V/m5uYwMjKCZDKJnTt36v/nRMtAg5PrwsKCSijbEhF8BjbPgpOQzUjnVgzvmYELnw0jc7Y/A9ZEIqEpm7wXbtXYLHMASpZioGhzZm6HipH8ffPNNzUSInGBg8ZWhmRHt8FHGZSnAAAHC0lEQVRGBZa1ZOgsOPDJBLXz122yAh0ZI0Zbl50dlw6TzFE6Il6PRCI68/L3+QAikQhmZ2dXUN3pTHk/hUJBVy6hUEgdEnVHgGVnwUHICYERh012YiTLCYT2UFeEbcFlNwfwrSjqbJPOzk79HC4tc7kcjDE4deqUUtbJXKQUAJ2NLadqfGYtSVt0dGTyMSLP5XI6+G1CFwBtR4o3UYGRjiGfz69YsbEPcaDTYTU3NyObzSopBcAKqQL78JRbT5Qq4CrJnkxJNqNWSzabVZ1v2snV1pUrVwBAy911dHRgcnJS+x77JNNC29raVEKWTm1xcXFFxE/OAx1TKBTC1atXNepk37p+3Sv4vmbNGi23yGibOfvRaBRzc3M66dC5cwVUV1eHqakpbedoNIpwOKxVqOjIeODY2tqqh6QAdMXE/082LCNmPm8WwwiFQujo6FBNHlYm4sTJDBRO/lNTU0p0YtaKzS6fnZ3F9evXlenK55LL5VaMb64UqGPDyJzjhZW2ZmZmNMLn+wwo8vm87iLYyrMknJHAZbNYSfZjAMOAb2FhAc8888xtI/eKcO4DAwPm7bff1p85G5LtxUHF/T8OXEblANSpctkFLFeJ4XKO+fOcSelAqIXC9wlGwuyU/EwOmKqqKqUEc2a2i3NwIM/NzWl+vu2IGHmQmUiHzEmJSzVgWRPdHnh0dOw0rO3KaMSO0mtrvTqb3HLh5GOzXLmdxE7K94tFr75qLBbTbBXaCACXL1/WepXJZFLbAYCmZ5IKnkwm1dFwL7GpqUmjXUZYnJxZyYjtAXjRjC2sdbNgGtmAtJ9txUmfg2NyclL3uDlRkdzCZ8HnHIlEdBlPJ80Jnk6dfYVtS+lXPjv2N9qQzWaVcUtZDE5uU1NTKBaLmjnCfs+cfwYTvG+uQlh7lltOXFFQs4h9j8+CUe/w8LDKHDc0NGB8fFzbhjbOzc0pKY2OkXbzmdfW1mJmZgbXrl3TCT0ej+t4YJ8tFouYmJhANpvVlFaumqgBxfMiZrZEIhEda3xmxhid/LnlxkPxUqmEWCymssOtra3KuAaWtdgPHTqEmZkZPP3004hGo5idndXtXlvAj4EPgxI7eOTWKPs2GddsQ65clpa8IvCFQgE9PT3a73lOxFVksVhUoTsqqNpaS0zDrHjnLiJTAK4BmC73vZQBSTi7VxOc3asL99rubmNMy63eqAjnDgAicux2M1CQ4exeXXB2ry6U0+6KEA5zcHBwcLi7cM7dwcHBIYCoJOd+y0T8VQBn9+qCs3t1oWx2V8yeu4ODg4PD3UMlRe4ODg4ODncJZXfuIvJNEbkoIpdE5Nly38/dhK+WOSkiZ6xr94nIGyIy7H9vst57zm+HiyKyvTx3/dkhIl0i8g8ROS8iZ0XkR/71QNsuInUi8p6InPTt/qV/PdB2EyISEpH3ReSw/3Pg7RaRURE5LSInROSYf60y7CZZpRxfAEIAPgCQARAGcBLAQ+W8p7ts39cADAE4Y137NYBn/dfPAviV//oh3/5aAGv9dgmV24ZPaXcawJD/uhHAv337Am07PEXUqP+6Bp409sag223Z/1N4ulKH/Z8Dbzc86ZXkTdcqwu5yR+5fAXDJGDNijLkO4M/win0EAsaYIwBmb7r8HXhCbPC/77au/19FTioV5vYFXgJtu/GQ93+s8b8MAm43AIhIJ4BvAfitdTnwdt8GFWF3uZ37pyrs8QVHyvhqmv73Vv96INtCVhZ4Cbzt/tbECQCTAN4wxqwKuwH8BsDP4NVZJlaD3QbA30TkuIh8379WEXbfiSrkvcQdFfZYJQhcW8hNBV6o6XGrX73FtS+k7cZTQB0QkQSAV0Tky5/w64GwW0S+DWDSGHNcRLbeyZ/c4toXzm4fjxpjxkSkFcAbInLhE373c7W73JH7fwB0WT93Ahgr0718XpgQkTTgaeLDi/CAgLWFeAVeDgH4ozHmr/7lVWE7ABhjsgD+Ca9IfNDtfhTALhEZhbe1+nUR+QOCbzeMMWP+90kAr8DbZqkIu8vt3P8FoE9E1opIGMCTAF4r8z3da7wGYK//ei+8Cle8/qSI1IrIWgB9AN4rw/19ZogXov8OwHljzAHrrUDbLiItfsQOEakHsA3ABQTcbmPMc8aYTmNMD7wx/HdjzHcRcLtFJCIijXwN4BsAzqBS7K6A0+ad8LIpPgDwfLnv5y7b9id49WWX4M3a+wA0w6tONex/v8/6/ef9drgIYEe57/8z2L0Z3nLzFIAT/tfOoNsOYD2A9327zwD4hX890Hbf1AZbsZwtE2i74WX5nfS/ztJ/VYrdjqHq4ODgEECUe1vGwcHBweEewDl3BwcHhwDCOXcHBweHAMI5dwcHB4cAwjl3BwcHhwDCOXcHBweHAMI5dwcHB4cAwjl3BwcHhwDiv52BLX3m0/pAAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "imshow(img, cmap='gray')" - ] - }, - { - "cell_type": "code", - "execution_count": 70, - "metadata": {}, - "outputs": [], - "source": [ - "def make_prediction(img):\n", - " processed = img / 255.0\n", - " processed = np.expand_dims(processed, 0)\n", - " processed = np.expand_dims(processed, 3)\n", - " pred = model.predict(processed)\n", - " pred = np.squeeze(pred, 3)\n", - " pred = np.squeeze(pred, 0)\n", - " out_img = pred * 255\n", - " out_img[out_img > 255.0] = 255.0\n", - " out_img = out_img.astype(np.uint8)\n", - " return out_img\n", - "\n", - "def path_leaf(path):\n", - " head, tail = ntpath.split(path)\n", - " return tail or ntpath.basename(head)" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [], - "source": [ - "pred = make_prediction(img)" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 66, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAADECAYAAABk6WGRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydd3zURf7/X7vJpmx62TTSSSMECBA6AlGUEwRF9BQUhZ8eHnZOVDj14MATDhVFREFABTT0KjUQUiGkkt4TkmyyyWaTbHY3m+37/v2R2/kmJKFjudvn45FH9tNm5vOZmffnPfOZeQ2HiGDGjBkzZv674P7WCTBjxowZM/ces3E3Y8aMmf9CzMbdjBkzZv4LMRt3M2bMmPkvxGzczZgxY+a/ELNxN2PGjJn/Qu6bcedwOH/icDjlHA6nisPhrLhf8ZgxY8aMmb5w7sc4dw6HYwGgAsDDABoAZAGYT0Ql9zwyM2bMmDHTh/vluY8FUEVENUSkBbAPwOP3KS4zZsyYMXMd98u4DwIg7LHd8J99ZsyYMWPmV8DyPoXL6Wdfr/4fDoezBMASALCzsxsdERFxn5JixowZM78fdDodOBwOLC0todfrYTQaYWFhAYPBAB6PB41GAysrK3C5N/e9c3JyWolI0N+x+2XcGwD49dj2BSDqeQIRfQfgOwCIiYmh7Ozs+5QUM2bMmPn9IJFIoNfr4e3tDQDo6uoCn88HAHR2dkKr1cLV1fWWwuJwOHUDHbtf3TJZAEI5HE4Qh8OxAvAsgBP3KS4zZsyY+cNgZWUFDw8Pts3h/F9HB4/Hg5OT0z2J574YdyLSA3gdwDkApQAOEFHx/YjLjBkz/WMwGH7rJJjpB6VSCdMoRaPRCFtbW3aMx+PBwsLinsRz38a5E9FpIgojosFE9K/7Fc+9QK1WQyQS3fzE3wiJRIKKigq23dDQgNLS0vsSl16vh1gshkqluudht7W1QafT3fH1arUav7ZEtcFggFQqhUajueuwdDodGhoa0NbWdg9SdmOkUuk9Da+zsxNNTU13FYZcLkdJSfdo6K6urnuRrFvGYDBALBbfsJ4TEbq6uu5JXg+ETCaDSqViL16j0djruE6nu6s60hPzDFUA+/fvx4kTv69eI1PmP/3008jKysKlS5ewd+9evPfee/D19UVgYCAaGhrY+WVlZRg/fvxdx2tpaYlZs2YhLi7unhvSL774Ap2dnXd0rdFoxJYtWzB37ly27/Dhw2htbQXQ/bw+/vhjSCSSe5JWExYWFti0adNNvalz587hs88+G/B4V1cXeDwe5s6de9/L2qpVq7B79278/e9/v6twTNe3tbVBoVBg9+7ddxQOEWHlypXIycmBi4sLRo4ciSlTptxV2nqi1+vR2dmJp59+esD4LSws8Oyzz2L//v0DhsPhcNDZ2Ynt27ffMwN7PQqFAoMHD4a1tTX0ej0AQKPRQCaTwWAwQK1Wg8fj3ZO4/ueNu0QigYWFBXJzc28pQ2/X4N2JB6zRaJgxcXV1xcyZM/Hiiy8iMzMTdnZ2AABbW1ssWbKEXRMREYErV67cdlz9YWtri7Fjx/bqC7wXpKWlwcXF5Y6uJSK88847OHbsGPN2Dh8+DHd3d7Zta2sLgaDfgQN3THNzM86fPw9LS8s+eW+qnACwY8cOtn19d4hWq2VptLOzw4QJE+5pGk2Y4t23bx/eeust/Pvf/76r8LKysgB0v/ATExMxe/bsOwpn7dq1WLZsGWJjY+Ht7Y158+bh448/vi0DeqN6Z2FhAXt7exw8eBBqtbrPcVM51uv1mDZt2g3juXDhAubMmXPPDOz1mD6YGgwGWFpawtLSElZWVnB0dASHw7mtFo1cLr/h8f95415VVYXp06fDwsJiwAxtampCR0cHAAzYZdHe3t6rKdzW1obOzk7Y2tqySt/S0nLDZmF7ezvkcjmsra0hl8tRWlqKtrY2NDU1obm5GSkpKQCAa9euQS6XQyaTsWvr6+uhUCgAgHmzRDRg81yhUPQyTiaICG5ubvD29mbh9KSzs7OP9216HmKxuN9rKisrodFoYG1tDaD75dXS0tLnvOrqahBRr2PNzc3st1gsZmk/efIkqqur2f3GxcWBz+f3Cbe9vZ09J4lEwjx7kUjUryHoSVVVFQoLC+Hp6Qng/4yE6TlbWnYPNqutrUVeXh57nqYXc3NzM+RyOaysrGBvbw+1Wg1HR0fY2tqivb29V1wDGS+5XN7nZSGVSvvt2uns7IRMJmPp69m90PO5EBHLs7a2tj7luaGhAZWVlSwv7e3t0djYCBsbG0gkkn4NUFtbG+rq+h+4weFwenXpTJ06FWPGjIFcLmdx92yFmuj5jDgcTq9zetYjmUyGmpoadHZ2wsbGBkC3Ib/+GUdFRYHH4/Ubl4m8vDz4+PiwbbVa3ac7yvTsBypDDQ0NrI4QUa90lJeXg8PhsLIkkUhQWVmJxsbGPnktlUr71DWNRgOJRAKFQsHK34AQ0W/+N3r0aPotSE1NpcOHDxMR0Z/+9Kd+z2lubqbs7Gx6+eWX6eLFi9TY2EjDhw9nx+Pj4+nnn38mIqLXXnuNZDIZnTt3jpYtW0axsbFERDRv3jwSi8W0ceNGmjZtWp84ampq6NChQ6RUKik+Pp5ycnJY2PX19ey82NhYMhgMpFQqSafT0SeffEJSqZS2b99Ob7zxBg0dOpRKSkro7NmzNHHiRFIoFLRmzRqKiopiYXzxxRdUXFxMRETz588nsVjcKy2FhYW0dOlSamlpobfeeotMeaPT6Wjz5s1UUlJCRESOjo6kVqvphx9+oEWLFlFkZCQREUVHR9OGDRuIiKigoIDWrl1LXV1d1NHRQYsWLaKUlBTKycmhyMhI+vTTT0ksFtOOHTuIiGjs2LH0zTff0IkTJyg8PJxWrFhB2dnZtHTpUkpJSSFfX1/au3cvyeVyWrx4MS1cuJCUSiURET3//POkUCioq6uL5e2+ffuoqamJFi5cSHK5nHJycujll1+m5ORkIiIaMWIEEREZjcZezyAnJ4fWrFlDREQxMTE0f/589gzWrl1LRESbN2+m6upqds3TTz9NCoWCba9du5Z0Oh01NzfTypUriYiooqKCPvroI1IqlWwfEdGGDRuosrKSNmzYQBUVFUREVF5eTn/9619ZWFKplJKSkmjPnj1kMBjogw8+oLi4uD5lSafT0bvvvktarZaIiJYvX047duwgg8FAsbGxVF5eTjt27KDXXnuNvL29SavV0ueff04SiaRXOImJiZSUlMTKn7u7O7u/qKgoUiqVpNfraefOnfTBBx8QEdHly5dJKBT2SZPBYKDJkyfT8OHDKSYmhoiI1Go1lZWV0fz586myspIuX75M48ePZ3mXmppKLS0tLI8kEgnl5+fT1KlTKSkpiWQyGW3bto2IiC5dukRHjx6l/fv3ExFRRkYGlZeXExGxtF27do1WrlxJMpmMVq5cSVKptE86jx8/TuHh4Wx7z549VFhYSEREwcHBpFAoaM+ePfTCCy9QYmIiERFFRETQrl27WLpff/11Ft+XX35JxcXFdOXKFdq4cSMRET3yyCNUXl5O9fX1lJqaStu3b6fk5GQaMWIEzZo1i5qamig7O5v+/e9/E1G3bXj//ffJaDTS5s2bqaysjHQ6Hb3zzjtUWlpKALJpALv6P+25FxUVwcfHBwqFAjqdjnk8PVGr1fD09ISFhQUmTJgAhUKB2tpadvzbb79FREQENBoNNBoNuFwuLC0t4eLiwjwAX19fCAQCxMbGYurUqezarq4uaLVabNu2DdOnTwefz0dUVBRMY/4VCkWvbgwbGxtwuVzw+XzI5XJERESgoaEBoaGh8PDwwLRp08DlcmFjYwOBQAB7e3tIpdJeHvrevXvh7OwMsViM0NDQPt0Y1dXVmDlzJgQCAWxsbKBUKgEAV65cwblz52CabObv7w+DwQALCwtYWVmxPtTW1lbmjX377bdoaGiAra0tHB0d8eijj0IikYDP54PP58PX1xfFxcUYNmwYAOD111/HpEmT4O3tjeDgYDz99NMYPXo0ZsyYAX9/f4SGhsLX1xcODg6orKxEWFgY89QqKythb28PW1tbSCQSbN68GWFhYSgpKYG1tTVUKhV8fX1ha2uLmJgYdq8A+nQ/7dixg3mGfD4fc+bMAQAkJSUhLCwMAODg4MDGKQNAeHg47O3tez1HS0tLODk5ISQkBABw9epVPP300+Dz+cy7LykpQWhoKEJCQmBjYwN/f38AQGZmJo4cOQIAePbZZ+Hs7IytW7ciMjKSlbXBgwfjetrb2xEcHMxaoQ0NDZgxYwa4XC5ycnJQUFAAR0dH8Hg8zJo1CzweDy+88ALc3d17hVNbW4shQ4bAz697usqwYcPY/bW2tqKrqwsGgwHvvPMOHB0dUVVVhfLyctbK6QmXy8WxY8dw/Phx2NjYoLy8HEqlEh4eHhAIBAgJCcGgQYOQk5MDANizZw+GDx8OZ2dnFkZrayu8vLwwduxYTJ06FR0dHSzNXl5e8PPzY2PFKyoqUFVVBQB47bXXAHR75C+88AIcHR0B9N9Sqq2tZa3LsrIybNmyBVFRUQC6u/wsLS3h4+MDPp+PBx54AABQV1fHwvrpp59Y2be0tERwcDBsbGzg5uaG4OBgAEBNTQ2srKxgY2OD7du3IyQkhOV5aGgovLy88NVXX0Eo7J7g7+DgAF9fX+zbtw8fffQRXFxcUFVVBTs7u5t/4B7I6v+af7+2567Vaik/P59EIhEREWk0GvLz86Pa2tp+z//++++Zx7127VqaPXs2EXV7SVZWVkREfTyW6OhoSk1NpYsXL9LFixeJqNuzSU9P75WOdevWEY/HY/u+/fZbunLlCkmlUnr++efZfqVSSQcOHGDbJo/YRGxsLEvD3LlzadOmTSwdptZJV1cXLV26lORyOfPsDAZDr3Q//vjj7HdUVBTzhgIDA2nu3LlERCQWi2n79u3svMmTJ1NdXR0RET3xxBOkUCjIYDCQnZ0dJSUlERHR3r17qaOjg13zwgsvsP8mr/HAgQMkEolILpfTl19+2Std7e3t9M9//pNtjxo1ihobG0mj0RARUUhICBF1t7RWrVrF8sWEwWCgAwcO0EMPPcT2TZ8+nfR6PV0Pn8+nhIQEIiKaMmUKicViEolEzItsbW3t9ZzWrVtHzc3NbHvLli0UHx9PRN3en0QiIZFIRA899BCp1WoiIho3bhw1NTXRI488wq574oknWL40NzdTfn4+vfDCC7Rz504i6m4tEXV7hQMRHx9PVVVVLO7U1FR2TCAQUEZGBhF1t0jEYnGf+zcYDFReXs48bI1GQxKJhLVYiouLac+ePUTU7VW+9957REQkl8v7TU9rayvl5uZSV1cXyWQyamxspO+//56IiM6fP89at5s3byZ7e3v68ccfqdss9a53SqWS9u7dy+rsp59+yloSOp2OZs2axc6Ty+V04sQJevzxxykuLo5aW1vpiSeeYGkaOnQoERF71iZCQkLYfQ4fPpzVP4lEQtu2bWPxmdJMRPToo49SU1MTERHxeDxKSUkhIqJDhw5Rc3MzyWQyWrhwITt/+PDhlJubS2VlZWRnZ0ddXV1UV1dH48ePpytXrhAREZfLZfl25MgRam9vJ3d3d1q9ejW1tLRQa2srEXW3BGH23P8PlUoFsViMpKQkeHt7o7OzE1ZWVrCwsGBvy+vZt28fZsyYAQDYuXMn3nnnHcTFxcHS0hKBgYEAuj+U9RxNUF5eDh8fH3zxxRcICAgA0N0XO2bMGHYOj8dDZGQkRowYwcKura3FuHHjIBKJUFZWBqB7eFR8fDz7ECeRSHDw4EFcvXoVAHDy5Em89dZbLFyhUIhFixahoaEBY8aMwYQJEyCVSmFrawsulwuNRoPKykpkZ2f3meLc2NgIoNtTGj16NCZOnIjW1laEhYVh8uTJ0Gq1WLduHSZOnMg8h7lz58Lf3x+bNm3Chx9+iJaWFnC5XAwdOhQRERE4dOgQNm3axPpk1Wo1G9v7008/wc3NDZWVlZgwYQK8vb1x8OBBvPjiiyxNEokEM2fOxF/+8hdcuHABYrEYY8eOhUAgQGZmJkpKSjBp0iRcvHgR6enpeOCBBzB8+HAAQH5+Pn744QdwuVzs2LEDkydPBgB88803+OCDD3Dq1Kk++R0TE4PQ0FAcPnwYjz32GCoqKmBnZ8fybvXq1aiqqkJKSgqEQiG++uoreHp6IikpCUB3Cys8PBy7d+/G2rVrUVpaCmtra/j6+sLa2hrXrl3DM888Ay8vL9byMPXtXr16FTU1NXjqqacQFBSEOXPmYPTo0QDA7snDwwO7du1CTU1Nr3RLpVIcPnyYefRubm4s/M8++wy5ubkYO3Ys2tvb8dJLL8HDw6PPKCAul4ujR4+iqqoKzc3NICLEx8fjueeeQ01NDZYvX44//elPSEpKQlBQEDQaDYgIBoMBFy9ehFar7RVefn4+KioqYGlpyT56mvL2008/RWRkJKRSKerq6pCWlgZXV1eMGjUKALBt2zbMmDEDBw4cAJ/Px65du1hr6euvv0Z9fT0SEhIwe/ZsyGQyZGVl4eDBg1i2bBlmz56N+fPnIygoCCkpKayfXSgUIjIyEhkZGX365JVKJR555BEUFhaCz+dDIBBAo9Fg/fr1iIyMZH3tptb39u3bsWzZMvZtIigoCD4+Pjh58iQ+/vhjeHp6wmg0IjU1FcD/tS5NLZShQ4eCiHDw4EFWRwHgwQcfhI+PD3bv3o3169fDxcUFM2fOhEwmg0AggJWVFU6ePMk8/oGwWL169Q1P+DX47rvvVvcc+XE/EYlEOHToELKzszFnzhxYW1sjPT0dP/30E1xcXBAbG9vnmqNHj+Lhhx9GWFgYUlJSWLPM3d0dRASFQoHm5mYIBAIMGjSINYG1Wi28vLwgFAohEokwZMiQPrPPPDw8wOVy0dzcDA6HgyeffBJOTk44evQoMjMz8de//hVGoxEXLlzAqFGjYGNjA3t7e1RWVkKpVGL8+PEQCoWoq6tDV1cXwsLCkJ2djVmzZsHa2ho7duxAQEAA6/q4fPkyLC0tIRQKMWrUKNaUNZGeno4nnngCXC4X33zzDQICAhAZGQknJyeUl5fD2toaTU1NaG1txQMPPMAqSHBwMFpaWlBXVwej0Yjg4GC0tbWhtrYWHA4HLi4ukEqlGDNmDCwtLXH69GkQEezs7CCVSqFUKhEeHg5ra2ucP38eEydOZF0LHA6HVQw+n48hQ4YgOTkZlpaWGDt2LLy9vXH+/HlEREQgMjISwcHB4HA4UKlUaGpqgp+fH/z9/XH48GHMnDkTISEhUCgUKC8vx5QpU/rkSXt7OyoqKmBjY4Nr166hvb0dsbGx4PP5KCgoQGdnJ1paWjBx4kSEhoZCIpHAysoKBoMBwcHB8PT0xOHDh2FrawsPDw8oFArExsbC3d0dWVlZKCwsxAsvvABra2vw+XxkZ2dDoVCgoKAAo0ePxpAhQ2AwGKDX63H58mXMnTuX6ZC0t7ejtbUVAoGAGXsTnZ2d2LNnDxYsWAAAEAgEOHToEKRSKXQ6HWJjYyGVSiEWi2Fpadlvtw7Q/ZIwxREVFYXk5GSMGDECVlZWuHbtGqysrNhH9+LiYlRXV7PuCFMXlIm0tDSUlpZCIpFAKpXCzs4Ofn5+4PF4WL16NcLDwyEWizFjxgxERERAIBBAqVRCpVKxbsWJEyfC19cXp06dwlNPPcXKsb+/P2JiYlBcXAxXV1eMHDkSAQEB6OzshEgkQnV1NZ599lkkJiZCp9PhiSeegEKhwKVLlzBhwgSEhIT06pJLTU0Fl8vF5MmT4e7ujvb2dvB4POh0OjQ1NWHq1Kmor6+Hq6srAgMDIZVKUV5eDgAICwsDh8OBSCRCWloaiouL8eabb8LGxgY2NjZobm5GS0sL0tPT8cwzzyAiIgIymQwajQYcDgeNjY2wt7fHyJEjweVyIRQKkZaWhqqqKrz22msQCAQQCoXg8XiQy+Xw9/eHTCbD5s2bm1avXv1df/l4X/Tcb5c/graMVquFlZXVrxJXUVERzp49C0tLSzz22GMICQnBjBkzcO7cOXaOXq9nX8t7/r4XmPrS+2OguIxG402Fjm7lnPvNje7tRuj1enC53Du+x577e/6+F3mnUqmQkZEBmUyGffv2Ye/evXcVXn/odDr2su35G+ju2r3RsNmeI6V6smjRIvz4448sPIPBAA6HAy6Xe8f5ZEKlUvWa+Xm/qa2thZ+fHywsLPDqq6/C3d0da9as6XWO0WhEXV0dBAIBLCwsYGNjA6lUykaoAd0j81xdXWFtbY2//e1vUKlU+Pbbb/uNs7S0FJGRkTlEFNPf8f+5bpk75dcy7ACwdOlSBAQEYPz48bhw4QKkUilrCppexj0Nwr007ABuWKkGiutWjPZvbdg7Ojr6dGXcKpaWlnd1jz339/x9L/Kuuroan3zyCcRiMZ555pm7Dq8/ehrz64cM32w+xPWGnYiQl5cHW1tbaDQaFp6FhQV7Nnc7Bf/XNOxA95yLrKwsiMVieHh44LnnnutzjkKhAJ/Ph7W1NWxtbcHhcODs7NzLtmRmZuLSpUvQaDRwcHDAwoULB4zzZmXH7LmbMWPmV8f0rUupVN7xxLbfE2q1mo1K6zlq6k7DksvlvcTF+kMkEmHQoEEDeu73S/LXjBkzZgbEZAB/zRbx/cT04fpehXUvwjN3y5gxY8bMH5CbqX7+7o270Wjso5zWE71e/6srzP0eUalUveQI/sgYDIbbFhi7U0GyO0WlUt1Tdcfr5STuFRqNpt/JebfKneRFf8jl8n4lJ/7oGI3GW34+Wq32ps9AoVDcVBbDxM261H/3xr26uhqXLl0a8LhMJsMnn3zyq6XnXkup3isqKiqwadOm3zoZd4xWq2VCUkqlEh9//DGqq6tv+GLvydmzZ9m8gF8DIsLq1avxyy+/3JPw9Ho91q9ff0/C6gmXy0VFRQVOnjx5W2kxvWgUCgVOnz7NZnzeCRUVFTh16hT279+Pffv2DTif5HYxGo1M8+m3wmAwoKSkBOfPn7/heXq9Ho2NjTeVE66trb2hcmVPbvaB/3dv3Ds6OnppmffEYDDAzc0Nubm5v1p6XnnllfsmB3o3ODk5IT4+/rdOxh3zySefsBaYnZ0d6uvr4enpecvKlCUlJWw42f1Gr9eDz+ejuLj4nn0MdHJyui9rCvB4PHA4nD7zGW7Ehx9+yBRHnZycUFRU1K+swK1QVFSE+Ph4zJ8/H2+88QZOnjwJPz+/u5aTNhqNKC8vx5///Oe7CuduMQ3htLa2HrCbxPSBdMmSJX1GuOh0OhgMBhARdDod+Hz+LY/0uZmH/7s37oMGDcJLL73ECkNnZyckEgkbB1tVVYU33ngDDQ0N/aoc9sSkKtgTk5KeCYPBwCZkiEQi9luv1yMtLQ0PPvhgr2FaWq0W7e3t7EHrdDomxSmVSvttavenOX59xdbpdH1m0N2IEydOICQkpN9nYDAY+u26am9vHzCOnh7z9eeIxWLmgZhedO3t7ewaU17J5XLWZO0ZRktLCzQaDfR6PVpaWlBWVobjx4/DyckJTU1NaGpqwrPPPgt7e3umbQN0dzFcn4emCvXLL79AIBD0aVk1NDT0UQFsaWlhHp9MJuvl/d1KE9t0nzExMZg8eTK7X1MFNaFQKNixlpaWXsfa2tp6pauurg7z5s3rFY9UKu2VDz21RHruN/2+XgLWdC/nzp3Dgw8+2Etutj9UKhW0Wi0SEhIwatQoSCQSdHR0IC0tDQ4ODujo6OgzA7Wzs/OG3aI///wzpkyZwp6xaWif0Whk1/XUajJxfUvBlK9isRhtbW3o6OjAsWPHYGlp2et+Ojs7+63nQHcZNKXDaDT2uheDwdCr9SiRSG5qPE11Oy0tDVOmTGF2oaOjg9Vxo9EIvV6PH3/8sY/KY0NDA9RqNdPp4fF4OHbsGKZOndqvEqhJ892ESSdnIH7Xo2UqKiqwceNGvPzyy0zsKSUlhckFxMbGor6+Hh0dHYiPj4dGo8HSpUv7DSs1NRVGoxHx8fGYMmUKAgIC0NTUhNLSUhARwsLC4Ofnh4SEBLS0tCA8PBzXrl1DTU0NVqxYAalUiv3792P48OHIy8tjU6QzMjKgUqlQW1uLJUuWIDk5GUKhENHR0dDpdEhJScHy5ct73ZNMJkNeXh57UeTm5kKr1aKoqAgPPPAAbG1tceHCBQgEAhQWFmLMmDE39bxSU1OhUChQVFSEiooK5tGIRCKUlJTAysoKzc3NbL9MJkNRURGICIGBgUwioWc6c3NzwePx4O3tDZFIhCeffBKZmZnQ6/VoaGjAggULwOPxUFJSgvb2dlRVVWHRokXgcDjIycmBTCaDUCjEk08+iaSkJKjVagwbNgytra2oq6vDokWLwOPxsHfvXrS3t6O6uho8Hg+ZmZmYOHEiuy+BQICYmBjk5eWhpaWll664VquFra0tysvLkZaWhqSkJHz44YcA/m/lH71eD7VajZCQEMjlcpSXl7PZpJmZmejo6EBERAQcHR2RkJCAZcuWDfic8/PzoVQq0dDQwGb9cjgcXLlyBZ2dnWhtbYVGo8Fzzz2HCxcuoKKiAu+//z5Wr16Nxx57DDNnzkR1dTUrW1OnTkV4eDhyc3OZSBXQPQPTVCZef/11AN1a4z4+PnBxcUF5eTlGjRqF8PBw5OTkQKPRoLa2FlwuFwsWLEBRUREaGxuhUChw9uxZrFy5Evb29ti6dSumTZvGBOB6otfrUVlZCYlEAkdHRzYeva6uDkVFRThz5gxmz57Nrs3IyICFhQWqq6sxb968fsddDx48GB9++CHWr18PJycnTJs2DQ0NDSgvL0dSUhKmTp2K9vZ26HQ6hIaGsrInEomQmZmJ2bNnQyKRoKCgAFqtFj4+PsjKysKkSZMQFxeH0NBQVFdXIywsDESElJQU2NnZ4eLFi5g1axYzgFVVVVAoFCguLsbzzz+PkydPIi8vDx988AEUCgUyMjKYxEh+fj5UKhXkcjkmT57cb90rLCxkq2qdOXMG7777bq+84/P5EAqFWLhwITo7O3HhwgX4+/ujtrYWAQEBrP6ZXiJ/+ctfAHRr6I8bNw6lpTp3UXEAACAASURBVKXw8fFBWFgYW3nNysoK1tbWbFGegSSWGQOJzvyafwMJh9XX11N4eDiT19y+fTudOnWKsrKymBTnc889Ry0tLVRdXU0uLi59wtDpdFRdXU2PPvooSSQSSk5OJrFYTHK5nObNm0cJCQmUmppKBQUF1NraSo2NjfTGG2+QVqulmpoasrW1ZeJajz76KAtXqVTSihUr6PTp05SQkEBLliyh5uZmqq+vp9dff51J0To4OLBr3n77bSZvu2nTJjIajTR58mRau3Yt1dfX0+eff04Gg4E2bNhAp06dIiLqJTR2IyZOnMgEyiIiIkihUJBGo6H33nuPhEIhyWQymjx5MhER1dbW0rhx46i1tZVkMhnV1NT0CqumpobOnz9PK1eupLfffpuIiPLy8mjZsmV05swZIuqWkiUi+vnnnyk/P5+IiL788ksyGAy0fv16SkxMpPPnz9OKFSuYBGtPwSUfHx8mJDZ79mwmE0xELJ2pqamUlZVFhw4dIiKiWbNmsefXk66uLiahKxKJmDCVSdwrMzOTkpKS6IsvvmB5mJqaSufPn6fi4mKKi4tjAmfLli3rI39r4vLly0xI7fvvv2fpz87Ops8++4xEIhFt3ryZ1q5dSyUlJZSRkUGPPfYYERGtWrWK9u7dS8uWLaNHH32UFAoFrVu3jk6fPk0ikYimTJnC4vnkk0+ovr6ejEYjNTc30/79+6mgoIB++eUXVu5XrlxJISEhVFFRQVu2bKGOjg7as2cPvfnmmySRSOjEiRNERLR7926aNGkSC/v8+fP93lvPZ9kzLxYtWkTLli0jom7Z32+++YaIiLKysmjDhg3U0tJC3377LRNw64/MzEx66aWXKCQkhJqamuj06dNUX19P06dPp0uXLhHR/4l5zZ07l6XRtO/UqVNUV1dHr7zyChERE0aLjo6miooKVj9ffvll0ul07JhGoyGNRkNLlixh8sDnz5+n8vJyqquro6+++ooaGxtp27ZtFB0dTUTdQmn4j3CZSTyuv/vZvXs3GQwG+vHHH9m1lZWVNH/+fJYek9AcUbdQW21tLRUWFtKhQ4do6tSpVFdXR8uXL6fw8HBmL3qWg4iICKqvr6eXXnqJiLrF5EwickTd9QF/VOEwLy8vREZGMsGsBx98EGvWrMHVq1cRHh6OqqoqdHZ2QiAQoKSkBA4ODn3CMBqN8PDwQHZ2NpYvXw4ulwsPDw9cvXoVHh4ecHd3Z14Yn8+H0WiEWCwGj8fD1atX4ePjw5q9165dA9DdhCstLUVcXBy8vLzQ3NyM2bNnw9XVFUSExsZG9rY3SaaqVCrk5uYy72DBggVsXcrQ0FAkJSXh8ccfB5fLxYwZM7Bq1Sq8+uqrTFTsZnh4eGDo0KHo7OzEtGnTYG9vj5qaGnh4eMDX1xdlZWXMswoICMDw4cPx7rvv4rPPPkNQUFCvsNzd3REREYGCggJMnToVer0eer0eP/30E6ytrXH8+HGMHz8eRUVFWLduHdM4mTdvHnJzc/Hll1+y+GfOnIlRo0bBw8MDXl5eALqbmHPmzGGaLiUlJb3SYBJiGjx4MKqrqxEZGQmgu5tj6NChfe69rKyMnSMSicDj8ZCbm4vQ0FCcPHkSRUVFGD16NDZu3Ah/f3+cOXMGpaWlmD59Otzd3VFWVoYhQ4YA6O4+uV7+1sSRI0cwcuRIAN3CcKb079+/HwKBAKWlpXB3d8czzzwDPz8/FBYWsm8GPj4+CAkJwYEDBzB48GAkJSVh/PjxGDNmDMrLy9koCqFQiP3798PPz48tzNDV1YWoqChcvXqVabfU1tZCp9Nh79698PT0RHFxMTw8PPD888/j4MGDLJ3FxcXMIwaA6dOn37Ac5eTk9NKcqaurY7OjXV1d2SIbu3btgr+/P1JTU+Hp6dnvePWWlhaoVCqMGTMGmzdvxpo1a5Cbm4sRI0bAy8sLvr6+TIxNo9FArVZDLBYzobSHHnoIANg5pnSY+v+9vb0RGhrKPizW1dWxMh4bGwsrKyuo1Wrk5+ezehcTEwOBQIDW1lZkZmbCx8cHBQUFrGfAysoKEREReOWVV1BUVAQAfb6xmcoBl8tlgncAcOrUKZSWloLL5cJoNPZqYXp5eSEgIACWlpZISUlBW1sbXFxcIJFIEB0dDZlMxrSoelJTU8Pyo7a2FmFhYcwe9VxUpF8Gsvq/5t9AnvuhQ4eYN5qRkUGfffYZEXVLn3711Ve0ceNGtsjG888/T2vXrmWSpj357rvver0Nr127RitWrOi1UIVJPnflypX0zDPPEBHRn//8Z1q3bh1VV1eTUqmkcePGkUQioePHj9Onn35KlpaW7Pr29nYi6l4cYN68eUREVFdXR//6178oJyeHLl++TEuWLGHn6/V6qqioYN5ITzZv3kxE3R7Cv/71r17h90d1dTUdOXKEiLpbN9nZ2SQSiWjr1q1Mxnj16tX0l7/8hUQiEZ04cYJ++eUXIiJ66qmnmITo9Tz//POk1WpJpVLRsmXLiMPhEFF3a0gul9Pnn39OdnZ2RESkUCioqamJPvjgA7K1tWVhmCRuRSIRnT59moiIdu7cSfn5+WwhiIkTJxIR0ZUrV0itVtOkSZPo6tWrRERkbW3NwnrzzTf7yLRqNBp66623qLKykgoLC+mVV14hg8FAcXFxpNPpSKVSEVG3vCufzyepVMrCEAqFVFpa2msxk1GjRpFQKGTSsiaEQiGTEU5MTKQRI0ZQS0sLtbe395IX7ujoYNKwNjY29P7775NKpaLjx48TERGHwyGFQkFKpZJJND/yyCP01FNPUX5+fi/PzGg00qxZs5jE79ixY9mx6OhoSk9Pp4CAALZPoVCQXq/vtS8mJoauXLlCYrGYOjo62PPoD4VCwSSR29raiKjbO2xra6PCwkLasmULCYVCysvLI2dn5wHDMfHmm29SYWEhKwMpKSksX2tqaujYsWNERBQXF0e5ubnU2tpKb731FhERnTt3jjIyMljZ/Oc//0lqtZotrPLzzz8zWVyhUEiVlZWshXHq1CnKyMggsVhMtbW1rN6ZrlWpVPTRRx+Rk5MTEXVLP1+4cIHkcjnl5eWRWCymhIQEcnV17fe+euZ3VFQUXb58mbq6uggAvfvuu0TUXUauXLlCCoWC8vPzWUu8uLiYtFotzZs3j7q6umjEiBH0ww8/UGlpKSUlJbHFRfbv30+nTp1irTipVEofffQRlZWVUWlpKR08eJDq6+tv6Ln/rlUhP/30U0yZMgUSiQT29vZIS0uDm5sbRowYgb/+9a/YuHEj3njjDQQHB+Pzzz/HnDlzEBIS0mcEw/Hjx+Hu7o4zZ87g7bffRlhYGLy8vJCeng69Xo+rV6/C1dUV9vb22L59O5YsWYKgoCB8/fXXePbZZ+Hl5QU3NzckJCQgODgYI0eOhI+PD1xdXeHm5oasrCw0NjYiNDQU27dvx+LFixEaGorExEQEBQUhICAAQ4cOZfF1dnZCoVAgLCyM9WHLZDJIJBJ4enri2LFjcHBwQEJCAt5++23weDz4+/vjvffe6/f5/fDDD5g/fz54PB7Wr1+PiRMnoqWlBTExMcjPz2eLJYwdOxaBgYFITk6GRqNBRUUFeDweHn300T5hVldXw9fXF2FhYUx90crKCnZ2dhCLxcjPz8f8+fOhVqvh7OwMkUiEyspKLFy4EDqdDm5ubigqKkJ9fT2Cg4Oxbds2PPfcc2hsbMSOHTswbtw4NDY2Ijg4GFlZWQgNDUVgYCA4HA7y8/Ph4+OD0NBQJue7ZcsWzJkzp4/MqYWFBYKCgpCRkYGysjJMmjQJoaGhiIyMxNatW8Hn81FTUwMvLy/weDzU19dDp9OhrKwMQUFBiIuLg62tLebMmYPq6mqUlpZi2LBhbFEOE46OjnB3d4dKpWJ6P4MGDUJERARiYmLYUoyFhYVsgYby8nLY2NjAzs4ODz30ELhcLhwcHCAUCkFEKC0txZAhQ5CYmIiAgAAIBAKMHz8eq1evhp2dHdLT0zFjxgxER0cD6JbtnTJlCk6cOIEVK1Zg+PDhGDp0KFsSLzs7Gw4ODggICIBSqURTUxNycnLg4+OD0aNHw9PTE+Hh4b369nuiUChw7tw5jBo1CvX19eBwOPD19cWIESOQkpICGxsbWFlZYdSoURg/fjyamppQW1uLioqKfhUmN2zYAIPBAJ1Oh/T0dKjVauaN79y5EzY2Nujq6kJHRwcefvhh8Pl8XL58GUD34jCWlpYQiUQQCAT47rvv8Nxzz7GWUGVlJfvgGRwcDFdXVyQnJ4PP5yMxMRF8Ph8dHR0YNWoUMjMzwefzIRKJkJeXh8GDB4OI0NraisjISFy+fBl+fn4YM2YM62s/duwYFi9ezL6rmCAiODs7Q6vVQiwWIyMjg7WE3d3dwefzodfrkZSUBIPBgDFjxqCpqYkpQDo7O2PLli1swZrU1FSEhYUhPDycKbq2tLRAqVTi6aefhq2tLSoqKpCfn4+srCxER0fDwcEBZ8+eRVRU1P1TheRwOLUAFAAMAPREFMPhcFwB7AcQCKAWwJ+J6IaDwwfSlklPT4eHhwdcXFxgY2ODyspK6PV6+Pr6wtPTExcuXEBUVBS8vLyQkpICBwcH1hztSXV1NTOokyZNgkKhgJWVFYqKisDhcBAcHAwnJydwuVycP38ew4YNg5eXF65cuQJ7e3uEh4eDx+MhMTERw4YNY4W8tbUVIpEIPj4+4PF4EAgESEhIYB9nTfKqvr6+sLe3R0lJCdra2hAaGgpbW1s4OTkhPz8fWq0WgYGBsLOzA5/PR0lJCZM8DQoKgpWVFTZu3Ii//e1vfe7NaDQiNzcXw4YNg7W1Na5evcq0qF1cXJCbmwt7e3u4ubmhubkZ4eHhaGtrQ2trKyQSCQYPHtyvLnRnZyfkcjlr+hERmpqaUFNTA2dnZ7i7u8PT0xNNTU1MGtXHxwc2NjYQiUSQSCRwd3fHoEGDYDAYUFhYyJqy+fn5sLOzg6+vL6vQXl5ebLWaixcvYuzYsbC3t0dycjKCg4Px0UcfYd26db1WPzJBREhLS4NAIEBgYCCbun358mUIBAJYW1vDw8MDEokENTU1cHJygqOjIwIDA5GdnQ0Oh4MxY8agq6uLfYjrbwhmfX09mpubMXjwYJSWliI8PBwCgYB9uLWxsYGvry+Tgs7Ly0NrayuGDBnC5IoVCgVqamrg6uoKBwcHeHh4oKSkBK2trYiKioKrqyuTTnZ2dmZdKlqtFitWrGAfrCMjI2FhYYGuri4Wt729Pdzd3aFWqyEUCuHp6clGokycOBEHDx7EtGnTbriIeEZGBtzd3eHq6go7Ozt0dXXB2dmZpcnb2xu+vr7o7OxEbW0tnJyc4O3t3edjqkKhQH5+Pts2rbzl4eEBo9GIRYsWYdWqVWhtbUVwcDAr+0lJSdBqtYiJiYFIJIKjoyNsbGxQWFjIXgxA92iWhoYGeHl5wdnZGTweD9nZ2XBzc4OTkxPa29vh7u4Od3d3lJSUQKFQICgoiBnVlpYWCIVCuLi4sHV6TVr/pg/uPj4+vVaDMtHQ0ACxWAwfHx9cu3YN9vb2GD58ONra2tDS0gJ7e3vY2dmhsrIS48aNA9DddajX6+Ht7Q2VSsWkk/Pz8+Ht7Y3JkyeDx+MhPT0dnp6eGDRoENzc3NDV1YXc3FwMHTqUjZbx8PCAUqmEVCq9oSrkvTDuMUTU2mPfBgDtRLSew+GsAOBCRO/fKByzcNjNkUql/xUCS7dDfX09fvzxR/j6+kKtVuPVV1/9rZP0m7Fz504EBQVhypQp91wF9NeEiPD9999jxYoVvYY0/69iMBhgMBjY0okDfe8ZCA6H86sKhz0OYNp/fu8CkATghsbdzM35XzPsQPcch0GDBrEPb/dat/6PgkQigU6nA4fD+cPfv2kRldmzZ/fRhf9fxMLCAnq9HhwO57YN+824W8/9GgApAAKwjYi+43A4HUTk3OMcKRHd0DKZPXczZgaGiGA0Gv+nPVwz/XM/PfdJRCTicDgeAM5zOJxbFvfgcDhLACwBcNO1AM2Y+V+Gw+GYDbuZ2+auxrkTkeg//1sAHAUwFoCYw+F4A8B//vcrg0ZE3xFRDBHF3OgDz73kblopfyRuJgX6e+H3lB+/p7T8L9JTNuNWxeLM3Jg7Nu4cDseOw+E4mH4DeARAEYATAExL178I4PjdJDA9PR1ffvnlgMeNRiNeeOGFW6qcRqMRCxYswIMPPojCwkJs3br1tpUUTfH0p4fRE7FYfEvqbrejkPf111/3muY8EHV1dXjllVduOdw75fvvv78jGdfy8nJs27YN1dXVeO+997Bv3z6Ul5fjs88+w/bt228pDJMBWLBgAbKysm47DSZ++uknbN++HWfOnBnwnJqaGiQmJiItLQ2JiYnYu3cvEhIS7jjOntTV1d1V+ntiGoJ3+fJlJCYmYt26dX20YG6GSCS66b3pdDo2sc1EWVkZjh49eltxabVafPHFF0hKSsLp06cxevRoXLlyBfX19XjrrbcAdH9Uv1H9/yPS1NSEtLS0uw7nZiqod+O5ewJI43A4+QAyAZwiorMA1gN4mMPhVAJ4+D/bd4xYLEZ9fX2/x9RqNZM0vRX1QAsLC1RUVMDJyQkRERGIjo6+oZzwjYiLi7uh/nZbWxuKi4sHPK7RaGAwGLBr165bli01regO9J011xMLCwvk5eXdUph3immI1p1cV1BQgOjoaISEhKCurg4TJkxAeHg47Ozsblla1jQrsaam5q7Wy7xw4QIcHBwGlGLNyspCYmIi/P39MXnyZIwePRp///vf74lx1+v1bFjcveDw4cPIyMhg46YvXbp0WysdyeVyKBSKm+rK83g8tLS0IDU1le1zc3NjwmO3ikkkLiwsDM7OzigoKMCoUaMQGBjIRMVsbGxuKpP7R0Kj0aClpQX79+9nImImVcjbpT8Bwp7ccZ87EdUA6DM3nojaADzU94o7Y9asWXjiiSfYSImeIyZsbGzQ3t6OBQsW3DQc04rzEokEb7/9NpRKJRPgUSgU/UoX9AeHw2ETGUzTz/vD39+/z+rnPbG2tkZXVxdsbW37HUvbH6mpqUyK9UajDM6dO3fT9RcHQq1W39ISXxYWFvj6668BdLdmtFptvyvcX09bWxuUSiUb/5uZmclEy0aOHHlb6dZoNPh//+//ISoq6pbTfT3+/v549tln+3RlGY1GVFVV4dChQ1i+fDkEAgGICI6OjoiNjcVjjz3W72gPUzm7FSwtLeHr69uviJdWq4WVldVtjRCaP38+SkpKAHTLJk+bNg1Ad/7civPD5/Ph5eWF8PDwfo/3TMulS5cQGxvLjgkEAowdOxYdHR23nIclJSWIjY2Fj48PNm/ejOHDh7OXkUkO4PDhw3e86LdKpbqtF7/pxWRlZQW5XA4+n3/Lz/5W8l0qlaKoqAg5OTlM2sD0QnRycur1fDs7O2FpaQkigrW1Nbhcbp/y1lMxtT9+1+OqysrKsGzZMrz66quYPXs25HI5Tpw4AY1GA61Wi6VLl+LAgQPg8/k4dOgQLly4gK1bt/YbFpfLBRHB1dUVM2fOBJfLxZdffokJEyYww56SksLO7ejowKxZs3D27FkYDAYIhULMnDkTbW1t+OGHH/Dwww+jvb0dnZ2d2LhxIzw9PTFy5Ejk5uZiypQp2L59O6ysrLB9+3YYDAbk5uZCJpOxGZ0eHh745JNP8OSTT6KsrAwRERFIS0uDUCiEr68vVCoVHnnkESQkJKCpqQmDBg2CnZ1dv6uqA90z9oqLi+Hm5oaNGzfi008/BRGhoqICBQUF8PLyglQqRWhoKNra2iAWiyGRSDBp0iTU1NRAqVRi0KBBqK+vR3R0NJuZZzQasWvXLvj4+CAxMRHr169Ha2srTpw4AT8/Pzz88MM4duwYWlpa4OrqCltbW2RlZeEf//hHvx8B/fz8sGjRIhZ2zxek6WW7f/9+WFlZQSgU4uGHH0ZISAjKy8tRUVHBJrVMnjwZ+/btw+TJk9Hc3IytW7fivffe61fBTywWIzMzEzqdDrW1tWwy2PHjx+Ho6Iiamho2gapneVmwYAF6juIyGcjPP/8cTk5OSEhIwPHjx7FixQp0dXVh9+7d+Pjjj5GWlobTp09j/PjxcHJyQmJiIl566SUcPnwYw4YNQ2pqKlavXo3U1FT89NNP+Prrr8Hj8XDx4kUcOnQIa9aswY8//ojLly/jyJEjSEtLQ3V1NVOENBm+65k0aRKefvppvPTSS1i4cCFTI+VwOIiLi4ObmxvS09OxevVq7Nu3DwUFBWyy2JUrVzBnzhysX78ecXFxAIBDhw6htLQUkyZNgkgkwrRp0+Dl5YWTJ0/i73//O44cOdJr+J5UKr2lF7yJnjOjjx07hmeeeYYZ5EWLFkGr1WL37t144IEHkJiYiDNnzmDDhg3QarXIycmBhYUFNBoNOBwOxo8fz3SERowYgdzcXPD5fDz66KPo6OiA0WiEra0tYmJi+rzokpOTcezYMfztb3+Dn58f1q1bh5UrV2LNmjV49dVX4e7ujtOnT4PH46Gurg4vv/wy4uPj0dHRgYaGBkyfPh0//PAD3nvvPaSnp8PFxQVlZWVYsGABK98qlQp1dXWorKzEt99+i4ceegi5ublwcXFBZmYmU8zct28fFi5ciJycHGRnZ+Pxxx9HUVERvL290draioaGBkRFRWHChAk3XYj7dy0cZmNjg7KyMvZGLS8vh0AggLe3N2vGVFRUYPTo0YiIiEBycvINw2tubkZQUBBEIhGOHTuGiIgIJu967tw5HDhwAFOmTIG/vz/a29uRmZmJTZs24bHHHoNAIICrqyuGDx8Oa2trjB07Fq6urigrK0NLSwuSkpIwbdo0yGQyVFVVwcHBgXlRxcXFOHnyJGJjYxEcHIzCwkIEBARApVJh1qxZ8PX1BdCtfc3j8TB48GCmf56YmMjijY6OHtA7MBXsBx54ABqNBkOHDgWHw8Hhw4dRUlKCBx54AFOnTsW5c+fQ0dGBgIAA7NixAwaDAZMmTUJmZib8/PwwcuRI5lUAQFJSEuzt7TF69GjWFdHa2gonJye2fJtp1l99fT0efPBByGSyW9Ki7+rqwpAhQ3o1SWUyGZqamjB37lz2Qu7o6MD333+POXPmYMSIEexlLBQKweVyYWlp2Uey2IRarca+ffsQGBiIKVOmoKCggB2zsLDAn/70pz6G3ZSegboDXVxcIJfL4e3tDR6PBxsbG+Tl5eHMmTMQi8UoKytDVVUVhg4diqlTp8Lf3x87d+7ElStXMHLkSKarLhAIoNPpWHy+vr7Izc2FXq9ns1yBbiPL4/EQGBh4Q5nX5cuXIyYmBl9++SUOHToEnU4Ho9GIzMxMAEBISAgyMjKgUCjg6+uL6upqjBw5EtOnT0d7ezscHR3Zy5GIEBERgYyMDPj5+SEkJASlpaWwtLSEg4MDnJ2d+0zNFwqFsLKyuu1+fqC7m3HatGnM8FpYWKCmpgYtLS0ICAjApEmTcO7cOQDdHvapU6cwYsQIBAQEoLW1FUKhEN7e3ti7dy8CAgIwatQoODg4ID4+HuPHj0dgYCDKysr6fKzt6uqCnZ0dVCoV84oLCwsBAA4ODiAidHZ2MmkLa2trqNVqWFpawtbWFnK5HMOHD0dQUBCOHz+O7OxsxMbGwtPTs9dz0Gg0UKlUcHZ2hqurKyZPngxfX1/odDpYW1ujoqICOp0Op0+fRldXF3Q6HROOO3XqFAQCASIiInDo0CEm8XzTtQcGEp35Nf8GEg7r6uqi119/nW2XlJRQVFQUvf3229TR0UGFhYX0xBNPEBHRrl27yN3dvd9wTGzevJmJMBERxcbGUlNTExmNRho8eDB9+umndOLECUpISCCdTkelpaX01FNPUXR0NC1dupSIiJKSkigmJqZXuNOmTaMzZ86QXq9n+6ZPn86EkcLDw+nHH38kpVLJxJ0uXbrEJE21Wi0dOXKEtmzZQqWlpXTu3DkiIsrJyaEDBw4QEVF8fDyTe/3uu+/oiy++oH/84x9ERCSXy+nFF1+kjo4OUiqV9M9//pOIuoWSYmNjSavVMklWk5zuxYsXKTs7m4iILl26xH5v3ryZ8vLyiIiosLCQxo0bR0REZ8+epeHDhxMRkVQqpVWrVjGxNY1GQ8uXLyelUkkVFRU0derUG+aDiaNHj9KFCxfYtsFgoMWLF1NraytJpVJatGgRO7Zr1y4aPXo0kw2urKyk4OBgiouLYzLE/fH888/TjBkz2PaiRYuYsNfatWtvKKTl7+/fZ59JfEutVlN+fj5Nnz6diIjmz59PL7/8Mnsm1z+DLVu20Pjx42n69Ol0/PhxMhgMVFVV1UsGubq6mknqXr58mQ4ePEjnz5+nL774gqqrq+nMmTNM0vZ6KisriYiYKJqzszNt3bqVzpw5Q+Hh4UTU/XwlEgnJ5XLKz8+nhx9+mO03Go2UlZXVKz0FBQVMkvfChQu0e/duIiJaunQpLV68uFf8CoWC1ZHbQavVkkwmoz//+c/s2Zl4++23WZhSqZTJTCcnJ9O//vUvio+PZwJ4RN0yzibpZqJuyd5Vq1ax80xSvD1Rq9WUnp7ORNmEQiHt2bOHlEol5eTkUG1tLT3++ON06dIlOnjwICuvOp2O1q5dy8TItFothYSE0JkzZ+js2bMUHx/fKx6NRkOlpaV06dIleu2116i4uJikUilVVFTQO++8Q+fPn6eSkhKaN28eVVZWUnFxMV25coU2b95Ma9asoYqKCqqvrycnJyd69913Sa1WU3Jy8g2Fw35zw043MO4bN26knJwcKigooCNHjrCM9vLyop9//plee+01eu6554io25ju3LmTqc5dT3NzM8vA9vZ2kslkTIFOpVKRra1tL13ztrY2euWVV6irq4uSk5NZBZk6dSotXryYCgsLWUHqaYSIiCoqKmjTpk3U1dVFGo2GLC0tqb6+nhkVhUJBIplRtAAAIABJREFUsbGxNHfuXBIKhawC91RnlEql9P7771NdXR0REb344ot07do1Sk5OJr1eT21tbawySCQSeuedd4ioW0kzKyuLLl26RAqFgu2XyWS0efNmOnr0KBER/eMf/2AF8+WXXyaZTEZE3S+8+vp6unz5Mv373/9m2taLFy+mDz74gDIyMkitVlN0dDSJRCK6dOkSpaSksJfARx99RIsXL+5XnbMnGo2GHnvssV4Vuq6ujnx8fIio2/CPHz+eZDIZrVq1ij3rV155hZRKJS1fvpwWL15MHR0dFBgYOKCR5vP59NFHH5HBYKDdu3fTtGnTiP4/e2ce19SV/v9PEsIaIOyLrAKCqLjvuFUR21q3urbVTpexdTrtdK/ttD/b2nVGO77sMtZqN6tttYtrpeIOAoqoLLKEPQQSkhBCErInz+8Pes+AgKC1TjtfP68XLyXcnHvvufec85znPOf9UMfAx7HWe+swuYGws7jcAlwduru7E1EH6//ChQtUUVFBRMTIoBaLhVEGiYj++te/Mgpmeno6bdy4kcrLyxmpUKfTERHRjh07qK6ujv7+9793eS85hnxnFRYW0vLly4mogzaq1+vpzjvvpJycHHr11VeZ0WOz2RiJcvXq1bRixYou5cyYMYPWr19PEomErFYrY/kTES1btoxRVD09PSknJ4cRW4k6+PYcldFisTDGf3+0e/fuHjnzM2fOpMzMTGpra6P33nuPqqurqbKykjZt2kQ1NTWsPTU2NhLRf3IMcHr99depuLj4qkRVu91Ozz33HCOePvroo1RdXU35+fm0Y8cOqqqqopCQEGa4mc1mam9vp1OnTrGBUKFQUFlZGT355JPkdDpJpVJRU1NTl3dSKpXSpUuX6OWXX6Y9e/ZQfX09SSQSOnXqFMXHx5NWq6VTp07RihUrqKamhvLy8hg19vPPP6f6+no6dOgQvfzyy+wdKysr++Py3LOysiAQCNDW1ga9Xo+LFy8CAKZOnYrU1FRIJBJGzKuqqkJUVFSvoXnFxcVobGwE0LHYpNFo2BRUpVIhMTERxcXFsFgsKC0thdFohEQigVarRVVVFW677TYAQGNjI8RiMerq6qDX69HQ0NCNuV5UVISBAweipaUFrq6uGDNmDBoaGmC321nmKJPJBF9fXxYpMXLkSBZdU1JSAqfTiVGjRrGogbNnz6KpqQktLS0QCAQQiURssUgoFCIwMBAOhwMnTpyA2WxmJE1ukfHIkSNQKBS488470dLSgry8PDYFLi8vZ8dVVFSwtYGEhARG+8vPz4fNZoPVaoVWqwURoampCVFRUTh//jxbkCwsLITD4UBLS8tVn21ubi6ys7O7+D/9/PyQkpICk8mEc+fOwWAwoKmpCY2NjWhvb0dTUxNcXV3h6emJvLw8DB48GCaTCREREb2GEy5duhQCgQBWqxVlZWV4/PHHAXT4obk0d71tEFqyZAl754COhXeOpw50kCI5LERrays0Gg1cXFyg1WoxePBgOBwOuLi4dEmvxufzma+5uroasbGxUCgUcHd3h0AggFwuh9lsxpAhQxAREYHZs2czAFdlZWWPi8Yc7ZB7r3JycvD8889jxIgRmDhxInOflJWVseiw2tpaxknn1NDQgCFDhqCmpgZCoRCZmZmMWBodHc0WSr28vBAUFNSlrUkkEkyYMAEOhwNnz57FK6+80mOdXim73Y4TJ070uAjb3t6OhIQEqFQqZGRkQCgUwtfXF3PmzGHthiMuAugWXjhs2DBIJBLodDpG4rxSAoEAPj4+8PLygt1uR2trKxobG1nmrujoaNx1112QyWQwGAy4dOkSzGYzysrKurgluQVRqVSKmpoa5joBuqbIs9vt4PP57N1ub2+HSCSCQqGAQqFgnBmn04nQ0FAMHz4carUaGo0GMpkMaWlpzC3Z5yJ5b73+zfzpzXJvamrqYs0qlUoqLy9nv58+fZpZnLW1tcwq6UnFxcWMa81ZrFVVVcxy1Ov1VF9fT21tbezvGo2GGhoaqLq6uts1cNaeyWRi18BJo9FQeXl5l2lgc3MzqVQq9j29Xt8tA1JdXR1JJJIu1qxUKmUW09VcCHK5nIqLi1n2Hk5arZbVjcViYS6azvXIuWSIOjjbXFYYu93OuOMWi6VL/UqlUubq6cxmr6urY26dq6mwsJBOnjxJEomky+cOh4N9ZjKZmMWkVCrZ/RF1cLG5empvb+82pefkdDpJqVQyrj0RsevmZhvc/fb03ba2ti7uts4yGo3U0NBAGo2GdDodO4fFYun2rLjr71zWxYsXmRXG3W99fT0plcou35VKpczt0pNqamqotraWiouLSSKRsExSHEfdZDKx7EWcysrKut13bm4ulZaWst/T0tJILpdTUVFRl+MuXLhAFy5cYM+iqampi9ujpaWFuXP6kk6no9zcXCorK+s2K6mqqmLtpbW1tUu2J6lUSlKptEsb47KWOZ1OstvtZDQaqbi4mORy+VWvwWq1UlNTEykUCrJYLFRVVdXtmXfuA4g6XMTnz58ni8VCRqOR1Go1nTp1inbu3EknT56kc+fOsfrR6/XU0tLC+pgTJ07Ql19+SQ888AAdPnyYiouLKScnhyoqKuj06dPU0tJCZrOZtFotaTQaOnnyJMnlcpadi6uTviz3X8WWuVG6XraM0+lkCxLXo98SXMQt3FwtPOpawuZ+SxH1L1TuZpXX3/C/6z3PTz/9hKFDh+Kee+7Bli1bkJSU9IcGct2o9+jK+nz00UexZcuWq9azwWDAyZMnMXny5BsOt7PZbODz+Wxm1RdB8ma2J7PZDDc3N7S2tsJisUAoFLL+xOl0wsXFhWVm4/pYrh6VSiXy8vJw/Phx3HvvvYiMjIRIJIKLi0uXaC+73c7qwM3NjbULrpyioiIMHz78plIhb5q4m75e/ZZEuv68ZL+Hjh3ox/TuJpfX3472es9TWVkJh8OB7du3Iz8/HxqNBlOnTr2usn4PulHvEVefRqMR586dw8iRI/scQIkIM2fO/FUbyXrTle2zL77OzWxPnHvMaDRCo9GwVJNXisfjdau/0NBQLFiwAPPmzbvqNbu4uHRpC9z/+/ve/6E791u6pesRt7Ud6AhRu3jxIsRica8N9P+aPD09MX36dLYJ6mrq7+a//1UZDIbr3jD4awejnvZ0dNatzv2W/k9r4MCBaG1tRXl5OcvOdEu31F8FBATccA57f9XnTOYmXcct3dLvUiKRCFOnToVYLMbBgwf7jPK5pVsCOmZ8RISgoCDweDzY7fabTrPsa7PYH6pzt1gstxofOpgcXN5NmUzGfoD/Hi7VZDJdE+ES6MACtLZ2pNe12Wyora1lv99M8Xg8REZGwtvbm4HZ+iO1Wo2ampobdh297Yi92aqvr2fJp/9XdKMDR65cixAIBF3cLBaLBQqF4jd5n7kQzL70u+/cMzMz8fHHHwPoqNCvv/66xwZ44MCBfuFw/+ji4vC5ZNNeXl7w9fXFU089xba+f/XVV9fEdOfqtb8vzZV67bXXsG7dOmzevPmavmexWBhyWSgUYtWqVdi2bdt1XcOvVVJSElatWoUPP/ywX8cTEWQyGZYsWdLnsZ988km/yIYcAO+/HcG2dOlS7Nmz56acS6FQMEzHbykej4dPPvmk7y37AF566aU+j+m838BsNjP20Q8//IDc3Fy0t7ejoaEBr7zyyg3r4J1OJ+x2O3788ccuycd70+++c9+xYwd7+O7u7sjIyGAbazrrzjvvxNtvv32zL++my83NDfv27cOIESPg7+8PkUgEb29vzJo1C/7+/khOTsZ99913TZEkOp0OlZWV/Voc66nj2b17N5YsWdKvRtFZp0+fxsqVK5mVaLfbsWDBgmsq40aJi2ror+XO4/GQlZXVLbKnp6ny119/3a+oLm6B7EZEL/2aAYLH43WBev2Wys/PvylhqK2trZBKpX3CtnQ6He6++240NzdDrVazzyUSCfR6PZRKJdvMZ7PZYLPZIJPJIBKJsG3bNjz88MOIjIyERqNBbGws5s2bB19f32sytjrPvjt/j9v8dPvttyMmJqbPd+p33bm3traitLQU/v7+bPTj2OdNTU1dRuGamhr2ktjtdmg0mn7Bq4COXW6dK9TpdKKlpaULM52bMkulUsjlchARuxZudyInvV4Pk8kE4D/c9c6JDerr66/KY+9NXINtaGjAqFGj4HQ6UVFRAavViiFDhoDP5zPGOp/P72IxKJVKtttVpVKxZCMGgwESiQT+/v79On95eTlzAXW+JofD0Se6WKVSsd2vQId7KSYmhllBsbGxPQ7cPYnbMQh01H9zczOICLW1tT0+d6VS2c1q61wPSqUSWq2WxSZfTQqFAmq1Gs3NzRg9enSXvzU0NLBOgduZ6u3tzSBena+fex6c3N3doVQqu8HBtFptt0GjqakJ9fX1rExuZiCTydDa2soGCO759kcKhQJ6vR6jRo1CQEAA+9xms6GlpaXb9XaWxWJh71tJSUmP7lOj0QidTsf+r1AocOrUqT47KZvNBqVSCZ1Ox+7XarWiubm5yzPt7NZSqVTsd4PBgJqaGvj5+XXrZJuamliZHCxQKpVCpVKhqqoKp0+fRklJCSQSCaqqqtDc3Izq6mo4nU72nri7u0MkEqG1tRUqlQrV1dVoaWmBl5cXBg8eDKCjj2luboZCoejSfrjz6vV6Vr+1tbWoqKiATqdDWVkZysvLQUQwGo0oKSlBWVkZXF1d+xwwftedu8ViwbBhw/DMM8+wDRJcw6mpqcHnn38OoIMJPmPGDHz00UcAgEWLFuH06dPYs2cP3njjjV7LP3jwINavXw+RSIR58+ahpaUFX3zxBd577z3w+Xw8/fTT2L59Ow4cOID6+noMHjwYvr6+yMjIwIcffgiJRIKFCxfCz88Px48fx4ULF/DNN9+gpqYG06dPR01NDU6fPs2sk/Pnz+Ouu+5CUFAQ3njjjX4PPpy4BltcXIzk5GR89tlnICK4urpi8uTJOHHiBF577TW2zfn8+fOYMGECMjIysG3bNqxduxbbt2+Hm5sb9Ho9Dhw4AJFIhNdeew0PPfTQVRdo3njjDWzfvh1hYWHIyspi5Mjq6mrMnz8fcXFxvVqc69atw65du6DRaLB+/Xq4u7vjwIED2L17N1vxP378OB544IF+hYfJZDJIJBKsWbMG+fn58PPzw8qVK3HmzBkMGDAAQ4cOxffffw+ggwf+/vvvg8/n47XXXmOM/RdffBFCoRAbN25EZmYmgoODceTIETzzzDNXrYfvv/8ejY2N8PHxwXvvvcewuhy6d8CAAXjrrbdQXl4OLy8v5OTk4OGHH4bBYACfz8enn36KTZs2QSwWIygoCBUVFQA6BoWRI0fCZDIx15BCocAPP/yA+vp6hgowGo344IMP2KaWzMxMAB1b77/66isEBgbixRdfBNDRRn766SfEx8dfdVZls9nwwQcfwNfXF2+99Raee+459iw/+ugjnD17FlqtlvH7r5TRaMSxY8fwzDPPYPfu3bBarV0GvdLSUuzatQuNjY04dOgQzp49C09PT2RnZyMvLw9hYWE9lmu32yGRSPDhhx/CaDTimWeeAZ/PR11dHXbv3g0vLy/8/PPPqKqqYlv/09LScOzYMVRXVzMuvcFgwKZNm7pkbFu7di0++OADaDQaPPjgg9DpdPjhhx9QUFCADRs2QKlUorKyEk888QR27doFgUCARYsWQSKRMOOOx+NBKBQiJCQELi4uWLx4MR5//HGsWrUKL7zwAv75z38iLCwMFy9exKFDh5Ceng6TyYSPPvoIa9euRV5eHr799ltotVqkpaVh7dq1qK+vR1ZWFl5++WUcO3YMYrEY8+fPh81mQ3V1NS5duoS//OUvMJvNfbqYftedu5ubG0aNGsWmUkSERYsWAQCio6PZxonY2FiIxWLGmdm5cyeMRiMiIyMxd+7cXsvnOitvb28MHToUQqEQn332GeLi4uDn5wdvb2/s378fQ4YMQWxsLBYuXAhfX18sWLAAY8aMQVxcHGt0o0ePRm1tLUQiEQYOHIi4uDiIxWKUlpay+Om8vDzs3r0bHh4eWLx4cb+s5StlNpsxcuRIJCUlobGxkYVhtbe3w8XFBbGxsRAIBBAIBBg0aBAmTJiAOXPm4N5778Xzzz+Pbdu2wcvLCyEhIezaOXxpb1l7nE4nSkpKkJaWBrFYjHHjxrFMT35+fkhMTERveXBtNhvkcjkmT56MoKAg1uCKi4u7WGxnz57tNUlET2Vyz3/o0KEAOhZ0x44dC1dXV6jVarYBZs+ePRCJRAgMDIS7uzu2bdsGq9WK7777Dl5eXlixYgUmTpzI6jAhIeGqluSPP/6I0aNHw9XVFcnJyQgPDwfQsTYUFBQEd3d3hIeHIykpCa6urjCbzUhMTGRc7+zsbAwcOBAikQgJCQmIjY1lM7D58+cjOjoaAoEAarUamZmZmDx5MgICAuDj48Pu85tvvoGrqys8PDwwYcIEaDQa7N27FwMHDgQRISQkBECHy0MoFILP5yM9Pb3L7LGz8vLyEBkZCQ8PDwQGBmLAgAEAOgZuDw8PjBo1CnFxcThw4ECPHYrZbEZcXBz4fD7mzJkDV1dXNvtwOBz47LPPMHPmTCQkJGDSpEkoKCgA0GGoORyOXt87m82GHTt2YN68eYiJicHw4cNhMBjw+eefY/bs2RCJRJg+fTouXrwIlUqFqKgojB07FsOGDUNAQACbyfv5+cFqtSIwMBAuLi5wOByQSCQICwtDe3s7Wltbcfr0aZjNZkRGRiIsLAw+Pj6IjY2FUCiEUChEeHg4+Hw+RCIRq1+uPjmu/IABA7Bw4ULce++9CA0NxaeffgqdTgeTyQSHwwF/f38EBATAYDDg559/xqFDhzBgwADExsbCbrdDJBKBiBAQEAAej4fw8HCEhISwWTe3Jubi4sJ2x15VvXEJbuZPb2yZdevWMa4KEdH3339PSqWStFotrVu3jjFUWlpaaMuWLey4o0eP0r/+9S+Kjo5m+NwrpdfrGc2RY8M4HA5G1yMiGj58OD388MNERPTll192Y8Hs2LGDGhoaSKfT0aZNmygrK4uIOvC8GRkZREQ0depUdnxhYSEdOHCARo0aRfn5+T1eV18qKCigHTt2EFEH++VKBgZHI9TpdLRv374uTBKLxUI1NTX03HPPUVhYGPt8xIgRREQ9Mjjsdjtt3bq1C3/m3XffZfjh48ePd+HUXKlPP/2UER03bdrEuDPx8fH07LPPks1mo+rqaoqOju6VztiTvvzyS7r99tuJqIOdwhEelUol3XfffVRXV0etra10//33s++MGTOGkUUlEgmtX7+e1q5dS2azmZqammjy5MlXPadcLmckxaamJtq8eTPZ7XbSaDS0YsUKstvtVF1dTdOnTyer1UpKpbIL+lcqldKqVavI6XQyumF7ezu1t7ez59bQ0EBDhw6ljz/+mOLj44moAxfMcXCIOvg9r7zyCkVERJDdbqd///vfJBaLiaiDocQxYxQKBW3dupVSUlLYu3mllEolDRkyhEwmEzU1NVFaWhrj9HRGWzc2NrK20JN27NjBEMJr166lBQsWMBaMp6cnEXW0s7///e/0008/EVFH+1q/fn03NhNRBx/m/fffJ19fXyIi1g9s3LiRAgIC2HHvvPMOe7927NjB+oQXX3yRFi5cSEREW7ZsYWjm1tZW2rlzJz333HO0a9cu+tvf/kavv/46/fjjj5SVlUWPPfYYbd68mYqLiykzM5P++c9/0qlTp+jdd9+lxx9/nHGPLBYLqdVqstvtJJfLqby8nIqKikiv11NRURE1NDRQcHAwNTc3k8PhoLvvvps2bNjA6nX16tUUFRVFGo2GJBIJTZgwgQ4cOEByuZw2b95Mc+bModraWsrPz6fp06dTc3Mz2e12mjNnDj3xxBPU0tJCBw4c+ONSIT/55BMEBwfjhx9+gF6vxw8//ICgoCD88MMPOHToEKRSKXJycvD2229jwYIFKCkpwQcffIBRo0bhySefRHJyMvPxXSkutZVKpYLNZkN+fj4qKysREBAAi8WC/Px8PPDAA9iwYQMUCgV2796N2NhY9v2WlhZ88sknCAwMxDfffIPi4mKkpqYC6LBCOWvdy8sLubm5uHDhAh577DGkpqZi3bp1zLL529/+xjJA9aW2tja8++67bDbCWeicDh48iOXLl+PgwYPw9vbGJ5980oViWFxcjAsXLuCpp57qktFp4sSJOH78eI85X51OJ/sBOtwPFy5cYBbzkSNHek2UAXT4nblr/OGHH0BEuHjxItzd3bFy5UpcvHgRe/bsgdPpREFBAXJycvDee+/1mcdzz549uOOOOwB0uGkWLlwIoMMKfeSRR6DX6+Hi4gKhUAir1YrCwkI89NBD+Oijj5Ceno68vDw8+eSTMBqNaG9vx+7du1FYWIjm5uarussCAwNhsVjw5ptvIj09Hbm5uQA60vUJBAJs2rQJdrsdFy5cwPfff4+SkhIYDAa0trbC4XCwXKNvvvkm5syZwyiDnLtxw4YNePzxx5GcnMxcCN9++y0WLlyI8vJyzJ07F1lZWfjrX/+KBQsWQKPRICAggM1uBw4ciKNHj2LdunX485//jD//+c/YtGnTVf2zYWFhsNvtWL9+Pdra2lBYWIiqqipERkYyX/6GDRuuGrCwa9cuzJo1iz3nNWvWYN++fXA4HGzW0tzcjPb2dsyePRtAh9to0aJFOHLkSLfyeDweRCIRmxF6eHhg7969iIqKYp9t2LABZWVlGDduHDsv5749cuQInn76aVRUVODrr79GYGAgDh48iK+++gr79+9HcnIyhg0bhpqaGgQHB6OlpQUqlQqZmZkYNGgQBAIBysvLkZiYiKioKOTl5WHx4sVsLY3H4yEgIAAOhwM+Pj44deoUzp49C5lMhsDAQEilUmzevBne3t7g8/morKzEyJEjkZWVhUWLFmHDhg2w2Wzw8/NDYWEhZsyYgQEDBsDV1RUHDx7EpEmT4Ofnh7y8PNx9992oq6tDWVkZPDw8sGLFCtjtduTk5PT6PIDf+Q7VuXPnoqKiAlFRUbDZbGz6Fh4eDl9f3y5uCKPRCFdXV8yfPx8ymQyXL1/Gn/70J9x11109lu3v788empeXF8LDw5GYmIjg4GAcO3YMMpkMDz30EEQiEerq6rr4BZ1OJ1QqFZRKJSQSCZxOJ1asWMH+PnnyZBQWFkKr1SIwMBB2ux0TJ05Eeno6FAoFampq8Je//AVAx0s4c2b/Us6eO3cOGo0GlZWVSEpK6hbd4uXlBT6fj6ioKABgjYB+8Q+GhISAz+cjPz+fDT7cdFUsFrOMUJ0lFAqxbNky7NmzBzabDe3t7XjkkUcAdCwEFRYWwt3dvVcGyYIFC3DmzBn4+/sjOjoaEokEqampGDx4MEMpl5SUYNCgQdBqtZg8eTL+3//7f7j33nvh5eXVayRFQEAAQ9kqlUrmkhMKhWhvb4dYLIZIJEJcXBwKCwtx6dIlzJ8/HwCwcOFCJCYm4uTJk3jyySfh6+uLlJQUTJs2DdXV1Zg0aVK381mtVoSGhiIgIAAXLlxAUFAQLl++DJFIBD8/P3h6euL48ePw9/dHUlISNBoNJk2ahBEjRkClUiE2NhZ+fn7w8fFBZmYmoqOjcfnyZfj7+8Pf3x+PPPIIsrOzMXXqVMydOxdGoxGLFy9GZWUloqKi4HQ64enpicWLFyM6Ohq5ubkYOXIkgoKCsGDBAsjlcmRlZbGsWIsWLUJYWBiqqqpQVlbGcu9eqaCgICxfvhzZ2dkICQlBYmIiLBYL4uPj8fDDDyMvL49l67raTsygoCCMGTMGDocDwcHB0Ol0GDVqFAQCAe6//34cPXoUra2tWLVqFRvs58yZA5lMhuTk5B7LHD9+PFtX4dw3M2fORH19PY4fPw4vLy88/PDDzD3r4+PD+ggfHx/o9XoMHz4c4eHhMBgMkMvlEIvFSE9Ph9PphEKhwLRp01hmKofDgQEDBjCXjq+vL8RiMQQCARISEtDa2spy6ZrNZgiFQhARbDYbPD09oVar2aBos9kwceJE8Hg86HQ6uLi4oKWlBU6nE3fccQe8vb1x5513Ms6Rp6cnVCoVa9fJycksSKG5uRkhISEsQ5fT6ezy/vemPzQVsjddS1Lha/leZ4rk2rVrUV5ejr179/7q6+xvcue+qHcWi+VXgdSuRdnZ2XA6ndiyZQvLuXmjxO3+62/C6+t93r83/R7v41pJi9wg31/iqtFovCojpTejob9yOp04fvw43N3dER0djcDAQFitVhatExoaygxDLt9DTEwMiAh6vR5xcXGwWq1ob2+Hu7s7vLy84HA4YDAYYDKZEBYWBo1Gg9bWVtTU1MDNzQ1jx45lA47BYEB2djZkMhnmzZsHHo+HoKAgdt/cM+9MvFSpVF3WsAwGA0QiUbe6qqqqQkJCwv8mFbI3XW8D6et7nV/WS5cu9brKf63n628n1lcju1kdO9ARRTF79ux+waWuR/2tE+D6n/fvTb/H+7hWuBXXEfeXuNoX/OpaO3auk2xra4PdbmfWulgsRlhYGAse8PX1hVAoZNayTqdjeWR5PB48PDzA5/Nhs9ng4eHB8Lt6vR4eHh6sY3Y6nTCbzQgLC2Phrnw+n11HY2Mjzpw5Ax8fH+h0Ouba5e6be+ad3atXBidwLrcr66qv96VPy53H430KYC4AJREN/eUzfwDfAogBUAdgKRG1/vK3FwE8BMAB4Aki+vmqJ8CNt9x/azkcDjQ2NkIoFP7qDv6PqubmZgQHB99wXPAt3dKvUVtbGxQKBdtfEBISwnz+3LoDF+qq0+kgFAqZ28Tb25v5yG02G5qbm+Hj4wNvb2+WMQno2MditVrZAMDj8ZhrhSuf47vX19dDLpfDzc0N8fHxcHNz63MjVX9VX1+PmJiYXi33/gzLnwOYc8VnawEcI6IEAMd++R2qaCh+AAAgAElEQVQ8Hi8ZwHIAQ375zkc8Hu/q6LI/oAQCAaKiov7PduwAEBIScqtjv6XflaRSKUpLS2GxWBAYGIjQ0FD4+vrC6XTC4XCwxB+cJW4ymWA0GtHU1MRcI5w1zs082traIJPJWMfN+ddVKhWcTifc3d0Zd50zlF1dXcHj8ZivfuzYsRg/fjwCAgJYuQ6H41djJn41FZKITgO4MnxgPoAvfvn/FwAWdPr8GyKyEFEtgCoA467lgm/plm7plq5FNpsNlZWVMJlMCAkJgb+/P4xGI4tsaW9vh06n68L34Sz10NBQtrvbbrfDarWCz+fDaDSygAVuAxq3FsTn86FWq6HT6WC1WmG1WnvsaLnPOruoONepQCD41cZRX5DA6w2FDCEiOQD88i9Hqx8AoDMaUPbLZ79aer0eCoXiRhR1zTKZTL2GVP7epdfr0dzc/N++jG4yGAx9hjv+t9W5g/hflVar7YY7+LWyWq1wOBy/WZvh3CvcoqbT6URMTAwSExPh7+8Pb29vREdHw9PTEwcOHEBbWxv8/PxgMpmg1+uZG4bzm/v6+sJqteL8+fNsx3BpaSnsdjs+++wzvPvuu6ioqIBAIGBWeXJyMoKCguDq6sqSm3Pumc7XyS0Im83mfsHjrkV9lXej49x7Gop6nHvweLzVPB7vPI/HO69Sqa5aaENDAwoKCpCdnY2jR49elXFxPaqpqbnqi9jc3IzNmzf3C6ebk5PDmDM3UwUFBSgqKur2eWZmJr788subfj1Xk91ux969e7F9+/bfpHybzYYbsYZz9OjR/3nE9LFjx/DZZ5/96nKMRiN777nIk4yMjBuKoC4uLsbp06dx7tw5qFQq2O128Hg8uLq6MutYLBbDx8cHnp6e8Pb2RmRkJGMeeXl5sQ66tbUVJpOJLdwTEcLDwxEcHAyn0wmr1QqDwYCUlBSMGDECBoOhywImF6ZrtVrZPXJlcr8LBAJYrVbI5XLGk+pcH5xbhtt0dDX19PffqnNv5vF4YQDwy7/KXz6XAYjsdFwEgKaeCiCirUQ0hojG9LZ1HQCeffZZHDt2DNOnT8fixYuxa9euG56vUaFQsEWUK2W322E0GvHzzz/3K3KgM8DsZorjil/JRTl37twN6ehupAQCAQoKClBWVvablM8RAH+tCgsLrwsR8UeRw+HAP/7xD3zxxRd9H9yHSktLu5Ak7XZ7N8b5r1FZWRm++uoruLu7M/a+WCyGm5tbF/eGXq+HXC4H0LHRcNiwYRCJRAxsxuPxwOfz4e7uDnd3dwb9czgciImJgVgshtFoxIgRI2A2mzFixAhoNBq4ubl1QS+4ubmxiBhudufn58cWWTlxETZr165FXV1dlzLa29tZh89hCoCOhV4ObcB16hyJsrP66meut+b3A7j/l//fD2Bfp8+X83g8Nx6PFwsgAcC56zwHgI5dqtOmTWNT+IULF4LH43W50f5wyLkR02AwdElE4HQ6kZKSApFI1MU3xk2vXFxcsG/fvn6Fdmm1Wtx3331dVsP7mpV0vrbe1NsIzQ1Gra2tmDdvHlJSUrpxOiQSCV555ZU+r0Gr1bIXinsB9Xp9l2nm1er5SvdFb/AthUIBHo+HvLw8LFmypFtSCO6enE5nF7dNby65K/kaTqcT3t7ejEHU27V3jpjo7dr37t3Lpu6crrSguLpqa2vrkfVxpcFgtVq7ldG5/GuFyV2vZWwymXD48GE8/vjj3ToJrj56Y9Go1Wo0Nf3HZqutrUVGRkYXv/OPP/6I8ePH9/h9vV7f5dl2Rute+Szq6uqQl5eH6upqLF++HGKxGBaLBU1NTVCr1ZBKpYwQW1JSwmLV8/PzUVpaiuzsbLS2trJ3WSaTIS8vDxaLBUqlEq2trTCbzQgMDER5eTlqa2tRUFAAvV4Pg8GAd999F4cPH4ZAIIDdbodarYbBYEBRUREuX74MvV4Pi8WCkpIS6HQ61NfX4/z588zVVV1djRMnTiAyMhJarZYZpk6nEzabDaWlpaiqqoJOp8PFixfR0NAArVaLpqYmlJWVobGxEWazGUajkRE+uXe5L2u/TxOTx+N9DWA6gEAejycDsA7AOwB283i8hwBIASz55WSXeTzebgClAOwAHiOi/oOMe5CXlxfefvttPProoxg1ahTS0tKgVqtx6dIlVFdXY9SoUWhsbERycjIGDRoEuVyO0tJSCAQCBAYGYujQoSgpKUF5eTkGDRoEqVQKg8GAyZMnIzw8HFKpFD/++COefvppAEBubi4D9cyaNQtBQUG4dOkSQkNDkZWVBSLC1KlTe7zW3NxcDBgwACkpKdDpdDh+/DgsFgsmTpzIdo1eqaysLBiNRlgsFgZd6iy5XI6qqio0NjZi+fLlqKurg7u7O0JDQ/HBBx/gqaeeQklJCRoaGnDPPffAZDJBq9WipKQEYrEY7e3tDG5lsVhw+PBh2O12iMVizJo1C0SEQ4cOQaFQICkpCXFxcfj++++RnJwMPp8PrVaLBQsWsEbh5+eH8PBwlJSUYMmSJbBYLDhx4gSADoBbYmIimpqaUFxcDJvNxqBrQAdeNTs7G+Hh4QgKCkJqamqXeHYiQn5+PhobG5GSkoLCwkIEBATAw8MD9fX18PDwwIIFC2Cz2SAQCFBSUoLq6mqIRCKMHTsWZrMZ9fX1yM7OxogRIzBz5kyGxm1sbMTAgQMxatQoNDU14ezZsxCLxazBL1u2jDU8zhLkYF0uLi6or69HcXEx+Hw+IiMjMWzYMFy6dIltUVer1VAoFJgyZQrLw1pYWIiGhga4uLhg+vTpEAqF+Omnn1BZWYnnnnsO+fn5MBgMmDFjBoAO67eyshKurq6YMWMGq5uioiKUlZXB398fI0eOhK+vL8xmMyorK1FXVweRSIS0tLRrWqArKCiATCbD4MGDuxgPZrMZ+/btA5/PR2xsLC5evIjY2FikpaUB6BgM8/LyoNfrERkZyQB2mZmZ8Pb2RkVFBRITE3H+/HnceeedOHnyJOx2O0MTKBQKnD17Fk6nE8OHD8fAgQNx5swZiMVihIaGoqKiAh4eHpg5cybKy8tht9vh6uqK9vZ2HD58GPPmzYPBYMD58+cxadIkNDc3Q6lUYvTo0SxD2YQJEyASiaBSqXDx4kXweDyGdx4+fDhiY2MZvtdkMmHcuHHQ6XTIzMwEEWHcuHE4c+YMmpubceTIEZhMJlRVVbEwycuXL6OmpgYGgwF33XUX1Go1jh8/jkmTJsHHxwcZGRkMXtbY2Ijs7GwEBQWhrq4OqampcDqdMBqNICIUFBSgra0NSUlJaG9vR01NDRISElBfX88WaxctWgSVSgWVSoXy8nKYzWasWrWqzzXI/kTLrCCiMCISElEEEW0nohYimklECb/8q+l0/JtEFEdEiUR0uN9vWy+6fPkynnjiCWzZsoVZkfn5+RCJRDhx4gQsFgsWLFiAe+65BwaDAY899hhmzpyJ0aNHIyMjA0DH6H/o0CG88847mDt3LmbPno0jR47g66+/RmtrK9th6XA48MADD2DAgAGIjo5mI2NjYyOWLVuGKVOmYOXKld2sCyLCkSNH4Obmxvjgd999Nw4fPoxly5ahvb292305HA688cYbcHNzQ3p6OjZu3AhXV9duo/H58+eRkpKCjRs3AgBeeOEF1ulwoV8hISHYtWsX27ixfv16DBkyBCNHjsSKFSvg7++PgwcP4vbbb8eECRNgNBoZz2PTpk0ICAhAVFQUgoODUV5ejurqahw5cgTTp09nNMCCggKMGTMGu3fvRlJSEqZOnYrVq1dj7dq1mDNnDgYOHIjq6mq88847eOGFF5Ceno7XXnsNR48eBdBhsb788stYunQpYmJicOedd3bblJGdnQ2BQIDNmzcjMTERS5cuxdq1azFp0iSsWLECa9asYbHJU6ZMwalTp7Bw4ULW6XKWNodSBcBw0YsXL0Z6ejqOHDmCc+fOYcyYMfjyyy8xa9YsTJgwocsMQigUwmAwMFokALz55psIDQ3FuHHjcOjQIQAda0EzZszAxo0bkZaWhtTUVBw8eBBAR+d59OhRzJw5E1VVVVCr1bhw4QIiIiJYnaxbt465psrKynD8+HHMnz8ftbW1zH/NWZ/Lli1DUVERi6GeN28eDh06hAULFuDYsWP9yjDUWRUVFViwYAEjGnI6fvw4hEIhtmzZgrFjx6KoqAhPPvkkAGDjxo146KGHcPvtt2PFihVYsWIFAgMDkZqaCqVSiU8//RTx8fH4+eefIZPJkJSUhOnTp+Oxxx4D0GHhP/3005g/fz5iY2PR1NSEqqoqDB06FN9++y3jIH333XfIyclhaAe1Wg03Nzfk5uYiMDAQWq0WOTk5sNlsmDJlCrZu3YrvvvsO06dPxzvvvIMTJ07AYDCgpaWFvRupqalYv349du7cieXLl8NisSA6Ohpff/01DAYDtFotxGIxTp48iejoaNTW1sLd3R3Jycl44oknMHLkSJw/fx7vv/8+lEol4xHp9XrU19fD4XAgPz8fbm5uOH/+PB588EFUVlZi0KBBMJlMePbZZ5GWloa2tjbw+XwQEdRqNcLCwnDmzBkEBwczTPjHH3+MlpYWBAQE4NKlS6itrcXGjRths9kQHBzMMkrd7AXVGyqVSgU/Pz8MHToU77//Pvbt24fc3FwMGzYMcXFxCAsLY7AujpfOWQgSiQTp6ekAgNtuuw3Nzc1YtWoVgA5LaNiwYVi8eDFGjRqFQYMGAejwBWu1WjzyyCNQq9UIDu4IAho2bBgmTpyIlpYWzJ49u5vPn8fjYerUqWhoaGCME19fX3z//fd49dVXGbC/s0pLS3H48GF2PDelvXIqPHr0aGg0Gtx2221QKBQszhboYG8kJydDJBLhvvvug6+vL5RKJQYPHozw8HCUlZWxBnPs2DG0tbUhLy8PNpsNK1euBACMGDECb731FqqrqzFo0CDMmDEDzc3NzK1hNpvhdDpx++23QyAQYNq0aYxTc/DgQYZlzcjIwB133IFDhw6xTpHja9jtdjQ2NmLUqFEAOgZbDjfcWTExMYiPj8eQIUMAdFiJXFlmsxnz5s1jA5vT6cTs2bNhsVig1Wrh4+OD1NRUDBkyBFKpFMuXL0dZWRlGjx6NpKQkAB0uj+bmZsyePRtubm6M6TNo0CAGnOKUn5+PGTNmwGg0ora2FiNHjsTIkSNRVlbGgGUzZ86ESqXC/fd3eChrampYspHDhw9jzZo18PDwgFKpREREBOLj41FZWQkvLy92T1OnToXFYsGBAwcwd+5cmM1myGQyhIaGAgCrV6BjFhceHs7YJvHx8fj4449xzz33wNvbG7W1td3qtCc1NDTA6XTizJkzyMnJ6WJ8DBs2DPX19Yz30tDQgGnTpgHogLUNHjyYBTRw7hy9Xo+wsDCEhYVBp9OhqKiIfb+5uRmTJ08G0LFdXiQSYffu3cjKysKECRPYrG7atGksBn3kyJFwc3ODt7c3vLy8EBsbCw8PDyQnJ7MdolqtFkFBQdBqteDxeJgxYwasVitiYmIwZMgQTJ06FfPmzYNMJkNkZCRD97a3t0OtVsPHxwc2mw0GgwFSqRTt7e1oampCaGgo3N3dERYWhuTkZOYykcvlqKioQEZGBqKjo2E0GuHu7g4fHx94eXnBbDZjzJgxCAoKgtPpxJQpU2A2m1FYWAiFQgF3d3fo9XrGo+F2ygoEAhgMBha+GR4eDq1WC6FQiJaWFqSmpqK+vp6FbWq1WgQHB0MoFPY9U+sNF3kzf3pD/j711FNE9B/c53fffcdQs62trbR3714iIjp06BAdP36ctmzZQk1NTaTX6+n5558nrVZLly9fJiKiRx55hKxWKzU2NtKCBQsYZvTo0aNUVFREbW1tDOdJRAwrWl1dTd999x0REW3dupXy8vJIKpX2eL1Lliwhp9NJRUVFDA+8YsUKqq+v73bs2rVrydvbm4g6kMV/+9vfekTuEhGtX7+eGhsb6bPPPqOkpCQiIqqvr6fGxkbSarW0cuVKIurAt27YsIGamppY/alUKiIiCgoKotdff52V2dLSQpWVlXTkyBEiIpowYQLt3LmTiIjuuusuMhqN1NLSQvfeey+1trYSEdGbb75JZrOZlRETE0Nms5mhXeVyOfn4+FBlZSV7BkqlksxmM/3zn/+kuro60mg09Pzzz1NNTU0XHDGnqqoqhkvOzc2lrKwsstvttGXLFqqtrSWpVEpqtZr+/ve/ExHR+fPnKTU1lTQaDStjzZo1ZLVa6cUXX2T3397eTnfddRdptVpWpyaTqcf6Jup4blKplNrb22nTpk1UV1dHREQvvPACqdVq9h6+/PLLZLPZSCqVUnp6OrW2tlJ7ezv5+fmxstLS0shut5PT6aTExERavXo1ERE9++yzVFxcTBUVFRQTE0NERPn5+ZSamkrl5eXU0tLCcLlExJDENTU19OKLL7LPNRoNORwOKisr64al7klcu+Fw0RMmTCAiYs928uTJDMu7cOFCOnv2LDU1NZG7uzsrv7W1lSF1s7KyGHaXiGjcuHHsHDt37qTTp09Tc3Mz/f3vfye9Xs/eF+58mzZtovz8fCooKKC9e/dSQUEBEREZjUaSSqX0008/0Z///GfKzMyk6upqysnJoYcffpj0ej1VVVXRihUrqKCggE6dOkVvvvkmSSQSMpvNJJVKadmyZaRSqSgvL4/uu+8++vTTT2nx4sVUUFBAhw8fpvT0dNq2bRudOnWKbr/9dnrrrbdo//799K9//Yt27txJs2bNoh07dtDFixdpzpw55O7uTkVFRfTUU0/RypUrKSsriyoqKig9PZ3OnDlDEomEJk2aRFu2bKGqqipasmQJzZkzh6qqqkgikTAcM1EHqvrZZ5+lhQsXUmNjIxUUFNDChQvpwIEDpFaryel0UnNzM82aNYv27dtHbW1t9Morr9Dy5cupra2Ntm3b9sdF/p44cQInTpxAdXU1duzYgcbGRoaa/eSTT2C321FYWIjs7GzMmDEDKSkpuHz5Mnbt2oXy8nLs378fUqkUMpkMc+fOZT5PLrQSALZu3QqDwcCm4g0NDaisrMTatWsBdOBWOZ/onj17YLVae0xbJpVKERgYiIKCAmi1Wpw6dQplZWWIiYnp0d+enp6OuXPnori4GDt27ACfz+81Hj0hIYFN811cXJCdnY0zZ84gPDwcu3btQn5+PgoKClBeXo7Zs2cjNzeXLc7s378fxcXF+Pbbb9HS0oLKykp8+umnKCwshEQiwYULF3DhwgUsWrQIM2bMQF1dHSIiIqBUKrFz507Mnz8fYrEYtbW1yMrK6sKvuf322/HJJ5+gqKgIP/74I0QiEe655x5cunQJX375JRwOBzIyMuDm5oYZM2aguLgYO3fuRE1NDc6cOdPjgt0XX3yB0aNHw2AwYPfu3Rg8eDBkMhlOnTqFlpYWNDc3w9vbG06nE+fPn8e///1vmM1mlhlKJpPhvvvug1AoxJo1a/DNN9/gwoUL+Pzzz7F//374+vqiuroaJ0+e7JVfo1KpYLVaceLECXh6emLmzJm4ePEiysrKcPnyZezfvx91dXWoqqrCiRMn4OLigsOHD6O+vp4d8/nnn6O0tBRFRUXMQuPxeBgxYgR8fHywZ88eCIVCnD17FoMGDcJTTz2FixcvYvPmzfD19UVtbS2ICB9//DHy8vJw7NgxDBw4EEAHXri1tRUnTpxAZmYmLl++DLPZjAkTJrAMVD2ppqYGL730EmOXCAQCnD59GgqFApWVlSy7U2hoKMaPH4+amhr4+/uzBcc1a9YgMzMT33zzDT744AOWuP7ixYvQarXIz8+H2WxGcHAwxo4di9LSUnz44YdwOp3w8/PDSy+9hK+++gqFhYU4cOAA8xkfPHgQWq0WbW1tGD16NEsW0t7eDh6PB6lUCqVSCbPZjICAAOh0OsTHx0OlUqGyshJpaWnw8/NDQUEBvL292WItl4Ly6NGjOHfuHB577DEkJSVhzJgxcHFxgd1ux9KlS5Geno6IiAj4+/tjxIgRiI2NxZQpUxAYGIiJEyey5Bz33HMPHnnkEeTk5MBkMmHQoEGoqalBbW0tS86Sm5uL+Ph4DBgwAGKxGAaDAZGRkZBIJBAKhfDw8GCJrtvb21FdXY2FCxdCKBRi0KBBeOyxxyCXy1FXV4fCwkIUFRXh5ZdfRmNjIwoKClBbWws/Pz/IZLI+F98Fr7766lUPuBnaunXrqz0hSVNTUxlfOyUlBWlpaYyiuG3bNkyaNAn19fVYs2YNBAIBIiMjIZVKMW3aNMyaNQvBwcGYMGEC1Go1Ro8eDT6fjyFDhrAsKyEhIRgwYADCwsIQFBSEsLAwSKVSVFVV4f7772d0u5iYGAiFQkRGRiI8PBwpKSnddqT5+vrC29sbI0aMQFxcHAwGA1ss8/Hx6RYSFhISwnjz06ZNQ2BgYK/RBUOHDsWlS5ewevVqzJ49G0qlEiNHjmRZkDhccUpKCkJCQlgGnTlz5iA0NBQpKSmIjY1FVFQUysrKMHXqVAwfPhwRERGIjo5GeXk5Vq1aBbFYjHPnzsHf3x98Ph/Dhg3D+PHjWeYXLy8v5jIBOqbwAoEARITIyEgMGDAAAwcOhMPhwLRp0+Dv74+4uDgEBwdDJBJBrVZj+vTpuO222xAaGtot85LD4QCPx8OQIUNARPDy8kJycjLEYjEGDBiA4OBgxMfHw9XVFW5ublCr1Vi6dCluu+02BAcHw8/PDz/99BPGjh0LT09P5qYym80YNmwYi2H28PCAm5sbwx5fKS8vLyQkJCAsLAyhoaHw9vZm+VCXLVuGwMBATJgwAVqtFgMHDmRscG9vbwwaNAjDhw9HQkIC5HI5i5AYP348eDwehg4dCn9/f1b/ERERzKXR2NiIFStWYPLkyXBxcUFCQgKio6Oh1WqhUCjg4eGBiRMngs/nIyIiAk1NTfD19cWUKVMgFAoRHByMqVOnss7oSsnlcqjVavasgI41nYiICHh6eiI+Ph52ux3h4eFISEhgvPKoqCiEhoZi0KBBjGo4bdo0ht9ISEiAwWCAp6cnIiIi4Ofnh+HDhyMwMJBlNgoNDWUGlNVq7ZIvNyMjg63BCAQCiEQiuLu7s52f3PtjNpvh4+MDmUwGp9OJpKQkGI1G+Pv7M1cKAAQHB4PP5+Pw4cOIioqCn58fxo0bh9jYWPD5fERHRyMvL4+1Q09PT2a0cFmYjEYjY9FzOGyz2Yzw8HCEh4cjNDQUYWFhiI+Ph6+vL8LDwzFgwAB4e3tDp9Mx9xAHKUtMTERoaCg8PDyYO4XH46GyshLJycldEl63tLQwF5OPjw+ioqKYa2fmzJmsr7BardixY4f81Vdf3drT8/5DIn+3b9+Of/zjH2w3WX+wpNeKLv2/qqVLl2L37t3/7cu4Zr3xxhtIS0vDd999hzfffLPX1G03QxqNBt9++y3WrFmDzZs34+GHH+6Cd72aOqNya2trkZubi7vvvhvbt2/HokWL/meYPna7HUVFRSgpKWG+d6FQyGLOAbC1lPz8fFitVsyYMYOlzCssLERMTAwMBgPCw8Ph4uKC2tpatsaUnZ2N1157Dc8++yymTJkCT09PyGQyyGQyxuQfMmQIxGIxPDw8YDAY2DoH/bKmUVpaysKudTodY7lz18CBA4kIDoeDsdabmppYHP3VIGF2ux1ms7lPkBi345fP58PNzY3tei0vL8fgwYP/d5C/drsdZ8+ehb+/P+Me96fTvtWx9y2NRsPCTFNSUv4QdcYN2pcuXYK/vz/S0tJ+NZDpRlzTqVOnEB4ezqxioH9I3yv3Whw5coTx/rnO548uLvxUJpPB398fPj4+7L51Oh1cXV3hdDpZp+fh4cHojSKRCHa7HTExMSxnrU6ng5+fHwICAtg5qqurIRQKUVJSgqSkJIb49fLyglqtxvDhw+Hm5gaxWAwiYouUHGPGxcUFPj4+qKurQ0tLCwYOHAiFQgGTycSgZN7e3ow3w+1n4PP5CAkJgc1m6/N5u7i49IsQeaWhwg3ufbXPP6Tlfku3dEt/XOXm5kIulyMuLg4hISEQi8Vs/UOj0cDd3Z0NiGazGSaTiXFbuA7YZrOhsbERYWFhzN0jFAohEAjg5+fHOkCDwQCn04nGxka4uLjAYrGAz+ezTEeZmZlITEyEzWaDv78/PD09Gc7b6XRCq9Xi3LlzmDdvHurq6uDi4oLk5GRYLBbGmXF3d+/RAu9PEh7OCr8Wcd+prq5GfHz8/47lfku3dEt/TBERzpw5g8DAQAwePBiurq4wGAy4fPkyRo8eDQBddoo3NzejoqICw4YNY3yY6upqGAwGDB8+HHa7HZ6envDw8GBp9Xg8Xhf2OvevyWRCQUEBRowYAT6fz3InJyQksFh/LjOTq6srPD09YbFYEBsbC7vdjoMHD8LhcEAqlSIsLAwmkwkKhQIxMTEIDQ2Fp6cnHA4HSwcI/Ces2eFwsNDJK3e6X4+LjftOX8jfW537Ld3Sb6Rb6zz/kdlsRkVFBcxmM0JDQyEWi6FWq+FwONgeA6fTyTpGpVIJm80Gi8XSBeHg4+PDonpEIhHc3NzQ1tYGh8PB2OpXujE4MioH9eLY60ajkblezGYzi8Ly8fFh5QiFQsawGTFiBFJTUxEZGQm5XA6r1Qq1Ws0WuLlZAheD7nQ6WVjilfmOb4Z+929ea2srduzYgdWrV/+qfKV/JBFRNybNv/71r5t2fr1ej4aGhj6P27Jly1XZJgaDoV9sHU7PPfcc24R2pUwmE+rq6nrkt/Qlo9GI2bNnX/MuzqupvLwcL730ErKysvCnP/0JGzZsQEZGBv7yl7/gwQcfhFQqxcsvv3xTkMFqtbobo6cv2Ww2tLW1YcqUKcjNzf2Nruw/ysnJgU6nY8nLASAwMBDe3t7Q6/UsoYZMJkNJSQmMRiPEYjFLAi2TydjGpYiICNZpcgk2uAVOh8PRrS5qamogl8sxfvx4tgbi4uLCGEzDrXIAACAASURBVO08Hg++vr6IiIhAXV0d5HI59u3bhz179rDw5BkzZsDf359t1Hr99dfx0UcfobS0FDabDeXl5Szaq6WlBQ0NDfDx8WH++xtJxwQ6+sX33nvvqsf87jv3y5cv49ixY4iPj7+hjfP3LIPBgNLS0i6f3cyBzWw249///vdVj3E6ndi7dy+jUfYklUqFt99+u9/nFQgEPTYCLo/lRx99hNzc3GtuKJ6enuDz+TcsvRkAnD17FvPnz2eRGCtXrsScOXMwfvx46PV6REVFISgo6IYTTHvS5s2bcfbs2Wv6jlAohK+vL3NF9AYJuxGSSCSw2+1QKBTw9fVFQ0MDi9EWiURsxy7nsuAscI7X3tDQAIlEgtbWVlRXV7OO3NPTE0KhEC4uLmzBlM/nw2AwdLkfNzc3OJ1OZm1zbhxXV1e4uLggODgYJpMJarUaISEh8PDwYKGoAoGA1VVQUBCEQiG7F5PJhISEBERGRjL4GRclw61lEtFvMoPz9PTsAlzrSb/7zv3IkSPw9fXF888/j/vuu++/fTk3RdnZ2Rg+fHiXz06dOtXjsb/FgvjJkydRXl5+1WP4fD4yMjJY6FlPunz58jVZhceOHcODDz7Y7XPOt1hcXNxrbHpf6onvc70qLy+HRqNBcnIy7HY7MjMzERISAqvVivDwcMybNw8HDhzAPffcc8POeTUdP378uuqlvr4ezz77LGJjY3uM7LgR1mZlZSUaGxuRmJiIuLg4XLp0CVu3bkVBQQGam5vR1taG+Ph41vlFREQgNjYW/v7+CA0NhZubG/h8PiZOnMjcMNXV1airq2OkRG5BValUwsXFBQEBAV06d1dXV4wdOxZisRhNTU1oa2uDwWCAWCyG1WpFaGgoYmJicPLkSeTm5qK2thbjx4+HUChkKAh3d3dcuHABJ06cgNlshlwuxx133IGQkBDU1NQgKioKPB4PRASBQMBQ0RaLBUajEVarlbFguGTbAHqcafQlIkJpaWmvMEJOv2ufu1KpxD/+8Q+sXr0a2dnZMBgM+Pbbb/Hoo4+iqKgIgwcPRmpqKr7++mvI5XI0Nzfj3Xffxfbt2yGXyxEVFYUhQ4bgu+++w1133QWBQIAdO3Zg9OjReOCBB3o85xdffAGn04nKykq89dZbAMB2cxIR4uLiMHr0aGzevBkNDQ2YP38+Ll26hAv/n70zD2+ySvv/N0mbpGnadF/pXmgplaVALaWMICCCiOIy44YjOI7i6Ig/xWXcEORi3MBBRl5EQdkVEKEIZW0prbTQPS1d0yZNmmbfk2bt+f1RnzMttOA2vs47fq/LS5o8z5PzbOfc5z73/blrarB8+XKUlZXhwoULNFbc7XZj+/btCA4OxsWLF/HYY48hMDAQ+/fvR11dHVasWAGNRgOxWIwnn3wSly9fxpNPPolPPvkE0dHRUCgUOHPmDO655x6a5HTmzBn09vbCZrPhgQceoLwVRseOHUNrayu0Wi1uueUWbNu2DVu3bkVPTw/kcjlOnDiBBQsWoKCgAAaDAYWFhairq8MjjzwCHo+Hl19+GVOnTkVtbS1UKhWqq6spw6a1tRWzZs3CJ598gnvuuYeyfRhq5MWLF7F48WIAwCuvvIK5c+eiq6sLKSkpcLlc2LlzJ0aPHo39+/fjnXfeoSREPz8/REZGYuHChVfdk8bGRjQ2NiIiIgKdnZ2Ijo6G0WjE2bNnIRQKUVtbi0cffZQm73zwwQdITk5GR0cHnn/+ecjlcixZsgR1dXUoLi7G6NGjsXDhQuzcuRMCgQBNTU14/fXXv/dzmZmZSXk1g+V2uzFjxgzw+XzceeediIiIQGlpKTo6OvDyyy8DGJhOl5eXg8/nQyaTYenSpdBoNPjmm28gFAoRFxeHGTNmABjg26jVapSWluKWW265ymVls9mwfft2jB8/HrW1tbj55psBDGRSe71etLa24vHHH0dsbCw0Gg2OHj2KwMBAhIeHY86cOSgsLMRtt92GxsZGHDp0CAsWLMDkyZPx7rvvIjY2Fu3t7XjzzTdHvA4HDhxAU1MTzGYzdRF89NFH0Gg0yM7Oxu7du2nWMsP1iYqKwqRJk2CxWNDe3o7k5GQ4nU5a2JrNZsNiscDpdCI/Px+9vb1QKpVob29HVVUVHnroIbS3t+OLL77A0qVL0dLSArfbjSlTpkAul+Py5ctYsmQJRCIRLl68iK+++grLly9HXV0dxGIxpk+fjqamJlgsFvz+97+nz5TJZILb7UZNTQ0ef/xxdHR0QK/XQyqVQqVS4cCBA8jPz0dGRgZ2796N5cuX47bbbsOFCxdQUlKClJQUTJ06FZcuXUJubi6SkpIAgPraz5w5g8bGRtTW1mLDhg203WlpaTCZTDh9+jSN/Ln11lthMBhQXl6O/v5+1NTU4LXXXoPFYkFTUxPcbjd2796Ne++995rP6a/aco+KikJ2djaeeOIJyOVyCIVCnDx5EjfeeCPmzJkDu92OkpISjBkzBitWrKCc47CwMEydOhVSqRSTJ0+mxEKmc1y9evWwv6fT6dDZ2YmlS5cOqaa0a9cueL1ezJ07FwqFAmazmSJmjUYjli1bhvPnz0MoFOKuu+4aYmXv2bMHc+fOxf3334/NmzfDarWivr6eIm0zMzOxYMECfPTRRwgKCsKkSZPg5+eH2bNnIzo6GtHR0Zg2bRouXbpEj1lTU4OMjAy43W5alGIwIU6tVuPGG29EYWEhZs6cifnz5+PgwYPYvn07brrpJjgcDly4cAGEELz33nv44x//SAl6mZmZCA0NxSuvvIKQkBD09fVBrVYjMjISt9xyC5RKJRoaGjBr1izqCmhpacELL7yAe+65Bw0NDSguLsa0adMQHByMhx9+mMKhNm/ejFtvvRU33XQTtFotTCYT3n//fdx///2YNWvWkKo5gzV+/HiEhITg2Wefxe9+9zv4+/tj27ZtkEgkmD9/Pvr7+6FQKAAMhNnl5eXhzjvvpNfGZDIhLi4OAQEBWLBgAS23dvnyZSxYsICSPH+MCCGYNGkSANDMSmDAILjxxhtx7733UqInMECCrKmpwZw5c3DhwgWYTCa88847kEgkuP322yn8q6qqCnv37sXChQths9mGbaNQKERaWhqefPJJ2rEDA4MhQwNlUvxXr16NZcuWYdGiRXQ9hQHzsdlsmgEMDKCZH3roIXR1dY3I5a+uroZUKoW/vz92795Nj5eTk4MPPvgAhBDExMTAbDbj1KlTSExMRFBQEJqbm8HhcBATE4PIyEgIhUJYrVZs27YNH3zwAU0EKisrQ1FRET7++GMIBAJMnjwZVVVVtJ5pbW0tAgICkJOTgwMHDqCrqwvZ2dn48ssv6UASFBSE2tpa8Hg8TJ06FYWFhZDL5UhNTcWRI0dw4cIFWCwWfPDBB6iurkZaWhocDgfq6upgNptht9tx4MABXLp0CVlZWTRxKiAgAJmZmeDz+RCJRAgNDYVcLodMJkNJSQl4PN4QTAcwYK2npaXh0qVLsNvtaGtrg0wmAyEEdXV1EAqFyM3NRW1tLRQKBc6ePQuFQoEpU6ZAoVBAJpPh4sWLMBgMmDx5MiQSyfUXaUeCzvyS/40EDiOEkDvvvJO43W7idDqJw+Egf/rTn+h3zzzzDCkoKKB//+EPfyCEEGK1WsmKFSvo5wyoiRBC8vPzyYsvvjji76Wnp5PExETy7bffEkIIOXbsGHnnnXfIxYsXyY4dOwghhPT395OmpiZy6623EkIIOXz4MG3H7t27KdyLEEJuuukm2qaVK1fSz5944gmyfPly+t1TTz1FCCHE7XbT82D0zjvvEKvVSrq7u8ny5csp8Mrn8xGr1UrWr19P1qxZQ5566inS19dHXC4XEYvFpLi4mEK/YmNjyZo1a8iWLVtIXV0dBW198cUXRCAQkPr6evp7d9xxByGEUMhRfn4+IYTQ3+3r6yPr1q2jf6emppKvvvpqSJtNJhO5++67CSGEWCwW4na7SXp6OiksLCSff/45MRqNRC6Xky+++IIQQsjJkycp5G04MXA0RvPnzydWq5V4vV56HZk25+TkkDFjxhCPx0MIIeT1118nL730EtHr9UOOkZGRQWbPnk0qKioIIf8Caf0QHTlyhJSWllIYFiPmGfP5fLR9crmc5ObmkiNHjpCNGzcSjUZDCCFkw4YNJDMzk9xyyy10/+joaPLuu++Sf/7zn+TcuXMUeMaIaeudd9455PPly5dToNyKFSuIz+cjhBDy7rvv0vMlZAA8N378ePLpp5+SEydODDlGaGgo4XK5pKamhvT39w/5zul0kurqapKUlEQIIeTDDz8kEydOpN8zUDOZTEaam5vJp59+SpYuXUrWr19Pjh8/TrRaLZHJZMRqtRKTyUSkUimRyWQkISGBrF27lshkMtLd3U3OnTtHIiIiyHvvvUdkMhmRyWQkJiaGFBYWkq+++oq88sorRC6Xk2PHjpG//vWv5PLly+TQoUPkscceIxKJhHR1dZHa2lry+OOPE6lUSnbu3ElWrlxJOjs7yc6dO8kzzzxDjhw5QrZs2UI2bNhAenp6yFdffUXefvttcv/995Pz58+Tf/zjH+TQoUOktbWVrFq1ijz//POkurqaPPXUU+TChQtELpeTjo4O8uqrr5Lz58+TzZs3k4kTJ5KWlhZSUVFBTCYT6evrI06nkyiVSnLw4EGybt06Ul9fTxYuXEh6enqIXC4n+/fvJ3w+nwQFBZF9+/aRl19+mSQmJpI1a9aQlStXks8//5xs2bKFpKenE4lEQtrb20leXh4pLy//zwSH+Xw+dHZ2YsmSJfD39wePx4NYLMajjz5KfVQHDhygSNiNGzeiqakJwACboaysjB6LqS3qdruRlJSE5557jlp6jDo7O/HMM8+gvb0dMpmMFu84deoUnnnmGUydOhVLlixBZ2cnWCwWPvzwQ+oGOH78OMWabt26FVu3boVcLodKpaLY4ZKSEjz55JN0RtDY2IhHHnkEbrcb//znP/G3v/0NCoUCX3zxBdatW4c9e/agtrYW+/btQ3l5Obq7u8FisfDNN99Q69BqtUIgEODZZ5/Fq6++io0bN9K6ktu2bcPMmTPpYhUwEI3y5z//GcHBwQgNDcWyZcvAZrNht9txxx13QK/XY8+ePfjggw8AgFp4DAOEEZ/Pp+6G1tZWGAwGzJgxA1arFQqFAi0tLaisrMRzzz0HYGBWIZfLsWzZMixcuBAPP/wwuFwu/v73v+Pee++FRCLBnj17wGazcebMmauehX/84x/UzbVx40Y0Nzdj6tSpEAqF2LNnDzo6OqBQKHDy5Em8/PLLqK6uxoULF/D3v/8dMpkMRUVFWLduHZ588kl4PB6cPXsW69atQ0tLC2bPno1nnnkGACgC9vvKYrFg+/btmDFjxpDwu+LiYsydOxdOpxN///vf8cYbb1Bw1KxZs3D77bfj6aefRkBAAHQ6HTIzM9Hc3IyCggJas5UQgieeeAJPPvkkIiMjR4xpPnToEABgy5YtUCqV+PDDDxEXF4eSkhJcunQJbDYbx44dw/jx49HS0oLc3FxotVqsX78eCQkJWLBgAZYsWUKn/ytWrIDBYIDL5UJeXt5Vcdg8Hg979+6laz0dHR2YN28eampqYDQaKcufySZtbGxEVlYW7r77bkyfPh18Ph+JiYkQCAQQCoUIDg6mvv1bbrkFbDabfubn54eMjAxUVFTg6NGjWLx4MVJTU3Hp0iVMnz4dfX19TCIPRX4vXrwYMpkMycnJqK+vx1133QUul4vg4GBkZGTAaDSiqqqKFsbp7u5GaGgoPB4P+vr6cPr0aRQUFKCvrw8Gg4H60VtbW5Geng6FQoHCwkKMGTMGXq8XMpkMR48eRXBwMEpLS+mMlVkz4/P5NCa+sLAQ2dnZKC4upvVWz549i7KyMjQ2NmLt2rU4ePAgqqurIZfL8dJLL2H16tWIiIjAl19+iba2NgoNmzhx4nUj2n61nTuHw4FMJsPo0aPpZ1VVVYiPj6cv0tSpU+mKsUQiwVtvvQUAKCwspPvYbDaMGjUKwIC7IjAwECwW66ooBofDQae+Op2OkiDz8vJQXl4OYMAHykxza2pq6HS8qqqKJmFotVqEhYVBrVYjNDQUQUFBkEgkOHz4MAghlF4oEAgQGhqKrq4unDhxgsb6Mm30eDyIi4vD22+/DYFAAIPBgJCQEOTm5tI2X7x4cYj7iMmW0+v1dEGUiUCIjo7Gt99+C7VajbNnz8JgMEAikdDEjJUrVyI8PBydnZ3gcDiUpaHT6VBQUACv10sHFYb5YTKZEBgYiJiYGDQ1NSEwMBClpaXwer2oqqqiECWPx4OkpCRK6+vo6KDVrSQSCWpqalBSUgKdTjds6KRaraYsaz6fj/j4eHrex44dg9VqhUajQWtrKw1dq6+vx8KFC1FSUkJdCwKBAGKxGHV1dXTAl0ql1JddXFyMffv2XfX7w8nn80EqleLMmTNXLYg1NTUhLi6OXmuRSAQWi4Xg4GBoNBpYLBZK6Gxra6PnLJPJ6GCcnJyMiooK6HQ61NbWXjXoMJ292WyGy+WCwWBAXFwcbr/9dgADMK7Q0FA0NjaioaEBRqMRvb29MBgMiIyMRGtrK26//Xa43W6MGTMGZWVlEIlE1EBSKBR0neBKJSQk0AXQw4cPIywsDF1dXbDb7aipqUFcXBwNNxw7dix8Ph88Hg+USiW9b2w2GxwOh8aQBwUFQSqVwufzQS6Xg8fjYdSoUTAYDBAKhejt7cWSJUsQFhaGiIgIcLlcWCwW9PT0YNKkSYiOjkZJSQnCwsKGlJ+MiYmB1WpFdXU1EhISEBUVhfLycggEAgoE0+v1sFqt6OjoQH5+PlJTU1FcXIz6+nrY7XacOXMGlZWVcDqd6O7uhlAohMlkgtFohN1up+X1ysrKkJ6eDr1eD7PZDJ1ORyO9mMLdhBCIxWIQQtDc3EzDey0WC5RKJRYvXozf//73tH4EEyaam5uLUaNGwW63Qy6Xw+v10lqxI+lXjR94++238eyzz1LWRGtr61WFL/R6PUUBMw9cR0cH+Hw+7dT1ej1EIhH8/PzQ3t6O4ODgYcl5NpsNOp0O/f39SE1NpSFM7e3tMJvNSEpKorhUiUSCpKQkWoKNWUCRy+VgsVj0t41GI/z9/SEUCtHe3o74+HgIBAJYrVbqMzMajRQ8BAx0OLGxsTTZwmAw0NCt/v5+XL58maZBDyer1QqlUonRo0fTa8LEnLNYLCQmJtKFK7fbDY/HQ4l6zPWLiYlBYGAg7HY72Gz2VVWTmDUQZlDS6XTQaDQ0o08oFFI8KbON1WqldEPm+rS0tNDFSY1GQ/2+g0UIgUqlQlBQEKXq9fT0wGq10mtgMpkgFApht9uh1WoRExNDS635fD7ExMRQHzwTZTD4vvl8Puj1eoSFhX0vBoxGo4HZbKYJNampqXC5XODxeGhubqbPqcPhoFYon8+HVCqFVqtFamoqeDweBAIBWltbQQhBZmYmnE4nBAIB+vr6oNfrwWazaQjecJLJZOByuYiNjUV/fz/cbjetbmSxWEAIgUgkQm9vL3g8HrhcLlwuFywWC8LCwiASiaBQKBAcHEzjshsaGhAZGYmoqKirfMeMtFotHA4HkpKSYLPZ0NPTA7vdjpaWFtx88800/NRms8FsNkMgECA6OnpYoJter4dOp4Ner6eEVpPJBLVaDYlEgszMTCQkJCA4OBg9PT20H4iOjkZrays9JvO+RkdH0/c2Pj4ezc3N6O3txa233oq+vj46yDCE07a2NlgsFtxwww3w+Xyw2WywWq0QCoXIzMyETqfDnDlz8MILLyAhIQEikQgJCQnUu2A0GqFUKnHo0CG88sorsNvtUKlUmDt3LoRCIdhsNkQiEbRaLZqamhATE4POzk4UFxfj+eefh16vh1KpREpKCgghEAgEUKlUtC/Jzs4Gn8+HwWCgxMn6+npEREQgNzd3RPzAr7Jzv3TpEgwGA86dO0cjVr6PrsVy8Pl8tALKLyHyPZkR34cU+L+l/wsZlkyF+u97Lt/3vo0ks9lMa23+mGP9Es+D2+2mHeJgCuVPUU1NDQwGA1JSUhAfHw+Px0MTk5iwwJHkcDhoog8TV240GmEwGGAymZCRkUGZ+CaTCTabjcLGXC4XHdS9Xi+Fh3k8Huj1egQGBtLF0ZiYGLBYLCiVSlitVoSEhND7FBQUBIFAAKlUCoVCgfj4eAQGBiIyMhJarRarV6/Go48+ilGjRiE+Pp4mPz333HO44YYbEBoaCqlUinnz5iE2NpbmDzAUT6fTSevBMvdAoVDA398f4eHhEIlEAAYY9FarFYGBgRg7diy0Wi0CAgIoII3BVavVatjtdmRlZf1nde7V1dWoqanB/PnzqYX3m37Tb/r1qrm5GQ6HA5MnT6ZExJ8ySDKzoCvFIAm4XC64XC4dvK8nr9dLUb6VlZWYOnUqLZzNJLip1WrqmrXZbIiKioJCoUBSUhINt+zt7aVhzWlpaSgvL0dYWBi8Xi/uvfdeGgFjs9kosZap58C0efAA7nK54Ha7h0S+aDQaOoMczkBgBufrIX9/lZ37b/pNv+nXob/97W8wmUx46623RrTACSFob29HSEgIIiIiftbZ3rU6b5vNhvb2drr2NZI8Hg+8Xi9lotvtdnA4HBp8IBQKERAQgObmZni9XgQFBaGlpQVmsxnjxo0Dm81GREQEent74XA4wGazodfrIRQKaaEQg8FA6+Xm5ORQhs5wHXNfXx/tzCsqKjB58uQR3V/XUmNjI2644YbfqJC/6Tf9ph8mg8EAjUYDnU6HpqYmmlx1pbxeL8LCwhAaGgqHw/GzYh6YznE4C5bD4YxYdWqwmOLYzc3NCAoKQnh4OPz9/eF2u9Hf30871oSEBHC5XLjdbmRlZcFmsyE2NhYGg4GuRURHR0MsFtNEPYlEQtd7mPJ9DMRsuJkLM/CR77g4TPFuq9VKLfvh9iPfYQw4HA51P1/PML9u585isbYBWAhAQwjJ/u6zVQAeA8CENvyNEHLsu+9eBvAoAB+AvxJCTlzvN37Tb/pv1ff1y19rO71eTyNPfk6FhYVRK3Tq1KnDbmOz2eByuRAeHk4LV/wUXbkGMLhzH/y3Xq9HSEgI4uLi6PdM5zecmNqwzOI+E3QhEongdDqHFAdh/Ok8Ho/igxsaGlBbW4uCggKMHj0afD4fcXFxSE1Nhcfjgc/ng0qlAiEEYWFhQ+6V2+2G1+ulawsBAQGwWCzg8/l0psOEe6tUKkRGRqKvr4/655kOnzk3Zl3xes/N97HcPwOwCcCOKz7fQAh5b/AHLBYrC8B9AMYBiANwmsVijSGE+L7H7/wqZLPZ4HQ6ERISctXClslkokVvf07r5JeWw+Gg1L1fYjFXp9NBIBBcFXFzpQgh0Ol0CAkJ+cmLfIM7nV9aLpcLUqkUIpHomtWTfD4f3G43NBoNrFYrLf4+nEZ6kY1GI7q7u9Hf349Ro0YhIiLie/mgv6/uuececDicEQMVOBwOeDweRdza7XYEBQXB4XDA6XTSyKZrnduVxxss5rwZRsvFixeRkpJCQxz9/PwQGBgIp9M5JLz5SneOn58f7dgB0BBMk8lEF9uZd9pkMqGxsZEyZbxeL3p6evDXv/4VQqEQnZ2dQ94bBvPLRE4Nts7dbjdYLBblvTN1mQEMaS/TmZvNZnA4HAQEBAxZ+B5O13tHruscI4SUArh2me1/6Q4A+wghLkJIF4AOALnX2edXI5vNhk2bNmH79u3DVhbftm0b2traIBQK8cEHH4yIp/2xYmo13nfffdfcbt26dTh37hy8Xi+sVivef//9q5KyriWBQICdO3eio6Pjpzb5mmJwt/Pnz8e2bduuuz2LxcLGjRt/FkyuzWbDli1bRkyf/3eJ8e3u3LkTFotlxG3eeustcDgcirR+9NFHv9fxr7SM2Ww2/t//+3/YvHkzoqOjvzdm2eVyoaamBm+99dY1E7eioqJGHCAZBgzTKSqVStTW1sLr9cLf3x96vR4ejwePPfbYsPfU6/VCKpUOOaeR/PVM5/nhhx+isLAQNpuN1ihl8jsY8Ny5c+eG4DhGOp5arcbhw4dht9vR3NyMkydPwmaz4dtvv8WqVavQ3NwMl8sFn8+HyZMnIz4+HhEREcjKykJUVBRFdtTV1dFZw2CQntlshslkgtPpRFdXF7hcLkVUDB5ogH8NahkZGRTLIBQKweVyKVaFyadgasxeb5b0U1Y+nmKxWA0sFmsbi8ViWhoPYHDalOK7z36y/p0Lv8zF6u3txfTp07Fy5cph462PHj2KMWPGwGq1YsWKFTh9+vTP2g5/f38EBwdj3759w5LimEy+4uJipKWlgRACf39/PPfccz8oqshms+HYsWPDwq9+qpg44cHy8/PDrbfeet19mdjfK0FoP0bFxcW48847f1Ch7J9KQWQKHgsEApw/fx5JSUnD8uetVivtgIODg3H+/Hnk5+cP2ebKgeFKt4TL5UJvby+sVit0Oh29vgz6trGxETKZDGazGVqtlhar8Hg8cDqd4PF4yMnJwauvvvqDZ6Ferxdmsxlms5kmXfX19VH0bm9vL0pLSxEeHg61Wo1p06bBarWir6+PPhvMtU5OTqZhk8y18ng8Q4wVxuI1Go3QarUYM2YMwsPDh8wmWCwWQkJCsH79emRmZkIgEFzVZzB/e71eiEQipKenY86cObDZbOju7qa0Tz8/P0yfPh033ngjDa8MDAxEb28vZDIZTpw4QTnuWVlZSE9Pp9niLpeLEin7+/sREBAANptN3TxMgpPVaoXX64VCoYBer4fBYKB5GIPb6/V6odPpsGvXLlgsFlgsFnA4HHovr6Uf27lvBpAGYCKAXgAMGWm4ueOwvTKLxfozi8WqYrFYVSNZGh6Phz7kTBXywdJqtcNaHUaj8boWG/muIIbX6wWHw4FSqcTZs2eHvWDMzWLaabfboVar6eeDucrDoclCZQAAIABJREFUWQw2m21IO5lQqSvFvLCDs0FtNhvNutRoNJSNrVar4e/vD6VSCY1Gc9WxmOy44c5bIpHQl8fhcND9mfPT6/WQyWR0n8GdHpM2zYhJl2eOzeFw4PP54HK5EBAQAIfDQWOfGTkcDpqtyqi7uxutra1XWTSDj3/l9tfiWavVatohDJZWqx3y24NfqMHPTH9/PxwOB01gAwaex87OTvT19aGxsRFarZZeG6bqTmlpKaRSKa0gdKUlKpFIcPbsWVpiTigUwmAwIDk5GWVlZUM6IJfLBb1ej/r6evT09Aw5Do/Hg5+fH43PTkxMpK4Lo9FIUbtMslpbWxsuX75MkbVarZZmSw/WlffF6XRCo9HQa2Oz2VBUVASxWAyVSkUBWIwFn5iYCK/Xi8jISCiVSpSVleGmm26ii4BdXV1oaWmhdENgAA/S09ODnp4eNDc306xMtVqNmpoaynJvb29Heno6TR5kxLBUNBoNuru70d7eDrFYjPLycpjNZtTW1qKiogJ2u51eX2ZAiYqKQnd3N81AbW9vR2JiIlQqFc1i9nq9CA0NhUAgoJnsDocDgYGB1DBkjAir1YqGhgbI5XJ6ThqNBkFBQejr60NzczMsFgs0Gg36+vrQ29uL3t5eupBrMBjQ1dWFzs5OOgAEBATA6XRCLBZDr9fDZrNBqVQOeUeH04/q3AkhakKIjxDSD2Ar/uV6UQBIGLTpKADKEY7xMSFkCiFkCpP1eaXEYjFmz56NTZs2ARgg6n23L959911ERkaiv78fhw8fhslkwqZNm3D58mWcPHkSS5YsGant2LVrF8RiMfr6+rB8+XI6OovFYkyYMOGqgYF5kJKSkiCTydDc3Izc3FwUFRWhuLgYp0+fxrRp06BWq/HGG28MCc3atGkTVCoVhEIhpk2bhvPnz6O2thYPPvggNBoNTp8+TaMQ6uvrcf78eezfvx8AUFdXh66uLkRHR2PlypWIiYmBQqHA2rVrkZOTQ7fPyckBMNDBFBYWoqSkBP39/cPy71ksFr7++mtkZGRAp9Ohvb0dGzZsQGVlJaqrq5GdnY3w8HBYrVbKmGGz2Xjvvfewa9cuCAQClJaW0nTr+vp6in1gEKRGoxE7dgws0ZSXl2PZsmUICAiAXC7H8ePHweVy8frrr+Ozzz5DbW0tTZV//fXXqV++uroar776Kng8Hm688Ua89tprAICPP/4YDQ0N+Pbbb4dNj3e73WhpacGGDRsQGxuLNWvW4KWXXoJYLMbx48fR39+PN998E3v37sXBgwexevVqWu3n5MmTOH/+PGw2G/bt24eqqip4vV4sX74c7e3tOH36NN566y2cOHECo0ePRk5ODj766CPYbDacOnUKf/vb35CWlgaVSkUzUJnkFObZi4iIQFFREV555RVqXfb09GD06NGQSqX4y1/+gvb2dmi1Wrz//vuoqqrCN998g+eff/6qcxWJRDhz5gzWrl2L7OxsSlesq6uDz+fDkSNHYLfbodfrUVtbiy+//BIdHR1Yv349du7ciSVLlqCxsRE2mw1lZWVYs2YN6urqsGDBAnz++efYt28fWltbER4ejpUrV1JURk1NDZYuXYq6ujq8/vrryM3NpVzy+Ph42iF5PB4UFRUhMzMTDocD5eXlqK+vh0KhwHvvvYfOzk6cP38er7/+OphQ6IcffhiNjY3YtWsXLl68CADYvn07uru7advi4+OvWnRtb2/HhQsXUFlZiZCQEDQ2NuKf//wnjh8/jsrKStx7771QqVSQSqXo6uqCXC5HfX09lEolLl++jNraWjzwwAPIyMjAp59+iqysLISEhGD+/Pl44YUXKEZk37590Ov1OHfuHLZt2wYul4vw8HBaIrCpqYmSaw0GA95//33s2bMHGzduxK5duyASiVBeXg4ul4v169eDxWKhsLCQDnarVq2ig9+GDRtgt9tx5MgRyuNZuXIljh49is7OTtpPjKQf1bmzWKzYQX8uBsCYAEcA3MdisXgsFisFwGgAF3/MbwBAYGAgDV0yGo0oKSkBMGDNHT58GH19fQgODsYdd9yBoqIifPbZZzQZISEhYdhjlpeX4/PPP0d6ejpNQ1epVDQ1ODw8fMQU6VmzZkEulyMxMRGZmZmIjY0Fh8OhLHLG5zl4cDh58iT1w4WHhyM2NhYCgYBOKxsaGugMICwsDHFxcRAKhTT0qr29HQDw5z//GcBAunlqaipYLBb6+vqQlpaGMWPGABjohFevXo2ZM2ciKiqK8m6uVHV1NSwWC3g8HiZMmIA//OEPAIDU1FS6T3h4OLXivF4vLl26hJkzZ4LH42HcuHEoLi5GZGQkYmNjh2AemPvGpHZLJBIarrZ69WrExsbC7XaDw+EgKSkJ27dvx+TJkxEUFAQ+n4+ZM2cCGMADSyQSCIVCOJ1OREdHY//+/XjttdcQExNDq/pcKS6XC7FYjJSUFBo9cubMGWzZsgXR0dG0yj2DRYiMjKTx29XV1QgKCsLp06exfv165OTkYOzYsfDz80NzczMtFjFhwgTweDyKlvDz88PBgwepy8Dj8WDOnDlX+b6ZNRWZTAaRSESt6ptvvhmxsbFIS0tDUFAQQkNDUVlZicOHD9NiFampqUOOxaT1y+Vy2rkcPXoUmzZtQlBQEOx2O2w2GxwOByIiIsBisRAfH4/c3Fzk5ORQRovJZKLAsYCAAISFhSEjIwMejwcff/wxxo8fDw6HAy6Xi+bmZrDZbBo+OGnSJMTFxVFsgcPhQG9vL0JDQ8Hn86klLRKJ0NHRgbVr11IWS0JCAs28ZEB2kZGRmDJlClwuF/bv309nfrGxseByuVCpVDRz1OPxwOFw0Jm9zWZDUFAQoqOjIRAIEBkZCZ/Ph+TkZMTFxVFXCFMc22Qyoa+vj7KYQkJCqA/fbrdTH7tIJAKfz0dLSwvefvttfPbZZ+ByudTVwoipmwqAPpfx8fF44IEHkJqaCrFYjMDAQEgkEvj7++PChQvw9/dHdHQ0MjIyEBgYCLFYjNDQUIwaNQoxMTHYu3cvamtrcfToUVohKjAwEH19fTRB6poaCRfJ/AdgLwZcLx4MWOaPAtgJQAygAQMdeuyg7V8BIAHQCmD+9Y5ProH8dbvd5P777yeEEPKPf/yD5OXl0e+amprISy+9RNhsNiFkAFP61FNPEZ1ON+yxGKWnp5NHHnmEEEKIy+Ui2dnZxOfzkaKiIjJnzpwR93vzzTdJS0sL/XvNmjX03/fccw9Zt24dIYSQ8ePHk88++4wQMoB7XbhwISGEEIPBQLZt20YIIeT06dOksrKSEDKA0129ejVxu93EarWSxYsX03O3WCzk888/J7NmzSKHDx8mUqmU5OfnD8Gw3nnnneSNN94ghAwgZZcsWUI8Hg9FvQ6nKVOmkKKiIjJ//nxy8eJF+vmBAwdIQ0MDIYSQ1atXE7PZTAgh5NNPPyXFxcV0u+3bt5OzZ8+Svr4+ivUlhNB71dDQQJRKJens7CRZWVn0+/j4eEIIIa2trfQzt9tN/z1r1iyi0+mI0+kkAEhJSQkhhJDp06cTt9tNEhMTycsvv0zkcvmI50YIITNnziSPPvooIYSQcePGkaVLlxIej0e8Xi8Ri8VDthWJRGT//v2ktraWLFq0iPh8PpKdnU0efPBB4vF4iFQqJenp6USlUhFC/oU/drlcpKCggEgkEtLT00O4XC45dOgQ0Wq1ZOvWraS8vPyqdnm9XrJ27VqSlpZGfD4fcblc5JNPPiGXLl0ihAzgegsLC4lWqyWxsbHkvvvuI3K5nBiNRtLZ2TnkWBqNhmi1WpKVlUV0Oh3RarVkypQp5PbbbyeHDx8mbW1tRK1Wk/b2diKRSEhubi4pKioiYrGYSCQSsmLFCpKfn0/Onj1LLl26RIRCIamtrSUlJSWkurqa/M///A9ZuHAhsVqtxOl0EpFIRJqamkhLSwuZP38+WbduHVGr1WTMmDHk2WefJXV1daStrY3Y7Xbi8XiIXq8nL7zwAklLSyMSiYS8+eabJCgoiEgkEqLT6YjFYiFWq5UYjUaSnJxMVCoV6ezsJIWFhWT37t1k8eLFpLKykrS3t5Pu7m4il8tJZmYm6enpIUajkdjtdtLf30+sViuxWCxEo9GQO+64g9TX1xOVSkU2bNhAZs+eTXp6esg777xDFi9eTLxeL5HL5USj0RC9Xk9kMhmxWCxkypQp5MUXXyQdHR2ku7ubzJo1izQ1NZE9e/aQ/Px8smXLFrJv3z4yZswY8vXXX5O6ujpSV1dHqqqqiF6vJ2azmbhcLtqOvLw8smrVKqLRaIjZbCbPPvssefvtt0llZSVRKpWkoaGBTJkyhcjlcuJwOIharSanTp0i06dPJzU1NUQul5Nz586R2267jXz44Yf0uTxz5gzZsWMH6e7uJtu2bSO7d+/+achfQsj9hJBYQog/IWQUIeRTQsgSQsgNhJDxhJBFhJDeQduvJYSkEUIyCCHHr3f8a8lisVCrb8+ePViwYAHq6+vxwAMPQKfTYfXq1fj9738PALjjjjsQFBQEDoeDyspK1NXVDXvMrKwsamk+99xzWLlyJdhsNl2kHE5yuZxWaOro6MCyZcvwyCOP0GljR0cHHnvsMSiVSkyYMAFz585FX18f2Gw2CgoKQAjBG2+8Qas/NTc301kDm83GwoULIZfLMW/ePFitVlRUVGDHjh145pln8PDDD2P58uWIjIxEYWEhFAoFdDodPvnkE9hsNng8Hjz99NPo6urCqFGjwGKxKKdjOHyuQqHA/PnzMW/ePGi1WlRXV2Pr1q2w2Wz48MMPccMNN2DLli3Ys2cPLBYL9u7dOyRiYs+ePaioqMCsWbNo4RHmnJgwsdbWVsTGxmLHjh2w2Ww4efIkmpub6b0MDQ3Fvn370NXVRembe/fuxc0330yJgHl5eUhMTMSOHTvw0EMPwd/fH9OmTYPD4cCoUaNQWlo64oK2wWAAm81GR0cH7rrrLmzbtg3x8fHgcDiIiIjAoUOHqBsgODgYU6dOxZdffgk+nw82m40bbrgBfn5+6Ovrw/vvv48//elPtJza1KlTKab5xRdfhNvtRnR0NIKDg5Geno6TJ0/ik08+GTZ0kMPhYOPGjSgoKMCJEycgkUhw8OBBTJw4EcDAgn1/fz/Ky8uxcOFCGjcuFosprZER43u9fPkyqqqqIJfLkZaWBg6Hg3HjxkGtVqOsrAyffPIJenp6YDAY4PP5UFFRAUIIjh07hj/96U8wGAzw9/dHdnY21Go1TVoSCAS0ePW5c+ewYsUKcLlcBAYGorW1FaNGjUJjYyP4fD5mzJhBC1QIBAK6kPjll18iMzMT9fX1mDhxIiZMmEDXGi5dugSTyUSLZpvNZrS2tqKgoAB5eXmIiYlBXFwcvY9OpxOEDJSXk0qlNLSQy+VCr9fT46nVang8Hpw5cwZ5eXlgs9nYsWMH8vPzIRaLER0dTZHGzPnI5XLMmTMHlZWVOH/+PAW4tbW1YfHixRg7dixiYmKQmpoKf39/8Pl8WK1W+kwFBwdDLBZT4iePx8OMGTNgsVhgNBoRFxeHqKgoxMXF0cgZZnZlt9vR09MDPz8/Cozr7e1FcXExPv74Y8ybNw9xcXGorq7GF198gSlTpiAmJgZFRUXXLbPHWbVq1TU3+CX08ccfr2LcDoMlEAiwZ88eOgXmcDiYOXMm+Hw+XC4Xjh49ioCAAMyZMwexsbGQyWTg8/nweDwYO3bsVS9Yf38/goODIZVK0d/fj5iYGMyfP58WXx43bhwtGzdY/v7+6O7uplPoqqoqOuVPTExEXV0dbr31VvD5fGzZsgVjx45Famoq2Gw2Tp06BZ/Ph7Vr11K+udlspthPhvA4Z84clJaWIj09HVlZWYiOjqbRBUwnxcTfZmVlQSQSISUlBRcvXkRCQgLS0tLg7+9POwGz2YysrKyrrkFXVxdCQkIwevRotLS0QCwW4+6776Y+2kceeQQqlQotLS2YMGEC4uPjkZ2djVOnTkGj0cDn8+G2225DVFQUzZJTq9WwWq2orKxEYmIiwsLCkJiYCL1eD61Wi0mTJmH8+PE0dNNsNiM0NBTZ2dno7e2l7glmIXLy5MlgsVhQqVQoLS3FokWLEBcXh7CwMOrS8Hg8mDx58lXnxySrqNVqsNlsLF68mEKmdDodrFYrwsLCkJ6ejoCAAFy6dAmxsbF49913MXXqVMybNw8ikYguykdERODuu++GQCBAT08PAgMDkZaWBo1GA7VaDT8/Pxot4fF4KA3R5/MNmxZfV1cHm82GvLw8JCUloaioCPfddx/6+/vxzTffID8/H1FRUUhNTYVGo0FAQAC4XC4iIiIouZOJeBEIBKirq8OUKVOQmppKa5GGhYVBKBTC398fJSUllCiakJCAmJgYjB49GkVFRcjIyMDEiRMRFBQEf39/migVGxuL8PBwmmrvcDgwYcIEpKWlwWw24+LFi7jjjjsQEBCACxcuYPz48Zg6dSp8Ph+EQiFN7ZfL5QgMDMSMGTOQlJQEPp8Po9GIqKgocDgcBAUFob+/H+fPn0dGRgYEAgFGjRoFHo+Hzs5OpKen0+zXuLg4HD9+HHl5eUhOTqbYbiZjU6vVQqvVoqCggFZsmz9/PiIjI3Hq1CnExsYiJycHAQEBtJKZn58fOBwOTpw4gbS0NPpMM+AuJhAiMTER6enpcLvdCAwMRFRUFAQCAeLj42l2q9frBY/Hg0wmg0wmw8yZM2mYpr+/P63vyiQupaSkQCqVUkRxWloawsPDaTasSCSiVankcjmsVisOHjyIBx98ECEhIfj6669x5513YuPGjb2rVq36eLh+9T+CLTMS0Y95GH+KmBCtZ555Bhs2bLhm0skPlUKhwKhRo3Dq1Cls2bIFBw4cADAQNnYlT34kMYlTjPr7+8FisYac9w8lCZLvQqyGS4K4VuLE9SBNP/Z+XHlcnU4Hp9OJqKgovPrqq1i7du2PTmq6FvWwsLCQWkKrVq3CfffddxVS+tcqnU4Hr9dLE76YDpWZtbHZbLq4zPj+Z8+eTQdYpmMbnFjG7Nfd3Q2v1wuLxYLY2FgEBgbCZDJh1KhR8Hg8aGlpQUBAAFJSUtDX10dDKZ1OJ/z8/OizSL5L4hnMTRn8Lvf19dH6qcnJyVedI/NcMP+32Wy0HsOVx2PCCVNSUuB2u7Fr1y64XC7k5uYiPj4eTqcTHA4HBw8eRE5ODnJzc+HxeGAwGOBwOJCZmUkLbTNJYQkJCWhpaYHP54PVaoXdbofD4UBOTg7YbDYyMzPhdrtp5qlQKIRKpaIDGBPRxOfzYTabIRKJhrxfg99bs9kMNptNmTOEEDQ1NcFoNCI9PR2nTp3C119/jU8++QSEEBqgkJmZ+Z/Nlhlp4eCnduzAQMjczp07cdddd/3sHfvrr7+OtWvX0ukmox8CCboypXy4a/FDs0wZa2I4XSsu/HqZjz/2flx53IqKCthsNtx0003XbOv30bX2bWlpQXBwMORyOW677bb/mI4dAGX/D9aVzwGXy0VqaioiIiJoUh4Ti848R4Ppiz09PYiPj6f8fvIdW5xJpgFAOzXmujIdOwPnYrPZtNNirNnBGvz8MqGcI70PzHPB/H+4eHzmeExxD0II9Ho99u7dC4/Hg9mzZ9NFYKaOaUNDAwoKCuDz+aDRaKBQKBAdHU1/h+lgGRdTf38/DStms9loa2tDTEwMxYiz2Wya7Wo2m2m9XpvNhoCAAPT399NF1sHv1+CBWCQS0QQ4DocDj8cDrVaLL774An/5y1+g0+mQm5uLsLAwOBwOCASC67+P/wmW+79bPxfT+koxsc7/yaiC/w11dnZCr9eDx+MhMTHxZ2emDJbD4aBgqf+LYuL1GYu3v78fVqt12EijwYVTmDyMK59do9F4VS4CU0AGwPfCDPwcM+6R5PV6odFoUFFRgeTkZIwZMwY8Ho+CwpxOJxoaGhAcHIyQkBBcvnwZSqUSUVFRyM/PB5fLpS692NhYmkshEomoC6+2thZnz57FokWLcNttt8Hn88HPzw96vZ7ifRlO/GAC5A8VUykNAOLi4q6atXxXoew35O9v+k3/jdJoNGCxWLSCmNfrZTjgMJlMtGxcQkIC+Hw+3G439b8zLsve3l5aHYnD4SAjIwN2ux3Av6xck8kEgUAwJCTwl5bH46EQr8EdKpNYyMwQ1Go1bDYbIiIiaIUvphwkU/wjNjaWlgdkQGANDQ1gs9mU/xIXF0cXZn0+H13QVigUiImJgZ+fHy3a8e+QwWBAeHj4iJ37f3aZnd/0m37TNRUUFDTE+vbz80N3dze6u7upS4GJSwcGXDNer5eyUgwGA6xWK4KDgxEVFTXEJzzYX8/n82kn90tLo9HQyDGGx36lGLeK1+ul1Y18Ph8yMjLo2oXZbKaZtlwuF6GhodTX7+fnh+zsbCQlJSE6OhoFBQVITU2lQLyYmBjqJmFKejLlJv+39B/hc/8ldD0C2/cRs2jyU+TxeGiK9E9tz69JTH3S72PV/RzX8b9FDApipAX6Kz/3+XwYM2YMxQRMmDCBfsdYvYyLinyHyWDq1nI4nCHVhVQqFfUbK5VK6gL5Kert7YXJZEJERARGylxn2qpUKuFwOBATE0Prrw5HHuVyuTT228/PD+PGjYPX66UIAoZbn5OTAy6XS9lGjK/cz88PNpuNhnoyC6n+/v5ISkqivnJGLBaLoiUCAgJGrCr1U/WTqZD/Deru7h42Jvz7yOFwABiY+r3//vvX2fraOn/+PI4ePYrS0lIcO3YMSuWw5IYfpf8Ni2qw2tracPz4yGkPTEei1+uv2VaXy4W2tjZ8+eWX/45m/iJqb2+HXC4f8fsf4io1Go147733rnm8weJwOJRBfiV7yOl0DoG2SSQStLW1gcfj0UzSwQYH87dUKsW5c+dgMBh+Mm5YoVDg5MmT+OKLL665ndVqhVgsxubNm2EymSCVSik98UqxWKwhRTAkEgmMRiNMJhOtxpScnIyenh709vZSXhAz0DHrFi6Xi+YeMHVZAQyJEPJ4PAgMDKSoYJ/PB4fDMSwV86e6xK+3/2+dOwYSc5qbm3/wfq2trSgpKYFOpwOPx0NpaemPbsOKFSuQnZ2NRYsWIS8vD+Xl5TSu+aeqo6MDx44d+1mO9WPFYrEQGBg4InnRbrejoaEBS5YsQVhY2FUANpfLRal4TALIf6IcDgfNNRhJ7e3tKCws/F4DclxcHCQSyYi4jeHEZrNpqv3gDoKBnzGMeSaaRSQSwW630zBc5h4y4L6enh44HI6fjFe2WCx46623oFQqR6yw5PP54PV64Xa7UV1djY6ODgQFBSEtLQ0CgYCiPK4cuAZH6bjdbvh8PpjNZtTU1GDTpk3o7u5GUFAQwsLC6CIqAyvs6+ujUUPBwcFob2+HWq0ekmPBuG+YQh9+fn7o6emB2+1GeXk59uzZcxUskBlsfqzhNRJUj57zjzrq/yHV19dj0aJFEIvFV32n1+tHZHIDwDfffIOcnBwalsZQ44ZjqzNsi5H0zjvvIDQ0FEajEUKhEH/84x/pi8Q8sFeO/sMx55mZBFMl3uVy4cCBA5g0adKQh2jwcQfL6XTSYwBDrQPm38x+NpttWCLlYNlsNni9Xpw4cQLTp0+nLxnDWOnu7obb7YbJZMLx48fpy8Tj8UAIgVKphF6vh9PphF6vR0BAAA4dOoSFCxdeZblYrVZK0ht8Plqtdthrf72Xg8nqHPw7brcbRqORXkuTyURdCRKJhF475nvGzca82Ha7HTfeeCN1FZjNZqjVaprw1drais2bNw+LY2ZYMYxUKhUaGxvxyCOPXPM8mHYzGnwtGEYR88xGR0eDxWLRXAjGaCkrK6MDLoPHjYyMhFarxaFDhzB37lyEh4dDpVLBYrHAarWitrYWCoWC/vaV94uJTXe5XFCr1bBYLOjs7ERWVhbuuuuuq9qtUqmgUChoMZ2ysjJMmTIFHR0dMBqN0Ol0QwpP19XVwWQyob29HQ0NDairq6OMF61Wi56eHrS1tcHf358mLYnFYlitVkilUmg0GhiNRlRXV1NiLHM/vV4vent76T1ubm6GyWRCR0cHdDod5VVJJBIUFRVBKBQO6UsIIVCpVAAGBh65XA6tVova2lpIpdJhaaaD+5X+/v5hseCD9V/tc2em+HffffdVLz+TGcqERF3JSz99+jT27t2LWbNm0dVwQgjKy8vR0NBAs1EB0EQSq9WK0NBQTJs27aq2BAUF4dChQxg9ejTN3mQwxGq1GosXL8bFixcxc+ZMxMTEwOVy0RcuLS0NOTk5kMlkqKioQGZmJrxeL86dO4epU6di//79FInAZKYyi1A+nw+33347bX9dXR2N+01JSUFDQwO0Wi3++Mc/0ugJxqr69ttvERoaColEgpycnKv8ih0dHZBKpejt7UVFRcWQOOz6+nq4XC50d3djyZIlsFgsOHLkCMaOHQulUkmzIauqquiU+Oabb4afnx/KyspQUFCAmpoapKSkYMyYMXC73SgrK6MIXIaUWVFRQfGx999/P4CBDrmrq4t2EgzBcbCcTicuX74MtVqNKVOmICkpCTabDS0tLVAoFAgKCkJ2djYuXLiAuro6zJkzB1KpFH5+fvjd734Hg8EApVJJE9bS09OhUqlw7tw5aDQa3HvvvWCz2WhqakJfXx/sdjvmzp2LY8eO4fjx45g/fz5YLBaioqLQ1tYGu90OlUqF2bNnw+VyUWyzWCymNM7BOnHiBDQaDfLz8xESEoLy8nLk5ubCZrPh6NGjWLFiBd22ra0NarUabrcbUVFRyM7Ohs1mw5dffonbb78dVqsVJpMJ586dQ1JSEs6cOQOTyYRVq1bBarWiu7sbSUlJcDgcqKyshM/nw/jx41FTUwMej0fbxwwkra2tMJvNFH07b948eL1etLW10VBNxr3D4AUsFgtaWlrgcrkwadIkhIWFobe3FyKRCOfPn4fH48GiRYvA5/NC5tT4AAAgAElEQVQREBAAqVRKjYcbbrgBPB4PFy5cgEQiQVpaGk1SqqqqgkAggMlkojHkTIUsFouFS5cuUdeNv78/vF4vjEYjLBYL2tvbERUVRVEP48ePh8/nw969ezF58mR6f6urq1FQUAC1Wg2hUAij0QiDwYCWlhbMmjWLkh/j4uIQGRmJ4uJizJgxAxMmTKDGiVKphNPphMlkQnZ2NsrLy/9tPPf/E9q/fz84HA61ngZbfHPnzkVKSgrS0tKusnAJIbBYLPB6vTTF/PTp03j66acxd+7cITz5ixcvYv/+/RCJRJBKpfjqq6+GbUtHRwcuX76MF198kVotlZWVEIlEKCoqQmpqKiorK/HQQw+hsLAQDz/8MBYtWoT33nsPX331FVQqFSQSCcrLy5GRkYHJkydj2rRptHjA5MmTkZCQgPb2drS0tGDatGno7OzEq6++CmCANrlixQqKbaiuroZWq8XEiROp//Orr76ineDWrVtpbU2fz0ctbUaNjY2oqalBfn4+9Ho99Ql3dXXhiSeewKRJkzBjxgxs3ryZlvzzer1YtmwZYmNjcerUKTz99NMYO3YsamtrsW7dOsTFxYHP58PhcGD8+PGYN28eli5dCpvNhgcffBDz58+nXHTmt9555x3MnDmT4qGNRiOWLVtG645KJJKrOvZt27bhySefREFBAcrKynDs2DGsWbMGd999N/XBPv744xCLxWCz2Th48CBCQkKQm5uLN954A2azGV6vF/PmzYNEIqHWqVKpBJfLxeHDhxESEoJbbrkF69evpwuHbDab1tjMzs4Gn8/H119/jd27d0Mul6O9vR2dnZ349NNP0dzcjLS0NBw4cAA6nQ4ymQxSqRSdnZ24cOECtFotxGIxNm3ahJqaGso5T05ORnNzMzo7OwEAGzZswObNm5Gfn4+3334bx48fR29vL4xGIzQaDTgcDuLj4+HxeFBaWgqFQgGRSITPPvsMHR0dqKyshNVqBYvFoviIrq4usNlsbNq0CVqtllqYLpcLCoUCb7zxBjgcDiZNmgQej4etW7ciIiICmZmZeOyxx4bgul0uFxoaGjBr1izExsbSsoJisRi5ubmYMGECkpOT8eGHH4LNZsPlcuGjjz5Cd3c3Ro8eje7ubuzduxdyuRwejwdnz56FTCaj0T/ffvstnn76aeTl5UEkEiEjIwPR0dHw8/PD559/jsrKSowaNQp8Pp9idsvKymh0TUlJCbq7u3Hu3DkEBAQgLy8PL730Es6ePQs2m00HAmbgOXHiBI4cOUJdSF1dXdBqtTh37hwOHjyIm2++Ga+99hoOHTpEZw8vv/wypkyZgtDQUEilUuzatQuFhYXXDbj4r+7cL1++DIfDgZaWFgADrhNmOhQbG4tXX30VW7ZsGZIazfjUqqqqhkR0lJSU4He/+x2AgY6aufCfffYZ0tLSYDAYEBQUhAcffPCqdvT29iIhIQErVqzAjh07KNRq/PjxaGxspEWAmaotn3/+Of2My+UiNzcXXq8X48ePR1dXF/UF3njjjThy5Ai1tPl8PrZv346cnBz4+/vDYDAMAXdVVFRAJBIhLy8Pf/jDHyAQCKBWqylutq6ujvqKc3Nz8fbbb2PVqlXUSh4cCXPo0CHk5ORAIBDAbDZj+vTpAICDBw+io6MDQqEQDocD2dnZNNmLgSsxnJWOjg6EhYXBYrEgPz+fckQGV8my2Wyora2l6xNGoxHjx48HAKSkpKCkpARLly6lUR51dXWIjIyk7O3hWEI7d+4Ej8dDf38/Zs2ahdzcXOzYsQOJiYkU92owGBAZGYmEhAQkJCQgPj4ebrcb48aNQ2hoKIRCIZKTk/Hxxx9DKpUiJiYGIpEIZrMZqamp4HA4yPr/7L15fJTluf//niQzk2WyTfZ9Z0lC2MIqu4ALoqAtKuBSPait1mrdqr9Wbc8pLtX2eHCrlW8RhaIiagFlJ4ASCCQsCdn3mezbJJPJbJncvz/iczcr4NJz7Dl+Xi9eQDLzzDPPcz/3ct3X9f6kppKbm8umTZvw9vbG09OToqIiAgMD6erqwmQysX37djQaDR0dHcTFxVFeXs6WLVsYN26cRF4rqXtKTNzX15fw8HC6urqYO3cuOp1OMozq6uro6OiQVZc7duyQjCBPT09iYmKkq5DD4SAmJobIyEi6urqIi4tDrVZTVVUlz7GmpoaQkBC0Wi2BgYGYTCbCw8Olb6iSLgj/YO3HxcUxceJEXC4XfX19NDc3o1arMZlMREZGDhpsnU4n/v7+jBkzhl//+tcybbG6uporrriCyMhITCYTOp0ODw8PCgsLOXDgAMHBwfT09ODp6Ulrays6nU62uwULFpCQkIC/v7+0AlRW1cnJydIKLy0tjbfffptnnnkGNzc38vLy2L9/P25ubnR3d5OZmUlUVBRBQUG0trZKM5LQ0FDCwsKkaXZsbCwqlQqLxcLhw4clj0mZqSsOTmPGjKGtrY3IyEhSU1MlqDA2NhYvLy9iYmIIDQ1l586dkpF1Mf2f7dw//vhj1q9fz9q1a5k1axYGg4Hu7m4CAwPp7Ozkl7/8Ja+99hq5ubmD4spKw3vllVd49tlnOXjwIGfPnuXEiRPodDo2bdpEc3Mz1dXVtLe3s3XrVlavXs2CBQu47bbbSE1NHXQetbW1/OEPfwD6N2V0Op1MT0tKSuLvf/871157LQB5eXmSKf3QQw/R2tpKamoqc+bMQa1WExwcLM9PKWvesmULTzzxBPv27aO5uZkdO3ZImtzBgwe55ZZbaG5u5s0335Tl98HBwajVasaOHcuuXbt49NFHOXDgAGfOnAH6O9QjR47w5ZdfMnv2bDZs2DDs+r7xxhuSY5+VlcXNN99MS0sLf/rTn2Q8OT8/n5/97Gd0dnayc+dO7r//fvR6PW1tbWzatInx48fj4+NDeXk5d955p4zxK+Gdt956i5deeomsrCzuu+8+Ghoa+OCDD0hISKC9vZ3XX3+dnp4eNm7cKOmhR48e5fnnn2fevHn85Cc/GTb7UYwrli1bRnV1tWSEKCuOwMBA3nzzTaZMmUJYWBhnz57lpptuwmKx8Nlnn3HXXXfJ2eVbb73F6dOnJT0yJCSE7du3c80113Du3DluuOEGPv30U0pLS3n22WfR6/UcPHiQ1atXS1hUfn4+a9euZcWKFVx11VWcPXsWg8FASEiIZLYrHPbY2FgSEhLw8/PDaDRKkJuvry+nTp0iICBAZok0Nzdz+vRpTpw4waJFi6isrCQsLEzCtaxWK4GBgQgh0Ov1GAwGIiIi0Ov1ZGVlsXTpUjw8PPj000+ZM2cOjY2NaLVa3nzzTWbNmkVnZydjx46VAwL0h7o++ugj5s+fj8ViwWAwsGnTJm699VZ6eno4fPjwMNidTqfjs88+47bbbuO+++7jd7/7HfX19RiNRqZMmUJoaChHjx7luuuuw2QyceDAAXJycoiMjKS1tZWCggKWL19OUFAQx48fZ/ny5cTFxeHm5kZRURETJkzAZrNJRr0yQSksLMRisbBjxw5mz57Nnj17UKvVfPrpp8ydO5cJEyYwbtw4EhMT5cAC/UkWK1euZOLEiaSkpPD+++8zd+5cdu3aRXR0tKTbJiYmMn78eAlP8/DwYOHChRw9epQVK1aQnJyMj4+P3KtqamqSA+rhw4f5t3/7t1H9GhT9n4u5t7S0kJ2dzf79+1m5ciXQb9Lg4+NDdnY2cXFxOBwOysvLMRqNpKenExoaOgxelpGRIWcGBQUFMia/b98+QkNDaWxsJDo6mrfffpsjR47ICr9p06YNOp8zZ85w/PhxsrKy0Gg0VFRUsGnTJvn7hoYGhBB88MEHfP7554wZM4a77rqLZcuWYbFY8Pb2JisrixtvvJH6+nruueeeQSCu9PR0rFYrQUFBhIaGcvfdd/PFF1/I8uvGxkYyMzN5+umnqa6u5tSpU7S0tBAYGEhwcDDTp0+noKBAkjCPHj3KnDlzqK6u5uzZsxw+fFg6ZA3Uo48+yhdffCHTwrq6uggKCuKee+7BYrGQm5vLvn37mDp1KtOmTcNms1FeXi4NSJ555hlaWlooLi6WVNCBnJNPP/0Ul8vFkiVLGDduHMePH8fpdJKbm0tiYiLTpk2THJDCwkJ5TR966CE+/PBDZs6ciYeHx7CNy+bmZtauXUtxcTHR0dE0NTURFhbG6tWryc3Npb29nYceeoiJEyfi7e2N0+lkypQp1NXVsX//fmbPno3T6cTT05O+vj6qq6vZvXs3TzzxBMXFxbI4JjQ0lPr6ehwOB1deeaWMhytl7ElJSURFRfHEE09w7Ngxxo8fj5eXF0uXLqW1tZWzZ8+Sl5fHihUr8PDwGDTb1Wq1nD9/nqCgIJKTkzEajXJFUVxcLBHFwcHBrFy5kpqaGlwuF6GhoZw5c4Zx48Zx8OBB4uPj6e7ulnaOmZmZ+Pn5odVqSUlJQaPREBkZicVikVaB0dHRnDt3Tm52KkYoLpcLl8vF9ddfT2VlpaRsrlmzhilTptDQ0MDp06cHhWQGAvYUEuz69euZOHEib7zxhty3KC0tZcmSJXh4eDBt2jS5F+FwOLjiiiuYOXMmFouFmJgY5s6dS2BgIFqtVqKDFdtKh8OBw+HA09OTlpYWGV4NCQlh8eLF+Pn58cILL8hVuIKxPnXqFPfddx9qtRqj0Sj7Cz8/P0kw7ejoQKVS8eyzz1JWViaLpxTuz1VXXUVISAhlZWXExcXhdDoJCwvjuuuuo6SkRNonJiQk8OKLL5KVlTVqRpGi7zXy958hJQMjJCSEsWPH4u7uTmtrK7GxsYSEhBAfHy9Nfz08PJg7d+6IxTdxcXHo9XomTZqEVqslIyODmJgY4uLiSElJYdq0aahUKlJSUuSsZurUqcPygJV4uBKW8PHxkR2OUhxyzTXXoNFoJPc7ISEBi8XCtGnTCAkJITo6mrCwMLq6uoiMjBxUoRcdHU1QUJBkfowbNw6DwYBer2f+/PnodDr8/f1JTU0lKiqK7u5upkyZQmRkJBqNRjKmp0+fTkpKCsnJybJzslgsTJ48WXLkByopKYmGhgaCgoKYPXs2/v7++Pv7M2HCBEJCQvD392fevHk4nU5iYmLw8fEZtPpIS0uTA9C8efMIDAyUBEStVktQUBCTJ0+WxzUajWRmZjJv3jz0ej2JiYlMmDCB2tpa+VqlqrKrq0sug4fG2y0WC2PHjsXf3x+9Xk9cXBx+fn6kpKQA/e46EyZMkKEG5R77+vrKEERMTAx+fn6YzWYcDgczZ87E09MTDw8PfHx8iIyMJCMjQ25qp6enk5KSIl1/wsLCpDtVbGwsarWamJgYwsLCiIqKIjg4WA7YGRkZsqNVpFarqaysZPz48UyfPl22g7i4OIKCgiRQy8/PT5o/p6WlSUqkgqtW7pebmxsajYaMjAy5YalsuAcFBRETE0N0dDSdnZ3ExsbidDo5f/48c+fOZdy4cTIzyGq1SjCZh4cHSUlJREREEBAQQHl5OYcPH2b58uV4e3sPAsZFRESgUqkIDg6WFaItLS3MnDlTHltxJLPb7SQmJqLRaIiIiJBoZyWkp5heK25M8fHxTJo0CbVaTWFhIS6Xi5SUFOx2uwwRhYWFERkZKbnsQghiY2NlBavJZCIxMZHw8HDCwsJwc3MjISGB4OBgIiMjiYuLIyoqiqioKCIiIqR3a0REBDqdDrPZTEREBFFRUURHR2Oz2UhKSiI4OFgigaOiouSzGhkZSVVVFXq9nr/+9a//2sjf/4sSX3m9xsfHS4/VH/Td6WLwqtEQ05ejb/NeQGZmfNtiIJvNJjtl6N8rcnd3x93dXX7vvr4+WTfgcDjw8vLCbDZTV1dHRESE3COAfxjnaDSaYawW6J9l5+fnk52dLVn0d9xxxyWJpUIInnjiCTk4KF65I6m7uxt3d3dJXLRYLHh6euLr6yuvuVIZqtyH5uZmuUejrGiVzDiFrCiEkKmUJpOJ6dOnU1pair+/v2Ssd3Z2SltNpepUqVIFJHZ5tEr3rq4udDrdt2obQ1VZWUlSUtK/NvL3/6IMBgOtra3SH/UHfbe6GAbh2zyA3/bh/br45tHU3d0teS+ALKwZKDc3N2muoYQ/rFarDKUBMgQxcFAYutpRwlDKRrW/vz/jxo27rO+ieBMkJibKJIHRNJDT4u3tLb1UB15znU4nQ0BqtZqQkBBZKaqct1I/orzPZDLhcDgIDAzE4XDQ29srXZpsNht2u13yd5T0YWX1pxiHDPzMoWpsbESn0+F0Or8RhmC0icilPCF+mLl/z3Upg4wf9H9bQx/8vr4+mXKoLPsvJoUNDv20RKvVSnx8vDyusufgdDpHHCC+b6qqqqK3t1eG0UaS8p2VdGVl1VJfX49OpxuENFayjBTj89raWhITE6VBuTKbh9E74X8W4rixsZGIiIgfZu7/qvqhY/9Bo2mkTsPNzQ0vLy9iY2MHzWhHc+saOPsLDAzEx8eHtrY2rFYr0dHR0uHpuwwnfFtdzHnM09NT7pmNpL6+vkGpmVqtVg6AFy5cID8/n7S0NNLT02lsbEStVtPS0kJwcLCs41CpVDJDRnk+lXDQSPpnIZCVkNlo+v7csR/0g75HUtjeX1dms/kbve+baOCqW2GuKLRExaBC0eW4mWk0mkHpgA0NDbIewWKx/LdMNBS+zcWu4cVWD8HBwYPgZwOlcHN6e3ux2WyYzWa50mlsbOT06dO89dZbZGVl0dfXh6+vL15eXrLOQ8mxh/7rfblWmd+1lKLKS62ifujcv0M5nU4efvhh3nzzze/keAp7AvqXzK+++up3ctzLUV9fnyzAgeFsi//tslgsrFmz5mtDnV577TXuuOOOYT+32WyUlJRcsmT860jphBVLNg8PD0klHKrROnej0YjJZAL6C8Dq6urw8/MjLCwMu92OWq2WqwFFlxPKFUKQl5c3Iv/oYjKZTBw7dowNGzZ8bSqqgvVtaWkZxPYZKJvNxp/+9CcMBoPMdS8rK6Ojo4OpU6dy8803k5mZKV2cxo0bR0xMDHq9XmYQwfCOdaBv7KVkNptlLcxHH32E3W6Xg9qlvp/y9759+2SV8Wi6ZOeuUqliVCrVYZVKVaRSqS6oVKpffPVzvUql2q9Sqcq++jtwwHueVKlU5SqVqkSlUl11yW/7v0SK23xCQsKo9MPLUW9vL8ePH5d5+ArHYuzYsd/VqV5SClNHgaI999xzrF+//r/t8/8nNRDIpBhXXI5cLheFhYX4+fkN6lSEEJhMJjZs2DCMDPhtpHTu9fX1lJaWAv3hAaUwZqCU2d7ApbzT6SQ6OpqAgABZwBcfH09fXx9qtZr4+Hg6OzuxWq3Y7XZqampoa2vDbDbLjcahqwen0ylxuD/96U85evToiNC2kQZNBV3wxBNPsHbt2mGD1NDvpHwXIYQsRFJ47MprlfJ/6GfotLa2smPHDlwuFwEBAbhcLtLT00lISCAlJYVf/OIXrFq1iq6urmGfFxISgoeHxyB4m8PhGOTBAAwDhClSkA7vvPMO999/P1VVVajVaiwWC25ubnKjVzEdGSqVSiU5Sy+++OKIpuIDdTkz917gESHEeGAmcL9KpUoFfgUcFEKkAAe/+j9f/e4WIA24GnhdpVL9twaOv03HOlRfd+YWERFBUlLSt2I1K/ZcysaOSqWioaGBuLi4b3zMb6LFixfLOGJ5efklG9M30cWW3wMfoovpu7rfSifk7u5OQ0MDU6dOpauri56eHvr6+i6JtHV3d6eqqmpYYZQS4qmurpYhg8vB4/b19clZtaKhZEghBO7u7rKtKHgEZabe19dHd3e3zOgYmK2hQLLgH5koSlqgci2UjlwIgdPpxGq10tfXh6en56AQjvL9FZSExWIhICCA6OjoETNEhoZ4nE4nNptNzvTDw8OHde6Kxd3A797b24sQQjpHKT9XBteB9ESn00lzczPBwcGoVCrMZjNjx46VkDCtViuzixTSpNFopLu7GyGEzIhR/FGh/z4qKwbof1aVCt+hfUBDQwPd3d0UFhZy9uxZAMaOHSvDZw6HQ9630VZaHh4emM1m4uPjL0qshcvYUBVCNAANX/3brFKpioAo4AZgwVcvewfIAp746ufbhBB2oEqlUpUD04HsS33WUJ06dYoPP/yQGTNm4O7uTn19PT/72c8wGo3k5eVJVOaKFSswGAzU19dTUlLC9OnTOXv2LGvWrMFsNnPhwgUKCgqYMmUKV199NdBfGdrQ0CCXUzExMZSWllJTU8OkSZM4e/Ysa9eupaenh/z8fNzd3enp6WHmzJmcOXOGEydOkJaWhlqtJiAggIULF2I2m9HpdDgcDh5++GH8/f3593//d2w2GydOnMDlclFZWcktt9xyUdPclpYWnn76aTIyMigqKmL8+PG8/PLLbNq0iePHj7Nv3z6eeeYZbDabLEcPDAwkLy+PJ554YtCxqqqqqK2tZc+ePTz44INUVFSQnp5OQEAAjz/+OD/5yU84evQoAQEBJCcns337du6++27y8/MRQnDjjTdSVlZGUVERd9xxh9w4ysvLk7MWDw8PUlNT+fvf/47D4SA5OZlPP/2UF154QZIra2trufnmm4c5LJWVlfH73/+eVatWsWjRIjZu3Mgvf/lLVqxYwZ133smKFStkvv+FCxeYPXs23t7evPTSS3h5eTFv3jxJ12toaCAiIoLDhw+zcOFCZsyYMeizGhsbKSoqwmKxUF1dzaxZs8jIyKC1tZUTJ07g5eVFV1cXq1at4tixY1xzzTUUFBRw5swZli5dytixYykoKKC1tVWaTM+aNYvi4mJOnTqFt7c3vr6+/PjHPx4U725tbeXRRx+luLiYY8eOMXbsWElHVHDHMTExLFq0SL5HSfErKCjAZDKh1Woxm81MnjyZ6upqHA6HZIovWrSIpqYm/vrXvzJ27FiioqLkOev1erKzs/Hx8ZFEx5UrVxIYGChTHKE/9fbIkSNAf5VuSEgIt912m5xQuFwuVCoVX3zxBWFhYfT19UlqYWFhIUFBQRJMpgCyUlJS6OzsHJb3bbPZ5CzabrfT0dHB6tWrqaur47XXXgP6c7iVWLfNZkOlUrFnzx60Wi0Oh4PJkydjs9lobm6Wqxaz2Ux0dLSEqF1xxRXSbOTqq68mKCiIU6dOER0djbe3NxqNhvr6ei5cuICbmxsul4u4uDjq6uo4c+YMM2bM4MKFCxQVFfGrX/2Kn//851xxxRVMmDABq9WKzWZj5cqVVFVVUV1dTW1tLXa7nZtvvpm6ujrpFOXj48PixYuJiYnB3d0df39/dDodL7/8Mi+//DLd3d18/vnnnD17lhtuuIHjx49TVlbGfffdN8wgvrm5mc2bN+Pr68uXX345ah8CXzPmrlKp4oHJwEkg7KuOXxkAFJpTFDDQFsb41c++tjw9Pfnwww8xm80sWLCA8vJyALZt20ZtbS3XXXeddK8vLy8nMjKSHTt2MH/+fEJCQigsLCQ7O5vrr7+ejz/+WI6WgGQ8pKamUlpaKk1td+7cSWRkJHq9nsLCQo4fP84111wjO/zW1lY0Gg07duygo6ODefPmsXnzZrq7u6mpqSEjI0NWtu3atQuA7Oxs/vznP0vA16Vm9SEhIXh5ebFu3TqCgoKA/pi7Xq8nMzOTXbt2YbFYcLlcHD58GLvdLhv1UJWUlKDT6di2bRsul4vs7Gw528rLy+P8+fOSb52eno6fnx82m23Q4JOSksIVV1zB9ddfL2d4u3fvZvLkyYSGhmIwGDh79iy9vb0YDAamTp0qyZKvvPIKS5cuHXGZCf151AUFBTQ3N9PS0kJWVhbQv+Rubm5m//79eHh4MGfOHC5cuEBra6scrI8dO8aYMWPw8fEhJyeH7u5uUlNTMRqNkrWtyOl08umnn3Lu3DmmT59OTU2NhMQVFxdjs9mYOnUq5eXl9PT0UFFRIWeOfn5+9PT00Nvby86dO0lKSiIpKYlz587R3t7Ozp07mTNnDnq9Hk9Pz2EDmLL8z8jIYMyYMej1ej755BMiIiLIyMjgyy+/ZNu2bYPeo0wmcnNzCQ0NZcKECXzyySeUlpayY8cOUlJSmDJlCn/84x+pqKigtbWV8+fPc+LECcaPH88777zDmTNnaGpqkis/hQmk8PLVarUcXAoLC+nt7SUqKorTp08P8zdwd3enpKSEv/3tbwQHB9PR0YHVauX06dMcPHiQ4OBggoODaWhoIDIyEh8fH+bPny+pqYrsdjt5eXns2rWLkJAQ0tPTeeuttzCZTISEhNDT00NaWprs2BW5ubnx2muvMX36dPr6+mQB1r59+9BqtSQnJ8syfX9/f4mjSEpKwmAwoNVqCQgIwM/Pj7S0NFwuF35+fhw7dgw/Pz/Gjh1LYWEhbW1tCCHkXlNYWBhHjhzB5XKxZ88e8vLy6OnpQafTkZOTQ3NzM3v27MHf3x9fX1/Kysrw9PSksLCQ6upqiQj28PCQq52AgAC8vb25cOECLS0ttLS04O7uzpkzZ+jq6iI2NpaPP/4YrVYrVx3K86NUMU+dOvWSK/nL7txVKpUO+Ah4SAhxsfXASHk/w3ozlUp1j0qlOq1SqU63tLSMeKDk5GQWLlzIHXfcQUBAgLTA2rJlC3feeSednZ3U1NQQHBzM6tWrCQ8P58knn8TlcrF27VqeeOIJuewUQrBs2TKgv1NTZnWFhYXcdNNNLF68mODgYB5++GGSk5NZu3Ytjz32mHx/Xl4ed9xxB/Hx8SxcuJAlS5awZs0afH19JcXx7bff5qmnniIgIIBTp04xc+ZMoD/FrL6+nlWrVtHQ0DDqbr6i3t5exo4dy8KFC2V13fLly2XJv0L6KygokB1aSUnJiKYNS5YsQafT8dBDDxEdHc0XX3whO58777yTZcuWcf78eR544AG0Wi1PPvkkqampHDt2TKKHs7Ky+PnPfy5nowcPHsRsNrN3715ycnL48Y9/zLx586isrOTxxx9HpxBGpKMAACAASURBVNOxfv16AgICsFgsXHfddZK9PVQTJ04kMTGRFStWsHv3bvmahx9+mOXLl3P33XczY8YMDh8+zO23387MmTNJT0/HZrOxbt06IiMjuffee3nmmWfIyMggJyeHe++9V9IuFRmNRvbu3cuNN94oKw6VsnN3d3eeffZZlixZwooVK6isrOTEiRPs3LmTzMxM1q5dy/jx49m+fTs1NTV88cUX1NfXc91113Hq1ClJYezo6OD2228fscgnPz+fdevWyWX13/72NxISEoD+2djhw4cHhZd6enqorq5mz549ZGZm4unpyWOPPcbPfvYzAgICCA4OJjQ0lIqKCtzc3FiyZAm9vb2sXr1a8smbmpoIDw+npqaGiRMnUlBQIBk/fn5+EndtNBr59a9/TWpqKkFBQbhcLmbMmDHofJTjdXZ2snr1anbv3i2fF6vVyrlz53C5XMyZM4e2tjaysrIkHmKgVCoVd911Fx4eHmg0GgICAsjOzpbgLrPZPIzUqWTPOBwOFi1axObNm4mLi+M///M/+ctf/oLFYqGmpoZbb72VtLQ02eaCgoK4cOECkydPxt3dnY8++og33niDpUuXkpuby8aNG3n77bfR6/UUFBRw3333MWbMGBobG5k1axYRERF0dXXJkv+f/vSntLS0sGLFCsaPH88vfvELHn30UTZs2IDVaiU4OJh169bR2dmJh4cHGzZskDA0m80mn581a9bwl7/8hdjYWMaMGSOpkHa7nVmzZsnUy7i4OFmUNXDD1mg0Mn36dJKSkoY9TwN1WZ27SqVS09+xbxFCKEDyJpVKFfHV7yMABZ1oBAZ6fkUDw7a9hRBvCSEyhRCZoxnhKtRAlUrFe++9R0FBAX19fdxyyy3odDqOHz9OYWGhzOL4r//6L1asWCFnpvn5+Vx55ZUYjUaWLVuGTqfDbrdz4MAB5syZg8FgYPPmzZJd8eabb3LNNdfIz79w4QILFiygtraWHTt2yOWUwWDgqquuQqfTsWXLFl588UV0Oh3FxcW4ubnJooinnnqKyspKTp48yeeff87zzz/Pxo0bgf6Y3mhOQNu2beOBBx4A+pekJ0+e5KabbsJut7NlyxZ++ctfUllZybFjx1i3bh1Tp06VJLmhUhq1QkU8d+4cAKdPn2b69OnodDpJe1RWFBs3bqS0tJQzZ85QW1vL73//e6ZOncr58+eBfvTvAw88wPz587nttttobGxEpVJx6NChQVZnBw4c4MMPP+Q3v/nNIBjaQPX09JCamkpYWBhvvvkmCQkJdHV1STZIV1cXCQkJXHHFFYSGhtLQ0EB9fT0xMTHSZAT645UZGRnMmDGD1NTUYZ1Ke3s7er1ews8Uh6fPPvuMrVu3UlJSwuOPP87evXt56623CAwMZNGiRXzyySdYrVZUKhUlJSWsXbuWa665hsWLFyOE4NixY6SlpVFRUcH7779PcnIytbW1gz67ra2NiRMnStbNiRMniImJkdWZLS0t/PrXvx5WadnZ2SnZIxqNBg8PD8rKygZNTFavXk1iYiJ1dXWkpKSQlJTE6dOnWbFiBddeey1dXV28+eabzJkzh1OnTpGWljbIeFoxuD537hyhoaG0tLQQFBREUFAQbm5uOJ1O6QFaWlrKk08+yYMPPig55h0dHaxcuZJZs2axdOlSnE4nWVlZHD58mICAAHJycgZdC6vVSkVFBTNmzECv11NTU8OaNWtklktgYOCw2b6bmxu7du3i6aef5sYbb+TkyZN0dXVx6NAh0tLSmD59OhkZGej1eqxWK1988QUzZswgJiaGDz74AH9/f/bu3YvZbKa0tBS73c6CBQuIjIwkKioKrVbLFVdcQVtbm7TRmz17NgkJCezcuZOlS5fy61//mh07dpCbm4uXl5e05Dt37hzXXnstKSkpjBs3Dm9vb/bu3cupU6fYu3cvn3/+OR999JG8t8oeiRL+bW5uxs/Pj9OnT5OcnIzT6WTLli0sX74cg8EgK4iVwrSTJ0+Sn5/PmDFjhu3HDNXlZMuogI1AkRDijwN+9XdAyfm6A/h0wM9vUalUWpVKlQCkAIPv8GXKYDBIr0yj0cj9998vnchdLhdffvklbW1tlJeX09vbKzsfZZRLS0vDZrORm5uLw+GgpaUFrVZLamoqDQ0NnDp1itzcXKA/zl1WVjZog0jZbDlz5gw5OTmUlZUB/fFAJTWsoqKCJUuWAP+wwVNAYdAfqzxy5IiMM86ePRuAxx57bFTP1aamJoKDg6mvr5ffKyUlRS4BlY27a665RnYkra2tMvY4VIqDDPxjg81oNEr0r7I6Ub77wYMHcTqdmEwm1Gq1vK5NTU0ATJo0SRINq6qqZCNTjqPM+JR9hqqqqmEPrCLFWxX64+9OpxOj0ShnJQqG2Gq1cvDgQcrKymhrayM+Ph6bzSY3wSZMmCDt75T7NFBarRa9Xo/JZKKoqEguhysrKykuLqa7u5vGxkbS09Opra2VRMSSkhLy8/Pp7e0lMzNTDsiKRVx8fDwul4vz589TU1ODwWAYtunX1taGp6cnbm5utLa24uXlJUM1+fn5rFq1igULFgw6X8Ws2tfXV5rDtLW1MXnyZJqammhvbyc/P5/rr79eHle59iUlJSxZsgRPT0+52Wo0GsnNzSUmJmZYhktAQAAzZsygpaVFGlkoPqPKvVXsBBXbwVmzZuHj4yONpZWQXFdXFx0dHWi1Wrq7u4dt+rm7u5Oenk5XVxcGg4HTp09zww03oNVq6enpITk5eRiGQOGi22w2XC4XixYtQqVSsXDhQjnhUqlUtLe309TURFNTk5xkmUwmVCoV+fn5nDhxgpaWFtra2iS4LCAggNbWVqxWK1VVVXR3d2MymXB3d5fuV4GBgRw9ehRfX1/pOKVgnRctWiTTGFtaWujq6qKlpYXW1la6urpYtGgREyZMkM+Gh4cHLpdLeiMoufZK+E2xikxKShr0PCkb8EM3uy+mS+IHVCrVHOAYkA8o67Sn6I+7fwDEArXAj4UQ7V+95/8D7qI/0+YhIcTotveMjh+4+uqree+992hpaSEuLk5WllVXV9PT08O4cePkAx0aGsrp06fJzPxHJW5TUxPNzc1MmDCBsrIymb8L/TPn3//+9xQUFHDq1CnsdruMxypqaGigsbGRyZMny4df2Wj6z//8T4xGozTogP6ZlMJrNxqNBAQEoNPpMJlMGI1GSdyDfrefSZMmSaOLgers7KS0tJTExESCgoKoqqqSS/jGxkaCg4PlAFZZWSlNL0aTEEI6yihL3PDwcBk/H7h5Bf2uTC0tLWRkZKDRaGhubqa3t5fQ0FD5uWVlZbhcLoKCgqSLUm1t7aB0zYaGBmnBplgRjiQlVpqamkptbS0ajUYaHAghqKmpQQiBv78/Go2GpqYmSSZUNgTr6+vp6+tDr9fLdjJQLpeLkpISzGYzEydOpKioiJSUFHQ6nUzxCwwMJCEhgSNHjpCRkUFAQAD79u1j7NixxMfHYzabqa+vJyAgAK1Wi5eXF0IIcnJyyMjIkLCugWYigDRJVmLMVquVgoICvLy88PT0HHHFBf2ZMcrrFCRAU1MTHR0dhIaG4nK55AD96quvSvqgv78/CQkJaDQaysrKqK6uxmKxsGHDBl566SXGjh2Ll5eXTL8D5KaiYhAdFxcnaYoqlYry8nLZ4el0OnQ6Hd7e3nR3d9PU1ERUVJQcwJSJUlRUFDExMcPuR0FBARaLhfDwcNzd3QkJCaGxsZGDBw+SmJg4bKCD/sG9rKxMrr4G2tW1t7fT19dHQkICPT097N27l9DQUEJCQmT2jqenJ/Hx8RQUFODh4UF6erqcOAUHB5OYmEhPTw+BgYFs27aNn/zkJxgMBgwGA729vUycOJHS0lICAwPR6/WD9qSqqqrw8vIiIiKC3t5evLy8pB9uenq67HOUimKz2SxDSZmZmbi7u7Nt2zYmT55MUlISH374ISEhISxYsEA+b7W1tfT09PDnP/+Zjo4ONm3aRE1NDfHx8aPiB77XbJlp06Zx6tQpOjs7JQZU+bIXK0G+mBoaGqiqqmL27Nn8/Oc/JyIigqeeeuqy33/27FnWrVsnB4TLBQEpLjkqlQqn0yln5wONCYa+/nLKxy+XQtjT0yNd7y9Hl7q+l8vLuNxrpMxSRquCtNls8tydTueIgCb49uc99HparVbUavWo11455qWqN5WsImXAgssrSx/KFlL+r6T/Qf+D//zzz/PUU08RGRk5iH3/1FNPMWXKFHx9fWlpaeHqq6+WtQsDpXDbR7tXSshE6exHuo7Kz5RrOFrbVGbT/v7+uFwuOjs7ycvLk3z60WB5A/nuQz9XWTW7u7vLAS05OZmkpCTq6+sJCgqSqxCtVktTUxNqtRovLy/Cw8PRaDT09vbicDhkiKu2thZPT0/pENXa2ioHWZvNJs9FMUxxd3eX98fhcGCxWPD19R3UdpQq4qFFYZ2dnRJzbLfbh71v165dtLa2StexX/3qV/+6VMjGxkbmzp2Ly+UacSPumwKMysrK2LVrFzabjQceeGDUWdNIcrlcHDhwgBkzZlyUJTGSBjZytVo9zHD7Yq+H0b/v5TI/RprNXkyXur6Xy8u43MHvUp3jwEFptI4dvv15D72eFysx/zq8lYHn/HVYI0Ovi/J/BQtst9s5deoUDQ0NtLW1odfr5cpB2Yfo6ekhPDycq64avZ7wUm156L7YSN9B+dmlcrUHpvYpnPVz584xderUi1JQR7sXiqWfouTkZMLDw2VVqZJVonBlFEMUp9MpV5omkwkfHx+5ulaY7QpkbCCuV+mclcFL2ZQfOBBrNJoR0b8eHh7D2qhKpRp0TUZ6VjUaDSaTiR//+Meyz7pk1OX7PHP/QT/oB11aZrN5UMUl/N8FzvX19dHT0yO580PrSZTN+vr6ejw9PdHr9Xh4eNDb2ztoAtHZ2SndlBQMstVqxeVyDTqmUrfg7+//jXC+30ZlZWWMGTNm1Jn7D2yZH/SD/sXl6+srV7dOp/Nrd+wXq/B1uVyDcAzfhf6ZE8ru7m5sNpvMLhpYzatUl3Z0dNDb2ytNSqxW6zDCore3N1qtVvoau1wuWaQ2UJ2dnTJMNlT/bIDcpSqzf+jcf9D3UkpJd1dX14idy5EjR3j88cf/B87s+yklJHK5eyoDdbHwktVq/UbHHE0KMuBiGthpfd2BQKPRyPPt6OgYdO7u7u50dXUREREh9/Camprw9fUd5keqVqtRq9XyXEYK9/X29sp8+u/yGo2kka7DJUOM/6yT+a7U3t4uH+6Bm0jfpdra2mhra7tko1NSAYeqvr7+azNovgt9W0qj3W4fNdf+f1KlpaWUlJRQV1eHm5sb586dIysri5KSEtkWfH19v/Y+wqU+Mz8//2tTDL9rDTQK/+8ImY72PCnPwsX2N76ulI69r69Ppsgq5txKWl9XV9egSmur1SpTV0fTQEibVquVg5Wfn5+sdC4uLqa2tlbeX2VjMyYmZsRjKrrYwDe0wx96v77LMI0yyWloaCA/P5+enp5Lto/vfef+ox/9iNdffx3o3yC69957v7Njb968mQceeAAvLy+2bNnC7bffPuLrlMZz8803j/j7ZcuWjVqkM5KG8l8uV2+//bYsOAJGPd/L1W9/+1teffXVUQet/wkJIcjKymLlypXU1dVJU+k33ngDo9EoO6MpU6ZwOebuu3btYvPmzSP+zmazyapiZca3Zs2a7+Q7fFM1NTXx4YcfcuLEicta1g/tnIfWGlxKI3VeA9+rZMZ8F5OX7u5uCQg7fPgw/+///T9effVVTCYTvr6+9Pb28vnnn9PY2Mh//dd/YTQasVgsPPLII6xfv37UyZdaraajo0MCwLy9vXG5XHh6etLc3Cy5TB988IHk4sTHx+Pr6zts8FLunZKvP1QDB5Kh5zOaC9PA114OMG6oHA4H/v7+CCE4c+YMP/rRj4BLh32+9527w+Hg2muvlUS7t94a0ej7G+mVV17hjjvuwNvbmzvuuENCi4ZKiWEqcbmhN0ij0Ui0weXohRdeuGzi4UC98cYb6HQ6iY39NmxwpYrxmWeeGbYk/Z9Ub28vWq2W6Oho3nzzTZxOJ2azmfnz53PllVdedLaudEBWqxWn04nD4eD999+XRVhD5enpyR/+8AcyMjKAfqLnn//858s6z9EeLIW5MvD3l+oYnU6nNI1oaGjg6NGjJCcn4+bmdsl4d21trUwDVJC7gPQNVXSpuDr0149UVVWhUqlwc3Oju7tbmlsondPXGbgGfqbdbpepzB0dHezbt48777yTZ555Bjc3N5nCOm/ePFpbW6moqCAoKAgvLy/a29tZvnz5sPY+EGM8FHPs7u4uKZbV1dWsXbuWlStXDnKVGuikJISQ8fru7m58fHzw9fWV119BHw8cDDw8PCgoKJD9gcJlHyiVSoXNZhuEDxjaHoQQw6qaB0rJunE4HJw+fZqEhATc3Nwu6jgF/wKde2RkpHRTV8rcgUGlt62trYNA9oDEh15spFQalNFoxGw2D/JONJvN1NbW0tnZKRuDkjalXOza2lr6+vpIS0uTqUwKdGo02Ww2bDYb3t7e9PX1yQFDIciN9BAqBMCgoKBhQK/u7m4MBsOg1/f09NDW1nZRdvj58+eHbbx1dXXJSkdAdpBKg4d/dCAKkU95XXNz86DXKNfnUiXSQyWEoLm5mVmzZpGVlSVRtUqZvPKwdXR0yMIlRUqRkslkkqS+0tJS9Hr9iJ9lt9ul+1BhYSEFBQVERETIjlm5jorMZrMsqLLb7RKH29fXR319PS0tLbJzt1qt9Pb20tzcPAxiNlTFxcU0NjbK6kjFJUgJJQxUR0fHoJWW1Wrl/Pnzsg3Y7XZaW1upra0d1C5KS0tHHGT6+vqwWq00NjaSn58vy/N7enpoamqiu7sbq9XKmTNnhrVPs9ksq8NHkvLcFBcXy0I4h8NBWVkZlZWVuFwurFYrvr6+8rWtra1UVlai1+tpamqSOeYhISGyvQ2t9jQYDNhsNhobGwe1387OTqqrq7FardTX15OYmCgL7np6emRbamxspKysTN7v+vp6mpqaaGtro6enB6PRSHt7uyzoKysro7u7m97eXtRqNQaDgYqKCtzd3SWyYaCam5sxGo1y89ZiscjBvKamhvLycmw226D3KRXQys/a2towmUy0tbURFxd3WQDC73XnfujQIe655x40Gg11dXUcO3aMd999ly+++IKcnBzmz5/P7373O7RaLfPmzcNut+NyuXjhhRd49913aWxs5JFHHhnx2ApvJjg4GK1Wy4svvih/t3PnTgoLCzl69CgPP/wwAJ999pnkvQC89NJLGI1GHn30UR577DE8PT2pqKhg165dJCQk8Itf/EI2RkWfffYZJ06cYNq0aRKA9sgjj7Bo0SL0ej0ZGRm8/fbbw85Vo9Fw4sQJHnvsMcLCwtDpdJSXl0v395deekk+dCUlJWzZsgWn0znqhmNpaSkff/wxkZGRGI1G6uvreeutt6iqquLDDz+U5g8ff/wx48ePx2Kx8Jvf/IZ7772Xv/3tb6SmptLZ2ckHH3zAjBkzZPhEwTAA3HbbbXh5eclKusuVy+UiKiqKVatWkZmZicViIScnh0mTJuFwOHB3d+dXv/oVBoOBxsZGXn31VTo6Oti/f7+sLM3Pz+fBBx/EarUSGxs7iEEzUBUVFcyfP5+kpCRCQkJ4/vnncXNzQ6vVkp2dzdatWyksLGTu3LkcOnSIkydPcuWVV0oi4VNPPYVKpWLr1q088MADrFixAuhfYe3du5fNmzfj5ubG+vXrh5WK22w2Hn30UR544AFqamrYtm0b3t7ebNu2jerqatzd3SkoKOCDDz6QE43nnnuO6upqiouLeeqpp6ivr6ehoYGbb76ZV155hePHj3PrrbfS3NzM8uXLefHFF3nyySe59957aWho4OTJk4POQWG3Hz16lDfeeAOtVkteXp4E0h0/fpwHH3yQAwcOUFRUxCeffILT6aS4uJgNGzaQk5PDrl27uOuuu4Zd256eHv7yl7/w8ssv43Q6ee655/jd736HyWQiOzubc+fOIYTAy8tLdtLQP2AcOHCAm2++mfDwcBoaGggICKCjo4OHH35Yvq+trY0XXniBU6dOcejQIR599FGCg4MloqCmpoba2lqqq6tZt24dqampWK1WPvvsMw4fPsz+/fu56667OHv2LJ9//jnPPvss+/bto7m5meeff54DBw7w05/+lIMHD8rQ0JdffklTUxNz5syRGTabNm3iueee49y5c5w4cWLY3l1WVhZHjx5l+fLl7N27l48//ljehyeffJKSkhIuXLgg0Q0ff/wxf/3rX7FYLLzyyiu88sorvPHGG5w7d47m5ma2b9/OggUL0Ol0l9wj/F537rm5uRInEBoaio+PDwkJCdTX18tS4yVLluDr6yvZE93d3WRlZREbG4vJZBqxAMrpdKLT6Zg4cSIxMTFoNBqJHTh+/DgbN27Ey8sLh8Mh2Sa5ubmMGTMGu91OaWkpISEhZGZmEhsbKzkoZ86cwdvbGzc3NwkqGyhfX18mTpxIRESELJLw9vZmwoQJeHp60tTUhIeHx7DVRl9fHwaDgfT0dLlyyc3N5bbbbiMwMBCNRiPDPBs3biQmJga1Wj0ifVJJAVOyBqKjo9m6dSuLFi0iLS2NzMxMdu/eTXd3N97e3gQGBhIUFMTPfvYzrr76aul0ExISglqtRqPREB8fj16vlzNUd3d3/vznPxMcHMyYMWO+lslIS0sLoaGhaLVapkyZQnFxMU6nk5CQEIk4OHz4sAQqxcTEYLFYiIuLkxWWS5cu5f7778fb25vJkyczYcKEYZ/T19eHTqcjOTmZ5ORk/Pz8qKmpQaVSYTKZ+Mtf/sLcuXPx9PRECIGfnx8ajQYvLy/sdjsGg0GGAhTI1tixY0lMTGT16tW88847BAQE0NfXR2RkpIwDK1Kr1bz33nucOnUKtVpNXFwcdruduro6Fi9eTHJysuTn2Gw2ioqK0Gg0hISEEB8fz86dO2loaMDT01NWWp49e5bm5mbCw8O5+eabmTZtGlu3buX8+fN4eHgQEhIyLJ5utVo5duwY48ePZ8yYMSQkJFBYWIjNZiMhIQEfHx9CQ0Px8vLCZDJJzO9nn30m878HeowCctXy2muv4e3tTXR0NCaTid27d0uXqqCgIMaMGSOfEQU/rNFo5GerVCrOnj3LNddcQ2RkJE6nUw7qx44do6mpCXd3d9RqNaGhoWg0mkEr2+DgYImD0Ov1HDt2jG3btqFSqYiIiMBsNlNVVSXj/ampqcTHx3PjjTfKauLk5GSioqIoKirCy8sLjUaD2WyWufTbt2/nwIEDMrTW3Nw8KPau1WqJiIggKCiI8PBwDAaDnJTt379fmoRERUXh5eXF7t27pbWhh4cH4eHh7NmzhzFjxhAREUF8fLxEnFxSSsrZ/+SfqVOnioHq6+sTQggRHx8vhBDCarUKs9ksli5dOuh1L774ohBCiNzcXLFx40YhhBCvv/66KCwsFEII0d7eLnp6esRI2rt3rygvLxdCCPH000+LkpISIYQQU6ZMEf/2b/8mWltbhdlsFna7XTQ3N4vk5GThdDpFe3u7mDp1qnA6nUIIMeiczGazeP7550V8fLzIyckZ8XM3b94sSkpKRFdXlxBCiJUrV4qKigrR2dkpFixYIIqLiwe93mQyCSGEmDhxohBCiNbWVtHT0yOuu+46IYQQOTk5Ij09XV63wMBAIYQQ+fn5wmw2D/v8vr4+sXv3bjF79mz5s+DgYHnsP/3pT+L48eNCCCHuvfdesWHDBnk/hBBiyZIl4g9/+IMQQogrr7xSvPvuu0IIIU6ePCn/LYQQf//730VMTIyIiooa9D0upsbGRvHll1+K8vJyYTAYxNmzZ8Xy5cvFkSNHRE9Pj7Db7WL9+vXi4MGDIjs7W9TV1Yn6+np5fsuWLROPP/64sNlsorGxUTz99NPDrqcih8MhPvzwQ7Fz505htVrF7373O7Fw4UJRX18vfvOb3wi9Xi8sFovYuHGjWLZsmbDZbCIvL09cf/31oqurSzz77LPi0KFDwuFwiPb2djFnzhxx4MABUVdXJwwGg/D39xdlZWUiOzt72H2w2WyioaFBvPPOO+LZZ58VoaGh4tZbbxXV1dXi+uuvF2fOnBEXLlwQDz74oDhw4IDIyckRmZmZwmKxiPz8fLF7927xox/9SBQVFYmcnBwxZ84ccfr0aXHTTTeJNWvWiI6ODrF7925x9OhR8cknn4h///d/F4GBgaKyslJ0dHQIm80mhBDCYrGIP/7xjyIyMlKYTCZRUVEhnnnmGfHuu+8Kg8Egtm7dKt566y1x4cIFcf/994u9e/eK1tZWkZycLFauXCmOHTsmKisrRWdn56DvZ7FYRF5enli1apVoamoSlZWVYsaMGeKWW24RBoNB3HrrreKxxx4bdk+cTqd48sknxaRJk0R7e7s4cuSIuO6660R1dbXo6OgQSUlJoqqqSrS1tYlx48aJ/Px8cezYMSGEGHYOTqdT9Pb2inXr1onc3FzR0tIiJk2aJJYtWyZqampEVlaWmDlzpti/f7/405/+JJYsWSLq6+tFQ0ODqK+vFy+++KJYuHChKCwsFOfOnRMrV64Uzc3NYvfu3WLt2rXCZDKJpqYmsWXLFvH000+LgIAAcezYMZGfnz+sTb/00kvipZdeEgaDQVx55ZWiuLhYFBYWiueff17MnDlTBAQEiJKSEtHZ2SlCQkJEXl6e2LlzpzzPO++8U9jtdpGdnS02bNggrFarEEKI8+fPC+C0GKVf/V7O3JWRz+FwcOLECcrKylizZg1+fn6S4rh161aJsf3DH/7AsmXL+Pjjj0lPT5dLYMVhZyQdOnRIzup37NhBU1MTmzdvZsaMGTgcDoKCguSS7f3335d0SA8PD4KCgvDw8MBut9PV1cXJkyfJzs5m3MWbAAAAIABJREFU9erVPPHEE7z77rsjptR9+umn/PGPf5TFEL29vcyePZu4uDj27NnDU089JTfHFCnnWFhYiMlk4r333mP37t3SPPs//uM/mDlzJu+//z6VlZXy9bGxsSM6tahUKumSo8wklb+VEMisWbOAfpjU4sWLB804e3t7ufrqqzGbzXR2drJ48WJqamp45plnmDNnDhUVFbz66qvMmTOHF154QdIghRD89re/HfFeWK1WDAYDJSUl7NmzR5o9REVFcf78edLT02XpuaenJxaLhcTERGlCsWvXLiorK/n888+ZO3cu77//PiUlJWzfvp3AwEBOnjw5LD6pVqs5ePAgY8eORavV8t5775Gamkp2djaBgYF4eHjg7e3NG2+8wZIlSyQ3xdPTk+PHj/PJJ5/g5+fH/v37aWtrY/HixcyfP5+IiAgiIyPx9/cnOjpaUiYHSvEhqK6u5pZbbmHFihWSjxIbG4tOp+PQoUP8/e9/R6fT4eHhQUREhHRv2r59O4888ghBQUE4nU5p/vDpp5+i0+mor69n2rRpdHd3U1xczK233srq1atlmbtaraarq4vKykoqKirkfoZiRhEQEIDT6eSTTz5h4cKF+Pn5cfDgQTQaDUVFRUyePJmgoCDS0tKwWq3D0hSVNEOLxUJTUxMXLlzgqquu4rnnnkOv11NVVcW8efMGhRXsdjvu7u5s3bqVtLQ08vPzqa2tlbC+3NxcMjIyqKiooLKykpSUFLy8vKRb2UD6ZFtbG319fbS0tHDkyBHa29vlnlpHRwfHjx8nKyuLZcuWMWfOHI4cOcLcuXMJCwsjPDwcu93O/v37mTt3LvHx8fT09DBjxgxsNhs5OTksWLCAoqIiTp48SWVlJTfddBM33XSTNP1Q5HQ6cTqdXLhwgeTkZBwOh6Synjhxgr6+PtavX8/KlSvp6uqSBirR0dEkJydTWVlJaWkpGo2Gjo4Odu3aRXp6uqSeXmqj/nvZuStSGk9aWpqsOFNUV1cnSYMKEzssLIzp06fLRnvhwoVRswTOnDkjN0EVJOfkyZNZtWoVoaGhFBcXYzQa8fPzIzw8nOjoaLq6uvD19eWGG27gyJEj5ObmkpKSgtPpZNasWcyfP5/S0lIZmx0qhWSoYIybm5uZNm2arHBTdulH0lVXXUV1dTVXXnkldXV1kgLp7e2N0+kkNDSUhIQE1q1bJ+N4o+XolpSUMGbMGBn+ue+++6R14EDDj/HjxxMVFTXouo8bN46IiAh6enpYsGCBXOoHBgbS0tJCUlISV155JU1NTRQVFcn00YaGBg4dOjTi+Sh0xYMHD/LZZ59JiFV3dze33nqrzJDRaDRcffXVlJeXU1dXR1VVFf7+/jQ1NVFYWMjChQupq6sjKiqK1NRUJk6cSE1NDQEBAYOWygq2tqioCH9/f/r6+pg6dSo2m43ExESmTp3KTTfdRHZ2NuXl5cTFxeHu7i7xvgqlVDFeMZvNTJo0SS7l3dzcuPPOO8nOzqa2tnZYipxWq8Xf35/ExESamppITU1l1apVlJWVMWHCBPz8/NBqtTJkkZyczLXXXsvp06cl9jchIQEvLy98fHzw8vIiNzeXhQsX4ubmhtlsRq/Xk5ycTGJiIvX19ZK3Av0bnRqNBovFwrXXXsuaNWvIy8sjMjKSpUuX4u/vj4+PzyA4mPLdkpKSuO2220hMTJROQ0MzmLRaLSkpKcTGxpKfn4/JZGLFihUEBwfLbBPFaUqRw+Ggs7OTzMxMVCoVkZGR1NXVyTCL8j5vb2+ioqK4/vrrKSoqoqSkRBpK9/b2ys1o5TwiIyMlrXLmzJnU19eze/duoqOjuemmm1CpVDI8p2zeBwYGotPpSEtLo62tjZqaGsaPH4+npydhYWHY7Xb0ej0hISFERUVRXl5Oamoqer1+UHaTYjQ+ZcoUKisrKSsrIzk5GZPJxIwZMxg3bhxtbW1MmTJFmqTceOON7N27l8rKSlQqFZMmTSIoKIjKyko8PT1lda3yPFxM/5JsmYtRAS+lkydPMnnyZFavXs327du/0TG+qYYSCweSDr+JLpccqUip2Lv77rtZv3498SMYXg98oIcSCbu6uvDy8kKtVst7MNK9UAbUgSbNyp+RGqSSNaTkJg/cqxjp+AOvm/JQO51OfH19aWtrk9aEo2nXrl1kZGTwy1/+ku3bt+N0OiXN0N3dndLSUmkf+PLLL/PCCy9IzK+7u/tl3Ufx1WalTqcb8Tt3dXXJQiWdTofVasXDwwO1Wi3TXRW2iVJO73A4MBgMeHp6Eh4ejlqtpqysjJCQEDo7OyVrXDlP6N+gtFqtEiam0Enb29tRq9USpaykPBoMBmJjY1Gr1TIDJTAwUKbeKbHg0e6NooFZVYqhdm5uLh0dHZw/f5577rlHWs8pg4NimalQUxUEsY+PDyqVisbGRiIiIuR9H9rWFcJkZ2enHHi3bt3KI488gsvlkqmminesr68vvr6+VFVVDdqX6evro7GxEb1ej1qtRqVSSaqqMgjp9Xq5d9fc3IynpyfBwcGDron4quhSpVLR3Nws94QUnIFiwqJMZJTno6GhgZCQEHx8fORg4ePjMyy7zWg0EhMT86+J/P1naPXq1WRmZqJWq/n5z3/+3/KZ3xedO3eOAwcO4Ovry+rVq78W1fL7rMvFHiu6/fbbiYuLIyYmhnvuuUeaUStkv7vvvptVq1bJ4y5evPiis6Rvip/+JqqurpYrQBg+AF+O6uvr5Yx/4MxbSU1UQo7+/v60tLTg5eX1jdqKgug1Go14enry17/+Fa1Wy5gxY7jiiiuGcVoGDkyKT8NIeOKLqb29nZaWFl5//XWmTZsmVxAajYagoKBBOeeAZM1cakKg6Ou0tYu9duB9U667MtjC5WEk2traCA4O/qFzV6TE40az9vvfrtbW1q/9wPxvk9FoxGazjYp7zsnJwdvbm7CwsO9NOxm4ovq2MhqNBAUFyVJ9u92OWq2mpaUFb29venp6cHd3Z9OmTdx+++0EBgZ+7ZXywPPt6OjAx8eH0tJSGhsb///2vjy6qev6el/JloRH2fI8YMAx2GEmFAyYBBvCWAKZCk3JQAvJWs3wNV9o2vSXpqZdWemvSb6VliTN0IQm4JSQMiYQZgzYzKaewDY2eJCRPEiyrNGafL4/5Hsrj0AmHEd7LS1Lz3p697zh3HvPPWdv3HHHHejs7BSc5TwWbTKZEBkZed1ww0DQaDRwuVwoKipCWFgYUlNT0dnZKYSleVz7q86av8nr8HXRlVnmd+5++PF9h8iC6BoN3qymAAdPOeQsic3NzWhsbBQatWPHjkVTUxPGjx+PjRs3YsmSJQgICOiTT51LHQ7k8Jqbm8VMw2g0wu12o729HbGxsRg2bJhYpwC8YSSXy9Vv4dn1oFarxQKzwWDA6tWr8dxzz/USTG9raxOL5t8FPTKfxfTXOXyVTkOr1SIhIcFP+evH4MRgGFx8X8CdII+Pf1UqXj4y5qpKwcHBiIqKQnBwsEhcsNvtmDp1qtDx7S8zoy/H3vOa+taaKJVK2O126HQ6yGSyXqpSvjH4m4XH40FcXBxCQ0MFJ87SpUuhUql6VXQGBQWJ+PZ3AV+myr7wVWYD13t2Bq0SU0+0tbUhMDAQQUFBQrrNd8HuZmKuXwf9SX3dalgsFhBRrzjmYIVvnPHbFDngTtBut3+jIRYu/ehwOL7TtQuHwwGn0wmbzSZS7b4ucyNfWHS5XIiLiwMRISIiAlu3boVUKoVSqYTBYOjTTrvdLqg0+DPIuV6oiwWypaUFycnJwoEZDAa0tLRAo9GAiOB0OkUKIS9K8gUXoPbVfO0LUqlUjMIlEgmUSiVWrFiBgIAAaDQa5OXloaioCG+99RakUinCwsL6LHL8NsDbfbMzErfbLXRre84wrkdCNuhH7uXl5Xj33XeFevurr76K6upqHDhwAJcvXwbgJeL6rnD8+HHs2rXrOzvejaK1tfUbJVX7urheabRv6tqhQ4e+EpHajUAikUCr1WLnzp039P2amppuOdP9fefs2bP45z//+ZUZNT0eDywWy03bzXV4z5w5g6NHj6KiogLAfzMzqEuPs6CgoBsvTs/6iZ5tAbwdR0xMDGpra0WYgI/u+xpN+2Y5ccfOs0A4URgRCf4Vl8sFh8MBiUQCg8Egfjc+Ph5Op7NbyMl3pmCz2b4SK6VMJoNSqRSdIc86qq6uhkKhEBkzX+W3fffhI2hf5kedTnfd+6gvWCwWwY/T0dGByspKbNmyBU6nsxdZ3fc6zx3wFistWLAA48ePR2dnJ2bNmoUJEyYgJydHnIQTJ04AuHGa06+Kt99+G3K5HCNHjvxWj3OzcDgccLvd2L17961uCgDvdSgtLcWOHTsA/Ld9vpDL5VAqlVAoFKioqPjWeOU5u+GN0Od2dHSgtLRUlM/3BY/Hg3Xr1qGoqAjx8fFfmVGTiJCbm4s1a9Zcd3rd3t4upvO82On1118H4HUozc3NIt3OaDRCrVajuLgY165dE5z/Op2um4P3HfVxTiaTyQSbzYY//vGP2Lx5M6RSqZil8nixL/OhRCIRGTccUqkUMplMtFmv10Mul+PPf/4ztFotqqurcfjwYdTX14vzx6kffPl3fEepXL/0RkMXvrYZjUaRb75o0SK89NJLOHPmDEwmk+hwBoq59+VAPR5Pt07ZtxaAp2MWFhYiPz+/1303EH++TqfDkSNHsGXLFhgMBsjlctFJcII74L+dyfUGBoPeuT/22GMiH3vTpk2YOnUqiAg2mw3p6enQ6XR4/vnn+w3N9Pfg9Nw+0Eizs7MTZrMZx48fx/jx4wVF7M38Pj8GZ07sjwGys7PzulS+LperG+OjXC7Hxx9/3KsAit/kvm3pKSfWH/pic+y5L38YHQ5Ht1hiS0sLTpw4gczMTNFeq9UKjUbTy4m3trZi1apVSExMhFqtRmtrK4gIarV6wNEm17Nsamoa8Ca3WCzYsWOHWCTk4Cx8vAqQx7F5YRbX1XS5XNBqtWIU1tjYiDNnziApKQnz58/vM1RBRLh69ar47PF44PF4RKWkXq9HeXk5zpw5gxkzZoiitr7abjKZcPr0aZjNZhARmpqacPr0acEpk5ycjPDwcLjdboSEhCAiIgKhoaF47LHHMHr0aOj1ehQUFAgyrfr6esGGSUQoKChATU0NAG9hYFVVFa5cuYL09HQMGzYMWq1WOGWr1SpsM5vN0Ol0aG9vR0lJCVwuF4xGozhOY2OjKBLct28f4uPjYTKZEB4ejkWLFuHnP/+5yIFvbm6G2+1GZWUlmpubodFooFarYTab4Xa7UVdXh6amJtjtdpHtxul5fdHQ0IDTp0/D6XRCr9fDbrfjypUromgtIiICbrcba9asER1Tc3MzqqurcfbsWTEI6OzsFDUFvMNTq9XQ6XRoaWlBXV0dHA5Hn2seHo8HjY2NOHnyJGJiYkTojI/mez5DnICMz3gOHDiAjIwMMSjhny9duiSeecaYCE0PhOvG3BljyQA+BhAHoBPAe0T0V8ZYLoC1AHjt8e+IaG/XPi8A+AUAD4BniGj/9Y7TH7gaul6vR0lJiViY4DdOSUkJDAYDduzYAZvNhocffljse+jQIcHtfd999wEADhw4AIvFAoVCgcWLF4tKxYqKCrjdbtx///29cpYlEglaW1tx6tQpXLp0CYmJiQgPDxdUn0lJSZg+fToqKytRVVWF2NhYaDQamM1mZGdn49ixY5g8eTKuXbuG1tZWzJ49G5cuXYJer8fSpUtF3O/gwYMwmUwgIkHI3xf27dsHh8OBMWPGICMjAwEBAaiursbEiRMBeMnP1Go1ZDIZ7r33XjDGYDabcenSJVRWVmLKlCl9kmlx7N27FzabDUlJSZg8eTLkcjm++OIL2O12qFQqTJgwASEhIdizZ4/IeKipqUFQUBBmz56N8vJy7Nq1CxMmTIBer0daWhra2tpQUlKC1tZWJCYmYv78+bBYLDh8+DDS0tIwbdo0HDt2DDabDdOmTUN+fj7kcjnWrFnTZwyWixzn5+dDJpNh8uTJ4l7h4KPckpISzJo1C59++inmz5+P5ORklJWVoaioCBkZGaioqMBdd92F6OhovPnmm1i/fj06Ojpgs9mwf/9+mM1mqFQq3H333Th37hykUinMZjMsFku3GCp3VBUVFWhuboZMJkNSUhJaWlpQVVUlCmNSUlJQU1ODhoYGaLVaEVP1BafHra+vx6FDh9DR0YHhw4fj2rVrOHbsGIKDgwUbJ38mZDIZLBYLTp06hbFjxyItLQ2XLl1CRUUF7rnnHhw6dAiMMSEW09LSgoKCAiQlJSEgIADt7e1oaGhAeHg4DAYDqqurcfnyZcjlcpjNZlGp/cknn2DMmDGYOXOmoLo1GAyQSqWora1FYWEhHA4H4uPj0djYiKNHj0KhUKCxsRFutxvvvfceli9fjkmTJkGhUKChoQEmk0mEXsePH4+mpiaYzWZMnjwZx44dQ1NTE6ZOnYrs7GxoNBqcOnUKwcHBmDBhApKSkoQ9W7duRWRkJFJTU2E2m0W1emxsLKRSKY4fP4577rkHMTExaGlpwTvvvIPc3FxUVFSIinFOgXz16lUkJiYiNTUV+fn5iIqKQnR0NI4cOYJFixb1+Qy1t7djz5492LlzJ6ZNm4b09HSEhITg5MmTsNvtqK2txYoVK8TamFwuF6EYjUaDo0ePYtKkSSAiTJw4ETU1NZg/fz42bdqEBQsWICsrC3K5HBqNBrW1tf0+w+IBGOgFIB7AlK73oQAuA7gdQC6AdX18/3YAJQDkAEYCuAJAOtAxehKH9YW3336b0tPTu22zWq20cuVKam9vp7q6OoqOjiYiIofDQU8++SSVlJQQEdHzzz9PbrebmpubadGiRURElJ+fT0RER44coTfeeIOIiD744IN+icaIiBYsWEBEXlKi+fPnC4KtmJgY2rdvH23cuJGefvppWrZsGRF5SYN2795NK1asoNWrV4vfWbduHRERXb16ld59910i8hIfPf7446IdfaGiooJ+9rOfkcPhICIvyRnH9OnTqaysjOrr6yk7O5uIiPbu3Sv+/+CDDwoyszFjxtD777/f7bc9Hg8REWVnZ5PBYCAiolWrVpHZbKa5c+eSTqcjIqLVq1fTsmXLqKioiD7//PNuxGmc6I2IaOrUqUREVF1dTTt37qT777+fLl++TC+88AJFRUWR0+mkL7/8krZu3UobNmyg4uJiKi4upgcffJB2795NhYWFFBoa2if5mclkoldeeUWcr3PnztG6det6fdfhcNCOHTvEdVuzZg1NnTqVzpw5Q0eOHKE5c+aQXq+nsrIy2rBhAxUVFdGUKVNIr9dTa2srpaamUktLC7W1tdEjjzxChYWFdOXKFXrhhRcEeROH3W4nq9VKkyZNoqysLCouLqZ9+/aRVqulefPm0blz5+jixYv00ksv0fnz56m4uJieffZZunbtWjdiNrPZTDU1NbRr1y5avnw5lZaW0qlTp+gnP/kJnT9/ni5fvkx/+MMf6OjRo9Te3k5Op5OuXbtG7e3t5Ha76cSJE7Rx40Y6ePAgqdVq2rVrF91///2CjEomk1FpaSkdOHCApkyZQmVlZVRdXU0nT56k1tZWysvLo8LCQiLyPhsPPvgg3XvvvXT27Fl655136NNPP6UVK1ZQVlYWXbhwgWbNmkX79++n6upqOnHiBD355JNUVVVFGzZsoLy8PKqvr6dly5ZRZWUlabVaKi0tFURpx48fp927d9MLL7xAZrOZGhoaKCQkhN5++206ceIEzZw5kz766CM6f/48/exnP6PW1lbatGkTPfDAA1RfX0//+Mc/BFGXx+Mht9tNNpuNNBoNPfvss/TYY4+RwWCguXPn0l/+8hcqKiqiLVu20MaNG+nkyZNUWVlJU6dOJa1WSzt27KBXX32Vqqurad68eXThwgW6cOECvfLKK7Rt2zbas2cP3XvvvbRr1y7Ky8ujZ599lqqrq3s9Q21tbfTKK6/QmDFjyGw2U319PeXm5tLBgwdJq9XSvn376PXXXyer1Sr2U6vV1NzcTGq1mqZPn05ERA0NDfTZZ5/RwoULSa1W07333ksjRowgg8FAxcXFtHz5ctq3b9/XIw4jIi0RXeh6bwZQASBxgF2WAdhCRA4iqgVQA2Da9Y5zPVRWVnbTO+Sr8G1tbQgLC0NxcbFY3Lp8+TI+//xzdHR0YNOmTZg5cyakUiliYmJQXl6OdevWCW6Wzz77DPHx8XjzzTfhdDoHzIThYZ+ysjJkZWUJgi3AWzyxePFiNDc344knnoDT6URsbCyWLl2KtrY2QXJWXl6OZcuWAYDgdge8ccV//etfeOaZZ3DXXXf1OjYR4dNPPxXxN5fLJfhrPB4P0tLSkJaWhvDwcFRWVuLpp5/G9OnTAQDV1dWYMGGCOJZare4lZMJtU6vViIiIQGdnJ1577TWEhISgtrYWKpUKbW1tkEgkgpvmypUrYvTCZyGAN+TBR1M2mw0nTpxAfX09IiMjYbFYsHjxYnR2dmLKlCkwmUwYP348VCoV2tvbYbFYMHv2bFRWVvaZIQB447F5eXmIjIyEw+FAY2Mjli5d2mua7Ha7cerUKYwaNQqdnZ1oa2uDwWBATEwMkpKSBE1weHg4VqxYAZVKhdjYWERGRoopNF9A/+Uvf4nY2Fg0Njbi9ttv71UAwyXh4uPjce7cOXz00UeYMWMGdDodamtrcfnyZRw/fhxLlizBiBEjBG2ATqfrFks2Go0ICAjAnj17UF5ejoSEBCiVSoSHh4uS/itXriAoKAhEBKlUiqCgIISFhaGtrQ1BQUFobGxEVFQUQkJCREiMU2VHR0cjIiICwcHBCA8PR25uLt59911ERkbCYDCgvLxccKSkpqaira0Ny5YtQ2BgIJYvX46pU6dCIpFg7ty5cLvdCA4OFlqle/bsQWJiIhhjuPvuuyGRSET8mZfPR0REID09HZmZmdBqtfjyyy8xbdo0kTkVExOD0NBQyGQyZGRkYPLkyWhvb4fNZsNTTz2F1NRUwYUfHR0t6LZ5hTEv5z927BhGjRoFiUQChUKBMWPGiNH6xIkTERoaimHDhiE0NBRhYWGIiopCZmam0LAtKipCfn4+5s2bh8mTJ8PhcGDevHlIS0uDTqfD9OnTe90DEokEISEhaGhoEKG91tZW7N69G1OmTIHdbofT6URra2s3LVepVAq5XA69Xi9m8Vw7NiUlRYRqMzMzxUzUbDYPKAoE3GTMnTE2AsBkAJz1/ynGWClj7EPGGJcxSgTgKw3UiIE7gwHBHdD27dvxyCOPdFMj37x5s3gwPvzwQ5FJc/DgQTidTkybNg0PPPCAcIKff/45Ghoa8Nprr2HOnDkAgG3btuEnP/kJnnrqqQH1WZubm3Hfffeho6MDO3fuxNq1awF446IzZszAggULoFKpEB0djUWLFnWrsktJScHChQths9mwefNmZGRkQKPR4P3338fw4cNhs9lQXl4Ok8mEN954o0/SMcYYNmzYILicL168KNgKz549ixUrVkAul+P8+fPQaDTYsGEDZs6cCcC7VvH4448D8DreCRMm4IEHHuhVCVhfX4/77rsPZrMZEokEYWFhKC8vx5o1a9De3o6IiAgcOHAAv//97xESEoItW7Zg1qxZALyhnIceegjAf4VQ8vPzERYWhr/97W8YOXIkQkJCUFpaiszMTFE0s2nTJowcORJutxsXLlxAXFwclEoltm/fjrVr1/aZh2y1WlFeXo67774b9fX1OHnyJJKTk7vx13O1q08//VRkdRQXF+P5559HbGws9Ho95s6di+DgYCQnJ0OlUqGoqAhr166FwWCA3W7Hww8/jOzsbDz66KNIS0tDWFgYTp8+LWz2RXJyMvR6PZ566ilcunQJLS0tqKmpgdFoxJw5c7Bw4UKsXLkSw4cPh1wuR15eHpYuXYrk5GQhZ8edd0hICLZv346ZM2eira0Ne/bswcmTJ0W83mg0Yty4cUL5Z9iwYULJKyEhAV988QUiIyMF7zl3Hh9++CFycnKg1WphsVjw4osv4plnnkFxcTH279+PP/3pT3j//fdRX1+PEydOQKFQIDIyEpmZmVAqlYJXpqKiAnPnzkVVVRXmzJkjMp8++ugjrF69GkSEkSNHIjAwECUlJRgzZgwCAwNhMBiwbt06/PjHP0Z6erqogp06dSpCQkJw6NAhPPfcc5g7dy6MRiMmTZokONCJCC+++CJ0Oh2eeOIJrFu3Dk888QSOHz/e7TpIJBKEhobi2rVryM7ORkNDg7in8vPzxbkZPnw4dDodgoKCBKvliBEjoNVqkZ2djYULF+IXv/iFuDcPHz4sCNsKCgqQkpLSKxWxubkZBoMBe/bswerVq1FaWor3338flZWVYIzBarXigw8+wH333SfCygEBAVAqlQgNDUV9fT2WLl2KsrIyJCUlYfv27RgxYgSIvNJ8y5Ytg1qtxt69e/GHP/xhwNAqcBPOnTEWAmAbgF8RkQnA3wGkApgEQAvgdf7VPnbvtbrIGHucMXaeMXZ+IGXzlpYWlJeXo6OjA4mJid1GZ4WFhXjsscfgdruh0WiQnp4uRhqLFi1CZWUlSkpKsHnzZlgsFhw5cgTt7e344IMPhOLRm2++iWPHjuHw4cPYtm1bv+04cuQIcnJyEBAQgDVr1mD37t2oqqrCyy+/LFR0Ll68iMWLF4t9HA4H/vOf/4gRbXFxMU6fPg2VSoV///vfsNvtuHDhAnQ6HbZt24aamhps3bq1X7HtX/3qV5DJZDh16hTeeustGI1GGI1GIS/G0+PMZjM2b94s9EAfeughfPLJJzh9+jRee+01nDhxQggscLjdbiQkJECv16O5uRmnTp1CcXExxo0bB7VajZKSEvz1r3/F6dOn8dBDD4HIK2Axc+ZMXL58Ge+99x4CAwPhcDiEZqnRaER9fT3WrVuHESNGoK6uDlarVeRlV+0lAAAOfUlEQVRmNzY2Ijo6GqWlpZDL5fjss89EnL6yshLjxo3rpWYFANHR0ViyZAkaGhqwd+9eVFVV4fjx4906K4VCgc7OTjz66KMidfDvf/87Vq5cCalUim3btonRJeDlbNmwYQOSkpJgsVhEJ9rU1IRz587h4sWLCA0NFdev52I4L+bRaDSCsZQLbigUCrS2tuLKlSuorKwU3CkajQalpaWCXEwqlQrn/7vf/Q6BgYGorKyEXC7Hv//9b0ycOBGnT58WTslqtaKxsRFNTU1ISEhAaGgozp8/j4SEBJw7dw7BwcG4ePEifv7zn8NoNMJsNmP8+PGIiopCUVGRINiaPXs20tPTkZycjOHDhyM8PBzDhg0T7KOcAIzLyslkMpSVlaGwsBBBQUGoq6tDaGgonnjiCVRXV4uOetKkSUhOTsa4ceNw8eJF2Gw2uN1uZGVlwe12IzY2Fs899xw2bdqEzZs3o7a2Fj/96U8hk8mwZcsWLFy4ENHR0di5c6cQySgrK0NdXR2qqqrw6quv9prl8lTRRx55BEVFRdiwYQNWrlyJO++8E3l5eWCMoba2FsHBwfB4PAgMDERTUxNkMhmioqIwceJEBAcHw2az4erVqzh16hQCAgJw9epVZGRk4Nq1a6iurhb1Br7gNA1ut1vI5D300ENYtWoV6urqUFdXhxUrVojOjIPIKy1ZWFgo1i8MBgMSExORlZUFtVoNlUoFpVKJ2267DS+//DKKioquK2EpvREFecZYIIBdAHYQ0YcAkJuba83NzaXc3Fxav379VQC/zs3NfXv9+vUTAYTn5uYWAMD69eufBPB5bm5uo+9v5ubmFuXm5r6Xm5v7Xl5eXi4fWfbEnj17oFarkZSUBIVCgVGjRonQSUdHB+666y5ERkYiKioKUVFRmDFjBiIiIjBt2jQUFhZCIpFgzpw5YpHl4sWLCAsLEzfF2LFjUVBQAKVSiUmTJvVbBPTll19i7ty5oqflmQdLly5FbGysUJC57bbbRPs4b8aYMWNE5V1AQADuuOMOxMTEYMSIEUhPT8eoUaOQkpKC2tpauN1uLFq0qM82jB07FoGBgYiKisLy5cvR1NQklJ3a2toQFRWFnJwcnDt3DkFBQZg3bx4AryINz0S58847BbOeL/i0lo8gVCqVUKcKCwtDTU0NpkyZIpSpODf46NGjERsbK85/QkICAgICcO3aNSxZsgSpqakIDw8XCk6jRo1CTEyMWGjiNK9xcXEwGAy48847MXLkSDQ3NyMiIgLz5s3rtcBtsVgQFxcHp9OJOXPmIDMzE/Hx8YLSloMxhoiICFgsFowYMQJZWVnweDwICgqC0+lEamqqmFpz6leZTIYJEyYgMDBQLPZFREQgJiYG4eHh2LBhA9auXdsnWRhXuGpoaEBWVhaSkpIEkyJfTB01ahTCwsKEStPo0aOFdq/dbodMJhP52XK5HB0dHZg2bRrGjBkDt9uN/fv3o729HatWrYJCoYBOp4PFYkFERATMZrMI1yQkJEClUsFoNGLu3LlISUkR6lo5OTlCMYwrXnV0dCA7O1uoL2VkZMBkMkEikSA2NlYIRoeGhqKtrQ0ymQzp6elISkpCYmIiwsLCIJVK0dTUBI/HI3RPOXVwSkoK4uLikJiYCIVCgZiYGAwfPhxxcXGw2+0YPXo0VCoVIiMjMWzYMLhcLsTGxiI0NBRyuRxxcXHIycnByJEjYbVaIZFIkJOTg/Dw8G5hLbvdLgqYWltb4XK5xKIrJ1z70Y9+BKVSibCwMGg0Gtx+++1ISUlBWFgYhg0bBr1ej7KyMiQkJCAjIwMqlQparRYzZsyATCaD2+1GRkZGr5EzdRVtcZWv+++/H+Hh4UhMTBRaC7Nnz+5VM8C57JOTk1FbW4ucnByoVCoQEZKTkxEfH4/29naMGjVKzIJ0Oh30ej12796tzc3N7bPA5brcMsx75j4CYCCiX/lsjycibdf7ZwFMJ6KVjLGxAD6BN86eAOAwgDQi6jfjfjBzy7z55psijrd69WqRjnQzVYk2m+0rl1T3BV9GObfbLYpNeuKbqF68HvqiHebMgryT4+3tj6ZVKpUKvhOg+/lqbGwU8fv+cKOsjDfDxcLbwLNWHn30UUyZMgUBAQH49a9/fUO/AXhnL9SVhserm/tL262rqxOc8VKpFDqdTjhHi8WC3/zmN7j99tsxe/ZskWbq+92oqChYLBa4XC7RYfAUVV/xdwDdzjcXb05KSoJMJhOFRgEBAWKdpbOzU2QGcbFoDrlcLtKFo6KicOXKFcjlcqGBwPPUe15LvvbDOyG+tnD48GHEx8fjjjvuQEBAwE3dxy6XC4cPH4bFYsG4ceOwceNG5ObmCrIw3+vvex348Tl4+mrP0OX17rW+6K4Hqp73vX5ctpFLfPKOhLN08tRH/pstLS2IjY39WtwyswA8DCCHMVbc9VoM4C+MsTLGWCmAbADPAgARXQSwFcAlAPsAPDmQYx/skMvlqKurw/DhwwF4nfrNlpt/k44d6F7gERAQ0G8Rxrft2AH0unF5IYfvwjQfQPT1UPC2+z5EvuerP/GSnse8EdzMdeNtcLlc8Hg8YkbjKwJ+I1Aqld0KgZxOp9Db9K2t4E7QV7WJh0d4B2M0GpGZmSmSAQD06hglEkm3vHmu5NQTPKPi6tWrMBqNsNlsIn/a9/+RkZFCNYw7Lj67DQkJEfvK5XLBNso7xZ7Vqz076cDAQJjNZlitVoSHh4u2qlQqBAcHi3b3dR/3V9tgt9uh1WpRUFCAgoICEUrli52+4Bqy/Dr5gi/O9sT17jWJRNLtmRjIsfNZhsvlEvcbv1f4sfnxuPC3729ej37Azwo5AHg5N+C92N8lh833GS0tLUIcYrBjoFGh7/Vua2sToagbhe/+Op1OhBB6jkZbW1vhcDhESGD06NEixg14H2a+IBkYGIjIyEhBk6tQKOBwOEBEUCgUsFgsMBqN3RxpX5zvvNy/ra1N1EiEhYV1G5maTCaYTCZER0eLzsfj8Qgn6TsT0mq1IsuKO6iWlhYx+uSCJE6nE3FxcdDr9aiqqhLEYWlpad041X1H+TxsERAQAJPJBMZYn+FTzijJj8c7LKvVisjIyG4Vt/wcBAYGwmg0wul0dlvkZIyJIit+rbiICg9p9ecPTCZTn+L0PWG32wUfDr8+7e3t4n1ISIiYmfMqW25naGgon/UMbsrfO+64g7g2qh/fTxARDAaDWDQLCQkR3CEKhUJUaPqGarhD4tN6/lBxeDwe8fmrdKrkQ6Pa14PIY8qAd8TUnwPk6xH88/WI4zhnTmdnJ+RyOex2u3g4uSADH2Hzka1EIhFVt1yZx2QyISkpqRvVLx/xc9uCgoKEA+M2c2ZHuVwOj8cjOonAwEB4PB5YrVbhgBQKhajojI6OFnH45ubmbhWQDocDdrsdSqUSERER8Hg8YjTJbeROT6/Xw2aziUXisLAwUZEdHh4uJPbq6upEyEqpVIp7gtvJOy4u4BEcHAyr1QrGmBjZBgYGCkUrqVQKi8UiMoh4J2W1WtHR0SFmA9TFw8NfH3/8MYxGIx555BHExcXBbDbD6XQiKCgIcrlcZKvw7/PzztvqcDjgcrkEXw2/Fowx4ZD5sTkjptVqRXJycrd70m63w+FwQKFQiH35rMNsNovwq8vlQlRUFKxWK5RK5eB27oyxVgBWALpb3ZZbgCj47f4hwW/3Dwvftt0pRNQn3emgcO4AwBg7318PNJTht/uHBb/dPyzcSrv9AWQ//PDDjyEIv3P3ww8//BiCGEzOffAoTXy38Nv9w4Lf7h8Wbpndgybm7ocffvjhxzeHwTRy98MPP/zw4xvCLXfujLGFjLEqxlgNY+y3t7o93yS62DJbGGPlPtsiGWMHGWPVXX8jfP73Qtd5qGKMLbg1rf76YIwlM8aOMsYqGGMXGWP/p2v7kLadMaZgjJ1ljJV02b2+a/uQtpuDMSZljP2HMfZF1+chbzdjrK6rUr+YMXa+a9vgsLs/ovfv4gVACq+YxygAMnhFPm6/lW36hu27E8AUAOU+2/4C4Ldd738L4H+73t+0yMlgfaF/gZchbTu8jKghXe8D4aXGzhzqdvvY/3/h5ZX6ouvzkLcbQB2AqB7bBoXdt3rkPg1ADRFdJSIngC3win0MCRDRcQCGHpuXwUvEhq6/y322f+MiJ7cC1L/Ay5C2nbzg4raBXS/CELcbABhjSQCWAPiHz+Yhb3c/GBR232rn/o0Ke3xPEEtdbJpdfzkJy5A8F6y7wMuQt70rNFEMoAXAQSL6QdgN4A0Az8Ors8zxQ7CbABxgjBUxxjhv+aCw+8bo9L493JCwxw8EQ+5c9BR48eWN6fnVPrZ9L20nLwPqJMaYEsAOxti4Ab4+JOxmjP0YQAsRFTHG5tzILn1s+97Z3YVZRKRhjMUAOMgYqxzgu9+p3bd65N4IINnncxIAzS1qy3eFZsZYPODlxId3hAcMsXPRJfCyDUAeEW3v2vyDsB0AiMgIIB/AQgx9u2cBuIcxVgdvaDWHMbYZQ99uEJGm628LgB3whlkGhd232rmfA5DGGBvJGJMBWAlg9y1u07eN3QAe7Xr/KLwKV3z7SsaYnDE2EkAagLO3oH1fG8w7RP8AQAUR/T+ffw1p2xlj0V0jdjDGhgGYB6ASQ9xuInqBiJKIaAS8z/ARIlqFIW43YyyYMRbK3wOYD6Acg8XuQbDavBjebIorAP7nVrfnG7btX/Dqy7rg7bV/AUAFrzpVddffSJ/v/0/XeagCsOhWt/9r2J0F73SzFEBx12vxULcdwAQA/+myuxzAS13bh7TdPc7BHPw3W2ZI2w1vll9J1+si91+DxW5/haoffvjhxxDErQ7L+OGHH3748S3A79z98MMPP4Yg/M7dDz/88GMIwu/c/fDDDz+GIPzO3Q8//PBjCMLv3P3www8/hiD8zt0PP/zwYwjC79z98MMPP4Yg/j8FjSEjUKJl8AAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "imshow(pred, cmap='gray')" - ] - }, - { - "cell_type": "code", - "execution_count": 73, - "metadata": {}, - "outputs": [], - "source": [ - "output_dir = 'test_preds'\n", - "if not os.path.exists(output_dir):\n", - " os.makedirs(output_dir)\n", - "for x_test_file in x_test:\n", - " img = cv2.imread(x_test_file, cv2.IMREAD_GRAYSCALE)\n", - " img = cv2.resize(img, input_shape[::-1], interpolation=cv2.INTER_AREA)\n", - " pred = make_prediction(img)\n", - " filename = path_leaf(x_test_file)\n", - " filepath = os.path.join(output_dir, filename)\n", - " cv2.imwrite(filepath, pred)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "lightweight-gpu-kernel", - "language": "python", - "name": "lightweight-gpu-kernel" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.1" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/test/apis/live-reloading/python/mpg-estimator/cortex.yaml b/test/apis/live-reloading/python/mpg-estimator/cortex.yaml deleted file mode 100644 index 002bafa67c..0000000000 --- a/test/apis/live-reloading/python/mpg-estimator/cortex.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- name: mpg-estimator - kind: RealtimeAPI - handler: - type: python - path: handler.py - multi_model_reloading: - path: s3://cortex-examples/sklearn/mpg-estimator/linreg/ diff --git a/test/apis/live-reloading/python/mpg-estimator/handler.py b/test/apis/live-reloading/python/mpg-estimator/handler.py deleted file mode 100644 index de1296d555..0000000000 --- a/test/apis/live-reloading/python/mpg-estimator/handler.py +++ /dev/null @@ -1,24 +0,0 @@ -import mlflow.sklearn - - -class Handler: - def __init__(self, config, model_client): - self.client = model_client - - def load_model(self, model_path): - return mlflow.sklearn.load_model(model_path) - - def handle_post(self, payload, query_params): - model_version = query_params.get("version", "latest") - - model = self.client.get_model(model_version=model_version) - model_input = [ - payload["cylinders"], - payload["displacement"], - payload["horsepower"], - payload["weight"], - payload["acceleration"], - ] - result = model.predict([model_input]).item() - - return {"prediction": result, "model": {"version": model_version}} diff --git a/test/apis/live-reloading/python/mpg-estimator/requirements.txt b/test/apis/live-reloading/python/mpg-estimator/requirements.txt deleted file mode 100644 index cbcad6b321..0000000000 --- a/test/apis/live-reloading/python/mpg-estimator/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -mlflow -pandas -numpy -scikit-learn==0.21.3 diff --git a/test/apis/live-reloading/python/mpg-estimator/sample.json b/test/apis/live-reloading/python/mpg-estimator/sample.json deleted file mode 100644 index 2dbbca46dd..0000000000 --- a/test/apis/live-reloading/python/mpg-estimator/sample.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "cylinders": 4, - "displacement": 135, - "horsepower": 84, - "weight": 2490, - "acceleration": 15.7 -} diff --git a/test/apis/live-reloading/tensorflow/README.md b/test/apis/live-reloading/tensorflow/README.md deleted file mode 100644 index 8fea3d3f63..0000000000 --- a/test/apis/live-reloading/tensorflow/README.md +++ /dev/null @@ -1,9 +0,0 @@ -## Live-reloading model APIs - -The model live-reloading feature is automatically enabled 1 for the TensorFlow Handler. This means that any TensorFlow examples found in the [examples](../..) directory will already have this running. - -The live-reloading is a feature that reloads models at run-time from (a) specified S3 bucket(s) in the `cortex.yaml` config of each API. Models are added/removed from the API when the said models are added/removed from the S3 bucket(s) or reloaded when the models are edited. More on this in the docs. - ---- - -*1: The live-reloading feature for the TensorFlow handler is disabled when Inferentia resources (`compute.inf`) are added to the API and `processes_per_replica` > 1.* diff --git a/test/apis/model-caching/onnx/multi-model-classifier/README.md b/test/apis/model-caching/onnx/multi-model-classifier/README.md deleted file mode 100644 index e57dfe47c4..0000000000 --- a/test/apis/model-caching/onnx/multi-model-classifier/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# Multi-Model Classifier API - -This example deploys ResNet50, MobileNet and ShuffleNet models in one API. Query parameters are used for selecting the model and the version. - -Since model caching is enabled, there can only be 2 models loaded into memory - loading a 3rd one will lead to the removal of the least recently used one. To witness the adding/removal process of models, check the logs of the API by running `cortex logs multi-model-classifier` once the API is up. - -The example can be run on both CPU and on GPU hardware. - -## Sample Prediction - -Deploy the model by running: - -```bash -cortex deploy -``` - -And wait for it to become live by tracking its status with `cortex get --watch`. - -Once the API has been successfully deployed, export the API's endpoint for convenience. You can get the API's endpoint by running `cortex get multi-model-classifier`. - -```bash -export ENDPOINT=your-api-endpoint -``` - -When making a prediction with [sample.json](sample.json), the following image will be used: - -![cat](https://i.imgur.com/213xcvs.jpg) - -### ResNet50 Classifier - -Make a request to the ResNet50 model: - -```bash -curl "${ENDPOINT}?model=resnet50" -X POST -H "Content-Type: application/json" -d @sample.json -``` - -The expected response is: - -```json -{"label": "tabby", "model": {"name": "resnet50", "version": "latest"}} -``` - -### MobileNet Classifier - -Make a request to the MobileNet model: - -```bash -curl "${ENDPOINT}?model=mobilenet" -X POST -H "Content-Type: application/json" -d @sample.json -``` - -The expected response is: - -```json -{"label": "tabby", "model": {"name": "mobilenet", "version": "latest"}} -``` - -### ShuffleNet Classifier - -At this point, there are 2 models loaded into memory (as specified by `cache_size`). Loading `ShuffleNet` as well will lead to the removal of the least recently used model - in this case, it will be the ResNet50 model that will get evicted. Since the `disk_cache_size` is set to 3, no model will be removed from disk. - -Make a request to the ShuffleNet model: - -```bash -curl "${ENDPOINT}?model=shufflenet" -X POST -H "Content-Type: application/json" -d @sample.json -``` - -The expected response is: - -```json -{"label": "Egyptian_cat", "model": {"name": "shufflenet", "version": "latest"}} -``` - ---- - -Now, inspect `cortex get multi-model-classifier` to see when and which models were removed in this process of making requests to different versions of the same model. diff --git a/test/apis/model-caching/onnx/multi-model-classifier/cortex.yaml b/test/apis/model-caching/onnx/multi-model-classifier/cortex.yaml deleted file mode 100644 index 9e127bc99d..0000000000 --- a/test/apis/model-caching/onnx/multi-model-classifier/cortex.yaml +++ /dev/null @@ -1,20 +0,0 @@ -- name: multi-model-classifier - kind: RealtimeAPI - handler: - type: python - path: handler.py - multi_model_reloading: - paths: - - name: resnet50 - path: s3://cortex-examples/onnx/resnet50/ - - name: mobilenet - path: s3://cortex-examples/onnx/mobilenet/ - - name: shufflenet - path: s3://cortex-examples/onnx/shufflenet/ - cache_size: 2 - disk_cache_size: 3 - config: - image-classifier-classes: https://s3.amazonaws.com/deep-learning-models/image-models/imagenet_class_index.json - image-resize: 224 - compute: - mem: 2G diff --git a/test/apis/model-caching/onnx/multi-model-classifier/dependencies.sh b/test/apis/model-caching/onnx/multi-model-classifier/dependencies.sh deleted file mode 100644 index 0577b68228..0000000000 --- a/test/apis/model-caching/onnx/multi-model-classifier/dependencies.sh +++ /dev/null @@ -1 +0,0 @@ -apt-get update && apt-get install -y libsm6 libxext6 libxrender-dev diff --git a/test/apis/model-caching/onnx/multi-model-classifier/handler.py b/test/apis/model-caching/onnx/multi-model-classifier/handler.py deleted file mode 100644 index 8ff7f62e04..0000000000 --- a/test/apis/model-caching/onnx/multi-model-classifier/handler.py +++ /dev/null @@ -1,118 +0,0 @@ -import os -import numpy as np -import onnxruntime as rt -import cv2, requests -from scipy.special import softmax - - -def get_url_image(url_image): - """ - Get numpy image from URL image. - """ - resp = requests.get(url_image, stream=True).raw - image = np.asarray(bytearray(resp.read()), dtype="uint8") - image = cv2.imdecode(image, cv2.IMREAD_COLOR) - return image - - -def image_resize(image, width=None, height=None, inter=cv2.INTER_AREA): - """ - Resize a numpy image. - """ - dim = None - (h, w) = image.shape[:2] - - if width is None and height is None: - return image - - if width is None: - # calculate the ratio of the height and construct the dimensions - r = height / float(h) - dim = (int(w * r), height) - else: - # calculate the ratio of the width and construct the dimensions - r = width / float(w) - dim = (width, int(h * r)) - - resized = cv2.resize(image, dim, interpolation=inter) - - return resized - - -def preprocess(img_data): - """ - Normalize input for inference. - """ - # move pixel color dimension to position 0 - img = np.moveaxis(img_data, 2, 0) - - mean_vec = np.array([0.485, 0.456, 0.406]) - stddev_vec = np.array([0.229, 0.224, 0.225]) - norm_img_data = np.zeros(img.shape).astype("float32") - for i in range(img.shape[0]): - # for each pixel in each channel, divide the value by 255 to get value between [0, 1] and then normalize - norm_img_data[i, :, :] = (img[i, :, :] / 255 - mean_vec[i]) / stddev_vec[i] - - # extend to batch size of 1 - norm_img_data = norm_img_data[np.newaxis, ...] - return norm_img_data - - -def postprocess(results): - """ - Eliminates all dimensions of size 1, softmaxes the input and then returns the index of the element with the highest value. - """ - squeezed = np.squeeze(results) - maxed = softmax(squeezed) - result = np.argmax(maxed) - return result - - -class Handler: - def __init__(self, model_client, config): - # onnx client - self.client = model_client - - # for image classifiers - classes = requests.get(config["image-classifier-classes"]).json() - self.image_classes = [classes[str(k)][1] for k in range(len(classes))] - self.resize_value = config["image-resize"] - - def handle_post(self, payload, query_params): - # get request params - model_name = query_params["model"] - img_url = payload["url"] - - # process the input - img = get_url_image(img_url) - img = image_resize(img, height=self.resize_value) - img = preprocess(img) - - # predict - model = self.client.get_model(model_name) - session = model["session"] - input_name = model["input_name"] - output_name = model["output_name"] - input_dict = { - input_name: img, - } - results = session.run([output_name], input_dict)[0] - - # interpret result - result = postprocess(results) - predicted_label = self.image_classes[result] - - return {"label": predicted_label} - - def load_model(self, model_path): - """ - Load ONNX model from disk. - """ - - model_path = os.path.join(model_path, os.listdir(model_path)[0]) - session = rt.InferenceSession(model_path) - return { - "session": session, - "input_name": session.get_inputs()[0].name, - "output_name": session.get_outputs()[0].name, - } diff --git a/test/apis/model-caching/onnx/multi-model-classifier/requirements.txt b/test/apis/model-caching/onnx/multi-model-classifier/requirements.txt deleted file mode 100644 index 46a61881fb..0000000000 --- a/test/apis/model-caching/onnx/multi-model-classifier/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -opencv-python==4.2.0.34 -onnxruntime==1.6.0 -numpy==1.19.1 -scipy==1.4.1 diff --git a/test/apis/model-caching/onnx/multi-model-classifier/sample.json b/test/apis/model-caching/onnx/multi-model-classifier/sample.json deleted file mode 100644 index 4ee3aa45df..0000000000 --- a/test/apis/model-caching/onnx/multi-model-classifier/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "url": "https://i.imgur.com/213xcvs.jpg" -} diff --git a/test/apis/model-caching/python/mpg-estimator/README.md b/test/apis/model-caching/python/mpg-estimator/README.md deleted file mode 100644 index 9cd05e5200..0000000000 --- a/test/apis/model-caching/python/mpg-estimator/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# MPG Estimator API - -This example deploys an MPG estimator model of multiple versions in one API. Query parameters are used for selecting the model and the version. - -Since model caching is enabled, there can only be 2 models loaded into memory (counting the versioned models as well) - loading a 3rd one will lead to the removal of the least recently used one. To witness the adding/removal process of models, check the logs of the API by running `cortex logs mpg-estimator` once the API is up. - -The example can be run on both CPU and on GPU hardware. - -## Sample Prediction - -Deploy the model by running: - -```bash -cortex deploy -``` - -And wait for it to become live by tracking its status with `cortex get --watch`. - -Once the API has been successfully deployed, export the API's endpoint for convenience. You can get the API's endpoint by running `cortex get mpg-estimator`. - -```bash -export ENDPOINT=your-api-endpoint -``` - -### Version 1 - -Make a request version `1` of the `mpg-estimator` model: - -```bash -curl "${ENDPOINT}?version=1" -X POST -H "Content-Type: application/json" -d @sample.json -``` - -The expected response is: - -```json -{"prediction": 26.929889872154185, "model": {"name": "mpg-estimator", "version": "1"}} -``` - -### Version 2 - -At this point, there is one model loaded into memory (as specified by `cache_size`). Loading another versioned model as well will lead to the removal of the least recently used model - in this case, it will be version 1 that will get evicted. Since the `disk_cache_size` is set to 2, no model will be removed from disk. - -Make a request version `2` of the `mpg-estimator` model: - -```bash -curl "${ENDPOINT}?version=2" -X POST -H "Content-Type: application/json" -d @sample.json -``` - -The expected response is: - -```json -{"prediction": 26.929889872154185, "model": {"name": "mpg-estimator", "version": "2"}} -``` - -### Version 3 - -With the following request, version 2 of the model will have to be evicted from the memory. Since `disk_cache_size` is set to 2, this time, version 1 of the model will get removed from the disk. - -Make a request version `3` of the `mpg-estimator` model: - -```bash -curl "${ENDPOINT}?version=3" -X POST -H "Content-Type: application/json" -d @sample.json -``` - -The expected response is: - -```json -{"prediction": 26.929889872154185, "model": {"name": "mpg-estimator", "version": "3"}} -``` - ---- - -Now, inspect `cortex get mpg-estimator` to see when and which models were removed in this process of making requests to different versions of the same model. The same algorithm is applied to different models as well, not just for the versions of a specific model. diff --git a/test/apis/model-caching/python/mpg-estimator/cortex.yaml b/test/apis/model-caching/python/mpg-estimator/cortex.yaml deleted file mode 100644 index 4f340dfd9a..0000000000 --- a/test/apis/model-caching/python/mpg-estimator/cortex.yaml +++ /dev/null @@ -1,11 +0,0 @@ -- name: mpg-estimator - kind: RealtimeAPI - handler: - type: python - path: handler.py - multi_model_reloading: - paths: - - name: mpg-estimator - path: s3://cortex-examples/sklearn/mpg-estimator/linreg/ - cache_size: 1 - disk_cache_size: 2 diff --git a/test/apis/model-caching/python/mpg-estimator/handler.py b/test/apis/model-caching/python/mpg-estimator/handler.py deleted file mode 100644 index f2b0c2cd50..0000000000 --- a/test/apis/model-caching/python/mpg-estimator/handler.py +++ /dev/null @@ -1,25 +0,0 @@ -import mlflow.sklearn - - -class Handler: - def __init__(self, config, model_client): - self.client = model_client - - def load_model(self, model_path): - return mlflow.sklearn.load_model(model_path) - - def handle_post(self, payload, query_params): - model_name = "mpg-estimator" - model_version = query_params.get("version", "latest") - - model = self.client.get_model(model_name, model_version) - model_input = [ - payload["cylinders"], - payload["displacement"], - payload["horsepower"], - payload["weight"], - payload["acceleration"], - ] - result = model.predict([model_input]).item() - - return {"prediction": result, "model": {"name": model_name, "version": model_version}} diff --git a/test/apis/model-caching/python/mpg-estimator/requirements.txt b/test/apis/model-caching/python/mpg-estimator/requirements.txt deleted file mode 100644 index cbcad6b321..0000000000 --- a/test/apis/model-caching/python/mpg-estimator/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -mlflow -pandas -numpy -scikit-learn==0.21.3 diff --git a/test/apis/model-caching/python/mpg-estimator/sample.json b/test/apis/model-caching/python/mpg-estimator/sample.json deleted file mode 100644 index 2dbbca46dd..0000000000 --- a/test/apis/model-caching/python/mpg-estimator/sample.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "cylinders": 4, - "displacement": 135, - "horsepower": 84, - "weight": 2490, - "acceleration": 15.7 -} diff --git a/test/apis/model-caching/python/translator/README.md b/test/apis/model-caching/python/translator/README.md deleted file mode 100644 index ddb1c5cb1f..0000000000 --- a/test/apis/model-caching/python/translator/README.md +++ /dev/null @@ -1,123 +0,0 @@ -# Translator API - -This project implements a multi-lingual translation API, supporting translations between over 150 languages, using +1,000 large pre-trained models served from a single EC2 instance via Cortex: - - -```bash -curl https://***.amazonaws.com/translator -X POST -H "Content-Type: application/json" -d -{"source_language": "en", "destination_language": "phi", "text": "It is a mistake to think you can solve any major problems just with potatoes." } - -{"generated_text": "Sayop an paghunahuna nga masulbad mo ang bisan ano nga dagkong mga problema nga may patatas lamang."} -``` - -Priorities of this project include: - -- __Cost effectiveness.__ Each language-to-language translation is handled by a different ~300 MB model. Traditional setups would deploy all +1,000 models across many servers to ensure availability, but this API can be run on a single server thanks to Cortex's multi-model caching. -- __Ease of use.__ Predictions are generated using Hugging Face's Transformer Library and Cortex's Handler API, while the translation service itself runs on a Cortex cluster self-hosted on your AWS account. -- __Configurability.__ All tools used in this API are fully open source and modifiable. The deployed service and underlying infrastructure run on your AWS account. The prediction API can be run on CPU and GPU instances. - -## Models used - -This project uses pre-trained Opus MT neural machine translation models, trained by Jörg Tiedemann and the Language Technology Research Group at the University of Helsinki. The models are hosted for free by Hugging Face. For the full list of language-to-language models, you can view the model repository [here.](https://huggingface.co/Helsinki-NLP) - -## How to deploy the API - -To deploy the API, first spin up a Cortex cluster by running `$ cortex cluster up cortex.yaml`. Note that the configuration file we are providing Cortex with (accessible at `cortex.yaml`) requests a g4dn.xlarge GPU instance. If your AWS account does not have access to GPU instances, you can request an EC2 service quota increase easily [here](https://console.aws.amazon.com/servicequotas), or you can simply use CPU instances (CPU will still work, you will just likely experience higher latency). - -```bash -$ cortex cluster up cortex.yaml - -email address [press enter to skip]: - -verifying your configuration ... - -aws access key id ******************** will be used to provision a cluster named "cortex" in us-east-1: - -○ using existing s3 bucket: cortex-***** ✓ -○ using existing cloudwatch log group: cortex ✓ -○ creating cloudwatch dashboard: cortex ✓ -○ spinning up the cluster (this will take about 15 minutes) ... -○ updating cluster configuration ✓ -○ configuring networking ✓ -○ configuring autoscaling ✓ -○ configuring logging ✓ -○ configuring metrics ✓ -○ configuring gpu support ✓ -○ starting operator ✓ -○ waiting for load balancers ...... ✓ -○ downloading docker images ✓ - -cortex is ready! - -``` - -Once the cluster is spun up (roughly 20 minutes), we can deploy by running: - -```bash -cortex deploy -``` - -Now, we wait for the API to become live. You can track its status with `cortex get --watch`. - -Note that after the API goes live, we may need to wait a few minutes for it to register all the models hosted in the S3 bucket. Because the bucket is so large, it takes Cortex a bit longer than usual. When it's done, running `cortex get translator` should return something like: - -``` -cortex get translator - -status up-to-date requested last update avg request 2XX -live 1 1 3m -- -- - -metrics dashboard: https://us-east-1.console.aws.amazon.com/cloudwatch/home#dashboards:name=*** - -endpoint: http://***.elb.us-east-1.amazonaws.com/translator -example: curl: curl http://***.elb.us-east-1.amazonaws.com/translator -X POST -H "Content-Type: application/json" -d @sample.json - -model name model version edit time -marian_converted_v1 1 (latest) 24 Aug 20 14:23:41 EDT -opus-mt-NORTH_EU-NORTH_EU 1 (latest) 21 Aug 20 10:42:38 EDT -opus-mt-ROMANCE-en 1 (latest) 21 Aug 20 10:42:38 EDT -opus-mt-SCANDINAVIA-SCANDINAVIA 1 (latest) 21 Aug 20 10:42:38 EDT -opus-mt-aav-en 1 (latest) 21 Aug 20 10:42:38 EDT -opus-mt-aed-es 1 (latest) 21 Aug 20 10:42:38 EDT -opus-mt-af-de 1 (latest) 21 Aug 20 10:42:38 EDT -opus-mt-af-en 1 (latest) 21 Aug 20 10:42:38 EDT -opus-mt-af-eo 1 (latest) 21 Aug 20 10:42:38 EDT -opus-mt-af-es 1 (latest) 21 Aug 20 10:42:38 EDT -opus-mt-af-fi 1 (latest) 21 Aug 20 10:42:38 EDT -opus-mt-af-fr 1 (latest) 21 Aug 20 10:42:38 EDT -opus-mt-af-nl 1 (latest) 21 Aug 20 10:42:38 EDT -opus-mt-af-ru 1 (latest) 21 Aug 20 10:42:38 EDT -opus-mt-af-sv 1 (latest) 21 Aug 20 10:42:38 EDT -opus-mt-afa-afa 1 (latest) 21 Aug 20 10:42:38 EDT -... -``` - -Once Cortex has indexed all +1,000 models, we can now query the API at the endpoint given, structuring the body of our request according to the format expected by our handler (specified in `handler.py`): - -``` -{ - "source_language": "en", - "destination_language": "es", - "text": "So long and thanks for all the fish." -} -``` - -The response should look something like this: - -``` -{"generated_text": "Hasta luego y gracias por todos los peces."} -``` - -The API, as currently defined, uses the two-letter codes used by the Helsinki NLP team to abbreviate languages. If you're unsure of a particular language's code, check the model names. Additionally, you can easily implement logic on the frontend or within your API itself to parse different abbreviations. - -## Performance - -The first time you request a specific language-to-language translation, the model will be downloaded from S3, which may take some time (~60s, depending on bandwidth). Every subsequent request will be much faster, as the API is defined as being able to hold 250 models on disk and 5 in memory. Models already loaded into memory will serve predictions fastest (a couple seconds at most with GPU), while those on disk will take slightly longer as they need to be swapped into memory. Instances with more memory and disk space can naturally hold more models. - -As for caching logic, when space is full, models are removed from both memory and disk according to which model was used last. You can read more about how caching works in the [Cortex docs.](https://docs.cortex.dev/) - -Finally, note that this project places a heavy emphasis on cost savings, to the detriment of optimal performance. If you are interested in improving performance, there are a number of changes you can make. For example, if you know which models are most likely to be needed, you can "warm up" the API by calling them immediately after deploy. Alternatively, if you have a handful of translation requests that comprise the bulk of your workload, you can deploy a separate API containing just those models, and route traffic accordingly. You will increase cost (though still benefit greatly from multi-model caching), but you will also significantly improve the overall latency of your system. - - ## Projects to thank - -This project is built on top of many free and open source tools. If you enjoy it, please consider supporting them by leaving a Star on their GitHub repo. These projects include Cortex, Transformers, and Helsinki NLP's Opus MT, as well as the many tools used under the hood by each. diff --git a/test/apis/model-caching/python/translator/cluster.yaml b/test/apis/model-caching/python/translator/cluster.yaml deleted file mode 100644 index e6e7e63020..0000000000 --- a/test/apis/model-caching/python/translator/cluster.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# EKS cluster name for cortex (default: cortex) -cluster_name: cortex - -# AWS region -region: us-east-1 - -# instance type -instance_type: g4dn.xlarge - -# minimum number of instances (must be >= 0) -min_instances: 1 - -# maximum number of instances (must be >= 1) -max_instances: 2 - -# disk storage size per instance (GB) (default: 50) -instance_volume_size: 125 diff --git a/test/apis/model-caching/python/translator/cortex.yaml b/test/apis/model-caching/python/translator/cortex.yaml deleted file mode 100644 index 331c17fa87..0000000000 --- a/test/apis/model-caching/python/translator/cortex.yaml +++ /dev/null @@ -1,12 +0,0 @@ -- name: translator - kind: RealtimeAPI - handler: - type: python - path: handler.py - multi_model_reloading: - dir: s3://models.huggingface.co/bert/Helsinki-NLP/ - cache_size: 5 - disk_cache_size: 250 - compute: - cpu: 1 - gpu: 1 # this is optional, since the api can also run on cpu diff --git a/test/apis/model-caching/python/translator/handler.py b/test/apis/model-caching/python/translator/handler.py deleted file mode 100644 index d1cbae34c1..0000000000 --- a/test/apis/model-caching/python/translator/handler.py +++ /dev/null @@ -1,24 +0,0 @@ -from transformers import MarianMTModel, MarianTokenizer, pipeline -import torch - - -class Handler: - def __init__(self, config, model_client): - self.client = model_client - self.device = torch.cuda.current_device() if torch.cuda.is_available() else -1 - - def load_model(self, model_path): - return MarianMTModel.from_pretrained(model_path, local_files_only=True) - - def handle_post(self, payload): - model_name = "opus-mt-" + payload["source_language"] + "-" + payload["destination_language"] - tokenizer_path = "Helsinki-NLP/" + model_name - model = self.client.get_model(model_name) - tokenizer = MarianTokenizer.from_pretrained(tokenizer_path) - - inf_pipeline = pipeline( - "text2text-generation", model=model, tokenizer=tokenizer, device=self.device - ) - result = inf_pipeline(payload["text"]) - - return result[0] diff --git a/test/apis/model-caching/python/translator/requirements.txt b/test/apis/model-caching/python/translator/requirements.txt deleted file mode 100644 index 78dded5eeb..0000000000 --- a/test/apis/model-caching/python/translator/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -transformers==3.5.1 -torch==1.7.1 diff --git a/test/apis/model-caching/python/translator/sample.json b/test/apis/model-caching/python/translator/sample.json deleted file mode 100644 index 9180fc52d7..0000000000 --- a/test/apis/model-caching/python/translator/sample.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "source_language": "en", - "destination_language": "es", - "text": "So long and thanks for all the fish." -} diff --git a/test/apis/model-caching/python/translator/sample2.json b/test/apis/model-caching/python/translator/sample2.json deleted file mode 100644 index a0241dc5dd..0000000000 --- a/test/apis/model-caching/python/translator/sample2.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "source_language": "en", - "destination_language": "he", - "text": "So long and thanks for all the fish." -} diff --git a/test/apis/model-caching/tensorflow/multi-model-classifier/README.md b/test/apis/model-caching/tensorflow/multi-model-classifier/README.md deleted file mode 100644 index dc24b7ff1b..0000000000 --- a/test/apis/model-caching/tensorflow/multi-model-classifier/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# Multi-Model Classifier API - -This example deploys Iris, ResNet50 and Inception models in one API. Query parameters are used for selecting the model. - -Since model caching is enabled, there can only be 2 models loaded into memory - loading a 3rd one will lead to the removal of the least recently used one. To witness the adding/removal process of models, check the logs of the API by running `cortex logs multi-model-classifier` once the API is up. - -The example can be run on both CPU and on GPU hardware. - -## Sample Prediction - -Deploy the model by running: - -```bash -cortex deploy -``` - -And wait for it to become live by tracking its status with `cortex get --watch`. - -Once the API has been successfully deployed, export the APIs endpoint. You can get the API's endpoint by running `cortex get multi-model-classifier`. - -```bash -export ENDPOINT=your-api-endpoint -``` - -When making a prediction with [sample-image.json](sample-image.json), the following image will be used: - -![sports car](https://i.imgur.com/zovGIKD.png) - -### ResNet50 Classifier - -Make a request to the ResNet50 model: - -```bash -curl "${ENDPOINT}?model=resnet50" -X POST -H "Content-Type: application/json" -d @sample-image.json -``` - -The expected response is: - -```json -{"label": "sports_car"} -``` - -### Inception Classifier - -Make a request to the Inception model: - -```bash -curl "${ENDPOINT}?model=inception" -X POST -H "Content-Type: application/json" -d @sample-image.json -``` - -The expected response is: - -```json -{"label": "sports_car"} -``` - -### Iris Classifier - -At this point, there are 2 models loaded into memory (as specified by `cache_size`). Loading the `iris` classifier will lead to the removal of the least recently used model - in this case, it will be the ResNet50 model that will get evicted. Since the `disk_cache_size` is set to 3, no model will be removed from disk. - -Make a request to the Iris model: - -```bash -curl "${ENDPOINT}?model=iris" -X POST -H "Content-Type: application/json" -d @sample-iris.json -``` - -The expected response is: - -```json -{"label": "setosa"} -``` - ---- - -Now, inspect `cortex get multi-model-classifier` to see when and which models were removed in this process of making requests to different versions of the same model. diff --git a/test/apis/model-caching/tensorflow/multi-model-classifier/cortex.yaml b/test/apis/model-caching/tensorflow/multi-model-classifier/cortex.yaml deleted file mode 100644 index 38c9c56858..0000000000 --- a/test/apis/model-caching/tensorflow/multi-model-classifier/cortex.yaml +++ /dev/null @@ -1,30 +0,0 @@ -- name: multi-model-classifier - kind: RealtimeAPI - handler: - type: tensorflow - path: handler.py - models: - paths: - - name: inception - path: s3://cortex-examples/tensorflow/image-classifier/inception/ - - name: iris - path: s3://cortex-examples/tensorflow/iris-classifier/nn/ - - name: resnet50 - path: s3://cortex-examples/tensorflow/resnet50/ - cache_size: 2 - disk_cache_size: 3 - config: - models: - iris: - labels: ["setosa", "versicolor", "virginica"] - resnet50: - input_shape: [224, 224] - input_key: input - output_key: output - inception: - input_shape: [224, 224] - input_key: images - output_key: classes - image-classifier-classes: https://s3.amazonaws.com/deep-learning-models/image-models/imagenet_class_index.json - compute: - mem: 2G diff --git a/test/apis/model-caching/tensorflow/multi-model-classifier/dependencies.sh b/test/apis/model-caching/tensorflow/multi-model-classifier/dependencies.sh deleted file mode 100644 index 057530cb85..0000000000 --- a/test/apis/model-caching/tensorflow/multi-model-classifier/dependencies.sh +++ /dev/null @@ -1 +0,0 @@ -apt-get update && apt-get install -y libgl1-mesa-glx libegl1-mesa diff --git a/test/apis/model-caching/tensorflow/multi-model-classifier/handler.py b/test/apis/model-caching/tensorflow/multi-model-classifier/handler.py deleted file mode 100644 index 83d4aac6b9..0000000000 --- a/test/apis/model-caching/tensorflow/multi-model-classifier/handler.py +++ /dev/null @@ -1,61 +0,0 @@ -import requests -import numpy as np -import cv2 - - -def get_url_image(url_image): - """ - Get numpy image from URL image. - """ - resp = requests.get(url_image, stream=True).raw - image = np.asarray(bytearray(resp.read()), dtype="uint8") - image = cv2.imdecode(image, cv2.IMREAD_COLOR) - image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) - return image - - -class Handler: - def __init__(self, tensorflow_client, config): - self.client = tensorflow_client - - # for image classifiers - classes = requests.get(config["image-classifier-classes"]).json() - self.image_classes = [classes[str(k)][1] for k in range(len(classes))] - - # assign "models"' key value to self.config for ease of use - self.config = config["models"] - - # for iris classifier - self.iris_labels = self.config["iris"]["labels"] - - def handle_post(self, payload, query_params): - model_name = query_params["model"] - model_version = query_params.get("version", "latest") - predicted_label = None - - if model_name == "iris": - prediction = self.client.predict(payload["input"], model_name, model_version) - predicted_class_id = int(prediction["class_ids"][0]) - predicted_label = self.iris_labels[predicted_class_id] - - elif model_name in ["resnet50", "inception"]: - predicted_label = self.predict_image_classifier(model_name, payload["url"]) - - return {"label": predicted_label, "model": {"model": model_name, "version": model_version}} - - def predict_image_classifier(self, model, img_url): - img = get_url_image(img_url) - img = cv2.resize( - img, tuple(self.config[model]["input_shape"]), interpolation=cv2.INTER_NEAREST - ) - if model == "inception": - img = img.astype("float32") / 255 - img = {self.config[model]["input_key"]: img[np.newaxis, ...]} - - results = self.client.predict(img, model)[self.config[model]["output_key"]] - result = np.argmax(results) - if model == "inception": - result -= 1 - predicted_label = self.image_classes[result] - - return predicted_label diff --git a/test/apis/model-caching/tensorflow/multi-model-classifier/requirements.txt b/test/apis/model-caching/tensorflow/multi-model-classifier/requirements.txt deleted file mode 100644 index c8e3dcc781..0000000000 --- a/test/apis/model-caching/tensorflow/multi-model-classifier/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -Pillow==8.1.1 -opencv-python==4.4.0.42 diff --git a/test/apis/model-caching/tensorflow/multi-model-classifier/sample-image.json b/test/apis/model-caching/tensorflow/multi-model-classifier/sample-image.json deleted file mode 100644 index 95200916c7..0000000000 --- a/test/apis/model-caching/tensorflow/multi-model-classifier/sample-image.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "url": "https://i.imgur.com/zovGIKD.png" -} diff --git a/test/apis/model-caching/tensorflow/multi-model-classifier/sample-iris.json b/test/apis/model-caching/tensorflow/multi-model-classifier/sample-iris.json deleted file mode 100644 index 67c03827f2..0000000000 --- a/test/apis/model-caching/tensorflow/multi-model-classifier/sample-iris.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "input": { - "sepal_length": 5.2, - "sepal_width": 3.6, - "petal_length": 1.4, - "petal_width": 0.3 - } -} diff --git a/test/apis/onnx/iris-classifier/cortex.yaml b/test/apis/onnx/iris-classifier/cortex.yaml deleted file mode 100644 index 9ad09cf772..0000000000 --- a/test/apis/onnx/iris-classifier/cortex.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- name: iris-classifier - kind: RealtimeAPI - handler: - type: python - path: handler.py - multi_model_reloading: - path: s3://cortex-examples/onnx/iris-classifier/ diff --git a/test/apis/onnx/iris-classifier/handler.py b/test/apis/onnx/iris-classifier/handler.py deleted file mode 100644 index 93e3cb0420..0000000000 --- a/test/apis/onnx/iris-classifier/handler.py +++ /dev/null @@ -1,37 +0,0 @@ -import os -import onnxruntime as rt -import numpy as np - -labels = ["setosa", "versicolor", "virginica"] - - -class Handler: - def __init__(self, model_client, config): - self.client = model_client - - def handle_post(self, payload): - session = self.client.get_model() - - input_dict = { - "input": np.array( - [ - payload["sepal_length"], - payload["sepal_width"], - payload["petal_length"], - payload["petal_width"], - ], - dtype="float32", - ).reshape(1, 4), - } - prediction = session.run(["label"], input_dict) - - predicted_class_id = prediction[0][0] - return labels[predicted_class_id] - - def load_model(self, model_path): - """ - Load ONNX model from disk. - """ - - model_path = os.path.join(model_path, os.listdir(model_path)[0]) - return rt.InferenceSession(model_path) diff --git a/test/apis/onnx/iris-classifier/requirements.txt b/test/apis/onnx/iris-classifier/requirements.txt deleted file mode 100644 index a8ef6d8489..0000000000 --- a/test/apis/onnx/iris-classifier/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -onnxruntime==1.6.0 -numpy==1.19.1 diff --git a/test/apis/onnx/iris-classifier/sample.json b/test/apis/onnx/iris-classifier/sample.json deleted file mode 100644 index 252c666b3a..0000000000 --- a/test/apis/onnx/iris-classifier/sample.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "sepal_length": 5.2, - "sepal_width": 3.6, - "petal_length": 1.4, - "petal_width": 0.3 -} diff --git a/test/apis/onnx/iris-classifier/xgboost.ipynb b/test/apis/onnx/iris-classifier/xgboost.ipynb deleted file mode 100644 index f06ad9eac9..0000000000 --- a/test/apis/onnx/iris-classifier/xgboost.ipynb +++ /dev/null @@ -1,231 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "iris_xgboost.ipynb", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.8" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "IiTxCwB7t6Ef", - "colab_type": "text" - }, - "source": [ - "# Training an Iris classifier using XGBoost\n", - "\n", - "In this notebook, we'll show how to train a classifier trained on the [iris data set](https://archive.ics.uci.edu/ml/datasets/iris) using XGBoost." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "j6QdLAUpuW7r", - "colab_type": "text" - }, - "source": [ - "## Install Dependencies\n", - "First, we'll install our dependencies:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "BQE5z_kHj9jV", - "colab_type": "code", - "colab": {} - }, - "source": [ - "pip install xgboost==0.90 scikit-learn==0.21.* onnxmltools==1.5.* boto3==1.*" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yEVK-sLnumqn", - "colab_type": "text" - }, - "source": [ - "## Load the data\n", - "We can use scikit-learn to load the Iris dataset:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "tx9Xw0x0lfbl", - "colab_type": "code", - "colab": {} - }, - "source": [ - "from sklearn.datasets import load_iris\n", - "from sklearn.model_selection import train_test_split\n", - "\n", - "iris = load_iris()\n", - "X, y = iris.data, iris.target\n", - "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.8, random_state=42)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "obGdgMm3urb2", - "colab_type": "text" - }, - "source": [ - "## Train the model\n", - "We'll use XGBoost's [`XGBClassifier`](https://xgboost.readthedocs.io/en/latest/python/python_api.html#xgboost.XGBClassifier) to train the model:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "jjYp8TaflhW0", - "colab_type": "code", - "colab": {} - }, - "source": [ - "import xgboost as xgb\n", - "\n", - "xgb_model = xgb.XGBClassifier()\n", - "xgb_model = xgb_model.fit(X_train, y_train)\n", - "\n", - "print(\"Test data accuracy of the xgb classifier is {:.2f}\".format(xgb_model.score(X_test, y_test))) # Accuracy should be > 90%" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Hdwu-wzJvJLb", - "colab_type": "text" - }, - "source": [ - "## Export the model\n", - "Now we can export the model in the ONNX format:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "AVgs2mkdllRn", - "colab_type": "code", - "colab": {} - }, - "source": [ - "from onnxmltools.convert import convert_xgboost\n", - "from onnxconverter_common.data_types import FloatTensorType\n", - "\n", - "onnx_model = convert_xgboost(xgb_model, initial_types=[(\"input\", FloatTensorType([1, 4]))])\n", - "\n", - "with open(\"gbtree.onnx\", \"wb\") as f:\n", - " f.write(onnx_model.SerializeToString())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ipVlP4yPxFxw", - "colab_type": "text" - }, - "source": [ - "## Upload the model to AWS\n", - "\n", - "Cortex loads models from AWS, so we need to upload the exported model." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "3IqsfyylxLhy", - "colab_type": "text" - }, - "source": [ - "Set these variables to configure your AWS credentials and model upload path:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "lc9LBH1uHT_h", - "colab_type": "code", - "cellView": "form", - "colab": {} - }, - "source": [ - "AWS_ACCESS_KEY_ID = \"\" #@param {type:\"string\"}\n", - "AWS_SECRET_ACCESS_KEY = \"\" #@param {type:\"string\"}\n", - "S3_UPLOAD_PATH = \"s3://my-bucket/iris-classifier/gbtree.onnx\" #@param {type:\"string\"}\n", - "\n", - "import sys\n", - "import re\n", - "\n", - "if AWS_ACCESS_KEY_ID == \"\":\n", - " print(\"\\033[91m{}\\033[00m\".format(\"ERROR: Please set AWS_ACCESS_KEY_ID\"), file=sys.stderr)\n", - "\n", - "elif AWS_SECRET_ACCESS_KEY == \"\":\n", - " print(\"\\033[91m{}\\033[00m\".format(\"ERROR: Please set AWS_SECRET_ACCESS_KEY\"), file=sys.stderr)\n", - "\n", - "else:\n", - " try:\n", - " bucket, key = re.match(\"s3://(.+?)/(.+)\", S3_UPLOAD_PATH).groups()\n", - " except:\n", - " print(\"\\033[91m{}\\033[00m\".format(\"ERROR: Invalid s3 path (should be of the form s3://my-bucket/path/to/file)\"), file=sys.stderr)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "NXeuZsaQxUc8", - "colab_type": "text" - }, - "source": [ - "Upload the model to S3:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "YLmnWTEVsu55", - "colab_type": "code", - "colab": {} - }, - "source": [ - "import boto3\n", - "\n", - "s3 = boto3.client(\"s3\", aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)\n", - "print(\"Uploading {} ...\".format(S3_UPLOAD_PATH), end = '')\n", - "s3.upload_file(\"gbtree.onnx\", bucket, key)\n", - "print(\" ✓\")" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/test/apis/onnx/multi-model-classifier/README.md b/test/apis/onnx/multi-model-classifier/README.md deleted file mode 100644 index cdab23c686..0000000000 --- a/test/apis/onnx/multi-model-classifier/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# Multi-Model Classifier API - -This example deploys ResNet50, MobileNet and ShuffleNet models in one API. Query parameters are used for selecting the model. - -The example can be run on both CPU and on GPU hardware. - -## Sample Prediction - -Deploy the model by running: - -```bash -cortex deploy -``` - -And wait for it to become live by tracking its status with `cortex get --watch`. - -Once the API has been successfully deployed, export the API's endpoint for convenience. You can get the API's endpoint by running `cortex get multi-model-classifier`. - -```bash -export ENDPOINT=your-api-endpoint -``` - -When making a prediction with [sample.json](sample.json), the following image will be used: - -![cat](https://i.imgur.com/213xcvs.jpg) - -### ResNet50 Classifier - -Make a request to the ResNet50 model: - -```bash -curl "${ENDPOINT}?model=resnet50" -X POST -H "Content-Type: application/json" -d @sample.json -``` - -The expected response is: - -```json -{"label": "tabby"} -``` - -### MobileNet Classifier - -Make a request to the MobileNet model: - -```bash -curl "${ENDPOINT}?model=mobilenet" -X POST -H "Content-Type: application/json" -d @sample.json -``` - -The expected response is: - -```json -{"label": "tabby"} -``` - -### ShuffleNet Classifier - -Make a request to the ShuffleNet model: - -```bash -curl "${ENDPOINT}?model=shufflenet" -X POST -H "Content-Type: application/json" -d @sample.json -``` - -The expected response is: - -```json -{"label": "Egyptian_cat"} -``` diff --git a/test/apis/onnx/multi-model-classifier/cortex.yaml b/test/apis/onnx/multi-model-classifier/cortex.yaml deleted file mode 100644 index 7579b83e0c..0000000000 --- a/test/apis/onnx/multi-model-classifier/cortex.yaml +++ /dev/null @@ -1,18 +0,0 @@ -- name: multi-model-classifier - kind: RealtimeAPI - handler: - type: python - path: handler.py - multi_model_reloading: - paths: - - name: resnet50 - path: s3://cortex-examples/onnx/resnet50/ - - name: mobilenet - path: s3://cortex-examples/onnx/mobilenet/ - - name: shufflenet - path: s3://cortex-examples/onnx/shufflenet/ - config: - image-classifier-classes: https://s3.amazonaws.com/deep-learning-models/image-models/imagenet_class_index.json - image-resize: 224 - compute: - mem: 2G diff --git a/test/apis/onnx/multi-model-classifier/dependencies.sh b/test/apis/onnx/multi-model-classifier/dependencies.sh deleted file mode 100644 index 60aa91e20a..0000000000 --- a/test/apis/onnx/multi-model-classifier/dependencies.sh +++ /dev/null @@ -1,2 +0,0 @@ -# install opencv dependencies -apt update && apt install libsm6 libxext6 libxrender-dev -y diff --git a/test/apis/onnx/multi-model-classifier/handler.py b/test/apis/onnx/multi-model-classifier/handler.py deleted file mode 100644 index 8ff7f62e04..0000000000 --- a/test/apis/onnx/multi-model-classifier/handler.py +++ /dev/null @@ -1,118 +0,0 @@ -import os -import numpy as np -import onnxruntime as rt -import cv2, requests -from scipy.special import softmax - - -def get_url_image(url_image): - """ - Get numpy image from URL image. - """ - resp = requests.get(url_image, stream=True).raw - image = np.asarray(bytearray(resp.read()), dtype="uint8") - image = cv2.imdecode(image, cv2.IMREAD_COLOR) - return image - - -def image_resize(image, width=None, height=None, inter=cv2.INTER_AREA): - """ - Resize a numpy image. - """ - dim = None - (h, w) = image.shape[:2] - - if width is None and height is None: - return image - - if width is None: - # calculate the ratio of the height and construct the dimensions - r = height / float(h) - dim = (int(w * r), height) - else: - # calculate the ratio of the width and construct the dimensions - r = width / float(w) - dim = (width, int(h * r)) - - resized = cv2.resize(image, dim, interpolation=inter) - - return resized - - -def preprocess(img_data): - """ - Normalize input for inference. - """ - # move pixel color dimension to position 0 - img = np.moveaxis(img_data, 2, 0) - - mean_vec = np.array([0.485, 0.456, 0.406]) - stddev_vec = np.array([0.229, 0.224, 0.225]) - norm_img_data = np.zeros(img.shape).astype("float32") - for i in range(img.shape[0]): - # for each pixel in each channel, divide the value by 255 to get value between [0, 1] and then normalize - norm_img_data[i, :, :] = (img[i, :, :] / 255 - mean_vec[i]) / stddev_vec[i] - - # extend to batch size of 1 - norm_img_data = norm_img_data[np.newaxis, ...] - return norm_img_data - - -def postprocess(results): - """ - Eliminates all dimensions of size 1, softmaxes the input and then returns the index of the element with the highest value. - """ - squeezed = np.squeeze(results) - maxed = softmax(squeezed) - result = np.argmax(maxed) - return result - - -class Handler: - def __init__(self, model_client, config): - # onnx client - self.client = model_client - - # for image classifiers - classes = requests.get(config["image-classifier-classes"]).json() - self.image_classes = [classes[str(k)][1] for k in range(len(classes))] - self.resize_value = config["image-resize"] - - def handle_post(self, payload, query_params): - # get request params - model_name = query_params["model"] - img_url = payload["url"] - - # process the input - img = get_url_image(img_url) - img = image_resize(img, height=self.resize_value) - img = preprocess(img) - - # predict - model = self.client.get_model(model_name) - session = model["session"] - input_name = model["input_name"] - output_name = model["output_name"] - input_dict = { - input_name: img, - } - results = session.run([output_name], input_dict)[0] - - # interpret result - result = postprocess(results) - predicted_label = self.image_classes[result] - - return {"label": predicted_label} - - def load_model(self, model_path): - """ - Load ONNX model from disk. - """ - - model_path = os.path.join(model_path, os.listdir(model_path)[0]) - session = rt.InferenceSession(model_path) - return { - "session": session, - "input_name": session.get_inputs()[0].name, - "output_name": session.get_outputs()[0].name, - } diff --git a/test/apis/onnx/multi-model-classifier/requirements.txt b/test/apis/onnx/multi-model-classifier/requirements.txt deleted file mode 100644 index 46a61881fb..0000000000 --- a/test/apis/onnx/multi-model-classifier/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -opencv-python==4.2.0.34 -onnxruntime==1.6.0 -numpy==1.19.1 -scipy==1.4.1 diff --git a/test/apis/onnx/multi-model-classifier/sample.json b/test/apis/onnx/multi-model-classifier/sample.json deleted file mode 100644 index 4ee3aa45df..0000000000 --- a/test/apis/onnx/multi-model-classifier/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "url": "https://i.imgur.com/213xcvs.jpg" -} diff --git a/test/apis/onnx/yolov5-youtube/README.md b/test/apis/onnx/yolov5-youtube/README.md deleted file mode 100644 index 5832b93d8c..0000000000 --- a/test/apis/onnx/yolov5-youtube/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# YOLOv5 Detection model - -This example deploys a detection model trained using [ultralytics' yolo repo](https://github.com/ultralytics/yolov5) using ONNX. -We'll use the `yolov5s` model as an example here. -In can be used to run inference on youtube videos and returns the annotated video with bounding boxes. - -The example can be run on both CPU and on GPU hardware. - -## Sample Prediction - -Deploy the model by running: - -```bash -cortex deploy -``` - -And wait for it to become live by tracking its status with `cortex get --watch`. - -Once the API has been successfully deployed, export the API's endpoint for convenience. You can get the API's endpoint by running `cortex get yolov5-youtube`. - -```bash -export ENDPOINT=your-api-endpoint -``` - -When making a prediction with [sample.json](sample.json), [this](https://www.youtube.com/watch?v=aUdKzb4LGJI) youtube video will be used. - -To make a request to the model: - -```bash -curl "${ENDPOINT}" -X POST -H "Content-Type: application/json" -d @sample.json --output video.mp4 -``` - -After a few seconds, `curl` will save the resulting video `video.mp4` in the current working directory. The following is a sample of what should be exported: - -![yolov5](https://user-images.githubusercontent.com/26958764/86545098-e0dce900-bf34-11ea-83a7-8fd544afa11c.gif) - - -## Exporting ONNX - -To export a custom model from the repo, use the [`model/export.py`](https://github.com/ultralytics/yolov5/blob/master/models/export.py) script. -The only change we need to make is to change the line - -```bash -model.model[-1].export = True # set Detect() layer export=True -``` - -to - -```bash -model.model[-1].export = False -``` - -Originally, the ultralytics repo does not export postprocessing steps of the model, e.g. the conversion from the raw CNN outputs to bounding boxes. -With newer ONNX versions, these can be exported as part of the model making the deployment much easier. - -With this modified script, the ONNX graph used for this example has been exported using -```bash -python models/export.py --weights weights/yolov5s.pt --img 416 --batch 1 -``` diff --git a/test/apis/onnx/yolov5-youtube/conda-packages.txt b/test/apis/onnx/yolov5-youtube/conda-packages.txt deleted file mode 100644 index 131fce12b5..0000000000 --- a/test/apis/onnx/yolov5-youtube/conda-packages.txt +++ /dev/null @@ -1,3 +0,0 @@ -conda-forge::ffmpeg=4.2.3 -conda-forge::youtube-dl -conda-forge::matplotlib diff --git a/test/apis/onnx/yolov5-youtube/cortex.yaml b/test/apis/onnx/yolov5-youtube/cortex.yaml deleted file mode 100644 index 6badade222..0000000000 --- a/test/apis/onnx/yolov5-youtube/cortex.yaml +++ /dev/null @@ -1,12 +0,0 @@ -- name: yolov5-youtube - kind: RealtimeAPI - handler: - type: python - path: handler.py - multi_model_reloading: - path: s3://cortex-examples/onnx/yolov5-youtube/ - config: - iou_threshold: 0.5 - confidence_threshold: 0.6 - compute: - gpu: 1 # this is optional, since the api can also run on cpu diff --git a/test/apis/onnx/yolov5-youtube/handler.py b/test/apis/onnx/yolov5-youtube/handler.py deleted file mode 100644 index 354ee0f2d9..0000000000 --- a/test/apis/onnx/yolov5-youtube/handler.py +++ /dev/null @@ -1,90 +0,0 @@ -import json -import os -import io -import uuid -import utils - -import onnxruntime as rt -import numpy as np -from matplotlib import pyplot as plt - -from starlette.responses import StreamingResponse - - -class Handler: - def __init__(self, model_client, config): - self.client = model_client - # Get the input shape from the ONNX runtime - _, _, height, width = model_client.get_model()["input_shape"] - self.input_size = (width, height) - self.config = config - with open("labels.json") as buf: - self.labels = json.load(buf) - color_map = plt.cm.tab20(np.linspace(0, 20, len(self.labels))) - self.color_map = [tuple(map(int, colors)) for colors in 255 * color_map] - - def postprocess(self, output): - boxes, obj_score, class_scores = np.split(output[0], [4, 5], axis=1) - boxes = utils.boxes_yolo_to_xyxy(boxes) - - # get the class-prediction & class confidences - class_id = class_scores.argmax(axis=1) - cls_score = class_scores[np.arange(len(class_scores)), class_id] - - confidence = obj_score.squeeze(axis=1) * cls_score - sel = confidence > self.config["confidence_threshold"] - boxes, class_id, confidence = boxes[sel], class_id[sel], confidence[sel] - sel = utils.nms(boxes, confidence, self.config["iou_threshold"]) - boxes, class_id, confidence = boxes[sel], class_id[sel], confidence[sel] - return boxes, class_id, confidence - - def handle_post(self, payload): - # download YT video - in_path = utils.download_from_youtube(payload["url"], self.input_size[1]) - out_path = f"{uuid.uuid1()}.mp4" - - # get model - model = self.client.get_model() - session = model["session"] - input_name = model["input_name"] - output_name = model["output_name"] - - # run predictions - with utils.FrameWriter(out_path, size=self.input_size) as writer: - for frame in utils.frame_reader(in_path, size=self.input_size): - x = (frame.astype(np.float32) / 255).transpose(2, 0, 1) - # 4 output tensors, the last three are intermediate values and - # not necessary for detection - output, *_ = session.run( - [output_name], - { - input_name: x[None], - }, - ) - boxes, class_ids, confidence = self.postprocess(output) - utils.overlay_boxes(frame, boxes, class_ids, self.labels, self.color_map) - writer.write(frame) - - with open(out_path, "rb") as f: - output_buf = io.BytesIO(f.read()) - - os.remove(in_path) - os.remove(out_path) - - return StreamingResponse(output_buf, media_type="video/mp4") - - def load_model(self, model_path): - """ - Load ONNX model from disk. - """ - - model_path = os.path.join(model_path, os.listdir(model_path)[0]) - session = rt.InferenceSession(model_path) - print("get_inputs", session.get_inputs()[0]) - print("get_outputs", session.get_outputs()[0]) - return { - "session": session, - "input_shape": session.get_inputs()[0].shape, - "input_name": session.get_inputs()[0].name, - "output_name": session.get_outputs()[0].name, - } diff --git a/test/apis/onnx/yolov5-youtube/labels.json b/test/apis/onnx/yolov5-youtube/labels.json deleted file mode 100644 index c86f2f812a..0000000000 --- a/test/apis/onnx/yolov5-youtube/labels.json +++ /dev/null @@ -1,82 +0,0 @@ -[ - "person", - "bicycle", - "car", - "motorcycle", - "airplane", - "bus", - "train", - "truck", - "boat", - "traffic light", - "fire hydrant", - "stop sign", - "parking meter", - "bench", - "bird", - "cat", - "dog", - "horse", - "sheep", - "cow", - "elephant", - "bear", - "zebra", - "giraffe", - "backpack", - "umbrella", - "handbag", - "tie", - "suitcase", - "frisbee", - "skis", - "snowboard", - "sports ball", - "kite", - "baseball bat", - "baseball glove", - "skateboard", - "surfboard", - "tennis racket", - "bottle", - "wine glass", - "cup", - "fork", - "knife", - "spoon", - "bowl", - "banana", - "apple", - "sandwich", - "orange", - "broccoli", - "carrot", - "hot dog", - "pizza", - "donut", - "cake", - "chair", - "couch", - "potted plant", - "bed", - "dining table", - "toilet", - "tv", - "laptop", - "mouse", - "remote", - "keyboard", - "cell phone", - "microwave", - "oven", - "toaster", - "sink", - "refrigerator", - "book", - "clock", - "vase", - "scissors", - "teddy bear", - "hair drier", - "toothbrush" -] diff --git a/test/apis/onnx/yolov5-youtube/requirements.txt b/test/apis/onnx/yolov5-youtube/requirements.txt deleted file mode 100644 index a93726566c..0000000000 --- a/test/apis/onnx/yolov5-youtube/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -ffmpeg-python -aiofiles -opencv-python-headless -onnxruntime==1.6.0 -numpy==1.19.1 diff --git a/test/apis/onnx/yolov5-youtube/sample.json b/test/apis/onnx/yolov5-youtube/sample.json deleted file mode 100644 index 8421278f58..0000000000 --- a/test/apis/onnx/yolov5-youtube/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "url": "https://www.youtube.com/watch?v=aUdKzb4LGJI" -} diff --git a/test/apis/onnx/yolov5-youtube/utils.py b/test/apis/onnx/yolov5-youtube/utils.py deleted file mode 100644 index 37e56d0325..0000000000 --- a/test/apis/onnx/yolov5-youtube/utils.py +++ /dev/null @@ -1,128 +0,0 @@ -import youtube_dl -import ffmpeg -import numpy as np -import cv2 -import uuid - -from pathlib import Path -from typing import Iterable, Tuple - - -def download_from_youtube(url: str, min_height: int) -> Path: - target = f"{uuid.uuid1()}.mp4" - ydl_opts = { - "outtmpl": target, - "format": f"worstvideo[vcodec=vp9][height>={min_height}]", - } - with youtube_dl.YoutubeDL(ydl_opts) as ydl: - ydl.download([url]) - # we need to glob in case youtube-dl adds suffix - (path,) = Path().absolute().glob(f"{target}*") - return path - - -def frame_reader(path: Path, size: Tuple[int, int]) -> Iterable[np.ndarray]: - width, height = size - # letterbox frames to fixed size - process = ( - ffmpeg.input(path) - .filter("scale", size=f"{width}:{height}", force_original_aspect_ratio="decrease") - # Negative values for x and y center the padded video - .filter("pad", height=height, width=width, x=-1, y=-1) - .output("pipe:", format="rawvideo", pix_fmt="rgb24") - .run_async(pipe_stdout=True) - ) - - while True: - in_bytes = process.stdout.read(height * width * 3) - if not in_bytes: - process.wait() - break - frame = np.frombuffer(in_bytes, np.uint8).reshape([height, width, 3]) - yield frame - - -class FrameWriter: - def __init__(self, path: Path, size: Tuple[int, int]): - width, height = size - self.process = ( - ffmpeg.input("pipe:", format="rawvideo", pix_fmt="rgb24", s=f"{width}x{height}") - .output(path, pix_fmt="yuv420p") - .overwrite_output() - .run_async(pipe_stdin=True) - ) - - def write(self, frame: np.ndarray): - self.process.stdin.write(frame.astype(np.uint8).tobytes()) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.__del__() - - def __del__(self): - self.process.stdin.close() - self.process.wait() - - -def nms(dets: np.ndarray, scores: np.ndarray, thresh: float) -> np.ndarray: - x1 = dets[:, 0] - y1 = dets[:, 1] - x2 = dets[:, 2] - y2 = dets[:, 3] - - areas = (x2 - x1 + 1) * (y2 - y1 + 1) - order = scores.argsort()[::-1] # get boxes with more ious first - - keep = [] - while order.size > 0: - i = order[0] # pick maxmum iou box - keep.append(i) - xx1 = np.maximum(x1[i], x1[order[1:]]) - yy1 = np.maximum(y1[i], y1[order[1:]]) - xx2 = np.minimum(x2[i], x2[order[1:]]) - yy2 = np.minimum(y2[i], y2[order[1:]]) - - w = np.maximum(0.0, xx2 - xx1 + 1) # maximum width - h = np.maximum(0.0, yy2 - yy1 + 1) # maxiumum height - inter = w * h - ovr = inter / (areas[i] + areas[order[1:]] - inter) - - inds = np.where(ovr <= thresh)[0] - order = order[inds + 1] - - return np.array(keep).astype(np.int) - - -def boxes_yolo_to_xyxy(boxes: np.ndarray): - boxes[:, 0] -= boxes[:, 2] / 2 - boxes[:, 1] -= boxes[:, 3] / 2 - boxes[:, 2] = boxes[:, 2] + boxes[:, 0] - boxes[:, 3] = boxes[:, 3] + boxes[:, 1] - return boxes - - -def overlay_boxes(frame, boxes, class_ids, label_map, color_map, line_thickness=None): - tl = ( - line_thickness or round(0.0005 * (frame.shape[0] + frame.shape[1]) / 2) + 1 - ) # line/font thickness - - for class_id, (x1, y1, x2, y2) in zip(class_ids, boxes.astype(np.int)): - color = color_map[class_id] - label = label_map[class_id] - cv2.rectangle(frame, (x1, y1), (x2, y2), color, tl, cv2.LINE_AA) - tf = max(tl - 1, 1) # font thickness - t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0] - x3, y3 = x1 + t_size[0], y1 - t_size[1] - 3 - cv2.rectangle(frame, (x1, y1), (x3, y3), color, -1, cv2.LINE_AA) # filled - cv2.putText( - frame, - label, - (x1, y1 - 2), - 0, - tl / 3, - [225, 255, 255], - thickness=tf, - lineType=cv2.LINE_AA, - ) diff --git a/test/apis/pytorch/answer-generator/cortex.yaml b/test/apis/pytorch/answer-generator/cortex.yaml deleted file mode 100644 index d047b77683..0000000000 --- a/test/apis/pytorch/answer-generator/cortex.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- name: answer-generator - kind: RealtimeAPI - handler: - type: python - path: handler.py - compute: - cpu: 1 - gpu: 1 - mem: 5G diff --git a/test/apis/pytorch/answer-generator/generator.py b/test/apis/pytorch/answer-generator/generator.py deleted file mode 100644 index d901e944cb..0000000000 --- a/test/apis/pytorch/answer-generator/generator.py +++ /dev/null @@ -1,42 +0,0 @@ -# This file includes code which was modified from https://colab.research.google.com/drive/1KTLqiAOdKM_3RnBWfqgrvOQLqumUyOdA - -import torch -import torch.nn.functional as F - - -END_OF_TEXT = 50256 - - -def generate(model, conditioned_tokens, device): - generated_tokens = [] - while True: - result = recalc(model, conditioned_tokens, generated_tokens, device) - if result == END_OF_TEXT: - return generated_tokens[:-1] - - -def recalc(model, conditioned_tokens, generated_tokens, device): - indexed_tokens = conditioned_tokens + generated_tokens - tokens_tensor = torch.tensor([indexed_tokens]) - tokens_tensor = tokens_tensor.to(device) - with torch.no_grad(): - outputs = model(tokens_tensor) - predictions = outputs[0] - logits = predictions[0, -1, :] - filtered_logits = top_p_filtering(logits) - probabilities = F.softmax(filtered_logits, dim=-1) - next_token = torch.multinomial(probabilities, 1) - generated_tokens.append(next_token.item()) - return next_token.item() - - -def top_p_filtering(logits, top_p=0.9, filter_value=-float("Inf")): - assert logits.dim() == 1 - sorted_logits, sorted_indices = torch.sort(logits, descending=True) - cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1) - sorted_indices_to_remove = cumulative_probs > top_p - sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone() - sorted_indices_to_remove[..., 0] = 0 - indices_to_remove = sorted_indices[sorted_indices_to_remove] - logits[indices_to_remove] = filter_value - return logits diff --git a/test/apis/pytorch/answer-generator/handler.py b/test/apis/pytorch/answer-generator/handler.py deleted file mode 100644 index 0b879b53da..0000000000 --- a/test/apis/pytorch/answer-generator/handler.py +++ /dev/null @@ -1,34 +0,0 @@ -import wget -import torch -from transformers import GPT2Tokenizer, GPT2LMHeadModel, GPT2Config -import generator - - -class Handler: - def __init__(self, config): - medium_config = GPT2Config(n_embd=1024, n_layer=24, n_head=16) - model = GPT2LMHeadModel(medium_config) - wget.download( - "https://convaisharables.blob.core.windows.net/lsp/multiref/medium_ft.pkl", - "/tmp/medium_ft.pkl", - ) - - weights = torch.load("/tmp/medium_ft.pkl") - weights["lm_head.weight"] = weights["lm_head.decoder.weight"] - weights.pop("lm_head.decoder.weight", None) - - model.load_state_dict(weights) - - device = "cuda" if torch.cuda.is_available() else "cpu" - print(f"using device: {device}") - model.to(device) - model.eval() - - self.device = device - self.model = model - self.tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - - def handle_post(self, payload): - conditioned_tokens = self.tokenizer.encode(payload["text"]) + [generator.END_OF_TEXT] - prediction = generator.generate(self.model, conditioned_tokens, self.device) - return self.tokenizer.decode(prediction) diff --git a/test/apis/pytorch/answer-generator/requirements.txt b/test/apis/pytorch/answer-generator/requirements.txt deleted file mode 100644 index effba0ef1b..0000000000 --- a/test/apis/pytorch/answer-generator/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -torch -transformers==2.3.* -wget==3.* diff --git a/test/apis/pytorch/answer-generator/sample.json b/test/apis/pytorch/answer-generator/sample.json deleted file mode 100644 index aa91c9d2eb..0000000000 --- a/test/apis/pytorch/answer-generator/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "text": "What is machine learning?" -} diff --git a/test/apis/pytorch/image-classifier-alexnet/cortex.yaml b/test/apis/pytorch/image-classifier-alexnet/cortex.yaml deleted file mode 100644 index f4436c71be..0000000000 --- a/test/apis/pytorch/image-classifier-alexnet/cortex.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- name: image-classifier-alexnet - kind: RealtimeAPI - handler: - type: python - path: handler.py - compute: - cpu: 1 - gpu: 1 - mem: 4G diff --git a/test/apis/pytorch/image-classifier-alexnet/handler.py b/test/apis/pytorch/image-classifier-alexnet/handler.py deleted file mode 100644 index a560a8cb9d..0000000000 --- a/test/apis/pytorch/image-classifier-alexnet/handler.py +++ /dev/null @@ -1,37 +0,0 @@ -import requests -import torch -import torchvision -from torchvision import transforms -from PIL import Image -from io import BytesIO - - -class Handler: - def __init__(self, config): - device = "cuda" if torch.cuda.is_available() else "cpu" - print(f"using device: {device}") - - model = torchvision.models.alexnet(pretrained=True).to(device) - model.eval() - # https://github.com/pytorch/examples/blob/447974f6337543d4de6b888e244a964d3c9b71f6/imagenet/main.py#L198-L199 - normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) - - self.preprocess = transforms.Compose( - [transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize] - ) - self.labels = requests.get( - "https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt" - ).text.split("\n")[1:] - self.model = model - self.device = device - - def handle_post(self, payload): - image = requests.get(payload["url"]).content - img_pil = Image.open(BytesIO(image)) - img_tensor = self.preprocess(img_pil) - img_tensor.unsqueeze_(0) - img_tensor = img_tensor.to(self.device) - with torch.no_grad(): - prediction = self.model(img_tensor) - _, index = prediction[0].max(0) - return self.labels[index] diff --git a/test/apis/pytorch/image-classifier-alexnet/requirements.txt b/test/apis/pytorch/image-classifier-alexnet/requirements.txt deleted file mode 100644 index ac988bdf84..0000000000 --- a/test/apis/pytorch/image-classifier-alexnet/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -torch -torchvision diff --git a/test/apis/pytorch/image-classifier-alexnet/sample.json b/test/apis/pytorch/image-classifier-alexnet/sample.json deleted file mode 100644 index eb72ddb869..0000000000 --- a/test/apis/pytorch/image-classifier-alexnet/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "url": "https://i.imgur.com/PzXprwl.jpg" -} diff --git a/test/apis/pytorch/image-classifier-resnet50/README.md b/test/apis/pytorch/image-classifier-resnet50/README.md deleted file mode 100644 index d3d873d8ae..0000000000 --- a/test/apis/pytorch/image-classifier-resnet50/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Image Classifier with ResNet50 - -This example implements an image recognition system using ResNet50, which allows for the recognition of up to 1000 classes. - -## Deploying - -There are 3 Cortex APIs available in this example: - -1. [cortex.yaml](cortex.yaml) - can be used with any instances. -1. [cortex_inf.yaml](cortex_inf.yaml) - to be used with `inf1` instances. -1. [cortex_gpu.yaml](cortex_gpu.yaml) - to be used with GPU instances. - -To deploy an API, run: - -```bash -cortex deploy -``` - -E.g. - -```bash -cortex deploy cortex_gpu.yaml -``` - -## Verifying your API - -Check that your API is live by running `cortex get image-classifier-resnet50`, and copy the example `curl` command that's shown. After the API is live, run the `curl` command, e.g. - -```bash -$ curl -X POST -H "Content-Type: application/json" -d @sample.json - -["tabby", "Egyptian_cat", "tiger_cat", "tiger", "plastic_bag"] -``` - -The following image is embedded in [sample.json](sample.json): - -![image](https://i.imgur.com/213xcvs.jpg) - -## Exporting SavedModels - -This example deploys models that we have built and uploaded to a public S3 bucket. If you want to build the models yourself, follow these instructions. - -Run the following command to install the dependencies required for the [generate_resnet50_models.ipynb](generate_resnet50_models.ipynb) notebook: - -```bash -pip install --extra-index-url=https://pip.repos.neuron.amazonaws.com \ - neuron-cc==1.0.9410.0+6008239556 \ - torch-neuron==1.0.825.0 -``` - -Also, `torchvision` has to be installed, but without any dependencies: - -```bash -pip install torchvision==0.4.2 --no-deps -``` - -The [generate_resnet50_models.ipynb](generate_resnet50_models.ipynb) notebook will generate 2 torch models. One is saved as `resnet50.pt` which can be run on GPU or CPU, and another is saved as `resnet50_neuron.pt`, which can only be run on `inf1` instances. diff --git a/test/apis/pytorch/image-classifier-resnet50/cortex.yaml b/test/apis/pytorch/image-classifier-resnet50/cortex.yaml deleted file mode 100644 index 18ecac1dfb..0000000000 --- a/test/apis/pytorch/image-classifier-resnet50/cortex.yaml +++ /dev/null @@ -1,13 +0,0 @@ -- name: image-classifier-resnet50 - kind: RealtimeAPI - handler: - type: python - path: handler.py - config: - model_path: s3://cortex-examples/pytorch/image-classifier-resnet50 - model_name: resnet50.pt - device: cpu - classes: https://s3.amazonaws.com/deep-learning-models/image-models/imagenet_class_index.json - input_shape: [224, 224] - compute: - cpu: 1 diff --git a/test/apis/pytorch/image-classifier-resnet50/cortex_gpu.yaml b/test/apis/pytorch/image-classifier-resnet50/cortex_gpu.yaml deleted file mode 100644 index 2f2526ea1e..0000000000 --- a/test/apis/pytorch/image-classifier-resnet50/cortex_gpu.yaml +++ /dev/null @@ -1,14 +0,0 @@ -- name: image-classifier-resnet50 - kind: RealtimeAPI - handler: - type: python - path: handler.py - config: - model_path: s3://cortex-examples/pytorch/image-classifier-resnet50 - model_name: resnet50.pt - device: gpu - classes: https://s3.amazonaws.com/deep-learning-models/image-models/imagenet_class_index.json - input_shape: [224, 224] - compute: - gpu: 1 - cpu: 1 diff --git a/test/apis/pytorch/image-classifier-resnet50/cortex_inf.yaml b/test/apis/pytorch/image-classifier-resnet50/cortex_inf.yaml deleted file mode 100644 index 3b59d08f94..0000000000 --- a/test/apis/pytorch/image-classifier-resnet50/cortex_inf.yaml +++ /dev/null @@ -1,14 +0,0 @@ -- name: image-classifier-resnet50 - kind: RealtimeAPI - handler: - type: python - path: handler.py - config: - model_path: s3://cortex-examples/pytorch/image-classifier-resnet50 - model_name: resnet50_neuron.pt - device: inf - classes: https://s3.amazonaws.com/deep-learning-models/image-models/imagenet_class_index.json - input_shape: [224, 224] - compute: - inf: 1 - cpu: 1 diff --git a/test/apis/pytorch/image-classifier-resnet50/dependencies.sh b/test/apis/pytorch/image-classifier-resnet50/dependencies.sh deleted file mode 100644 index 057530cb85..0000000000 --- a/test/apis/pytorch/image-classifier-resnet50/dependencies.sh +++ /dev/null @@ -1 +0,0 @@ -apt-get update && apt-get install -y libgl1-mesa-glx libegl1-mesa diff --git a/test/apis/pytorch/image-classifier-resnet50/expectations.yaml b/test/apis/pytorch/image-classifier-resnet50/expectations.yaml deleted file mode 100644 index 5e1d38f9ae..0000000000 --- a/test/apis/pytorch/image-classifier-resnet50/expectations.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# this file is used for testing purposes only - -response: - content_type: "json" - expected: ["tabby", "Egyptian_cat", "tiger_cat", "tiger", "plastic_bag"] diff --git a/test/apis/pytorch/image-classifier-resnet50/generate_resnet50_models.ipynb b/test/apis/pytorch/image-classifier-resnet50/generate_resnet50_models.ipynb deleted file mode 100644 index 7a91250571..0000000000 --- a/test/apis/pytorch/image-classifier-resnet50/generate_resnet50_models.ipynb +++ /dev/null @@ -1,119 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Generate Resnet50 Models\n" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import numpy as np\n", - "import os\n", - "import torch_neuron\n", - "from torchvision import models" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Load Resnet50 model" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "model = models.resnet50(pretrained=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Compile model for Inferentia. Should have worked with 1 NeuronCores, but it appears that setting it to a minimum of 2 is required." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:Neuron:compiling module ResNet with neuron-cc\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Compiler args type is value is ['--num-neuroncores', '2']\n" - ] - } - ], - "source": [ - "model.eval()\n", - "batch_size = 1\n", - "image = torch.zeros([batch_size, 3, 224, 224], dtype=torch.float32)\n", - "model_neuron = torch.neuron.trace(model, example_inputs=[image], compiler_args=[\"--num-neuroncores\", \"2\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Save both models to disk" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "model_neuron.save(\"resnet50_neuron.pt\")\n", - "torch.save(model.state_dict(), \"resnet50.pt\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.9" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/test/apis/pytorch/image-classifier-resnet50/handler.py b/test/apis/pytorch/image-classifier-resnet50/handler.py deleted file mode 100644 index 85dbeb2cac..0000000000 --- a/test/apis/pytorch/image-classifier-resnet50/handler.py +++ /dev/null @@ -1,86 +0,0 @@ -import os -import torch -import cv2 -import numpy as np -import requests -import re -import boto3 -from botocore import UNSIGNED -from botocore.client import Config -from torchvision import models, transforms, datasets - - -def get_url_image(url_image): - """ - Get numpy image from URL image. - """ - resp = requests.get(url_image, stream=True).raw - image = np.asarray(bytearray(resp.read()), dtype="uint8") - image = cv2.imdecode(image, cv2.IMREAD_COLOR) - image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) - return image - - -class Handler: - def __init__(self, config): - # load classes - classes = requests.get(config["classes"]).json() - self.idx2label = [classes[str(k)][1] for k in range(len(classes))] - - # download the model - model_path = config["model_path"] - model_name = config["model_name"] - bucket, key = re.match("s3://(.+?)/(.+)", model_path).groups() - s3 = boto3.client("s3") - s3.download_file(bucket, os.path.join(key, model_name), model_name) - - # load the model - self.device = None - if config["device"] == "gpu": - self.device = torch.device("cuda") - self.model = models.resnet50() - self.model.load_state_dict(torch.load(model_name, map_location="cuda:0")) - self.model.eval() - self.model = self.model.to(self.device) - elif config["device"] == "cpu": - self.model = models.resnet50() - self.model.load_state_dict(torch.load(model_name)) - self.model.eval() - elif config["device"] == "inf": - import torch_neuron - - self.model = torch.jit.load(model_name) - else: - raise RuntimeError("invalid handler: config: must be cpu, gpu, or inf") - - # save normalization transform for later use - normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) - self.transform = transforms.Compose( - [ - transforms.ToPILImage(), - transforms.Resize(config["input_shape"]), - transforms.ToTensor(), - normalize, - ] - ) - - def handle_post(self, payload): - # preprocess image - image = get_url_image(payload["url"]) - image = self.transform(image) - image = torch.tensor(image.numpy()[np.newaxis, ...]) - - # predict - if self.device: - results = self.model(image.to(self.device)) - else: - results = self.model(image) - - # Get the top 5 results - top5_idx = results[0].sort()[1][-5:] - - # Lookup and print the top 5 labels - top5_labels = [self.idx2label[idx] for idx in top5_idx] - top5_labels = top5_labels[::-1] - - return top5_labels diff --git a/test/apis/pytorch/image-classifier-resnet50/requirements.txt b/test/apis/pytorch/image-classifier-resnet50/requirements.txt deleted file mode 100644 index df61209f31..0000000000 --- a/test/apis/pytorch/image-classifier-resnet50/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -torch==1.7.1 -torchvision==0.8.2 -opencv-python==4.4.0.42 ---extra-index-url https://pip.repos.neuron.amazonaws.com -torch-neuron==1.7.1.1.2.3.0 diff --git a/test/apis/pytorch/image-classifier-resnet50/sample.json b/test/apis/pytorch/image-classifier-resnet50/sample.json deleted file mode 100644 index 4ee3aa45df..0000000000 --- a/test/apis/pytorch/image-classifier-resnet50/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "url": "https://i.imgur.com/213xcvs.jpg" -} diff --git a/test/apis/pytorch/iris-classifier/cortex.yaml b/test/apis/pytorch/iris-classifier/cortex.yaml deleted file mode 100644 index 3494493ebb..0000000000 --- a/test/apis/pytorch/iris-classifier/cortex.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- name: iris-classifier - kind: RealtimeAPI - handler: - type: python - path: handler.py - config: - model: s3://cortex-examples/pytorch/iris-classifier/weights.pth diff --git a/test/apis/pytorch/iris-classifier/deploy.py b/test/apis/pytorch/iris-classifier/deploy.py deleted file mode 100644 index 9124b5d206..0000000000 --- a/test/apis/pytorch/iris-classifier/deploy.py +++ /dev/null @@ -1,22 +0,0 @@ -import cortex -import os - -dir_path = os.path.dirname(os.path.realpath(__file__)) - -cx = cortex.client() - -api_spec = { - "name": "iris-classifier", - "kind": "RealtimeAPI", - "handler": { - "type": "python", - "path": "handler.py", - "config": { - "model": "s3://cortex-examples/pytorch/iris-classifier/weights.pth", - }, - }, -} - -print(cx.deploy(api_spec, project_dir=dir_path)) - -# cx.delete("iris-classifier") diff --git a/test/apis/pytorch/iris-classifier/expectations.yaml b/test/apis/pytorch/iris-classifier/expectations.yaml deleted file mode 100644 index 93b58bac83..0000000000 --- a/test/apis/pytorch/iris-classifier/expectations.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# this file is used for testing purposes only - -response: - content_type: "text" - expected: "versicolor" diff --git a/test/apis/pytorch/iris-classifier/handler.py b/test/apis/pytorch/iris-classifier/handler.py deleted file mode 100644 index 1ceb99984b..0000000000 --- a/test/apis/pytorch/iris-classifier/handler.py +++ /dev/null @@ -1,43 +0,0 @@ -import re -import torch -import os -import boto3 -from botocore import UNSIGNED -from botocore.client import Config -from model import IrisNet - -labels = ["setosa", "versicolor", "virginica"] - - -class Handler: - def __init__(self, config): - # download the model - bucket, key = re.match("s3://(.+?)/(.+)", config["model"]).groups() - s3 = boto3.client("s3") - s3.download_file(bucket, key, "/tmp/model.pth") - - # initialize the model - model = IrisNet() - model.load_state_dict(torch.load("/tmp/model.pth")) - model.eval() - - self.model = model - - def handle_post(self, payload): - # Convert the request to a tensor and pass it into the model - input_tensor = torch.FloatTensor( - [ - [ - payload["sepal_length"], - payload["sepal_width"], - payload["petal_length"], - payload["petal_width"], - ] - ] - ) - - # Run the prediction - output = self.model(input_tensor) - - # Translate the model output to the corresponding label string - return labels[torch.argmax(output[0])] diff --git a/test/apis/pytorch/iris-classifier/model.py b/test/apis/pytorch/iris-classifier/model.py deleted file mode 100644 index a43a362544..0000000000 --- a/test/apis/pytorch/iris-classifier/model.py +++ /dev/null @@ -1,58 +0,0 @@ -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch.autograd import Variable - - -class IrisNet(nn.Module): - def __init__(self): - super(IrisNet, self).__init__() - self.fc1 = nn.Linear(4, 100) - self.fc2 = nn.Linear(100, 100) - self.fc3 = nn.Linear(100, 3) - self.softmax = nn.Softmax(dim=1) - - def forward(self, X): - X = F.relu(self.fc1(X)) - X = self.fc2(X) - X = self.fc3(X) - X = self.softmax(X) - return X - - -if __name__ == "__main__": - from sklearn.datasets import load_iris - from sklearn.model_selection import train_test_split - from sklearn.metrics import accuracy_score - - iris = load_iris() - X, y = iris.data, iris.target - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.8, random_state=42) - - train_X = Variable(torch.Tensor(X_train).float()) - test_X = Variable(torch.Tensor(X_test).float()) - train_y = Variable(torch.Tensor(y_train).long()) - test_y = Variable(torch.Tensor(y_test).long()) - - model = IrisNet() - - criterion = nn.CrossEntropyLoss() - - optimizer = torch.optim.SGD(model.parameters(), lr=0.01) - - for epoch in range(1000): - optimizer.zero_grad() - out = model(train_X) - loss = criterion(out, train_y) - loss.backward() - optimizer.step() - - if epoch % 100 == 0: - print("number of epoch {} loss {}".format(epoch, loss)) - - predict_out = model(test_X) - _, predict_y = torch.max(predict_out, 1) - - print("prediction accuracy {}".format(accuracy_score(test_y.data, predict_y.data))) - - torch.save(model.state_dict(), "weights.pth") diff --git a/test/apis/pytorch/iris-classifier/requirements.txt b/test/apis/pytorch/iris-classifier/requirements.txt deleted file mode 100644 index 12c6d5d5ea..0000000000 --- a/test/apis/pytorch/iris-classifier/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -torch diff --git a/test/apis/pytorch/iris-classifier/sample.json b/test/apis/pytorch/iris-classifier/sample.json deleted file mode 100644 index 0bc6836266..0000000000 --- a/test/apis/pytorch/iris-classifier/sample.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "sepal_length": 2.2, - "sepal_width": 3.6, - "petal_length": 1.4, - "petal_width": 3.3 -} diff --git a/test/apis/pytorch/language-identifier/cortex.yaml b/test/apis/pytorch/language-identifier/cortex.yaml deleted file mode 100644 index 0ee39707c4..0000000000 --- a/test/apis/pytorch/language-identifier/cortex.yaml +++ /dev/null @@ -1,5 +0,0 @@ -- name: language-identifier - kind: RealtimeAPI - handler: - type: python - path: handler.py diff --git a/test/apis/pytorch/language-identifier/handler.py b/test/apis/pytorch/language-identifier/handler.py deleted file mode 100644 index d065179b29..0000000000 --- a/test/apis/pytorch/language-identifier/handler.py +++ /dev/null @@ -1,16 +0,0 @@ -import wget -import fasttext - - -class Handler: - def __init__(self, config): - wget.download( - "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin", "/tmp/model" - ) - - self.model = fasttext.load_model("/tmp/model") - - def handle_post(self, payload): - prediction = self.model.predict(payload["text"]) - language = prediction[0][0][-2:] - return language diff --git a/test/apis/pytorch/language-identifier/requirements.txt b/test/apis/pytorch/language-identifier/requirements.txt deleted file mode 100644 index a342ff2914..0000000000 --- a/test/apis/pytorch/language-identifier/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -wget==3.* -fasttext==0.9.* diff --git a/test/apis/pytorch/language-identifier/sample.json b/test/apis/pytorch/language-identifier/sample.json deleted file mode 100644 index 225c357392..0000000000 --- a/test/apis/pytorch/language-identifier/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "text": "build machine learning apis" -} diff --git a/test/apis/pytorch/multi-model-text-analyzer/README.md b/test/apis/pytorch/multi-model-text-analyzer/README.md deleted file mode 100644 index 907c739eb6..0000000000 --- a/test/apis/pytorch/multi-model-text-analyzer/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# Multi-Model Analyzer API - -This example deploys a sentiment analyzer and a text summarizer in one API. Query parameters are used for selecting the model. - -The example can be run on both CPU and on GPU hardware. - -## Sample Prediction - -Deploy the model by running: - -```bash -cortex deploy -``` - -And wait for it to become live by tracking its status with `cortex get --watch`. - -Once the API has been successfully deployed, export the APIs endpoint. You can get the API's endpoint by running `cortex get text-analyzer`. - -```bash -export ENDPOINT=your-api-endpoint -``` - -### Sentiment Analyzer Classifier - -Make a request to the sentiment analyzer model: - -```bash -curl "${ENDPOINT}?model=sentiment" -X POST -H "Content-Type: application/json" -d @sample-sentiment.json -``` - -The expected response is: - -```json -{"label": "POSITIVE", "score": 0.9998506903648376} -``` - -### Text Summarizer - -Make a request to the text summarizer model: - -```bash -curl "${ENDPOINT}?model=summarizer" -X POST -H "Content-Type: application/json" -d @sample-summarizer.json -``` - -The expected response is: - -```text -Machine learning is the study of algorithms and statistical models that computer systems use to perform a specific task. It is seen as a subset of artificial intelligence. Machine learning algorithms are used in a wide variety of applications, such as email filtering and computer vision. In its application across business problems, machine learning is also referred to as predictive analytics. -``` diff --git a/test/apis/pytorch/multi-model-text-analyzer/cortex.yaml b/test/apis/pytorch/multi-model-text-analyzer/cortex.yaml deleted file mode 100644 index 864599086a..0000000000 --- a/test/apis/pytorch/multi-model-text-analyzer/cortex.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- name: multi-model-text-analyzer - kind: RealtimeAPI - handler: - type: python - path: handler.py - compute: - cpu: 1 - gpu: 1 - mem: 6G diff --git a/test/apis/pytorch/multi-model-text-analyzer/handler.py b/test/apis/pytorch/multi-model-text-analyzer/handler.py deleted file mode 100644 index 234ba5f5b1..0000000000 --- a/test/apis/pytorch/multi-model-text-analyzer/handler.py +++ /dev/null @@ -1,23 +0,0 @@ -import torch -from transformers import pipeline -from starlette.responses import JSONResponse - - -class Handler: - def __init__(self, config): - device = 0 if torch.cuda.is_available() else -1 - print(f"using device: {'cuda' if device == 0 else 'cpu'}") - - self.analyzer = pipeline(task="sentiment-analysis", device=device) - self.summarizer = pipeline(task="summarization", device=device) - - def handle_post(self, query_params, payload): - model_name = query_params.get("model") - - if model_name == "sentiment": - return self.analyzer(payload["text"])[0] - elif model_name == "summarizer": - summary = self.summarizer(payload["text"]) - return summary[0]["summary_text"] - else: - return JSONResponse({"error": f"unknown model: {model_name}"}, status_code=400) diff --git a/test/apis/pytorch/multi-model-text-analyzer/requirements.txt b/test/apis/pytorch/multi-model-text-analyzer/requirements.txt deleted file mode 100644 index 3f565d80e4..0000000000 --- a/test/apis/pytorch/multi-model-text-analyzer/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -torch -transformers==2.9.* diff --git a/test/apis/pytorch/multi-model-text-analyzer/sample-sentiment.json b/test/apis/pytorch/multi-model-text-analyzer/sample-sentiment.json deleted file mode 100644 index de3a18a92a..0000000000 --- a/test/apis/pytorch/multi-model-text-analyzer/sample-sentiment.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "text": "best day ever" -} diff --git a/test/apis/pytorch/multi-model-text-analyzer/sample-summarizer.json b/test/apis/pytorch/multi-model-text-analyzer/sample-summarizer.json deleted file mode 100644 index b19a1406d4..0000000000 --- a/test/apis/pytorch/multi-model-text-analyzer/sample-summarizer.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "text": "Machine learning (ML) is the scientific study of algorithms and statistical models that computer systems use to perform a specific task without using explicit instructions, relying on patterns and inference instead. It is seen as a subset of artificial intelligence. Machine learning algorithms build a mathematical model based on sample data, known as training data, in order to make predictions or decisions without being explicitly programmed to perform the task. Machine learning algorithms are used in a wide variety of applications, such as email filtering and computer vision, where it is difficult or infeasible to develop a conventional algorithm for effectively performing the task. Machine learning is closely related to computational statistics, which focuses on making predictions using computers. The study of mathematical optimization delivers methods, theory and application domains to the field of machine learning. Data mining is a field of study within machine learning, and focuses on exploratory data analysis through unsupervised learning. In its application across business problems, machine learning is also referred to as predictive analytics." -} diff --git a/test/apis/pytorch/object-detector/coco_labels.txt b/test/apis/pytorch/object-detector/coco_labels.txt deleted file mode 100644 index 8d950d95da..0000000000 --- a/test/apis/pytorch/object-detector/coco_labels.txt +++ /dev/null @@ -1,91 +0,0 @@ -__background__ -person -bicycle -car -motorcycle -airplane -bus -train -truck -boat -traffic light -fire hydrant -N/A -stop sign -parking meter -bench -bird -cat -dog -horse -sheep -cow -elephant -bear -zebra -giraffe -N/A -backpack -umbrella -N/A -N/A -handbag -tie -suitcase -frisbee -skis -snowboard -sports ball -kite -baseball bat -baseball glove -skateboard -surfboard -tennis racket -bottle -N/A -wine glass -cup -fork -knife -spoon -bowl -banana -apple -sandwich -orange -broccoli -carrot -hot dog -pizza -donut -cake -chair -couch -potted plant -bed -N/A -dining table -N/A -N/A -toilet -N/A -tv -laptop -mouse -remote -keyboard -cell phone -microwave -oven -toaster -sink -refrigerator -N/A -book -clock -vase -scissors -teddy bear -hair drier -toothbrush diff --git a/test/apis/pytorch/object-detector/cortex.yaml b/test/apis/pytorch/object-detector/cortex.yaml deleted file mode 100644 index 6cd2e90e49..0000000000 --- a/test/apis/pytorch/object-detector/cortex.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- name: object-detector - kind: RealtimeAPI - handler: - type: python - path: handler.py - compute: - cpu: 1 - gpu: 1 - mem: 4G diff --git a/test/apis/pytorch/object-detector/handler.py b/test/apis/pytorch/object-detector/handler.py deleted file mode 100644 index faa320e31c..0000000000 --- a/test/apis/pytorch/object-detector/handler.py +++ /dev/null @@ -1,47 +0,0 @@ -from io import BytesIO - -import requests -import torch -from PIL import Image -from torchvision import models -from torchvision import transforms - - -class Handler: - def __init__(self, config): - self.device = "cuda" if torch.cuda.is_available() else "cpu" - print(f"using device: {self.device}") - - model = models.detection.fasterrcnn_resnet50_fpn(pretrained=True).to(self.device) - model.eval() - - self.preprocess = transforms.Compose([transforms.ToTensor()]) - - with open("/mnt/project/coco_labels.txt") as f: - self.coco_labels = f.read().splitlines() - - self.model = model - - def handle_post(self, payload): - threshold = float(payload["threshold"]) - image = requests.get(payload["url"]).content - img_pil = Image.open(BytesIO(image)) - img_tensor = self.preprocess(img_pil).to(self.device) - img_tensor.unsqueeze_(0) - - with torch.no_grad(): - pred = self.model(img_tensor) - - predicted_class = [self.coco_labels[i] for i in pred[0]["labels"].cpu().tolist()] - predicted_boxes = [ - [(i[0], i[1]), (i[2], i[3])] for i in pred[0]["boxes"].detach().cpu().tolist() - ] - predicted_score = pred[0]["scores"].detach().cpu().tolist() - predicted_t = [predicted_score.index(x) for x in predicted_score if x > threshold] - if len(predicted_t) == 0: - return [], [] - - predicted_t = predicted_t[-1] - predicted_boxes = predicted_boxes[: predicted_t + 1] - predicted_class = predicted_class[: predicted_t + 1] - return predicted_boxes, predicted_class diff --git a/test/apis/pytorch/object-detector/requirements.txt b/test/apis/pytorch/object-detector/requirements.txt deleted file mode 100644 index ac988bdf84..0000000000 --- a/test/apis/pytorch/object-detector/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -torch -torchvision diff --git a/test/apis/pytorch/object-detector/sample.json b/test/apis/pytorch/object-detector/sample.json deleted file mode 100644 index 5005f13bad..0000000000 --- a/test/apis/pytorch/object-detector/sample.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "url": "https://i.imgur.com/PzXprwl.jpg", - "threshold": "0.8" -} diff --git a/test/apis/pytorch/question-generator/cortex.yaml b/test/apis/pytorch/question-generator/cortex.yaml deleted file mode 100644 index ebd9e13bb4..0000000000 --- a/test/apis/pytorch/question-generator/cortex.yaml +++ /dev/null @@ -1,8 +0,0 @@ -- name: question-generator - kind: RealtimeAPI - handler: - type: python - path: handler.py - compute: - cpu: 1 - mem: 6G diff --git a/test/apis/pytorch/question-generator/dependencies.sh b/test/apis/pytorch/question-generator/dependencies.sh deleted file mode 100644 index 27980fbf99..0000000000 --- a/test/apis/pytorch/question-generator/dependencies.sh +++ /dev/null @@ -1,2 +0,0 @@ -# torchvision isn’t required for this example, and pip was throwing warnings with it installed -pip uninstall torchvision -y diff --git a/test/apis/pytorch/question-generator/handler.py b/test/apis/pytorch/question-generator/handler.py deleted file mode 100644 index 17fe084183..0000000000 --- a/test/apis/pytorch/question-generator/handler.py +++ /dev/null @@ -1,34 +0,0 @@ -from transformers import AutoModelWithLMHead, AutoTokenizer -import spacy -import subprocess -import json - - -class Handler: - def __init__(self, config): - subprocess.call("python -m spacy download en_core_web_sm".split(" ")) - import en_core_web_sm - - self.tokenizer = AutoTokenizer.from_pretrained( - "mrm8488/t5-base-finetuned-question-generation-ap" - ) - self.model = AutoModelWithLMHead.from_pretrained( - "mrm8488/t5-base-finetuned-question-generation-ap" - ) - self.nlp = en_core_web_sm.load() - - def handle_post(self, payload): - context = payload["context"] - answer = payload["answer"] - max_length = int(payload.get("max_length", 64)) - - input_text = "answer: {} context: {} ".format(answer, context) - features = self.tokenizer([input_text], return_tensors="pt") - - output = self.model.generate( - input_ids=features["input_ids"], - attention_mask=features["attention_mask"], - max_length=max_length, - ) - - return {"result": self.tokenizer.decode(output[0])} diff --git a/test/apis/pytorch/question-generator/requirements.txt b/test/apis/pytorch/question-generator/requirements.txt deleted file mode 100644 index 13f0b839bc..0000000000 --- a/test/apis/pytorch/question-generator/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -sentencepiece==0.1.95 -spacy==2.1.8 --e git+https://github.com/huggingface/transformers.git#egg=transformers ---find-links https://download.pytorch.org/whl/torch_stable.html -torch==1.6.0+cpu diff --git a/test/apis/pytorch/question-generator/sample.json b/test/apis/pytorch/question-generator/sample.json deleted file mode 100644 index 88c9fb0c92..0000000000 --- a/test/apis/pytorch/question-generator/sample.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "context": "Sarah works as a software engineer in London", - "answer": "London" -} diff --git a/test/apis/pytorch/reading-comprehender/cortex.yaml b/test/apis/pytorch/reading-comprehender/cortex.yaml deleted file mode 100644 index b38a8f7173..0000000000 --- a/test/apis/pytorch/reading-comprehender/cortex.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- name: reading-comprehender - kind: RealtimeAPI - handler: - type: python - path: handler.py - compute: - cpu: 1 - gpu: 1 - mem: 4G diff --git a/test/apis/pytorch/reading-comprehender/handler.py b/test/apis/pytorch/reading-comprehender/handler.py deleted file mode 100644 index 733b96b813..0000000000 --- a/test/apis/pytorch/reading-comprehender/handler.py +++ /dev/null @@ -1,23 +0,0 @@ -import torch -from allennlp.predictors.predictor import Predictor as AllenNLPPredictor - - -class Handler: - def __init__(self, config): - self.device = "cuda" if torch.cuda.is_available() else "cpu" - print(f"using device: {self.device}") - - cuda_device = -1 - if self.device == "cuda": - cuda_device = 0 - - self.predictor = AllenNLPPredictor.from_path( - "https://storage.googleapis.com/allennlp-public-models/bidaf-elmo-model-2018.11.30-charpad.tar.gz", - cuda_device=cuda_device, - ) - - def handle_post(self, payload): - prediction = self.predictor.predict( - passage=payload["passage"], question=payload["question"] - ) - return prediction["best_span_str"] diff --git a/test/apis/pytorch/reading-comprehender/requirements.txt b/test/apis/pytorch/reading-comprehender/requirements.txt deleted file mode 100644 index 13dd5fbdba..0000000000 --- a/test/apis/pytorch/reading-comprehender/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -allennlp==0.9.* diff --git a/test/apis/pytorch/reading-comprehender/sample.json b/test/apis/pytorch/reading-comprehender/sample.json deleted file mode 100644 index 14f60455bc..0000000000 --- a/test/apis/pytorch/reading-comprehender/sample.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "passage": "Cortex Labs is building machine learning infrastructure for deploying models in production", - "question": "What does Cortex Labs do?" -} diff --git a/test/apis/pytorch/search-completer/cortex.yaml b/test/apis/pytorch/search-completer/cortex.yaml deleted file mode 100644 index 91ee7211d4..0000000000 --- a/test/apis/pytorch/search-completer/cortex.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- name: search-completer - kind: RealtimeAPI - handler: - type: python - path: handler.py - compute: - cpu: 1 - gpu: 1 - mem: 4G diff --git a/test/apis/pytorch/search-completer/handler.py b/test/apis/pytorch/search-completer/handler.py deleted file mode 100644 index 18c4b75b81..0000000000 --- a/test/apis/pytorch/search-completer/handler.py +++ /dev/null @@ -1,18 +0,0 @@ -import torch -import regex -import tqdm - - -class Handler: - def __init__(self, config): - roberta = torch.hub.load("pytorch/fairseq", "roberta.large", force_reload=True) - roberta.eval() - device = "cuda" if torch.cuda.is_available() else "cpu" - print(f"using device: {device}") - roberta.to(device) - - self.model = roberta - - def handle_post(self, payload): - predictions = self.model.fill_mask(payload["text"] + " ", topk=5) - return [prediction[0] for prediction in predictions] diff --git a/test/apis/pytorch/search-completer/requirements.txt b/test/apis/pytorch/search-completer/requirements.txt deleted file mode 100644 index 16b9215d31..0000000000 --- a/test/apis/pytorch/search-completer/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -torch -regex -tqdm -dataclasses -hydra-core diff --git a/test/apis/pytorch/search-completer/sample.json b/test/apis/pytorch/search-completer/sample.json deleted file mode 100644 index dfd2a2f433..0000000000 --- a/test/apis/pytorch/search-completer/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "text": "machine learning is" -} diff --git a/test/apis/pytorch/sentiment-analyzer/cortex.yaml b/test/apis/pytorch/sentiment-analyzer/cortex.yaml deleted file mode 100644 index 9e691a786a..0000000000 --- a/test/apis/pytorch/sentiment-analyzer/cortex.yaml +++ /dev/null @@ -1,8 +0,0 @@ -- name: sentiment-analyzer - kind: RealtimeAPI - handler: - type: python - path: handler.py - compute: - cpu: 1 - # gpu: 1 # this is optional, since the api can also run on cpu diff --git a/test/apis/pytorch/sentiment-analyzer/handler.py b/test/apis/pytorch/sentiment-analyzer/handler.py deleted file mode 100644 index e7ee5e5ceb..0000000000 --- a/test/apis/pytorch/sentiment-analyzer/handler.py +++ /dev/null @@ -1,13 +0,0 @@ -import torch -from transformers import pipeline - - -class Handler: - def __init__(self, config): - device = 0 if torch.cuda.is_available() else -1 - print(f"using device: {'cuda' if device == 0 else 'cpu'}") - - self.analyzer = pipeline(task="sentiment-analysis", device=device) - - def handle_post(self, payload): - return self.analyzer(payload["text"])[0] diff --git a/test/apis/pytorch/sentiment-analyzer/requirements.txt b/test/apis/pytorch/sentiment-analyzer/requirements.txt deleted file mode 100644 index 3f565d80e4..0000000000 --- a/test/apis/pytorch/sentiment-analyzer/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -torch -transformers==2.9.* diff --git a/test/apis/pytorch/sentiment-analyzer/sample.json b/test/apis/pytorch/sentiment-analyzer/sample.json deleted file mode 100644 index 7622d16ae0..0000000000 --- a/test/apis/pytorch/sentiment-analyzer/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "text": "best day ever" -} diff --git a/test/apis/pytorch/server-side-batching/cortex.yaml b/test/apis/pytorch/server-side-batching/cortex.yaml deleted file mode 100644 index c11f0419b7..0000000000 --- a/test/apis/pytorch/server-side-batching/cortex.yaml +++ /dev/null @@ -1,11 +0,0 @@ -- name: iris-classifier - kind: RealtimeAPI - handler: - type: python - path: handler.py - config: - model: s3://cortex-examples/pytorch/iris-classifier/weights.pth - server_side_batching: - max_batch_size: 8 - batch_interval: 0.1s - threads_per_process: 8 diff --git a/test/apis/pytorch/server-side-batching/handler.py b/test/apis/pytorch/server-side-batching/handler.py deleted file mode 100644 index 1b27bd316e..0000000000 --- a/test/apis/pytorch/server-side-batching/handler.py +++ /dev/null @@ -1,49 +0,0 @@ -import re -import torch -import os -import boto3 -from botocore import UNSIGNED -from botocore.client import Config -from model import IrisNet - -labels = ["setosa", "versicolor", "virginica"] - - -class Handler: - def __init__(self, config): - # download the model - bucket, key = re.match("s3://(.+?)/(.+)", config["model"]).groups() - s3 = boto3.client("s3") - s3.download_file(bucket, key, "/tmp/model.pth") - - # initialize the model - model = IrisNet() - model.load_state_dict(torch.load("/tmp/model.pth")) - model.eval() - - self.model = model - - def handle_post(self, payload): - responses = [] - - # note: this is not the most efficient way, it's just to test server-side batching - for sample in payload: - # Convert the request to a tensor and pass it into the model - input_tensor = torch.FloatTensor( - [ - [ - sample["sepal_length"], - sample["sepal_width"], - sample["petal_length"], - sample["petal_width"], - ] - ] - ) - - # Run the prediction - output = self.model(input_tensor) - - # Translate the model output to the corresponding label string - responses.append(labels[torch.argmax(output[0])]) - - return responses diff --git a/test/apis/pytorch/server-side-batching/model.py b/test/apis/pytorch/server-side-batching/model.py deleted file mode 100644 index a43a362544..0000000000 --- a/test/apis/pytorch/server-side-batching/model.py +++ /dev/null @@ -1,58 +0,0 @@ -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch.autograd import Variable - - -class IrisNet(nn.Module): - def __init__(self): - super(IrisNet, self).__init__() - self.fc1 = nn.Linear(4, 100) - self.fc2 = nn.Linear(100, 100) - self.fc3 = nn.Linear(100, 3) - self.softmax = nn.Softmax(dim=1) - - def forward(self, X): - X = F.relu(self.fc1(X)) - X = self.fc2(X) - X = self.fc3(X) - X = self.softmax(X) - return X - - -if __name__ == "__main__": - from sklearn.datasets import load_iris - from sklearn.model_selection import train_test_split - from sklearn.metrics import accuracy_score - - iris = load_iris() - X, y = iris.data, iris.target - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.8, random_state=42) - - train_X = Variable(torch.Tensor(X_train).float()) - test_X = Variable(torch.Tensor(X_test).float()) - train_y = Variable(torch.Tensor(y_train).long()) - test_y = Variable(torch.Tensor(y_test).long()) - - model = IrisNet() - - criterion = nn.CrossEntropyLoss() - - optimizer = torch.optim.SGD(model.parameters(), lr=0.01) - - for epoch in range(1000): - optimizer.zero_grad() - out = model(train_X) - loss = criterion(out, train_y) - loss.backward() - optimizer.step() - - if epoch % 100 == 0: - print("number of epoch {} loss {}".format(epoch, loss)) - - predict_out = model(test_X) - _, predict_y = torch.max(predict_out, 1) - - print("prediction accuracy {}".format(accuracy_score(test_y.data, predict_y.data))) - - torch.save(model.state_dict(), "weights.pth") diff --git a/test/apis/pytorch/server-side-batching/requirements.txt b/test/apis/pytorch/server-side-batching/requirements.txt deleted file mode 100644 index 12c6d5d5ea..0000000000 --- a/test/apis/pytorch/server-side-batching/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -torch diff --git a/test/apis/pytorch/server-side-batching/sample.json b/test/apis/pytorch/server-side-batching/sample.json deleted file mode 100644 index 0bc6836266..0000000000 --- a/test/apis/pytorch/server-side-batching/sample.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "sepal_length": 2.2, - "sepal_width": 3.6, - "petal_length": 1.4, - "petal_width": 3.3 -} diff --git a/test/apis/pytorch/text-generator/cortex.yaml b/test/apis/pytorch/text-generator/cortex.yaml deleted file mode 100644 index 025d41c87d..0000000000 --- a/test/apis/pytorch/text-generator/cortex.yaml +++ /dev/null @@ -1,8 +0,0 @@ -- name: text-generator - kind: RealtimeAPI - handler: - type: python - path: handler.py - compute: - cpu: 1 - gpu: 1 # this is optional, since the api can also run on cpu diff --git a/test/apis/pytorch/text-generator/handler.py b/test/apis/pytorch/text-generator/handler.py deleted file mode 100644 index 1ba965c30d..0000000000 --- a/test/apis/pytorch/text-generator/handler.py +++ /dev/null @@ -1,16 +0,0 @@ -import torch -from transformers import GPT2Tokenizer, GPT2LMHeadModel - - -class Handler: - def __init__(self, config): - self.device = "cuda" if torch.cuda.is_available() else "cpu" - print(f"using device: {self.device}") - self.tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - self.model = GPT2LMHeadModel.from_pretrained("gpt2").to(self.device) - - def handle_post(self, payload): - input_length = len(payload["text"].split()) - tokens = self.tokenizer.encode(payload["text"], return_tensors="pt").to(self.device) - prediction = self.model.generate(tokens, max_length=input_length + 20, do_sample=True) - return self.tokenizer.decode(prediction[0]) diff --git a/test/apis/pytorch/text-generator/requirements.txt b/test/apis/pytorch/text-generator/requirements.txt deleted file mode 100644 index 4d131781b1..0000000000 --- a/test/apis/pytorch/text-generator/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -torch==1.7.* -transformers==3.0.* diff --git a/test/apis/pytorch/text-generator/sample.json b/test/apis/pytorch/text-generator/sample.json deleted file mode 100644 index dfd2a2f433..0000000000 --- a/test/apis/pytorch/text-generator/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "text": "machine learning is" -} diff --git a/test/apis/pytorch/text-summarizer/README.md b/test/apis/pytorch/text-summarizer/README.md deleted file mode 100644 index 39715961a9..0000000000 --- a/test/apis/pytorch/text-summarizer/README.md +++ /dev/null @@ -1 +0,0 @@ -Please refer [here](https://sshleifer.github.io/blog_v2/jupyter/2020/03/12/bart.html) to learn more about BART. diff --git a/test/apis/pytorch/text-summarizer/cortex.yaml b/test/apis/pytorch/text-summarizer/cortex.yaml deleted file mode 100644 index 439bfb3cb9..0000000000 --- a/test/apis/pytorch/text-summarizer/cortex.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- name: text-summarizer - kind: RealtimeAPI - handler: - type: python - path: handler.py - compute: - cpu: 1 - gpu: 1 # this is optional, since the api can also run on cpu - mem: 6G diff --git a/test/apis/pytorch/text-summarizer/handler.py b/test/apis/pytorch/text-summarizer/handler.py deleted file mode 100644 index c65df86f34..0000000000 --- a/test/apis/pytorch/text-summarizer/handler.py +++ /dev/null @@ -1,16 +0,0 @@ -import torch -from transformers import pipeline - - -class Handler: - def __init__(self, config): - device = 0 if torch.cuda.is_available() else -1 - print(f"using device: {'cuda' if device == 0 else 'cpu'}") - - self.summarizer = pipeline(task="summarization", device=device) - - def handle_post(self, payload): - summary = self.summarizer( - payload["text"], num_beams=4, length_penalty=2.0, max_length=142, no_repeat_ngram_size=3 - ) - return summary[0]["summary_text"] diff --git a/test/apis/pytorch/text-summarizer/requirements.txt b/test/apis/pytorch/text-summarizer/requirements.txt deleted file mode 100644 index 5afceb377e..0000000000 --- a/test/apis/pytorch/text-summarizer/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -transformers==2.9.* -torch diff --git a/test/apis/pytorch/text-summarizer/sample.json b/test/apis/pytorch/text-summarizer/sample.json deleted file mode 100644 index e54b77f18c..0000000000 --- a/test/apis/pytorch/text-summarizer/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "text": "Machine learning (ML) is the scientific study of algorithms and statistical models that computer systems use to perform a specific task without using explicit instructions, relying on patterns and inference instead. It is seen as a subset of artificial intelligence. Machine learning algorithms build a mathematical model based on sample data, known as training data, in order to make predictions or decisions without being explicitly programmed to perform the task. Machine learning algorithms are used in a wide variety of applications, such as email filtering and computer vision, where it is difficult or infeasible to develop a conventional algorithm for effectively performing the task. Machine learning is closely related to computational statistics, which focuses on making predictions using computers. The study of mathematical optimization delivers methods, theory and application domains to the field of machine learning. Data mining is a field of study within machine learning, and focuses on exploratory data analysis through unsupervised learning. In its application across business problems, machine learning is also referred to as predictive analytics." -} diff --git a/test/apis/realtime/Dockerfile b/test/apis/realtime/Dockerfile deleted file mode 100644 index 2955a09120..0000000000 --- a/test/apis/realtime/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -# Use the official lightweight Python image. -# https://hub.docker.com/_/python -FROM python:3.9-slim - -# Allow statements and log messages to immediately appear in the Knative logs -ENV PYTHONUNBUFFERED True - -# Copy local code to the container image. -ENV APP_HOME /app -WORKDIR $APP_HOME -COPY . ./ - -# Install production dependencies. -RUN pip install Flask gunicorn - -# Run the web service on container startup. Here we use the gunicorn -# webserver, with one worker process and 8 threads. -# For environments with multiple CPU cores, increase the number of workers -# to be equal to the cores available. -CMD exec gunicorn --bind :$CORTEX_PORT --workers 1 --threads $NUM_THREADS --timeout 0 main:app diff --git a/test/apis/realtime/cortex.yaml b/test/apis/realtime/cortex.yaml deleted file mode 100644 index 9f4e231cea..0000000000 --- a/test/apis/realtime/cortex.yaml +++ /dev/null @@ -1,15 +0,0 @@ -- name: realtime - kind: RealtimeAPI - pod: - containers: - - name: api - image: 499593605069.dkr.ecr.us-west-2.amazonaws.com/sample/realtime-caas:latest - env: - NUM_THREADS: "8" - compute: - cpu: 200m - mem: 512Mi - autoscaling: - max_queue_length: 16 - max_concurrency: 8 - target_in_flight: 10 diff --git a/test/apis/realtime/hello-world/.dockerignore b/test/apis/realtime/hello-world/.dockerignore new file mode 100644 index 0000000000..12657957ef --- /dev/null +++ b/test/apis/realtime/hello-world/.dockerignore @@ -0,0 +1,8 @@ +*.dockerfile +README.md +sample.json +*.pyc +*.pyo +*.pyd +__pycache__ +.pytest_cache diff --git a/test/apis/realtime/hello-world/cortex_cpu.yaml b/test/apis/realtime/hello-world/cortex_cpu.yaml new file mode 100644 index 0000000000..071ba16b88 --- /dev/null +++ b/test/apis/realtime/hello-world/cortex_cpu.yaml @@ -0,0 +1,16 @@ +- name: hello-world + kind: RealtimeAPI + pod: + port: 9000 + containers: + - name: api + image: quay.io/cortexlabs-test/realtime-hello-world-cpu:latest + readiness_probe: + http_get: + path: "/healthz" + port: 9000 + compute: + cpu: 200m + mem: 128M + autoscaling: + max_concurrency: 1 diff --git a/test/apis/realtime/hello-world/hello-world-cpu.dockerfile b/test/apis/realtime/hello-world/hello-world-cpu.dockerfile new file mode 100644 index 0000000000..a496f7c881 --- /dev/null +++ b/test/apis/realtime/hello-world/hello-world-cpu.dockerfile @@ -0,0 +1,17 @@ +FROM python:3.8-slim + +# Allow statements and log messages to immediately appear in the logs +ENV PYTHONUNBUFFERED True + +# Install production dependencies +RUN pip install --no-cache-dir "uvicorn[standard]" gunicorn fastapi + +# Copy local code to the container image. +COPY . /app +WORKDIR /app/ + +ENV PYTHONPATH=/app +ENV CORTEX_PORT=9000 + +# Run the web service on container startup. +CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/realtime/hello-world/main.py b/test/apis/realtime/hello-world/main.py new file mode 100644 index 0000000000..a164dbf79a --- /dev/null +++ b/test/apis/realtime/hello-world/main.py @@ -0,0 +1,16 @@ +import os +from fastapi import FastAPI + +app = FastAPI() + +response_str = os.getenv("RESPONSE", "hello world") + + +@app.get("/healthz") +def healthz(): + return "ok" + + +@app.post("/") +def post_handler(): + return response_str diff --git a/test/apis/sleep/sample.json b/test/apis/realtime/hello-world/sample.json similarity index 100% rename from test/apis/sleep/sample.json rename to test/apis/realtime/hello-world/sample.json diff --git a/test/apis/realtime/.dockerignore b/test/apis/realtime/image-classifier-resnet50/.dockerignore similarity index 70% rename from test/apis/realtime/.dockerignore rename to test/apis/realtime/image-classifier-resnet50/.dockerignore index 3e4bdd9fbb..6482ea768a 100644 --- a/test/apis/realtime/.dockerignore +++ b/test/apis/realtime/image-classifier-resnet50/.dockerignore @@ -1,5 +1,6 @@ -Dockerfile +*.dockerfile README.md +client.py *.pyc *.pyo *.pyd diff --git a/test/apis/realtime/image-classifier-resnet50/client.py b/test/apis/realtime/image-classifier-resnet50/client.py new file mode 100644 index 0000000000..5b0b23eb6b --- /dev/null +++ b/test/apis/realtime/image-classifier-resnet50/client.py @@ -0,0 +1,69 @@ +"""A client that performs inferences on a ResNet model using the REST API. + +The client downloads a test image of a cat, queries the server over the REST API +with the test image repeatedly and measures how long it takes to respond. + +The client expects a TensorFlow Serving ModelServer running a ResNet SavedModel +from: + +https://github.com/tensorflow/models/tree/master/official/resnet#pre-trained-model + +The SavedModel must be one that can take JPEG images as inputs. + +Typical usage example: + + python client.py +""" + +import sys +import base64 +import requests + +# the image URL is the location of the image we should send to the server +IMAGE_URL = "https://tensorflow.org/images/blogs/serving/cat.jpg" + + +def main(): + # parse arg + if len(sys.argv) != 2: + print("usage: python client.py ") + sys.exit(1) + address = sys.argv[1] + server_url = f"{address}/v1/models/resnet50:predict" + + # download labels + labels = requests.get( + "https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt" + ).text.split("\n")[1:] + + # download the image + dl_request = requests.get(IMAGE_URL, stream=True) + dl_request.raise_for_status() + + # compose a JSON Predict request (send JPEG image in base64). + jpeg_bytes = base64.b64encode(dl_request.content).decode("utf-8") + predict_request = '{"instances" : [{"b64": "%s"}]}' % jpeg_bytes + + # send few requests to warm-up the model. + for _ in range(3): + response = requests.post(server_url, data=predict_request) + response.raise_for_status() + + # send few actual requests and report average latency. + total_time = 0 + num_requests = 10 + for _ in range(num_requests): + response = requests.post(server_url, data=predict_request) + response.raise_for_status() + total_time += response.elapsed.total_seconds() + prediction = labels[response.json()["predictions"][0]["classes"]] + + print( + "Prediction class: {}, avg latency: {} ms".format( + prediction, (total_time * 1000) / num_requests + ) + ) + + +if __name__ == "__main__": + main() diff --git a/test/apis/realtime/image-classifier-resnet50/cortex_cpu.yaml b/test/apis/realtime/image-classifier-resnet50/cortex_cpu.yaml new file mode 100644 index 0000000000..cb8db93493 --- /dev/null +++ b/test/apis/realtime/image-classifier-resnet50/cortex_cpu.yaml @@ -0,0 +1,15 @@ +- name: image-classifier-resnet50 + kind: RealtimeAPI + pod: + port: 8501 + containers: + - name: api + image: quay.io/cortexlabs-test/realtime-image-classifier-resnet50-cpu:latest + readiness_probe: + exec: + command: ["tfs_model_status_probe", "-addr", "localhost:8500", "-model-name", "resnet50"] + compute: + cpu: 1 + mem: 2G + autoscaling: + max_concurrency: 8 diff --git a/test/apis/realtime/image-classifier-resnet50/cortex_gpu.yaml b/test/apis/realtime/image-classifier-resnet50/cortex_gpu.yaml new file mode 100644 index 0000000000..bc840bea3c --- /dev/null +++ b/test/apis/realtime/image-classifier-resnet50/cortex_gpu.yaml @@ -0,0 +1,16 @@ +- name: image-classifier-resnet50 + kind: RealtimeAPI + pod: + port: 8501 + containers: + - name: api + image: quay.io/cortexlabs-test/realtime-image-classifier-resnet50-gpu:latest + readiness_probe: + exec: + command: ["tfs_model_status_probe", "-addr", "localhost:8500", "-model-name", "resnet50"] + compute: + cpu: 200m + gpu: 1 + mem: 512Mi + autoscaling: + max_concurrency: 8 diff --git a/test/apis/realtime/image-classifier-resnet50/image-classifier-resnet50-cpu.dockerfile b/test/apis/realtime/image-classifier-resnet50/image-classifier-resnet50-cpu.dockerfile new file mode 100644 index 0000000000..e64f7cdfa5 --- /dev/null +++ b/test/apis/realtime/image-classifier-resnet50/image-classifier-resnet50-cpu.dockerfile @@ -0,0 +1,17 @@ +FROM tensorflow/serving:2.3.0 + +RUN apt-get update -qq && apt-get install -y -q \ + wget \ + && apt-get clean -qq && rm -rf /var/lib/apt/lists/* + +RUN TFS_PROBE_VERSION=1.0.1 \ + && wget -qO /bin/tfs_model_status_probe https://github.com/codycollier/tfs-model-status-probe/releases/download/v${TFS_PROBE_VERSION}/tfs_model_status_probe_${TFS_PROBE_VERSION}_linux_amd64 \ + && chmod +x /bin/tfs_model_status_probe + +RUN mkdir -p /model/resnet50/ \ + && wget -qO- http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp32_savedmodel_NHWC_jpg.tar.gz | \ + tar --strip-components=2 -C /model/resnet50 -xvz + +ENV CORTEX_PORT 8501 + +ENTRYPOINT tensorflow_model_server --rest_api_port=$CORTEX_PORT --rest_api_num_threads=8 --model_name="resnet50" --model_base_path="/model/resnet50" diff --git a/test/apis/realtime/image-classifier-resnet50/image-classifier-resnet50-gpu.dockerfile b/test/apis/realtime/image-classifier-resnet50/image-classifier-resnet50-gpu.dockerfile new file mode 100644 index 0000000000..a9f4e79478 --- /dev/null +++ b/test/apis/realtime/image-classifier-resnet50/image-classifier-resnet50-gpu.dockerfile @@ -0,0 +1,17 @@ +FROM tensorflow/serving:2.3.0-gpu + +RUN apt-get update -qq && apt-get install -y --no-install-recommends -q \ + wget \ + && apt-get clean -qq && rm -rf /var/lib/apt/lists/* + +RUN TFS_PROBE_VERSION=1.0.1 \ + && wget -qO /bin/tfs_model_status_probe https://github.com/codycollier/tfs-model-status-probe/releases/download/v${TFS_PROBE_VERSION}/tfs_model_status_probe_${TFS_PROBE_VERSION}_linux_amd64 \ + && chmod +x /bin/tfs_model_status_probe + +RUN mkdir -p /model/resnet50/ \ + && wget -qO- http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp32_savedmodel_NHWC_jpg.tar.gz | \ + tar --strip-components=2 -C /model/resnet50 -xvz + +ENV CORTEX_PORT 8501 + +ENTRYPOINT tensorflow_model_server --rest_api_port=$CORTEX_PORT --rest_api_num_threads=8 --model_name="resnet50" --model_base_path="/model/resnet50" diff --git a/test/apis/realtime/image-classifier-resnet50/sample.json b/test/apis/realtime/image-classifier-resnet50/sample.json new file mode 100644 index 0000000000..36c70ef46c --- /dev/null +++ b/test/apis/realtime/image-classifier-resnet50/sample.json @@ -0,0 +1 @@ +"{\"instances\" : [{\"b64\": \"/9j/4AAQSkZJRgABAQAASABIAAD/4QBYRXhpZgAATU0AKgAAAAgAAgESAAMAAAABAAEAAIdpAAQAAAABAAAAJgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAB8qADAAQAAAABAAAC0AAAAAD/7QA4UGhvdG9zaG9wIDMuMAA4QklNBAQAAAAAAAA4QklNBCUAAAAAABDUHYzZjwCyBOmACZjs+EJ+/8AAEQgC0AHyAwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMFBQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkqNDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/EAB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMRBAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/bAEMAAwMDAwMDBQMDBQgFBQUICggICAgKDQoKCgoKDRANDQ0NDQ0QEBAQEBAQEBMTExMTExYWFhYWGRkZGRkZGRkZGf/bAEMBBAQEBgYGCwYGCxoSDxIaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGv/dAAQAIP/aAAwDAQACEQMRAD8A/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA//Q/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAphkQHBPWs7Vr5LG2aZ2CADqelfM3iT4xi0vXtIiQAcHaCxz9elceKxsKHxHVh8JOt8J9VLIjZ2nOKfkV8i6V8b4knRL2TAPXcMY/Kvb/AA/4+0fVYg8UwZ26DPSs6GZUqul7F1sDVp6tHpdFU7W9huVzG24eo6VcrvTT1RxtWCiiimIKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAoopuD3oAdkVl32pR2ti94CPlBYA9wOuPwHFc/4v15NHsRzhp38gexbBB/Lj8a+aPiH8W4reF47YM7iR2WNTk7DwFbHQDnP1rzsZmFOgmnud2FwM6zVloeqal8Wb20AkhgjdWJBGGyp4ODz6Zrn5fjjqILeTbwELgc56/8AfX6V8Z6h4y1fW5XK+YkR9Rhc+u3/ABrn5by+d1zcyZHXIxn8hXzM85xDekrH0MMopJaxPvzSfjeJ5fL1KKNOM5QHqO3U8+le36Vrdhq8CTWkqy7gCdhyBkZxn2zX4/XOtX1vOWW4kVsjGCAK+hPg38aDp1/FpWqShI5mAB7474PGM4ruwWcT5kqzuu5x4zKYqPNTWp+itFZul6pbarbLc2Z3Ie46frjNaVfURkmro+daadmFFFFMQUUUUAFFFFABRRRQAUUUUAf/0f1SooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA4/xtDI/h68aIciNjnr2r8utQv531CRrmYxMWPJYknnoF7nH1xX6y6pBHPYzRydGUjn3Ffkx4306TSfEt3DJIUjikYOTgMQW4A47181nsWnGZ9Fkck+aJZj1CRiY7aJ5PlyzyyYzjuccL+Jq9p2q6pA3m2l2I3B4CMxH8h+dYlnkxCSVSsQ5CHvju3cn0/lW2keoIu52FujDqw+Zh6hBjj/eP4GvmozbZ9BOmtj3LwZ8aNS0t1steAZV4DKePxr6s8MePNH123VracSuwyQO31r85vsqy5gjk3Njuo499qrx+JzWhomuax4Vu0ubGYFMglfu5/4CTXq4XNa1F2lqjy8TllOqrx0Z+pCOHUMO9Pr53+HnxbsNZjWC/kw+Bwa99gv7S4UNFIG3dPxr6vDYunWjzQZ8zXw06UuWSLdFFFdRzhRRRQAUUUUAFFFFABRRRQAUUUUAFFFGaACikzmloAa7pGpdzgDqTXLa14ostNRkDZfkfQ/5IrD8ba69hauiHBfIX6pn+or5V8ZeOJrW22pIXuZR8q9SCTliR+tePmOZ+x9yO56eCy91tXsT/FD4jy3M32dJMTh2AABbGM44Hfnqa+ZWubi9uDNdF23MfmYDBPf5T0xV6WC5vZ5JxcebI3UyDGS3Tv07ZqKJ7QpJBe+bbmNgkgJyFbOMjPI/zmvjqs51ZOTZ9hRoRpQ5USh/Kb94+z0dfusPTjpWZe3kYXEcy7v9ruPqOP0FW5rKaydWeQz2sp271OQM8ZYdRn6cGubuLNhMd4G8NnrjnoQQeDms+SyNVqZdyp37ojkEHK9SOf5Vzhml068W4jJVkbPHGD6iupLRxn7m3HYjke1c5evBNHjvmrpysxTjc/TL4AePm8QeHbeEy5dBiRCAACOMhgO/v+dfT6uGAI71+SHwO1+5stYexkuxbFhuUkkKSp54APX0NfqV4V1JdT0uGYAkgAFuxPtnkfSvr8nxTnD2T6Hx+a4b2dTmXU6eiiivbPICiiigAooooAKKKKACiiigD//S/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigCC5iE0JQnFfmv+0J4fl07xS9zApWOUhg20s2f5Amv0vPSvl79ojwi2qaA2oR5DQDPy9ffHvXk5xQ56N10PVyiv7Ouk+p8RaFKZSGUDcvCknIGO/px+prrBHa3MwMjltzcsG+8fQcZ/wA+p48z0+5fTfOtlOZIuCc9MkKFX/PauwtdRt4Ll45kCBBtCgbpAmdqr14Z/wA+TniviIxaZ9pNX2N+NJL2V47MMtsh+YJ8qlvfux9yfoO9Wl0y2CmVNgGcEhdxOOvPP88UttN9uzPOSYrcfJbxjChv9oggMfY8Dqa0IbD7SRc34aTLAJFEOOO+TyRnjI6npXbGHMjkk7GGI7vT5vtGns6MDleAP0zXq/hv4p6lb+RFNKUkiK5DcZ2nIOPQ+1ctDopkcrdxxwq3IQ8yH6ryR/wIk+wqld6GmP3TrGQQV4BK/iD39KpRnT1g7GM1CppJH2L4T+KNnf7LW/YRyM4BJPGDx1/WvXbXULW8j82Bww27vwr80rXUr/S5fKmw4HI7AjvjPevYNA8e6lBEpilYRbSrZ7Dtn6GvVwudSh7tZXPIxOUp+9TPtqivDdL+J4ljTzhyuC3uB1x/Ot6++JWmQGRYX3so+THfJ6/lXsxzTDOPNznlPA1k+XlPVKK8Yt/ilar5T3KkCQHcPQr0H41txfEzRzaLM5+dskr6c/4VdPMMPP4ZEywVaO8T0yivMW+KGjZUxqSGIHX1qhf/ABU0y2nMYIKhSMj+/jj8Kc8ww8VdzQRwdZuyieu0V4Jc/F+1D7IFLDGfyOKzZvjI0pWRIsIeMZ6H/wCtXLLOsIvtm8crxD+yfRpIHBPWjINfM6/FG/vWhKpho95Y5659PTAqW1+JWsW0i+eodYwfYc5Az+FT/beH8ynlVc+k6QjNfPTfFe+a4K+ViIALx16/4V0Nj8VLctJ9tjKc/KB2Hv8AWrhnGFk7cxnLLa8Vex7KBiqOo6hbabavdXLbVQE1wl58R9Hghyr5YxhsDrls4H4da8N8b+P7i6hlW4YqnJRRz2AyfypYrNqNOD5HdlYfLqtSSUlZGZ8SviFDcRsYBtCOzxjOT8x6/QE8V81Sy3erXr3quxZWxgLgbu+Wz3PrV67nuNdvJWY7Vc5ZsYCrxhSzFRnn7q+tdlpuhTmBIuYLhMKjkAowPOGA6D3zXyM3Urz5p7s+upU4UIcqONWynnjYMuZAM/NwSAc9cDp7ipbyx+2Wkd9kTb8RS9x8vC7vRsYB9cda7e50OaH968DQujZUlSQrezDsagbTZpnOQrrMMSJ1U+uVPOPQg5FbwoeRLrLdHm8dkIYJbG53NbnBIccqp4PPp/LrXIalbPZ3DWty2UJIVsg5HY5Feo39lLHGEVpIymcB1J+XoMZ+8OOevFcNr1g1xBHKEQZG3IUEe3XGQR+NTUpWNYVNTz28MxbfuJA7/TiuPmkKuyKeSTj2rq7u3uY2KBc44+XpWDHbEy7nxjmudRsbPU6bwHdG38TWrruBzggduMEmv1C+DF/LcWjISHC4B24GPr3/ADr8r/D8LQ65bvuGwMDk4GB6/j2r9K/gzaTiU3UROw8Ntxlc9m/pXrZTJrEK3Y8TN4r2ep9Q0Ui7sfNyaWvsT5IKKKKACiiigAooooAKKKKAP//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigArm/FmkR6zodxZv1ZDg/4V0lNdd6MvqMVFSCnFxfUqEnGSkj8f/F2hS+HNcubMsEkaXIB5IAPUDux7ZrNtWkH+qbY0kgz/E7ueg49SfwAr6T/AGivBH9mXY1vywFcEs69c+nqa+dvDTQXN5bxheAS20nbt/vM30A/OvgsTRlCq0z9AwtZVaKmj0S3vLWxaHQ5ys05BeTcV2KiDJkdR1GegPYZPpWzavJf3X7p5pJJMFtmRJkj5Qo528dAMBRySSad4e8OWc1vPeXETAXZDyuW+fDEFRkA47bVAJ79a9M0v+z9PtfskSLawAliu3YxyernJJJ68kk9zXXSo3V5M5q1VR2V2c/a6HevIdyeUH6gEuxx/fbOAB2A789aZc6SY02i4YOpxmPA5/HNdW+rW8qljF/o4bHHG4kdgOWJ9P6VgXN9eSJhLTyEH3Y1KbiPQ9Pxxj8a1lGFvdOVTm3qcldaSxVvMJcDoXbJz9SBWXZXkukuAGDd/LOAduMHBXIIre1OPWGbJjVARyGdePbHeuD1a0ZFErSYxzsXJ5Hc9h9RXBVp9UdlN3VmdvbawJleS1PlvHnfHnOR2K9609N12OZgCPmJPH5153o199kuEMhyoBHI5I6j8Aenaqs0503UmhaQssjZXnpk7v8A9Vc3mVKNtD3KW8gnRuh24P6cn9awLi/lSXybT5s+/Q1TsUM9ivk/MXbj8OT+FaenaIrYuI2yAH3HPJOf6VnOHOKKUdytObhnSKFjgthsfn+FP8iWFkE55UYck/3uldlBFaQHeuCxJI47471z+oRG4eaViFVSBnHH/wCvrzU1KSUUy4TV7WOVUSQCUoCzb8H/AHTip4Z1Nx5bkKWclfQL1/QVfk8hbBVkz5s4Ue55Y/4Uy5tWEalX8oAFR77yeOPQCsFS6o3c11NbTtTtvLkkiZCisB1HDNyf8KfcXMUmYGbndycjB5yao+H7SEWRjH3gxY+vJ4HuaTUYoIvk4aSVhs/LJNdCbsYNLmNl7qKGJHj4XHGevPGapLcPLI6xEbR8xP8AeOM4/Ws77OLpke4yIyOcjHAOf1qzDdQSXQsLfDHPJ9Mdf0q4tt6ktJEmta6ukwiOMLJcnacP0QHA5xXmk+vRzFri/Ly7i2BnGQoyff0/DjioPF9/LNqsrQuFEQAweCQMgH8OTXnWoXi2phiacR7V643MQPuqB2GeSc8nsTSd5Ssb06aUbnsGmSSSItxdSLHGRjyiAx555Y4x9AK3YruRz5cdrgx9PMlXaR6ADJ5+grx3TtWluLtLSJpGj2/KOUwM8n5DuP4kn1Ir0drY3BEH2m3tQw/1cyI+Rj2yxJ9ya9CjHQ5q2j1OhXW2tI2W/KwAHOULTAD68kfyqI+J7KchRcx3IYk7RlHx9M9q5G802SHMGnOINgYnyZUDAdN2HC4H+6vHrXkWvapqKnypLiKeTJB2ljKdvcruBb6iutRdrr+v69TKNNSPoz+3NGv8wzE5OCBsJAx7jpz61gXvh/Tpo3Nmcbzltvzqx/2kJyD6EV8yNrF6wVDcSW/ozAx5/wCA9fx/OrVvql1A/wA7NvxjdnaxH1HUfnSkpdTWOHtrFnaeI/D89shkjkBzn7vLH/H8ea4J7dbaMgnB756/WtSTVJ4fmmlabfx8xz/kiueur0yEq/8AEOorhnG7OuN0rMvaO5l1OIICQrLkgDPXtX6l/BuK/ist0ixNGRgOOXHsSD2+lfm58P8ATln1qxVxyzMVXqXKgnGPUYr9TPhppj2OkRMzhgw3Kdmxh6qw45B9RXp5PC9e66Hh51P3LHqlFFFfWHyoUUUUAFFFFABRRRQAUUUUAf/U/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA83+J3hBPFvhqezUDzVUsp75HSvzh0zRprHxSumyr5cibotxUEljnJGRwBk5Jr9ZWUMpU9DXxD8evAw0vUW17TP3P2gYZunzH0Pb/wCvXz2d4e1q8fmfQZLi7N0X12POb/xvo2kXC6dYzbvJHGw/Jk9SXwcsxznsPWuevPGFvdsIJtszkjCCXaozjoGA/Mj6ZrzO1DbxBOxjRWB4xufHbkdOO/15NdJo+leJ9Vui5/dKAW3EAIvP3iT8zEA8KPxwK8SNSU9UfRewhBXZ041vUpZozY2kS2iZVXaRgm0dcAAPKxPXbhc9TXULq18BstoFVtvL9CPXAJ2gZ4yWPPSsx47LS2MWLjUrhFziMMQCP78uOT6IgAH6muJfFtwoWxs0sYF4KFtr8c/OcnH4kn8a6InPJJ7IsTm38wGSHa8g5JVdzfTHzY/AA1mTW7pCIleSRGPIZiu38+g9K1LWC581luctITk4c4YdOCeMenHPtTrmwulbcibQPlYpHgj+7yTyB34rOqnYSZwxRAmE+c9VOfTkr2zXUnRxrFxBeoMIzKzDqPlAOM/h+lXLbRYpmzcLhkOCQMZHXP41uSkROYoV2L1BH94Zz+vNeXVnym8feNG3uYNLtTASDtPy/jWnp2qCbCR8bwTjtnArza/mmBKseuR9R/8AWp2g6hcy35hGVVAQSeg61zKtJs1dFWbO81XUngkBhPrkY5ABxx+lJqF5K9qlsowZdvPf5jgViTKJpHM5wWT5eeucevYVrpIvmJLs3CMgDPP3ec/hVqV3YmySRjW7qdXs4JwzKu8/98L/APXq/NcRy3cUZbKld+PTOaclmtnJNkbnV2MfHODjI9qyrSOZdR+2yYC54HYg8H8qPhuh76o20uUsrgup3ALnA9c9/wADWet+txqEczfdQAjPoetYOtXpt7mNs4GMH3wcf04pLOVrmRJ8AbwFyOePvfn2qOdrYpQT1Z2mryzSWx8oHZIOo+nFcppV5FpP2m7vOJEUlWbrkgcV0lzdPFagDO1FLEfgAP5153rcAv4JZnk2JGhLhOCSeAg+tbueqMEtLHmWt3893qJjjcgznIUDcSc5UAe55yeABUR0hUjI1GF3zwsioRIT/sYzwO5OM9qZdaVeQOivCZbuddwWMnESMcBSw/iYDoOceldhpPhVrC3e41mMWkBA3BZGWSUsOR1DBR+v05r0KdLRMcqtkc/bamNIjWxtlmSCUnMsICuD6qzEkkd8rjsOma0P7WdZ49M1icTIqlw32hCxQ9CyNGdwPvyM1pS6Hf3iS6hYyqlo67VGI0TYv91eWIXuWbnrXOP4VttUVPmAgiG9jA2ckdxJwD14AyPYV1xgk9TGUkzvdP1Tw00iaV9vktcHPBUoB2wRkD2+7TtX8C2OrRC7s75Z8qSwOwMew3K0bkfUH8689/4Rm5uLOTTVaG/hGfLS5BhuIixzkSxg8Z/vDHeoYtI8W+Hbdbq8nlEFuxAZVacbCuWVigb046D8a3gu39fqZtdmZV14V1HTrloEI6FsR4lcD1OX3gfVMVjtaGzVpZJBKD0KqU/MN/StDVdZ8RWNmTfPb6jazcxMr7Jx7eW4Xkd8AE9a4aXV/td0oVTCwAypGD+NOdzopyfU6B4JWAdzkFQ351SSMySIHIALZ/AV1+nXUVpYxSSxrKOQc44H41mazZzG8NxbHdDtGwk889vY15dSerSOhHpXwaistb+LemQt8lpZW8i84wXZTnuB37mv1Y0WC3tYEhtSSrYIznIUDGTnnntmvzA/Zq0wf8JHqWsMq5tIQi7hn5pGx+PAxj3r9MvDVwJrfibzmJ+dlA5I498KOg/HHFfRZLFKnfqfK53L97yrojrKKKK908MKKKKACiiigAooooAKKKKAP//V/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAK8b+Nmiw6x4Ou0lG4hDt9jXslch42s3vdBuIkA+4SfpiuXGw5qE15HRhJ8taMvM/M3w5pVtBE1/rJZvJ4ZQpy/PyhfX8P5V0Fxr8d5GYEjZWX/V2sJB5HTzJCcE/wCyOB7VX8R2V7bWT2gDD5yAFbaCDy2cdP8ACvNrtDaQhbEhHJ+Z1Hf+6uece/U+1fBRqSi7H6AqaqK53hn1eaF4YpILEZBMSSh5WbOPmbDhRnk4Vjx8oHWrNtbLczeUZJ9WnQ/6qFGggAzjAbLSN/tO5rzBIbnUn+wm7MbjmZtwwB3XAIOAPTqeOgr6X+Hfhq2to47u2mVkyMtyXkx03N3Yf/Wx3r1MOubY5MTJU1dnR6J4T/0NDqlqLfI/1ceWVc9QW6tn1ySadfeH7axZltgie0WM7fqRn8K9MWa3iGRhAOuRgj3OOv5VzeszQGNWjQHPQriuzFKEaZ5FKrOU9TzEwyjdDJ8yqT+HuPp3FQG3eQHI6de45HWti6LS/v0BHPJPb61T89YSUbAyOCOf84r5mtFN3PZpt2OM1aJvJZh1Ukfh2NP8PKx8ySRQCxA9+oyao6vdCa58tMk87ucdOePwrpdKgNnaAznceCG7HP8ASuOK1OmWkR968ZkMZ+Xym2luvy4J/nTlnFtKNykEl8egz09ua57ULlpdQFuPus4BI5OScHPoKtIbi8iljAOYuFA9evHf1qkm3oTolqb32oTTWzZw0ikEf7QIPP1qtfI9vGFQFsKWH45IGPxpLVo474goQSVb6ZGD/wCPU/W3jVGlZsED5cHp3FW/MjrZHmt9ema4EUvEhJAHbJ6fqDXY6Y2YUAOVb5go65HX8K4XUvKub1rqFtz5JwO3A4H410miXJa3Dn5cJkL3z6VLWxq9jrr25RN6hs8bfXjI/WuUhs/tEjXtwB9nViFT+8x/ngUs140hECnLs3bng5q2NrQwo+T5QBKg9c54+projHqzmbsUkjaO5a4tTvu7gqMjkxqOMIF9up4rrLTSpLmNWuGfc5O0OFI/Uc+5zgdBz0qWJSHdPMDGeXcrxwBgD8+n09q6SwuBLsuCvlr0T+8wHoTyBXfRmc1S/Qzp/DEUtu1xf3DyTKQI0hXa68cBWGCPyrgPEWi3VjCUsZ3jveHEarnzQOPnAzggf3fvHrnmvclmhwGuQHC9IxyXJ7HP/wBYV5d4xb+1LecpMXbOEgsEyPlPCNJuTdz15Ar0lytaHPGcr2Z4fp+s6nsktLpEuIELYN0vkFW4GEZXRcnPYqT6VvweKYbRZ7LxNo84gVUJmhUncT0AAJB7DGeMHr1rkNe8B+I9de2lvYpWkhZjKGCyO5GNigoNg6c88VDDpniLSk/sy9jmSNidgh3s1u548wN8oJx2J55JxxVtNf1/X5G6SZJr7eF/EF3KtlfzQ26H5zDM2BIv8DwzAj5Rx8revXNefNZ6le3git5/tMY+6NzDGPUOFwPpmqOv6b4ntbhIDPIVA2RRxOWURk/fZ/VvQ5JPWruk2N/aWbwvM5dyCzbiQB6c1MnZXOinE0JGnhBgJOQe/tXo3hm0ku4Htb0gIvzKeB06/rwM1wGkaPPPqii5dnjjBJQcZOeM8c8V7Vodt5LF3G6V+igcIteZiLJnQ37p1PwKsZbHUNetHwcGE7RzySf5Dgmv0T8HuY9MjRsDPAAAHA47dq+KPhpo8lq2pXcW7N1NGuR97CDn8MnrX274TsWtNPiLDkqOTnP0yf6D8TX0uUp8iZ8jm81Ks3/Wx1tFFFe0eQFFFFABRRRQAUUUUAFFFFAH/9b9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigArN1gE6ZcADOUNaVMlQSRtGejDFTNXi0OLs0z88vGVlc/aZUAQRyk7lPcH615RqOjyGDChgwPynHIx0zzz9a+r/H3hwW95NEqfeywJxjB9civD7/7TZrsVRDg/eYjGB1I718BVpuM2mffYTEKUE0ct4N+E19rFyl/ffJDncMFhyPY4yP5V9P2OmQaPbpAIlBUDnuQOnP/ANavC9E8SalARFFfROwzgyPhf/QT/jXfQa9qzw7tQVSOzRnK/h616FCtCENFqceMjVqT956HT3WpurMqsR7Hkfga527upZPnBJ6npVmZopFE44DDr15rEu3Cjfnp+ePauLFVm9bhQppFGa5kQtkfK/A56Vi38rpGVjGCuCe9SzSmRiScZ/DNZ91cMVBJ+YHaccH/ACa8mc+Y9GMbHEXErJfgOdpkJBPp2Ga9EtHd9PRZOrDcc84H/wCuvOdTtZ7md2QYCr25OPX8D60nh7xLPcf8S28wOMehOKmETSeqOg0h5H8RtDL8+1xJgf3CwBJ/OvSYdMe1vLdI03ASukh9QjHj8M5rndDhgi8Sw3T8R3KyWjccBiMrz/wEYr3HStOWZPLbq+JQf9vaAx/EivZwGEVSNzzcXieRnid3pstlrkk3mfuWYoR/dPpz+B/GuU8U3TNB6ZHr6V6F46dbXU5geEnCHPTDL0P9Pwrz3UtNudQs/LY4ffvHsrZx+o/WvNxELVWkdlGV4qTOTsrYzwQmOTa27nHOQxH8q6jyYEt3uIyBGgCrjj5geSf89ax4YJIE2YP7pOe3OT/LpUP2nMUaxL8jMWAHPAJz+FZJO5pJ3NolVVJARuZCp9RkY/PrWjaQbcM5wEQkY7EjjPuaxrhJ2hDRrgF+P93j+dbTyC3eEEgAHzGHqccCuqN2kYSLVvIZBlvkXCrgjOMgA/jitV7hkCSxnduAC5+uOf8ACsCBWnMSFuZCWznso5PsOa0fMY+SsRG4jIHXqeP0NbRRjIuyTkxuZwWydiqM5dj16f5x6VesoNPhVVvz9ocEnltsYPZQq4zgfhWXEhXYCcKg69yT7e9UrmS6t5d6IT2T6nk9a6YTcdjJxvoegPf26lS3k2qAYCooU4zxjqf0Fc1repaS0DxmYIeeMAtyepDAj865yLzZiFQh5DycEkD1J/8Ar8V5/wCInmgcwxzF5Dn5t2FXngLxyffpW3tW9yYUlc89+IPmxyhhcquTu2vguw6A4AJHtnGfSuU8PsLkEzyZwdp962da0eW4AjjnSVi/ziMlsE+pbGD6muet7SfRrloG+bzeVx7d/pTc7xstzvp6aM9U064toPkiABAH1J7Zr0PSrOTyxc3TMo4bkYGR+PWvNvCemSzyCeToSG3YBP4civbfszqYY1UmMkZyOw7V5zjedhVqtlZHtXw40hp7aETzmNHbeRk7j+A/rX13aQQwQokK7VAGM8nH1r5y8C6c8t8hi2QqiAAsM7R32jp+J+g5NfScQ2xqASeOp619tgKajTsj4rFzcptj6KKK7zkCiiigAooooAKKKKACiiigD//X/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAPKfiDp0NzEZM4dOlfKHivTmZXEIChucjk5/Gvr7xrcIITgD5e9eBXNtBcS/v1LDnv/AEr5HMYp1pJH0mXTcaabPn/RPCV3dXzNcR7UXHTaCQfUY/8Ar16la6YumL5MUxkjxnnqB6HqDXXeVEkey1QKMcN7/WsKfadyP949RXmzXKen7RzepWkdVUiPkeg6evSs2SP7WV2dR07H/wCvVi4gkHMeRnv9arWl3HFNtvwSgP8AnHFcsnzOzNYqyujEu7G6hzuXjpz6ensRXPXSSRgrjGBnvyM17W9lp2p2vmwsA6jB75x2OK881pIU3RNwQMH29/pSxGF5FzJ6F0K/O7MoaXYx3Nv5pXJIK9DjB9a8k1G2GneLolRDsZwv4k//AFq+idDt/L0/KL0z+R615zrOird649xHzLBJEyDvtLAZ/AmiNPReY4z95o6uKwk/cQIuZYphIo9SQT+lfTFjZbY45gu1imcemR/SuTsvDcb6nBdYG2PqMZHzL/8AXr1CO2VIxH/dGPwr6vLMG6cW5HzuPxSnZI+cfFWmLPrUcc3zKcqdwyOexrnLnSzbIsMg5RcZHoOnP1Fe3a9pGZvOKkrnt2/ya4PxDCFjYxjDFe/9K8fGYVwlKTPSw+J5oxijwnUdkczxIu3JOfoxzUWm6W0ske/O1R2HQY5znvz+vtVi5KRXCMVIIyST04zzXe+HrMXarwAOO2R+NeZShzTsejUfLC5CmmgwRxui713O2TkjP3R+Vc7PDFLcOWj3IoHDcLx1JPU5xx7V3OtXttaZsrI7mA5GOWPcn1/lXCCK8lmlMrhjkBUXJA+uOv0reslB2iY0m5K7IFZpZGf5VVvvED+HsBWnE6uJGjXHXpycYx1p5sblgzyFE42/Pldv0Azz60FZLYeXbOX3/edRjP8Au9h7miMhyQIRDtHKhRk45JPHAqtLatesvnDG4846Adl+vrWgkiWq8KC/dmO7HsoFTRN55DsvA5Abj8hWikZNHPXljLCBb2rAxkcgZ+b6nso9PxrDuvCd5qakxys+7KkopX6jcckD3HJ9q9CnEPE0cSs7nuSefZafG91G224+RjjIPX8AKnnaY13PKn+HtpC6iNN+zqAensM+vfvW7aeEoY49pgQ5xuyM/wAxXp0ckbMFEeAPoD9eelWGtzjMakg9iarfW4nVZwlhoFnAxMW1JFz2B5+hrbjsZVnUyNnBHTp+lSzsUzwAyk85B/wqG3vvOuY7flixA4/+tSh8SIm21c+o/h5DboTLkyucHaoyAAOCT0Htk17WpyAa8t8D2XlqsTsWWJRhc8A9yQPy5r1OvusOrQR8hWd5MKKKK3MgooooAKKKKACiiigAooooA//Q/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKa5+U/0p1Nf7pHrQwPIvGkB+zs0ORg5OTzXkJaJE8x8jPYn9a9h8YoUh/dsx7c9PevEJpI8iEpsKnORXx2NdqzPpMGr00X5HMdqHDAHtz1z71zN0yZzL+Z4/Wr107Ngswb0A5rIlkAmUuCM9Aozn8zXmVp3dj0aUeo6GESMZfMDBeuWwB75I/lSXdudm5tkg7YGT+dOutesNLtztT7RNgEKCAB6lm+6oFeL+Ivj3oNlcyQxtDdzxn5vKVpEX6udqfln2pqMbcq1fkVzS3PdbKRIbc7pSoboCp25/L+tec+I5gbxRy3bjkfpXzndftPRxT7LiJSvIzCsikj3JO38hiuj0n4xeDvFUyQtcGGRgB8wxg1rXw9XkXuu3oKjUipN3PrLw9AF0yNV+YngZ9DU+l+E2u9Zhu3Xd5TgPnuByPzPFc94U1QwwLaTkbVI2kdCuOCD+Ne+aBGrW4mGCWPUdwK78Dho1Gr9DixWIlTu11Ny1tFjxgY4H51rEYFMjUdae5ABHoK+nSsj55u7MXUY1eMqehrxbxLhQcjgZB/CvXdRuQiNXh3izUFjZmBPPbqTntXh5tZxPWy6/MfP3iS/EFywzhcE57ACu48K6ysOmhwTGAPmY8du3px+NeY6+yWxn1G5OYUIK7j0HXHPB54r5v8AEnxR1R7trDSpCQn8QJ2qT1wP6mvCo4ac5e4e7Uqx5bM+1J9b0yS5Ms1/HFnKhWcA7h1ODk/nV7StX0GZALa7EjgckFd2fYEg498V+b0fjrxa12sFjI00ruMAKDlugA/wrpP+Fn+KtLums9bsofPhO1w0YRlI90wf1rteWVl7yVzk+uU/hufopE+mToxEm/Pdcbvw3Lj9TVG9u9Hj2xrJI5XtINwyP9wgV8Y2Xxe/drJfQSxsoBDwyFW+pz8rfQj8a9c8N/EfUvFkQV4vMQHIm3YbA6cco30IrCVGaTvE1hJN6M9be/lllyJMjsuCn6d66K0UvGC7Mm7nIAHH868vj1u1j+a4wpHLYUJU6+OLSBglpl29Sefzx1rzlLleh0yg2j2W1gtLL98AzOc43N/QVJK9tg7l2N3APLfXH8s15Gnji7aQbtoJHAyCR7nFbEHiKW9GHcIWHRVOfxOMfzrdVU9DB05J3OnS/EUhiBGOe3f8P8avxXSsDucrnuAST9BjAriWX5PO3YXH/LPPftzjn8atW+pRsnlCXjcQATnIHHP/AOupUmnqOUU0dVcp5illJB7bufzrK0G1kGsxyEkhTxsGD+tRtLbRqv2gk7uyOo/qa6Lw5brJqUY6AHIBJJA/CunDrmqRsc9V2gz6t8EbxbKGQru+bGPvH3PfH4CvRq4XwlZxwQiT5tzgdeM/hgH88/hXdV91S+FHyU9wooorQgKKKKACiiigAooooAKKKKAP/9H9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAqKVgqnNS1RvCQhpPYaPMvGE37rhTg8DsK+etUlMMjuCcLzkAnmvoDxax8n5evc9/oK8F1kGMSSK21hkgDknNfH5qrVHY+ky5+7Y8r1DXL2SVxZZ3DgK2efyBx+NQofFd+BGJYokyAxDHdj64/pUltaveXTLGzAk8vnHPsAea9d0pEjVWKscDBbA/XjivGoUFOWp7NSqoLRHlvijwrP8A8I0yzMzs+Nw5A54/zmvhPVrA2Wi3c7KN6yuv0IcqfyAGK/VbXtIj1TQrlLXhvLJXB6lecHqa/PnxxoYF5caPGnli93Tw9MOxP7xf94da9dxVBW6bnHCbrRfc+W9Y1PRbm10+LSjOZfs4a9aZEQC5LNlYdrNmMJtwzYYtngACqV3rdrNo9jpsFjDBc2sk0kl4hfz5vM27EfLbQsYX5QoBySSTVbWdIm0i+e2uFIwTjjH4VVt4TcSLBCuWbgDHNfUxrRac4bNfgeH7Np2e6P0G/Zh8ay+K9Pk8Ma45e7sFEsMjHl4iQMH1Kn9K++/CrNDD9mfPydPoTxX5P/BWZvCnxJ8MxEjN27wSgH+CRD1/ECv1k0oYKOo68V5uD5XNyhsb4xNQXMdyjAZx6VUuptqE5qVG+WsTVpmSAgd816knZHlxV2cPr2pPGvB5NeR6tDLqFyRuwqjn26/4V3mqt506555rH1S0WGylMY2syHn265z9ea8WvT57tnsUHyWSPir47619hthptp8rPmRgP7qD+XpXzHptqtrpM2oOvmsE3nPGSeTzXs3xavDqniy4UEyIuIx6Y6kD615RpEJkt59LkP7xASAehTtiudR5KN13VzumnzW8jzSHUJYrmO8G1pI3DgMoK/Kc4Kngj1HcVY1vXNR8Rarc61qjK1xdSNK+xFiQFyWIVEAVVGeFUAAcCnanpU+lXBEiHYeVbHY1TgilvJlgt1MjscADmvejVi4Xi9NzxnT967Wp6N4UtZtSsH/eiIFWUk8ZXByCfQ19ifBbwzBF4Tt5TFucRiU7x0LuSMjPPH5V8++EfD0sKw6JEP8ASZ1BkI5Ecf8AExHqei+pPtX6N+APCsei+G1S5hbdJ8zKMfKB90fUAV4Ln7VyUdm/wPUS9nFSZwkvhC3u5WkMeR1BGCgz6Z//AF1xureDdPthuhiecrnCkhFH5DNen63fxqzERkKp6ucDr0HTNcjJrGZvsu1W3Y3KcHj0AGa8ao4KVkd0JTtc8ourMRNssYVjY5zsO4D23HFZUV5fWcxWQtuHT5gT19icV7zfaakluWFkhBGdxHA9MDOa8p1p49PhkkW02c5O05J/76/pSdEuNZPQ6uwvWmjRpR5hA53nJH41biMD5SEqSpyVPHXtgf415dpHia3mfyYIPJz1YMOD7g8A/Su9h1iNgInnWTH96Rdw/LNZvTRiaOhY3NvGXhQomMcbf55zXY+CLmd9Sg4KZ7g5z+Irzldc0+NgId0znjBZiP0BFen+CtUN1qEZiVYsdTg9fxArqwlnVic2IuqbPsTwpBdiDzHhMUbfxEjc34dh9ea7ivPvCchlAWSYOV/hXJ/E+leg191T+E+QnuFFFFaEhRRRQAUUUUAFFFFABRRRQB//0v1SooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACoZoxIuKmooBHnviawtorKWaUc9vrXzn4g3xK46s2Tg4HPv3r601ezjuIw0kfmFfuj0z/WvnbxhpgSZlZPlYfd/wAa+Zzmm17yPcy2or2Z4DpVxGdUeyaWISMdwVVzge7cAfmc1ozXN5pF6YjIyK2MED5SPqKzpNIgsdZW4s1CEn5mAzj+f6YrotTto71BJICCvdh/9fvXz0oNw03R7vMlLyZ0ejeKnKm01FswNwGZgMH8eo9a8e+JHgCHURJdQJ9ps5X3hocmSGTu8Z7/AExzW1dWk0CEwKsnfHXP4VvaV4guLVRbztuTHzRvjb9ARjFaUsW2lGp06h7LlblDr0PirxF4VaVjDqFmNRU8CaEgOf8AejcrtP0JFYeneHLTThjStCnaUnBeYxxqD6El+BX6Fz2fgjVjuvbUI0hxnaCD+IGKuwfDrwJMyz2llE7j5i5Ab68dM1301zLlg9Oyb/I55ezT5pJ/cfDHgnwlqw8Z6d4jvmR/s0yuwj+4o6ABj1xnsOa/UXQZhNYxyDuq9K+XfFOmm21QfZbdUSE4wvAAH06/hX1R4aiEGj2qydREufrjn8q9LK6rnUd1a2hzZrCMaUXHqdQvMYx1xWBrOfL2ryK0ROVYqvPFZl5i4jIfo1e1PVHiQVnc8uuIZftrOQSAQM/WszxbdTQaTK6DjB5HOPpXX6jGEO7bjHX1rhtdt5L3TJIZG+RyowBggE/0/WvMrxtFpHpUmnOLZ8P+IPDM+satcy7uGcbRz1x2NcrdfDrUopvtjC4t3QfJMib1Ge5wMlfUYr6DsNLkXxHdaXKdht5sgHjj6fka+j9HtojpyNfxgov3tuDn9OteZSqTb5EexiHBK7Vz8210XVArWGo21vqHP31doz+TIDW5oXw91a6n8vRtOW2aT+PmVh7rwoB9zn6V+gV1Po1smYbVbl1ySCgDAE8HGMn69qoPr1vbWYWygigYnnK7frjI5NYVJ04XX+f5bGMUpaqL/A8v8B/Dix8JL9t1cqkmd7b23M7gdXbufQDAHau68S/EG28kWOlkxbcgnzC2c9RtX5fzNc/qcySsJ76XMg6AuCPqQv8AKsBIbSJPPt4DcEgYbb8uT2BbmvOq42esYM6VQi2pTIb7Ubq5jEkvzSDuw5x7AZ/MmptEtY7MC4chmc5Cqfm/EVQFvd3VyxkOxF6jAOOenr+dWr+5i06MzK6wxqPvTBgTn0xWNODfvMc5fZR15mSYGWdTx93c24fp0/OuQ16zt5o2a8yARxiQjj22jj8TVDS/EUd4+UkQM2MZUnOO4yMYrpp4ru8GQx5HHOAffAwRXbGSasc3K4s8UvdDWc+dZsIgTnEyqc+nzAg8/SrVho2oqMm3THTOQBj1BzXpF3pdykS+c25vYlwB6kHmmSzW9vAkUywXB/uSRl+fbjioku5upvoZljoMi7ZLqTYAMBQRn9OtezeCrZ7e5TycjaO/615rpkFu0vmrbR2xY56dvrgY/HNe3eG1MKggbzj1H9K3wME6qZy4ubUGj27wvrrC+WzVERRj7uMlvfuTXs6klQTXiHhKWOG7yyrvbAyw5H+78pGT6mvbYyCgI6e9faUG2tT5SokmPooorczCiiigAooooAKKKKACiiigD//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAI5V3IR3Irxjxnp0bltmWYjOa9rIzXMaxpC3VvI7tl3HX0HtXn5jh3Vp2R14SryTufEes2Est2FDnZnpjjjtUes6jp1hCiXVwVAHOR0Hpn/AArvPF2kz2F08XK9SCK8E8RW8kiMJGCDoDyTkemOc/TH1r4uUnFOJ9ZTSnaR0X9oW0A3xuXB5UxnqPfioIb61uX25ckHJG0nv2JAFcTp93qj2eLqFo4Y/uyNjc/bkc/1+tdboiRXimSAYQfebG0k+xwM1xuMuax06JXOu0/THusRwOAp6BnzjPsB1H1r2Xw94afSrUyR8u464JH4lu35D2rM8F6fYReXIRMzOMBucYPuGIr2eCzjZf3mTgY5Oa+qyzL0o+0lueDj8a78i2PLJ/Dh1K4V7lck8qMdh3P1Pau8hiSIR2yfKFHStmeFI4zjgjpXOfaAt2hA7f5FezSoRpXa6nnTrSq2T6G5thjQ4A9v6VkXTxmPeuOmc/Wpb2VlXCYySOPY1iXNztUkEHcfm7Be38q0lImEOpyWrSckAkfz/nXN+V9ttDEVww5/Kn6/eiPLyHksOO+O2fQVT0557q4SS3kGwfe759sDv9K85zvOx6Cg1C50vh/wJp9tC018guJ3eRw7gbgJCCV3d8dB3xW7NoqxM4t1+VhjB4+tb1ioSNS2V6fnWuio3TJ+tdP1eFrJHE8RO+rPD77w9eQXHmQLuLD+LnPGM4PPHf1rzzU4VhkxeReU6EjLIXUjPHfAz9BX1NeRRsCEC5HUEc15D4nlTzNlzbouPlEgYbwPUgbTj868LHZfGKcos9XCY1ydmjwm/klhQzRQMwOdoSNdpwepPWuZfxDdTstsYwijqCPmXHoWIH4ium1qaztGB84sjEgHC5/DKgiuTuLVp22wRGUdc7tpGeeg6/nXz86bjLU9hSTRv/bJbaEH7SuOuFAc59toGK8+11b7VJTPCAXTplXf8SckA/pXW2WmXBA89iR/dXgjPsf/AK9dFaaQQplSPJHOR8vH4E/pXRTTMHJJniNvdahYF47mEKc8tna36Cu60vWdRt1T908i5xw+7j24/pXeTabHIux7VAPVuSPwbIrKXwuZXzEvkr0ygUNj/eHP60Sg90UqkXuE1ydQVAeQTn/aX8QMit/TdJgVFe7m3dcCT5m/PH9auaT4TtbZ/PJdjnJJxn8a6wW6uBHERj0OK1hSk9zGdVLSJmWemws4+zKeec7q9B0WEwyKjgD6c1iWsFvEvzIQ3+fStvTnKSBuQB2r1MJTUZJnnYio5Kx32m3z2eoKs0m1X4AGBn8e34V7/p063NokqABSOMcivmWZBI8cx6gjI45r3nwzqUl1ZxiX5R0HGBx2HTOPYYHrX0OHlq0zxa0TrqKKK7DmCiiigAooooAKKKKACiiigD//1P1SooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACijpyaj81T9z5vpzQBJUcqb1Ipu6Y/dUL9T/Qf40hSYj5pAPoP8c0mNHkfjnw1LcQNdRgNIK+atR8OW89yJZ0JYnlR/WvuG6to54yjSySZ6gf/AFhXinijQGs7g3UUDbepJC8fzr5nMcvjGftY7dT3MDjXy+zZ4M+jozYkzsThU7n8O31NWrewRZFaCIjtjOE/kK71zZcyrD5hPoq8/iQKz3vNJhP/ACD8sDnIfbz9Oc/lXN9Xho7nV7eW1juvCumrHbiWYgluuzAH6V6IsqRKB0FefeFrp75SII5ERf7xXA/EAGu5XMQEcq7s+2a+jwyXIuU8bENubuWpgWTPSuLnZINRVXYbSCfxNdSZxKWReNvWvDPiJ4hufD2rRRDG25GYy3+z1FViKihHmZpgqDq1PZxPQr3UIWYDdgk8DPXFYl5eQxQku4AH3vXjjrXk58cGQZlGGTpj1PWuW1nxhcyxeXEcJzwOp/8ArV588bHc92nk9XRMteJ9Ua71FDbEKm4/e4HzDGTz/OvUvCdnYQRBrYqWflmBzyB1z/gK+LvEHxEnSX7LprxXN2x4UfPtwOWOz09K+qfgzb3Nv4UtW1An7RIN77jk5IyefXP0x0rDDTbnfuLMKPs6fL2PdI2JQZ4HWq8+rR28ywMj/OeMLkfy4qit+8kzQwI+U6tt4P0J61JFNrRkYzIqx/wkcnHvnvXq8x4HL3NJ7pGXjPPrXKaxBbX8DxnI4wGTBI9xnNWjpl3fD/Trggbs7UAXIznHU1g63Y6nEc6dcYRR9w/Ln8a5MQ3yu6NqSSejPNNW0vVbJ3CXTXELdFZeMehU8Vy7WkjPtNnsHfAwPy5FdPc3lyNxuZYgwJ48xzz68cViGYufkeDPfnP8xXzdRRvorHrxlKwyKwES5ZGAPOMFR+gpzTKM+VG4PuDUkMNzMR+8gAGeyd/wrZt9NuOu1MjnKhMH8qhR7Dcu5nW2JF/eRjjqc8/rWvHDC46KfXjB/MVOsMqDcVwfoMfhUbPMGG5VQ9N23GfrxWsVYhyuNKjA8uQg+mOv5VIkQdMrnI74qSOIOf3hGT3FX1gfdlX/AMa2jEzlIzhJIMCVmA+hre02SEMOSfqaqbvL+WQBj7Zq3Es24OB8p9RXVSVncxm7o7Zo1nth0yBXongeDAy6KCeN2WLN7AHov0A+teb6ZcsqeXMCVP6V3Hh/XrfSrr7LP9x/utnHJ7EngfXrXs0mrqTPNqReqR7TRVe2m86MOOnr0z9ParFeicIUUUUAFFFFABRRRQAUUUUAf//V/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigBCQKQ7j04p2AKKAGeWv8AF831pdwHAGfpS7c9aWgYzDt32/qaaIh1PJ9+alqBpiWMcI3MOueg+v8AhSduoIkJVFyxAArC1a1+327xJGcMOrfKP6k/lW0kIB3uS7ep7fQdqSSUHcqY+Xqx6ConHmVpFRlZ3R88y6Z/Z9xJB5atzxkkj8uKoTWtwp3RmKL0wij9SP8AGvTNd0txIblUwGPU8Mf8B7VxU8e0kdTXluioLlPSVXm1JdCe+iRlMoK+u3H+Gfyrak1S7hUgRbj6+v4Vx7PcQsohJDdgP5nsBV6LUruMbCPNbHJHU1rTq2XKROF3c3v7dBYpNCyYGc//AKq8k+Jnhm08Z6dhXMdxFny2BIK56ke/pXXXPibYVEsDBDxwOCa53VNesJi1ugdGQZYnj/PH4Ac1OIqRlBps1wynCanFHxmkuteGrWbStWleaS2P7uST7zJnGCe+PWuR1zX5NX0maxWUxmQbXKnBI7j8a978bJo2twrFJukaQlUYAjb7k8cCvmiLTLDS/FUcV7KZLZmJG7uU5BNeGopyufZUcdenaS1PUfh18ONNtbJLqeIok+Nx7EA5x6g5HXNfWelXrxR+d5LCNR0UAuw/+vXgtprUjuHtYWKxxqVGVCtnnp0H6V3Np4yvliRI4DuYgbEC4Hbk5/lWlOryy5pHi4pSqHqsfiDxDdxILHTtqscFpHwVA9VI5NWZW8Ru4InWNOpHUflXmv8AwlHiKdlaxljhA+8r8tj2we1XHvtZuSv22+2oRjKgEc9+1dX1uLR57oNPZHoBXVJjie9TylGdqjaSfqSax7m80u0Yu11uc/7W7n0xxXFTwWkWTd3TTKnUhiSPwrOub2NxsSITQcfP/EPr3/Aiuari/IuNE0bu4srqfM8Ko3Y7SufzK/oTWfPZ2G7OGjPX723j/gagf+PUkM9vGuxHMat64Cn+aH8QKtq6oAjDaGPBX5Mn6cofwxXA5c250JWIY7SWPDRSfIe7pkf99LuFaEcN1gFQHHrG2f5H+lLDZlPmhOc/3fkcfh0P4GtJGdMNJhwOpxhh9e4qoxFKRVW08wgSnBPcirrWpiTdksvcZ/l2NTfasx/u2ORyVb5hVJrlFb5Djn8PyrZKKM7tiHaBmL8j2/GhAr8kEZPQ9KWUoy+bGwYfTkexqh9p2/Iw3c8EcY/DpTvYRr+fEo8tMbvftWlZOXx5nJrnhH5uGKhsfhVqK4aNsFCuPxropy1uZTid5BIQAQoWr88giEd5tG5SDnk59q5OzvpZBtAb8Oa7CyMVxAYXZskcgivTpy5lY45qz1PbPCurrq2nrIiFNvB9Pw/wrqK8I8H6kdH1c2LysI5zgf3dw6ce9e6qwdQw716NCfNHXc4asLS0HUUUVsZBRRRQAUUUUAFFFFAH/9b9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAoopCM8HpQBGwaQ7VO1e57n6U9VVFCqMAU6msSTtH40hkcjFsgHao6n+gqP5VUPJ8qj7q/wD1vWkZgSoQbsfdUd/c+1DqsSm4mOWHf69lHvUsoo34M0ewqSW6IPvH6nsK8d1YixuDEcb888cD6V7OEMcL3F1+7BGW57emf85rw7xfqUElypX5MHoPT0rixklGPMzqwsXKXKilLOoTbGMs361kTXT2wKxncT99vX2HsP1NSefuGd2MelUZW5yOQK8+dRvY7lCxmaneX7jELhdvLHGT/uj+tecarquqQOZHUMmCPlHJrvrmRzgY9cdhz3rFmjjdSjLkH2rgrTk3udVKy6HgOsz6zdwzSRQiJpF2lu+D2UV51c+CdQeaK5vXDXEozjrjjpn1GOfSvqqXSo5c7F5Jqj/Z0MDtLMu9lGBx0H90VnGdjrjVPnHSdG1y3uJEs5CXBCqh+6wxlRz6811VrJrke6aIjkD5WH+fpXoj6couwuBkLuz/ALdTNZJNllGOc8+//wBfrUTqXLc0zjYZdaxHM6gB+GHQ5z1/KurspblY1jP3R69j6/SrkcDQoUl+7nH0I6VnXOqRwqQq4J4weP1FYuXYzbNln8va4b5W6Z52nuvuPT2qi17bl2MJ8uSPqB/Me1csNVe7kZW/1UgGO+G7H8DTLUXPnb3Pzp07/h/ntUO5J18U8c5ZWG3d1I6fXH+Fa1t5tuQmeG5x1U4rmoy6tvg+63+SK1ILiYy7GGOMAduKcUJs66O4idv+eZHUdVIPXg1YaaSEgSZZBx6jHseufxrmnRjgqSCR19D7+1XIZpyOchuOR7eorVSZm0XC+9iYGzjt/hUWS3z55HFJEke7cp2P1xVpBC52PhvQjgj/ABpqIFZXYNhBtJ64rShthLgnj3Ax+YpVsWjA3nK9iCCKvoiqo2sHz+YreEO5nKXYlhthF941rxW3ngDBOe9QWtszYOc/pW3BDsIHeu+lA5akhINOSP7oPHatKImDDBSv41LGCRgnP1qfZhcHIFd0YpbHNJ33I5EimdLnJDKeq9a928P6gb2wjMhJbHUjGR9eQf8AOQK8LQRjMY6n1PFeg+C9QVZWsmXY3UAHr749fXH4iuik7S9TCqrxPUaKB0orsOUKKKKACiiigAooooA//9f9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooADVWZ2yIIxl25PoB6mppH2DIGSeAPU01EESlnPzNyx/z2pPsNDo4xGOuSep9apGWMyNczsFjiyFz0z3P9B+Nc34k8WW+jQMF3PIeBhTx6nPtXzP4s+J+r3ebG1l8kvgKAv3B+fWvPxeYUaCtI7cNgqlbVHuHirxnaws0TS7Y4xnaCBk9s/4V8+3WsNrGoNdE/JuwB1yfXA615Fd6xf3twz+Y+1iAoJC+wzk9+pP1rq9Gv4XQOju0ScEqMZ+nTOf0/Svl8Vmcq8lfRH0GHwCox8z1uwjijjEW1pJX+bLdAB3OeM/n9KvtFztUdB26fmawdF+23mREggSQ7nLdWC8AZPQevb0Ga63y1jwu7e3oBgf48V6mH96CaOKtpIwJLPqcVnS2y8jH5V1jx7ztH5VDJaYHA5q5UbmSmcXLAVXEYAz3NZ0tqSp3HiuzltEU5K7jWNdwuynoormnRsaxqHE3SxRnccZFQx27SKZApAPTtWs9mJZxHGC5PftXQLphWIBuPSsI0WzV1LHBzwHaQe45/wAa4XWIhGA5OdxIA+nWvXr2w2qSDxXnGuWbSTC1jYKz4LH+6o6/nWcoa2LjK5SstKJhQ4wT0+lbdvpJIG0YwME/TpXY6fou+OOVRhWAAz6f/XrSmsBbHbjKk801h3a7JdRbHGafppTNu68A5H49a05rEK6PGvCnk100lhmQTJxjg4rQSyQxgN1atY0OhDqnLpY+egHQ9anEAtyEmQD0btW7Havb53DoP61PJapdQYPKtVqiS6hmNYQ3CAgDNKNFglXLEq3qKnhtLmzwo+ZfQ1rxM7KGIrWNNPdESk+jMAaVJBxFcZHcEVehsn3AsM+4wK3YlDkbhiriWidfu1tGguhlKozPigRCOTWjGoKgKc49RUiRFM7SM1MmBw4x9OK6YRsZNlmGMkDB5q0Y5FXpuHoDg1HDk/dbcP1q6FLDDHB/KuiK0MJGU4YHK7gB2atTTpFjuY2mJUZGGB5U9iKhljfHzZYeoNQxMw4PGOmapB0PfrKfzoVJIY46qeKuVz3h68S5skyVLAAAqc8fjyPpXQ13Rd1c4mrMKKKKYgooooAKKKKAP//Q/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooqrcXlvaoXmcKB6nFJtLcaTexapGZUUs5wB1JrzHXviZpOkq3luGI9AWyfoBXh+vfGnVb9HisIQir13KT+YJrzMTm+Go6OV35HfQy2vV2VkfS2o+KNI01Gmup1UqOBkcD/GvD/FXxngBez04cgffJ4H0x1rwG/8Q6lq05uLoZ2jOMlAT24H51nwuCw/djnPIyR6/ebNfOYvP6tT3aWiPcw+TU4e9U1Oj1DXr3UZPt158yIM4PUntgH8K465u2kEr3DyFnYqNuOvU+/tV6a5txH5kqlt5zjPYdPzzmsLUb22gIjSInYdoA4AY9T6V406jk9WepCCWyM2Qrcb1hBfGVXryz9Tx7Aiuo0aWZFS3x0+6vRj7nsBXMKzGzVSAqszMwHUjA6nsK3vD7xQzM0Z3SMcZHRe2fr6VKKlsexaLqUwQ2cIIUY3lfvu3pu7D6c/SuutBc3WSHCx5424xj0XH6nmuDsLJUaMNlA3ylR1Oe3HPPf19a9HilihiSCABCgxjoEHfJHf/wDUK+iwMpNWm9jxsUkneK3NeC1VFAxz70+SFB93k+tEO5UAf5e54/n7+1TMwAGOvpXuRSseTK9zGnthjJ61iz2bSfKB/hXViAk7n6mq9wiouAMfzrOVJPcqM7HNW1gsR+Ufj61cls3K8DNbNtbNjzHH4VNLEzdeB6UKirA6mpwl7p/loWOXduFUdz7VwmtaUtnCsAG+5u3VCfQE817Q1sFDXMvHYZ7CuYi00XetRTSDhDkVzVMOrqxtTq23NK3sFht1UjGwAfSm3tsiWzDGWxxn1rqGhVR83T0rPmtvOJZunat5UrKyMVUu7mJBZgICef8AOaYkQW4eMjiM5B9mreWDbGo6Fev0FUJUAnbjnA/WodOyRalcrMmWw2BtP5ioo7dYgVHCnt71flRmUnutKIgycjg8fSly6hfQorHuO09quJDGORz9KcIjuJPBFTLEcF1OR3FUoktiLEpGTz9P60pG0ZGRj8aa4OcZwaZuZDg8eo7VexJZQ7scA4q6qJ3z+FU4wfoTV2OQ4wwx+taxfciRMiRjBP5irOExnPHrUajseM+lSHK+/wChrVGbK7oDnaA30ODVOMAOQDz6E1bkIIJBGfcVm/vDKMqM9ip/pSbGj0Dwrqht5vs8oAQnB3Njk9COMc9DzzXqgOa8a8PJMmoBWyC/r0b2I6H/ADg17BAgSIKo2jsPT2rqot21OaqknoS0UUVsZBRRRQAUUUUAf//R/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACmPJHEpeRgoHUmsfWdfsNEt2nu3AwOB618x+Nfirf6p5lpYr5MI7jPzV5mOzWjhlZ6y7HdhMBUrvTbuexeKfijoOjBreKYSSDrt5r5o8SfEnXtcmkj092WIepwfy61xrW97cM11cJ52exPr7mswxmGfYgRPZTz+OetfHYzNcRX0k7LyPqMLltGjtqy9CLy5YyXMjeZ6nJ+mBmnXFpjJkDFR0y/PT0q7DGJkxg78YzuwPxwavwrbRZV9uS3Uct/XFeeo6bna3ZnN+RJHEAm2IH5mySWPp+lEFsDG0mNxGR3Ix3+g9TW3L++y4jOGJ5f07Yz/hVKSM7WEzgonqcjI9ccfhSUNbj5uhRnd2VQo2+WvRMcn/erjbrbkzSssarkFiSx9Sff866m7lcIdiMcqAM/KB6/h+Nci0aSF2275cfKF5Vcep6D6CmwSHGI3Wz+GCMk+WcjHTBf1b0BNXtIuo0kIU7Qp4749/rnp7/SqcqPgvIwVd3zFccsewx1J/SqVr5oljjjb5Q2QvU8dzn/AD6VSYmj6S0ScR2P2ll/ePwOcnGOpP8An8a2NMvLqWQRpgKDkDHft16n0rgNAvWuLVUkwiqOSTknPcn0z1I+grprS6hhm+0EkAdz94A/3ey7vXsPevWw9bWOuh5tWlvoekfaGgAhyWYdhzg/zJ7f54uRkx7WuPvt91B/WuMgvpDulXCJEMsx+4mexPVm9h+PpXRWAPmYQEuBlyx59h7Dua96jX5tjyqtKy1N7qOeT3pIrXzH8yUfQU63AlYqhyBWmgUERiu+KvqcUtCMQoo6VHJEuNxFaOwd6gkTNaNGdzBuIjJy3QdB70y1sSkwbHOOa2vIy3ParEaAAn1qFBN3K57KyMtoNxIIzWfcMY8ZHFdDInU/lWPcW/mgpnjOamcdNBxfcz92XaMDoM59qaLPcd/8Qq8lu6Mrt34qcJj5ug71nyX3L5uxkSRkZGODUcce1ijDjofoRxWsyK/H51D5Q6N1X+lLlHzFFojkEdB0P9DUghIO5O/UVaVMNg9DzUwXAx6UKInIymjRhtXgjsf6VEsJU5XtWmYBISfTv7VKIVPXgjvRyXDmKMZJGCuQaspCD8yHj0zzT/JZCQg+o/qKN5X5iMr3x1qkrbibvsOVWA55H608uQNrDingq+Np696hfcowfzFWQU5WUg7QRj/PSswXLb9u5XA9cZFWrqcpk7d304P+FZTyQu4mkj2jPBweP6j+VZSlqaxR6P4c+d0aRsoCDkE8duR6V69GNqgV4v4entjMgDbS3Gc8H+ley2+fKXJzxXfRehx1tyeiiitjEKKKKACiiigD/9L9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAriPFPjjSvDsRRpFknP8ACG+79az/AB34zg0KxeC3IaZhj2H/ANevknVNTv8AV7kM6Ha3TjP4/WvnM2zr2T9jQ1fV9j2cuyz2v7ypsdF4m8TXviC4eZWwh6Dt/wDXrjxbys3zISAeDj/CplgW3JaWRiwxxn1/lTvtZjbaIjyPvHpXyE5OUuab1Pp6cVFcsEV57G4kQebnBzgMP554FUzYK75j4Zf8/Sqt/qEpl8rpgdeh/PkVY09S0XmeTuychiMk596zTTexrZpFlNMypZnMzr/tAD/CpY7GdJDIWWLGcbckir6Rkk7js9hjGPqxwKoXUyBfLRTI2T1fj9MA1o4pamd29ChlhON++bHO4kk59AOBUTshnUC3Z5emAMbR7iqa3Lgnc6x5PVMbhj/PrWvbIJyWZHbsC7bQfXnv+FELNFS0Od1Zd8YEo8sHnbgMf0P86wigUrDAC2QNzOcAH0CL1/H9a7DUbezjzJIFGOOrHHscn9BiuNuJZ7htkA8iJBgt32k9AB0z6Dk9zS2Y1sZjxSzXAi8whE3fe5b3OewH/wBarVrttr8IyiQt8scXJLMO745Cjr7/AErRKLbJvVQjYBO/kADpn1PsKyrGSRNQ807laRtpZx8/94sfQkc+1VEJHeWU7wzNHcAeYTlx1yMcL6D3rqbebzWMrD92gLqnZn/vH2WuMkASdZB845Az0Yn+fuTXRwXWYE89tmOScE7iOnHoOo7cVUJO5lJaHc6VK9xNClx8zDLIpwPm7yOOw/u/4129k32pdlkc2zH5nHWQ57e1eTvfbx/Z8LFJrsjz5By6oeFQH+8Rx+fpmvXdJdbYx21suNgCDP3UVcfmfU9M8V9Bl81LQ8jGQtqdZGv2ZBbxD5z+g9T71owxqg45J6msuGRTIVjJxnk9yR1P/wBatJJB/qxwF619DCx4syxj0pCnc0BxjI79KQuMhRWtzKxGUydtEhCY9qsDHWqsnzHPpSY0MkJxjvVPBaQ46VeAHSmiPBz61L1GUGySB+NI3y+4JyKshVZmI7GoJOAcfwmoKKO3DFenWnkg49e5/lTXDMSV+opURs4btUFDcgkL0owQ3XkfqKkeIHGOpJpwUN97jAp2AIwN3y8VIwXqOD6UnTgilLbxtxzVIlkTqGHHIH5g1ErqSQetSHI5H5VVkAchkJ3e3X8qljQ+RAvK8A1SkeSPljkepqXzWYfKfriovm+8mSO+P6j/AAqG+xaRVkYMpypx1yvI/KsiSIEF4JDGTnjPH5VsMARlBjPpx+orOvFmEZZDz9A2azki4mjpc6uO6uvccV79o00stojO24FQQSMH/wCvXzNpl4qzBWbBY7fYEfWvonwnPHPpMZQ/MvDD0NdeEncwxMbK509FFFdpxBRRRQAUUUUAf//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigArl/FHiW18O2LXEzLuI+UE9/pV3XddsNBspLu+lEYVSR3r4k8W+L7zxRqcmCWTPAJ7dif8K8POM0WHj7On8T/AAPUy3L3XlzS+FEniPxJLr109xLymTySAB9KwYb4sdiJkdyeuPxrO4+UzsPb0AHXA7D3rTgMKxh1xjj5cZz747/yr4fmbd5M+uUFFWS0NJo5powbceXGvBYABR+I4qg2mRiPzbiVkx1zgZ/rV5LiV8NyFQ98YH4dKytSvk25D7snqSBgfjVSUbXFHm2Mq5Fqv3jnHTnH8+taMDGYBidyjgbnIT8h/wDXrDQwSXSKxDhjwqLliP8APvXbWUFrDs3Y2ngLgM3P0JwKmmm2XN2QsEdsIt7RcjPJG0cHsCSf0rEvruMErtVQT/EefyP+FdzKhEe23iRFxwAOBn/PfmuZvbJlTaQuWPoP51rVg0rIypzu9TlLVfOuBIFRu67hwM988fov411SDYE2yKzN1252jHv3/OobXSbYSEySqMH5gMH9MY/PFbUMUJPmwBpPSV+B+A6fgOKdOOmo6kjltTtB5LlW+Zz8zHrjrxx0964wKHukhtznbk7VGW56sx7fj+Ar03VdPlnQuANgPLkkk8flXmV/JLF/olsdqEktjgE+5HJ/PjvWdRWlsXTd0VrphGCARLMCQBnhTkAe3A6CseeNpr6HyiW28swJwf8AZHY5PU/StG1s4mVsjOB8zDgENyQB2z3PJIqxIuJFRywOcKFBOF9SP0AJGT146i8hs6mzXcE+0Zc4HA6D/ZHrnuatCOe5lM0mdxbYBngEckfQDg/lU0I2ugUbEQBVycnheWPvzz6Us8saiOyQ/O2AAOe+efbPp1qktLGXU1fDiifUUnIyuSy+vPGT6E9B7fWvX4LlZHaK0O1Y8B5QMgHqFT1xnJ9WxXjenLHA0kMb7dp3O2eSPqOnp711theTW0W24OyMMSFHGWOcZ+g5Nelg63s9GcmJpc+p65pu5VMmMKq8kc4Hb6k960IrhFQb/lJ5I/z6VzVpfxiyRJX3STp5jgcEID8qgds55+uK2bdSz/apgFQkdeOnT8B1/D2r6alUTirHhVIWbubMUkjAyuNuecHsKsJwCx7Vh296t3KJfuxgblH+yP4z/T86sS38aYij+Z2BYD8M7j7V0KorXMJQd7GtvBJx3/pTJHVBj0qhbykRbmOWbGCe59celSRFXdpCc5OBVc1yHEshiy7iKfIdv4YpjSBRgdqqtKWUu3fHHtTuKw1jsVmH8RqIyDbgDkkflRPKqw575/Wos8YIxgVDZSRYhVc8c44qVo8AnHSooSFBPbBP5VYLDaSeOtUiWVyAOvSmOB1B5pk0yquCeCKqeeSwH8VS5IpJlsPkevtTCw/h6e/amFgenDVF5nzfz/8A1UmwsK7lTkDj+VQlwwJHOKSSZVqjLh8unbrt6j6iolItInlUsNykn19RVYybcEHPcEcYoVpWG+JskenWk3LMN5O1uhPSouUkKbnzFOQC3TPr9ayZpwARICh/T65qe5SWP5miyR3H/wBbism4u4dm2XKqffB/A1nOdty4x7Ebv5c4fuec9z9cdf1r3fwCUezMgIycjKk/kR0+hr5ze4jZtsE2/oQOjcfoa95+Gt2TC8Hl43c54z/n+VaYOd52JxMfcPWaKKK9c8sKKKKACiiigD//1P1SooooAKKKKACiiigAooooAKKKKACiiigAqC4nit4WlncRooyWJwAPXNPlljgjaWVgqKMknoBXyX8V/iDJeTS2FlKWt0yFUHhj0zgYz+NedmWYwwlPmereyOzBYOWInyrY5v4neOBq+rHTLKTz4g3XkL9fevN4woXbjbzz/tH3/wAKq6ZZTXI+07QsshPzOedvfA6DFakgit8kMGIwAwHA9ev86/PalSVSbqT6n2tKlGnFQj0GCJ9yMo+6OAe/19vatCB7VWMkwZwOw6k/lwPQVyj3xhk2QqZJJOdzDP8AOtC2v7mOEtNJub+6nQfjjr9BUQavoaSTOlaXz0xEghTHHGW/M9K5S83zTPFb89ieBgd8kZ4/WrzzXSxia+bYp+7CAN7E+pPT271mRaq8JVZrf5c8RoAQfXPBzj15rSWpES9p+m20lwH2+aSPuoSB6dckmvQ9Mso7cgGOG0D/AHfl8yQ/QAnmud0m5M/zSM1onoACcegxgA/rXbabKxO63ilKk/eLIg9TgjJP511Yanqn/X+ZhXm7W/r/ACNKa1WO3BkWWRSON+I1/pXFX6Pknaqgduo/Wu1lBmLeWgZmxjLF2AHuK5fUrEoSZsDnnHHP61ri46aGeHlrqYUUzg+QoQ4+YAYAHueBWxHIk8mLu4ds9ewx6ADt+nvVKKFUJWEcn/ZJx79q1rOxiE4e6GXYbiM5OB3bsK5qSkb1LFTVI0uItqgsgIwvb0/z3NcLqGmQq77tzseQTwAOnAHH9PavWpRCwAhiDLjA4647nP8A9asOezklLfZxg4JkfPIA9D/hVVqV3cmlVsrHlaW00CyNCBu/vMcbT7Z/njNVrKB5J5BayMzhvmKjJ49z+lbuqWy6a73DuDkdevHooH6mjTFk+zmRotkLnJA6yE92Pb2Fc67M3fcmNvLDbRPM/QYYZJGeu0HuP7x78Uy5doCGU4lGW47DtVq7nSNVjOJjGOeyj0VR9fzrN02P7Y7ySKNjOxc9SQvbd6MevsMCmmr2Ia0udHo1ukUQ8+TdJI4bHUe31J/zxmuxihju5P3TBtuQMnhc8lm9yBxXBpI0c+wOB/IZxk59cdvoPWvSNJsiLMQvkCQ5+bjK/wB5v89K7aGr5bHNW0XMRWrTxyrISSJGDFh1KqcKPZQefeukuNd88nI/chgqqT97bxk+2f0FU7wWywkwg4cdT7cAH+eKyEjDPukBYrztx0GOPpnrnsK7FUqU/cizm5Yz95o7aO/URGGNd807ZIPAA64+n9Kj04ZRmllLKxO+TvIerH2UdB7VgfvUiAwFONueg3HqPwHWte2XESoH4GAM8c+p/wAK7oVnJq/Q5J00k7HSi6Z9qRL878Aeg/8ArD+tX/OSJc8bYzx7k9K5kSrAi4yA2eT1IHU/jWfJc3F5KinKjeWx+GBmuv6zy77nN7C5082oqkIkByd2wf7TDqfoDViGYuxAPAA59BjNc0sMl1KigYjiLfQAdTWm7MEkVeDJgfRT1P5YrWFVvVmcqaWiJSzzSqDwuQfwq68iqxz0HP4AVSZuQg4IHP4/4Uk0qDnOdxx+FUpWJcbloXBS2dm6rj8z1p3nko4Y/d4/Mf41itdJG5VjlTgAevc1QnvjuMinK4Ofp7VPt0h+yOiYLNGI88/4io4gUYbhlen0rDg1RAEd2wcfnjity1uoZVAyCcU41Iy2FKDiWZBwADn0P0qpJMSwDdexp8kioD3U/wA/8ayppt5AUg81cpWJjEkeQsenTt/WjGRkcEfp+NSqoIy3U9D6fjUUjPEQ2M1HmV5AAW5Iww7jjNRyMXyG4cd+n59jUu+J1DJ39DVaZ2C7vvr0NDGZ1xNMg2n5R6jlf8/jWJPdW8wZZGBK9z8w/wCBd1+tatw3lbp4MOvcY6ewwePxrjtSuEu13wKFk5zkd/8AeGCP+BCuOtOx0U43K93LHDMqsuwnldp3D8j/AEr6H+FZZ4WZSNuM7SORnuPY18nbszCPJGD91s5H4HtX2D8MbM2+mJIVChlA4ORnr07GtstvKdyMdpCx6vRRRXvnihRRRQAUUUUAf//V/VKiiigAooooAKKKKACiiigAooooAKKCcVRv7pLW0knkfywo69efYetTKSim2NK7scD8RvEMdhpU2nwMDcOuSD0Vff3PavjO5DXV4XX5znJY9hXpnj3X4ry+Nra52gksWOWJ75rgIdyoEjPLHJ4r88zPFvE13J7I+yy7DKjS82CWowfMJ257jlj7+w9KwtVmVmJVuE4OBhR/ia3p51C+WDhh6HoP5Cse8SPYrL0XgADA/Adz715stdD0Y9zjUju7qbYVCKD0IOT9ea3XkMMaRI5WV+gXBx74AGTUsoeCNSx3buoX5QPfn+v5VRub2O32x26s8jDO8dh+X60r9EUacFooQSzRH/enfGD6n39qvqoK/Iy7W7r8xI/3R/U1kxzmFUZ4cZxt3AFvqAentnFbNlZXZ+afJMnO3IwB+FapENm7psd4oJVdg7Fhlj7egruNPg1FmWS8xEpIOXbIwPpXKW0H2cDdIqEH7qDcfxxwK63TnuQd22QDozuy5Gen3uB+Wa9DC6OzOKu7rQ3J1iMZd23Bhnp1+ijgfrXJ3lsWJkVfl9TwOa6tp7KAMrXDyyMOcHIz/vAD9K5TUWmkIMabQejHr+Ga6cXa1zHD3uY2145MGTYf9kY/KtbelhAsYGHc5yRzk9yT39MA+1Yq5SQsGBxwWHcn0PJrV0uCdpfNQEksSZGOMY64znGPWuCje9kdlTa7NJIQGSa/YgMCQhPp/sjOPqcmor6RXi8uBfkBzycfi39B19avratNJznYeWfucehPb/P0r3youLaBguTgnr+tdU4NRZzRkrnj2vPtm2YUk8sTnHX27ewp1le3KhI3YiTaZArcEJ03t/dX09eg9m+JGto5z5fDZPJG5sZ4IXpz6msjTFAmkeSJnJKuEI3NIw4VpOzbf4F+6OpzXlq3Mz0Ps3NW6lR0UIScg/OeOvUgfyqjbyyXDP8AZgyQWwwenXPAPr6n8q27izmlZ57puGB4zjGTyc+tUYrR5JVghP7odQBgAdTj0HqepqVfmHpY7mzs0ljinRR8pAAPGT1J98dq6eyE16v2gMy2+7AOfmfb1I9F4x9K422lmkKE/LEq4H8PA6k/Xkn0AxXplu8cemx7U4kwEUdSn07Z4HP6V7OFpqVzy8RNozmlVsmZPkJHlr+mT9T+lTyXHkKhiXdJI2WLenqfYDp+FTSwxoXnuyAU6KP4fQD3Oaw55JHnyUwRwF9O5/8Ar1rNunuZx940EuY1uxO/711+SCMdAc9QP5k966S3eFBsOHlbsPfjk/h2/wAa4GO5a3L4UtI/DOf5fl2HA+tb2m3MjI0mcg/eYHBJ/ur6ADjPQCtMNXV7E1qTtc27hlEyoxDsxIZh047D2FRwARysWPKqWYn+HJ4z7k9vSssalHHKwTBAA3N/Cg7KvqavytnyrVx88rbio6/ifYdK6YyjJ3MHFrQ14JGkjwnCuRj/AHBx+bNmr7N5Nu079Wxj6DpWatyqcnBxgceo4AHsP6GsbVtWkkim+baiZC+2B1/PmuqVWMI3Zzqm5SsX5dR8uN5V+Y8n3JPA/rVFr9pGdiTj7qjucdT+NcpFeyXN3HDED5MYUMP9rbwPwGPzNT3Bk8wJFyvVj6+34nrXnvEuWx1exUdzVEsu8SEH584z+VSkqIGiA5zjHuetLaIfsyzzZ3McAn070WrRsdo5JYE/yNaRX4mTZQj0q5kYKx4Q5H9a2o7KW0G8E5U5rWhlh8tmHPWlkYzAKvrj866YUYxV0YyqN7kIlaVDnPv/AIirMcMUgEg6t1x3I7/WlRFi6jkdakJjHzIcA10xXcyfkTBABjOaqt8pKHlfTqRSGZlPTg96Z56FsP1HT2/Gq5kJJkDwgBnXp3wcj6/55qi0kUQ8wnA9c8VduZBGdwYr7j/PNc3qE8gRpvLKnu0XRh7jpWFSSjsaQTZNctA585Hw7dcAEke471zl/wCWqsWIViOu3bn1yOQa5DUtVgik2TqYXB4LLtBP1U4NZa6+88ZimQFccFCTx9Tz+BrzamJTdmdsKLRbeOOfUoCrDaXAODxn19q+5vBlsIdHhVWJwBkNg/qOtfn7p8xfXEJBeLIwUIJ//WK+/PAhm/sSIyAqCOAR6ccEZGP5V7GWdjhzDZHcUUUV7J5AUUUUAFFFFAH/1v1SooooAKKKKACiiigAooooAKKKKAEIryz4oayumaQEyFZ84OefoP6mvSr29ttPtmurt9kadT/Qe9fGfxM8Y3HiHUHkiXbBb7ggPQKO59zXh55jI0qDpp+8z1Mrwzq1VK2iOEleB55L68O4t1Hp6AemamF0ot3mlATrgf41gWazT7JSpkZ+V3HAX3NWblI2iEpzjooH8XqfpXwvMz6/l6Fa6kiWMSEjHXc3Cgn27j61k/aJLwiGDdIzY+bGT9R6/wBKrTNczXBd9rZ6L6A9zWpZ7IlYZIQk5K/JuPueTj2BqEaNFpbVyhjchWPqQ2B6kf8A6vpWPd3kiSeQn3vpzn1P/wBc1XuNXlMn2S0G1RyNi8D8f69agWOdtimRd7dEHX6k8CnzXBRtuT29zEj+Y4adv4mYjb+A/qa7S0lkMcRR/J3dY40zKfrkYH61woL2jGV5QwHIAAAX6Dp9c10ejSSXLtKx3I5GQqE59B1yfzxVQepM0el6SbVY96AKqcfvCSc9Dwvf8a6gSQvtdIBxwCRtUfRRnr71y1k1yUUpG0Y6buARj6A4/Ct5NsW4PMAw5bqzj8zgV61GVlY86qrs12u4o9yQqpYew4H5cfjXK6jeSysWk+c9ieg/PGalnvjErRxRsN38TEAnNc7cISxMzYZf4eWPPr1x/Os8TWclZFUaSTuyncPICPLJJJwPXn07/lXU6XblYjNdSZhTkgnClh2OOuPQcZ61z1rE7S7ijFh2/iOffnA+nNdTBHPdyCW+A8qPHA+WNR6Z749Kyw0Nb9TWvLSxoxzSTIbm5XZABiJO7HscfyzWReSs2ZAOTngdB7D29TWvc3Cld7H5UAHA7dgB6n8653UbyRUYY2MB84HVF7AnsT6Ct8RNJWuY0otu9jzfUfMF1IW+aQ/OPRewwO5+vStLS/IhjYsd7knJY9SOMKOpA9fWs+9jR75Zp8hA28qM/NjoD3wPzNbCRF5Nyx5eUABQOQnUZ9AOpFefHV3R2y2sJeM9zjaCyLj2FZxE4jMUL73c8gdFGffHbk9810N5gcOwHXAXHygdTjpnHGT0rOEEsifuBsGMnAJ4/vHPUnoPepcHcFJWMyC68p40dyURtpPUuxOAB69PoBXoOi6yby2WcMFklO2MDkRoOB9WOD/M8V5lPaSrMcMd2SBn+AHgnP0/M1v6M8to6YUoFGEUjpnAGffHQds5NdGGruDsZVqSkj1W22uQ7NiOIE5OT9Tk9WJ7/lVO+Rty4B3uQqoBzj3NMaeSRktkf5wdzMOg44/+tU7vDEmVPlxL95+5HoD7160mpKx5yTTuY9xaTRSBCCGJ2oi8n8/5n61ZVLiGBbQKQwGWx79gP5Vu6dbsxM8KFGYZLt1Vew9QT/L3NbkdnHbqZIh88hA3t0/AfyFOlhL+8hVMRbQ5e1toNPi+03n3oxvWP0PYt6n+Z59K0LaZrWEXt4S1zMP3aD7w3cD8z/nilu7cNOI1b5Ac7j3Pc8/kPz7VjTyBkeZTjnJY8t0wAPw6f/XrRv2e3Qz+PU0Li6LSpYQsvmRgs5HQf/qrImYXqmyGGQKXZvUe59/6Vz8EspaSaQ7FlGCc87SeBn3/AJVuWCNb2MjyHL3ICqPQHP64rBVXUfkaOCgi3a2iCItH1Dlif1q0pTgMByCT9OgH4mq5vEjXy1/jyOOgUcH8TWZJKxjAHLE7ifbt/jWt4wWhi7yeptXMzSCO3U8DA49e9VS7WUSugxuIY59zgf402FsKhbqcGo5/NuI4oz3G7/vnpVt316k26G5bTxeWWQ8Nz+nFMi1ZmuHUfd4IPr7VnCMjYIjjKge2arrhJtvG0g5/r+NU6slYnkTOplvHnUPCfnHH1FVLXVI5TtfhumPUj+tc5NeGzlUZzFMOG9GHY1KphmQy7uTjd7HsT/jV/WG2T7JJHYLfREbE+Y91Pf6VRvL2ONcEfKc8ntXHSyTWsg3HhvuntUUmqzPmFzz0wx4P0Yf1pvFNqzF7HXQ1pNVSJC6uJkHOAfmUVkN4gtWBe3bYw4Kt3z79KxpkFxJvwUcenIP8qiuNPSZcg4fHOPlb/wCvWDqyextGEepXvtdtGR1eNm6/d5x9UbI/EcfSuammVQLjdhW6EKB+Py8fhVu90h4o/NUhWXkNjDY/rXGSmcHyXcbs/eAAz7Gud3clc6Eklod34Js9OvdVzcEFWb/dI9wfUdcd+3Nff/h6yistPjSBmKMq8Mc4OOoPoa+Hfhr4fvbzW7Z7GULKG5BGCQPzBFfeenQCC1ji2hdo6DpX0uWx0bPFzCWqRfooor1DzQooooAKKKKAP//X/VKiiigAooooAKKKKACiiigAooqnqFytpZyXD4CoCSW6Af1+lTKSim2NK7seIfFjxHJBALcv5UXIAB5c+v0r5Sc/2gzahetts4/uL/FKw6Z9h+prvviH4hOu6rJKXymdo4xwOOBXJJFBNJE33ghwq9Bnv/8Arr84x2I9vXlUvc+3wND2VFREsgWjae4Tg8HsMf3cfz9TTdR3SKCi+WG+7nk/gB+grWuLwISjISF6Dp+H9Sa52aRppnmuHLMw2fL0GeSAOw7Dua5GlsdavuYc9rcy4itxlW++R0z9e59uAO9W0svLjLSOHb0HOMerdB/KtuO1eWBYbdFVDxycE/gO3vV1dGhto1NwfOkbOFx8qj6dM/Wj2T3Y/aLY4iXfMuIIjj+HB2qR67jyR78ZqqiiPdKxHHBK8L/30eT9c/hXY3kMYG3diMnBA5LH0z6fp71nx2kEf72T92qHCrgZ/XpUcrvZFqWhg2+nJP8A6Tdx+ZHxjzDtXHYBOTj64z6V1emRkoGct6YxtH0XPIH5U2KREYlysRUfKoIZiT3Jx19eKlhe6k+feuwngtuJwfQYz+NCdnoJ6nW217PHEqIqh2OFznj0Ax1P6VuWM8iQGSZ1iVsbj0JJ6ALyxrJ00KCqru7bn+7ge3Uk1uSG3WIICsaAZ4wpP1J6fqT616lFWXNc4Ku9rFSa73MSkZcjOSflH9SP51z9zdzyuI7YKg9cd++B/wDXrXuBDJGCh2qBgdPzP/1zWU62yHAOR3PQH+p/zzWFZyelzWmkugtpckSGL77gYJ4O3146D3JrsrHddp5znaqjK57gd8Ht79TXI2sSykuzeXAnLM2ACB2A4AFdRa3TX6iOzj/dH+I8Z2+nr7D8cCt8K+jMq/kW5EH2Yyg5ZjkYAzn1HtXJXjSRt5eOhyecjPqSepr0SW0McH7wheP++cD+f9TXn+sRtFG0pXoMKDnH19zRjabjZsnDTT0PO9YkleTNsTuUgk4xtUn+Zx1POBXX2jrNZx4cRjIUEDJJHU4HU/oK4yd5I5HtY873+aR2P3PTkdz2A+tbmjXBugLSBfKigQZPUgHnkn+LA6du9cVJnZUWhtP5EgcRfMyjaSBwD2X3Pc44HfNPiiEcJ8sjL/ebOT+Z6n+VRxSRfZmMI226sQ2PvSE/wg9ceuOTSW264uPtCKQFO0eik9FUe3860nZSsZK9iyLHIWGJMySEHHUnHbnsO5NSz6e0GHfLHPBHAwep/PpWnZRP5Zhjb945O4ryce59P0ropLNfljPMjDGCc7V9T05rqp4bmjcwnWcWc9Z/NwQRCvUDguew/H9K6eyt7e9cXkwUxRDCZ+5kd1HcD17npVSa2hiVlwAgyDz1x1A+veo4tVTzI7KBDLcSsAqjhFC8kn0Ue1dlGKg+WZy1G5K8TqYZYolM92NqHlIx1Oe7e57DtUcl1b5+037bdnQDAVQOgHqfp9KzZ7iEgAyKVRhlUHVvc/zrCmuNy/a3IJ3fIT0A7ED+v5etdk6/IjnjS5jYvrtHcrIfLXALf7K44GOpPtXL3lzBP80g8u1Q5255YdyT79B7fhWXcaqqK8u7zHc9evzN3J9T+grKfzLmNY3YkDLMPTP82P6Z9q8ytieZ6HZToWL9u51B8sAAz/IMdcf4dAK2JrrdOI41+UHCjvxx/M1Db3DW+Z1RVaNSsX91MYG4/wAgPasCXU3RAIQS4DOCep2tx+ZNKDtEUldm7aypKZZclgAwX8Op/E1ppBmP5upxk/lXN2EzwWsKT/fbCnt0yxx+JArqY2b/AFOPuxoPxHJ/wropWa1MJ6MnX/VxepHP51a8j59g6Lx+eRVFMCdd3GP5VpW8wZC5PpXTGzMZFecbLdcdQP5f5zWPJMjSSleAc+5BH+FX9Rn2qyp2IIxWHHMkjEg4Y+vtWNWWti4LS5LFgqYLrHlSdGHQH1Hp71WYTWjlVOdvB+lXJkj2lwSufvr/AF/z25rEu2dR945XofasZaFrUmkvxgxvjb3HpjuKrSXKAYlXPo4/kaotcK/zPjf0PofQ57GqsjsrbofuE4I9D9KnnZXKaguo4wHVxn+7nB/D/Cl/tOwvgbecmOQcBtuMfWssfZn+8SCfamyx2RGXAZh68H8D1q1JhYz9UaeKN0Ys8f8AeXJBA9Qf6GuLjuPOuzBwQDwTnP455r0m3SMoRbz4LfwtyP1rPs9Nd9SPmRo27PQrkn1HQ1rCKbuDlpY9l+DXmm6J8rzI8gA7gQG7Yz0b05GelfXMX3ehH1rw34T6F/Z8LzMWV5eDkDa4H3WHHUchh9DXui9K+owUWqSueBi5XqOw6iiius5QooooAKKKKAP/0P1SooooAKKKKACiiigAooooAK8k+K+tPY6T9jjbb5vLegA9fqeAK9Zd1jQuxwAMmvkH4neJF1rWXsoW/dQt8xz1I9P6V4ueYn2eHcE9ZaHpZXQdSsn0R5XDEbt3cnapPLH19BWrFZmBo8DbxlR32+v4+9Z1vI10+6VTHDGcBVHXnp9T1J/CtG7uXAMiLyRwcEhQO1fDxStdn1zvsYWoyRRJvI+d8hQT2Hc1iDb5qlyxLfKicZbPU+2fXH0qO4e5uZ23sdqdAD2HTJ/pVrT0EbGTcWkfPz9So9j/AC9fpWSd2bWsjrUc2y759qM2FKry2R0XPapLy7f/AI94+XUZPPOT249uvpT7dQjRhPvL6/eJ7/T3NUrlmjbEimMscYUZPXoP6mt5uy0MUrszRsO4AGaX8cJ7f4k/QVTuYZJcJE5e4cY6AhM/XCgn1NK90tu+PLO92+SJR949ief896s6bDwZJQJp+vllsopP95gACfYCsormNG7ai6bpTR4e6myP4v8Alo5P+9wPy4rSmZEHlwRMkeOSx6j3ParUM8jB2hdEA43ffbJ7D/AVHeWSyKZplYAfxMR19eSBn8Kpx090nm11KcGpW8JI+YyAcKmWCn19/wAeBW5ZSPqDeZJHlV/ikPyqfXA44rkY4o4v3qDKnkdBu+vrUqfbp33Tsqxrnhhx9MHj9CadKo1o0E4Jq6OwvpklAijke5ZeNkY2x/8AAm6fhyT6VmOsm7E0aqB0AGMn37/nU9lJbxoFLSXEzdgdq5/22+6o/U9BVqeSExlY8FVyrMDgZ/uqR/QH0rqlHmXMYRfLoUGV5SomfCKeExgH/gPUk+9dxozKAqwgvKMgjsi9/YZ+uTXGpbp5gUyrbLwTg5kIPUeorpH1FYLZLLTozEmcAEHdIfUDOSPyHv1NbYW0XzMzr+8uVHUieJ0MEJzI5ySTk+5PoPT9K5rVkVlzKSFPy56Zx2A61q6OIYbYyy/vHfkhMHJPAyRx/QdqoayplLqxwdo4XIGD0APX/Hk9K6sSnKnc5qTSnY8g1C6SKdo7VNu05z6E+/8AePcnoBxRo8YLtHcDCcE88sTjH06Z+gFbWsWcdjE+EDP3Xoo471zGmvNaTSS6gTGZQCU4LkHgDHbeeMdcDHrXjRVnZnqN3jdHXLOkirEOY0yeOmfr6f55phuwkDTTZC/diVeM5POPc+tZdxcQRMJr5jtQZWBepz3b+np7k0Wn2q6uxf3UYATKwxE4UNjqT64/IcdabkSonf6PeGBlghVVdFyVPPPq2e3p6mus3RWkDuWLSkjOP7zdvXPtXD6LbSBhsPznlnxgFj7df89q60RRx+XHEd7ZIjz6/wAUje/YDrXp4Wo+Q4MRFcxkXMsrFsv83IJ7L6//AK6fZFlJNqvloeGb+JvRfUknk/lW1Par5YVPmZflwOgJPOSO/sOlRC12DOduzjeRwueoQd29TVqnJSIc48tjMuLuC3jCIgRFyBuOScnnPuTXHanqSySGAPlgOSeka/QdT6Dqe+BWxqbbInmhXZuB8sHsOm8/0/SuIttKnvW8yZsqTu92+vp7dvrXHWqu/KdNKCtcvxXGVV7cbyPlXA3ZJ7Dtn+83rwK27m4W0hEVuge4KgADnLNxyfrz9BUNtBcF8wrtWNcIF4VR9e5xyT0rd0zSohKzu2doLSueigD/AAq6NJt2RFSaRkyWs12qWaEiCMAtjqzcd/oP1rUGlJh7+cdcH2AHOB7dq6W3todu8jb5zBR+PP8ALH51Xvb6OOLy0G4K+MZ7AZOfxr0FQUY3kcUqrbsjiol83UZEI4gx+f8AFXXsQrMCRuBA/HG4/lXNmAWrGYHOSQ3uWGf61K85aWd0PzIMgepY/N/h+FZwfKOSubsVvKEE754BH+fwqhbzlbeRSc7QCPwrW0u+ju4pLLO1mHyg9MkZH4Hp7E1y0rGFSGU+XKvB6445FaTaSUkZq7bTK1xdurMRnswx6/8A6qmjKyjzVAAPzY+tczBcyB/JkG7b8p9cA8flW9aiRBuHTv8A41xqd2bNWRff5kwGPHr1x6fhXN3cjKXjyQcA/QjjI9Qe4rWvJhBtkPAJwSOx7f8A1q5bULxGbcSA69e34g1U2KJW+1fN5brg4P4j2P8ASrcRcjZgkHgg+n171ioGmYlgAc8EdD71tWpmhA3KSPUc1ES2WI7OZe/X8aklsnkXBchvQ9KvwSiRQc81caQOgQgMB271vGKaM22cva2VxCzsdgI5BIOD+NaNmZJrgeYysD0PQD0PTNaBkjEbFAeuCcjj8f8AECqdoqxys4dtp6g9Pr/9euqnFInmufSPwr1m+mZtLnl3Bfuq3PA9D3+uele9r0rxr4aaN5WlebMDvLBlLegPY+o7g+1eyJ92vpsKmqaueDiWnN2HUUUV0nOFFFFABRRRQB//0f1SooooAKKKKACiiigAooqnqFwbWzkmVSzAHAHc0AeS/EjxnJp9rNZ2UoRBhHZfvZPVQfp6fU4r5QvLszMSSQZicHPT3z/n861fiB4guNR1yVnbKQkqqj7o9cVzFraNfFVlysaqCzZ+6o7Accnv6V8DnOI9piWl0PscsoKnQTfU17ONZjHawgtGo+Zzxx359zVzUS8lv5EZ+VxxxwFHcfXoo/GmWim8k+XEcCAgD68Zpbx2MJRAS8hC7h/CMfzx/OvMXwnf1OQleK1IJ+Ux8c9Ccd/XHU+pxWhp9u8WLiXJkfBjD4JIz94j1OeB2piWttJMnngnB4Uc59AT6nvXXWcKp/pNwBtOAo4JZufzH8zSpwvqVOehJbxGGI9cnl5DyeOuP5DH1NZGpSHmNdyMBzkcge4/xrqriOWMhM7ZpOeeNq+v+ArBuUUny4gTHzliPmkOeT7Ln861qQstTKnK7OLMDeW8odkB43McO2OoUDt61Kl1KAsFp8zYAZuh2/0H61auYJJZGI3YHykjjOP4R7fSorayRcxldqnjAJH16Vx3aOrQ37C5kjZlC5EYyXPA49McADtn8q1kuXu8yRR7woyDtGFz7k9fSsa3tLoFVEgSPg7QCT7cf1rdmsrifYkjbUX+Hdg59cD5R+OTW9NyaMJ2TOWvjFGN9yfKBPGWyze/Hb2rOkuJCAI0IU8cnLY7jpgfzrsJ7Lyj5sUkaMOM9W/MnrXMXluxnKSzE9ehCjJ9TjP5VMo2LjK4y0W6d/tFwCIlyFG4genHYe7cn0rprfUBEWzCTsGBJwiRr1wnp9eprBtvkkCF0IU44GQo92Of0q1PeQ4AEgk9MqQPw4J/KtKc+XqTOPN0NZrmaf5bZPLDnICcf99OwyT9M49q1LdoYQIY1a4uHHT7qkdySSW2n1PWsaOO6QiTCu+35VJyRnr8vv3P8qYZ5UkAjUSscBneT5cjrkLgH8TXVCp1ZhKPRHo8d1b2iCCL95IxBZ/4dwHYDAIQdO2T3qhctOEaRHBlYjLnnDnsM9wMf/WArnY7+eaQ28D7fMYGSXAUH0RM+vUnHA9K0SiX2YIpP9HgwksycBjnLqhPOexP4Cu7n542RycnK7syriG3SP5j8395uTnufc15pqlwI73Fqryt0B6HPTJPbjJ/ya9DvJDc3jwx/JDCPmxxgdhnt05PX8a5DULS3CtN8wIA3AdW54AHbJ4HevNqQtsd1OXcq21tGzxyzttDsPl7se+Py49K7CDEtxF+62KgyFyMAD6/nz+tYDJHBHFeSN1Unt8qjjI9uw9akhklv5TAquUfkID8xGeGkb37AY4rnirOzNZbXR6BaahExzF8+T26nsM+3oByTW5aFnm6jzOjNyQo/ugDjjv71iaXZRwqIY8s7cOy9AO6r6e7eldrDZBVESYVQOdvyjHYfT/Jr2cPSbSPLrTSbK8bLM0iL8sacHGAfx+vXAqLULiHyVVOpOPTKjsPTJ696aZ433eRjykOOOAxPue2OSaw55FlleVjlD93Ax8vTIz0BP51rUqcsbIyhG8tTA1V4pAZZ3AReWycgnoAo7+gH6VRikYgI0OE5Y7unH971Y+g4Axn0qldNLeXf7vJSM5G0cE9sZ7Due/auz03So8RrMTJO5Bcg52gdF9M55wOAeSeBXnUKbqTbR3VZKEbMLW2ubtWlkULsOQG6bj0z9OuO/5VYvENtZR6bacCZsyMerH39s8kfSt+YJH/AKPapiKBTubr+A9WJ/KsS1ia7nZn6IpQe2PmP54xXr+y5Vyrc81zvqyxDO8stv5fMaTMFPqF7/iRx7ViyxhHkicniMZP+0Tmunkt0hiiEY5JOPrjoKzpYftE8uOCXP5ADAoqQdrExkrmbJEZVLHlZFP1DoeP5Cs64UQ7ZVBO8duue9dTFbeQ2YxlT0/4FVDUIRDumCDyznPbafX25rKdPS5SnrY465nlhc3Fq5DKMj+nWoLq/e+QzqRGZjv44CydTx2B/wAamv2EM4mxjaMMOzDrnHt3/OsC4ge3dxDzDN8yZ6euP6VySbWhsknqWYzK0hM2cg9T1H410lq0oQA88ZU96xrCNpRvIw46+49a2yuyIkcKDhh02k8g/j2pQXUJvoZepzqkbMccjGCOMehHpn8q4WWQyP5bghTypB5HqPcV0mtXDSRnGHzwynr9RXFWbyhzESWGejc/lSm9QitDqrJNoC5BrooQF5/yawrRcDArWWbbjdwaIBJF54kxmPg+3Iqk8jxtsmOM9CKcJwDt6Z/Wonl3EJIA4PY8H8K6ImbJfMHlt5jK47FRhufb/A0unNEuY3zsY9eR+RHf+dZsUUTSbYZGX+8hOf068fpXqWheCZZ0inkJMbkE4O5WX1VgTyO4NdtGnKWxEpxivePojwEWj0aC3EizIqjDA54x9AcexGR06V6IvSuS8N6bFptqlvEchBXWr0r6ekrRSPn6rvJtC0UUVoZhRRRQAUUUUAf/0v1SooooAKKKKACiikBoAWuP8c+ILTw94fubq4+ZmQqijuTXXswRS7cAV8zfG7xC5gjsIuDJwq9Tjux9PYVz4qsqVKVR9DfD0nUqKCPle/vWvtTZiCzSMWx2/GuktYRFGlk5Yyzctk87f/risG2gS2mluJV8xoyAFyAAx5JJ9fQV0lnLI0MmoS8yHgDHc8KP6n2Ffms5OcnKW7PuklGKii5cOVjIh+XadqgYxuPH546VKZY7ezMikbgu1e+49yPWsFboGTaxJI4X8eCePUd6vrMbiRZHj3RxHaoz1yeB9KUZDcS3CFESyzrjI5HTgds+nrioBqDxXAeQAOwBjU9Qo7+gyeB6CqOp3bwAIG3zO3Poo9MdPpWVYXAuLmSRCCzMQCWyBt6k+yjj6/jTUtbAo6XZ6Bb7ERrq7b963zOSOQSOuAfThRV1bNpJFkkG6QgBIVzu/wCBY6Y9P61zum3KvO00LkBeQzHnPeQj/wBB9Pwrozq8EMWzb5UZ+bv5sg6DOPmC4+n866lKLV5GElJP3SebThsHmhccYUHj6DHpUUejIR5r7trc7Y1GCPqThRU9jN9pIk28cYQjk46cdABVu4LqdxOTnOCcn8ugqnGDXNYjmkna5QQyW6F44gFPH49ueM/yrPngvbxyZHKqP7uefx/wrWkkiEiiSUPJgZI6Y9qlN1byhlWViEGG24GP8B+prJxUnZsvma1SObXSZPMLLgN0LHnr6ZqnLoJdjufdjjPH6Zrt0lswoQLkAHvlmPqT6UoOX3eXz6noPYCl9Xi9Lh7aSOEHhtAn15J/+sOtL/Y5iUbIzx0yelejfZlnUeccBfbA/Cs+fS7EhvOTCg59M+9a/Uo9CfrT6nGRpd2oJUtGx67QAv4kDJ+lVgss5XzXL5YZLqVAHbHYnPrXWtZW8oZIfMZvQE4A9SenFZz20w3IqkMMfNjOMdBik6DiwVVMzn0mVGUQRkhSxZskHnvuPQU/+04tPgisYm8pYyQi4LHdwSxx3Azj3xWluug3kD7pIXc/AAbsqKD1NU7y2iuVEafM+ck4HY8DA6fnWi93WJN76SKxlW30/ZCAXdsbm5+Y9OnGQOe+OvWuei3tFGJFLIGZ+vzOegwPX+VQ3FzexzC0SMCLIVnXcwAz8xHHXAwoHU1Umu3+3FrYbCSI1B/hxyQR2AHU+2KG+axSVjoBp8uppL5qjzdyqFAG2MDgBccEgcDsK6W3sY7WKO0tkBMjckHJb3dvT2Hauc8OXUqyRqFIVizru5Zs8bz9e34V391IplisrfYspA3yDog7gDueM9h+FKnTi9QnN7GzZJDFEUc7iv3seg6DjufTsKszySyRqk7bIyMt2P0z7CoIhBBGJERiCCY0zlmJPBOOme1QX5nRlg489wBs7IPc/TrmvSu1A4GryIrm481fKhjLRqAAq9y3RfbPc9cVjaiHjUxjEspOCR9xpO4x/dQfyrbdks7ZDGSTkhSerE/eb8uB6CuV1KVijEfLGqdfb09+etcuKlaGptQjeWhgx7llJuCBEhySvGSPQ9z9Onau2sbmQRBmxHkfKO/1C+gH4k9eK8ut7xZ7wBBuMXJzn5fy6cf4D1rq4dR8sfZ45A8xzJIzcBF6gsew7geg+grDB1LG2Jhc7WWZrXTyZhzKScHkhVBbn+Z9Sa1PD9u72yPMMNMCdvoME/yxXF6fcvqzbiT5bncS393+EfiFyf1rv0nENr9oj/hXao75cZz+Ve1QkpPm6I8yrFpcvUbcGNntnyAu4n8zVGQpbt5yjgtnPoR/+qlmZWmhgUfL8uP6fzP5VUtJftAls5fvoSV98cEVUppuxCjoTSqYbh4cjZIPkJ6YbkfUetYlxdXMRkDp8ycbG7/Q/wD6waS8uWMaR5wQTtPoR1H9RWZqd2H09jKN8ZXa/OGVv4WH8vbFc8pp3sWo9zzjVtagmmjaNGhkUlW2nKnHRh+tTWd0JgIpeVPb0PqK52QR3N0ZRw/cdN3v9a6Cxg2YGMgH6GvNcm2dVtDrbFDAB3XPB9P/AK1WNQmWPaR/GNp9CBUFozRxAxncoHQ1k3crNI0XQKNyfzre9omdrsxNScncrdV4NY9rEWYMeoOD/wDXrQuHdi0q8n+YH/1qkgVPMZk4Dc+3NYPUtFiNzEwJFX1kBXL4OOtV3RWDoOoPT69qzpJ2jYSA4I59iKa0HY2WMU0nkhgkn8Of4vb6+lZs9zNA54wO6E4H1B9ayL+SObKplNp+Rwc7f9k+3pTvOmv4fs10w81R8r98/X+frWkX0FY39Cv2utURJEEjA4+YDdt/r+dfc3hPQkt9KhkUKCR8wA/mPUfrX50+EtYu9M8SkyRf8e55XOAR6iv0T8DeJpta02KR4Aq4HK8fp0/I19Fljjszysw5t0dlBbiJiq1oAYpAMEn1p1e2eS2FFFFAgooooAKKKKAP/9P9UqKKKACiiigBGPFNHWhjTlFAFTULiK1s5bmf7kalj+FfAfxB1+41vXLnUOQM7UGOg/zivrz4j67JpujTlFAjVSCzdCxHAA7/AMq+E7ktcSl5mxvJdmPfvj2zXzPEeJtTjRXU9/JKF5Oo+gqw5hiidtrOMtn8yfqT1rppIoY7WO3jOcfNj+p/nXKWTrfzmXd/rMqMDoAeceg4/Oupjie8uSsYIjVQFzycHrmvkI7n0kjHthIDNNcdByxPp0wAK1EnIt/MIB8z5gFHGOgAH8hVqdEZhBHnyVJLseASOcn8eB+PepX/ANTlRsVcDn/a749T2x0qoxtcTdzgNZuvIjKkkySHaxHUdiR79lH41zkd7JEzR2KABUAOeTn+FeOvqavakBJfCAne3IO7ooHr7mrOh6VJcRrcyKdhO5COM9uAP0NZxXY2ubelx3MURMjkAY3HoS56jP49AOK63TolmmI2GSXILHsMdMnoFHpVI6ftQRyEb8HGScIO4Udz6n14rrLC2ihtltoSBHgOzYyW/DuPTPGfWrhGTlqZTkkjSW8jgVg77io+b+n0HoPSueu9QklJEC4X6457kkce2K0roQrEIU+VTz6fUn1J7+9ZBga6JAUlegA7+vtgdzWlWcn7qM6cVuyk05RfKU5ycsRxk+gI5xWna28skQByijBA7fUg4yfrVq1sUjOM8+wz+H/6q3I4fL+ZkIxjGRz/APrpQot6sc6iWiKkVsY4yxOB6nqfb/69W4ZCq4AOTwOwH+NE2xzhOq+nAAqJCm7BO4jr/hW0dHoYvUuu0zjKybE/U/5/KrUWnGch5GBA654Az+pqmG2MGAJY++MVKsxLbd+eeABwP6mumFRX94wlHTQ6SEoV8lACv5Dj0FRXEAkYE4RU6E+vr/hVOKT5sSuNvQ9s4qF7/wA9ysAwpH1IA7D0r0FVi42Zy+zaehlT6Wwn3RYJY/w/5/Wsm6W1gdllQggHaT+Wf144rsZJP3TDbhjkj1wB/WqqRxzkRLErepPOAeT+XGKwlRjfQ0VR9ThNQ0aC6wtlGAoYAsCfu9jx3rzoadJa3l0LxgLaTo7nAVSfmUD3zxyc17y6RJI8cciqqK25R2BzzXEa54dnvLYSIm5guQSMncjBh+n86xqUuXVG0Kt9Gc7JNNFOXTFvEoT5h2UjhB/ujH8q6rSLy3c7xhIA+WZj1J4C89fXr/SvOru+MlmIMM8rMXynOEGOv+8x7elWNLmQyImoEiPAEcWcu5YjJI5CjsT36etYuVp3Rvy3jZntWj3ltfTztYOZQjFTKB1YcMFP91emRgZ4FaEsSyyGJCFXq5B4AHPJ9+pH59qxoLmNbFbS22oCOFU4yM47D1zjP19hkzahJPdCyiICAEyHJ2qB1ye+P58V2SrxjFI5FScm2X729E06wQN8ozlzxtUdcfU8Vw+t3yysIYc+VnCjoBjoW9enAqa81iK2t5mgDFjlVz3AHAHuep9OlcRrcsz2kUMrkbAGnKnB+b7qg/3mP5CvMr1uY7qVHlFOrKqvFZjIQk7hwN36Akfz+lXtNV54I7dyFRiZJjnmSRui+4UVRtod0Rto1WNYiEwvKhiMke+3ufU11EFvEsas/KxDcwAxx6E+/fHPaoptp2RU0rHciBEMemxkDKBS3f7o3tj36fQe9aGoawbi8MNqpCBnIx3JwB+HT8qxLWC5WNr+5Y+Y4YDt15Yn+Q9qktrabyludvQH8yf/AK9eoqsrWR57gt2WbzUJDMm07fLVSSPVEIX8Oc1lSX06FpEciUNuz7nn+f8AOp7q2ky8bckjae3+eBVDymhILkHcoGPXFQ5yb1DlikE96lyzGI/J98Z6gE8j/gJzVC8d7m1fZwcEHHPI7gdx6j8RWNcOLaUO3yIw+9z8pJ4P09aItQeIEgg4POP7w/kf0PWlGbb1FKPY5xICH3jBxzxzkV1dlh0GOGXg9/pTI7RZibi3GAxJxVu3tzu2jgiklqSy8zeUmE439u2fT/CsyYZlGeCBx/WtOeMyIEPDH+fY/j0qjcEsgkxyMcfzB/nWrRCOWlO07Dwe2aWBieP9lf1p2oKDMjg/5P8A+qktmDsX/hxj8qytqWWLmXaflP8ArBwffNY4Du6xn3GPUf8A1jVlFdkIl5A+b/HFakMAlf5QAcbh9euKSTZT0MaO0mD84PTPoR2NWLu0RLcSxg/StlDGAOMFePwP/wBeq13cCNmtZ0PI3Kw/iH+I/wA8dOmEEZOTNfwNo2hajchtYZombC+aB93/AHh1wPUV9oeA9Ht9EtPs1vOky5O0owKsp7qf5j/J+LPB9z511EI1WYCTbhiVDc9MjkV91eG7e1+zo8ds9szANgsHDAjqGHDfU8172Weh5ePVjsKKKDXtHlBRRRQAUUUUAFFFFAH/1P1SooooAKKKRulADepokcRxliQMetA61nazcw2mmz3NwcJGhY/hQB8sfHDxKtxcxaLasWSP53x3avAEUqgEgzJcj7oH3U/yK6fX74axqt3qdwCY1PA/kBXOQCbzzdHHI9Op6Ko9Bnp9K/Os1xHtcTKXQ+3y+j7OhFGxYW+LtbdYx8g6DvxhF/mTXaW1ulvAYBnzJMea/wDtHog9gOTWPoURt4nuTlpHyqnHfH7x/wAOg/GtdrlhIVgA4BBPYZ4yPUnkVz0kkrm1Rtuxk3SJ5qW0RO0nnA6he/PesjV74RRLFFIcDcR6n1Iz+Wfritq53NKQv7xlBXP19+1cFrd1H9tdVIY7tvHQ4HIA9BWNR20RrBXMOy05pJXku1BkuvmEZzlI+o3ehbqc9uBXfWhg3xiEnCDluhY+w7e3tWPpdheToYSfKeY5dtvJ/wBlc8n/AD0FdjpdrbQ5SD5mBxnGcEck++PyFEU3sOTSJ4bQs3mzjDSkKiDrtHQAdh+vU1p3VzHZps+8R1UdiOmfp6dqfFGE3SgfP15OD+J7fQVi6g7b2CAEk4+X29Pxq5e6tDJe8yhPeTSuEBAZhyT0UZ+vX2qytyY1WFgfQK3U+5/wxxWN9naBjO5HHOSen0q1YnNwrrxu6seW5+vbFc0Zu50OKsd7YlEtw9y+1nPReuB0A9zVq4AkO6X5c/dj74qtYlDGsyDcxOF3D5j6YXGPp+dX5kkiUqyDPQ5OT9M/zr1knyHnS0kY7s54j6evqfQChZRGmFGB6+tPkBdhHH7ZJ44FU36bj6kKP6/4Vxu61OhW6lgSbQ2OWb0q35626iPP7wjn2J/rWQd0SbjjceFBqZESNA8rbmyBwO5pRmwlFG9CiSMN5zxwKv2sI3FkAAXv24rm7OcCTk4znH4delasWoSSAbCQo6HpwOB+td1KtHqctSm+hvPOgBG0MxHf17VnX5aLMYf92U+nzGoAftBIAJQfxHpjBoS3eV1edwwGTj1Ax1z7V2+0ckc3Kosx/KDXe5pAEAAAJxuwOGY/WnfbJHstlxgSqVQ4J5LHOR7mr93oyyFQ2c4BJHBwRg5rNNoUJWQgYI9s5xjH5Vg+aLtY1XLI831iyXQr5r12xaPFkgdAOAo498VkgC0DTR7TNIeTzwW447ZwDj0HNem61YJqGjvaFR8q7CMY3Acj8f8ACvIJ/NtYorS5x52ZMMBk+ikL7D1rCcUtEdEHfU37XUVjk8iKUStn5VUZJxxyegH9B787j3EvkmHG9nwrbBkfTPfHc965XQ7SVkE+RHaxAhd5DM79C8jdCeuFHH8q9MtraMoiMegySRt7dh1+prn9jNo1dWKZwcsM0b+dMRhMhUUZycdPw71lPYzsRdTryzEoDyTI3G4+/Yeg4HevVjZRS/NCMjGN2OMe3rVOfQpkjLFCG24HGCq+3pn86x+r1FsjT28Op5g0Lxtb2Q5yT8o4475I7nufc4r0HR9PkvJYlnQCPjCjhQByT+Ap9hoMKzSTy/My8N7Z6IPc8V3EViYYCqDEknyAfTk/0FdmEw0pO7OfEV0lZBfwRuqwxfdC/Me2Rycew7/lWi1uFjgt0HBwcd8Djn8qgMsVvvR13BEA/wCAxjcx/ElR+NWbe6Zrwsw+baFHPfODj6Yr2FCN/U8tydjLu7dvNZh/ECx+vIrm7vDiMY68DHr0/mK6x5fJn8sgkjDHPp1rKvrdCpEYz8hcD15OcfhWU6Sd2hxn3OPmtlcESL8wyCvqO+Pf2rn7vS2RC9t8w9OzL2x3BHpXYSkHJYb84IYDJ9v/ANVZ5mSTfAevUj09x6g/5wa5/Zo1U2Z+hygSfZz/AMtPuj1Pp9cVrPb7H82M4Knn6HkH8e9c0++OQoGz/FGeuGHI59D0rSOpO8YnJ2sPvZ9Cev4HqPxoTVrMT3L8rkuCOGH6H/A1i3M/Lc8Nkj6jtVma7jJBGAcfl7fT0/Cudvb2JQN3RiPwPQUSYJFG6nLE5OM4wfekhBWBgeC3A+orKabzVz/zzYD+ecVsRI0jR56qcmsupZrQRhoVRhknPP1x/wDXqeGF45M+nf2/+tToomyFPp+grQWMnlumc/gf/r1rFENlN4OTkc8gjuQf88Vy+qNJcRfZN33TlW7o319DXYyFS2yU4I6H+VcpfSMJf3i/d7dCD9R69a1WhKO4+HPh5pZ4vnyCw8zBw3JxuGfQ1956ZaNb2kQfAkAAfHCsRxux2J618b/DaNZdVt5I8ZXGVzgsp64zkfnX2nbkeUAoxjseD/h+XFfQ5ZFcjZ5OPleSRaooor1DzgooooAKKKKACiiigD//1f1SooooAKa1OqNjigBVry34s6kbTw3JCr7d/wB76elemeaFPNfOHxv1pY5YNNTJ3fMw9SOg/OubF1VToym+iOjDUnOrGJ82XrLFahpu5yV/ln6020tJruQZ+RV5yOxx7d8cDjvUl6pMyyuu5YiT9ZO/1A6fWtqytjp8KichpZmDKvXaO5/E1+cW5pOTPuE7KyNgZtLYIgZndR8oGDtzwo9Af5CpIEJCgDcQTuYcAtj+QGcVn3jzF5JlkDPgxR9ueN7/AJYUfjUjyLZ2oRGX92m12PAyBliPT0rRPqRYjuZwd0UJ2b3JAUfdwMDP071yY01BcmVySVwoJ4GO/PqOp9zir0+rKEzD94puOATjJ+Ue+euPSs8yz3UYe4U4TGyMDGT6n6Ht6msZtM1imbqeSQtvb7meZcE8Agdznn0/zjFdbCIkUW1qqrFEAmTnk9cDucck1wltJ9mDK0oaSTLOeNoHufQen5da6Oyu/wB0FjAXJyvHJB6sQO59KuE1YicOp0O8sCWIQDkcdDjjP8+KpvblV84DzC4+UYP4E/XtWdbyqeJd0yL8xAOFwegz/k10clzGYV8wqCc/KDwMDoT7Dr+VaxSktTJ3i9DnWsHZA0sak9gemc8fh/OpLawCEzNtIHB9Sc9O9WVmW4YzEYEhwD329MDHTPPPoPerUssCxRlHWIbQFwOBk4GB7cmpVOD94tzlsdVpMcfmLIx+aMdM8KB157k9z/kak8KyR4bCE84/ur/PmuUg1aKGF1tvljiIUEkZwo5Y+/p2Ga0ba6N9cKASFzudz1OACT7DsK9anOHLyI8+cJc3MTtpayfu0b5EGW9ye1Z02nNGPMc4RM8e/oK7GEbVEaqOcE47ZOcn8BWTdSjUJQkQCwofvew6nHrj+dOrhoJX6ip1pX8jh5IHch8cnp1wBnNNEZyFPPPH19a66SwR8sB+7Xt3JHr7VDLYpaRtLJ/rCceyj0zXnSwclqzrWIWxgGEqSQuSwx6YHens+0NHG27nBwMDP+f5VEzPOSqk89f6VdtrdjKI4gPlwST9ayjBt2RcpJK7LcIkDCMErgct3+gHpW5a+Woy3GB37/Wqa2uZjKzcbSP8/ia0444wxV+COT+HavUowaZw1JJof5jStuT5STg44I7/AKVz+owyRglQMFsgZ5B649xxW3KpWPeM5OOB6nGDVe+t4zYEXIwyYJ+jZH4VvUp8yaMoS5WjmHuEW1klJwyuFH+6QCD+GMVwnibT1EX2pc/JkZHdTg9uvX8q7bUGLjyOC23A/wBrB7/571QWGC8hudNuh86qpQ9zjKn8wRXnSTb5WdkXZcyPI4NQNrMlywVFt8qC3KoT3wOrt2A5xxnrXoVhqH2pVS6J2cM+eNxPQE9z6gdK8e8W2N/b3EUVuwTYSTv+6h9WxyQPTvx2rf8ADlwCiKztPLINxd/vBPUjsXP3VHb2rTVpWG0e7JqsEAzGQWwNuP0/D6VYmuWkQBHBduASc5J5Y4HpXnXnLBliDJK5x16exP8AMCqr69NajzUBeRfugD5VHrz3z3/KpliraMlUL6o9TR7ezjijRsuCWJb+AEZLn/aPX8aR7pVn8tDtSJfmz2UcufYlj/TtXlMHieKLcZ8mbGVj64zjBOe5PPPQVvpeieAWfnAIcNdSk5yV5Kj2HT65raGKi1ZETw7W51QYyBrqbHztmT02g7yo+p2j8K0o5cGKUjJjUZ+pJP8AhXKSa9DqV0NK0seYsP3s/wAUr4x+XP8AnNb9oXkgSNAT5khYt1yFyB/jW8JXehhOLS1NS5KedyMZKgk+hGf0zWZcoY5sDgxybfbBHFbc6rNeKWHyuCg+vT+YrOuCJIVlcfNIgV/95Bj+VdM1uc6exzE8Ij+UDKc8dx3x/hXF38JQfaICQU+brge/Pv8Al616DOhyDnjgEj1I+U/Q/oa4rVphA8jJ93B3pjt3IH8//wBVclSJtFmHJcIZME4zjGe27kfqKsK6Sgg9R1Hsev5Hke1cqshMnkEjHVGHIx1GPyyPTpWhBLnK9COQfT/61c9zSw27n8oBQ3Tgf0/UVz1/c+bIsYOQB29+RVvVZxl4xwd2Qaw4w9xOO2cD6ev61LRSNbTY2kjUn+I5z68YrsdPt/lMjDAJOKyra18lAFH3eAPb1rp7R9sXIx0wPr/k1cYgy2kONzegx+ozVmJM/Kf4ePwIp7KI4zu7/KfqcVWeYrIhBxuG38eK12M3qMlEcqmMj5gOPUjv+INcTqZlVVdPni6Er1A9/ofXpWzq98bZlnIJVz1U8g9Dj36GuR1CacS742EiMM8HHXr/APXHak5Iaiz6S+DEcFxcrJIvmIpAPGduf85/CvsSNdqhc5wK+RPgNbTQ3IkliYRzr8rg8Z/usOhB7e/evr4da+oy5fujwsa/3gtFFFd5xhRRRQAUUUUAFFFFAH//1v1SooooADVdzU5qEjNA0ZdxvZ1Re5r43+KmsR3/AI0mgR/N8giIEdAV6/lX1b4t1OTSdFvLyDiRUwh9Ce/4V8BJO02ozTM3JDZPU5bj86+fz+u4UVTXU9nJ6PNUc30NCRfNkXeu1FywB646rn8OTSw3qvMX+8ygE55HOcA/Tn8K5q7v55JHtI2O5s7sY4wevtj+ddlpmneXZeeY/nf5Uzxub/ADGfavjI3ex9Q7Jal6EhVE7jDRgYyPXkAe5PJ9AK5bVbzzSLbAKjkg5IznofXHX3OK6Zl224jwJJDwuf4mPBOPaqC2dnbbkuvmnds4HJwOQCR+Zq2m0kiU9blS0s5TEZ2By7ZCnP3iOB9AP1rOv1WAFbYF5B8u/OAMdST6gH+g55rsod08YSMgZByx45I5x7Ac5qlcWaNtULsRULAE9FB6kepNTKnpoNTszl4wtvbZmG/glFHRmHTPrjqauxFzmEFpJXKhznk5PCj69+nFMnidlyvWTAGP4RjilguoYsKAV3kksOCxOAcH0CjH8qiNky3qdAk5EaRbRtJ654wByc9TjoAKtyzqFZZBufLAY6Kq+g9s/ia5ZtR37REdqYU7sYAH8KqPp1PvV6G5nM0ixLltoLueQik7jge3Sqc29CeWw+e9uUCbAfmHyp/EAeMnPt+VQG9fzA8XzM0h2kDCJhQAefQVQvP9FtVSQtvmQ9OXIJzj9Rknr79KzL17vIiGUjVAu0DgEnJ57+5qLtbl2TO1iuUYQ2tucKr7mbrnnr79sev0rrdN1KPZ9jgP7mMEysx+ZznJGe1eQWlvfMrtDnfIoHmMcKoPUKOOg/8Ar12emxTpbpbI4CKRtVRwTnr79uTXRQrO+hlVpq2p6jLqEnklZHw8oG9h2LZyB7jgVLYy29vZM8SgeXuCb/4mPVm9h2HfFc1FbyyxI7El1J69Pr/npU5LyARLl9pCgLzluw/+vXpRryvexwunG1rm2dVAgKQ5+UHDHqWbqT7/AKCsvUZ3bZCSBtUFs+p/r/KoZoHZ3VTtWP7xzxgcY/E1QuJvPkB24Qc89x7moq1ZWsx04RvdGnaxtcr56oEjOFjz0LtnH1wAWPtVm2nt8stq26OIDLf32PTn3PSsf+0Zm0/ywuC25Y/UAjBP5VNprpZwurbdiOrMW/vLyB74ohUimrDlB2dzq/kWF5CQMFQCe+DgHHu2aQSqzrDHglmBPPY8/wCNcuL1JLcRS5QyEM5b72wZIH/1qjtb9o5zJ/E4KAemR8v6Vr9ZjdGToM6uO8jmSKVcETHBHTA6kn+VPupeMSYPmfkMDIH5VhCSFYWOPlAYD8v61RlvLweTIpIiUsT65/yK2WISWpk6V3oRXsc0Eu9VLRIRx3Hf/P5Vm38VygF3Zt+8jZWP0U8HHt0b6V11xJa3Ba5YABMCVewyN2f89Olc7KS6O8BIO3I56qzdPwrKtTT2ZpTk0efeMLCPUtLuNVRAxAzg9QU7H8q818Ms0ql4GZWlOXkPDY9F/uqB+J4r2i4T7MZCU/dy5WQEZAz3x6fyzXiWtTyaTqbadFE1tEAMyDI75AReS7H1GAPrWVJ3TizaXdHoj3iRx+VbsoYfLz1H19D/AJNYt7LcSkQQt5ca4y2cM5Hv1IH5VQhmuZFVreHyxx97AWMe5/vevXmtm20ia5kLvk5PJOcfmf1NcVZNvQ3ptJamRFZzSzYtkDtGdxbGVBHTjufTPGeea3bK1ugGWbfkkHJP4hf6101paxwxbLYHpnIGBx6Z6D3x9K0RC0KeacZwQFXls92J7eg/WiNBilW7EGhWc1vJNLF8rSNhSDymeD9WOTn0H416PYSwwZYHcvRD7kH8MccewrzaCVYoWmd8x4UDacAAnoPx49WJrQ+3CEQ2ksoZl3ySspyAwjPyj/dzj8a9HDT5EclaLmeiSt/oyyOCHiVX/Niao3bK0M0AyGQhk+vOf0qpLdC7W9dCVjG2NQfSMDJ/GoNUulY+dGflkAHP/XPNd0pqxx8juUZGDgM5IV8qw9GXn/64rjNfEnAb/WgbkYcZ9wR69CP/ANVdW10LmzMK/wCtfBH++oyP++hkfUVyl5cQzRspJ2scjjO1uh4/mP61zzaaNIo8zllEExEiFCT0x0J9D0wfb8qu292q+YSe24Z/M1X1RiZTC2OvI7Y9s9K566uGtpvLz/D09ewP58VzpGhqXLreSbx0OVP+NadhbgOZX4xyfqOlZFgFReeg5APqa0hOq/KxwOp9lHrVD1OqjnTaMfxevpV23uVkdN3CsR+AriTqG8sxyEXgDuSeP0FbunSljHn+EZP480ORXKb818JVC55wx/HNUJ5/PdihwGyR7EHP9aqqGwhHpn6nNV7wrBJlchTnGOxHFZOoWoGbfXsnzWspyj/MCPUe/oR+VZxleaIwyY3fwtjg46Ejsfcfypt8sm5ZegHHHTP9Pas64dIxGYiY2yMAZ9eeO2etFN800E1aJ+g/wg0K40zw/byMNpZRuRuV5AIIPb9a9wHIzjFcF8OruSfwrYNKyyYjUFl6jA/iHY+9d9X3GHilTSifKV23N3CkHNKaQdK3MRaKKKACiiigAooooA//1/1SooooAYTTadjNBGFOaBnhHxx1VrDw21rG21ZDlznknso/rXxg1xJp2mGcAfabkkrnny1PAbHrjkfnXvHx/wBbe51+00KIfu4hu25+8fc9h6+1fO9zcJNflpXLwqAWPYkfyB9PQV8Zn9bmrqC6H1GUU+WlzPqWNDtna5UPgSSHccjJUfwj6mvVpZ4ra38lGBIHl8/Tnp6dSfavPNH8yBmvSAGZSUyfuqemffHPtx3rRaWXy9jjaihVYA9R94qPrkZrwVLl2PXkr7kt3rEdtjyk3u5CRgnHA64+p/rUOnXLXJd3beSRuI/vMcsfZcDA9etcld3b3Fw/lgliAcAYAXoAD2J6k9667RrVoIIpWyUDBpDxyegA9R247A0J9ymtDtlnWO3DOAmW4HUknGc5/ICqJLCRpJyP3nLE/wAIA2qPzJqja3cd1qNsX/gaSVh14Rfkz+eT7mpZbuGByZEMiQhQqn+OTgj68nJHtitr30MbWZBNZgyuu0tsBA4+9k9Pof5CsR7VWlCA7jHkEnnp1zjgD2rqW/132CQ77mdcORxhn5cY68dKzlsJX85IGChyVD44GTxgdyOBzyfpWM6ZpGZk29vPdTLDChRCxZAerEdyccDp+H1ropWEDG3hbzJGHmSuOgA4AA9BwAO55NV5JIrSMx2/II2MSeXIH3c9vc9u3NWtNUCaTzHBlnG6RgOEQcAD0AHIpwp23FKdzPkjedTPtJ2ZIz94tnAH5+tNi0+UDCvkKQgZuckY3EDpgdB6mtuCW1mtmMI2RLn8FDf+hEn8z7VSM8mTg4YEAADhck4H5kAD2q3TiSpsox2ksl03XyIMrnA+YgZIHocnoK6uGOKxRFkLNK6gOOm0HovsO57msxw1nIHK7zDtBzzl3bKqPXnk464x0rMfVLmVnhQkNkK5OSSxyTz64H5mhOMA1kdJcalIVkMjeWnAIGOFHQe5YkYH+TswXxskG3AZQ2B3GAc8+p9f6V51FIzOrzsqlWDhScgFeh+gPJ/AVblvELRyGTbGpwQeyjk5xzlj1+oHrVQxLV2EqCeh3MlysNpGHbLSHlAPvORz+Cj+dULmQed5e8DqpOeODg49TngVzBv7ieRro5VdpEaD7/P8R9CevsBk9qt2qp59vNP8xR0AXsMf0HNN17oSpWN+2iCiTJ4QYG7JPvj/ABNVpZCw8vGQ5+cduPesyG8ODGxOWlHfAJAbOfpnP1+lS3NwnmLGMhQ29l/3xgZ/LJoc48ocruXXOZhcynIGFC98dPwFIkZNyEJ+UEAEeqgg/lUNu+yzM7H5o5CeehUghj+HatGOMH95H0jYkZ913Y+uRinFJ7EyYs1yY4ljGcyHn2CdMfzp5laSM4AwOWHvinSRq8zvngEY/wCBDPH65qjbPslltmPB4U/7eOv41V3ciysXfOW5SYKdjgE89c/56VTW6UwmL7syKQB2f/8AXUrnDjnG9Dg/SsogFjGemQfp7/nWilJbk2RqMEv7YkZLgc+vHr+HGa8/8a6Y13owvLSXyZrc7fMx8wU8YOf1/PNdUolsbhLtXG09cnGB6Z9+2atFEvFeaDarn5Xzna6443DkZ9+4rSMk/Uhq3oeGafC/2mKNZ2YRc4GQDjqc98nqRgdsmvRbMRnCTO0jAdAfkHru5/MZNec3Wl2WjavNEYZFnkbLLywUH/aJwAe2P1rsLWURJm7YliMKi4VVH48/pTlGzC522AY9u9lV+cZAaQ9hx0H41Vu7hncW1viOCPgkDqB/CvesOPUoU2oWwyjc5BJ2jsP/AK351iXmswxusikbE5AX7qqPfuSetZzmkrFRgy5q0txHGJYcq8cilVb7oK/dYj/YHI98Vk+HtQNvPDDeP5gVmMpP3cswO0Hjg9D7VQuNUdylpKTuciRyTngckD3HT/8AVWZO7xQqIVyzfIEPUmQ55H0JJ/AVjzO90bJaWZ6dp/iR5ILq5ly8jAr04LsQ5OPTofYGrn9ryTW8QZtwU5/EI2P0rzlp0hEwictlGjQ/3pSBuY/ifyGK07KVktWZmwscarj/AGgME/ka0+sPYh0kdQ99LHiROsWPxKc/qKx9UmKyvJCf9YN4HZhjkj39aoRXLbd8bff+cDryDz+YqK7ZY4yuSsed6Ec7fb6A1SqGbgjnLu9hkJD5BHKnqMHg/ke1cfLdmW/aOXrtx7HHU1b8QSzKJHYbCw+cDoT2YYrjtOvZbs+Y3EjfLz2rshHS5zdT022n/dCRuW71HLcMW2k4Xgn+n/1vzqDTYfNGF5X7o98dfzNbz6aZVJA/z3/wFZTvc2jZGXAXYgyYC9cfXn+QA/Gu0sl+cjuFBP1PJrn4rQ+YiuMAf0rq7GLAkbB52/zrJ6lstCFsKf7vFQ3dr5pKjvmumt7UMWDAY2qM0klgwY8dKHB2I5zj/sBeIxuu4Dhx3x2P/wBeud1GxEclq0fAjlQliOi56n1HrXpM8ZjKug57H19jXMaxcwWl9YBEyss444wMnpzXRh6dpJmNSbaZ+jXhSwt7fQrLywmREuGjPHIz19PY11NcF4DETaLH5TEhAFwccDqOhI6V3or7SlblVj5ip8TGsT0FOoorQgKKKKACiiigAooooA//0P1SooooAKrXtzFZ2kt3OwVIlLMT6CrNeHfHbxK2g+EpESXy2lBAA6k/0AqZyUYuTKhFykoo+LPiB4nl8QeNLzUpn3BiwXB6L0/lXNCdfKjXIDO2QB2Lcc/hz9K5eH7XfTlpPkErYLY6YOSf5VqySg3CxKPLCtgAd8cD8z/KvzvG1Pa1XM+3w1P2cFHsdwt0rWcbSchpMYJ5PPAHtxk+1Xp2e33SSkZAO845LuATgewrm7RlkuolBIitw7HsBjj+Z/Gtn7WWkJRdxDbUB5O9gAM/Qcn0xXGnobPczIjbLcorxlMk8E/xejY9O/oSRW3ealIGt7aBsmRjyOFCgYBGPTOBWNFFEsvmyYZmZsY5J56H3ycmqLXGdTjjhXccmJD/ALowSO2F6D3peha8zqLS9RpN8Z2Ql3iQDkkABdxPoOg/+tViS8D3CPjAjcHHXBI4HPoozn1Nc8R9kkgQnc5IbA+6mMYB9uhNTQoZZhCuWHzscHnAGM4HVjk/SmmJnT6VcPMW1O5yxI+X9e/5kmtG71Ke1AS3GBgFmA6Z7L6nqSeg/DFc/fzG3tYLW3AUxKFYHkBiw446gAfia43Vtakk1BbSGT91GhLdySTySfp/PFXzdBKNzsre4tzcwrt3gMQuD0Q7ecnux78cD0FR3msPAt3HFzPP930G5tqL+eSfYCsDS5fs8YuDlpZZG4zyS+cZ9gqfrgVo6UJb7ULmbZgA/KW9Bk5/X9fak5N7D5UtzYsbk2emJYRnLtJEu7/ZQ8n2zg/hV571BOZICdgkaU9hgfKuffAP0HNcvtMiXH2diQHUE923nPHoNo/WtWOyjEH2QNmRzCq54DBgpkz9cEfQ1UW3uS0jsCyJvknJULjaO+SpJJ96IIIluIncAKwMxXA7jcP8D7D3FZ+s3cD3EqdVKsxxnB3BT27cgCsObUZJdSzHtaONAqbgeqoO2cdc03JISi2jqZLONIVuJvlllIYDoQvXHsCSST7H2rLOmGaRLdkCR56kYxkjHHrjn6mpkkuLx4xdZ2KPm2nGWxnH0HTHrV9HkjKPMm+UsehyqySE5/CNM+2feqVOMkS5NMY0UUe1DGM+WSMHnaMd+5b196iljmaIuiYwVDNnG3OTxUEerCa5DodxJK7iflCsQCQBjJABA56kmqM3iBHvI7RT5ah8nsAu04+uOT9TQ1DYacjTmVUEZdNhyQqevXk/XPI+lRsXkjjuJeXI+YdFGD3Hes3V7qe4vrKNCyjylL9j8z5IA/vNxk9hW/qEcEaLA65YF3wvOAoAUH1JPQfnWcqd22ilKyQi3kf2RpOsS4Ue5PX8wM1b0++Ml3LA3y7hyR/f9/xFYUpit7gaI0gNyQp25yQ56E+mM1GrmG4cSEBZcLn0JySfwI4+tJOSYNJo6+S8hFu0+eARj229f51VmlaLy7xR8n3sjoCvOP6VzsF2JdNadASjFgV65wo4/L+talpdAQC0mOBK7AkjI3EEqfbpg1spX0Zk422Ogu4ooo4LiEAoPMI9GEg4H4HisQykFH/iUAk9yp/qD1o0m5a7sFguQQS6oy9vnXg47YYD86vJFHNHhRtdc5/E4P5E10O72MdtGT200N7C0SqGJ4x0B9QQensfWube1ezm8y3YgfdZTkEew7Z9qttDd2su6Pg+nrVuJhcEiT/WgZ2n+MegPqO3ftzSTUtHoxNcu2xxXjDThdWa6js882ylgwGSnHJx3x6H8DXkdvrl5cqFt3Aj45U8uR3PevosSQpMQBtLDH1/x/KvDtc0K2sdZla2McHmsTtQYJJ7+34HHpWsk3GwoPXUgWSaaIi4Jbn5ix2k8dAvp7mpGQPAqHnd6Hnj+XoK0rLRmMYZRgL05wo/Crn9jR7PNlDbUOFXHX3PTiuKUWdKkjh2kEkzNy7kY+U8AAjgH6/nV5b+OCK6l3bp3JHm54jyAoI/vEjp+dbsujSCM8bd3IHfHb86ypdA3XG5iWwMhQO/r7AetKLtuW2mW7gRyXgt41YRp5b/APAiMgfkMn8K2XikjBDcJLMxOfRlP6c020sJ2EcZfJyWdsevYfQCusTTvtMoLDCLg8dBk4xWnI3sZOSRhw6a5IyeoxnoAcZ/nV6ezZ8LjDMMqT0yR0/GurjsBGqk87k/Vev6VNPYEjC4II4Hr36+vpW1OizCdU+bvExnsdyFcBgec8A/3SO49K4TSPJWbzZMglvlQd/9709h/KvUfilaLFZkyOsco5Qn7sgHbgEgjv8AmO9eF6bdtCFJ+8xyr5GMexGc16VOPuq5yN6n0Lo8qR4jkByOo7fSu5ilikjCxpg968l8LTs4RZTkHoB29zXtNhGsiK2PkH60vZofMyibMNMvAHety0s8ruA69PpViaBBKsan5sZY/wBK1LeE4GemKj2SuPnZPDbhlZh69ParMsX7vdjoeauQqqBeOox+dQyOTGSpAZB+YFU4pEXbOXvVEQJGWA53YyMe4/nXmHiKC7l1nTJ7Z2VFnTzI15Df3SOD/SvSr+5CHkFS2dp9SPf1rz6/uEn1WxQxNkyg+ZEMY543YOefp1opu0kOS0P0l8ESWtxodvcQxiJ2RQ6jPUDrzzg12Ncv4OkSbQLWZTksgz65xzn+ddRX11L4EfOVfjYUUUVoZhRRRQAUUUUAFFFFAH//0f1SooooAa7BELHsM1+d/wAfPGr6z4m/seBibe1yzH+83b8B2r7116RlsZDvEaKCXJ9B2/Gvyu8b39vqnjS68pi0e85b+8c/1P6V5Wb1HHDtLqellcFKsm+hWtXjVo++O/b1P+fetaK2SYhsDcRvGemeQCfYZJrAtmaadGYck8enP/1s/nWtMzQQvsJ3uNue+M8Zr4SW59d0LNsRHuQPu83aQehCjnt69fyqaCaSCeRo1JkT7gbqWfjPHYcmsn7SBckwHBRgp9SVHGcfhWhHIliJr24P7x2bap/uouM/QDJ+posIa17HbRzzZLCF1iHr0y38jUtlZi41Se43FYrUCNRkAAjGSxPH3j19qwriaV7a1wuQoEhUf3+ijj1Y5+lGrahJaeHfsdjjzJpI42fOSxzhj+DE496IrUpnS395A8qxW/zLwMjku7Y2j6AAsT34JqxozCzt7m/mOVJ8vP8As9Sq+pPy5+prjZ5yT5YYgRy43DgdAMZ/2VAHuTXQzSmVdO0kfKqymR1z1IYEZ+pIqoq7Jehtagq2oMpfbID82MnDP2HuAPwrlU0+NJmLg5UF29yx+X8B2H0rU1rUIr5pmVv3dqyN3wWlUkn1PyhQM+tPZXuEMeQqSgHI45Bwx/ADNEo9UCl0C3t33xkEFXyFIGAONoA9TnOPWuuFsILW98k9I/KTHdyCCfw5NILO2fYsK4SKSNFPbaqEc/nioZL623/Z5B5ccAaVx03YB3A9gNxHHfmjSIXbMuS9huJ7extcAuY2l54xtx19l/x71uW95A08N0wAYqJE5HAC4UfiMn8RXExNu1CW4ZQpO9FXP3UwFXOPXGSfatu5cLClwxEawZVyenGTgD1wAB+FLm0G4khujLumY7Q0m0n1C9APQZ4Fbnh2w22wuXjUyIcKDzlupGT2Ud65yG3SdYrdv3cUCeY2WIVe+W7kKo/Emuim8QiOx3WgECuQIh0YRbQWfnoTnj25NOnFfFIVR/ZRrX11FAtyI3KeWQpkB+bAGW2D1B4Huc1WkmuJfMikPlgKECJ/AhA+QHuxY/Mx5OKbZ28V3YwJL+8MjGZ3P8UmcKB7Dv7CtRrdY9vIbYBhuox90cepJY/lW129EZaI5q+W00q03R8MoDKo7/X/AD04rmWjVpGuA2S3B7EEjJ698D9a27yYz3QjgG5pSfmbnbjHUfSoW06C3gnZ2LrGwV27l3+Yge+MZ/KsOVLVGil3MqfU7hLqC8kQmQtGx9Bx0H4DjtyPaug0nUJFV4bt9kkUSPnOfnwDJnPHB4HvXOTW7yyPcXHQ48lV/vKDk+nB4H5+lIlsB5Ue3ejxvuYchUQbv1YEZPX8aIKSKk0zes76OTXQkC+ZcJvJc4JC8Pk565zxn0rVvpbWOeG25MxBkIY8krDkqT6gHn3xXE2X/Et1Ge7blzEIxz/G4VyB9F/QVorEi3UNyGIdrUIGPVWkYHP1OefYV0X01Rk0bWniWGB1lGYbeUpxxk4BYkezAj6U+Kdpnls3JDofNUDnKruHX8SPwqCG5iksntmf5ZSW2+uOSfrt5NZVrPLJJNqVk20uZRtA+7tdXAAPHBP61FlZMfkdlYmOVkkjfi4UMvruTr/MGti3nlR2kbknGfr/AHh9R1rjtKuoHjubVV+zyAm5jx90K+DuUdge4/hOexFbegXjanpUjZ8m4gBVx1B2nKn8q1jHXQznsdmMXcQnYDcMh1759R/Os69tCuJYmIOePcjn8DVmzmItwxG18fgf/wBVX5AwifzADk546Eeo967VTUlqcTk4vQ42ZmmUtt+Ykkr79x7eo965fVRb3CeckoRo+WDg7l57j+Jc/lXeS2yyszQkFiM4PGR7/wCIrzzxOEClpFMbjgNggq2O+Oo9uhqYxa3K5kyrbyoApeUbc8eWDz9STn9BWst5Z8yZ34Hc9fw615O+sXlmTuEaK5x5o5T8Sc/TAwfUVs6detOi7JlfPBb7mPrx+nWsZuxvGNz0QOJ5PkVt2MklflXtx6YqUWaRjZMfLZz90AE8ep/X2+tcsk7EeRJPtRiOWJDN745yPTNdRaW+3JHTuTyx/wD11EbMJJofaiOOdSq52ZyegAHXPv8A44roLBbeQyMeDOQMf3QoJ/PGM1h3UQRPIiOS3b174H45P4VBayXVrdbTwNxkYnqA/B/SrjNReqJcbrQ6qzaO4td+SBuyufQ/Kf51N5iMfIz85LbNw4Yjqvsc5xWBBcSLEtpzhXGD7nIH4etaDL5kDNKfldt4P91ud35EHPtW0ZpmU4nnnxJ0T+0NGuHKkoqksDnKH+9x1HqD9eor4itb6W31l7aXywFbaHGBwPoQCT6Yr9GLx5ZrZ4F+aZVwAeQw7g+4/UV8HeLdO0+TxBcRW8c1nJG53ROqmPIPODwQD2P54ruoy0szmmrHqnhi4UbdnzF+2c/mew9a9v0icuidwOfqa+bvC96LVAiHDHgk/McfU5zntXvmg3ieSpkbJb05x7Z7n1o6jO4Qt5gYnLHOT710toEMXv0rmYdsuGU4ArXtZdqbl9P51LdgsbZJdNydCeRVK6Yqu6Pl0HHoR6Gn7iqK/Qd/w71BLOjofLP7wZxnv7fUfrWMmXFHF6jPtaSMqXhk54PIPYj0I6VxOs6Y1xf2ctqFLxzKfnO0k8dD0z9RXpNyY5laSNVD56fwn1+lcRqXmXM0FjHIqM8ihCWCkMDkDnrmoop86KqS90/RvwI27w5a7i28IMhsZB+o7V2dcn4LSVfD9qLmIwzKihx68da6w9K+zpfAj5ip8TEXpS0i9KWtCAooooAKKKKACiiigD//0v1SpG4BpaZKcRsfagDzP4i6pBpugXV5csFijQliemPp61+XcV2dR1e+1PaUjlclR/dXsPqRX2F+0b4rH2KLw3A/+vbMgB6qvavk6CCOO1YLgs5BwPzr5fPcWm1RXQ+iyjDtRdRmxY+UybeFI79hnr/hRqEq/ZnmVwFR8KSeuByfoOT9MetYAmNqArPwxHf+f61jeLdX+x6dDZr/AKwhnbHbd/kV81GDbse42Wbe5bzY51J27Wlz0ycBUH5ksaveW9/eSRSPnYPKBz8oLkFm/wCAiuUtbnyYo5JWztGFGdwbYM8/ienrTrTVGEQUcuwO8f7Z469uo/KtHBvYEzck1YXV1JKj5t7dnkC57RINn0GR3681k2089wIbE8rGYArdw7LuJ/EmsaAtboUOcSsFf18tF3H8xXQI/l7Jx8rTGS4Y46fKRHx6LxT5bBc1kZoYnSX5lbcR6LtJbk9Mkgk4rW0S5lWGS8lCs6xnb2wQck+5JIA+lZkNutzbHByzxsVPqCQrZ9sc+2TV95Y7VBIr7k8oSMByuA44H5k1CXQG7m1Zxq9w8IPzvKGYdz5MeAM+gOfzrQkuI/N+xKd0ShwD2+UrnPqSSfzrk7bUthW5jYiV45WlPTglRx6HkfhWva34tra4uJSG8uRNgbsqgZJ/4ESSKUkxI27fxFbyaemXILR4QHuyEhuPdhj8a4uzu7u+vZI3XfHIC8/spwVUe5PGff2qt4gAg1RZIZBHErM0Yz0V+qnnqD1p+h3kSRXLMdspjLDjum5v6jFJx3aNIvS522k2UjsLu7df3yY+Xpk7l49Bnv8AX2rpY7M305R8GKCTpj5flznPv0Jz9KxdPu3SK0W4yrlFJ4GMKDhc9M8kn6VbbWo/3KwKWW4YLtA6+azck98459Mj1p8q6GbbbK8qxXLywqS8SmSMr1BaNVzkDk8uBj1qhNb3VxqP2Bn3HBaR8AgNIfkUZ6gAZIxjp2rZ+1W6Myq3loTJukxhvmIJxj3LN74FW9Gt5dUu/tAh8mDeuQ3+tcLwqk9FGAMgcnuaI67Dlpqy60674ULbQAwxn+HoB9TzUV/qM3mBYxhArEDudufm/M4H0rQFtCLvzcZRd2COnHDHPoBwPc+1VUEZtZdWnOEYlIwByw6AD25zTcXFXIvzM5SW7fSza7TlicMO5PBOT+B/OqMl9MYIrN5CEhLtJIOhd1JY/wDAASP96m3cjteR3Z5RSRtBzg5I/kKuNYpbWqxXvMjNkj1Zzu2j6/yqL2jc0S1MdtQeWZ7qJNi4XA7IsascDtxwPc1clvbZL6CwkO3fbFpMtwFt4m2qPqQXPviqN0qLOlrbHLSMrk9eFy+OPr0+lZ1xayPercnKmENCPcurE/zxTjUG4GnaXCXG++lUCV2dVJ7B0WIMeewz+Aqnf6s4v47ja3lWjxSbXPJA8xGBA7NkN+NIllOyRKhO0Bdy56lDtP4nqfY1A2npNfxyTSfJMvmFjx8q/L09F71cat9EQ4pFB/FCQa2t0TmKQSMmOhZB5YPPQFR9KnPihdH1mO0i3fZ7gtMJABx5o8wfz21Cnhh7drm0nTe9uzKjYyAh52t6DOSD6HNSajpyafLFJqMXnCHy4HXjIDLlGHr1I/Cr5r6JC0Oo03W7K/WKJphDdqdqnoCrLkf8B5/MYrttE3Iwa3O12jZSPcHkfUEdK8ts/DsDnYrgLKSIZN3GCM4PoD+hq7aX2seHrqO51COQAsY5uhxJ/C/HYjr7j3q42vczl2R9AWdzujVJUAdS2cdyP5VorNjCKdw5wO+Pb3x0riNJ1cX1tHcnBZgQ5H98DqPY10FvMkn7gkAj7p9h/hXdCRwzjqRTuPvRkr7dseo9K57VTbXlttvMusnAdOcHHRh/hWvcSbZN0q7gThsdj/eH+etcvqQ8qQ7SEL9Tj5W9MgfdPuKYkeXXOjQx3heznG5uCwfIdRx8w9e3OeamsNKAUmSYMp6BGJGT9BTdVmMV2Xb5MnP3TgH13Lg89/et7QrwzgsJlx36Fh+LDdz7VlVp31NqdRo3bLT0tYwxYFvX73+fwzXQ20VwVCdEXgdO/c4qK3MWP3Sl5D69B7k1pweY2CwVQOeTx/ke1ZRhYtyuI8cdpt/5aMwIZj1wOw9KR1jYvI+GG0Ej3IyFH0x+lWHg3ZZk+VcDP64xVJnYHzScOBwOuFPBz6kmnJISGK0RjmLEAKQAewIBA/Dv+FRpfNZzRxFt+5nkK9iQPnx+h965W7vCjpbnJVhk4PocAfTqPxqlJdSDVLZyfLDuQP8AeBOD7Dt9KwVW2xr7M7o6pDcRp5a4IAAPXcPT345XP0r5q+M3hZpL1dbsGVknBZl3gNuHXhsZHrglh6GvfNOCyItxJGTFgElf4SOuPp/KqXirRNK8S6TcaNqkRaN18yKZGwSQOGBHRh3HX0rtoVWndnLVh0R8leFjj5rpGAQ8Andk/wDAe1e9eHpnnkTA4GMA9AP8a8gtNAbRbtre6kynARgCwkH++OvuDzXqmhzC3ZUyO24qOnooz3rsbT2MD2O0mXZheff+ZrZicIMgZ6AVxtpcB1VTxz0Hb/69dJZ/vThTjAz+NZSZSRfutQjjDFThh0B6GuevNUB+eDIzw6N1Vh2+nv2rVurVbiPyzgcYFY8lgdrEjLgYPcNx0PofQ1xVJSudEErGVcamzqZmyG/hZeuff1/EVzEl8mrajaKYhcAyqSUHTB6Fex9D0zWpqsU9pCbiHIUY+Vxxj/636VzmhW0kvjnTryx4LOpZVOOfY/41eGm3NJk1orlbR+pngsxt4etXhkMibAATnI46EHuK6usbQhB/ZsMkKeWHUEj3x7cVsZ+bFfcQVoo+Un8TFHSiijIqyQooHSigAooooAKKKKAP/9P9Uqq3zbLOZ+mFJq1WRr84ttFvJyMhImOPXApMaPzF+L2pzXnjGbzmJCdMemf85rzqXUBaxGXglS2APU0eLtZOoeJry7LbwHIHcYFcZdXfnk28WcKPmPqc9q+Exq568mfZYVctGKNSKSS5ZJbhslwAR+n51y2uXLXV6sjnKOyoM9wvzH8OK1ZLtYrctn1P0APHNYcJM88UzkMbdtuMZAAUkk/57VlFa3NmTzXM6L9kVczMNgyfu8bifrmt4RrBG1wSSHDvyeCfvk/gM1gI8cc0d8hD7D8xJ4JJJJPtkfjXQRt5+nxrL/HCwHQ4dwsfH6k+1OWiBFZMXNlLPOf3jbYlHQfOWzx9F/Wutgt1ubSdVPyrAGX/AGlWMqB9c1h26w3tx5MOQqYkBAzngKMZ69atS6iIXuGhUqQse1en316H6ECsHd6Is14d9gUgZg32fCY/2W2hzVKZZJbK4tl272jZG6922L+ozS27mRBdDLSThiS2OpAP9PyrI1C6ngjZEHzMoyR3yc5/E0kndDNm0lVyyyybVYR4VV3Eq2CSf0/KrzHz0l05fnWVVQufXIzx3OTXPRSMs1kD8rZAYHsvPBx6L+tdfdQrarAbNcyXCvlge4zhuenzHFTJPoVoUbqKTUYmIO8tIERvVSwUNjr1zmrUq+Vq0kKDYrqYuw+WX5RjHfpVmWwEbwshKqFyP++vu/gKzrmVxdy3yHcETeOORtbYmD2wcEDvQrX0Jvobeoa2I18lcERyLAORypIDMPrnAo0q9AS1vJG3C3QsPQswOP1ArktWjW3iBcAtGsarjHDjPOP724856YrZhmWGCJZ0VvLJkKk8DjgH0Axlj7YqHqrRNIpLVnY+H7aQx/2pfA/u0PlqTkAE4Vvd3JwB9T0r03Qre5CMj/uzHw5H3slTgD0Pf2+tcTokiu4ku222tgw27+styR97HpECMDH3iB2re1PxImn2UiQt++Q4X18xgfT06nNbRtFXZjO8nZF24kingNhZDCHcHbH8Kj7oz3J4H6964/XtRW5vILC1bMVquQqj70hyOvoP1NQ201/cWoijBXzfm3E8hen5H9fxrXt7BrYi3sU3Tycl/wCLHfk9Pr2rmnVlN2RrGEY6so6fpbxpEswwSRnPXHtRrZS92vEoI4RO3J+Tdn/dB/ya6me1t9Ph33MmXfnPbHqvfHbJ615vqWpCaby7RcwQKWyRhQTwB6kk/pR7OV7C9otxiX9vZa7bk5lW2LZf+HkYAP1I/Ks7U9RUXe2D5i0weNR6JkE5PryazRY31y7PduNvLAdBzxwPoTyaz7y7ZtSYWeWKjYG5xznOB7ZPJ+taKlshc/U6nT79WjIuAInilYnB6oMY/EdD7Yp11fW1i0V26iVMvbyoBygckhh+HB9a5fymhXy1YlFDE45LFzk/pjFWAsl1HOrghlKFieQu3g/XGce5FappENNlvRL+50a8NtfMZ7WeJV8wgnMeONw647eoqbxM7XCxwWh3hCV2Z3AqvIwe4Ixj0rOaAz2W1ly1uTHnPJjbp+IYfrWdPFeQ232mMMstowfj+JM7c/kcH6CplK70LUbbl2z1O5s5jcoBNaSkGSPuFI5A+ma9BsZYtQguLJ33q4XY5OcgD5c+9eaMFZ3YL8kygtjsT6egNdTprvpm4qTJE4GfXB/zilGre1yZwPRdGtWhRVU8SAblJx0GP5VsTyusBuBkSQnD49OxqlpTOpVh86EAnPUZ5z9Ks6g01tE00ZyrfLz1UnsfUGvQprS6OCo9R0t+l1CtxD82eGx2I9q5q71KEo0FywVSPlb6eo68Hr6VjSXr2lwQCE83GAeFPtnsfesLV9RlQNHeREg9HC8n8OQfcjrXRFXMmzC8QajNYy+TcRiWPkjnkr/eRx1x6HNO8P39tLN5Ts21cEDcBjPcZ5I/GuD1K9jlDRQuCgOdnPykdwDyM96zdK8RT2E4EbM7xnA3HsfqD3rSUNBKWp9X6O6XalwRj1Jzg/TpXRAlgP3iqMYyvJ+mfU+1ebeFdaW/VfPjXnAyWzz7gY/WvSLXy5NvlKqkd8/4f41ycupvcvGRXIt4eNh5P8/x7Vk3cayeZkggjPp1IwK1JNqxFd3AGT259K5LVr5lHlQ/Kq8uxz0P+eBWdSSS1Lpxu9DndRt4iBOTjYRjrxz1/M1mTKpkZsZxIMBuxAwcfnVs3quzgEhY0z9STgcdO9YwvHlFwYSciMHJ7kEgt9cYzXC0tzrSZ2djetaSOY2O1SSQvI2nk8ewOSPT6VfubiVAU2b4/vLkfKc84BHQ9fx9q4e0uZgI5weUO5+fXgkf7px+FdXGeEaCUb4+sZ6YPf02n34raE+hjONtTyrxfoKxXyavYwxrDNgOMnG/OckDgH0PFVbKRbd0EkgkbtsOV/Pua9d1ezj1DTZtjbJcFk29yozgZ4OOoHWvJLWyjvALpwN7DG8DYT67l6AjvivToybjqcU42bPQNJlWUoEPXv8AzruLZ9jAD2ya8x0w7ZlZcqq8fh613kcjSIET75APtn/9VEnYSR1cbJcJvUDzEGPY5rFvXuIV8+LGGGCvuO1WrKfCsHGM/wAhRqMZKOMbt/PpyOo/EVyzd9jaKs9TjdR1F2QzRIZIDw4HJj9eO4rn/DsVvD4u0+YOcbwQyjkjPtjJFXbxJbF9okOwk5OOqnsfpVHw8qw+PrJLfAB2bk59eCO3NZU5e8mFVWi0fqvo/OmwNuD5QHcBjPHWrFw5jKsKi0sqdPgKdNg6jHb0qxcR+ZGQOor79fCfJvclVty5pOajtzlMHtUhHNUIfRRRQIKKKKACiiigD//U/VKvO/i1qLaT8NfEGoocGCzlYH3xXoleQfH0kfBvxSR/z4yfzFZ1XaEn5F0leaR+R8ty0ss8khyXk+ZvyOPxNZ0LtFtixmQsTn0+XI/n+lYFzqDmWKPI65Hrk5/+tU8N0JSJmbhF5zxyBtxmvh2ru7Ps00lYdqFy5U26N8p3MT6gcD8OtQLcGO0ZtxDdzjkkkf061KHjuNsmMgDaR/suCP0JGaxLV5CZBIS2FXH1LA5PtTS0BsnivH+zgclpUBX2IPp/Ku9tJwbGKLIX94kYbHOeWJOexNcZDbwsZPKziJQqgf3Vdef1re06YXLjzgcRO0qgHGQi7Rj8aU0rBF6mvbmZLZFVS7yDA46bpFPGParIWRRNdxgne5PP90rkA/StVzFZ3drclduXGVz0UyADPpUN+/2dYooxtVy5b1ARgOfqAa5tb6G2ljbifc1pHE2SJZGOMD5UXCj8cnP0p4sVlHmTgMI4trY6/ujxx7g4+tZlrtj1GBN4KqyHg8DcDn8iK32uI4r3epULLA+4Y67cbv5VNgZnadaC4ypCmQKyrkd9uMfmQas2t4WuhHKwCBBuBOQGXaCPqzsSfQVEr/Z2eInO+aNx6jzcEj2wRgfWktWt4o5rqQHYzSsp4wcyfp93FOwrs3SZnaS0243F85PTORnH1X9azikiziGIbpZtuEHZf7x/E4H4ntV6W5iivZZQu0qDu56855/GqtgWea9vWbZPIzIhPAQ7eST6KvQfT1rOK6lXKD2yXBlu8HMPRe7Ox2oSPx4Hqa6+3tLe2htA43PJ8xjOAo2EAFvXBAPPfntVzTrC3ggWYR5OVZN3HEecOx7ckn16VB50t0d0eI4cZaZ+mOo49CTwo68E1LfYerNNNWkZEZUDbFEkeeDliTuOe5J3ewq7pmiTXzebdKJNmHZj0LEc59uAMdags9PLSB5ydijzCpA3tg43v6Adh6/jXZTW0klukfmG2Rh8qj7yg9WPq3ueB2pcrm/ITlylISRRE2tmPOuScE4woA7n0x2Fb8cX2OHfJ88koy24cuT0B9FHZe/8oYINO0wJ5KgEYCRk5I7739SeuPSuZ1jxMqhltjl2Jw7dz65/w6VqlGCuZ6ydiHWZIFR57iQFySSzHOcDsOgHpXDtOkQSGMGR3O47z0GPvEenpnr6VVvNSkvrsbnE5UYB6Rrj09frjH1po84/LYANITueRucehOf0H4msnUN40ieV4bdXSV2kmuCNzOcEL14XsMc+p+lV1shcrJJHiCDd87fdLL/BEo7DHLHqamgs5EbAJeVieW5Yk8l2z69v/wBVb+leFrm8YtKT5YJz689QPc9zS9q2U4RS1M+LS47kAQ58tQeV/iJ6tn25rp7Hww8kRgWPAZQSo6gZyAT2Pc13Wm+HpHWOKNAAhHAH5ZNej2vh62tIBA+C2Mt7nqSfr0FdNHB1KvoclXFRhseNjwrEcELiNRnH949vw9KyG8PvvZ3X5CCpHXKnr+Ve5XFjDzF0XHb+Z9sVnXNmokJiTauMc9Rgbj/hXQ8DYxWLufOraC1rKN4ym1s+69R+taFppA2CIfeJ43dMk525+vSvT302Oa4BXiN41U8dCfX8aYdJ+yfuZo96sQwI5zg9vcYrCODd9TSWJ6GPbWstqiSEbdgwp9j2I9P8in394Gt2C9CMPH3Hr+XUGt2aRY7c+W26PJOeuPUMO2K4y4kt5/nuNqgnC84yQDgj2/8A1V304cuhyTlzamLqFqHi8qZ1KHlXIypU/wB7vj17g153rE95pcR8+Iz2hyrRk7sEdCp6jjv3r0W4kNrF5hbzM8qGGM54wwz0/WvIfEt19qQ3Vk6hj8rJ82Vx1XjnHcccV0wRi2zzPWbuFroXNpIxQ8gH7yexB649azYpg8gkjTc/8SnGGB9ORTLx3LbJGUr2PQ59uKoLaNKRuGCOjY4/L/OK3toZ31PdfB94IWRbWOONu4RIy5/FmJH4CvoPSr6ZolaRdo9B6+7cZ/Cvj7w3BdBU5XOdy7wcbRnnAwSc9MntX0BoWoTK0UbsWJxztbn2BPb6VxVYWdzqhK6PWCkzrufKlugI5OO+OwFcdq9o5/1mfLJyRnBb/Ae/5V0MGqhiyq+EX7zDkgf09gOamkmt57YySjABPLdz2H0FcVanzLQ6aU3E8seGfZckHfI5AHoB/T0qGK3dEMEbY3KVPHYL/ia7iXT4PNZCwAwHY9Mgep9z/KiysIZEa5YD5lZ8ew46/SuT2bvY6PaKxy8VsSIpJQVJAGPXs36VryG3jlt5IQU4IUnkbckFGHdSQfp2rZ+zn5VH/PYEZ9MHANU5rPLrGV2iMuyg9wSSR+WDVqNkQ5XZLFDItpMjRllIBKA5OR02H1H8Pr0rg2v7G5M0kbKxJIPGCSPXOMN6g9/evQVjENtucHf/AB55yh6Njvjow/EV4p4mSex1czQcSORz13D2PRh2zwR0I7120HbQ5KivqdlpjwsS78DAAFdvaQvEolPJwT9Djj9K80055JTGZeApBI9T2r0/T5fPwgO1exPfHGfzraaIRupCJF2qORyfpVhXUw7cElevsRVOW5eI+YvPHT168fSmm7QvHLG2C/4cjqDXJKSTNlFtGFrVisTtNgPE/wAxzxj1B/oa8+Se1svENvOMgxMGUcH6hT1+v516zMgu9275WyTtbnOeM5rktY8NPdQO1uA04Gfu9x2ot1iRJdz9GPA+u22ueH7W5t3MnyAHI5Bx0NdlX55/Cb4vap4bePRNSLJbg7cDBxjjo3TH0r7w0HXLTXbFLu1fepHXj+nFfX5fjoV4K2587isNKlLyNZEKOfQ1NRRXoHIFFFFABRRRQAUUUUAf/9X9Uq8b/aDbb8F/FTeli/8AMV7JXjP7QpA+C3iokZH2J/8A0Jayr/w5ejNaH8SPqj8UsCQrLIfmUA/iSRip8bLfyhhi6g8fUnP51Ru38t/KTAKqf++c8fnVtd7SxIBwF3Z/2ck18a1pc+uuR29xNAquvXZlh7delWIpRFYGZBwqsCMdR24HYd/bmrgtle1N+gyNw+UdcY5A/TApjaf5ZXY/7iTmNhxtJ5APqCOtTdFalewjuokkmQn5TxnoykrnB/CtXTrpV1eFlb92QUfP4ZFXk024miTzLcxNCpIAzjk8nA459OhrGn05/PLw5R0JGMHn2/D1pOSlcdmjsYpYr3TILwvvH2grIQOQFYn9c9PaujlW3nvD90O4QbegG8vjI9Mr+deSaffLpbm2mJ8mQZYA8Z6E/hXbvq9rcR+bnfIVEcmP4thZsj3OeD+FRODQ4yudXeMtteRiMAEL3wORkjH4YP1qJpViuVWTc4wVB7ASD5vxxVbSryPVmtpdwd1lABYfeGNvPueDUrzi/tUGQsmGbcO7RZX+WK57NaM0Kj30iDznJLufvAcZRWK/pinm5OyLTnyy8NJtHqN5/NjwKz2YXVq4UZY4Ck56E7SR9eKcJZo7tDAuZpnGAOTgKvT36VdgTOwKyraz3cimS4uCcAYyOQQMdskfgDzXSaTFIfLmm24KlwM/u+T8zt3OSOPUDsKyra1iESQuwuFi4ZMnaWPPzFevPYdQOa0w5urd443Hlg5lcDnC8HGOFHYKDwcDrnHNJ32LSNqa4W9Tz3DGCU+XEucM+eCwA6A+vYZNWYLCOe8hMhyqkCBF5GQOSFGee+egA9ScULC0mvZkvZGG2TKxogx+7HBwD64wSeMZ9a72wFzGGby0abZkAfcRe248Z9amKuwbsixFbQQEGbCow3FM5Z2HTd7DtnjNc9f6xcTXAgsUM0jn5mXn6AHv7f5Nbi6feXqO17JuQ85xgN6HHH4cAeg71UurMW8vl24AB4GBzz1J/wD1VUlLoRFrqc/9pngWUykPI24MYwX6dVT15+8x69uK5S8Fxejy3QgHjYME8diegHsK9JGiLy052IqjgsRwPXp+VPRNPjP7mJVCg/M3T9OT61m6cpPU1VRLZHnWn+GbuWZfMGQecdh9a7W18PMh+zRDJH3sjoTxkj19P8K6KxmtlbI+Zx0JHAJ/uj+prrrCS3tcNKBtJzxxz7+taU8OpPVkVK8ktjE0zwT5Cq8ikE8lm659fqa66LSIodsMfCL1an3GvWyg7+/YnAH9a5e+8SJIfLj+c9hnAPue4/nXdy0KW2px3rVNz0OK40+zj+VgdvQDnJ96z5NZWY+UgJd+MdT789gPWvN5dZ3rs44HRDxUqakI0I4HHCjAX8T1NU8x6RWgvqS3bPQjNbpEsSuGcnc2Oe/c1TnvYmDRnJ3H/Dqa4GXVJDkLJ7nbwPpmqEsl9McE4UfhWU8fJ9Co4SK3Z293qNtGdqEFj06ADH865+fVVjk2EmUEk4XsT1xWG6+UdrSFn7nqfpjtVSdoo8Ce4K54CqOSPr/hXNLGT6HRHDw6k9xLNcv8+1dx+5nOf+Ajr+NS2+l2hYXT7rmZOuQNoIHGB0znv0HvUUKQKwZSF9MAkk+5NbkYZ49rvtxyBwf06fnRTru+rCdJWsjnbrToLuLM581yNohT7gP0GWJPqTXnOtfDzxBqkjl4QIW4WMLl8EdCAR17biSO1e8W6nAKOeOpOOf6Vu2ckFqdxXczdSzdz6mu+niPOxxTpeR8qaL8ANXu5Wn1XFpGMbUzuP0OOp/r7c16Db/A1rpUtVZY4cAMAPuqOdoI5JY9eg/SveZNajQ7FCn1OcD8+v5Cta01NXA+6sajnA2j9eTXdCtGT1kc0qUktjxuL4HWX2mCeZlEEPzGJBjewGF3NnoO/b0ro7L4QEv5z3HkhioO3OAi8iNM4wO2a9Q/t21UgMwZh0Hf8AOaB4hkQb3URgev+Fb3ofakZWrdEc7F8NrK2hWKGRlAOdoAyx7ZJ6AVg6v4I1wCQ2Xluq/6pCcRr7k8s5z24Fd6/iyBT1OT0zirdtrsVyQQoH15P61MoYWekWUpYiOrR4jN4RvLWHydTnEhUjcFGNxPOSc9KqRhVkcOAEhUcZGBgbjnHHGRxX0JKthcp/pWHJ4wB2rAl0DSbkNHDGFUggjGcA9a554JL4WbQxT+0jxW1xHCgky7Fywz/ECev5VZlSM3L2znEgODgjqqgEjPrmu81bw2Qmy2cABSucc47fl6Vx82iXE088sCkb2LbyeVyAD16dK5J0nDc6I1FLUwNRDhGlU+Y6jeFHXvnb69OR7e9eRfEPTL24toNX0xC7j/AFkQIOR1yAepx6c17e2mXLq8dz8qE5Djs3UEZ9/0rjvF+nvd6JLFZQEycNHngsQckexz0rKLd7oqSVjz7wdcNf2+Y2yMZAJzj25r0uySeGXDLhX4/P8A+vXE+ErN7Z1ab/XPyVIGRnn5sd8/j617Db2nmxZcZ9a73HmV0cvNysekalFY/dyAR1wG/wAKnTTlGQw8xcg5A9uD9Oa07ezCgbujZB9eO9b9npUkaCUEg4xx3Fc0sO5PYtV7I56DTAi72jyg564P+cfnWtb6bFcA8Bh78EfiK7uzsIgRsXAPr1rYaOKGMhVHFdlHAdWznqYq58teLfCLWeoC9tElR+zA7xntjGK9H8BeNPE3hZzbsxlRSN4PIAIJxknrivQJdOguJlaQYwc8U5tA014fJCYBbJPcnjP54xURwNSFV1KMrEyrxlHlmrnpWm/FuzneGK6h2hiA7dME4HA5J616np2q2WrQmezYsgOMkEV8oTeG5Ul8+ybYR6/pj0re8PazeeH7qNbmVtmegJwc9eK9Wjj60HautO5wVMNBq9M+o6KzNL1ODU7VZ4WzkVp17UZJq6PPatowooopiCiiigD/1v1Srxz9oLj4L+KiBkixc4+hBr2OvL/jVaLf/CnxLaMM+ZYyjHA5xx1rOsr05LyZpRdqkX5n4fW/lvdLLL8yr27ggZP1Hep7e4WJ0k++A3U9MZzj6dawZfPW9ltYx843A/8AAsD+ldHaxJCEjkAIjHz55ycZx+Zr42pHl0Z9dF3JFfz5HhhGFmBxj+FwN6n9BWojefaNCNskUpyQQdyE8gkDPGc8de4qhZ2Qe7FypLLHkAD0xW6lmj+VDEm1Ao3Y7+hz7Vi2tC0SadPqMUgtJJQyx/wBjkbufXkd+n4V22mCAkiSM3Ibgo4BAI9GIzx61hraQXMsHk5WZMqWH4A59a2BEkcmyMk7BknHUDgcjnJzWFWz1RcXbRlhtD0K/JTULUiM4OxJBkn1BOCPyrnZ/AGjeZ52l6jLCWHCSx7h6jDLj+VdfAkMgLs3mIFB5IJDdx0zUrFJCHiBOe5JA7Vkqko7S/r5lcqfQ5G18Gazpd3Hdx3UE8W8OVVip+XB6MPr3qWWwu7HVRI8OLfngNuJDFt3T/ezXfW9iZl3zTHccfcJ/XrWkdP06Ebxh5TxluT6cU3Xn9odo9DyeHSdb8tVaB23EEEj+FenA6V02m6Bd3eoJJEjMseN7DIy2OEBIHy55bua7FruNX8yQbm6Y+n9KedanCBY3Ck8A9Mf59qTrtiUexBcaLKqrbuCqkY2qwRQc9vTPcmpbexjRIracgQxsT5a/KjEdPcgc/jVEyfaD5c8o685yRXRWNva5Qt8zZySf8Kwu3sabbnQaZYqkKQxEIzAZYLwB6DP5V2ELW0Uaw26F1U9CcjPUliepNcNJqHkE7NpwMcNk/jzTv7VncEOfoq9fzrZTsjJxvqdlc3sa53EMzdk6+wrLkv1gH3PLLAcdT7+/wCPFcjLrEyt5UeUPY5/wrImkklBxOArehLN9KmVRvYqMF1N6810SziBGL7cZwPu+1RRJd3Tgs2xRxwMEfnWZCFUbly+3uy8/hWlHtX19ueTWDTb1NVJLY6i3MNquAwPbAI/VqdNqT/xFWx0VTx+J71ySXcAO1gMD8cmrS38Eg8uOFmZhj0H51abWxD13NiWa6u1DMQF7Y4x9arpbzZx8oHODjqfw5rMIULhgR7E8D6D1q4sicKXKj39u2KVrjv2NWGMKB8vTj5jgA/QVYWO3iHmTMu7/PassOF+dn2r2z1/AU19RtoiRbx4duNz9fwFV6CNQzRb1bGMdA3r9BVSaUuhZnwPZSf04zVSO6IBctuYdh/XGKryXdxdsVMhKjsSF/QVEikicTRoCqxsxzyWJBP4DpVhYc/vpESIn2x+p5rKlaS3BcThAPTB4/U1mnUo92ZZDMc8E9PyrNRbKudMRHIco+BnqoPb61rRzY+RTwefTmuNi1GSYgIMk/7PC+/Na9vjcGZ/mPGTWqViZHWwSDfkjfj8qu795ywXHYDnH9K5+3uY412liR7nithbyJQGJA710RMJJ3NSINkFVUe5GavMHcgTMX9c/KKwlv4ACzMTjnIzmpP7UjXBjJx/nrzWimkZ8rZuc4IjTBxg46VQl39T87f7RrMn1CaZQMkp9MVX+2vs2qAAevGTUTqI0jBmv8ztiUggdhWlb7jINik+3QVz8M2Puvitm3llTBGR74p0pK9yaiZ1kMT4ElywQdQBTZbyJAY43z2wOlYnnM6hnfOR3OP5VA0oiyHIVR/dGc/jXofWbKyOP2OupqNcN/rJnUAeo/pWJfatbEeWqbyP73SoJZIuGKs/sScfpWXLP2ZAg6cdf0rmqYh2NoUVco3N09xIHmJ29Aq/KPz4rMuWuLpCkZ27uMjqR+PNX55ogMGJn7DIP8qqmSeRsEGNe+BjP9a43Wex0qmjMs/D8UMomkcK3HCjHPue5rt7GGKE5B4HU9Kz7aNEUGNcY7t1/CrouEAwRlfbmuqlWkupz1Ka6HTWos5G3cKRzjsK3I5FTJWQdOB6ivPGuyn3F2g+poGsSRLsTDdhurtp4xLRnLPD3PVYb2MDAOW9+gq7HJHIclgTXi8OszRuFkfZnoByTXoGj6q8kfKkZ9RjNdtDFqbsYVcM46nSSxjdlTzU1uw6MaqyXJY7UGTjJqSF8oG6HvxxXVfXQ5nF2L7EYOKxdSiW4iKggHGcj1rSdg6lT0PXPFcbqOqraylFk46gdf1oqTVtRRi+h6F8L9anjv30qc5x0PPSvoavl34ZW8t34me8j+4ByexzX1EK9DAfwjhxNufQKKKK7TnCiiigD//X/VKuQ8f6Z/bPgvV9LyV+0W0i5HXpmuvqG4hW4geFxlXBBH1pSV00OLs7n4A6lpwsvEN9A4xIJSvPXjPJHXvVqCZZP3MSYXGGx3yDyfYYr3H9pzwNJ4G8fPeWybLbUMshAwAW4NeGW9s9pAtx91iMqOvOeBz6ivj8dTcarTPrMLUUqakjZ0nTpAxkAyvJZugBAzgew5rYHlvEwUjywkZcng4PJHsSOB9aoreqBNAgPlsTgE43LjYpP86rtfItvIjrhskBTwSwwV/HA6VyWbZvfQ6OG4TTwbfjdJtdTjhWBzz9ScGpbeeZ53VGG4/KRkY69B9a5qFQtv5spLMV+XPTgir5litysiAhjt69ee34VnKOpSZ0LMkUxtZQzdNuOmDgg4610cVuBB8xLKoJ+XjBrmXn8yHzIhhhjPfp/wDrq8t8YAGlIjfoR1BFQ43RV3c6aO+8tNsYHAHf171VuLxZQ23IbHXoM/5Fc5JJaeZINgBZQcgHngHp6Y9OlVVuITMPsxbAPOG4/X0qOW4zVa7lUjfISBg9OxqaO4jeTzcbT27k1T81Ng4UZ7E5+nT1pSkojWQgBccruHT881PIUpG9HfAv8iKoXuSTmpVuJT8rSAhv7prmG+0OoSKNRk565yP8KmiWQRn7VlVXqVPH0470vZhzHQx6nDbEnkdvUVeTVJZFV49wQewwf/11yi3ljHhVXhh/FjJFLJqSqmI3VcdB2/Ok4Bc64XoZ8yKQSTnIH6VIt3bQjIP3fTA/+vXCJqTNyVJ9cEf061YGoMjfMDk8cYx+gzS9kx3R38N9CQCZME9Tgk/nUjanbRAbQGHr3rikuJHwJHAJ5/8A15NS/b1hTfwcd6OUR2a6nG4yi49SQBikF3CxwhJyeey5/ma4OTUVY7mcAEZGCP8AGlS83/efjHqM80nF7jPRPMjAzv59sf8A16WG7tlP7xgceuTXCR3RZf3Qb645/WpV2kBpGOO45FS7oDuzqNqcsWC546c1Vl1G2hDFFyT0PGa4xrqONti9O1PSZCPl+VuvJGam9xnSvqyldhGB3xms6410QKVjXGfQdKz3fgbiGqi6zyHjhT69P0qlFMLkr65JM7eWjOT9MUxL2/3F1Qfj2/OodgQ4Y/MPSpY3QY3NjPbg0+XsPmLi6hdqCFySeflx/wDrq3DqV3ECX+UD1z3qAW/AZhuB6YIBpj28VwAIXBkB6NhsD6ip0KOis/EAjX55GP8Ad2LnP410MOvKyhihbjqxAFcFbRzwyhJpYzjtgqRWoJNp4dsY5xhhSvqJo7FNTVmJfYR6A/8A1qurrCkeWsDMO3GP5DNcdFJKy/6MRx0IVc8/72BU0l1eRcs0iknuVQZHrxg/nRcVjrptTm+UGPA6/e5/KrC30zfcjCAj+IiuPN1qBUtFIip33gH9RTre8Df8fLRPj+6x+lQyrHcR3B7kJtrVgutxUPnB6NXnkWr2aS7bhmxnjcrEfz/lWxDqFpMP9HZsdwoxn86uLa1IkrneC5k2hI2xg/Tj61SknmQ5YEA8jvWAl/BkRsyoexk5/rUxvHWM4kVsHtjj861U2zHlsWzPMQww34n+lZk7XjtySqnsWx+maZJeyAAiSPB/vc/qKozaiYifLn2n0HNS9S0TSRXGcMWcjryB/I1ArPFjcEUn/aOayJ9RkUbmkaQZ5UDms06lAmCylD7kfyqNizv4bg52uQTWgLlCu0/L+GR/OvObfXF3EBRjOck10Ca5A0eFaJT7kg1pGbInFHQyMjNtLY49OazpWAPy7vr2rObU1wFUJIT3XJ5/Gq5vJpSRJiMLk5Jxn0Ap8zZnYeshSUskjBie3+Sa2rLWZ7Z8yM2c8ev65rAaQsudzAew4/oKbvA+XnH+fQYq4VGtglFPc9as/ElvjPlszE9WfJ/GurtdajLkS/LkDvxzXz7BqHkTB0/x4rqNOuNS1Fxb2sbZkPPHbHAr1sNipy0POxFKMdT03UfEtrCTAP3jP8uBjj61zuleH9R8S3iiKErEWwWxzzXp/gv4UO6C81TguQ3PX8q9/wBN0XT9KiEVpEFx7V7FHBTqe9U0R5VTEqOkDE8I+F7fw9YLEq/P3Peuxoor2IxUVZHA227sKKKKoQUUUUAf/9D9UqKKKAPnn9oj4Uw/EjwkWt0zfWJ8yL1bHVfxr8mfFVje6JEbG8iZJ4iTgg5O1gv5cgV+9bAMpU96+Zviv8APD/j2GSaNBbXjDAdR755/GvLzDA+1anHdHpYHGezThLY/JqCYSWKo3zTI5Vj6gjgfgazDetJqCWrnd5hYpn+HCZBz+le1fEL4E+LPhvGXmjNzaqeJUGc5Bzmvne5tb4TRXKIwMZGTj2wa8T2Eoyakj2Y1YyinFndQ322KJD/BuBHXryPzxTGuLiRhbt8zOQ6joc+lc3pmpKE/0kFXQ4II64/zmrEVys80ckb5aMlg305IP9K53Bpu6Nk9NDsrfU4oZWKEKdpYAZPAPTHr7VrRahbupud2Y2IBZSCFz1yOo59RXmsupGNwMbQQuJPQj1/HvVoX4RpJG3bm5zjG4H1I61Dplcx3+54iAx3AsCCxIyPY017v7OC5KsoPJ749+K89fXWiKkDaoPKqSB+PX9K0INfNxlh5ZUfwhiGx9TU+yktWiudHVf2las48qUKT74B/Gte2myMRzEP1ySCvv24riIr63Zy0bKsigBlkOAR9eKb/AGtFHmGF1icHgbgQefUVEoN6IpSXU7WUynEhl74ABP8A7LU0eo2JRUuA+TwME446mvOl8RxxTNs+fP3ieuavf8JBp6hsAoXGQEUde+eafs5W1QcyPQBc2wLISVA5yeAR6ZbOaadRsY8fZ3B56bg3Hvj+Veet4p3bfP8AMeLjIJ5H0yTSW/iGzdtq26uOTlmIP5YIpOi+qFzpHof9rSA5MZfkngHgfhUT6tJCChhYAdCwOOffiuIutTec4hLc+j8D8gKgiuyx2zXGW6D5iRSVKw+c9FjmaUBt+09gDgAe+etWfnXAllOScfKw/lXn0WoR28nliVdx7gn+taX9oJtCySFj2OeB+FTKmxqR1StaqxKFsdATg9OtPaexOTuxtPb/AOt0rlXuFlVkTDE9BnrUAE2Vbyh0weDx9fSjl8xnapqD5Bi+YDv0/LPWpDdSTLtCdfVsGubs5TjMjjI471ea/YfK649Mc1MoAmaQWbhpdpK9+OB+FT+dOPmRxtPXoOn05rHa+i7ptI7H+dVzqUSy7GXgdSTxmlyeQXOnS6hT5SM5Hfn+dRTakqrtBA9Oa546jbsTFCp3MBznIGaoy3F1uIVPlwME9PzoVNBc6Zb63cYDfMcA5IAFSC+to3G9iM9cc4/IVgxXgMYLOmO6HOT9DiqrXwk3LHblj65/wFPkC51s98GTdG7H0HP+FRxX1xHh88n14P4HFc4t/LCuHmEeR0bqAPoahfWE3ZFyoJ4KhSf1o9n5Dudn/beonBLgYOPm6/1FSnWJh805Td6jGfxxXHQ6orPsYsxzjdWit3agExn5/TOefyqPZLsPmOmXXkBPJJ6DDZp8ut3ZAUlhkj+70/Ec1zpnkmwOwHHHf1qXzJIRtRGwR1PPP5CmqC7Cc/M1I9buoVMkJA2/3cH/AL6UH9RUkOuXskmFSN1b0wck+hIFcxJvkOZTGGx/HHz14w4pht3ucg7o89dihl/kDVfVm+hPt0up6BHfqoG9fLbGSudp49MjFaVlrthGAfOYHvgq2Pwry7yxb/u3LMB35x+I5qazaWR1jtI5N3YKN/P8/wA6f1GT2RLxUV1PZI/EUAO77a21uoKD8+DUp8T2A5EzSf7WzI/ka5XSvBvjzVF82y0maRGGQ5h2gn6k13OnfCD4l3i4a3MB9WwP5dvrVxy6p2/r7jCWNprdmTJ4hsZEMIbzR1yu0Y/A4rOkvPtGBZY3H1cKRj0r06P9njx+xBkmjIPrj8jxWsf2d/GyDe0ylem09PqK2WVVukTJ5hS7ngMguA2XkZGPZmyP0rJkYQvv8yMA8nLMT/8ArzXr+pfs8/EVZG8kRyIxzggdPpjBxXAX3wJ+Kcc5SLTiRnIZTjp70/7NrdUWsfSf2jn49StY5Pn4IPVGx/OurtdZteiStk9ScHFSx/Az4o3EqLDp+MgYD+/XkV7l4U/Zu8QPGr66UiY8sE5qo5VVl9kynj6a+0ePLd+fzHK2T6qDVi3t9Qdyke2XIz93JODX2FpnwH0q2UC5O84xXY6X8JtCsJA5UEDtiuuGR1Hucsszj0Pie20LXLqQi3tnOeflXH867HS/hZ4k1Jl86GRPqCK+6LPQdLsVCwQKMe1aqxxr91QK7qeQ0lrNnPPNqj+FHy1oHwRkQA3vy+tez6D8P9J0bayxgsOenevQaK9OjgqVL4UcNXE1KnxMRVCgKowBS0UV1nOFFFFABRRRQAUUUUAf/9H9UqKKKADg0wpn3oIpvzDpQBkar4e0vWrZ7PUYVmifqrDIry68+Avw5uIXh/s2Nd/cCvaPMI608MGqZQi90XGclsz8/wDxj+xZYanetNot2YImOdp7fSvFNe/Y58aaD582iSC7UrwOhz61+tuKjKVzywdJq1jojjaq1ufgb4g+Hvjrw3dtDeafLHtznKEj+VcM5urYtDfRtGDx3ytf0N3ejaXff8flrHLn+8oNeaeIPgV8MvEhZtQ0iIM3UoNtcs8rh9lnVHM39pH4QvdeTIyqdwHI3D+tU3vJGIKKAtfs9c/sffCOdy32Z1B7A9K5PUf2H/hrcvusppoAeozms1lzRr/aUD8ivtby/I+5x79q0YU81PLjBTnsOtfrRb/sSfDmJQr3EzevNb1v+x78OLZQkbSDHc8mn/Z7sH9oQPx9XTXceWqn8Rmta20O9YgQKxPptzX68W/7JHw6iffNJPIPTgV19j+z14E00AWlvnH98Amj+z5Pdk/2jHofj5D4e1GXbuteFAGSOv8AKmy+G5Oph2HPPHFftAfgv4ObBaziDAYyFAzVab4HeC5wQ9qv5VKyrXcf9prsfjbF4cuQCqoTnBPbn8qVvDl87hljIx6f/qr9hW+AHgY9LfGfSq8n7Png1j8qbah5U+jGszXY/ISfw7fMf9W3H1/woi0e+i5EbFvfp+NfruP2evBw6x5pp/Z68HdoRS/sp2sx/wBpo/Ia5tdQziCEqW5bv+RqktzqNtmNkbjGNwyB785r9err9nrww6lYYgPwrj7/APZg0e5B8squfaollTtoi45nHqfmPDeXdwC0uAx5yoH+TWtbi6kbLy8dgy5BP5ivvGb9kaFixjuAAfQVAP2Tnh4ScEVxzyytsonRHMaXVnxBKbhV3SDn24rlby/u0lO2F9g6hiMfgetfoev7LNx0eYEVK37Ktuy4lkBJ9qUMrrdYjlmVLoz8zpNQuXbZAjBE/PP14qzBrd+oXaMMp4LqOM/Wv0kX9k3TmPzMAK04P2SdDXhm49MV0/2bO1uUy/tGHc/M8yavMm8DoeNi4xSm71/zQ8wkHTlSVyB9MV+pUH7KnhmPAJOB2rbh/Zn8IwjBjz9aSyur2QnmVPuflVHqGoPhWWZzg8k9j9akt7C8mG8QM3OSST+FfrFD+zt4Lh5EA9627T4K+DrPG22Q49s01lE+rsS80h0R+V+n+Etfv2UWVg5z15NezeHfgx4wv1UpamLOM5NfpDpngXw9aYMUCjHTiuzt9Ms7dQIowMe1dVPKIfaZz1M0n9lHwhpX7OeuSov2namRzzXd2H7M1vsC3cufWvsIIo6CnV2wy+jHock8dVl1PlyD9mHwtjFw5I9hW/Zfs2fDy25lhaT8cV9CUV0KhTW0TB1pvdniEf7Pnw3jO4Wjk+7mu40r4deENGAFhp8SEd9oz+ddvRVeyh2Jc5dyrHZW0ShI4woHYCpfKjHapaTAqrIm435R0FOAFGBRimIQqh6gUbE9BUbA9qAGzSGSbFHQU6kHSlpiCiiigAooooAKKKKACiiigAooooAKKKKACiiigD//0v1SooooAKTFLRQAwqDTdvcVLRigCMFhT91LgUYFABkUdaTFGKAA7u1MLMO1P5paAGKxPan0UUAFFFFABRRRQAUUUUAFFFFABRRRQA3mkwafRQMZg1GYsnmp6KLBcgEIFOCYqWilYLkew00x5qaiiwXK/kg0eQvpViiiyC5EsSjoKlAxRRTEFGaKMUAJmlzSYNGDQMWikwaUUAFFFFAgooooATFGKWigdwooooEFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAH//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA//Z\"}]}" diff --git a/test/apis/realtime/main.py b/test/apis/realtime/main.py deleted file mode 100644 index f13fb6f436..0000000000 --- a/test/apis/realtime/main.py +++ /dev/null @@ -1,19 +0,0 @@ -import os -import time -import threading as td - -from flask import Flask - -app = Flask(__name__) - - -@app.route("/") -def hello_world(): - time.sleep(1) - msg = f"Hello World! (TID={td.get_ident()}" - print(msg) - return msg - - -if __name__ == "__main__": - app.run(debug=True, host="0.0.0.0", port=int(os.getenv("CORTEX_PORT", "8080"))) diff --git a/test/apis/realtime/prime-generator/.dockerignore b/test/apis/realtime/prime-generator/.dockerignore new file mode 100644 index 0000000000..12657957ef --- /dev/null +++ b/test/apis/realtime/prime-generator/.dockerignore @@ -0,0 +1,8 @@ +*.dockerfile +README.md +sample.json +*.pyc +*.pyo +*.pyd +__pycache__ +.pytest_cache diff --git a/test/apis/realtime/prime-generator/cortex_cpu.yaml b/test/apis/realtime/prime-generator/cortex_cpu.yaml new file mode 100644 index 0000000000..ba5ea380f5 --- /dev/null +++ b/test/apis/realtime/prime-generator/cortex_cpu.yaml @@ -0,0 +1,16 @@ +- name: prime-generator + kind: RealtimeAPI + pod: + port: 9000 + containers: + - name: api + image: quay.io/cortexlabs-test/realtime-prime-generator-cpu:latest + readiness_probe: + http_get: + path: "/healthz" + port: 9000 + compute: + cpu: 200m + mem: 128M + autoscaling: + max_concurrency: 1 diff --git a/test/apis/realtime/prime-generator/main.py b/test/apis/realtime/prime-generator/main.py new file mode 100644 index 0000000000..59099d8202 --- /dev/null +++ b/test/apis/realtime/prime-generator/main.py @@ -0,0 +1,36 @@ +from typing import DefaultDict + +from fastapi import FastAPI +from pydantic import BaseModel + + +def generate_primes(limit=None): + """Sieve of Eratosthenes""" + not_prime = DefaultDict(list) + num = 2 + while limit is None or num <= limit: + if num in not_prime: + for prime in not_prime[num]: + not_prime[prime + num].append(prime) + del not_prime[num] + else: + yield num + not_prime[num * num] = [num] + num += 1 + + +class Request(BaseModel): + primes_to_generate: float + + +app = FastAPI() + + +@app.get("/healthz") +def healthz(): + return "ok" + + +@app.post("/") +def prime_numbers(request: Request): + return {"prime_numbers": list(generate_primes(request.primes_to_generate))} diff --git a/test/apis/realtime/prime-generator/prime-generator-cpu.dockerfile b/test/apis/realtime/prime-generator/prime-generator-cpu.dockerfile new file mode 100644 index 0000000000..a8f376e7d4 --- /dev/null +++ b/test/apis/realtime/prime-generator/prime-generator-cpu.dockerfile @@ -0,0 +1,17 @@ +FROM python:3.8-slim + +# Allow statements and log messages to immediately appear in the logs +ENV PYTHONUNBUFFERED True + +# Install production dependencies +RUN pip install --no-cache-dir "uvicorn[standard]" gunicorn fastapi pydantic + +# Copy local code to the container image. +COPY . /app +WORKDIR /app/ + +ENV PYTHONPATH=/app +ENV CORTEX_PORT=9000 + +# Run the web service on container startup. +CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/realtime/prime-generator/sample.json b/test/apis/realtime/prime-generator/sample.json new file mode 100644 index 0000000000..9f35f9c3e9 --- /dev/null +++ b/test/apis/realtime/prime-generator/sample.json @@ -0,0 +1,3 @@ +{ + "primes_to_generate": 100 +} diff --git a/test/apis/realtime/sleep/.dockerignore b/test/apis/realtime/sleep/.dockerignore new file mode 100644 index 0000000000..12657957ef --- /dev/null +++ b/test/apis/realtime/sleep/.dockerignore @@ -0,0 +1,8 @@ +*.dockerfile +README.md +sample.json +*.pyc +*.pyo +*.pyd +__pycache__ +.pytest_cache diff --git a/test/apis/realtime/sleep/cortex_cpu.yaml b/test/apis/realtime/sleep/cortex_cpu.yaml new file mode 100644 index 0000000000..ee64740962 --- /dev/null +++ b/test/apis/realtime/sleep/cortex_cpu.yaml @@ -0,0 +1,16 @@ +- name: sleep + kind: RealtimeAPI + pod: + port: 9000 + containers: + - name: api + image: quay.io/cortexlabs-test/realtime-sleep-cpu:latest + readiness_probe: + http_get: + path: "/healthz" + port: 9000 + compute: + cpu: 200m + mem: 128M + autoscaling: + max_concurrency: 1 diff --git a/test/apis/realtime/sleep/main.py b/test/apis/realtime/sleep/main.py new file mode 100644 index 0000000000..6dd675055c --- /dev/null +++ b/test/apis/realtime/sleep/main.py @@ -0,0 +1,15 @@ +import time + +from fastapi import FastAPI + +app = FastAPI() + + +@app.get("/healthz") +def healthz(): + return "ok" + + +@app.post("/") +def sleep(sleep: float = 0): + time.sleep(sleep) diff --git a/test/apis/realtime/sleep/sample.json b/test/apis/realtime/sleep/sample.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/test/apis/realtime/sleep/sample.json @@ -0,0 +1 @@ +{} diff --git a/test/apis/realtime/sleep/sleep-cpu.dockerfile b/test/apis/realtime/sleep/sleep-cpu.dockerfile new file mode 100644 index 0000000000..a496f7c881 --- /dev/null +++ b/test/apis/realtime/sleep/sleep-cpu.dockerfile @@ -0,0 +1,17 @@ +FROM python:3.8-slim + +# Allow statements and log messages to immediately appear in the logs +ENV PYTHONUNBUFFERED True + +# Install production dependencies +RUN pip install --no-cache-dir "uvicorn[standard]" gunicorn fastapi + +# Copy local code to the container image. +COPY . /app +WORKDIR /app/ + +ENV PYTHONPATH=/app +ENV CORTEX_PORT=9000 + +# Run the web service on container startup. +CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/realtime/text-generator/.dockerignore b/test/apis/realtime/text-generator/.dockerignore new file mode 100644 index 0000000000..12657957ef --- /dev/null +++ b/test/apis/realtime/text-generator/.dockerignore @@ -0,0 +1,8 @@ +*.dockerfile +README.md +sample.json +*.pyc +*.pyo +*.pyd +__pycache__ +.pytest_cache diff --git a/test/apis/realtime/text-generator/cortex_cpu.yaml b/test/apis/realtime/text-generator/cortex_cpu.yaml new file mode 100644 index 0000000000..3e502c40a1 --- /dev/null +++ b/test/apis/realtime/text-generator/cortex_cpu.yaml @@ -0,0 +1,16 @@ +- name: text-generator + kind: RealtimeAPI + pod: + port: 9000 + containers: + - name: api + image: quay.io/cortexlabs-test/realtime-text-generator-cpu:latest + readiness_probe: + http_get: + path: "/healthz" + port: 9000 + compute: + cpu: 1 + mem: 2.5G + autoscaling: + max_concurrency: 1 diff --git a/test/apis/realtime/text-generator/cortex_gpu.yaml b/test/apis/realtime/text-generator/cortex_gpu.yaml new file mode 100644 index 0000000000..1db64d7477 --- /dev/null +++ b/test/apis/realtime/text-generator/cortex_gpu.yaml @@ -0,0 +1,19 @@ +- name: text-generator + kind: RealtimeAPI + pod: + port: 9000 + containers: + - name: api + image: quay.io/cortexlabs-test/realtime-text-generator-gpu:latest + env: + TARGET_DEVICE: "cuda" + readiness_probe: + http_get: + path: "/healthz" + port: 9000 + compute: + cpu: 1 + gpu: 1 + mem: 512M + autoscaling: + max_concurrency: 1 diff --git a/test/apis/realtime/text-generator/main.py b/test/apis/realtime/text-generator/main.py new file mode 100644 index 0000000000..020c7fc9e2 --- /dev/null +++ b/test/apis/realtime/text-generator/main.py @@ -0,0 +1,42 @@ +import os + +from fastapi import FastAPI, Response, status +from pydantic import BaseModel +from transformers import GPT2Tokenizer, GPT2LMHeadModel + + +class Request(BaseModel): + text: str + + +state = { + "model_ready": False, + "tokenizer": None, + "model": None, +} +device = os.getenv("TARGET_DEVICE", "cpu") +app = FastAPI() + + +@app.on_event("startup") +def startup(): + global state + state["tokenizer"] = GPT2Tokenizer.from_pretrained("gpt2") + state["model"] = GPT2LMHeadModel.from_pretrained("gpt2").to(device) + state["model_ready"] = True + + +@app.get("/healthz") +def healthz(response: Response): + if not state["model_ready"]: + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE + + +@app.post("/") +def text_generator(request: Request): + input_length = len(request.text.split()) + tokens = state["tokenizer"].encode(request.text, return_tensors="pt").to(device) + prediction = state["model"].generate(tokens, max_length=input_length + 20, do_sample=True) + return { + "prediction": state["tokenizer"].decode(prediction[0]), + } diff --git a/test/apis/realtime/text-generator/sample.json b/test/apis/realtime/text-generator/sample.json new file mode 100644 index 0000000000..36a3627568 --- /dev/null +++ b/test/apis/realtime/text-generator/sample.json @@ -0,0 +1,3 @@ +{ + "text": "machine learning is" +} diff --git a/test/apis/realtime/text-generator/text-generator-cpu.dockerfile b/test/apis/realtime/text-generator/text-generator-cpu.dockerfile new file mode 100644 index 0000000000..b90ff40a7f --- /dev/null +++ b/test/apis/realtime/text-generator/text-generator-cpu.dockerfile @@ -0,0 +1,23 @@ +FROM python:3.8-slim + +# Allow statements and log messages to immediately appear in the logs +ENV PYTHONUNBUFFERED True + +# Install production dependencies +RUN pip install --no-cache-dir \ + "uvicorn[standard]" \ + gunicorn \ + fastapi \ + pydantic \ + transformers==3.0.* \ + torch==1.7.1+cpu -f https://download.pytorch.org/whl/torch_stable.html + +# Copy local code to the container image. +COPY . /app +WORKDIR /app/ + +ENV PYTHONPATH=/app +ENV CORTEX_PORT=9000 + +# Run the web service on container startup. +CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/realtime/text-generator/text-generator-gpu.dockerfile b/test/apis/realtime/text-generator/text-generator-gpu.dockerfile new file mode 100644 index 0000000000..2de181d169 --- /dev/null +++ b/test/apis/realtime/text-generator/text-generator-gpu.dockerfile @@ -0,0 +1,27 @@ +FROM nvidia/cuda:10.2-cudnn8-runtime-ubuntu18.04 + +RUN apt-get update \ + && apt-get install \ + python3 \ + python3-pip \ + pkg-config \ + git \ + build-essential \ + cmake -y \ + && apt-get clean -qq && rm -rf /var/lib/apt/lists/* + +# Allow statements and log messages to immediately appear in the logs +ENV PYTHONUNBUFFERED True + +# Install production dependencies +RUN pip3 install --no-cache-dir "uvicorn[standard]" gunicorn fastapi pydantic transformers==3.0.* torch==1.7.* + +# Copy local code to the container image. +COPY . /app +WORKDIR /app/ + +ENV PYTHONPATH=/app +ENV CORTEX_PORT=9000 + +# Run the web service on container startup. +CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/sklearn/iris-classifier/cortex.yaml b/test/apis/sklearn/iris-classifier/cortex.yaml deleted file mode 100644 index 42d13d506b..0000000000 --- a/test/apis/sklearn/iris-classifier/cortex.yaml +++ /dev/null @@ -1,11 +0,0 @@ -- name: iris-classifier - kind: RealtimeAPI - handler: - type: python - path: handler.py - config: - bucket: cortex-examples - key: sklearn/iris-classifier/model.pkl - compute: - cpu: 0.2 - mem: 200M diff --git a/test/apis/sklearn/iris-classifier/handler.py b/test/apis/sklearn/iris-classifier/handler.py deleted file mode 100644 index c9f6d100f0..0000000000 --- a/test/apis/sklearn/iris-classifier/handler.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import boto3 -from botocore import UNSIGNED -from botocore.client import Config -import pickle - -labels = ["setosa", "versicolor", "virginica"] - - -class Handler: - def __init__(self, config): - s3 = boto3.client("s3") - s3.download_file(config["bucket"], config["key"], "/tmp/model.pkl") - self.model = pickle.load(open("/tmp/model.pkl", "rb")) - - def handle_post(self, payload): - measurements = [ - payload["sepal_length"], - payload["sepal_width"], - payload["petal_length"], - payload["petal_width"], - ] - - label_id = self.model.predict([measurements])[0] - return labels[label_id] diff --git a/test/apis/sklearn/iris-classifier/requirements.txt b/test/apis/sklearn/iris-classifier/requirements.txt deleted file mode 100644 index bbc213cf3e..0000000000 --- a/test/apis/sklearn/iris-classifier/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -boto3 -scikit-learn==0.21.3 diff --git a/test/apis/sklearn/iris-classifier/sample.json b/test/apis/sklearn/iris-classifier/sample.json deleted file mode 100644 index 9e792863cd..0000000000 --- a/test/apis/sklearn/iris-classifier/sample.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "sepal_length": 5.2, - "sepal_width": 3.6, - "petal_length": 1.5, - "petal_width": 0.3 -} diff --git a/test/apis/sklearn/iris-classifier/trainer.py b/test/apis/sklearn/iris-classifier/trainer.py deleted file mode 100644 index a88859cf81..0000000000 --- a/test/apis/sklearn/iris-classifier/trainer.py +++ /dev/null @@ -1,23 +0,0 @@ -import boto3 -import pickle - -from sklearn.datasets import load_iris -from sklearn.model_selection import train_test_split -from sklearn.linear_model import LogisticRegression - -# Train the model - -iris = load_iris() -data, labels = iris.data, iris.target -training_data, test_data, training_labels, test_labels = train_test_split(data, labels) - -model = LogisticRegression(solver="lbfgs", multi_class="multinomial", max_iter=1000) -model.fit(training_data, training_labels) -accuracy = model.score(test_data, test_labels) -print("accuracy: {:.2f}".format(accuracy)) - -# Upload the model - -pickle.dump(model, open("model.pkl", "wb")) -s3 = boto3.client("s3") -s3.upload_file("model.pkl", "cortex-examples", "sklearn/iris-classifier/model.pkl") diff --git a/test/apis/sklearn/mpg-estimator/cortex.yaml b/test/apis/sklearn/mpg-estimator/cortex.yaml deleted file mode 100644 index 42b8f5a460..0000000000 --- a/test/apis/sklearn/mpg-estimator/cortex.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- name: mpg-estimator - kind: RealtimeAPI - handler: - type: python - path: handler.py - config: - model: s3://cortex-examples/sklearn/mpg-estimator/linreg/ diff --git a/test/apis/sklearn/mpg-estimator/handler.py b/test/apis/sklearn/mpg-estimator/handler.py deleted file mode 100644 index 21bd161579..0000000000 --- a/test/apis/sklearn/mpg-estimator/handler.py +++ /dev/null @@ -1,35 +0,0 @@ -import boto3 -from botocore import UNSIGNED -from botocore.client import Config -import mlflow.sklearn -import numpy as np -import re -import os - - -class Handler: - def __init__(self, config): - model_path = "/tmp/model" - os.makedirs(model_path, exist_ok=True) - - # download mlflow model folder from S3 - bucket, prefix = re.match("s3://(.+?)/(.+)", config["model"]).groups() - s3 = boto3.client("s3") - response = s3.list_objects_v2(Bucket=bucket, Prefix=prefix) - for s3_obj in response["Contents"]: - obj_key = s3_obj["Key"] - s3.download_file(bucket, obj_key, os.path.join(model_path, os.path.basename(obj_key))) - - self.model = mlflow.sklearn.load_model(model_path) - - def handle_post(self, payload): - model_input = [ - payload["cylinders"], - payload["displacement"], - payload["horsepower"], - payload["weight"], - payload["acceleration"], - ] - - result = self.model.predict([model_input]) - return np.asscalar(result) diff --git a/test/apis/sklearn/mpg-estimator/requirements.txt b/test/apis/sklearn/mpg-estimator/requirements.txt deleted file mode 100644 index cbcad6b321..0000000000 --- a/test/apis/sklearn/mpg-estimator/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -mlflow -pandas -numpy -scikit-learn==0.21.3 diff --git a/test/apis/sklearn/mpg-estimator/sample.json b/test/apis/sklearn/mpg-estimator/sample.json deleted file mode 100644 index 2dbbca46dd..0000000000 --- a/test/apis/sklearn/mpg-estimator/sample.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "cylinders": 4, - "displacement": 135, - "horsepower": 84, - "weight": 2490, - "acceleration": 15.7 -} diff --git a/test/apis/sklearn/mpg-estimator/trainer.py b/test/apis/sklearn/mpg-estimator/trainer.py deleted file mode 100644 index 7cec1e1925..0000000000 --- a/test/apis/sklearn/mpg-estimator/trainer.py +++ /dev/null @@ -1,23 +0,0 @@ -import mlflow.sklearn -import pandas as pd -import numpy as np -from sklearn.linear_model import LinearRegression -from sklearn.model_selection import train_test_split - - -df = pd.read_csv( - "https://www.uio.no/studier/emner/sv/oekonomi/ECON4150/v16/statacourse/datafiles/auto.csv" -) -df = df.replace("?", np.nan) -df = df.dropna() -df = df.drop(["name", "origin", "year"], axis=1) # drop categorical variables for simplicity -data = df.drop("mpg", axis=1) -labels = df[["mpg"]] - -training_data, test_data, training_labels, test_labels = train_test_split(data, labels) -model = LinearRegression() -model.fit(training_data, training_labels) -accuracy = model.score(test_data, test_labels) -print("accuracy: {:.2f}".format(accuracy)) - -mlflow.sklearn.save_model(model, "linreg") diff --git a/test/apis/sleep/cortex.yaml b/test/apis/sleep/cortex.yaml deleted file mode 100644 index 718610268d..0000000000 --- a/test/apis/sleep/cortex.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- name: sleep - kind: RealtimeAPI - handler: - type: python - path: handler.py - compute: - cpu: 100m diff --git a/test/apis/sleep/deploy.py b/test/apis/sleep/deploy.py deleted file mode 100644 index 67d226f9eb..0000000000 --- a/test/apis/sleep/deploy.py +++ /dev/null @@ -1,19 +0,0 @@ -import cortex -import os - -dir_path = os.path.dirname(os.path.realpath(__file__)) - -cx = cortex.client() - -api_spec = { - "name": "sleep", - "kind": "RealtimeAPI", - "handler": { - "type": "python", - "path": "handler.py", - }, -} - -print(cx.deploy(api_spec, project_dir=dir_path)) - -# cx.delete("sleep") diff --git a/test/apis/sleep/handler.py b/test/apis/sleep/handler.py deleted file mode 100644 index 6d316961f9..0000000000 --- a/test/apis/sleep/handler.py +++ /dev/null @@ -1,10 +0,0 @@ -import time - - -class Handler: - def __init__(self, config): - pass - - def handle_post(self, payload, query_params): - time.sleep(float(query_params.get("sleep", 0))) - return "ok" diff --git a/test/apis/spacy/entity-recognizer/cortex.yaml b/test/apis/spacy/entity-recognizer/cortex.yaml deleted file mode 100644 index 01782cf002..0000000000 --- a/test/apis/spacy/entity-recognizer/cortex.yaml +++ /dev/null @@ -1,8 +0,0 @@ -- name: entity-recognizer - kind: RealtimeAPI - handler: - type: python - path: handler.py - compute: - cpu: 1 - mem: 1G diff --git a/test/apis/spacy/entity-recognizer/handler.py b/test/apis/spacy/entity-recognizer/handler.py deleted file mode 100644 index 8f6c274641..0000000000 --- a/test/apis/spacy/entity-recognizer/handler.py +++ /dev/null @@ -1,20 +0,0 @@ -import spacy -import subprocess - - -class Handler: - """ - Class to perform NER (named entity recognition) - """ - - def __init__(self, config): - subprocess.call("python -m spacy download en_core_web_md".split(" ")) - import en_core_web_md - - self.nlp = en_core_web_md.load() - - def handle_post(self, payload): - doc = self.nlp(payload["text"]) - proc = lambda ent: {"label": ent.label_, "start": ent.start, "end": ent.end} - out = {ent.text: proc(ent) for ent in doc.ents} - return out diff --git a/test/apis/spacy/entity-recognizer/requirements.txt b/test/apis/spacy/entity-recognizer/requirements.txt deleted file mode 100644 index 568e4fc634..0000000000 --- a/test/apis/spacy/entity-recognizer/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -spacy diff --git a/test/apis/spacy/entity-recognizer/sample.json b/test/apis/spacy/entity-recognizer/sample.json deleted file mode 100644 index ae0f0f4120..0000000000 --- a/test/apis/spacy/entity-recognizer/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "text": "Lilium, a Munich-based startup that is designing and building vertical take-off and landing (VTOL) aircraft with speeds of up to 100 km/h that it plans eventually to run in its own taxi fleet, has closed a funding round of over $240 million — money that it plans to use to keep developing its aircraft, and to start building manufacturing facilities to produce more of them, for an expected launch date of 2025." -} diff --git a/test/apis/task/cortex.yaml b/test/apis/task/cortex.yaml deleted file mode 100644 index a9780bbdfa..0000000000 --- a/test/apis/task/cortex.yaml +++ /dev/null @@ -1,12 +0,0 @@ -- name: task - kind: TaskAPI - pod: - containers: - - name: s3-pusher - image: 499593605069.dkr.ecr.us-west-2.amazonaws.com/sample/task-caas:latest - command: - - python - - main.py - compute: - cpu: 100m - mem: 256Mi diff --git a/test/apis/task/.dockerignore b/test/apis/task/iris-classifier-trainer/.dockerignore similarity index 70% rename from test/apis/task/.dockerignore rename to test/apis/task/iris-classifier-trainer/.dockerignore index 3e4bdd9fbb..5bb0c5ed3b 100644 --- a/test/apis/task/.dockerignore +++ b/test/apis/task/iris-classifier-trainer/.dockerignore @@ -1,5 +1,6 @@ -Dockerfile +*.dockerfile README.md +submit.py *.pyc *.pyo *.pyd diff --git a/test/apis/task/iris-classifier-trainer/cortex_cpu.yaml b/test/apis/task/iris-classifier-trainer/cortex_cpu.yaml new file mode 100644 index 0000000000..d8ff2ff1d3 --- /dev/null +++ b/test/apis/task/iris-classifier-trainer/cortex_cpu.yaml @@ -0,0 +1,12 @@ +- name: iris-classifier-trainer + kind: TaskAPI + pod: + containers: + - name: trainer + image: quay.io/cortexlabs-test/task-iris-classifier-trainer-cpu:latest + command: + - python + - main.py + compute: + cpu: 200m + mem: 256Mi diff --git a/test/apis/task/Dockerfile b/test/apis/task/iris-classifier-trainer/iris-classifier-trainer-cpu.dockerfile similarity index 78% rename from test/apis/task/Dockerfile rename to test/apis/task/iris-classifier-trainer/iris-classifier-trainer-cpu.dockerfile index bfb7703693..06cdb86fcc 100644 --- a/test/apis/task/Dockerfile +++ b/test/apis/task/iris-classifier-trainer/iris-classifier-trainer-cpu.dockerfile @@ -1,17 +1,17 @@ # Use the official lightweight Python image. # https://hub.docker.com/_/python -FROM python:3.9-slim +FROM python:3.7-slim -# Allow statements and log messages to immediately appear in the Knative logs +# Allow statements and log messages to immediately appear in the logs ENV PYTHONUNBUFFERED True +# Install production dependencies. +RUN pip install numpy==1.18.5 boto3==1.17.72 scikit-learn==0.21.3 + # Copy local code to the container image. ENV APP_HOME /app WORKDIR $APP_HOME COPY . ./ -# Install production dependencies. -RUN pip install boto3==1.17.72 - # Run task CMD exec python main.py diff --git a/test/apis/task/iris-classifier-trainer/main.py b/test/apis/task/iris-classifier-trainer/main.py new file mode 100644 index 0000000000..440bd1c61e --- /dev/null +++ b/test/apis/task/iris-classifier-trainer/main.py @@ -0,0 +1,41 @@ +import json, pickle, re, os, boto3 + +from sklearn.datasets import load_iris +from sklearn.model_selection import train_test_split +from sklearn.linear_model import LogisticRegression + + +def main(): + with open("/cortex/spec/job.json", "r") as f: + job_spec = json.load(f) + print(json.dumps(job_spec, indent=2)) + + # get metadata + config = job_spec["config"] + job_id = job_spec["job_id"] + s3_path = None + if "dest_s3_dir" in config: + s3_path = config["dest_s3_dir"] + + # Train the model + iris = load_iris() + data, labels = iris.data, iris.target + training_data, test_data, training_labels, test_labels = train_test_split(data, labels) + + model = LogisticRegression(solver="lbfgs", multi_class="multinomial", max_iter=1000) + model.fit(training_data, training_labels) + accuracy = model.score(test_data, test_labels) + print("accuracy: {:.2f}".format(accuracy)) + + # Upload the model + if s3_path: + pickle.dump(model, open("model.pkl", "wb")) + bucket, key = re.match("s3://(.+?)/(.+)", s3_path).groups() + s3 = boto3.client("s3") + s3.upload_file("model.pkl", bucket, os.path.join(key, job_id, "model.pkl")) + else: + print("not uploading the model to the s3 bucket") + + +if __name__ == "__main__": + main() diff --git a/test/apis/task/iris-classifier-trainer/submit.py b/test/apis/task/iris-classifier-trainer/submit.py new file mode 100644 index 0000000000..2b63d55d25 --- /dev/null +++ b/test/apis/task/iris-classifier-trainer/submit.py @@ -0,0 +1,32 @@ +""" +Typical usage example: + + python submit.py +""" + +import sys +import json +import requests +import cortex + + +def main(): + # parse args + if len(sys.argv) != 3: + print("usage: python submit.py ") + sys.exit(1) + env_name = sys.argv[1] + dest_s3_dir = sys.argv[2] + + # get task endpoint + cx = cortex.client(env_name) + task_endpoint = cx.get_api("trainer")["endpoint"] + + # submit job + job_spec = {"config": {"dest_s3_dir": dest_s3_dir}} + response = requests.post(task_endpoint, json=job_spec) + print(json.dumps(response.json(), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/test/apis/task/main.py b/test/apis/task/main.py deleted file mode 100644 index 494e9bdfb0..0000000000 --- a/test/apis/task/main.py +++ /dev/null @@ -1,21 +0,0 @@ -import json, re, os, boto3 - - -def main(): - with open("/cortex/job_spec.json", "r") as f: - job_spec = json.load(f) - print(json.dumps(job_spec, indent=2)) - - # get metadata - config = job_spec["config"] - job_id = job_spec["job_id"] - s3_path = config["s3_path"] - - # touch file on s3 - bucket, key = re.match("s3://(.+?)/(.+)", s3_path).groups() - s3 = boto3.client("s3") - s3.put_object(Bucket=bucket, Key=os.path.join(key, job_id), Body="") - - -if __name__ == "__main__": - main() diff --git a/test/apis/tensorflow/image-classifier-inception/cortex.yaml b/test/apis/tensorflow/image-classifier-inception/cortex.yaml deleted file mode 100644 index 2dda257975..0000000000 --- a/test/apis/tensorflow/image-classifier-inception/cortex.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- name: image-classifier-inception - kind: RealtimeAPI - handler: - type: tensorflow - path: handler.py - models: - path: s3://cortex-examples/tensorflow/image-classifier/inception/ - compute: - cpu: 1 - gpu: 1 diff --git a/test/apis/tensorflow/image-classifier-inception/cortex_server_side_batching.yaml b/test/apis/tensorflow/image-classifier-inception/cortex_server_side_batching.yaml deleted file mode 100644 index 5e80d1ccd5..0000000000 --- a/test/apis/tensorflow/image-classifier-inception/cortex_server_side_batching.yaml +++ /dev/null @@ -1,14 +0,0 @@ -- name: image-classifier-inception - kind: RealtimeAPI - handler: - type: tensorflow - path: handler.py - models: - path: s3://cortex-examples/tensorflow/image-classifier/inception/ - server_side_batching: - max_batch_size: 2 - batch_interval: 0.2s - threads_per_process: 2 - compute: - cpu: 1 - gpu: 1 diff --git a/test/apis/tensorflow/image-classifier-inception/handler.py b/test/apis/tensorflow/image-classifier-inception/handler.py deleted file mode 100644 index f2f450fe94..0000000000 --- a/test/apis/tensorflow/image-classifier-inception/handler.py +++ /dev/null @@ -1,19 +0,0 @@ -import requests -import numpy as np -from PIL import Image -from io import BytesIO - - -class Handler: - def __init__(self, tensorflow_client, config): - self.client = tensorflow_client - self.labels = requests.get( - "https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt" - ).text.split("\n") - - def handle_post(self, payload): - image = requests.get(payload["url"]).content - decoded_image = np.asarray(Image.open(BytesIO(image)), dtype=np.float32) / 255 - model_input = {"images": np.expand_dims(decoded_image, axis=0)} - prediction = self.client.predict(model_input) - return self.labels[np.argmax(prediction["classes"])] diff --git a/test/apis/tensorflow/image-classifier-inception/inception.ipynb b/test/apis/tensorflow/image-classifier-inception/inception.ipynb deleted file mode 100644 index 804513e207..0000000000 --- a/test/apis/tensorflow/image-classifier-inception/inception.ipynb +++ /dev/null @@ -1,198 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "inception.ipynb", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.8" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "n8CwINQcEBKz", - "colab_type": "text" - }, - "source": [ - "# Exporting ImageNet Inception\n", - "\n", - "In this notebook, we'll show how to export the [pre-trained Imagenet Inception model](https://tfhub.dev/google/imagenet/inception_v3/classification/3) for serving." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "3221z3P69fgf", - "colab_type": "text" - }, - "source": [ - "First, we'll install the required packages:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "_SdQpq7g9LiI", - "colab_type": "code", - "colab": {} - }, - "source": [ - "!pip install tensorflow==1.14.* tensorflow-hub==0.6.* boto3==1.*" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "I-k0gUpxDGkU", - "colab_type": "text" - }, - "source": [ - "Next, we'll download the model from TensorFlow Hub and export it for serving:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "z6QLCzB4BKMe", - "colab_type": "code", - "colab": {} - }, - "source": [ - "import time\n", - "import tensorflow as tf\n", - "import tensorflow_hub as hub\n", - "from tensorflow.python.saved_model.signature_def_utils_impl import predict_signature_def\n", - "\n", - "export_dir = \"export/\" + str(time.time()).split('.')[0]\n", - "builder = tf.saved_model.builder.SavedModelBuilder(export_dir)\n", - "\n", - "with tf.Session(graph=tf.Graph()) as sess:\n", - " module = hub.Module(\"https://tfhub.dev/google/imagenet/inception_v3/classification/3\")\n", - "\n", - " input_params = module.get_input_info_dict()\n", - " image_input = tf.placeholder(\n", - " name=\"images\", dtype=input_params[\"images\"].dtype, shape=input_params[\"images\"].get_shape()\n", - " )\n", - " \n", - " sess.run([tf.global_variables_initializer(), tf.tables_initializer()])\n", - "\n", - " classes = module(image_input)\n", - " signature = predict_signature_def(inputs={\"images\": image_input}, outputs={\"classes\": classes})\n", - "\n", - " builder.add_meta_graph_and_variables(\n", - " sess, [\"serve\"], signature_def_map={\"predict\": signature}, strip_default_attrs=True\n", - " )\n", - "\n", - "builder.save()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "aGtJiyEnBgwl", - "colab_type": "text" - }, - "source": [ - "## Upload the model to AWS\n", - "\n", - "Cortex loads models from AWS, so we need to upload the exported model." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "fTkjvSKBBmUB", - "colab_type": "text" - }, - "source": [ - "Set these variables to configure your AWS credentials and model upload path:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "4xcDWxqCBPre", - "colab_type": "code", - "cellView": "form", - "colab": {} - }, - "source": [ - "AWS_ACCESS_KEY_ID = \"\" #@param {type:\"string\"}\n", - "AWS_SECRET_ACCESS_KEY = \"\" #@param {type:\"string\"}\n", - "S3_UPLOAD_PATH = \"s3://my-bucket/image-classifier/inception\" #@param {type:\"string\"}\n", - "\n", - "import sys\n", - "import re\n", - "\n", - "if AWS_ACCESS_KEY_ID == \"\":\n", - " print(\"\\033[91m{}\\033[00m\".format(\"ERROR: Please set AWS_ACCESS_KEY_ID\"), file=sys.stderr)\n", - "\n", - "elif AWS_SECRET_ACCESS_KEY == \"\":\n", - " print(\"\\033[91m{}\\033[00m\".format(\"ERROR: Please set AWS_SECRET_ACCESS_KEY\"), file=sys.stderr)\n", - "\n", - "else:\n", - " try:\n", - " bucket, key = re.match(\"s3://(.+?)/(.+)\", S3_UPLOAD_PATH).groups()\n", - " except:\n", - " print(\"\\033[91m{}\\033[00m\".format(\"ERROR: Invalid s3 path (should be of the form s3://my-bucket/path/to/file)\"), file=sys.stderr)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "czZkjb1IBr-f", - "colab_type": "text" - }, - "source": [ - "Upload the model to S3:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "M0b0IbyaBsim", - "colab_type": "code", - "colab": {} - }, - "source": [ - "import os\n", - "import boto3\n", - "\n", - "s3 = boto3.client(\"s3\", aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)\n", - "\n", - "for dirpath, _, filenames in os.walk(\"export\"):\n", - " for filename in filenames:\n", - " filepath = os.path.join(dirpath, filename)\n", - " filekey = os.path.join(key, filepath[len(\"export/\"):])\n", - " print(\"Uploading s3://{}/{}...\".format(bucket, filekey), end = '')\n", - " s3.upload_file(filepath, bucket, filekey)\n", - " print(\" ✓\")\n", - "\n", - "print(\"\\nUploaded model export directory to \" + S3_UPLOAD_PATH)" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/test/apis/tensorflow/image-classifier-inception/requirements.txt b/test/apis/tensorflow/image-classifier-inception/requirements.txt deleted file mode 100644 index 7e2fba5e6c..0000000000 --- a/test/apis/tensorflow/image-classifier-inception/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -Pillow diff --git a/test/apis/tensorflow/image-classifier-inception/sample.json b/test/apis/tensorflow/image-classifier-inception/sample.json deleted file mode 100644 index 667652007a..0000000000 --- a/test/apis/tensorflow/image-classifier-inception/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "url": "https://i.imgur.com/PzXprwl.jpg" -} diff --git a/test/apis/tensorflow/image-classifier-resnet50/README.md b/test/apis/tensorflow/image-classifier-resnet50/README.md deleted file mode 100644 index abcadf7a51..0000000000 --- a/test/apis/tensorflow/image-classifier-resnet50/README.md +++ /dev/null @@ -1,88 +0,0 @@ -# Image Classifier with ResNet50 - -This example implements an image recognition system using ResNet50, which allows for the recognition of up to 1000 classes. - -## Deploying - -There are 4 Cortex APIs available in this example: - -1. [cortex.yaml](cortex.yaml) - can be used with any instances. -1. [cortex_inf.yaml](cortex_inf.yaml) - to be used with `inf1` instances. -1. [cortex_gpu.yaml](cortex_gpu.yaml) - to be used with GPU instances. -1. [cortex_gpu_server_side_batching.yaml](cortex_gpu_server_side_batching.yaml) - to be used with GPU instances. Deployed with `max_batch_size` > 1. The exported model and the TensorFlow Handler do not need to be modified to support server-side batching. - -To deploy an API, run: - -```bash -cortex deploy -``` - -E.g. - -```bash -cortex deploy cortex_inf.yaml -``` - -## Verifying your API - -Check that your API is live by running `cortex get image-classifier-resnet50`, and copy the example `curl` command that's shown. After the API is live, run the `curl` command, e.g. - -```bash -$ curl -X POST -H "Content-Type: application/json" -d @sample.json - -["tabby", "Egyptian_cat", "tiger_cat", "tiger", "plastic_bag"] -``` - -The following image is embedded in [sample.json](sample.json): - -![image](https://i.imgur.com/213xcvs.jpg) - -## Throughput test - -Before [throughput_test.py](../../utils/throughput_test.py) is run, 2 environment variables have to be exported: - -```bash -export ENDPOINT= # you can find this with `cortex get image-classifier-resnet50` -export PAYLOAD=https://i.imgur.com/213xcvs.jpg # this is the cat image shown in the previous step -``` - -Then, deploy each API one at a time and check the results: - -1. Running `python ../../utils/throughput_test.py -i 30 -p 4 -t 2` with the [cortex.yaml](cortex.yaml) API running on an `c5.xlarge` instance will get **~16.2 inferences/sec** with an average latency of **200 ms**. -1. Running `python ../../utils/throughput_test.py -i 30 -p 4 -t 48` with the [cortex_inf.yaml](cortex_inf.yaml) API running on an `inf1.2xlarge` instance will get **~510 inferences/sec** with an average latency of **80 ms**. -1. Running `python ../../utils/throughput_test.py -i 30 -p 4 -t 24` with the [cortex_gpu.yaml](cortex_gpu.yaml) API running on an `g4dn.xlarge` instance will get **~125 inferences/sec** with an average latency of **85 ms**. Optimizing the model with TensorRT to use FP16 on TF-serving only seems to achieve a 10% performance improvement - one thing to consider is that the TensorRT engines hadn't been built beforehand, so this might have affected the results negatively. -1. Running `python ../../utils/throughput_test.py -i 30 -p 4 -t 60` with the [cortex_gpu_server_side_batching.yaml](cortex_gpu_batch_sized.yaml) API running on an `g4dn.xlarge` instance will get **~186 inferences/sec** with an average latency of **500 ms**. This achieves a 49% higher throughput than the [cortex_gpu.yaml](cortex_gpu.yaml) API, at the expense of increased latency. - -Alternatively to [throughput_test.py](../../utils/throughput_test.py), the `ab` GNU utility can also be used to benchmark the API. This has the advantage that it's not as taxing on your local machine, but the disadvantage that it doesn't implement a cooldown period. You can run `ab` like this: - -```bash -# for making octet-stream requests, which is the default for throughput_test script -ab -n -c -p sample.bin -T 'application/octet-stream' -rks 120 $ENDPOINT - -# for making json requests, will will have lower performance because the API has to download the image every time -ab -n -c -p sample.json -T 'application/json' -rks 120 $ENDPOINT -``` - -*Note: `inf1.xlarge` isn't used because the major bottleneck with `inf` instances for this example is with the CPU, and `inf1.2xlarge` has twice the amount of CPU cores for same number of Inferentia ASICs (which is 1), which translates to almost double the throughput.* - -## Exporting SavedModels - -This example deploys models that we have built and uploaded to a public S3 bucket. If you want to build the models yourself, follow these instructions. - -Run the following command to install the dependencies required for the [generate_resnet50_models.ipynb](generate_resnet50_models.ipynb) notebook: - -```bash -pip install --extra-index-url=https://pip.repos.neuron.amazonaws.com \ - neuron-cc==1.0.9410.0+6008239556 \ - tensorflow-neuron==1.15.0.1.0.1333.0 -``` - -The [generate_resnet50_models.ipynb](generate_resnet50_models.ipynb) notebook will generate 2 SavedModels. One will be saved in the `resnet50` directory which can be run on GPU or on CPU and another in the `resnet50_neuron` directory which can only be run on `inf1` instances. For server-side batching on `inf1` instances, a different compilation of the model is required. To compile ResNet50 model for a batch size of 5, run `run_all` from [this directory](https://github.com/aws/aws-neuron-sdk/tree/master/src/examples/tensorflow/keras_resnet50). - -If you'd also like to build the TensorRT version of the GPU model, run the following command in a new Python environment to install the pip dependencies required for the [generate_gpu_resnet50_model.ipynb](generate_gpu_resnet50_model.ipynb) notebook: - -```bash -pip install tensorflow==2.0.0 -``` - -TensorRT also has to be installed to export the SavedModel. Follow the instructions on [Nvidia TensorRT Documentation](https://docs.nvidia.com/deeplearning/tensorrt/install-guide/index.html#installing-debian) to download and install TensorRT on your local machine (this will require ~5GB of space, and you will have to create an Nvidia account). This notebook also requires that the SavedModel generated with the [generate_resnet50_models.ipynb](generate_resnet50_models.ipynb) notebook exists in the `resnet50` directory. The TensorRT SavedModel will be exported to the `resnet50_gpu` directory. You can then replace the existing SavedModel with the TensorRT-optimized version in [cortex_gpu.yaml](cortex_gpu.yaml) - it's a drop-in replacement that doesn't require any other dependencies on the Cortex side. By default, the API config in [cortex_gpu.yaml](cortex_gpu.yaml) uses the non-TensorRT-optimized version due to simplicity. diff --git a/test/apis/tensorflow/image-classifier-resnet50/cortex.yaml b/test/apis/tensorflow/image-classifier-resnet50/cortex.yaml deleted file mode 100644 index c6e2acf12d..0000000000 --- a/test/apis/tensorflow/image-classifier-resnet50/cortex.yaml +++ /dev/null @@ -1,17 +0,0 @@ -- name: image-classifier-resnet50 - kind: RealtimeAPI - handler: - type: tensorflow - path: handler.py - models: - path: s3://cortex-examples/tensorflow/resnet50/ - processes_per_replica: 4 - threads_per_process: 16 - config: - classes: https://s3.amazonaws.com/deep-learning-models/image-models/imagenet_class_index.json - input_shape: [224, 224] - input_key: input - output_key: output - compute: - cpu: 3 - mem: 4G diff --git a/test/apis/tensorflow/image-classifier-resnet50/cortex_gpu.yaml b/test/apis/tensorflow/image-classifier-resnet50/cortex_gpu.yaml deleted file mode 100644 index 90e951ae7f..0000000000 --- a/test/apis/tensorflow/image-classifier-resnet50/cortex_gpu.yaml +++ /dev/null @@ -1,18 +0,0 @@ -- name: image-classifier-resnet50 - kind: RealtimeAPI - handler: - type: tensorflow - path: handler.py - models: - path: s3://cortex-examples/tensorflow/resnet50/ - processes_per_replica: 4 - threads_per_process: 24 - config: - classes: https://s3.amazonaws.com/deep-learning-models/image-models/imagenet_class_index.json - input_shape: [224, 224] - input_key: input - output_key: output - compute: - gpu: 1 - cpu: 3 - mem: 4G diff --git a/test/apis/tensorflow/image-classifier-resnet50/cortex_gpu_server_side_batching.yaml b/test/apis/tensorflow/image-classifier-resnet50/cortex_gpu_server_side_batching.yaml deleted file mode 100644 index 8dd0b5b3d8..0000000000 --- a/test/apis/tensorflow/image-classifier-resnet50/cortex_gpu_server_side_batching.yaml +++ /dev/null @@ -1,21 +0,0 @@ -- name: image-classifier-resnet50 - kind: RealtimeAPI - handler: - type: tensorflow - path: handler.py - models: - path: s3://cortex-examples/tensorflow/resnet50/ - server_side_batching: - max_batch_size: 32 - batch_interval: 0.1s - processes_per_replica: 4 - threads_per_process: 192 - config: - classes: https://s3.amazonaws.com/deep-learning-models/image-models/imagenet_class_index.json - input_shape: [224, 224] - input_key: input - output_key: output - compute: - gpu: 1 - cpu: 3 - mem: 4G diff --git a/test/apis/tensorflow/image-classifier-resnet50/cortex_inf.yaml b/test/apis/tensorflow/image-classifier-resnet50/cortex_inf.yaml deleted file mode 100644 index 5ba87a891d..0000000000 --- a/test/apis/tensorflow/image-classifier-resnet50/cortex_inf.yaml +++ /dev/null @@ -1,20 +0,0 @@ -- name: image-classifier-resnet50 - kind: RealtimeAPI - handler: - type: tensorflow - path: handler.py - models: - path: s3://cortex-examples/tensorflow/resnet50_neuron/ - processes_per_replica: 4 - threads_per_process: 256 - config: - classes: https://s3.amazonaws.com/deep-learning-models/image-models/imagenet_class_index.json - input_shape: [224, 224] - input_key: input - output_key: output - compute: - inf: 1 - cpu: 3 - mem: 4G - autoscaling: - max_concurrency: 16384 diff --git a/test/apis/tensorflow/image-classifier-resnet50/cortex_inf_server_side_batching.yaml b/test/apis/tensorflow/image-classifier-resnet50/cortex_inf_server_side_batching.yaml deleted file mode 100644 index 9d587cecfb..0000000000 --- a/test/apis/tensorflow/image-classifier-resnet50/cortex_inf_server_side_batching.yaml +++ /dev/null @@ -1,23 +0,0 @@ -- name: image-classifier-resnet50 - kind: RealtimeAPI - handler: - type: tensorflow - path: handler.py - models: - path: s3://cortex-examples/tensorflow/resnet50_neuron_batch_size_5/ - server_side_batching: - max_batch_size: 5 - batch_interval: 0.1s - processes_per_replica: 4 - threads_per_process: 260 - config: - classes: https://s3.amazonaws.com/deep-learning-models/image-models/imagenet_class_index.json - input_shape: [224, 224] - input_key: input_1:0 - output_key: probs/Softmax:0 - compute: - inf: 1 - cpu: 3 - mem: 4G - autoscaling: - max_concurrency: 16384 diff --git a/test/apis/tensorflow/image-classifier-resnet50/dependencies.sh b/test/apis/tensorflow/image-classifier-resnet50/dependencies.sh deleted file mode 100644 index 057530cb85..0000000000 --- a/test/apis/tensorflow/image-classifier-resnet50/dependencies.sh +++ /dev/null @@ -1 +0,0 @@ -apt-get update && apt-get install -y libgl1-mesa-glx libegl1-mesa diff --git a/test/apis/tensorflow/image-classifier-resnet50/generate_gpu_resnet50_model.ipynb b/test/apis/tensorflow/image-classifier-resnet50/generate_gpu_resnet50_model.ipynb deleted file mode 100644 index cef7c720fa..0000000000 --- a/test/apis/tensorflow/image-classifier-resnet50/generate_gpu_resnet50_model.ipynb +++ /dev/null @@ -1,129 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Generate GPU Resnet50 Model\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import tensorflow as tf\n", - "from tensorflow.python.compiler.tensorrt import trt_convert as trt" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "input_model_dir = \"resnet50\"\n", - "output_model_dir = \"resnet50_gpu\"" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "conversion_params = trt.DEFAULT_TRT_CONVERSION_PARAMS\n", - "conversion_params = conversion_params._replace(\n", - " max_workspace_size_bytes=(1<<30))\n", - "conversion_params = conversion_params._replace(precision_mode=\"FP16\")\n", - "conversion_params = conversion_params._replace(\n", - " maximum_cached_engines=100)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Linked TensorRT version: (0, 0, 0)\n", - "INFO:tensorflow:Loaded TensorRT version: (0, 0, 0)\n", - "INFO:tensorflow:Running against TensorRT version 0.0.0\n" - ] - } - ], - "source": [ - "converter = trt.TrtGraphConverterV2(\n", - " input_saved_model_dir=input_model_dir,\n", - " conversion_params=conversion_params)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "WARNING:tensorflow:From /home/robert/.miniconda3/envs/py36-tf/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1781: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "If using Keras pass *_constraint arguments to layers.\n", - "WARNING:tensorflow:Issue encountered when serializing variables.\n", - "Type is unsupported, or the types of the items don't match field type in CollectionDef. Note this is a warning and probably safe to ignore.\n", - "to_proto not supported in EAGER mode.\n", - "WARNING:tensorflow:Issue encountered when serializing trainable_variables.\n", - "Type is unsupported, or the types of the items don't match field type in CollectionDef. Note this is a warning and probably safe to ignore.\n", - "to_proto not supported in EAGER mode.\n" - ] - } - ], - "source": [ - "converter.convert()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Assets written to: resnet50_gpu/assets\n" - ] - } - ], - "source": [ - "converter.save(output_model_dir)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.9" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/test/apis/tensorflow/image-classifier-resnet50/generate_resnet50_models.ipynb b/test/apis/tensorflow/image-classifier-resnet50/generate_resnet50_models.ipynb deleted file mode 100644 index c5358f1dc3..0000000000 --- a/test/apis/tensorflow/image-classifier-resnet50/generate_resnet50_models.ipynb +++ /dev/null @@ -1,176 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Generate Resnet50 Models\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import time\n", - "import shutil\n", - "import tensorflow as tf\n", - "import tensorflow.neuron as tfn\n", - "import tensorflow.compat.v1.keras as keras\n", - "from tensorflow.keras.applications.resnet50 import ResNet50" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Prepare export directories for compile/non-compiled versions of the model." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "model_dir = \"resnet50\"\n", - "compiled_model_dir = model_dir + \"_neuron\"\n", - "shutil.rmtree(model_dir, ignore_errors=True)\n", - "shutil.rmtree(compiled_model_dir, ignore_errors=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Instantiate a Keras ResNet50 model." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "WARNING:tensorflow:From /home/robert/.miniconda3/envs/py36-neuron/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "If using Keras pass *_constraint arguments to layers.\n" - ] - } - ], - "source": [ - "keras.backend.set_learning_phase(0)\n", - "keras.backend.set_image_data_format('channels_last')\n", - "model = ResNet50(weights='imagenet')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Export the model as SavedModel." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "WARNING:tensorflow:From :5: simple_save (from tensorflow.python.saved_model.simple_save) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.simple_save.\n", - "WARNING:tensorflow:From /home/robert/.miniconda3/envs/py36-neuron/lib/python3.6/site-packages/tensorflow_core/python/saved_model/signature_def_utils_impl.py:201: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.\n", - "INFO:tensorflow:Assets added to graph.\n", - "INFO:tensorflow:No assets to write.\n", - "INFO:tensorflow:SavedModel written to: resnet50/saved_model.pb\n" - ] - } - ], - "source": [ - "tf.saved_model.simple_save(\n", - " session = keras.backend.get_session(),\n", - " export_dir = model_dir,\n", - " inputs = {'input': model.inputs[0]},\n", - " outputs = {'output': model.outputs[0]})" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And then compile it for Inferentia to be used on only one Neuron core. `--static-weights` option is used to cache all weights onto the neuron core's memory." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:tensorflow:Restoring parameters from resnet50/variables/variables\n", - "INFO:tensorflow:Froze 320 variables.\n", - "INFO:tensorflow:Converted 320 variables to const ops.\n", - "INFO:tensorflow:fusing subgraph neuron_op_d6f098c01c780733 with neuron-cc\n", - "INFO:tensorflow:Number of operations in TensorFlow session: 4638\n", - "INFO:tensorflow:Number of operations after tf.neuron optimizations: 556\n", - "INFO:tensorflow:Number of operations placed on Neuron runtime: 554\n", - "INFO:tensorflow:No assets to save.\n", - "INFO:tensorflow:No assets to write.\n", - "INFO:tensorflow:SavedModel written to: resnet50_neuron/saved_model.pb\n", - "INFO:tensorflow:Successfully converted resnet50 to resnet50_neuron\n" - ] - }, - { - "data": { - "text/plain": [ - "{'OnNeuronRatio': 0.9964028776978417}" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "compiler_args = ['--static-weights', '--num-neuroncores', '1']\n", - "batch_size = 1\n", - "tfn.saved_model.compile(model_dir, compiled_model_dir, batch_size)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.9" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/test/apis/tensorflow/image-classifier-resnet50/handler.py b/test/apis/tensorflow/image-classifier-resnet50/handler.py deleted file mode 100644 index 3612a6455b..0000000000 --- a/test/apis/tensorflow/image-classifier-resnet50/handler.py +++ /dev/null @@ -1,61 +0,0 @@ -import os -import cv2 -import numpy as np -import requests -import imageio -import json -import base64 - - -def read_image(payload): - """ - Read JPG image from {"url": "https://..."} or from a bytes object. - """ - if isinstance(payload, bytes): - jpg_as_np = np.frombuffer(payload, dtype=np.uint8) - img = cv2.imdecode(jpg_as_np, flags=cv2.IMREAD_COLOR) - elif isinstance(payload, dict) and "url" in payload.keys(): - img = imageio.imread(payload["url"]) - else: - return None - return img - - -def prepare_image(image, input_shape, input_key): - """ - Prepares an image for the TFS client. - """ - img = cv2.resize(image, input_shape, interpolation=cv2.INTER_NEAREST) - img = {input_key: img[np.newaxis, ...]} - return img - - -class Handler: - def __init__(self, tensorflow_client, config): - self.client = tensorflow_client - - # load classes - classes = requests.get(config["classes"]).json() - self.idx2label = [classes[str(k)][1] for k in range(len(classes))] - - self.input_shape = tuple(config["input_shape"]) - self.input_key = str(config["input_key"]) - self.output_key = str(config["output_key"]) - - def handle_post(self, payload): - # preprocess image - img = read_image(payload) - if img is None: - return None - img = prepare_image(img, self.input_shape, self.input_key) - - # predict - results = self.client.predict(img)[self.output_key] - results = np.argsort(results) - - # Lookup and print the top 5 labels - top5_idx = results[-5:] - top5_labels = [self.idx2label[idx] for idx in top5_idx] - top5_labels = top5_labels[::-1] - - return top5_labels diff --git a/test/apis/tensorflow/image-classifier-resnet50/requirements.txt b/test/apis/tensorflow/image-classifier-resnet50/requirements.txt deleted file mode 100644 index fdb5b5287b..0000000000 --- a/test/apis/tensorflow/image-classifier-resnet50/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -imageio==2.9.* -opencv-python==4.4.0.42 diff --git a/test/apis/tensorflow/image-classifier-resnet50/sample.bin b/test/apis/tensorflow/image-classifier-resnet50/sample.bin deleted file mode 100644 index 921abf24a5c99cd3c1d1cd12d00f134a16391a77..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8680 zcmbVxYg`le*6v`cQB&uZa{9BbO)$v8W{2LJ+NxL z*0Y{xjkDd^g}nahij^x67Z(I^fq#heF%p4{b73CL^El@58jE=z|JrM;*CvdgFyYk~ zd(y-S>`Cki6DCfcILVcHz)w>qyG~*L!@SAgFC90Y#TxI*p1^)}$^Yodc?)r$h;$)c zESEQsaqccGcNga!1cP&qe|0=K?B5TUaje(IPhi6_Cc_04ufy?KEI9OdI5AwE0e?qc zb00r#Zs76>9xF zgAYTNghqTExgzS5m7lIz8~YV+-TDpjUw^Z8Tf+7oDSJ}C-z!WLWgo~nDE{$K?!S)w zeDv7yUrrRBDf#v6xzh6&F8)@2t>Sv+4aLp6`a2Dcs=L28J^bU();4YXqYj}*2L)_9WE}rJ!#(JFU_7?cDK0Bm>;Bj)53)IDzg8-!1n*IBKz;a{%>3z$P|_f z6p!VO;K*>(f-^M{>YxQ@auMcB9_hI|J5TOecjRR24cSbeirq2PN9JhTfQ@s=;N6k? z%)Ihq(O27!#ozlmCURxZ%!Q|>uO8o28aW+_iR5dB!;|=0{OWKha?j4I!FG&XzLKJ6 z?+-&&+(k~L?E9Nnb{itKVQ2|yxb@75#A~lOeiq^#;}w+^sxT@h0f4}L% zQ~c9YUH!S&79eMG7r5Lcp5QOF?^@)1C-T##UwepQFNI%DVHg22>(oDG<5QSgI}!e>I?31Wj}l;uP;4Wm3+hZDIwg(pq_Oyfi9^btyxD(AwoX84PVzeMp1KyAiUqT;k4a<~$m~v+?aFRDYVT*HD-f@yNZR$2$6M(G&4uueKmUL_St2 z&8w4g3pt7|C*qU9C%zT#r57&j`Py+d^{uM3c*Ape}$xV@=HZ zVcs)wN6!DLp+7el@PDDT3Gx4^yyrg+G49hRW}J!+R`-W#12)F7A)DQbJWaB6c4DDc z<(s5GSLQf~U&qU+NGI}!Npwzpx{?x4bnH+Ua6R4=+gJ2)LNr68t-dL|O)2zz(JcH_ zrqXdnEjBJ0IJAMIkS)>%0{YUm*}Wn!?PABlrMxS&W<$54_^?$z!*SY)=%cy_GR_;z z!>{}6QzF`LoqQ{&I3bhmxQOd-V1r)Jgx_DKLx?*PLqS=b#QtRu{#f~2TfRkk%ZNQ< z&yt6bmmCU1J{qOUdl%D*Os_Q;=-korut6CmNKvT?*O;<8d9iqCnP z=lgux9qMrMXjeu`gmy)E;&6m+W|kAtvB}t96Ea&Zva_;;!tPxEm&BAl!hBLVJL*-r z10`-eX`y>oP{PGMynEQX=`oRrr!;c9OWcK)NcOC_+|o$)W<}(-mSx^Qf90F7I{El} zRoc_ET7*&2L?zzimZ+9hX=Nl1oRz~fN37HOw>6D%c7~0X+ww@MBgY^iV>(%?zv8-X2!n%qk=TeuTj=^2T=E9AfCU9fr@m9vI%a=BGX9@bte z#V;f`Mv5YEzAc*Oi&mMHG0eRH%=zu4Z;urntCAG(%#Lj2%yWCeQ7hy(g)| zUw=$4Pkj2`obB_T`_(M(c~pMxkFf3bB|VuR(tl~F&H4apeqzVVR3;R2l+Bn`EpU_x z{K<%Qnp)4E&_;2kJ)T||K$a$Ib`o1nNkB8N?SZsTB5Zgb{wyH(q&zU(C%w6pPd|Z4 zpie8-woA)8Z5)Mti4)n}LkzLy2QodWU0Ph9Y?kXwse}H#l6I8O5TN5;MbycRsX)1c z_cCDqtvNj|cfsj#8El{K>Aof|;-A>4CmYPNZ)+(ayf_M9jHRsqCRkO7l;o zM8nhIX4!GAwS@M3I7U~L+UmnPt!bQ@U8m5yZND{^LGe0svp;K5?+?Pgh&%e4@;e(a zY7Per?X?J`c*R$hPGoYDfc`6qJmN(DY=8-5Pm!{vgy!0iykjvW>T%&q2{ifVcyEAy zVGSVWaUn)+P>yD6W6hVk2)H7LaJ4cV*$53UhkP9{bct<7YMm2#q`bo??lmlIMj9I$Ut1ZWsS;vhQd&&gW=F8bscpg zzciMu%M5{+P|24Q;^}`_d%QVEW$C*>cbMhtG{cMI=GvE}dysD$zC(Fh?%N8l7ugU0 z*;ijLx)(JC5HSdycX$ICt~ZesBFGM zH$^c3B)N+)sae_*;`S12rw>v0Cta^i2xqwKKZ z*O{^A&YEyJ84-rluh+kbqUPQlsWiqtw&6)_$NNQoBT>)S54DB|^xw}jtI5d6or5K` z`}ZT%rN5NAS%T%Gtr!`peLPs|xQML|^Pt_g6NO{(;u3(Q!+HOYoH^Qiq{O9@V*r=NL z+HZ6R{+kC?u?HCcPdYZbnQJYbYb3T~OER_0FiF5H^FVodQ&pr=M${$C`$9PigPV1% z#PV<2@1~lH4#yjn;do-eBLVFP9dJ^}QGU~eKU~gD_W%_tMjPtn*nJtg_#Fz z;p=Y+<1R65QD-)DB@Qve%kg`udlf?kWnF|%y1zwMUQD+XEdu6iB=n~&o*53a#E#w? z^P+*G59}S_-6za3(LM4k~X$Aw0kFQCfhD;!6yYBDyx zFWZjUx72<+g5JD_GvaHb)co(BxP!GTEvkku&|3z4kX;htD8YL|{3F!dv-Fde@W|DW zt?|N4u_gNY4ktndxb*sKg2IzF-iVG~WsNdTdNHOggp}XrIFVDUwP7p!w-etScAVb# zu3U0tub}NFHkGt|;Lr(X^-5=Z-I${l=;Zm5Yty2QajUi;dC1KU(7!Y6h^2q$9F&9+ zd)jD4SGfuLz?GXlC_nLRTb1M7J<*JRg-?y6?_q!8Bdu#`6*qgRG>N;vU|vxDsM*1JX>N}L;HEF9Fr@Cq^20)m-|T(fNH4h=a^&o zbomJl=&?E!8toE@$CXZ`7lzC8QyIaa&Ci=;+FK$HDBLN=VgntI4FwW|XDWm6{4pKf zPXBdTz0AP{--TJeJm05$Ehv0nhk zb{o)$gj|K|TYFNUq@yIyiA=;P9w$T|cHbt?k7H{;BzEr09n2Tp8Z+8vQhwgpY5Aw- znFY8mqy<{r;HIxmw25JcXzBTxi%hHbO@62S9xy<{7keYDvP%)9X5ZG@H0^g87YoFC zAO>#d*)(ZKDPc<+*_K5w_D%WWTvdKbgNjdklYc-lr!|V7eDzdpUtaKwC(MW)KKSbJ zc$N3_I1-Z)-)WDM1_Mma36iX_msJi0TpzlOSlMYlypJ$k8^Aki_dsIpjr18gVBV$; zS8f;@bsI1(?#ackUaB9)3pf=|5{T*ct>xkIH|V;{;VJN$Mo&dC*%@((ndE-wPosyN z$o3=_10zQ0ybz41;z*t>9o42v3t)h^1!oIH#O>u3PjRUGr=<1^Qddmv^1C(mT%=e( zBm^O1=7u?ugXhT$z?d-obu!|PVO<;kY*UF&V2Y6m4WObdx8++%gWVTChl&mgH9L{n z)a!71Oee6tbr3JJ=5MK4oG0d6RCUVS!tOozD67X>-&Y#~ z@@u4L#Qg*vQa{EkAPxF$1y=Q_i}c|+EjK6;Xe~M@jQ6%I^G}w2GCf~dVviC3In@EI z%rt?=gWdT6uCQcBSux3}HbNgAFc=Oh+G-0)o*9oIdBwndeMdifHyw4HbRs=VMq39u zN+2l*4Ez=fE#~=K*4WN#gKsmytGij!l`%z#MS#I%Q>zw)4ZVtWA{PYjm|sC+6CL7< z+EeT)5m(eXB1RxI?5+*bSeu5L>ra81g&(9a9l8&O$bi`KtJeFxSQ)Xo@92qs4(3oe z4n38altZ~MgVi~4)~E<_edlzS<2uHt56a8&uQ#DbR3gi!4gT|6N7gMOD-nv zrrfr+CJ+@XAYp+w;jA~bVN+$_%XqKS5CL7{C;_54-zW79LziK5=vK@V+VuhUvHS#) zpk|Or{pdt2er>_IV~MsdW*Q#iDg;aDcB!lN!DL8!&V?>wY-XY+L_N=pGYG?=9U%)U z-~^s!O~mhVUmBcnLo``T?c5u!bm(bV9lomX~uVEwD{?hu$P7F&Kw1dxP2P1-Cc>Q*|$3`!Dz zp!X2p?7zQiUxa2D&l>Yh6BGnLNYS5r)#gvg3t)ojF00|DvEhqdD7pMv=n$GddopkMu^1@}OO8uB@kEHJzyrh8@)BHk()u7pCx8$^ zke}Ft_v1MkvPeoZ?fEXdS*K zXV;j~G|N$0%uF~Tx5iGpZa2#zEA$ijzcNFZSDNLtpVSOs{g!B03PQ?+2*svZ7ZYxB z6sOSOoMQS>fNZ2{38Qqc|MeX8@35iT@q3;k@s zBmR^5t=x%112EH}hV((MyU?&PR0S2dRIeK-z$qu5adfS&_^pT~HU%^M3b zm^&Omfk)F|4Z7*o!eLsNwhT5Nwr)7)w9w%KkKZhfveX9C^nsIv)Y+_y?yKkF16`%@hV8GWaN(Zfw2RbGS zcD|6b>>i>V%(vt7SU4R-IgzV~D}H{gf{8KWm5Ib?zoSoxoo$%S#4buoAS`cadPi&~ z7zmXs#14gES+Z2~fxj@V`HFHs zUU}SfL7pCCF36J64+XDV|M_(8!w8oCB}#o^*LDKxW~v`n-RHqi$jj#!oCuD#W8~I> z+@}*jK4Fb)x_d4^s^AUn9v~NLmzi+fUnei8lFh#ejr6k2Mc%kBE?JiOpmX*x_dp_+ zleye9ZJs?4dym*GG(t1-FcuZ>C@ayHrMo$lLE+58%?NvhqZSn;q<}LKaAT}fvSAH03DhU`J`mx3;=_pZQmJski-^+yt*j4GkX284x*u;a|e^0|cE>zQ#E?n70 zFCe_g?`tzit$7k)aW8Zr0Q^VDZ&s919VV~BB!9|*e| zkEY-61sc0iKsy)|%NxM}he?eJMjt0&kO#(Pul%zRZ+#BL94z zwyhdAu;4b1wkAL}KLOTV;ol59x9fK84H(KlazoPBGTj{Lkd$cH>M`FoXayg0Vt;fF zQMN+S1)D1mNiIx?9M65cexrRZ^}6GvmEro(#h!hD3;Ut2e%?5Bul5ZYc57^a;XtJ&49?oHPn<<1gB^h5 z=)gHX1D*4cNfb57sDvUVs7V=ql}d)JL5MjW)@UDJ=%WmXn+I{js)YHVk~kia=~uw7 zM)-Iy;z`v;X?dv%HK}3p!E2{w#7CVTM(1joz4+AKRr#n2(nH+ceS_#I1q{vsncFR1 zN0b-cNL9l=u4wB%F)ZXechnS9sPzgnjJuh6gtK5(RC}G#G+ighm|uCQVLSH&H_;r` zZv=%0fJfyQxd7JN?^6&+pc;1etsHQaW;wVI3XLF6l9&l)X(kOLh@Yu|q%{jITB zcd*r(@nQXj+ETp$BU9>18{$81mH+r%Kw8DHmi-$A zfLFLlRE}##^_k;VQWg) zKS)zIMCZaTAq;TqDra;e6Lr2@4Qq?4fStu`!tM+Lg26OG8UU=nb3-f)jTK;z3w<+Y z6_W;-E@i{vC^TF{+P2Lap<$DT0i(x*U2QrhKOkJ8*MPnb)YsqSmN}6%o;wqzS1}dZ zv543iZ|6YqNI$$F47PV;Q&x-EM?suk#Z0q6i`kh5Hy4ogG~C4FpaFU>*@lCa;C*y)DWKKB3&HKP za3u2~QFqPEbGULj)Ws3X9~)0-N^}EF(4!=n`F6*S5o&&CKYdiH9XbxgA>uqN<86c# zKi^?fTxrGn7ixxn55i3yy~;AYd547h;gZ;*Qn?LtvR()*^Dlynr<%w?P-}|E4dSUbzXj zHn3&i=qTYb+aM7CuzC|*DP*R%;S^ev*1gRL@6!cf_QZ97paU4+BDOV3q9E_&Tic*3 z=!haz{xE5DD)@L1(HeHcO(Qiex{H9V>nmj>ddc-<>;j9a`uiCf!9ZQ$G+^YdW&?HA zzmxtBj4p0IYD)nYCCH)#V6Cst^%1&dJ~)-NfPU;mLYRox5s$CAv=W;cFCu32sNI+H zht2--zKbxG|4QB9iKJ^`DT2eTV<`9?y*z;^>nup@R6^j);jMs$$yJZ7@@0mw<9vs> nI+?jSC^F{xUjzvL0|#PdBqF?9&IK8 90%" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Hdwu-wzJvJLb", - "colab_type": "text" - }, - "source": [ - "## Export the model\n", - "Now we can export the model using [`Estimator.export_saved_model`](https://www.tensorflow.org/versions/r1.14/api_docs/python/tf/estimator/Estimator#export_saved_model):" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "AVgs2mkdllRn", - "colab_type": "code", - "colab": {} - }, - "source": [ - "def json_serving_input_fn():\n", - " placeholders = {}\n", - " features = {}\n", - " for feature_name in feature_names:\n", - " placeholders[feature_name] = tf.placeholder(shape=[None], dtype=tf.float64, name=feature_name)\n", - " features[feature_name] = tf.expand_dims(placeholders[feature_name], -1)\n", - " \n", - " return tf.estimator.export.ServingInputReceiver(features, receiver_tensors=placeholders)\n", - "\n", - "\n", - "classifier.export_saved_model(\"export\", json_serving_input_fn)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ipVlP4yPxFxw", - "colab_type": "text" - }, - "source": [ - "## Upload the model to AWS\n", - "\n", - "Cortex loads models from AWS, so we need to upload the exported model." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "3IqsfyylxLhy", - "colab_type": "text" - }, - "source": [ - "Set these variables to configure your AWS credentials and model upload path:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "lc9LBH1uHT_h", - "colab_type": "code", - "cellView": "form", - "colab": {} - }, - "source": [ - "AWS_ACCESS_KEY_ID = \"\" #@param {type:\"string\"}\n", - "AWS_SECRET_ACCESS_KEY = \"\" #@param {type:\"string\"}\n", - "S3_UPLOAD_PATH = \"s3://my-bucket/iris-classifier/tensorflow\" #@param {type:\"string\"}\n", - "\n", - "import sys\n", - "import re\n", - "\n", - "if AWS_ACCESS_KEY_ID == \"\":\n", - " print(\"\\033[91m{}\\033[00m\".format(\"ERROR: Please set AWS_ACCESS_KEY_ID\"), file=sys.stderr)\n", - "\n", - "elif AWS_SECRET_ACCESS_KEY == \"\":\n", - " print(\"\\033[91m{}\\033[00m\".format(\"ERROR: Please set AWS_SECRET_ACCESS_KEY\"), file=sys.stderr)\n", - "\n", - "else:\n", - " try:\n", - " bucket, key = re.match(\"s3://(.+?)/(.+)\", S3_UPLOAD_PATH).groups()\n", - " except:\n", - " print(\"\\033[91m{}\\033[00m\".format(\"ERROR: Invalid s3 path (should be of the form s3://my-bucket/path/to/file)\"), file=sys.stderr)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "NXeuZsaQxUc8", - "colab_type": "text" - }, - "source": [ - "Upload the model to S3:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "YLmnWTEVsu55", - "colab_type": "code", - "colab": {} - }, - "source": [ - "import os\n", - "import boto3\n", - "\n", - "s3 = boto3.client(\"s3\", aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)\n", - "\n", - "for dirpath, _, filenames in os.walk(\"export\"):\n", - " for filename in filenames:\n", - " filepath = os.path.join(dirpath, filename)\n", - " filekey = os.path.join(key, filepath[len(\"export/\"):])\n", - " print(\"Uploading s3://{}/{}...\".format(bucket, filekey), end = '')\n", - " s3.upload_file(filepath, bucket, filekey)\n", - " print(\" ✓\")", - "\n", - "print(\"\\nUploaded model export directory to \" + S3_UPLOAD_PATH)" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/test/apis/tensorflow/license-plate-reader/README.md b/test/apis/tensorflow/license-plate-reader/README.md deleted file mode 100644 index f650c86443..0000000000 --- a/test/apis/tensorflow/license-plate-reader/README.md +++ /dev/null @@ -1,173 +0,0 @@ -# Real-Time License Plate Identification System - -This project implements a license plate identification system. On resource-constrained systems, running inferences may prove to be too computationally expensive. One solution is to run the ML in the cloud and have the local (embedded) system act as a client of these services. - -![Demo GIF](https://i.imgur.com/jgkJB59.gif) - -*Figure 1 - GIF taken from this real-time recording [video](https://www.youtube.com/watch?v=gsYEZtecXlA) of predictions* - -![Raspberry Pi client with 4G access and onboard GPS that connects to cortex's APIs for inference](https://i.imgur.com/MvDAXWU.jpg) - -*Figure 2 - Raspberry Pi-powered client with 4G access and onboard GPS that connects to cortex's APIs for inference. More on that [here](https://github.com/RobertLucian/cortex-license-plate-reader-client).* - -In our example, we assume we have a dashcam mounted on a car and we want to detect and recognize all license plates in the video stream in real-time. We can use an embedded computer system to record the video, then stream and infer frame-by-frame using a web service, reassemble the stream with the licence plate annotations, and finally display the annotated stream on a screen. The web service in our case is a set of 2 web APIs deployed using cortex. - -## Used Models - -The identification of license plates is done in three steps: - -1. Detecting the bounding boxes of each license plate using *YOLOv3* model. -1. Detecting the very specific region of each word inside each bounding box with high accuracy using a pretrained *CRAFT* text detector. -1. Recognizing the text inside the previously detected boxes using a pretrained *CRNN* model. - -Out of these three models (*YOLOv3*, *CRAFT* and *CRNN*) only *YOLOv3* has been fine-tuned with a rather small dataset to better work with license plates. This dataset can be found [here](https://github.com/RobertLucian/license-plate-dataset). This *YOLOv3* model has in turn been trained using [this](https://github.com/experiencor/keras-yolo3) GitHub project. To get more details about our fine-tuned model, check the project's description page. - -The other two models, *CRAFT* and *CRNN*, can be found in [keras-ocr](https://github.com/faustomorales/keras-ocr). - -## Deployment - Lite Version - -A lite version of the deployment is available with `cortex_lite.yaml`. The lite version accepts an image as input and returns an image with the recognized license plates overlayed on top. A single GPU is required for this deployment (i.e. `g4dn.xlarge`). - -Once the cortex cluster is created, run - -```bash -cortex deploy cortex_lite.yaml -``` - -And monitor the API with - -```bash -cortex get --watch -``` - -To run an inference on the lite version, the only 3 tools you need are `curl`, `sed` and `base64`. This API expects an URL pointing to an image onto which the inferencing is done. This includes the detection of license plates with *YOLOv3* and the recognition part with *CRAFT* + *CRNN* models. - -Export the endpoint & the image's URL by running - -```bash -export ENDPOINT=your-api-endpoint -export IMAGE_URL=https://i.imgur.com/r8xdI7P.png -``` - -Then run the following piped commands - -```bash -curl "${ENDPOINT}" -X POST -H "Content-Type: application/json" -d '{"url":"'${IMAGE_URL}'"}' | -sed 's/"//g' | -base64 -d > prediction.jpg -``` - -The resulting image is the same as the one in [Verifying the Deployed APIs](#verifying-the-deployed-apis). - -For another prediction, let's use a generic image from the web. Export [this image's URL link](https://i.imgur.com/mYuvMOs.jpg) and re-run the prediction. This is what we get. - -![annotated sample image](https://i.imgur.com/tg1PE1E.jpg) - -*The above prediction has the bounding boxes colored differently to distinguish them from the cars' red bodies* - -## Deployment - Full Version - -The recommended number of instances to run this smoothly on a video stream is about 12 GPU instances (2 GPU instances for *YOLOv3* and 10 for *CRNN* + *CRAFT*). `cortex_full.yaml` is already set up to use these 12 instances. Note: this is the optimal number of instances when using the `g4dn.xlarge` instance type. For the client to work smoothly, the number of processes per replica can be adjusted, especially for `p3` or `g4` instances, where the GPU has a lot of compute capacity. - -If you don't have access to this many GPU-equipped instances, you could just lower the number and expect dropped frames. It will still prove the point, albeit at a much lower framerate and with higher latency. More on that [here](https://github.com/RobertLucian/cortex-license-plate-reader-client). - -Then after the cortex cluster is created, run - -```bash -cortex deploy cortex_full.yaml -``` - -And monitor the APIs with - -```bash -cortex get --watch -``` - -We can run the inference on a sample image to verify that both APIs are working as expected before we move on to running the client. Here is an example image: - -![sample image](https://i.imgur.com/r8xdI7P.png) - -On your local machine run: - -``` -pip install requests click opencv-contrib-python numpy -``` - -and run the following script with Python >= `3.6.x`. The application expects the argument to be a link to an image. The following link is for the above sample image. - - -```bash -export YOLOV3_ENDPOINT=api_endpoint_for_yolov3 -export CRNN_ENDPOINT=api_endpoint_for_crnn -python sample_inference.py "https://i.imgur.com/r8xdI7P.png" -``` - -If all goes well, then a prediction will be saved as a JPEG image to disk. By default, it's saved to `prediction.jpg`. Here is the output for the image above: - -![annotated sample image](https://i.imgur.com/JaD4A05.jpg) - -You can use `python sample_inference.py --help` to find out more. Keep in mind that any detected license plates with a confidence score lower than 80% are discarded. - -If this verification works, then we can move on and run the main client. - -### Running the Client - -Once the APIs are up and running, launch the streaming client by following the instructions at [robertlucian/cortex-license-plate-reader-client](https://github.com/RobertLucian/cortex-license-plate-reader-client). - -*Note: The client is kept in a separate repository to maintain the cortex project clean and focused. Keeping some of the projects that are more complex out of this repository can reduce the confusion.* - -## Customization/Optimization - -### Uploading the Model to S3 - -The only model to upload to an S3 bucket (for Cortex to deploy) is the *YOLOv3* model. The other two models are downloaded automatically upon deploying the service. - -If you would like to host the model from your own bucket, or if you want to fine tune the model for your needs, here's what you can do. - -#### Lite Version - -Download the *Keras* model: - -```bash -wget -O license_plate.h5 "https://www.dropbox.com/s/vsvgoyricooksyv/license_plate.h5?dl=0" -``` - -And then upload it to your bucket (also make sure [cortex_lite.yaml](cortex_lite.yaml) points to this bucket): - -```bash -BUCKET=my-bucket -YOLO3_PATH=examples/tensorflow/license-plate-reader/yolov3_keras -aws s3 cp license_plate.h5 "s3://$BUCKET/$YOLO3_PATH/model.h5" -``` - -#### Full Version - -Download the *SavedModel*: - -```bash -wget -O yolov3.zip "https://www.dropbox.com/sh/4ltffycnzfeul01/AAB7Xdmmi59w0EPOwhQ1nkvua/yolov3?dl=0" -``` - -Unzip it: - -```bash -unzip yolov3.zip -d yolov3 -``` - -And then upload it to your bucket (also make sure [cortex_full.yaml](cortex_full.yaml) points to this bucket): - -```bash -BUCKET=my-bucket -YOLO3_PATH=examples/tensorflow/license-plate-reader/yolov3_tf -aws s3 cp yolov3/ "s3://$BUCKET/$YOLO3_PATH" --recursive -``` - -### Configuring YOLOv3 Handler - -The `yolov3` API handler requires a [config.json](config.json) file to configure the input size of the image (dependent on the model's architecture), the anchor boxes, the object threshold, and the IoU threshold. All of these are already set appropriately so no other change is required. - -The configuration file's content is based on [this](https://github.com/experiencor/keras-yolo3/blob/bf37c87561caeccc4f1b879e313d4a3fec1b987e/zoo/config_license_plates.json#L2-L7). - -### Opportunities for performance improvements - -One way to reduce the inference time is to convert the models to use FP16/BFP16 (in mixed mode or not) and then choose the accelerator that gives the best performance in half precision mode - i.e. T4/V100. A speedup of an order of magnitude can be expected. diff --git a/test/apis/tensorflow/license-plate-reader/config.json b/test/apis/tensorflow/license-plate-reader/config.json deleted file mode 100644 index 0ff64d0a98..0000000000 --- a/test/apis/tensorflow/license-plate-reader/config.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "labels": ["license-plate"], - "net_h" : 416, - "net_w" : 416, - "anchors" : [15,6, 18,8, 22,9, 27,11, 32,13, 41,17, 54,21, 66,27, 82,33], - "obj_thresh" : 0.8, - "nms_thresh" : 0.01 -} diff --git a/test/apis/tensorflow/license-plate-reader/cortex_full.yaml b/test/apis/tensorflow/license-plate-reader/cortex_full.yaml deleted file mode 100644 index 27f8805467..0000000000 --- a/test/apis/tensorflow/license-plate-reader/cortex_full.yaml +++ /dev/null @@ -1,34 +0,0 @@ -- name: yolov3 - kind: RealtimeAPI - handler: - type: tensorflow - path: handler_yolo.py - models: - path: s3://cortex-examples/tensorflow/license-plate-reader/yolov3_tf/ - processes_per_replica: 4 - threads_per_process: 3 - signature_key: serving_default - config: - model_config: config.json - compute: - cpu: 1 - gpu: 1 - mem: 8G - autoscaling: - min_replicas: 2 - max_replicas: 2 - -- name: crnn - kind: RealtimeAPI - handler: - type: python - path: handler_crnn.py - processes_per_replica: 1 - threads_per_process: 1 - compute: - cpu: 1 - gpu: 1 - mem: 8G - autoscaling: - min_replicas: 10 - max_replicas: 10 diff --git a/test/apis/tensorflow/license-plate-reader/cortex_lite.yaml b/test/apis/tensorflow/license-plate-reader/cortex_lite.yaml deleted file mode 100644 index 5a9ffef71a..0000000000 --- a/test/apis/tensorflow/license-plate-reader/cortex_lite.yaml +++ /dev/null @@ -1,12 +0,0 @@ -- name: license-plate-reader - kind: RealtimeAPI - handler: - type: python - path: handler_lite.py - config: - yolov3: s3://cortex-examples/tensorflow/license-plate-reader/yolov3_keras/model.h5 - yolov3_model_config: config.json - compute: - cpu: 1 - gpu: 1 - mem: 4G diff --git a/test/apis/tensorflow/license-plate-reader/dependencies.sh b/test/apis/tensorflow/license-plate-reader/dependencies.sh deleted file mode 100644 index 057530cb85..0000000000 --- a/test/apis/tensorflow/license-plate-reader/dependencies.sh +++ /dev/null @@ -1 +0,0 @@ -apt-get update && apt-get install -y libgl1-mesa-glx libegl1-mesa diff --git a/test/apis/tensorflow/license-plate-reader/handler_crnn.py b/test/apis/tensorflow/license-plate-reader/handler_crnn.py deleted file mode 100644 index 3fee32cd2c..0000000000 --- a/test/apis/tensorflow/license-plate-reader/handler_crnn.py +++ /dev/null @@ -1,42 +0,0 @@ -import cv2 -import numpy as np -import keras_ocr -import base64 -import pickle -import tensorflow as tf - - -class Handler: - def __init__(self, config): - # limit memory usage on each process - for gpu in tf.config.list_physical_devices("GPU"): - tf.config.experimental.set_memory_growth(gpu, True) - - # keras-ocr will automatically download pretrained - # weights for the detector and recognizer. - self.pipeline = keras_ocr.pipeline.Pipeline() - - def handle_post(self, payload): - # preprocess the images w/ license plates (LPs) - imgs = payload["imgs"] - imgs = base64.b64decode(imgs.encode("utf-8")) - jpgs_as_np = pickle.loads(imgs) - images = [cv2.imdecode(jpg_as_np, flags=cv2.IMREAD_COLOR) for jpg_as_np in jpgs_as_np] - - # run batch inference - try: - prediction_groups = self.pipeline.recognize(images) - except ValueError: - # exception can occur when the images are too small - prediction_groups = [] - - image_list = [] - for img_predictions in prediction_groups: - boxes_per_image = [] - for predictions in img_predictions: - boxes_per_image.append([predictions[0], predictions[1].tolist()]) - image_list.append(boxes_per_image) - - lps = {"license-plates": image_list} - - return lps diff --git a/test/apis/tensorflow/license-plate-reader/handler_lite.py b/test/apis/tensorflow/license-plate-reader/handler_lite.py deleted file mode 100644 index e0e2b192e8..0000000000 --- a/test/apis/tensorflow/license-plate-reader/handler_lite.py +++ /dev/null @@ -1,113 +0,0 @@ -import boto3, base64, cv2, re, os, requests, json -import keras_ocr - -from botocore import UNSIGNED -from botocore.client import Config -from tensorflow.keras.models import load_model -import utils.utils as utils -import utils.bbox as bbox_utils -import utils.preprocess as preprocess_utils - - -class Handler: - def __init__(self, config): - # download yolov3 model - bucket, key = re.match("s3://(.+?)/(.+)", config["yolov3"]).groups() - model_path = "/tmp/model.h5" - s3 = boto3.client("s3") - s3.download_file(bucket, key, model_path) - - # load yolov3 model - self.yolov3_model = load_model(model_path) - - # get configuration for yolov3 model - with open(config["yolov3_model_config"]) as json_file: - data = json.load(json_file) - for key in data: - setattr(self, key, data[key]) - self.box_confidence_score = 0.8 - - # keras-ocr automatically downloads the pretrained - # weights for the detector and recognizer - self.recognition_model_pipeline = keras_ocr.pipeline.Pipeline() - - def handle_post(self, payload): - # download image - img_url = payload["url"] - image = preprocess_utils.get_url_image(img_url) - - # detect the bounding boxes - boxes = utils.get_yolo_boxes( - self.yolov3_model, - image, - self.net_h, - self.net_w, - self.anchors, - self.obj_thresh, - self.nms_thresh, - len(self.labels), - tensorflow_model=False, - ) - - # purge bounding boxes with a low confidence score - aux = [] - for b in boxes: - label = -1 - for i in range(len(b.classes)): - if b.classes[i] > self.box_confidence_score: - label = i - if label >= 0: - aux.append(b) - boxes = aux - del aux - - # if bounding boxes have been detected - dec_words = [] - if len(boxes) > 0: - # create set of images of the detected license plates - lps = [] - for b in boxes: - lp = image[b.ymin : b.ymax, b.xmin : b.xmax] - lps.append(lp) - - # run batch inference - try: - prediction_groups = self.recognition_model_pipeline.recognize(lps) - except ValueError: - # exception can occur when the images are too small - prediction_groups = [] - - # process pipeline output - image_list = [] - for img_predictions in prediction_groups: - boxes_per_image = [] - for predictions in img_predictions: - boxes_per_image.append([predictions[0], predictions[1].tolist()]) - image_list.append(boxes_per_image) - - # reorder text within detected LPs based on horizontal position - dec_lps = preprocess_utils.reorder_recognized_words(image_list) - for dec_lp in dec_lps: - dec_words.append([word[0] for word in dec_lp]) - - # if there are no recognized LPs, then don't draw them - if len(dec_words) == 0: - dec_words = [[] for i in range(len(boxes))] - - # draw predictions as overlays on the source image - draw_image = bbox_utils.draw_boxes( - image, - boxes, - overlay_text=dec_words, - labels=["LP"], - obj_thresh=self.box_confidence_score, - ) - - # image represented in bytes - byte_im = preprocess_utils.image_to_jpeg_bytes(draw_image) - - # encode image - image_enc = base64.b64encode(byte_im).decode("utf-8") - - # image with draw boxes overlayed - return image_enc diff --git a/test/apis/tensorflow/license-plate-reader/handler_yolo.py b/test/apis/tensorflow/license-plate-reader/handler_yolo.py deleted file mode 100644 index 3e1b0a7e1b..0000000000 --- a/test/apis/tensorflow/license-plate-reader/handler_yolo.py +++ /dev/null @@ -1,44 +0,0 @@ -import json -import base64 -import numpy as np -import cv2 -import pickle -import utils.utils as utils - - -class Handler: - def __init__(self, tensorflow_client, config): - self.client = tensorflow_client - - with open(config["model_config"]) as json_file: - data = json.load(json_file) - for key in data: - setattr(self, key, data[key]) - - def handle_post(self, payload): - # decode the payload - img = payload["img"] - img = base64.b64decode(img) - jpg_as_np = np.frombuffer(img, dtype=np.uint8) - image = cv2.imdecode(jpg_as_np, flags=cv2.IMREAD_COLOR) - - # detect the bounding boxes - boxes = utils.get_yolo_boxes( - self.client, - image, - self.net_h, - self.net_w, - self.anchors, - self.obj_thresh, - self.nms_thresh, - len(self.labels), - ) - - # package the response - response = {"boxes": []} - for box in boxes: - response["boxes"].append( - [box.xmin, box.ymin, box.xmax, box.ymax, float(box.c), box.classes.tolist()] - ) - - return response diff --git a/test/apis/tensorflow/license-plate-reader/requirements.txt b/test/apis/tensorflow/license-plate-reader/requirements.txt deleted file mode 100644 index 0fb87fcf23..0000000000 --- a/test/apis/tensorflow/license-plate-reader/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -keras-ocr==0.8.5 -keras==2.3.1 -tensorflow==2.3.0 -scipy==1.4.1 -numpy==1.18.* diff --git a/test/apis/tensorflow/license-plate-reader/sample_inference.py b/test/apis/tensorflow/license-plate-reader/sample_inference.py deleted file mode 100644 index 2bfeea9571..0000000000 --- a/test/apis/tensorflow/license-plate-reader/sample_inference.py +++ /dev/null @@ -1,98 +0,0 @@ -import click, cv2, requests, pickle, base64, json -import numpy as np -import utils.bbox as bbox_utils -import utils.preprocess as preprocess_utils - - -@click.command( - help=( - "Identify license plates in a given image" - " while outsourcing the predictions using the REST API endpoints." - " Both API endpoints have to be exported as environment variables." - ) -) -@click.argument("img_url_src", type=str) -@click.argument("yolov3_endpoint", envvar="YOLOV3_ENDPOINT") -@click.argument("crnn_endpoint", envvar="CRNN_ENDPOINT") -@click.option( - "--output", - "-o", - type=str, - default="prediction.jpg", - show_default=True, - help="File to save the prediction to.", -) -def main(img_url_src, yolov3_endpoint, crnn_endpoint, output): - - # get the image in bytes representation - image = preprocess_utils.get_url_image(img_url_src) - image_bytes = preprocess_utils.image_to_jpeg_bytes(image) - - # encode image - image_enc = base64.b64encode(image_bytes).decode("utf-8") - image_dump = json.dumps({"img": image_enc}) - - # make yolov3 api request - resp = requests.post( - yolov3_endpoint, data=image_dump, headers={"content-type": "application/json"} - ) - - # parse response - boxes_raw = resp.json()["boxes"] - boxes = [] - for b in boxes_raw: - box = bbox_utils.BoundBox(*b) - boxes.append(box) - - # purge bounding boxes with a low confidence score - confidence_score = 0.8 - aux = [] - for b in boxes: - label = -1 - for i in range(len(b.classes)): - if b.classes[i] > confidence_score: - label = i - if label >= 0: - aux.append(b) - boxes = aux - del aux - - dec_words = [] - if len(boxes) > 0: - # create set of images of the detected license plates - lps = [] - for b in boxes: - lp = image[b.ymin : b.ymax, b.xmin : b.xmax] - jpeg = preprocess_utils.image_to_jpeg_nparray(lp) - lps.append(jpeg) - - # encode the cropped license plates - lps = pickle.dumps(lps, protocol=0) - lps_enc = base64.b64encode(lps).decode("utf-8") - lps_dump = json.dumps({"imgs": lps_enc}) - - # make crnn api request - resp = requests.post( - crnn_endpoint, data=lps_dump, headers={"content-type": "application/json"} - ) - - # parse the response - dec_lps = resp.json()["license-plates"] - dec_lps = preprocess_utils.reorder_recognized_words(dec_lps) - for dec_lp in dec_lps: - dec_words.append([word[0] for word in dec_lp]) - - if len(dec_words) == 0: - dec_words = [[] for i in range(len(boxes))] - - # draw predictions as overlays on the source image - draw_image = bbox_utils.draw_boxes( - image, boxes, overlay_text=dec_words, labels=["LP"], obj_thresh=confidence_score - ) - - # and save it to disk - cv2.imwrite(output, draw_image) - - -if __name__ == "__main__": - main() diff --git a/test/apis/tensorflow/license-plate-reader/utils/__init__.py b/test/apis/tensorflow/license-plate-reader/utils/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/apis/tensorflow/license-plate-reader/utils/bbox.py b/test/apis/tensorflow/license-plate-reader/utils/bbox.py deleted file mode 100644 index 3126076037..0000000000 --- a/test/apis/tensorflow/license-plate-reader/utils/bbox.py +++ /dev/null @@ -1,109 +0,0 @@ -import numpy as np -import cv2 -from .colors import get_color - - -class BoundBox: - def __init__(self, xmin, ymin, xmax, ymax, c=None, classes=None): - self.xmin = xmin - self.ymin = ymin - self.xmax = xmax - self.ymax = ymax - - self.c = c - self.classes = classes - - self.label = -1 - self.score = -1 - - def get_label(self): - if self.label == -1: - self.label = np.argmax(self.classes) - - return self.label - - def get_score(self): - if self.score == -1: - self.score = self.classes[self.get_label()] - - return self.score - - -def _interval_overlap(interval_a, interval_b): - x1, x2 = interval_a - x3, x4 = interval_b - - if x3 < x1: - if x4 < x1: - return 0 - else: - return min(x2, x4) - x1 - else: - if x2 < x3: - return 0 - else: - return min(x2, x4) - x3 - - -def bbox_iou(box1, box2): - intersect_w = _interval_overlap([box1.xmin, box1.xmax], [box2.xmin, box2.xmax]) - intersect_h = _interval_overlap([box1.ymin, box1.ymax], [box2.ymin, box2.ymax]) - - intersect = intersect_w * intersect_h - - w1, h1 = box1.xmax - box1.xmin, box1.ymax - box1.ymin - w2, h2 = box2.xmax - box2.xmin, box2.ymax - box2.ymin - - union = w1 * h1 + w2 * h2 - intersect - - return float(intersect) / union - - -def draw_boxes(image, boxes, overlay_text, labels, obj_thresh, quiet=True): - for box, overlay in zip(boxes, overlay_text): - label_str = "" - label = -1 - - for i in range(len(labels)): - if box.classes[i] > obj_thresh: - if label_str != "": - label_str += ", " - label_str += labels[i] + " " + str(round(box.get_score() * 100, 2)) + "%" - label = i - if not quiet: - print(label_str) - - if label >= 0: - if len(overlay) > 0: - text = label_str + ": [" + " ".join(overlay) + "]" - else: - text = label_str - text = text.upper() - text_size = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, 1.1e-3 * image.shape[0], 5) - width, height = text_size[0][0], text_size[0][1] - region = np.array( - [ - [box.xmin - 3, box.ymin], - [box.xmin - 3, box.ymin - height - 26], - [box.xmin + width + 13, box.ymin - height - 26], - [box.xmin + width + 13, box.ymin], - ], - dtype="int32", - ) - - # cv2.rectangle(img=image, pt1=(box.xmin,box.ymin), pt2=(box.xmax,box.ymax), color=get_color(label), thickness=5) - rec = (box.xmin, box.ymin, box.xmax - box.xmin, box.ymax - box.ymin) - rec = tuple(int(i) for i in rec) - cv2.rectangle(img=image, rec=rec, color=get_color(label), thickness=3) - cv2.fillPoly(img=image, pts=[region], color=get_color(label)) - cv2.putText( - img=image, - text=text, - org=(box.xmin + 13, box.ymin - 13), - fontFace=cv2.FONT_HERSHEY_SIMPLEX, - fontScale=1e-3 * image.shape[0], - color=(0, 0, 0), - thickness=1, - ) - - return image diff --git a/test/apis/tensorflow/license-plate-reader/utils/colors.py b/test/apis/tensorflow/license-plate-reader/utils/colors.py deleted file mode 100644 index 3646fc1bc0..0000000000 --- a/test/apis/tensorflow/license-plate-reader/utils/colors.py +++ /dev/null @@ -1,97 +0,0 @@ -def get_color(label): - """Return a color from a set of predefined colors. Contains 80 colors in total. - code originally from https://github.com/fizyr/keras-retinanet/ - Args - label: The label to get the color for. - Returns - A list of three values representing a RGB color. - """ - if label < len(colors): - return colors[label] - else: - print("Label {} has no color, returning default.".format(label)) - return (0, 255, 0) - - -colors = [ - [31, 0, 255], - [0, 159, 255], - [255, 95, 0], - [255, 19, 0], - [255, 0, 0], - [255, 38, 0], - [0, 255, 25], - [255, 0, 133], - [255, 172, 0], - [108, 0, 255], - [0, 82, 255], - [0, 255, 6], - [255, 0, 152], - [223, 0, 255], - [12, 0, 255], - [0, 255, 178], - [108, 255, 0], - [184, 0, 255], - [255, 0, 76], - [146, 255, 0], - [51, 0, 255], - [0, 197, 255], - [255, 248, 0], - [255, 0, 19], - [255, 0, 38], - [89, 255, 0], - [127, 255, 0], - [255, 153, 0], - [0, 255, 255], - [0, 255, 216], - [0, 255, 121], - [255, 0, 248], - [70, 0, 255], - [0, 255, 159], - [0, 216, 255], - [0, 6, 255], - [0, 63, 255], - [31, 255, 0], - [255, 57, 0], - [255, 0, 210], - [0, 255, 102], - [242, 255, 0], - [255, 191, 0], - [0, 255, 63], - [255, 0, 95], - [146, 0, 255], - [184, 255, 0], - [255, 114, 0], - [0, 255, 235], - [255, 229, 0], - [0, 178, 255], - [255, 0, 114], - [255, 0, 57], - [0, 140, 255], - [0, 121, 255], - [12, 255, 0], - [255, 210, 0], - [0, 255, 44], - [165, 255, 0], - [0, 25, 255], - [0, 255, 140], - [0, 101, 255], - [0, 255, 82], - [223, 255, 0], - [242, 0, 255], - [89, 0, 255], - [165, 0, 255], - [70, 255, 0], - [255, 0, 172], - [255, 76, 0], - [203, 255, 0], - [204, 0, 255], - [255, 0, 229], - [255, 133, 0], - [127, 0, 255], - [0, 235, 255], - [0, 255, 197], - [255, 0, 191], - [0, 44, 255], - [50, 255, 0], -] diff --git a/test/apis/tensorflow/license-plate-reader/utils/preprocess.py b/test/apis/tensorflow/license-plate-reader/utils/preprocess.py deleted file mode 100644 index 8122c0bb09..0000000000 --- a/test/apis/tensorflow/license-plate-reader/utils/preprocess.py +++ /dev/null @@ -1,57 +0,0 @@ -import numpy as np -import cv2, requests -from statistics import mean - - -def get_url_image(url_image): - """ - Get numpy image from URL image. - """ - resp = requests.get(url_image, stream=True).raw - image = np.asarray(bytearray(resp.read()), dtype="uint8") - image = cv2.imdecode(image, cv2.IMREAD_COLOR) - return image - - -def image_to_jpeg_nparray(image, quality=[int(cv2.IMWRITE_JPEG_QUALITY), 95]): - """ - Convert numpy image to jpeg numpy vector. - """ - is_success, im_buf_arr = cv2.imencode(".jpg", image, quality) - return im_buf_arr - - -def image_to_jpeg_bytes(image, quality=[int(cv2.IMWRITE_JPEG_QUALITY), 95]): - """ - Convert numpy image to bytes-encoded jpeg image. - """ - buf = image_to_jpeg_nparray(image, quality) - byte_im = buf.tobytes() - return byte_im - - -def reorder_recognized_words(detected_images): - """ - Reorder the detected words in each image based on the average horizontal position of each word. - Sorting them in ascending order. - """ - - reordered_images = [] - for detected_image in detected_images: - - # computing the mean average position for each word - mean_horizontal_positions = [] - for words in detected_image: - box = words[1] - y_positions = [point[0] for point in box] - mean_y_position = mean(y_positions) - mean_horizontal_positions.append(mean_y_position) - indexes = np.argsort(mean_horizontal_positions) - - # and reordering them - reordered = [] - for index, words in zip(indexes, detected_image): - reordered.append(detected_image[index]) - reordered_images.append(reordered) - - return reordered_images diff --git a/test/apis/tensorflow/license-plate-reader/utils/utils.py b/test/apis/tensorflow/license-plate-reader/utils/utils.py deleted file mode 100644 index efadb6e69f..0000000000 --- a/test/apis/tensorflow/license-plate-reader/utils/utils.py +++ /dev/null @@ -1,158 +0,0 @@ -import cv2 -import numpy as np -import math -from .bbox import BoundBox, bbox_iou -from scipy.special import expit - - -def _sigmoid(x): - return expit(x) - - -def correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w): - if (float(net_w) / image_w) < (float(net_h) / image_h): - new_w = net_w - new_h = (image_h * net_w) / image_w - else: - new_h = net_w - new_w = (image_w * net_h) / image_h - - for i in range(len(boxes)): - x_offset, x_scale = (net_w - new_w) / 2.0 / net_w, float(new_w) / net_w - y_offset, y_scale = (net_h - new_h) / 2.0 / net_h, float(new_h) / net_h - - boxes[i].xmin = int((boxes[i].xmin - x_offset) / x_scale * image_w) - boxes[i].xmax = int((boxes[i].xmax - x_offset) / x_scale * image_w) - boxes[i].ymin = int((boxes[i].ymin - y_offset) / y_scale * image_h) - boxes[i].ymax = int((boxes[i].ymax - y_offset) / y_scale * image_h) - - -def do_nms(boxes, nms_thresh): - if len(boxes) > 0: - nb_class = len(boxes[0].classes) - else: - return - - for c in range(nb_class): - sorted_indices = np.argsort([-box.classes[c] for box in boxes]) - - for i in range(len(sorted_indices)): - index_i = sorted_indices[i] - - if boxes[index_i].classes[c] == 0: - continue - - for j in range(i + 1, len(sorted_indices)): - index_j = sorted_indices[j] - - if bbox_iou(boxes[index_i], boxes[index_j]) >= nms_thresh: - boxes[index_j].classes[c] = 0 - - -def decode_netout(netout, anchors, obj_thresh, net_h, net_w): - grid_h, grid_w = netout.shape[:2] - nb_box = 3 - netout = netout.reshape((grid_h, grid_w, nb_box, -1)) - nb_class = netout.shape[-1] - 5 - - boxes = [] - - netout[..., :2] = _sigmoid(netout[..., :2]) - netout[..., 4] = _sigmoid(netout[..., 4]) - netout[..., 5:] = netout[..., 4][..., np.newaxis] * _softmax(netout[..., 5:]) - netout[..., 5:] *= netout[..., 5:] > obj_thresh - - for i in range(grid_h * grid_w): - row = i // grid_w - col = i % grid_w - - for b in range(nb_box): - # 4th element is objectness score - objectness = netout[row, col, b, 4] - - if objectness <= obj_thresh: - continue - - # first 4 elements are x, y, w, and h - x, y, w, h = netout[row, col, b, :4] - - x = (col + x) / grid_w # center position, unit: image width - y = (row + y) / grid_h # center position, unit: image height - w = anchors[2 * b + 0] * np.exp(w) / net_w # unit: image width - h = anchors[2 * b + 1] * np.exp(h) / net_h # unit: image height - - # last elements are class probabilities - classes = netout[row, col, b, 5:] - - box = BoundBox(x - w / 2, y - h / 2, x + w / 2, y + h / 2, objectness, classes) - - boxes.append(box) - - return boxes - - -def preprocess_input(image, net_h, net_w): - new_h, new_w, _ = image.shape - - # determine the new size of the image - if (float(net_w) / new_w) < (float(net_h) / new_h): - new_h = (new_h * net_w) // new_w - new_w = net_w - else: - new_w = (new_w * net_h) // new_h - new_h = net_h - - # resize the image to the new size - resized = cv2.resize(image[:, :, ::-1] / 255.0, (new_w, new_h)) - - # embed the image into the standard letter box - new_image = np.ones((net_h, net_w, 3)) * 0.5 - new_image[ - (net_h - new_h) // 2 : (net_h + new_h) // 2, (net_w - new_w) // 2 : (net_w + new_w) // 2, : - ] = resized - new_image = np.expand_dims(new_image, 0) - - return new_image - - -def get_yolo_boxes( - model, image, net_h, net_w, anchors, obj_thresh, nms_thresh, classes, tensorflow_model=True -): - # preprocess the input - image_h, image_w, _ = image.shape - batch_input = np.zeros((1, net_h, net_w, 3)) - batch_input[0] = preprocess_input(image, net_h, net_w) - - # run the prediction - if tensorflow_model: - output = model.predict({"input_1": batch_input}) - yolos = [output["conv_81"], output["conv_93"], output["conv_105"]] - filters = 3 * (5 + classes) - for i in range(len(yolos)): - length = len(yolos[i]) - box_size = int(math.sqrt(length / filters)) - yolos[i] = np.array(yolos[i]).reshape((box_size, box_size, filters)) - else: - output = model.predict_on_batch(batch_input) - yolos = [output[0][0], output[1][0], output[2][0]] - - boxes = [] - # decode the output of the network - for j in range(len(yolos)): - yolo_anchors = anchors[(2 - j) * 6 : (3 - j) * 6] # config['model']['anchors'] - boxes += decode_netout(yolos[j], yolo_anchors, obj_thresh, net_h, net_w) - - # correct the sizes of the bounding boxes - correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w) - - # suppress non-maximal boxes - do_nms(boxes, nms_thresh) - - return boxes - - -def _softmax(x, axis=-1): - x = x - np.amax(x, axis, keepdims=True) - e_x = np.exp(x) - - return e_x / e_x.sum(axis, keepdims=True) diff --git a/test/apis/tensorflow/multi-model-classifier/README.md b/test/apis/tensorflow/multi-model-classifier/README.md deleted file mode 100644 index 61334ee155..0000000000 --- a/test/apis/tensorflow/multi-model-classifier/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# Multi-Model Classifier API - -This example deploys Iris, ResNet50 and Inception models in one API. Query parameters are used for selecting the model. - -The example can be run on both CPU and on GPU hardware. - -## Sample Prediction - -Deploy the model by running: - -```bash -cortex deploy -``` - -And wait for it to become live by tracking its status with `cortex get --watch`. - -Once the API has been successfully deployed, export the APIs endpoint. You can get the API's endpoint by running `cortex get multi-model-classifier`. - -```bash -export ENDPOINT=your-api-endpoint -``` - -When making a prediction with [sample-image.json](sample-image.json), the following image will be used: - -![sports car](https://i.imgur.com/zovGIKD.png) - -### ResNet50 Classifier - -Make a request to the ResNet50 model: - -```bash -curl "${ENDPOINT}?model=resnet50" -X POST -H "Content-Type: application/json" -d @sample-image.json -``` - -The expected response is: - -```json -{"label": "sports_car"} -``` - -### Inception Classifier - -Make a request to the Inception model: - -```bash -curl "${ENDPOINT}?model=inception" -X POST -H "Content-Type: application/json" -d @sample-image.json -``` - -The expected response is: - -```json -{"label": "sports_car"} -``` - -### Iris Classifier - -Make a request to the Iris model: - -```bash -curl "${ENDPOINT}?model=iris" -X POST -H "Content-Type: application/json" -d @sample-iris.json -``` - -The expected response is: - -```json -{"label": "setosa"} -``` diff --git a/test/apis/tensorflow/multi-model-classifier/cortex.yaml b/test/apis/tensorflow/multi-model-classifier/cortex.yaml deleted file mode 100644 index 8aa366346c..0000000000 --- a/test/apis/tensorflow/multi-model-classifier/cortex.yaml +++ /dev/null @@ -1,28 +0,0 @@ -- name: multi-model-classifier - kind: RealtimeAPI - handler: - type: tensorflow - path: handler.py - models: - paths: - - name: inception - path: s3://cortex-examples/tensorflow/image-classifier/inception/ - - name: iris - path: s3://cortex-examples/tensorflow/iris-classifier/nn/ - - name: resnet50 - path: s3://cortex-examples/tensorflow/resnet50/ - config: - models: - iris: - labels: ["setosa", "versicolor", "virginica"] - resnet50: - input_shape: [224, 224] - input_key: input - output_key: output - inception: - input_shape: [224, 224] - input_key: images - output_key: classes - image-classifier-classes: https://s3.amazonaws.com/deep-learning-models/image-models/imagenet_class_index.json - compute: - mem: 2G diff --git a/test/apis/tensorflow/multi-model-classifier/dependencies.sh b/test/apis/tensorflow/multi-model-classifier/dependencies.sh deleted file mode 100644 index 057530cb85..0000000000 --- a/test/apis/tensorflow/multi-model-classifier/dependencies.sh +++ /dev/null @@ -1 +0,0 @@ -apt-get update && apt-get install -y libgl1-mesa-glx libegl1-mesa diff --git a/test/apis/tensorflow/multi-model-classifier/handler.py b/test/apis/tensorflow/multi-model-classifier/handler.py deleted file mode 100644 index 7d1b0ffe97..0000000000 --- a/test/apis/tensorflow/multi-model-classifier/handler.py +++ /dev/null @@ -1,60 +0,0 @@ -import requests -import numpy as np -import cv2 - - -def get_url_image(url_image): - """ - Get numpy image from URL image. - """ - resp = requests.get(url_image, stream=True).raw - image = np.asarray(bytearray(resp.read()), dtype="uint8") - image = cv2.imdecode(image, cv2.IMREAD_COLOR) - image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) - return image - - -class Handler: - def __init__(self, tensorflow_client, config): - self.client = tensorflow_client - - # for image classifiers - classes = requests.get(config["image-classifier-classes"]).json() - self.image_classes = [classes[str(k)][1] for k in range(len(classes))] - - # assign "models"' key value to self.config for ease of use - self.config = config["models"] - - # for iris classifier - self.iris_labels = self.config["iris"]["labels"] - - def handle_post(self, payload, query_params): - model_name = query_params["model"] - predicted_label = None - - if model_name == "iris": - prediction = self.client.predict(payload["input"], model_name) - predicted_class_id = int(prediction["class_ids"][0]) - predicted_label = self.iris_labels[predicted_class_id] - - elif model_name in ["resnet50", "inception"]: - predicted_label = self.predict_image_classifier(model_name, payload["url"]) - - return {"label": predicted_label} - - def predict_image_classifier(self, model, img_url): - img = get_url_image(img_url) - img = cv2.resize( - img, tuple(self.config[model]["input_shape"]), interpolation=cv2.INTER_NEAREST - ) - if model == "inception": - img = img.astype("float32") / 255 - img = {self.config[model]["input_key"]: img[np.newaxis, ...]} - - results = self.client.predict(img, model)[self.config[model]["output_key"]] - result = np.argmax(results) - if model == "inception": - result -= 1 - predicted_label = self.image_classes[result] - - return predicted_label diff --git a/test/apis/tensorflow/multi-model-classifier/requirements.txt b/test/apis/tensorflow/multi-model-classifier/requirements.txt deleted file mode 100644 index 16682773e3..0000000000 --- a/test/apis/tensorflow/multi-model-classifier/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -Pillow -opencv-python==4.4.0.42 diff --git a/test/apis/tensorflow/multi-model-classifier/sample-image.json b/test/apis/tensorflow/multi-model-classifier/sample-image.json deleted file mode 100644 index 95200916c7..0000000000 --- a/test/apis/tensorflow/multi-model-classifier/sample-image.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "url": "https://i.imgur.com/zovGIKD.png" -} diff --git a/test/apis/tensorflow/multi-model-classifier/sample-iris.json b/test/apis/tensorflow/multi-model-classifier/sample-iris.json deleted file mode 100644 index 67c03827f2..0000000000 --- a/test/apis/tensorflow/multi-model-classifier/sample-iris.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "input": { - "sepal_length": 5.2, - "sepal_width": 3.6, - "petal_length": 1.4, - "petal_width": 0.3 - } -} diff --git a/test/apis/tensorflow/sentiment-analyzer/bert.ipynb b/test/apis/tensorflow/sentiment-analyzer/bert.ipynb deleted file mode 100644 index c6854e4c9e..0000000000 --- a/test/apis/tensorflow/sentiment-analyzer/bert.ipynb +++ /dev/null @@ -1,993 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "bert.ipynb", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "code", - "metadata": { - "id": "j0a4mTk9o1Qg", - "colab_type": "code", - "colab": {} - }, - "source": [ - "# Modified source from https://colab.research.google.com/github/google-research/bert/blob/master/predicting_movie_reviews_with_bert_on_tf_hub.ipynb\n", - "\n", - "# Copyright 2019 Google Inc.\n", - "\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "dCpvgG0vwXAZ", - "colab_type": "text" - }, - "source": [ - "#Predicting Movie Review Sentiment with BERT on TF Hub\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "xiYrZKaHwV81", - "colab_type": "text" - }, - "source": [ - "If you’ve been following Natural Language Processing over the past year, you’ve probably heard of BERT: Bidirectional Encoder Representations from Transformers. It’s a neural network architecture designed by Google researchers that’s totally transformed what’s state-of-the-art for NLP tasks, like text classification, translation, summarization, and question answering.\n", - "\n", - "Now that BERT's been added to [TF Hub](https://www.tensorflow.org/hub) as a loadable module, it's easy(ish) to add into existing TensorFlow text pipelines. In an existing pipeline, BERT can replace text embedding layers like ELMO and GloVE. Alternatively, [finetuning](http://wiki.fast.ai/index.php/Fine_tuning) BERT can provide both an accuracy boost and faster training time in many cases.\n", - "\n", - "Here, we'll train a model to predict whether an IMDB movie review is positive or negative using BERT in TensorFlow with tf hub. Some code was adapted from [this colab notebook](https://colab.sandbox.google.com/github/tensorflow/tpu/blob/master/tools/colab/bert_finetuning_with_cloud_tpus.ipynb). Let's get started!" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "chM4UttbMIqq", - "colab_type": "text" - }, - "source": [ - "First, we'll install the required packages:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "jviywGyWyKsA", - "colab_type": "code", - "colab": {} - }, - "source": [ - "!pip install bert-tensorflow==1.0.* tensorflow-gpu==1.13.* scikit-learn==0.21.* pandas==0.24.* tensorflow-hub==0.6.* boto3==1.*" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "hsZvic2YxnTz", - "colab_type": "code", - "colab": {} - }, - "source": [ - "from datetime import datetime\n", - "\n", - "from sklearn.model_selection import train_test_split\n", - "import pandas as pd\n", - "import tensorflow as tf\n", - "import tensorflow_hub as hub\n", - "\n", - "import bert\n", - "from bert import run_classifier\n", - "from bert import optimization\n", - "from bert import tokenization" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "KVB3eOcjxxm1", - "colab_type": "text" - }, - "source": [ - "Below, we'll set an output location to store our model output, checkpoints, and export in a local directory. Note: if you're running on Google Colab, local directories don't persist after the session ends." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "US_EAnICvP7f", - "colab_type": "code", - "colab": {} - }, - "source": [ - "OUTPUT_DIR = \"bert\"\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "pmFYvkylMwXn", - "colab_type": "text" - }, - "source": [ - "#Data" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "MC_w8SRqN0fr", - "colab_type": "text" - }, - "source": [ - "First, let's download the dataset, hosted by Stanford. The code below, which downloads, extracts, and imports the IMDB Large Movie Review Dataset, is borrowed from [this TensorFlow tutorial](https://www.tensorflow.org/hub/tutorials/text_classification_with_tf_hub)." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "fom_ff20gyy6", - "colab_type": "code", - "colab": {} - }, - "source": [ - "from tensorflow import keras\n", - "import os\n", - "import re\n", - "\n", - "# Load all files from a directory in a DataFrame.\n", - "def load_directory_data(directory):\n", - " data = {}\n", - " data[\"sentence\"] = []\n", - " data[\"sentiment\"] = []\n", - " for file_path in os.listdir(directory):\n", - " with tf.gfile.GFile(os.path.join(directory, file_path), \"r\") as f:\n", - " data[\"sentence\"].append(f.read())\n", - " data[\"sentiment\"].append(re.match(\"\\d+_(\\d+)\\.txt\", file_path).group(1))\n", - " return pd.DataFrame.from_dict(data)\n", - "\n", - "# Merge positive and negative examples, add a polarity column and shuffle.\n", - "def load_dataset(directory):\n", - " pos_df = load_directory_data(os.path.join(directory, \"pos\"))\n", - " neg_df = load_directory_data(os.path.join(directory, \"neg\"))\n", - " pos_df[\"polarity\"] = 1\n", - " neg_df[\"polarity\"] = 0\n", - " return pd.concat([pos_df, neg_df]).sample(frac=1).reset_index(drop=True)\n", - "\n", - "# Download and process the dataset files.\n", - "def download_and_load_datasets(force_download=False):\n", - " dataset = tf.keras.utils.get_file(\n", - " fname=\"aclImdb.tar.gz\", \n", - " origin=\"http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\", \n", - " extract=True)\n", - " \n", - " train_df = load_dataset(os.path.join(os.path.dirname(dataset), \n", - " \"aclImdb\", \"train\"))\n", - " test_df = load_dataset(os.path.join(os.path.dirname(dataset), \n", - " \"aclImdb\", \"test\"))\n", - " \n", - " return train_df, test_df\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "2abfwdn-g135", - "colab_type": "code", - "colab": {} - }, - "source": [ - "train, test = download_and_load_datasets()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "XA8WHJgzhIZf", - "colab_type": "text" - }, - "source": [ - "To keep training fast, we'll take a sample of 5000 train and test examples, respectively." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "lw_F488eixTV", - "colab_type": "code", - "colab": {} - }, - "source": [ - "train = train.sample(5000)\n", - "test = test.sample(5000)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "prRQM8pDi8xI", - "colab_type": "code", - "colab": {} - }, - "source": [ - "train.columns" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "sfRnHSz3iSXz", - "colab_type": "text" - }, - "source": [ - "For us, our input data is the 'sentence' column and our label is the 'polarity' column (0, 1 for negative and positive, respecitvely)" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "IuMOGwFui4it", - "colab_type": "code", - "colab": {} - }, - "source": [ - "DATA_COLUMN = 'sentence'\n", - "LABEL_COLUMN = 'polarity'\n", - "# label_list is the list of labels, i.e. True, False or 0, 1 or 'dog', 'cat'\n", - "label_list = [0, 1]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "V399W0rqNJ-Z", - "colab_type": "text" - }, - "source": [ - "#Data Preprocessing\n", - "We'll need to transform our data into a format BERT understands. This involves two steps. First, we create `InputExample`'s using the constructor provided in the BERT library.\n", - "\n", - "- `text_a` is the text we want to classify, which in this case, is the `Request` field in our Dataframe. \n", - "- `text_b` is used if we're training a model to understand the relationship between sentences (i.e. is `text_b` a translation of `text_a`? Is `text_b` an answer to the question asked by `text_a`?). This doesn't apply to our task, so we can leave `text_b` blank.\n", - "- `label` is the label for our example, i.e. True, False" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "p9gEt5SmM6i6", - "colab_type": "code", - "colab": {} - }, - "source": [ - "# Use the InputExample class from BERT's run_classifier code to create examples from the data\n", - "train_InputExamples = train.apply(lambda x: bert.run_classifier.InputExample(guid=None, # Globally unique ID for bookkeeping, unused in this example\n", - " text_a = x[DATA_COLUMN], \n", - " text_b = None, \n", - " label = x[LABEL_COLUMN]), axis = 1)\n", - "\n", - "test_InputExamples = test.apply(lambda x: bert.run_classifier.InputExample(guid=None, \n", - " text_a = x[DATA_COLUMN], \n", - " text_b = None, \n", - " label = x[LABEL_COLUMN]), axis = 1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "SCZWZtKxObjh", - "colab_type": "text" - }, - "source": [ - "Next, we need to preprocess our data so that it matches the data BERT was trained on. For this, we'll need to do a couple of things (but don't worry--this is also included in the Python library):\n", - "\n", - "\n", - "1. Lowercase our text (if we're using a BERT lowercase model)\n", - "2. Tokenize it (i.e. \"sally says hi\" -> [\"sally\", \"says\", \"hi\"])\n", - "3. Break words into WordPieces (i.e. \"calling\" -> [\"call\", \"##ing\"])\n", - "4. Map our words to indexes using a vocab file that BERT provides\n", - "5. Add special \"CLS\" and \"SEP\" tokens (see the [readme](https://github.com/google-research/bert))\n", - "6. Append \"index\" and \"segment\" tokens to each input (see the [BERT paper](https://arxiv.org/pdf/1810.04805.pdf))\n", - "\n", - "Happily, we don't have to worry about most of these details.\n", - "\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "qMWiDtpyQSoU", - "colab_type": "text" - }, - "source": [ - "To start, we'll need to load a vocabulary file and lowercasing information directly from the BERT tf hub module:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "IhJSe0QHNG7U", - "colab_type": "code", - "colab": {} - }, - "source": [ - "# This is a path to an uncased (all lowercase) version of BERT\n", - "BERT_MODEL_HUB = \"https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1\"\n", - "\n", - "def create_tokenizer_from_hub_module():\n", - " \"\"\"Get the vocab file and casing info from the Hub module.\"\"\"\n", - " with tf.Graph().as_default():\n", - " bert_module = hub.Module(BERT_MODEL_HUB)\n", - " tokenization_info = bert_module(signature=\"tokenization_info\", as_dict=True)\n", - " with tf.Session() as sess:\n", - " vocab_file, do_lower_case = sess.run([tokenization_info[\"vocab_file\"],\n", - " tokenization_info[\"do_lower_case\"]])\n", - " \n", - " return bert.tokenization.FullTokenizer(\n", - " vocab_file=vocab_file, do_lower_case=do_lower_case)\n", - "\n", - "tokenizer = create_tokenizer_from_hub_module()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "z4oFkhpZBDKm", - "colab_type": "text" - }, - "source": [ - "Great--we just learned that the BERT model we're using expects lowercase data (that's what stored in tokenization_info[\"do_lower_case\"]) and we also loaded BERT's vocab file. We also created a tokenizer, which breaks words into word pieces:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "dsBo6RCtQmwx", - "colab_type": "code", - "colab": {} - }, - "source": [ - "tokenizer.tokenize(\"This here's an example of using the BERT tokenizer\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "0OEzfFIt6GIc", - "colab_type": "text" - }, - "source": [ - "Using our tokenizer, we'll call `run_classifier.convert_examples_to_features` on our InputExamples to convert them into features BERT understands." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "LL5W8gEGRTAf", - "colab_type": "code", - "colab": {} - }, - "source": [ - "# We'll set sequences to be at most 128 tokens long.\n", - "MAX_SEQ_LENGTH = 128\n", - "# Convert our train and test features to InputFeatures that BERT understands.\n", - "train_features = bert.run_classifier.convert_examples_to_features(train_InputExamples, label_list, MAX_SEQ_LENGTH, tokenizer)\n", - "test_features = bert.run_classifier.convert_examples_to_features(test_InputExamples, label_list, MAX_SEQ_LENGTH, tokenizer)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ccp5trMwRtmr", - "colab_type": "text" - }, - "source": [ - "#Creating a model\n", - "\n", - "Now that we've prepared our data, let's focus on building a model. `create_model` does just this below. First, it loads the BERT tf hub module again (this time to extract the computation graph). Next, it creates a single new layer that will be trained to adapt BERT to our sentiment task (i.e. classifying whether a movie review is positive or negative). This strategy of using a mostly trained model is called [fine-tuning](http://wiki.fast.ai/index.php/Fine_tuning)." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "6o2a5ZIvRcJq", - "colab_type": "code", - "colab": {} - }, - "source": [ - "def create_model(is_predicting, input_ids, input_mask, segment_ids, labels,\n", - " num_labels):\n", - " \"\"\"Creates a classification model.\"\"\"\n", - "\n", - " bert_module = hub.Module(\n", - " BERT_MODEL_HUB,\n", - " trainable=True)\n", - " bert_inputs = dict(\n", - " input_ids=input_ids,\n", - " input_mask=input_mask,\n", - " segment_ids=segment_ids)\n", - " bert_outputs = bert_module(\n", - " inputs=bert_inputs,\n", - " signature=\"tokens\",\n", - " as_dict=True)\n", - "\n", - " # Use \"pooled_output\" for classification tasks on an entire sentence.\n", - " # Use \"sequence_outputs\" for token-level output.\n", - " output_layer = bert_outputs[\"pooled_output\"]\n", - "\n", - " hidden_size = output_layer.shape[-1].value\n", - "\n", - " # Create our own layer to tune for politeness data.\n", - " output_weights = tf.get_variable(\n", - " \"output_weights\", [num_labels, hidden_size],\n", - " initializer=tf.truncated_normal_initializer(stddev=0.02))\n", - "\n", - " output_bias = tf.get_variable(\n", - " \"output_bias\", [num_labels], initializer=tf.zeros_initializer())\n", - "\n", - " with tf.variable_scope(\"loss\"):\n", - "\n", - " # Dropout helps prevent overfitting\n", - " output_layer = tf.nn.dropout(output_layer, keep_prob=0.9)\n", - "\n", - " logits = tf.matmul(output_layer, output_weights, transpose_b=True)\n", - " logits = tf.nn.bias_add(logits, output_bias)\n", - " log_probs = tf.nn.log_softmax(logits, axis=-1)\n", - "\n", - " # Convert labels into one-hot encoding\n", - " one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32)\n", - "\n", - " predicted_labels = tf.squeeze(tf.argmax(log_probs, axis=-1, output_type=tf.int32))\n", - " # If we're predicting, we want predicted labels and the probabiltiies.\n", - " if is_predicting:\n", - " return (predicted_labels, log_probs)\n", - "\n", - " # If we're train/eval, compute loss between predicted and actual label\n", - " per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1)\n", - " loss = tf.reduce_mean(per_example_loss)\n", - " return (loss, predicted_labels, log_probs)\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "qpE0ZIDOCQzE", - "colab_type": "text" - }, - "source": [ - "Next we'll wrap our model function in a `model_fn_builder` function that adapts our model to work for training, evaluation, and prediction." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "FnH-AnOQ9KKW", - "colab_type": "code", - "colab": {} - }, - "source": [ - "# model_fn_builder actually creates our model function\n", - "# using the passed parameters for num_labels, learning_rate, etc.\n", - "def model_fn_builder(num_labels, learning_rate, num_train_steps,\n", - " num_warmup_steps):\n", - " \"\"\"Returns `model_fn` closure for TPUEstimator.\"\"\"\n", - " def model_fn(features, labels, mode, params): # pylint: disable=unused-argument\n", - " \"\"\"The `model_fn` for TPUEstimator.\"\"\"\n", - "\n", - " input_ids = features[\"input_ids\"]\n", - " input_mask = features[\"input_mask\"]\n", - " segment_ids = features[\"segment_ids\"]\n", - " label_ids = features[\"label_ids\"]\n", - "\n", - " is_predicting = (mode == tf.estimator.ModeKeys.PREDICT)\n", - " \n", - " # TRAIN and EVAL\n", - " if not is_predicting:\n", - "\n", - " (loss, predicted_labels, log_probs) = create_model(\n", - " is_predicting, input_ids, input_mask, segment_ids, label_ids, num_labels)\n", - "\n", - " train_op = bert.optimization.create_optimizer(\n", - " loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu=False)\n", - "\n", - " # Calculate evaluation metrics. \n", - " def metric_fn(label_ids, predicted_labels):\n", - " accuracy = tf.metrics.accuracy(label_ids, predicted_labels)\n", - " f1_score = tf.contrib.metrics.f1_score(\n", - " label_ids,\n", - " predicted_labels)\n", - " auc = tf.metrics.auc(\n", - " label_ids,\n", - " predicted_labels)\n", - " recall = tf.metrics.recall(\n", - " label_ids,\n", - " predicted_labels)\n", - " precision = tf.metrics.precision(\n", - " label_ids,\n", - " predicted_labels) \n", - " true_pos = tf.metrics.true_positives(\n", - " label_ids,\n", - " predicted_labels)\n", - " true_neg = tf.metrics.true_negatives(\n", - " label_ids,\n", - " predicted_labels) \n", - " false_pos = tf.metrics.false_positives(\n", - " label_ids,\n", - " predicted_labels) \n", - " false_neg = tf.metrics.false_negatives(\n", - " label_ids,\n", - " predicted_labels)\n", - " return {\n", - " \"eval_accuracy\": accuracy,\n", - " \"f1_score\": f1_score,\n", - " \"auc\": auc,\n", - " \"precision\": precision,\n", - " \"recall\": recall,\n", - " \"true_positives\": true_pos,\n", - " \"true_negatives\": true_neg,\n", - " \"false_positives\": false_pos,\n", - " \"false_negatives\": false_neg\n", - " }\n", - "\n", - " eval_metrics = metric_fn(label_ids, predicted_labels)\n", - "\n", - " if mode == tf.estimator.ModeKeys.TRAIN:\n", - " return tf.estimator.EstimatorSpec(mode=mode,\n", - " loss=loss,\n", - " train_op=train_op)\n", - " else:\n", - " return tf.estimator.EstimatorSpec(mode=mode,\n", - " loss=loss,\n", - " eval_metric_ops=eval_metrics)\n", - " else:\n", - " (predicted_labels, log_probs) = create_model(\n", - " is_predicting, input_ids, input_mask, segment_ids, label_ids, num_labels)\n", - "\n", - " predictions = {\n", - " 'probabilities': log_probs,\n", - " 'labels': predicted_labels\n", - " }\n", - " return tf.estimator.EstimatorSpec(mode, predictions=predictions)\n", - "\n", - " # Return the actual model function in the closure\n", - " return model_fn\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "OjwJ4bTeWXD8", - "colab_type": "code", - "colab": {} - }, - "source": [ - "# Compute train and warmup steps from batch size\n", - "# These hyperparameters are copied from this colab notebook (https://colab.sandbox.google.com/github/tensorflow/tpu/blob/master/tools/colab/bert_finetuning_with_cloud_tpus.ipynb)\n", - "BATCH_SIZE = 32\n", - "LEARNING_RATE = 2e-5\n", - "NUM_TRAIN_EPOCHS = 3.0\n", - "# Warmup is a period of time where hte learning rate \n", - "# is small and gradually increases--usually helps training.\n", - "WARMUP_PROPORTION = 0.1\n", - "# Model configs\n", - "SAVE_CHECKPOINTS_STEPS = 500\n", - "SAVE_SUMMARY_STEPS = 100" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "emHf9GhfWBZ_", - "colab_type": "code", - "colab": {} - }, - "source": [ - "# Compute # train and warmup steps from batch size\n", - "num_train_steps = int(len(train_features) / BATCH_SIZE * NUM_TRAIN_EPOCHS)\n", - "num_warmup_steps = int(num_train_steps * WARMUP_PROPORTION)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "oEJldMr3WYZa", - "colab_type": "code", - "colab": {} - }, - "source": [ - "# Specify outpit directory and number of checkpoint steps to save\n", - "run_config = tf.estimator.RunConfig(\n", - " model_dir=OUTPUT_DIR,\n", - " save_summary_steps=SAVE_SUMMARY_STEPS,\n", - " save_checkpoints_steps=SAVE_CHECKPOINTS_STEPS)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "q_WebpS1X97v", - "colab_type": "code", - "colab": {} - }, - "source": [ - "model_fn = model_fn_builder(\n", - " num_labels=len(label_list),\n", - " learning_rate=LEARNING_RATE,\n", - " num_train_steps=num_train_steps,\n", - " num_warmup_steps=num_warmup_steps)\n", - "\n", - "estimator = tf.estimator.Estimator(\n", - " model_fn=model_fn,\n", - " config=run_config,\n", - " params={\"batch_size\": BATCH_SIZE})\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "NOO3RfG1DYLo", - "colab_type": "text" - }, - "source": [ - "Next we create an input builder function that takes our training feature set (`train_features`) and produces a generator. This is a pretty standard design pattern for working with TensorFlow [Estimators](https://www.tensorflow.org/guide/estimators)." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "1Pv2bAlOX_-K", - "colab_type": "code", - "colab": {} - }, - "source": [ - "# Create an input function for training. drop_remainder = True for using TPUs.\n", - "train_input_fn = bert.run_classifier.input_fn_builder(\n", - " features=train_features,\n", - " seq_length=MAX_SEQ_LENGTH,\n", - " is_training=True,\n", - " drop_remainder=False)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "t6Nukby2EB6-", - "colab_type": "text" - }, - "source": [ - "Now we train our model! For me, using a Colab notebook running on Google's GPUs, training time is typically 8-14 minutes." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "nucD4gluYJmK", - "colab_type": "code", - "colab": {} - }, - "source": [ - "print(f'Beginning Training!')\n", - "current_time = datetime.now()\n", - "estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)\n", - "print(\"Training took time \", datetime.now() - current_time)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "CmbLTVniARy3", - "colab_type": "text" - }, - "source": [ - "Now let's use our test data to see how well our model did:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "JIhejfpyJ8Bx", - "colab_type": "code", - "colab": {} - }, - "source": [ - "test_input_fn = run_classifier.input_fn_builder(\n", - " features=test_features,\n", - " seq_length=MAX_SEQ_LENGTH,\n", - " is_training=False,\n", - " drop_remainder=False)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "PPVEXhNjYXC-", - "colab_type": "code", - "colab": {} - }, - "source": [ - "estimator.evaluate(input_fn=test_input_fn, steps=None)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ueKsULteiz1B", - "colab_type": "text" - }, - "source": [ - "Now let's write code to make predictions on new sentences:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "OsrbTD2EJTVl", - "colab_type": "code", - "colab": {} - }, - "source": [ - "def getPrediction(in_sentences):\n", - " labels = [\"Negative\", \"Positive\"]\n", - " input_examples = [run_classifier.InputExample(guid=\"\", text_a = x, text_b = None, label = 0) for x in in_sentences] # here, \"\" is just a dummy label\n", - " input_features = run_classifier.convert_examples_to_features(input_examples, label_list, MAX_SEQ_LENGTH, tokenizer)\n", - " predict_input_fn = run_classifier.input_fn_builder(features=input_features, seq_length=MAX_SEQ_LENGTH, is_training=False, drop_remainder=False)\n", - " predictions = estimator.predict(predict_input_fn)\n", - " return [(sentence, prediction['probabilities'], labels[prediction['labels']]) for sentence, prediction in zip(in_sentences, predictions)]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "-thbodgih_VJ", - "colab_type": "code", - "colab": {} - }, - "source": [ - "pred_sentences = [\n", - " \"That movie was absolutely awful\",\n", - " \"The acting was a bit lacking\",\n", - " \"The film was creative and surprising\",\n", - " \"Absolutely fantastic!\"\n", - "]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "QrZmvZySKQTm", - "colab_type": "code", - "colab": {} - }, - "source": [ - "predictions = getPrediction(pred_sentences)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "MXkRiEBUqN3n", - "colab_type": "text" - }, - "source": [ - "Voila! We have a sentiment classifier!" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "ERkTE8-7oQLZ", - "colab_type": "code", - "colab": {} - }, - "source": [ - "predictions" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "P3Tg7c47vfLE", - "colab_type": "text" - }, - "source": [ - "# Export the model\n", - "\n", - "We are now ready to export the model. The following code defines the serving input function and exports the model to `OUTPUT_DIR`." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "NfXsdV4qtlpW", - "colab_type": "code", - "colab": {} - }, - "source": [ - "def serving_input_fn():\n", - " reciever_tensors = {\n", - " \"input_ids\": tf.placeholder(dtype=tf.int32,\n", - " shape=[1, MAX_SEQ_LENGTH])\n", - " }\n", - " features = {\n", - " \"input_ids\": reciever_tensors['input_ids'],\n", - " \"input_mask\": 1 - tf.cast(tf.equal(reciever_tensors['input_ids'], 0), dtype=tf.int32),\n", - " \"segment_ids\": tf.zeros(dtype=tf.int32, shape=[1, MAX_SEQ_LENGTH]),\n", - " \"label_ids\": tf.placeholder(tf.int32, [None], name='label_ids')\n", - " }\n", - " return tf.estimator.export.ServingInputReceiver(features, reciever_tensors)\n", - " \n", - "estimator._export_to_tpu = False\n", - "estimator.export_saved_model(OUTPUT_DIR+\"/export\", serving_input_fn)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "tIFTmUbcwI0w", - "colab_type": "text" - }, - "source": [ - "# Upload the model to AWS\n", - "\n", - "Cortex loads models from AWS, so we need to upload the exported model." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "gByRzrnR_OBX", - "colab_type": "text" - }, - "source": [ - "Set these variables to configure your AWS credentials and model upload path:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "1bdCOb3z0_Gh", - "colab_type": "code", - "cellView": "form", - "colab": {} - }, - "source": [ - "AWS_ACCESS_KEY_ID = \"\" #@param {type:\"string\"}\n", - "AWS_SECRET_ACCESS_KEY = \"\" #@param {type:\"string\"}\n", - "S3_UPLOAD_PATH = \"s3://my-bucket/sentiment-analyzer/bert\" #@param {type:\"string\"}\n", - "\n", - "import sys\n", - "import re\n", - "\n", - "if AWS_ACCESS_KEY_ID == \"\":\n", - " print(\"\\033[91m{}\\033[00m\".format(\"ERROR: Please set AWS_ACCESS_KEY_ID\"), file=sys.stderr)\n", - "\n", - "elif AWS_SECRET_ACCESS_KEY == \"\":\n", - " print(\"\\033[91m{}\\033[00m\".format(\"ERROR: Please set AWS_SECRET_ACCESS_KEY\"), file=sys.stderr)\n", - "\n", - "else:\n", - " try:\n", - " bucket, key = re.match(\"s3://(.+?)/(.+)\", S3_UPLOAD_PATH).groups()\n", - " except:\n", - " print(\"\\033[91m{}\\033[00m\".format(\"ERROR: Invalid s3 path (should be of the form s3://my-bucket/path/to/file)\"), file=sys.stderr)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "WLT09hZr_bhm", - "colab_type": "text" - }, - "source": [ - "Upload to S3:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "jCN3BINl2sKN", - "colab_type": "code", - "colab": {} - }, - "source": [ - "import os\n", - "import boto3\n", - "\n", - "s3 = boto3.client(\"s3\", aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)\n", - "\n", - "for dirpath, _, filenames in os.walk(OUTPUT_DIR+\"/export\"):\n", - " for filename in filenames:\n", - " filepath = os.path.join(dirpath, filename)\n", - " filekey = os.path.join(key, filepath[len(OUTPUT_DIR+\"/export/\"):])\n", - " print(\"Uploading s3://{}/{} ...\".format(bucket, filekey), end = '')\n", - " s3.upload_file(filepath, bucket, filekey)\n", - " print(\" ✓\")\n", - "\n", - "print(\"\\nUploaded model export directory to \" + S3_UPLOAD_PATH)" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/test/apis/tensorflow/sentiment-analyzer/cortex.yaml b/test/apis/tensorflow/sentiment-analyzer/cortex.yaml deleted file mode 100644 index 194505692e..0000000000 --- a/test/apis/tensorflow/sentiment-analyzer/cortex.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- name: sentiment-analyzer - kind: RealtimeAPI - handler: - type: tensorflow - path: handler.py - models: - path: s3://cortex-examples/tensorflow/sentiment-analyzer/bert/ - compute: - cpu: 1 - gpu: 1 diff --git a/test/apis/tensorflow/sentiment-analyzer/handler.py b/test/apis/tensorflow/sentiment-analyzer/handler.py deleted file mode 100644 index 2f0508805f..0000000000 --- a/test/apis/tensorflow/sentiment-analyzer/handler.py +++ /dev/null @@ -1,27 +0,0 @@ -import tensorflow as tf -import tensorflow_hub as hub -from bert import tokenization, run_classifier - -labels = ["negative", "positive"] - - -class Handler: - def __init__(self, tensorflow_client, config): - with tf.Graph().as_default(): - bert_module = hub.Module("https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1") - info = bert_module(signature="tokenization_info", as_dict=True) - with tf.Session() as sess: - vocab_file, do_lower_case = sess.run([info["vocab_file"], info["do_lower_case"]]) - self._tokenizer = tokenization.FullTokenizer( - vocab_file=vocab_file, do_lower_case=do_lower_case - ) - self.client = tensorflow_client - - def handle_post(self, payload): - input_example = run_classifier.InputExample(guid="", text_a=payload["review"], label=0) - input_feature = run_classifier.convert_single_example( - 0, input_example, [0, 1], 128, self._tokenizer - ) - model_input = {"input_ids": [input_feature.input_ids]} - prediction = self.client.predict(model_input) - return labels[prediction["labels"][0]] diff --git a/test/apis/tensorflow/sentiment-analyzer/requirements.txt b/test/apis/tensorflow/sentiment-analyzer/requirements.txt deleted file mode 100644 index 273614922e..0000000000 --- a/test/apis/tensorflow/sentiment-analyzer/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -bert-tensorflow==1.0.1 -tensorflow-hub==0.7.0 -tensorflow==1.15.* -tensorflow-serving-api==1.15.* -numpy==1.16.* diff --git a/test/apis/tensorflow/sentiment-analyzer/sample.json b/test/apis/tensorflow/sentiment-analyzer/sample.json deleted file mode 100644 index c433e33216..0000000000 --- a/test/apis/tensorflow/sentiment-analyzer/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "review": "the movie was amazing!" -} diff --git a/test/apis/tensorflow/sound-classifier/README.md b/test/apis/tensorflow/sound-classifier/README.md deleted file mode 100644 index c1e492c8f2..0000000000 --- a/test/apis/tensorflow/sound-classifier/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Sound Classifier - -Based on https://tfhub.dev/google/yamnet/1. - -Export the endpoint: - -```bash -export ENDPOINT= # you can find this with `cortex get sound-classifier` -``` - -And then run the inference: - -```bash -curl $ENDPOINT -X POST -H "Content-type: application/octet-stream" --data-binary "@silence.wav" -``` diff --git a/test/apis/tensorflow/sound-classifier/class_names.csv b/test/apis/tensorflow/sound-classifier/class_names.csv deleted file mode 100644 index 9c07d818a6..0000000000 --- a/test/apis/tensorflow/sound-classifier/class_names.csv +++ /dev/null @@ -1,522 +0,0 @@ -index,mid,display_name -0,/m/09x0r,Speech -1,/m/0ytgt,"Child speech, kid speaking" -2,/m/01h8n0,Conversation -3,/m/02qldy,"Narration, monologue" -4,/m/0261r1,Babbling -5,/m/0brhx,Speech synthesizer -6,/m/07p6fty,Shout -7,/m/07q4ntr,Bellow -8,/m/07rwj3x,Whoop -9,/m/07sr1lc,Yell -10,/t/dd00135,Children shouting -11,/m/03qc9zr,Screaming -12,/m/02rtxlg,Whispering -13,/m/01j3sz,Laughter -14,/t/dd00001,Baby laughter -15,/m/07r660_,Giggle -16,/m/07s04w4,Snicker -17,/m/07sq110,Belly laugh -18,/m/07rgt08,"Chuckle, chortle" -19,/m/0463cq4,"Crying, sobbing" -20,/t/dd00002,"Baby cry, infant cry" -21,/m/07qz6j3,Whimper -22,/m/07qw_06,"Wail, moan" -23,/m/07plz5l,Sigh -24,/m/015lz1,Singing -25,/m/0l14jd,Choir -26,/m/01swy6,Yodeling -27,/m/02bk07,Chant -28,/m/01c194,Mantra -29,/t/dd00005,Child singing -30,/t/dd00006,Synthetic singing -31,/m/06bxc,Rapping -32,/m/02fxyj,Humming -33,/m/07s2xch,Groan -34,/m/07r4k75,Grunt -35,/m/01w250,Whistling -36,/m/0lyf6,Breathing -37,/m/07mzm6,Wheeze -38,/m/01d3sd,Snoring -39,/m/07s0dtb,Gasp -40,/m/07pyy8b,Pant -41,/m/07q0yl5,Snort -42,/m/01b_21,Cough -43,/m/0dl9sf8,Throat clearing -44,/m/01hsr_,Sneeze -45,/m/07ppn3j,Sniff -46,/m/06h7j,Run -47,/m/07qv_x_,Shuffle -48,/m/07pbtc8,"Walk, footsteps" -49,/m/03cczk,"Chewing, mastication" -50,/m/07pdhp0,Biting -51,/m/0939n_,Gargling -52,/m/01g90h,Stomach rumble -53,/m/03q5_w,"Burping, eructation" -54,/m/02p3nc,Hiccup -55,/m/02_nn,Fart -56,/m/0k65p,Hands -57,/m/025_jnm,Finger snapping -58,/m/0l15bq,Clapping -59,/m/01jg02,"Heart sounds, heartbeat" -60,/m/01jg1z,Heart murmur -61,/m/053hz1,Cheering -62,/m/028ght,Applause -63,/m/07rkbfh,Chatter -64,/m/03qtwd,Crowd -65,/m/07qfr4h,"Hubbub, speech noise, speech babble" -66,/t/dd00013,Children playing -67,/m/0jbk,Animal -68,/m/068hy,"Domestic animals, pets" -69,/m/0bt9lr,Dog -70,/m/05tny_,Bark -71,/m/07r_k2n,Yip -72,/m/07qf0zm,Howl -73,/m/07rc7d9,Bow-wow -74,/m/0ghcn6,Growling -75,/t/dd00136,Whimper (dog) -76,/m/01yrx,Cat -77,/m/02yds9,Purr -78,/m/07qrkrw,Meow -79,/m/07rjwbb,Hiss -80,/m/07r81j2,Caterwaul -81,/m/0ch8v,"Livestock, farm animals, working animals" -82,/m/03k3r,Horse -83,/m/07rv9rh,Clip-clop -84,/m/07q5rw0,"Neigh, whinny" -85,/m/01xq0k1,"Cattle, bovinae" -86,/m/07rpkh9,Moo -87,/m/0239kh,Cowbell -88,/m/068zj,Pig -89,/t/dd00018,Oink -90,/m/03fwl,Goat -91,/m/07q0h5t,Bleat -92,/m/07bgp,Sheep -93,/m/025rv6n,Fowl -94,/m/09b5t,"Chicken, rooster" -95,/m/07st89h,Cluck -96,/m/07qn5dc,"Crowing, cock-a-doodle-doo" -97,/m/01rd7k,Turkey -98,/m/07svc2k,Gobble -99,/m/09ddx,Duck -100,/m/07qdb04,Quack -101,/m/0dbvp,Goose -102,/m/07qwf61,Honk -103,/m/01280g,Wild animals -104,/m/0cdnk,"Roaring cats (lions, tigers)" -105,/m/04cvmfc,Roar -106,/m/015p6,Bird -107,/m/020bb7,"Bird vocalization, bird call, bird song" -108,/m/07pggtn,"Chirp, tweet" -109,/m/07sx8x_,Squawk -110,/m/0h0rv,"Pigeon, dove" -111,/m/07r_25d,Coo -112,/m/04s8yn,Crow -113,/m/07r5c2p,Caw -114,/m/09d5_,Owl -115,/m/07r_80w,Hoot -116,/m/05_wcq,"Bird flight, flapping wings" -117,/m/01z5f,"Canidae, dogs, wolves" -118,/m/06hps,"Rodents, rats, mice" -119,/m/04rmv,Mouse -120,/m/07r4gkf,Patter -121,/m/03vt0,Insect -122,/m/09xqv,Cricket -123,/m/09f96,Mosquito -124,/m/0h2mp,"Fly, housefly" -125,/m/07pjwq1,Buzz -126,/m/01h3n,"Bee, wasp, etc." -127,/m/09ld4,Frog -128,/m/07st88b,Croak -129,/m/078jl,Snake -130,/m/07qn4z3,Rattle -131,/m/032n05,Whale vocalization -132,/m/04rlf,Music -133,/m/04szw,Musical instrument -134,/m/0fx80y,Plucked string instrument -135,/m/0342h,Guitar -136,/m/02sgy,Electric guitar -137,/m/018vs,Bass guitar -138,/m/042v_gx,Acoustic guitar -139,/m/06w87,"Steel guitar, slide guitar" -140,/m/01glhc,Tapping (guitar technique) -141,/m/07s0s5r,Strum -142,/m/018j2,Banjo -143,/m/0jtg0,Sitar -144,/m/04rzd,Mandolin -145,/m/01bns_,Zither -146,/m/07xzm,Ukulele -147,/m/05148p4,Keyboard (musical) -148,/m/05r5c,Piano -149,/m/01s0ps,Electric piano -150,/m/013y1f,Organ -151,/m/03xq_f,Electronic organ -152,/m/03gvt,Hammond organ -153,/m/0l14qv,Synthesizer -154,/m/01v1d8,Sampler -155,/m/03q5t,Harpsichord -156,/m/0l14md,Percussion -157,/m/02hnl,Drum kit -158,/m/0cfdd,Drum machine -159,/m/026t6,Drum -160,/m/06rvn,Snare drum -161,/m/03t3fj,Rimshot -162,/m/02k_mr,Drum roll -163,/m/0bm02,Bass drum -164,/m/011k_j,Timpani -165,/m/01p970,Tabla -166,/m/01qbl,Cymbal -167,/m/03qtq,Hi-hat -168,/m/01sm1g,Wood block -169,/m/07brj,Tambourine -170,/m/05r5wn,Rattle (instrument) -171,/m/0xzly,Maraca -172,/m/0mbct,Gong -173,/m/016622,Tubular bells -174,/m/0j45pbj,Mallet percussion -175,/m/0dwsp,"Marimba, xylophone" -176,/m/0dwtp,Glockenspiel -177,/m/0dwt5,Vibraphone -178,/m/0l156b,Steelpan -179,/m/05pd6,Orchestra -180,/m/01kcd,Brass instrument -181,/m/0319l,French horn -182,/m/07gql,Trumpet -183,/m/07c6l,Trombone -184,/m/0l14_3,Bowed string instrument -185,/m/02qmj0d,String section -186,/m/07y_7,"Violin, fiddle" -187,/m/0d8_n,Pizzicato -188,/m/01xqw,Cello -189,/m/02fsn,Double bass -190,/m/085jw,"Wind instrument, woodwind instrument" -191,/m/0l14j_,Flute -192,/m/06ncr,Saxophone -193,/m/01wy6,Clarinet -194,/m/03m5k,Harp -195,/m/0395lw,Bell -196,/m/03w41f,Church bell -197,/m/027m70_,Jingle bell -198,/m/0gy1t2s,Bicycle bell -199,/m/07n_g,Tuning fork -200,/m/0f8s22,Chime -201,/m/026fgl,Wind chime -202,/m/0150b9,Change ringing (campanology) -203,/m/03qjg,Harmonica -204,/m/0mkg,Accordion -205,/m/0192l,Bagpipes -206,/m/02bxd,Didgeridoo -207,/m/0l14l2,Shofar -208,/m/07kc_,Theremin -209,/m/0l14t7,Singing bowl -210,/m/01hgjl,Scratching (performance technique) -211,/m/064t9,Pop music -212,/m/0glt670,Hip hop music -213,/m/02cz_7,Beatboxing -214,/m/06by7,Rock music -215,/m/03lty,Heavy metal -216,/m/05r6t,Punk rock -217,/m/0dls3,Grunge -218,/m/0dl5d,Progressive rock -219,/m/07sbbz2,Rock and roll -220,/m/05w3f,Psychedelic rock -221,/m/06j6l,Rhythm and blues -222,/m/0gywn,Soul music -223,/m/06cqb,Reggae -224,/m/01lyv,Country -225,/m/015y_n,Swing music -226,/m/0gg8l,Bluegrass -227,/m/02x8m,Funk -228,/m/02w4v,Folk music -229,/m/06j64v,Middle Eastern music -230,/m/03_d0,Jazz -231,/m/026z9,Disco -232,/m/0ggq0m,Classical music -233,/m/05lls,Opera -234,/m/02lkt,Electronic music -235,/m/03mb9,House music -236,/m/07gxw,Techno -237,/m/07s72n,Dubstep -238,/m/0283d,Drum and bass -239,/m/0m0jc,Electronica -240,/m/08cyft,Electronic dance music -241,/m/0fd3y,Ambient music -242,/m/07lnk,Trance music -243,/m/0g293,Music of Latin America -244,/m/0ln16,Salsa music -245,/m/0326g,Flamenco -246,/m/0155w,Blues -247,/m/05fw6t,Music for children -248,/m/02v2lh,New-age music -249,/m/0y4f8,Vocal music -250,/m/0z9c,A capella -251,/m/0164x2,Music of Africa -252,/m/0145m,Afrobeat -253,/m/02mscn,Christian music -254,/m/016cjb,Gospel music -255,/m/028sqc,Music of Asia -256,/m/015vgc,Carnatic music -257,/m/0dq0md,Music of Bollywood -258,/m/06rqw,Ska -259,/m/02p0sh1,Traditional music -260,/m/05rwpb,Independent music -261,/m/074ft,Song -262,/m/025td0t,Background music -263,/m/02cjck,Theme music -264,/m/03r5q_,Jingle (music) -265,/m/0l14gg,Soundtrack music -266,/m/07pkxdp,Lullaby -267,/m/01z7dr,Video game music -268,/m/0140xf,Christmas music -269,/m/0ggx5q,Dance music -270,/m/04wptg,Wedding music -271,/t/dd00031,Happy music -272,/t/dd00033,Sad music -273,/t/dd00034,Tender music -274,/t/dd00035,Exciting music -275,/t/dd00036,Angry music -276,/t/dd00037,Scary music -277,/m/03m9d0z,Wind -278,/m/09t49,Rustling leaves -279,/t/dd00092,Wind noise (microphone) -280,/m/0jb2l,Thunderstorm -281,/m/0ngt1,Thunder -282,/m/0838f,Water -283,/m/06mb1,Rain -284,/m/07r10fb,Raindrop -285,/t/dd00038,Rain on surface -286,/m/0j6m2,Stream -287,/m/0j2kx,Waterfall -288,/m/05kq4,Ocean -289,/m/034srq,"Waves, surf" -290,/m/06wzb,Steam -291,/m/07swgks,Gurgling -292,/m/02_41,Fire -293,/m/07pzfmf,Crackle -294,/m/07yv9,Vehicle -295,/m/019jd,"Boat, Water vehicle" -296,/m/0hsrw,"Sailboat, sailing ship" -297,/m/056ks2,"Rowboat, canoe, kayak" -298,/m/02rlv9,"Motorboat, speedboat" -299,/m/06q74,Ship -300,/m/012f08,Motor vehicle (road) -301,/m/0k4j,Car -302,/m/0912c9,"Vehicle horn, car horn, honking" -303,/m/07qv_d5,Toot -304,/m/02mfyn,Car alarm -305,/m/04gxbd,"Power windows, electric windows" -306,/m/07rknqz,Skidding -307,/m/0h9mv,Tire squeal -308,/t/dd00134,Car passing by -309,/m/0ltv,"Race car, auto racing" -310,/m/07r04,Truck -311,/m/0gvgw0,Air brake -312,/m/05x_td,"Air horn, truck horn" -313,/m/02rhddq,Reversing beeps -314,/m/03cl9h,"Ice cream truck, ice cream van" -315,/m/01bjv,Bus -316,/m/03j1ly,Emergency vehicle -317,/m/04qvtq,Police car (siren) -318,/m/012n7d,Ambulance (siren) -319,/m/012ndj,"Fire engine, fire truck (siren)" -320,/m/04_sv,Motorcycle -321,/m/0btp2,"Traffic noise, roadway noise" -322,/m/06d_3,Rail transport -323,/m/07jdr,Train -324,/m/04zmvq,Train whistle -325,/m/0284vy3,Train horn -326,/m/01g50p,"Railroad car, train wagon" -327,/t/dd00048,Train wheels squealing -328,/m/0195fx,"Subway, metro, underground" -329,/m/0k5j,Aircraft -330,/m/014yck,Aircraft engine -331,/m/04229,Jet engine -332,/m/02l6bg,"Propeller, airscrew" -333,/m/09ct_,Helicopter -334,/m/0cmf2,"Fixed-wing aircraft, airplane" -335,/m/0199g,Bicycle -336,/m/06_fw,Skateboard -337,/m/02mk9,Engine -338,/t/dd00065,Light engine (high frequency) -339,/m/08j51y,"Dental drill, dentist's drill" -340,/m/01yg9g,Lawn mower -341,/m/01j4z9,Chainsaw -342,/t/dd00066,Medium engine (mid frequency) -343,/t/dd00067,Heavy engine (low frequency) -344,/m/01h82_,Engine knocking -345,/t/dd00130,Engine starting -346,/m/07pb8fc,Idling -347,/m/07q2z82,"Accelerating, revving, vroom" -348,/m/02dgv,Door -349,/m/03wwcy,Doorbell -350,/m/07r67yg,Ding-dong -351,/m/02y_763,Sliding door -352,/m/07rjzl8,Slam -353,/m/07r4wb8,Knock -354,/m/07qcpgn,Tap -355,/m/07q6cd_,Squeak -356,/m/0642b4,Cupboard open or close -357,/m/0fqfqc,Drawer open or close -358,/m/04brg2,"Dishes, pots, and pans" -359,/m/023pjk,"Cutlery, silverware" -360,/m/07pn_8q,Chopping (food) -361,/m/0dxrf,Frying (food) -362,/m/0fx9l,Microwave oven -363,/m/02pjr4,Blender -364,/m/02jz0l,"Water tap, faucet" -365,/m/0130jx,Sink (filling or washing) -366,/m/03dnzn,Bathtub (filling or washing) -367,/m/03wvsk,Hair dryer -368,/m/01jt3m,Toilet flush -369,/m/012xff,Toothbrush -370,/m/04fgwm,Electric toothbrush -371,/m/0d31p,Vacuum cleaner -372,/m/01s0vc,Zipper (clothing) -373,/m/03v3yw,Keys jangling -374,/m/0242l,Coin (dropping) -375,/m/01lsmm,Scissors -376,/m/02g901,"Electric shaver, electric razor" -377,/m/05rj2,Shuffling cards -378,/m/0316dw,Typing -379,/m/0c2wf,Typewriter -380,/m/01m2v,Computer keyboard -381,/m/081rb,Writing -382,/m/07pp_mv,Alarm -383,/m/07cx4,Telephone -384,/m/07pp8cl,Telephone bell ringing -385,/m/01hnzm,Ringtone -386,/m/02c8p,"Telephone dialing, DTMF" -387,/m/015jpf,Dial tone -388,/m/01z47d,Busy signal -389,/m/046dlr,Alarm clock -390,/m/03kmc9,Siren -391,/m/0dgbq,Civil defense siren -392,/m/030rvx,Buzzer -393,/m/01y3hg,"Smoke detector, smoke alarm" -394,/m/0c3f7m,Fire alarm -395,/m/04fq5q,Foghorn -396,/m/0l156k,Whistle -397,/m/06hck5,Steam whistle -398,/t/dd00077,Mechanisms -399,/m/02bm9n,"Ratchet, pawl" -400,/m/01x3z,Clock -401,/m/07qjznt,Tick -402,/m/07qjznl,Tick-tock -403,/m/0l7xg,Gears -404,/m/05zc1,Pulleys -405,/m/0llzx,Sewing machine -406,/m/02x984l,Mechanical fan -407,/m/025wky1,Air conditioning -408,/m/024dl,Cash register -409,/m/01m4t,Printer -410,/m/0dv5r,Camera -411,/m/07bjf,Single-lens reflex camera -412,/m/07k1x,Tools -413,/m/03l9g,Hammer -414,/m/03p19w,Jackhammer -415,/m/01b82r,Sawing -416,/m/02p01q,Filing (rasp) -417,/m/023vsd,Sanding -418,/m/0_ksk,Power tool -419,/m/01d380,Drill -420,/m/014zdl,Explosion -421,/m/032s66,"Gunshot, gunfire" -422,/m/04zjc,Machine gun -423,/m/02z32qm,Fusillade -424,/m/0_1c,Artillery fire -425,/m/073cg4,Cap gun -426,/m/0g6b5,Fireworks -427,/g/122z_qxw,Firecracker -428,/m/07qsvvw,"Burst, pop" -429,/m/07pxg6y,Eruption -430,/m/07qqyl4,Boom -431,/m/083vt,Wood -432,/m/07pczhz,Chop -433,/m/07pl1bw,Splinter -434,/m/07qs1cx,Crack -435,/m/039jq,Glass -436,/m/07q7njn,"Chink, clink" -437,/m/07rn7sz,Shatter -438,/m/04k94,Liquid -439,/m/07rrlb6,"Splash, splatter" -440,/m/07p6mqd,Slosh -441,/m/07qlwh6,Squish -442,/m/07r5v4s,Drip -443,/m/07prgkl,Pour -444,/m/07pqc89,"Trickle, dribble" -445,/t/dd00088,Gush -446,/m/07p7b8y,Fill (with liquid) -447,/m/07qlf79,Spray -448,/m/07ptzwd,Pump (liquid) -449,/m/07ptfmf,Stir -450,/m/0dv3j,Boiling -451,/m/0790c,Sonar -452,/m/0dl83,Arrow -453,/m/07rqsjt,"Whoosh, swoosh, swish" -454,/m/07qnq_y,"Thump, thud" -455,/m/07rrh0c,Thunk -456,/m/0b_fwt,Electronic tuner -457,/m/02rr_,Effects unit -458,/m/07m2kt,Chorus effect -459,/m/018w8,Basketball bounce -460,/m/07pws3f,Bang -461,/m/07ryjzk,"Slap, smack" -462,/m/07rdhzs,"Whack, thwack" -463,/m/07pjjrj,"Smash, crash" -464,/m/07pc8lb,Breaking -465,/m/07pqn27,Bouncing -466,/m/07rbp7_,Whip -467,/m/07pyf11,Flap -468,/m/07qb_dv,Scratch -469,/m/07qv4k0,Scrape -470,/m/07pdjhy,Rub -471,/m/07s8j8t,Roll -472,/m/07plct2,Crushing -473,/t/dd00112,"Crumpling, crinkling" -474,/m/07qcx4z,Tearing -475,/m/02fs_r,"Beep, bleep" -476,/m/07qwdck,Ping -477,/m/07phxs1,Ding -478,/m/07rv4dm,Clang -479,/m/07s02z0,Squeal -480,/m/07qh7jl,Creak -481,/m/07qwyj0,Rustle -482,/m/07s34ls,Whir -483,/m/07qmpdm,Clatter -484,/m/07p9k1k,Sizzle -485,/m/07qc9xj,Clicking -486,/m/07rwm0c,Clickety-clack -487,/m/07phhsh,Rumble -488,/m/07qyrcz,Plop -489,/m/07qfgpx,"Jingle, tinkle" -490,/m/07rcgpl,Hum -491,/m/07p78v5,Zing -492,/t/dd00121,Boing -493,/m/07s12q4,Crunch -494,/m/028v0c,Silence -495,/m/01v_m0,Sine wave -496,/m/0b9m1,Harmonic -497,/m/0hdsk,Chirp tone -498,/m/0c1dj,Sound effect -499,/m/07pt_g0,Pulse -500,/t/dd00125,"Inside, small room" -501,/t/dd00126,"Inside, large room or hall" -502,/t/dd00127,"Inside, public space" -503,/t/dd00128,"Outside, urban or manmade" -504,/t/dd00129,"Outside, rural or natural" -505,/m/01b9nn,Reverberation -506,/m/01jnbd,Echo -507,/m/096m7z,Noise -508,/m/06_y0by,Environmental noise -509,/m/07rgkc5,Static -510,/m/06xkwv,Mains hum -511,/m/0g12c5,Distortion -512,/m/08p9q4,Sidetone -513,/m/07szfh9,Cacophony -514,/m/0chx_,White noise -515,/m/0cj0r,Pink noise -516,/m/07p_0gm,Throbbing -517,/m/01jwx6,Vibration -518,/m/07c52,Television -519,/m/06bz3,Radio -520,/m/07hvw1,Field recording diff --git a/test/apis/tensorflow/sound-classifier/cortex.yaml b/test/apis/tensorflow/sound-classifier/cortex.yaml deleted file mode 100644 index 3f40c961a9..0000000000 --- a/test/apis/tensorflow/sound-classifier/cortex.yaml +++ /dev/null @@ -1,11 +0,0 @@ -- name: sound-classifier - kind: RealtimeAPI - handler: - type: tensorflow - path: handler.py - models: - path: s3://cortex-examples/tensorflow/sound-classifier/yamnet/ - signature_key: serving_default - compute: - cpu: 1 - mem: 2.5G diff --git a/test/apis/tensorflow/sound-classifier/handler.py b/test/apis/tensorflow/sound-classifier/handler.py deleted file mode 100644 index a3608d4c27..0000000000 --- a/test/apis/tensorflow/sound-classifier/handler.py +++ /dev/null @@ -1,27 +0,0 @@ -from scipy.io.wavfile import read -import numpy as np -import io -import csv - - -class Handler: - def __init__(self, tensorflow_client, config): - self.client = tensorflow_client - self.class_names = self.class_names_from_csv("class_names.csv") - - def class_names_from_csv(self, csv_file): - class_names = [] - with open(csv_file, "r", newline="") as f: - for row in csv.reader(f, delimiter=","): - class_names.append(row[2]) - return class_names - - def handle_post(self, payload): - rate, data = read(io.BytesIO(payload)) - assert rate == 16000 - - result = self.client.predict({"waveform": np.array(data, dtype=np.float32)}) - scores = np.array(result["output_0"]).reshape((-1, 521)) - - predicted_class = self.class_names[scores.mean(axis=0).argmax() + 1] - return predicted_class diff --git a/test/apis/tensorflow/sound-classifier/requirements.txt b/test/apis/tensorflow/sound-classifier/requirements.txt deleted file mode 100644 index 9136ba9f03..0000000000 --- a/test/apis/tensorflow/sound-classifier/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -scipy==1.5.4 diff --git a/test/apis/tensorflow/sound-classifier/silence.wav b/test/apis/tensorflow/sound-classifier/silence.wav deleted file mode 100644 index 59b993f6d3ac02e9d0afd664d17940c9014f35b3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32058 zcmeIup$)=N00ht@bOIKjGjxCk1WmArR{xaL;l6Fsxlw$>7VpfRIrCo6X&UxqxzCU7 zQ~OJjb51$NHoi%hzKyM{wQGI5Cht;9+CvBsAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ PfB*pk1PBly@E7<2r8NqB diff --git a/test/apis/tensorflow/text-generator/cortex.yaml b/test/apis/tensorflow/text-generator/cortex.yaml deleted file mode 100644 index 824398d2bb..0000000000 --- a/test/apis/tensorflow/text-generator/cortex.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- name: text-generator - kind: RealtimeAPI - handler: - type: tensorflow - path: handler.py - models: - path: s3://cortex-examples/tensorflow/text-generator/gpt-2/124M/ - compute: - cpu: 1 - gpu: 1 diff --git a/test/apis/tensorflow/text-generator/encoder.py b/test/apis/tensorflow/text-generator/encoder.py deleted file mode 100644 index 518b4da96c..0000000000 --- a/test/apis/tensorflow/text-generator/encoder.py +++ /dev/null @@ -1,116 +0,0 @@ -# This file includes code which was modified from https://github.com/openai/gpt-2 - -import json -import regex -from functools import lru_cache - - -@lru_cache() -def bytes_to_unicode(): - bs = ( - list(range(ord("!"), ord("~") + 1)) - + list(range(ord("¡"), ord("¬") + 1)) - + list(range(ord("®"), ord("ÿ") + 1)) - ) - cs = bs[:] - n = 0 - for b in range(2 ** 8): - if b not in bs: - bs.append(b) - cs.append(2 ** 8 + n) - n += 1 - cs = [chr(n) for n in cs] - return dict(zip(bs, cs)) - - -def get_pairs(word): - pairs = set() - prev_char = word[0] - for char in word[1:]: - pairs.add((prev_char, char)) - prev_char = char - return pairs - - -class Encoder: - def __init__(self, encoder, bpe_merges, errors="replace"): - self.encoder = encoder - self.decoder = {v: k for k, v in self.encoder.items()} - self.errors = errors - self.byte_encoder = bytes_to_unicode() - self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} - self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges)))) - self.cache = {} - self.pat = regex.compile( - r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" - ) - - def bpe(self, token): - if token in self.cache: - return self.cache[token] - word = tuple(token) - pairs = get_pairs(word) - - if not pairs: - return token - - while True: - bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf"))) - if bigram not in self.bpe_ranks: - break - first, second = bigram - new_word = [] - i = 0 - while i < len(word): - try: - j = word.index(first, i) - new_word.extend(word[i:j]) - i = j - except: - new_word.extend(word[i:]) - break - - if word[i] == first and i < len(word) - 1 and word[i + 1] == second: - new_word.append(first + second) - i += 2 - else: - new_word.append(word[i]) - i += 1 - new_word = tuple(new_word) - word = new_word - if len(word) == 1: - break - else: - pairs = get_pairs(word) - word = " ".join(word) - self.cache[token] = word - return word - - def encode(self, text): - bpe_tokens = [] - for token in regex.findall(self.pat, text): - token = "".join(self.byte_encoder[b] for b in token.encode("utf-8")) - bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(" ")) - return bpe_tokens - - def decode(self, tokens): - text = "".join([self.decoder[token] for token in tokens]) - text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors) - return text - - -def get_encoder(s3_client): - encoder = json.load( - s3_client.get_object( - Bucket="cortex-examples", Key="tensorflow/text-generator/gpt-2/encoder.json" - )["Body"] - ) - bpe_data = ( - s3_client.get_object( - Bucket="cortex-examples", Key="tensorflow/text-generator/gpt-2/vocab.bpe" - )["Body"] - .read() - .decode("utf-8") - ) - bpe_merges = [tuple(merge_str.split()) for merge_str in bpe_data.split("\n")[1:-1]] - return Encoder(encoder=encoder, bpe_merges=bpe_merges) diff --git a/test/apis/tensorflow/text-generator/gpt-2.ipynb b/test/apis/tensorflow/text-generator/gpt-2.ipynb deleted file mode 100644 index 8c7508c5b1..0000000000 --- a/test/apis/tensorflow/text-generator/gpt-2.ipynb +++ /dev/null @@ -1,369 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "gpt-2.ipynb", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.8" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "kc5cIgeEmv8o", - "colab_type": "text" - }, - "source": [ - "# Exporting GPT-2\n", - "\n", - "In this notebook, we'll show how to export [OpenAI's GPT-2 text generation model](https://github.com/openai/gpt-2) for serving." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "RAWs29lAktOK", - "colab_type": "text" - }, - "source": [ - "First, we'll download the GPT-2 code repository:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "gHs3aaFaLUXq", - "colab_type": "code", - "colab": {} - }, - "source": [ - "!git clone --no-checkout https://github.com/openai/gpt-2.git\n", - "!cd gpt-2 && git reset --hard ac5d52295f8a1c3856ea24fb239087cc1a3d1131" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "A4al4P14nmni", - "colab_type": "text" - }, - "source": [ - "Next we'll specify the model size (choose one of 124M, 355M, or 774M):" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "3Y4bt6hkfuxY", - "colab_type": "code", - "colab": {}, - "cellView": "form" - }, - "source": [ - "import sys\n", - "\n", - "MODEL_SIZE = \"124M\" #@param {type:\"string\"}\n", - "\n", - "if MODEL_SIZE not in {\"124M\", \"355M\", \"774M\"}:\n", - " print(\"\\033[91m{}\\033[00m\".format('ERROR: MODEL_SIZE must be \"124M\", \"355M\", or \"774M\"'), file=sys.stderr)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "C6xRx0Monh_j", - "colab_type": "text" - }, - "source": [ - "We can use `download_model.py` to download the model:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "Kb50Z6NjbJBN", - "colab_type": "code", - "colab": {} - }, - "source": [ - "!python3 ./gpt-2/download_model.py $MODEL_SIZE" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "zz2ioOcpoPjV", - "colab_type": "text" - }, - "source": [ - "Next, we'll install the required packages:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "Vk4Q2RR-UZQm", - "colab_type": "code", - "colab": {} - }, - "source": [ - "!pip install tensorflow==1.14.* numpy==1.* boto3==1.*" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "KkVf5FmuUMrl", - "colab_type": "code", - "colab": {} - }, - "source": [ - "import sys\n", - "import os\n", - "import time\n", - "import json\n", - "import numpy as np\n", - "import tensorflow as tf\n", - "from tensorflow.python.saved_model.signature_def_utils_impl import predict_signature_def" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "6Ay7qiQFoWRn", - "colab_type": "text" - }, - "source": [ - "Now we can export the model for serving:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "GdnYXr1IKaF0", - "colab_type": "code", - "colab": {} - }, - "source": [ - "sys.path.append(os.path.join(os.getcwd(), 'gpt-2/src'))\n", - "import model, sample\n", - "\n", - "def export_for_serving(\n", - " model_name='124M',\n", - " seed=None,\n", - " batch_size=1,\n", - " length=None,\n", - " temperature=1,\n", - " top_k=0,\n", - " models_dir='models'\n", - "):\n", - " \"\"\"\n", - " Export the model for TF Serving\n", - " :model_name=124M : String, which model to use\n", - " :seed=None : Integer seed for random number generators, fix seed to reproduce\n", - " results\n", - " :length=None : Number of tokens in generated text, if None (default), is\n", - " determined by model hyperparameters\n", - " :temperature=1 : Float value controlling randomness in boltzmann\n", - " distribution. Lower temperature results in less random completions. As the\n", - " temperature approaches zero, the model will become deterministic and\n", - " repetitive. Higher temperature results in more random completions.\n", - " :top_k=0 : Integer value controlling diversity. 1 means only 1 word is\n", - " considered for each step (token), resulting in deterministic completions,\n", - " while 40 means 40 words are considered at each step. 0 (default) is a\n", - " special setting meaning no restrictions. 40 generally is a good value.\n", - " :models_dir : path to parent folder containing model subfolders\n", - " (i.e. contains the folder)\n", - " \"\"\"\n", - " models_dir = os.path.expanduser(os.path.expandvars(models_dir))\n", - "\n", - " hparams = model.default_hparams()\n", - " with open(os.path.join(models_dir, model_name, 'hparams.json')) as f:\n", - " hparams.override_from_dict(json.load(f))\n", - "\n", - " if length is None:\n", - " length = hparams.n_ctx\n", - " elif length > hparams.n_ctx:\n", - " raise ValueError(\"Can't get samples longer than window size: %s\" % hparams.n_ctx)\n", - "\n", - " with tf.Session(graph=tf.Graph()) as sess:\n", - " context = tf.placeholder(tf.int32, [batch_size, None])\n", - " np.random.seed(seed)\n", - " tf.set_random_seed(seed)\n", - "\n", - " output = sample.sample_sequence(\n", - " hparams=hparams, length=length,\n", - " context=context,\n", - " batch_size=batch_size,\n", - " temperature=temperature, top_k=top_k\n", - " )\n", - "\n", - " saver = tf.train.Saver()\n", - " ckpt = tf.train.latest_checkpoint(os.path.join(models_dir, model_name))\n", - " saver.restore(sess, ckpt)\n", - "\n", - " export_dir=os.path.join(models_dir, model_name, \"export\", str(time.time()).split('.')[0])\n", - " if not os.path.isdir(export_dir):\n", - " os.makedirs(export_dir)\n", - "\n", - " builder = tf.saved_model.builder.SavedModelBuilder(export_dir)\n", - " signature = predict_signature_def(inputs={'context': context},\n", - " outputs={'sample': output})\n", - "\n", - " builder.add_meta_graph_and_variables(sess,\n", - " [tf.saved_model.SERVING],\n", - " signature_def_map={\"predict\": signature},\n", - " strip_default_attrs=True)\n", - " builder.save()\n", - "\n", - "\n", - "export_for_serving(top_k=40, length=256, model_name=MODEL_SIZE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "hGfSohMrowmg", - "colab_type": "text" - }, - "source": [ - "## Upload the model to AWS\n", - "\n", - "Cortex loads models from AWS, so we need to upload the exported model." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "BfB5QZ82ozj9", - "colab_type": "text" - }, - "source": [ - "Set these variables to configure your AWS credentials and model upload path:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "B2RNuNk7o1c5", - "colab_type": "code", - "colab": {}, - "cellView": "form" - }, - "source": [ - "AWS_ACCESS_KEY_ID = \"\" #@param {type:\"string\"}\n", - "AWS_SECRET_ACCESS_KEY = \"\" #@param {type:\"string\"}\n", - "S3_UPLOAD_PATH = \"s3://my-bucket/text-generator/gpt-2\" #@param {type:\"string\"}\n", - "\n", - "import sys\n", - "import re\n", - "\n", - "if AWS_ACCESS_KEY_ID == \"\":\n", - " print(\"\\033[91m {}\\033[00m\".format(\"ERROR: Please set AWS_ACCESS_KEY_ID\"), file=sys.stderr)\n", - "\n", - "elif AWS_SECRET_ACCESS_KEY == \"\":\n", - " print(\"\\033[91m {}\\033[00m\".format(\"ERROR: Please set AWS_SECRET_ACCESS_KEY\"), file=sys.stderr)\n", - "\n", - "else:\n", - " try:\n", - " bucket, key = re.match(\"s3://(.+?)/(.+)\", S3_UPLOAD_PATH).groups()\n", - " except:\n", - " print(\"\\033[91m {}\\033[00m\".format(\"ERROR: Invalid s3 path (should be of the form s3://my-bucket/path/to/file)\"), file=sys.stderr)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ics0omsrpS8V", - "colab_type": "text" - }, - "source": [ - "Upload the model to S3:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "BnKncToppUhN", - "colab_type": "code", - "colab": {} - }, - "source": [ - "import os\n", - "import boto3\n", - "\n", - "s3 = boto3.client(\"s3\", aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)\n", - "\n", - "for dirpath, _, filenames in os.walk(\"models/{}/export\".format(MODEL_SIZE)):\n", - " for filename in filenames:\n", - " filepath = os.path.join(dirpath, filename)\n", - " filekey = os.path.join(key, MODEL_SIZE, filepath[len(\"models/{}/export/\".format(MODEL_SIZE)):])\n", - " print(\"Uploading s3://{}/{} ...\".format(bucket, filekey), end = '')\n", - " s3.upload_file(filepath, bucket, filekey)\n", - " print(\" ✓\")\n", - "\n", - "print(\"\\nUploaded model export directory to {}/{}\".format(S3_UPLOAD_PATH, MODEL_SIZE))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "IIMVPhe2qkU4", - "colab_type": "text" - }, - "source": [ - "We also need to upload `vocab.bpe` and `encoder.json`, so that the encoder can encode the input text before making a request to the model." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "YdN8MtZxsO9V", - "colab_type": "code", - "colab": {} - }, - "source": [ - "print(\"Uploading s3://{}/{}/vocab.bpe ...\".format(bucket, key), end = '')\n", - "s3.upload_file(os.path.join(\"models\", MODEL_SIZE, \"vocab.bpe\"), bucket, os.path.join(key, \"vocab.bpe\"))\n", - "print(\" ✓\")\n", - "\n", - "print(\"Uploading s3://{}/{}/encoder.json ...\".format(bucket, key), end = '')\n", - "s3.upload_file(os.path.join(\"models\", MODEL_SIZE, \"encoder.json\"), bucket, os.path.join(key, \"encoder.json\"))\n", - "print(\" ✓\")" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/test/apis/tensorflow/text-generator/handler.py b/test/apis/tensorflow/text-generator/handler.py deleted file mode 100644 index ac86b55cfb..0000000000 --- a/test/apis/tensorflow/text-generator/handler.py +++ /dev/null @@ -1,17 +0,0 @@ -import os -import boto3 -from botocore import UNSIGNED -from botocore.client import Config -from encoder import get_encoder - - -class Handler: - def __init__(self, tensorflow_client, config): - self.client = tensorflow_client - s3 = boto3.client("s3") - self.encoder = get_encoder(s3) - - def handle_post(self, payload): - model_input = {"context": [self.encoder.encode(payload["text"])]} - prediction = self.client.predict(model_input) - return self.encoder.decode(prediction["sample"]) diff --git a/test/apis/tensorflow/text-generator/requirements.txt b/test/apis/tensorflow/text-generator/requirements.txt deleted file mode 100644 index f064e1eb7e..0000000000 --- a/test/apis/tensorflow/text-generator/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -requests -regex diff --git a/test/apis/tensorflow/text-generator/sample.json b/test/apis/tensorflow/text-generator/sample.json deleted file mode 100644 index dfd2a2f433..0000000000 --- a/test/apis/tensorflow/text-generator/sample.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "text": "machine learning is" -} diff --git a/test/apis/traffic-splitter/README.md b/test/apis/traffic-splitter/README.md deleted file mode 100644 index 6bd70ac98e..0000000000 --- a/test/apis/traffic-splitter/README.md +++ /dev/null @@ -1,106 +0,0 @@ -# Splitting traffic between APIs - -This example shows how to split traffic between 2 different iris-classifiers deployed as Realtime APIs. - -To deploy this example: - -1. Determine your CLI Version `cortex version` -1. Clone the repo and switch to the current version; for example, if your cortex version is 0.18.1, run `git clone -b v0.18.1 https://github.com/cortexlabs/cortex` -1. Navigate to this example directory - -## `cortex deploy` - -```bash -$ cortex deploy - -creating iris-classifier-onnx (RealtimeAPI) -creating iris-classifier-tf (RealtimeAPI) -created iris-classifier (TrafficSplitter) -``` - -## `cortex get` - -```bash -$ cortex get - -env realtime api status up-to-date requested last update avg request 2XX -cortex iris-classifier-onnx updating 0 1 27s - - -cortex iris-classifier-tf updating 0 1 27s - - - -env traffic splitter apis last update -cortex iris-classifier iris-classifier-onnx:30 iris-classifier-tf:70 27s -``` - -## `cortex get iris-classifier` - -```bash -$ cortex get iris-classifier - -apis weights status requested last update avg request 2XX 5XX -iris-classifier-onnx 30 live 1 1m - - - -iris-classifier-tf 70 live 1 1m - - - - -last updated: 1m -endpoint: http://***.elb.us-west-2.amazonaws.com/iris-classifier -example curl: curl http://***.elb.us-west-2.amazonaws.com/iris-classifier -X POST -H "Content-Type: application/json" -d @sample.json -... -``` - -## Make multiple requests - -```bash -$ curl http://***.elb.us-west-2.amazonaws.com/iris-classifier -X POST -H "Content-Type: application/json" -d @sample.json -setosa - -$ curl http://***.elb.us-west-2.amazonaws.com/iris-classifier -X POST -H "Content-Type: application/json" -d @sample.json -setosa - -$ curl http://***.elb.us-west-2.amazonaws.com/iris-classifier -X POST -H "Content-Type: application/json" -d @sample.json -setosa - -$ curl http://***.elb.us-west-2.amazonaws.com/iris-classifier -X POST -H "Content-Type: application/json" -d @sample.json -setosa - -$ curl http://***.elb.us-west-2.amazonaws.com/iris-classifier -X POST -H "Content-Type: application/json" -d @sample.json -setosa - -$ curl http://***.elb.us-west-2.amazonaws.com/iris-classifier -X POST -H "Content-Type: application/json" -d @sample.json -setosa -``` - -## `cortex get iris-classifier` - -Notice the requests being routed to the different Realtime APIs based on their weights (the output below may not match yours): - -```bash -$ cortex get iris-classifier - -apis weights status requested last update avg request 2XX 5XX -iris-classifier-onnx 30 live 1 4m 6.00791 ms 1 - -iris-classifier-tf 70 live 1 4m 5.81867 ms 5 - - -last updated: 4m -endpoint: http://***.elb.us-west-2.amazonaws.com/iris-classifier -example curl: curl http://***.elb.us-west-2.amazonaws.com/iris-classifier -X POST -H "Content-Type: application/json" -d @sample.json -... -``` - -## Cleanup - -Use `cortex delete ` to delete the Traffic Splitter and the two Realtime APIs (note that the Traffic Splitter and each Realtime API must be deleted by separate `cortex delete` commands): - -```bash -$ cortex delete iris-classifier - -deleting iris-classifier - -$ cortex delete iris-classifier-onnx - -deleting iris-classifier-onnx - -$ cortex delete iris-classifier-tf - -deleting iris-classifier-tf -``` - -Running `cortex delete ` will free up cluster resources and allow Cortex to scale down to the minimum number of instances you specified during cluster installation. It will not spin down your cluster. diff --git a/test/apis/traffic-splitter/cortex.yaml b/test/apis/traffic-splitter/cortex.yaml deleted file mode 100644 index c8265eb493..0000000000 --- a/test/apis/traffic-splitter/cortex.yaml +++ /dev/null @@ -1,36 +0,0 @@ -- name: iris-classifier-pytorch - kind: RealtimeAPI - handler: - type: python - path: pytorch_handler.py - dependencies: - pip: pytorch_requirements.txt - config: - model: s3://cortex-examples/pytorch/iris-classifier/weights.pth - -- name: iris-classifier-onnx - kind: RealtimeAPI - handler: - type: python - path: onnx_handler.py - dependencies: - pip: onnx_requirements.txt - multi_model_reloading: - path: s3://cortex-examples/onnx/iris-classifier/ - -- name: request-recorder - kind: RealtimeAPI - handler: - type: python - path: request_recorder.py - -- name: iris-classifier - kind: TrafficSplitter - apis: - - name: iris-classifier-onnx - weight: 30 - - name: iris-classifier-pytorch - weight: 70 - - name: request-recorder - shadow: true - weight: 100 diff --git a/test/apis/traffic-splitter/model.py b/test/apis/traffic-splitter/model.py deleted file mode 100644 index a43a362544..0000000000 --- a/test/apis/traffic-splitter/model.py +++ /dev/null @@ -1,58 +0,0 @@ -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch.autograd import Variable - - -class IrisNet(nn.Module): - def __init__(self): - super(IrisNet, self).__init__() - self.fc1 = nn.Linear(4, 100) - self.fc2 = nn.Linear(100, 100) - self.fc3 = nn.Linear(100, 3) - self.softmax = nn.Softmax(dim=1) - - def forward(self, X): - X = F.relu(self.fc1(X)) - X = self.fc2(X) - X = self.fc3(X) - X = self.softmax(X) - return X - - -if __name__ == "__main__": - from sklearn.datasets import load_iris - from sklearn.model_selection import train_test_split - from sklearn.metrics import accuracy_score - - iris = load_iris() - X, y = iris.data, iris.target - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.8, random_state=42) - - train_X = Variable(torch.Tensor(X_train).float()) - test_X = Variable(torch.Tensor(X_test).float()) - train_y = Variable(torch.Tensor(y_train).long()) - test_y = Variable(torch.Tensor(y_test).long()) - - model = IrisNet() - - criterion = nn.CrossEntropyLoss() - - optimizer = torch.optim.SGD(model.parameters(), lr=0.01) - - for epoch in range(1000): - optimizer.zero_grad() - out = model(train_X) - loss = criterion(out, train_y) - loss.backward() - optimizer.step() - - if epoch % 100 == 0: - print("number of epoch {} loss {}".format(epoch, loss)) - - predict_out = model(test_X) - _, predict_y = torch.max(predict_out, 1) - - print("prediction accuracy {}".format(accuracy_score(test_y.data, predict_y.data))) - - torch.save(model.state_dict(), "weights.pth") diff --git a/test/apis/traffic-splitter/onnx_handler.py b/test/apis/traffic-splitter/onnx_handler.py deleted file mode 100644 index 93e3cb0420..0000000000 --- a/test/apis/traffic-splitter/onnx_handler.py +++ /dev/null @@ -1,37 +0,0 @@ -import os -import onnxruntime as rt -import numpy as np - -labels = ["setosa", "versicolor", "virginica"] - - -class Handler: - def __init__(self, model_client, config): - self.client = model_client - - def handle_post(self, payload): - session = self.client.get_model() - - input_dict = { - "input": np.array( - [ - payload["sepal_length"], - payload["sepal_width"], - payload["petal_length"], - payload["petal_width"], - ], - dtype="float32", - ).reshape(1, 4), - } - prediction = session.run(["label"], input_dict) - - predicted_class_id = prediction[0][0] - return labels[predicted_class_id] - - def load_model(self, model_path): - """ - Load ONNX model from disk. - """ - - model_path = os.path.join(model_path, os.listdir(model_path)[0]) - return rt.InferenceSession(model_path) diff --git a/test/apis/traffic-splitter/onnx_requirements.txt b/test/apis/traffic-splitter/onnx_requirements.txt deleted file mode 100644 index a8ef6d8489..0000000000 --- a/test/apis/traffic-splitter/onnx_requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -onnxruntime==1.6.0 -numpy==1.19.1 diff --git a/test/apis/traffic-splitter/pytorch_handler.py b/test/apis/traffic-splitter/pytorch_handler.py deleted file mode 100644 index 1ceb99984b..0000000000 --- a/test/apis/traffic-splitter/pytorch_handler.py +++ /dev/null @@ -1,43 +0,0 @@ -import re -import torch -import os -import boto3 -from botocore import UNSIGNED -from botocore.client import Config -from model import IrisNet - -labels = ["setosa", "versicolor", "virginica"] - - -class Handler: - def __init__(self, config): - # download the model - bucket, key = re.match("s3://(.+?)/(.+)", config["model"]).groups() - s3 = boto3.client("s3") - s3.download_file(bucket, key, "/tmp/model.pth") - - # initialize the model - model = IrisNet() - model.load_state_dict(torch.load("/tmp/model.pth")) - model.eval() - - self.model = model - - def handle_post(self, payload): - # Convert the request to a tensor and pass it into the model - input_tensor = torch.FloatTensor( - [ - [ - payload["sepal_length"], - payload["sepal_width"], - payload["petal_length"], - payload["petal_width"], - ] - ] - ) - - # Run the prediction - output = self.model(input_tensor) - - # Translate the model output to the corresponding label string - return labels[torch.argmax(output[0])] diff --git a/test/apis/traffic-splitter/pytorch_requirements.txt b/test/apis/traffic-splitter/pytorch_requirements.txt deleted file mode 100644 index 588cf3a19d..0000000000 --- a/test/apis/traffic-splitter/pytorch_requirements.txt +++ /dev/null @@ -1 +0,0 @@ -torch==1.7.1 diff --git a/test/apis/traffic-splitter/request_recorder.py b/test/apis/traffic-splitter/request_recorder.py deleted file mode 100644 index 355b4c7497..0000000000 --- a/test/apis/traffic-splitter/request_recorder.py +++ /dev/null @@ -1,10 +0,0 @@ -from cortex_internal.lib.log import logger as cortex_logger - - -class Handler: - def __init__(self, config): - pass - - def handle_post(self, payload): - cortex_logger.info("received payload", extra={"payload": payload}) - return payload diff --git a/test/apis/traffic-splitter/sample.json b/test/apis/traffic-splitter/sample.json deleted file mode 100644 index e17bbb2896..0000000000 --- a/test/apis/traffic-splitter/sample.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "sepal_length": 5.2, - "sepal_width": 3.6, - "petal_length": 1.4, - "petal_width": 0.3 -} diff --git a/test/apis/trafficsplitter/hello-world/.dockerignore b/test/apis/trafficsplitter/hello-world/.dockerignore new file mode 100644 index 0000000000..12657957ef --- /dev/null +++ b/test/apis/trafficsplitter/hello-world/.dockerignore @@ -0,0 +1,8 @@ +*.dockerfile +README.md +sample.json +*.pyc +*.pyo +*.pyd +__pycache__ +.pytest_cache diff --git a/test/apis/trafficsplitter/hello-world/cortex_cpu.yaml b/test/apis/trafficsplitter/hello-world/cortex_cpu.yaml new file mode 100644 index 0000000000..8d18685e83 --- /dev/null +++ b/test/apis/trafficsplitter/hello-world/cortex_cpu.yaml @@ -0,0 +1,67 @@ +- name: hello-world-a + kind: RealtimeAPI + pod: + port: 9000 + containers: + - name: api + image: quay.io/cortexlabs-test/realtime-hello-world-cpu:latest + env: + RESPONSE: "hello from API A" + readiness_probe: + http_get: + path: "/healthz" + port: 9000 + compute: + cpu: 200m + mem: 128M + autoscaling: + max_concurrency: 1 + +- name: hello-world-b + kind: RealtimeAPI + pod: + port: 9000 + containers: + - name: api + image: quay.io/cortexlabs-test/realtime-hello-world-cpu:latest + env: + RESPONSE: "hello from API B" + readiness_probe: + http_get: + path: "/healthz" + port: 9000 + compute: + cpu: 200m + mem: 128M + autoscaling: + max_concurrency: 1 + +- name: hello-world-shadow + kind: RealtimeAPI + pod: + port: 9000 + containers: + - name: api + image: quay.io/cortexlabs-test/realtime-hello-world-cpu:latest + env: + RESPONSE: "hello from shadow API" + readiness_probe: + http_get: + path: "/healthz" + port: 9000 + compute: + cpu: 200m + mem: 128M + autoscaling: + max_concurrency: 1 + +- name: hello-world + kind: TrafficSplitter + apis: + - name: hello-world-a + weight: 30 + - name: hello-world-b + weight: 70 + - name: hello-world-shadow + shadow: true + weight: 100 diff --git a/test/apis/trafficsplitter/hello-world/sample.json b/test/apis/trafficsplitter/hello-world/sample.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/test/apis/trafficsplitter/hello-world/sample.json @@ -0,0 +1 @@ +{} diff --git a/test/e2e/e2e/tests.py b/test/e2e/e2e/tests.py index cf2a5615a3..28b2c45f26 100644 --- a/test/e2e/e2e/tests.py +++ b/test/e2e/e2e/tests.py @@ -66,7 +66,8 @@ def test_realtime_api( client: cx.Client, api: str, timeout: int = None, - api_config_name: str = "cortex.yaml", + api_config_name: str = "cortex_cpu.yaml", + extra_path: str = "", ): api_dir = TEST_APIS_DIR / api with open(str(api_dir / api_config_name)) as f: @@ -79,7 +80,7 @@ def test_realtime_api( api_name = api_specs[0]["name"] for api_spec in api_specs: - client.deploy(api_spec=api_spec, project_dir=str(api_dir)) + client.deploy(api_spec=api_spec) try: assert apis_ready( @@ -89,7 +90,7 @@ def test_realtime_api( if not expectations or "grpc" not in expectations: with open(str(api_dir / "sample.json")) as f: payload = json.load(f) - response = request_prediction(client, api_name, payload) + response = request_prediction(client, api_name, payload, extra_path) assert ( response.status_code == HTTPStatus.OK @@ -137,7 +138,7 @@ def test_batch_api( deploy_timeout: int = None, job_timeout: int = None, retry_attempts: int = 0, - api_config_name: str = "cortex.yaml", + api_config_name: str = "cortex_cpu.yaml", local_operator: bool = False, ): api_dir = TEST_APIS_DIR / api @@ -147,7 +148,7 @@ def test_batch_api( assert len(api_specs) == 1 api_name = api_specs[0]["name"] - client.deploy(api_spec=api_specs[0], project_dir=str(api_dir)) + client.deploy(api_spec=api_specs[0]) try: endpoint_override = f"http://localhost:8888/batch/{api_name}" if local_operator else None @@ -220,7 +221,7 @@ def test_async_api( deploy_timeout: int = None, poll_retries: int = 5, poll_sleep_seconds: int = 1, - api_config_name: str = "cortex.yaml", + api_config_name: str = "cortex_cpu.yaml", ): api_dir = TEST_APIS_DIR / api with open(str(api_dir / api_config_name)) as f: @@ -234,7 +235,7 @@ def test_async_api( assert len(api_specs) == 1 api_name = api_specs[0]["name"] - client.deploy(api_spec=api_specs[0], project_dir=str(api_dir)) + client.deploy(api_spec=api_specs[0]) try: assert apis_ready( @@ -333,7 +334,7 @@ def test_task_api( deploy_timeout: int = None, job_timeout: int = None, retry_attempts: int = 0, - api_config_name: str = "cortex.yaml", + api_config_name: str = "cortex_cpu.yaml", local_operator: bool = False, ): api_dir = TEST_APIS_DIR / api @@ -343,7 +344,7 @@ def test_task_api( assert len(api_specs) == 1 api_name = api_specs[0]["name"] - client.deploy(api_spec=api_specs[0], project_dir=str(api_dir)) + client.deploy(api_spec=api_specs[0]) try: endpoint_override = f"http://localhost:8888/tasks/{api_name}" if local_operator else None @@ -402,7 +403,7 @@ def test_autoscaling( apis: Dict[str, Any], autoscaling_config: Dict[str, Union[int, float]], deploy_timeout: int = None, - api_config_name: str = "cortex.yaml", + api_config_name: str = "cortex_cpu.yaml", ): max_replicas = autoscaling_config["max_replicas"] query_params = apis["query_params"] @@ -422,7 +423,7 @@ def test_autoscaling( "downscale_stabilization_period": "1m", } all_api_names.append(api_specs[0]["name"]) - client.deploy(api_spec=api_specs[0], project_dir=api_dir) + client.deploy(api_spec=api_specs[0]) primary_api_name = all_api_names[0] autoscaling = client.get_api(primary_api_name)["spec"]["autoscaling"] @@ -513,7 +514,7 @@ def test_load_realtime( api: str, load_config: Dict[str, Union[int, float]], deploy_timeout: int = None, - api_config_name: str = "cortex.yaml", + api_config_name: str = "cortex_cpu.yaml", ): total_requests = load_config["total_requests"] @@ -534,7 +535,7 @@ def test_load_realtime( "max_replicas": desired_replicas, } api_name = api_specs[0]["name"] - client.deploy(api_spec=api_specs[0], project_dir=str(api_dir)) + client.deploy(api_spec=api_specs[0]) # controls the flow of requests request_stopper = td.Event() @@ -624,7 +625,7 @@ def test_load_async( load_config: Dict[str, Union[int, float]], deploy_timeout: int = None, poll_sleep_seconds: int = 1, - api_config_name: str = "cortex.yaml", + api_config_name: str = "cortex_cpu.yaml", ): total_requests = load_config["total_requests"] @@ -643,7 +644,7 @@ def test_load_async( "max_replicas": desired_replicas, } api_name = api_specs[0]["name"] - client.deploy(api_spec=api_specs[0], project_dir=str(api_dir)) + client.deploy(api_spec=api_specs[0]) request_stopper = td.Event() map_stopper = td.Event() @@ -742,7 +743,7 @@ def test_load_batch( load_config: Dict[str, Union[int, float]], deploy_timeout: int = None, retry_attempts: int = 0, - api_config_name: str = "cortex.yaml", + api_config_name: str = "cortex_cpu.yaml", ): jobs = load_config["jobs"] @@ -767,7 +768,7 @@ def test_load_batch( sample_generator = load_generator(sample_generator_path) api_name = api_specs[0]["name"] - client.deploy(api_spec=api_specs[0], project_dir=str(api_dir)) + client.deploy(api_spec=api_specs[0]) api_endpoint = client.get_api(api_name)["endpoint"] try: @@ -846,7 +847,7 @@ def test_load_task( deploy_timeout: int = None, retry_attempts: int = 0, poll_sleep_seconds: int = 1, - api_config_name: str = "cortex.yaml", + api_config_name: str = "cortex_cpu.yaml", ): jobs = load_config["jobs"] @@ -860,7 +861,7 @@ def test_load_task( assert len(api_specs) == 1 api_name = api_specs[0]["name"] - client.deploy(api_spec=api_specs[0], project_dir=str(api_dir)) + client.deploy(api_spec=api_specs[0]) request_stopper = td.Event() map_stopper = td.Event() @@ -921,7 +922,7 @@ def test_long_running_realtime( api: str, long_running_config: Dict[str, Union[int, float]], deploy_timeout: int = None, - api_config_name: str = "cortex.yaml", + api_config_name: str = "cortex_cpu.yaml", ): api_dir = TEST_APIS_DIR / api with open(str(api_dir / api_config_name)) as f: @@ -937,7 +938,7 @@ def test_long_running_realtime( api_name = api_specs[0]["name"] for api_spec in api_specs: - client.deploy(api_spec=api_spec, project_dir=str(api_dir)) + client.deploy(api_spec=api_spec) try: assert apis_ready( diff --git a/test/e2e/e2e/utils.py b/test/e2e/e2e/utils.py index fdaf54630f..013f1c0f57 100644 --- a/test/e2e/e2e/utils.py +++ b/test/e2e/e2e/utils.py @@ -14,6 +14,7 @@ import importlib import pathlib +import os import sys import threading as td import time @@ -169,10 +170,16 @@ def generate_grpc( def request_prediction( - client: cx.Client, api_name: str, payload: Union[List, Dict] + client: cx.Client, + api_name: str, + payload: Union[List, Dict], + extra_path: Optional[str] = None, ) -> requests.Response: api_info = client.get_api(api_name) - response = requests.post(api_info["endpoint"], json=payload) + endpoint = api_info["endpoint"] + if extra_path and extra_path != "": + endpoint = os.path.join(endpoint, extra_path) + response = requests.post(endpoint, json=payload) return response diff --git a/test/e2e/tests/aws/test_async.py b/test/e2e/tests/aws/test_async.py index a8a59ac944..1413c55c9c 100644 --- a/test/e2e/tests/aws/test_async.py +++ b/test/e2e/tests/aws/test_async.py @@ -19,10 +19,8 @@ import e2e.tests -TEST_APIS = [ - "async/iris-classifier", - "async/tensorflow", -] +TEST_APIS = ["async/text-generator"] +TEST_APIS_GPU = ["async/text-generator"] @pytest.mark.usefixtures("client") @@ -35,3 +33,20 @@ def test_async_api(printer: Callable, config: Dict, client: cx.Client, api: str) deploy_timeout=config["global"]["async_deploy_timeout"], poll_retries=config["global"]["async_workload_timeout"], ) + + +@pytest.mark.usefixtures("client") +@pytest.mark.parametrize("api", TEST_APIS_GPU) +def test_async_api_gpu(printer: Callable, config: Dict, client: cx.Client, api: str): + skip_gpus = config["global"].get("skip_gpus", False) + if skip_gpus: + pytest.skip("--skip-gpus flag detected, skipping GPU tests") + + e2e.tests.test_async_api( + printer=printer, + client=client, + api=api, + deploy_timeout=config["global"]["async_deploy_timeout"], + poll_retries=config["global"]["async_workload_timeout"], + api_config_name="cortex_gpu.yaml", + ) diff --git a/test/e2e/tests/aws/test_autoscaling.py b/test/e2e/tests/aws/test_autoscaling.py index 075bbed3d9..70789b5182 100644 --- a/test/e2e/tests/aws/test_autoscaling.py +++ b/test/e2e/tests/aws/test_autoscaling.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, Dict +from typing import Any, Callable, Dict import cortex as cx import pytest @@ -21,8 +21,8 @@ TEST_APIS = [ { - "primary": "sleep", - "dummy": ["sklearn/mpg-estimator", "tensorflow/iris-classifier"], + "primary": "realtime/sleep", + "dummy": ["realtime/prime-generator"], "query_params": { "sleep": "1.0", }, @@ -32,7 +32,7 @@ @pytest.mark.usefixtures("client") @pytest.mark.parametrize("apis", TEST_APIS) -def test_autoscaling(printer: Callable, config: Dict, client: cx.Client, apis: str): +def test_autoscaling(printer: Callable, config: Dict, client: cx.Client, apis: Dict[str, Any]): skip_autoscaling_test = config["global"].get("skip_autoscaling", False) if skip_autoscaling_test: pytest.skip("--skip-autoscaling flag detected, skipping autoscaling tests") diff --git a/test/e2e/tests/aws/test_batch.py b/test/e2e/tests/aws/test_batch.py index cabb7441a7..dda3464981 100644 --- a/test/e2e/tests/aws/test_batch.py +++ b/test/e2e/tests/aws/test_batch.py @@ -19,8 +19,8 @@ import e2e.tests -TEST_APIS = ["batch/image-classifier", "batch/onnx", "batch/tensorflow"] -TEST_APIS_INF = ["batch/inferentia"] +TEST_APIS = ["batch/image-classifier-alexnet"] +TEST_APIS_GPU = ["batch/image-classifier-alexnet"] @pytest.mark.usefixtures("client") @@ -46,11 +46,11 @@ def test_batch_api(printer: Callable, config: Dict, client: cx.Client, api: str) @pytest.mark.usefixtures("client") -@pytest.mark.parametrize("api", TEST_APIS_INF) -def test_batch_api_inf(printer: Callable, config: Dict, client: cx.Client, api: str): - skip_infs = config["global"].get("skip_infs", False) - if skip_infs: - pytest.skip("--skip-infs flag detected, skipping Inferentia tests") +@pytest.mark.parametrize("api", TEST_APIS_GPU) +def test_batch_api_gpu(printer: Callable, config: Dict, client: cx.Client, api: str): + skip_gpus = config["global"].get("skip_gpus", False) + if skip_gpus: + pytest.skip("--skip-gpus flag detected, skipping GPU tests") s3_path = config["aws"].get("s3_path") if not s3_path: @@ -68,5 +68,5 @@ def test_batch_api_inf(printer: Callable, config: Dict, client: cx.Client, api: job_timeout=config["global"]["batch_job_timeout"], retry_attempts=5, local_operator=config["global"]["local_operator"], - api_config_name="cortex_inf.yaml", + api_config_name="cortex_gpu.yaml", ) diff --git a/test/e2e/tests/aws/test_load.py b/test/e2e/tests/aws/test_load.py index ee14102140..0c0fe3e10f 100644 --- a/test/e2e/tests/aws/test_load.py +++ b/test/e2e/tests/aws/test_load.py @@ -19,10 +19,10 @@ import e2e.tests -TEST_APIS_REALTIME = ["tensorflow/iris-classifier"] -TEST_APIS_ASYNC = ["async/iris-classifier"] +TEST_APIS_REALTIME = ["realtime/prime-generator"] +TEST_APIS_ASYNC = ["async/text-generator"] TEST_APIS_BATCH = ["batch/sum"] -TEST_APIS_TASK = ["task/hello-world"] +TEST_APIS_TASK = ["task/iris-classifier-trainer"] @pytest.mark.usefixtures("client") diff --git a/test/e2e/tests/aws/test_long_running.py b/test/e2e/tests/aws/test_long_running.py index 93351998c7..b7a5d75590 100644 --- a/test/e2e/tests/aws/test_long_running.py +++ b/test/e2e/tests/aws/test_long_running.py @@ -19,7 +19,7 @@ import e2e.tests -TEST_APIS = ["onnx/iris-classifier"] +TEST_APIS = ["realtime/text-generator"] @pytest.mark.usefixtures("client") diff --git a/test/e2e/tests/aws/test_realtime.py b/test/e2e/tests/aws/test_realtime.py index e960b22292..71c4b2fd13 100644 --- a/test/e2e/tests/aws/test_realtime.py +++ b/test/e2e/tests/aws/test_realtime.py @@ -18,54 +18,59 @@ import e2e.tests -TEST_APIS = ["pytorch/iris-classifier", "onnx/iris-classifier", "tensorflow/iris-classifier"] -TEST_APIS_GRPC = ["grpc/iris-classifier-sklearn", "grpc/prime-number-generator"] -TEST_APIS_GPU = ["pytorch/text-generator", "tensorflow/text-generator"] -TEST_APIS_INF = ["pytorch/image-classifier-resnet50"] +TEST_APIS = [ + { + "name": "realtime/image-classifier-resnet50", + "extra_path": "v1/models/resnet50:predict", + }, + { + "name": "realtime/prime-generator", + "extra_path": "", + }, + { + "name": "realtime/text-generator", + "extra_path": "", + }, +] +TEST_APIS_GPU = [ + { + "name": "realtime/image-classifier-resnet50", + "extra_path": "v1/models/resnet50:predict", + }, + { + "name": "realtime/text-generator", + "extra_path": "", + }, +] @pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS) -def test_realtime_api(printer: Callable, config: Dict, client: cx.Client, api: str): - e2e.tests.test_realtime_api( - printer=printer, client=client, api=api, timeout=config["global"]["realtime_deploy_timeout"] - ) - +def test_realtime_api(printer: Callable, config: Dict, client: cx.Client, api: Dict[str, str]): -@pytest.mark.usefixtures("client") -@pytest.mark.parametrize("api", TEST_APIS_GRPC) -def test_realtime_api_grpc(printer: Callable, config: Dict, client: cx.Client, api: str): + printer(f"testing {api['name']}") e2e.tests.test_realtime_api( printer=printer, client=client, - api=api, + api=api["name"], timeout=config["global"]["realtime_deploy_timeout"], + extra_path=api["extra_path"], ) @pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS_GPU) -def test_realtime_api_gpu(printer: Callable, config: Dict, client: cx.Client, api: str): +def test_realtime_api_gpu(printer: Callable, config: Dict, client: cx.Client, api: Dict[str, str]): skip_gpus = config["global"].get("skip_gpus", False) if skip_gpus: pytest.skip("--skip-gpus flag detected, skipping GPU tests") - e2e.tests.test_realtime_api( - printer=printer, client=client, api=api, timeout=config["global"]["realtime_deploy_timeout"] - ) - - -@pytest.mark.usefixtures("client") -@pytest.mark.parametrize("api", TEST_APIS_INF) -def test_realtime_api_inf(printer: Callable, config: Dict, client: cx.Client, api: str): - skip_infs = config["global"].get("skip_infs", False) - if skip_infs: - pytest.skip("--skip-infs flag detected, skipping Inferentia tests") - + printer(f"testing {api['name']}") e2e.tests.test_realtime_api( printer=printer, client=client, - api=api, + api=api["name"], timeout=config["global"]["realtime_deploy_timeout"], - api_config_name="cortex_inf.yaml", + api_config_name="cortex_gpu.yaml", + extra_path=api["extra_path"], ) diff --git a/test/e2e/tests/aws/test_task.py b/test/e2e/tests/aws/test_task.py index 05fec51e75..75825c584d 100644 --- a/test/e2e/tests/aws/test_task.py +++ b/test/e2e/tests/aws/test_task.py @@ -19,7 +19,7 @@ import e2e.tests -TEST_APIS = ["task/hello-world"] +TEST_APIS = ["task/iris-classifier-trainer"] @pytest.mark.usefixtures("client") diff --git a/test/e2e/tests/conftest.py b/test/e2e/tests/conftest.py index 6fb520a551..147a906abb 100644 --- a/test/e2e/tests/conftest.py +++ b/test/e2e/tests/conftest.py @@ -114,7 +114,7 @@ def pytest_configure(config): "status_code_timeout": 60, # measured in seconds }, "async": { - "total_requests": 10 ** 4, + "total_requests": 10 ** 3, "desired_replicas": 50, "concurrency": 10, "submit_timeout": 120, # measured in seconds diff --git a/test/utils/build-and-push-images.sh b/test/utils/build-and-push-images.sh new file mode 100755 index 0000000000..ed06652899 --- /dev/null +++ b/test/utils/build-and-push-images.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Copyright 2021 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. >/dev/null && pwd)" +source $ROOT/dev/util.sh + +# registry address +host=$1 + +# images to build +api_images=( + "async-text-generator-cpu" + "async-text-generator-gpu" + "batch-image-classifier-alexnet-cpu" + "batch-image-classifier-alexnet-gpu" + "batch-sum-cpu" + "realtime-image-classifier-resnet50-cpu" + "realtime-image-classifier-resnet50-gpu" + "realtime-prime-generator-cpu" + "realtime-sleep-cpu" + "realtime-text-generator-cpu" + "realtime-text-generator-gpu" + "realtime-hello-world-cpu" + "task-iris-classifier-trainer-cpu" +) + +# build the images +for image in "${api_images[@]}"; do + kind=$(python -c "first_element='$image'.split('-', 1)[0]; print(first_element)") + api_name=$(python -c "right_tail='$image'.split('-', 1)[1]; mid_section=right_tail.rsplit('-', 1)[0]; print(mid_section)") + compute_type=$(python -c "last_element='$image'.rsplit('-', 1)[1]; print(last_element)") + dir="${ROOT}/test/apis/${kind}/${api_name}" + + blue_echo "Building $host/cortexlabs-test/$image:latest..." + docker build $dir -f $dir/$api_name-$compute_type.dockerfile -t cortexlabs-test/$image -t $host/cortexlabs-test/$image + green_echo "Built $host/cortexlabs-test/$image:latest\n" +done + +# push the images +echo "$DOCKER_PASSWORD" | docker login $host -u "$DOCKER_USERNAME" --password-stdin +for image in "${api_images[@]}"; do + blue_echo "Pushing $host/cortexlabs-test/$image:latest..." + docker push $host/cortexlabs-test/${image} + green_echo "Pushed $host/cortexlabs-test/$image:latest\n" +done From d7c03dcfe48c1c33dc8ef373fd3366a9272bb07c Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Thu, 27 May 2021 21:17:17 +0300 Subject: [PATCH 28/82] CaaS - fix cortex CLI (#2194) --- pkg/operator/endpoints/delete.go | 2 +- pkg/operator/endpoints/deploy.go | 2 +- pkg/operator/endpoints/get.go | 6 +++--- pkg/operator/endpoints/get_batch_job.go | 2 +- pkg/operator/endpoints/get_task_job.go | 2 +- pkg/operator/endpoints/info.go | 2 +- pkg/operator/endpoints/refresh.go | 2 +- pkg/operator/endpoints/respond.go | 12 ++++-------- pkg/operator/endpoints/stop_batch_job.go | 2 +- pkg/operator/endpoints/stop_task_job.go | 2 +- pkg/operator/endpoints/submit_batch.go | 2 +- pkg/operator/endpoints/submit_task.go | 2 +- pkg/operator/endpoints/verify_cortex.go | 2 +- pkg/operator/resources/realtimeapi/metrics.go | 5 +++++ 14 files changed, 23 insertions(+), 22 deletions(-) diff --git a/pkg/operator/endpoints/delete.go b/pkg/operator/endpoints/delete.go index c143216193..355590deb7 100644 --- a/pkg/operator/endpoints/delete.go +++ b/pkg/operator/endpoints/delete.go @@ -32,5 +32,5 @@ func Delete(w http.ResponseWriter, r *http.Request) { respondError(w, r, err) return } - respond(w, response) + respondJSON(w, r, response) } diff --git a/pkg/operator/endpoints/deploy.go b/pkg/operator/endpoints/deploy.go index 57784ad775..345951b1e3 100644 --- a/pkg/operator/endpoints/deploy.go +++ b/pkg/operator/endpoints/deploy.go @@ -48,5 +48,5 @@ func Deploy(w http.ResponseWriter, r *http.Request) { return } - respond(w, response) + respondJSON(w, r, response) } diff --git a/pkg/operator/endpoints/get.go b/pkg/operator/endpoints/get.go index 59b61835ec..821621c804 100644 --- a/pkg/operator/endpoints/get.go +++ b/pkg/operator/endpoints/get.go @@ -30,7 +30,7 @@ func GetAPIs(w http.ResponseWriter, r *http.Request) { return } - respond(w, response) + respondJSON(w, r, response) } func GetAPI(w http.ResponseWriter, r *http.Request) { @@ -42,7 +42,7 @@ func GetAPI(w http.ResponseWriter, r *http.Request) { return } - respond(w, response) + respondJSON(w, r, response) } func GetAPIByID(w http.ResponseWriter, r *http.Request) { @@ -55,5 +55,5 @@ func GetAPIByID(w http.ResponseWriter, r *http.Request) { return } - respond(w, response) + respondJSON(w, r, response) } diff --git a/pkg/operator/endpoints/get_batch_job.go b/pkg/operator/endpoints/get_batch_job.go index a292cfa826..ea78c7474e 100644 --- a/pkg/operator/endpoints/get_batch_job.go +++ b/pkg/operator/endpoints/get_batch_job.go @@ -86,5 +86,5 @@ func GetBatchJob(w http.ResponseWriter, r *http.Request) { Endpoint: parsedURL.String(), } - respond(w, response) + respondJSON(w, r, response) } diff --git a/pkg/operator/endpoints/get_task_job.go b/pkg/operator/endpoints/get_task_job.go index 75b08aeaec..dc0843fbdf 100644 --- a/pkg/operator/endpoints/get_task_job.go +++ b/pkg/operator/endpoints/get_task_job.go @@ -86,5 +86,5 @@ func GetTaskJob(w http.ResponseWriter, r *http.Request) { Endpoint: parsedURL.String(), } - respond(w, response) + respondJSON(w, r, response) } diff --git a/pkg/operator/endpoints/info.go b/pkg/operator/endpoints/info.go index 9d43babcfd..c0c04e0ce0 100644 --- a/pkg/operator/endpoints/info.go +++ b/pkg/operator/endpoints/info.go @@ -48,7 +48,7 @@ func Info(w http.ResponseWriter, r *http.Request) { NodeInfos: nodeInfos, NumPendingReplicas: numPendingReplicas, } - respond(w, response) + respondJSON(w, r, response) } func getNodeInfos() ([]schema.NodeInfo, int, error) { diff --git a/pkg/operator/endpoints/refresh.go b/pkg/operator/endpoints/refresh.go index f0bfc7e8ea..bcb67c4cc6 100644 --- a/pkg/operator/endpoints/refresh.go +++ b/pkg/operator/endpoints/refresh.go @@ -37,5 +37,5 @@ func Refresh(w http.ResponseWriter, r *http.Request) { response := schema.RefreshResponse{ Message: msg, } - respond(w, response) + respondJSON(w, r, response) } diff --git a/pkg/operator/endpoints/respond.go b/pkg/operator/endpoints/respond.go index 1edaa145a9..039b49fc2c 100644 --- a/pkg/operator/endpoints/respond.go +++ b/pkg/operator/endpoints/respond.go @@ -28,16 +28,12 @@ import ( var operatorLogger = logging.GetLogger() -func respond(w http.ResponseWriter, response interface{}) { +func respondJSON(w http.ResponseWriter, r *http.Request, response interface{}) { + if err := json.NewEncoder(w).Encode(response); err != nil { + respondError(w, r, errors.Wrap(err, "failed to encode response")) + } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(response) -} - -func respondPlainText(w http.ResponseWriter, response string) { - w.Header().Set("Content-Type", "text/plain") - w.WriteHeader(http.StatusOK) - w.Write([]byte(response)) } func respondError(w http.ResponseWriter, r *http.Request, err error, strs ...string) { diff --git a/pkg/operator/endpoints/stop_batch_job.go b/pkg/operator/endpoints/stop_batch_job.go index 98cc4d33a6..c4c16416c0 100644 --- a/pkg/operator/endpoints/stop_batch_job.go +++ b/pkg/operator/endpoints/stop_batch_job.go @@ -46,7 +46,7 @@ func StopBatchJob(w http.ResponseWriter, r *http.Request) { return } - respond(w, schema.DeleteResponse{ + respondJSON(w, r, schema.DeleteResponse{ Message: fmt.Sprintf("stopped job %s", jobID), }) } diff --git a/pkg/operator/endpoints/stop_task_job.go b/pkg/operator/endpoints/stop_task_job.go index 9e9d09e852..cb2ab6d3c4 100644 --- a/pkg/operator/endpoints/stop_task_job.go +++ b/pkg/operator/endpoints/stop_task_job.go @@ -46,7 +46,7 @@ func StopTaskJob(w http.ResponseWriter, r *http.Request) { return } - respond(w, schema.DeleteResponse{ + respondJSON(w, r, schema.DeleteResponse{ Message: fmt.Sprintf("stopped job %s", jobID), }) } diff --git a/pkg/operator/endpoints/submit_batch.go b/pkg/operator/endpoints/submit_batch.go index 65c96751b2..bd73f6bf51 100644 --- a/pkg/operator/endpoints/submit_batch.go +++ b/pkg/operator/endpoints/submit_batch.go @@ -94,5 +94,5 @@ func SubmitBatchJob(w http.ResponseWriter, r *http.Request) { return } - respond(w, jobSpec) + respondJSON(w, r, jobSpec) } diff --git a/pkg/operator/endpoints/submit_task.go b/pkg/operator/endpoints/submit_task.go index 0a122eb81b..7135757dd9 100644 --- a/pkg/operator/endpoints/submit_task.go +++ b/pkg/operator/endpoints/submit_task.go @@ -74,5 +74,5 @@ func SubmitTaskJob(w http.ResponseWriter, r *http.Request) { return } - respond(w, jobSpec) + respondJSON(w, r, jobSpec) } diff --git a/pkg/operator/endpoints/verify_cortex.go b/pkg/operator/endpoints/verify_cortex.go index 3232e82c4d..17468e8dab 100644 --- a/pkg/operator/endpoints/verify_cortex.go +++ b/pkg/operator/endpoints/verify_cortex.go @@ -23,5 +23,5 @@ import ( ) func VerifyCortex(w http.ResponseWriter, r *http.Request) { - respond(w, schema.VerifyCortexResponse{}) + respondJSON(w, r, schema.VerifyCortexResponse{}) } diff --git a/pkg/operator/resources/realtimeapi/metrics.go b/pkg/operator/resources/realtimeapi/metrics.go index 4c6c9fcef6..3f0178d9a6 100644 --- a/pkg/operator/resources/realtimeapi/metrics.go +++ b/pkg/operator/resources/realtimeapi/metrics.go @@ -19,6 +19,7 @@ package realtimeapi import ( "context" "fmt" + "math" "time" "github.com/cortexlabs/cortex/pkg/config" @@ -152,6 +153,10 @@ func getAvgLatencyMetric(promAPIv1 promv1.API, apiSpec spec.API) (*float64, erro } avgLatency := float64(values[0].Value) + + if math.IsNaN(avgLatency) { + return nil, nil + } return &avgLatency, nil } From 481e17e6f28875e829801d5da11fb7fd0ba249bc Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Thu, 27 May 2021 21:18:02 +0300 Subject: [PATCH 29/82] Fix build-and-push-test-images make cmd --- Makefile | 2 +- test/utils/build-and-push-images.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e8d59f851e..a5b6d7bdbe 100644 --- a/Makefile +++ b/Makefile @@ -173,7 +173,7 @@ test-python: # build test api images -# the DOCKER_PASSWORD and DOCKER_USERNAME vars to the quay repo are required +# make sure you login with your quay credentials build-and-push-test-images: @./test/utils/build-and-push-images.sh quay.io diff --git a/test/utils/build-and-push-images.sh b/test/utils/build-and-push-images.sh index ed06652899..dc3cd57e54 100755 --- a/test/utils/build-and-push-images.sh +++ b/test/utils/build-and-push-images.sh @@ -52,7 +52,6 @@ for image in "${api_images[@]}"; do done # push the images -echo "$DOCKER_PASSWORD" | docker login $host -u "$DOCKER_USERNAME" --password-stdin for image in "${api_images[@]}"; do blue_echo "Pushing $host/cortexlabs-test/$image:latest..." docker push $host/cortexlabs-test/${image} From ee712f36c40a970b15d5751dd3994f4b9e012caf Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Thu, 27 May 2021 16:32:55 -0700 Subject: [PATCH 30/82] Update docs (#2196) --- docs/clusters/observability/metrics.md | 77 -------------------------- docs/summary.md | 8 +-- docs/workloads/async/container.md | 34 ++++++++++++ docs/workloads/async/example.md | 2 +- docs/workloads/async/metrics.md | 27 --------- docs/workloads/async/webhooks.md | 54 ------------------ docs/workloads/batch/container.md | 36 ++++++++++++ docs/workloads/batch/metrics.md | 27 --------- docs/workloads/realtime/container.md | 46 +++++++++++++++ docs/workloads/realtime/metrics.md | 26 --------- docs/workloads/task/container.md | 11 ++++ docs/workloads/task/metrics.md | 27 --------- pkg/enqueuer/enqueuer.go | 2 +- pkg/workloads/k8s.go | 49 +++++++--------- 14 files changed, 153 insertions(+), 273 deletions(-) create mode 100644 docs/workloads/async/container.md delete mode 100644 docs/workloads/async/metrics.md delete mode 100644 docs/workloads/async/webhooks.md create mode 100644 docs/workloads/batch/container.md delete mode 100644 docs/workloads/batch/metrics.md create mode 100644 docs/workloads/realtime/container.md create mode 100644 docs/workloads/task/container.md delete mode 100644 docs/workloads/task/metrics.md diff --git a/docs/clusters/observability/metrics.md b/docs/clusters/observability/metrics.md index 55de3e85de..1bb3afe2df 100644 --- a/docs/clusters/observability/metrics.md +++ b/docs/clusters/observability/metrics.md @@ -69,80 +69,3 @@ the `Explore` menu in grafana and press the `Metrics` button. ![](https://user-images.githubusercontent.com/7456627/107377492-515f7000-6aeb-11eb-9b46-909120335060.png) You can use any of these metrics to set up your own dashboards. - -## Custom user metrics - -It is possible to export your own custom metrics by using the `MetricsClient` class in your handler code. This allows -you to create a custom metrics from your deployed API that can be later be used on your own custom dashboards. - -Code examples on how to use custom metrics for each API kind can be found here: - -- [RealtimeAPI](../../workloads/realtime/metrics.md#custom-user-metrics) -- [AsyncAPI](../../workloads/async/metrics.md#custom-user-metrics) -- [BatchAPI](../../workloads/batch/metrics.md#custom-user-metrics) -- [TaskAPI](../../workloads/task/metrics.md#custom-user-metrics) - -### Metric types - -Currently, we only support 3 different metric types that will be converted to its respective Prometheus type: - -- [Counter](https://prometheus.io/docs/concepts/metric_types/#counter) - a cumulative metric that represents a single - monotonically increasing counter whose value can only increase or be reset to zero on restart. -- [Gauge](https://prometheus.io/docs/concepts/metric_types/#gauge) - a single numerical value that can arbitrarily go up - and down. -- [Histogram](https://prometheus.io/docs/concepts/metric_types/#histogram) - samples observations (usually things like - request durations or response sizes) and counts them in configurable buckets. It also provides a sum of all observed - values. - -### Pushing metrics - -- Counter - - ```python - metrics.increment('my_counter', value=1, tags={"tag": "tag_name"}) - ``` - -- Gauge - - ```python - metrics.gauge('active_connections', value=1001, tags={"tag": "tag_name"}) - ``` - -- Histogram - - ```python - metrics.histogram('inference_time_milliseconds', 120, tags={"tag": "tag_name"}) - ``` - -### Metrics client class reference - -```python -class MetricsClient: - - def gauge(self, metric: str, value: float, tags: Dict[str, str] = None): - """ - Record the value of a gauge. - - Example: - >>> metrics.gauge('active_connections', 1001, tags={"protocol": "http"}) - """ - pass - - def increment(self, metric: str, value: float = 1, tags: Dict[str, str] = None): - """ - Increment the value of a counter. - - Example: - >>> metrics.increment('model_calls', 1, tags={"model_version": "v1"}) - """ - pass - - def histogram(self, metric: str, value: float, tags: Dict[str, str] = None): - """ - Set the value in a histogram metric - - Example: - >>> metrics.histogram('inference_time_milliseconds', 120, tags={"model_version": "v1"}) - """ - pass -``` diff --git a/docs/summary.md b/docs/summary.md index cd40165657..a407dda760 100644 --- a/docs/summary.md +++ b/docs/summary.md @@ -32,6 +32,7 @@ * [Realtime APIs](workloads/realtime/realtime-apis.md) * [Example](workloads/realtime/example.md) * [Configuration](workloads/realtime/configuration.md) + * [Container Interface](workloads/realtime/container.md) * [Autoscaling](workloads/realtime/autoscaling.md) * [Traffic Splitter](workloads/realtime/traffic-splitter.md) * [Metrics](workloads/realtime/metrics.md) @@ -40,20 +41,19 @@ * [Async APIs](workloads/async/async-apis.md) * [Example](workloads/async/example.md) * [Configuration](workloads/async/configuration.md) - * [Metrics](workloads/async/metrics.md) + * [Container Interface](workloads/async/container.md) * [Statuses](workloads/async/statuses.md) - * [Webhooks](workloads/async/webhooks.md) * [Batch APIs](workloads/batch/batch-apis.md) * [Example](workloads/batch/example.md) * [Configuration](workloads/batch/configuration.md) + * [Container Interface](workloads/batch/container.md) * [Jobs](workloads/batch/jobs.md) - * [Metrics](workloads/batch/metrics.md) * [Statuses](workloads/batch/statuses.md) * [Task APIs](workloads/task/task-apis.md) * [Example](workloads/task/example.md) * [Configuration](workloads/task/configuration.md) + * [Container Interface](workloads/task/container.md) * [Jobs](workloads/task/jobs.md) - * [Metrics](workloads/task/metrics.md) * [Statuses](workloads/task/statuses.md) ## Clients diff --git a/docs/workloads/async/container.md b/docs/workloads/async/container.md new file mode 100644 index 0000000000..58f9f21202 --- /dev/null +++ b/docs/workloads/async/container.md @@ -0,0 +1,34 @@ +# Container Interface + +## Handling requests + +In order to handle requests to your Async API, one of your containers must run a web server which is listening for HTTP requests on the port which is configured in the `pod.port` field of your [API configuration](configuration.md) (default: 8080). + +Requests will be sent to your web server via HTTP POST requests to the root path (`/`) as they are pulled off of the queue. The payload and the content type header of the HTTP request to your web server will match those of the original request to your Async API. In addition, the request's ID will be passed in via the "X-Cortex-Request-ID" header. + +Your web server must respond with valid JSON (with the `Content-Type` header set to "application/json"). The response will remain queryable for 7 days. + +## Readiness checks + +It is often important to implement a readiness check for your API. By default, as soon as your web server has bound to the port, it will start receiving traffic. In some cases, the web server may start listening on the port before its workers are ready to handle traffic (e.g. `tiangolo/uvicorn-gunicorn-fastapi` behaves this way). Readiness checks ensure that traffic is not sent into your web server before it's ready to handle them. + +There are two types of readiness checks which are supported: `http_get` and `tcp_socket` (see [API configuration](configuration.md) for usage instructions). A simple and often effective approach is to add a route to your web server (e.g. `/healthz`) which responds with status code 200, and configure your readiness probe accordingly: + +```yaml +readiness_probe: + http_get: + port: 8080 + path: /healthz +``` + +## Multiple containers + +Your API pod can contain multiple containers, only one of which can be listening for requests on the target port (it can be any of the containers). + +The `/mnt` directory is mounted to each container's file system, and is shared across all containers. + +## Using the Cortex CLI or client + +It is possible to use the Cortex CLI or client to interact with your cluster's APIs from within your API containers. All containers will have a CLI configuration file present at `/cortex/client/cli.yaml`, which is configured to connect to the cluster. In addition, the `CORTEX_CLI_CONFIG_DIR` environment variable is set to `/cortex/client` by default. Therefore, no additional configuration is required to use the CLI or Python client (which can be instantiated via `cortex.client()`). + +Note: your Cortex CLI or client must match the version of your cluster (available in the `CORTEX_VERSION` environment variable). diff --git a/docs/workloads/async/example.md b/docs/workloads/async/example.md index efe94570b3..9734cfc071 100644 --- a/docs/workloads/async/example.md +++ b/docs/workloads/async/example.md @@ -148,7 +148,7 @@ are `in_queue | in_progress | failed | completed`. The `result` and `timestamp` is `completed`. The result will remain queryable for 7 days after the request was completed. It is also possible to setup a webhook in your handler to get the response sent to a pre-defined web server once the -workload completes or fails. You can read more about it in the [webhook documentation](./webhooks.md). +workload completes or fails. ## Stream logs diff --git a/docs/workloads/async/metrics.md b/docs/workloads/async/metrics.md deleted file mode 100644 index fdebca7fe3..0000000000 --- a/docs/workloads/async/metrics.md +++ /dev/null @@ -1,27 +0,0 @@ -# Metrics - -## Custom user metrics - -It is possible to export custom user metrics by adding the `metrics_client` -argument to the workload handler constructor. - -```python -class Handler: - def __init__(self, config, metrics_client): - self.metrics = metrics_client - - def handle_async(self, payload): - # --- my workload code here --- - result = ... - - # increment a counter with name "my_metric" and tags model:v1 - self.metrics.increment(metric="my_counter", value=1, tags={"model": "v1"}) - - # set the value for a gauge with name "my_gauge" and tags model:v1 - self.metrics.gauge(metric="my_gauge", value=42, tags={"model": "v1"}) - - # set the value for an histogram with name "my_histogram" and tags model:v1 - self.metrics.histogram(metric="my_histogram", value=100, tags={"model": "v1"}) -``` - -**Note**: The metrics client uses the UDP protocol to push metrics, so if it fails during a metrics push, no exception is thrown. diff --git a/docs/workloads/async/webhooks.md b/docs/workloads/async/webhooks.md deleted file mode 100644 index f822725a7f..0000000000 --- a/docs/workloads/async/webhooks.md +++ /dev/null @@ -1,54 +0,0 @@ -# Webhooks - -Polling for requests can be resource intensive, and does not guarantee that you will have the result as soon as it is ready. In -order to overcome this problem, we can use webhooks. - -A webhook is a request that is sent to a URL known in advance when an event occurs. In our case, the event is a workload -completion or failure, and the URL known in advance is some other service that we already have running. - -## Example - -Below is an example implementing webhooks for an `AsyncAPI` workload using FastAPI. - -```python -import os -import time -import requests -from datetime import datetime -from fastapi import FastAPI, Header - -STATUS_COMPLETED = "completed" -STATUS_FAILED = "failed" - -webhook_url = os.getenv("WEBHOOK_URL") # the webhook url is set as an environment variable - -app = FastAPI() - - -@app.post("/") -async def handle(x_cortex_request_id=Header(None)): - try: - time.sleep(60) # simulates a long workload - send_report(x_cortex_request_id, STATUS_COMPLETED, result={"data": "hello"}) - except Exception as err: - send_report(x_cortex_request_id, STATUS_FAILED) - raise err # the original exception should still be raised - - -def send_report(request_id, status, result=None): - response = {"id": request_id, "status": status} - - if result is not None and status == STATUS_COMPLETED: - timestamp = datetime.utcnow().isoformat() - response.update({"result": result, "timestamp": timestamp}) - - try: - requests.post(url=webhook_url, json=response) - except Exception: - pass -``` - -## Development - -For development purposes, you can use a utility website such as https://webhook.site/ to validate that your webhook -setup is working as intended. diff --git a/docs/workloads/batch/container.md b/docs/workloads/batch/container.md new file mode 100644 index 0000000000..368e54ec9f --- /dev/null +++ b/docs/workloads/batch/container.md @@ -0,0 +1,36 @@ +# Container Interface + +## Handling requests + +In order to receive batches in your Batch API, one of your containers must run a web server which is listening for HTTP requests on the port which is configured in the `pod.port` field of your [API configuration](configuration.md) (default: 8080). + +Batches will be sent to your web server via HTTP POST requests to the root path (`/`). The payload will be a JSON-encoded array representing one batch, and the `Content-Type` header will be set to "application/json". In addition, the job's ID will be passed in via the "X-Cortex-Job-ID" header. + +Your web server must respond with status code 200 for the batch to be marked as succeeded (the response body will be ignored). + +Once all batches have been processed, one of your workers will receive an HTTP POST request to `/on-job-complete`. It is not necessary for your web server to handle requests to `/on-job-complete` (404 errors will be ignored). + +## Readiness checks + +It is often important to implement a readiness check for your API. By default, as soon as your web server has bound to the port, it will start receiving batches. In some cases, the web server may start listening on the port before its workers are ready to handle traffic (e.g. `tiangolo/uvicorn-gunicorn-fastapi` behaves this way). Readiness checks ensure that traffic is not sent into your web server before it's ready to handle them. + +There are two types of readiness checks which are supported: `http_get` and `tcp_socket` (see [API configuration](configuration.md) for usage instructions). A simple and often effective approach is to add a route to your web server (e.g. `/healthz`) which responds with status code 200, and configure your readiness probe accordingly: + +```yaml +readiness_probe: + http_get: + port: 8080 + path: /healthz +``` + +## Multiple containers + +Your API pod can contain multiple containers, only one of which can be listening for requests on the target port (it can be any of the containers). + +The `/mnt` directory is mounted to each container's file system, and is shared across all containers. + +## Using the Cortex CLI or client + +It is possible to use the Cortex CLI or client to interact with your cluster's APIs from within your API containers. All containers will have a CLI configuration file present at `/cortex/client/cli.yaml`, which is configured to connect to the cluster. In addition, the `CORTEX_CLI_CONFIG_DIR` environment variable is set to `/cortex/client` by default. Therefore, no additional configuration is required to use the CLI or Python client (which can be instantiated via `cortex.client()`). + +Note: your Cortex CLI or client must match the version of your cluster (available in the `CORTEX_VERSION` environment variable). diff --git a/docs/workloads/batch/metrics.md b/docs/workloads/batch/metrics.md deleted file mode 100644 index 50b0297d10..0000000000 --- a/docs/workloads/batch/metrics.md +++ /dev/null @@ -1,27 +0,0 @@ -# Metrics - -## Custom user metrics - -It is possible to export custom user metrics by adding the `metrics_client` -argument to the Handler class constructor. Below there is an example of how to use the metrics client. The implementation is similar to all handler types. - -```python -class Handler: - def __init__(self, config, metrics_client): - self.metrics = metrics_client - - def handle_batch(self, payload): - # --- my handler code here --- - result = ... - - # increment a counter with name "my_metric" and tags model:v1 - self.metrics.increment(metric="my_counter", value=1, tags={"model": "v1"}) - - # set the value for a gauge with name "my_gauge" and tags model:v1 - self.metrics.gauge(metric="my_gauge", value=42, tags={"model": "v1"}) - - # set the value for an histogram with name "my_histogram" and tags model:v1 - self.metrics.histogram(metric="my_histogram", value=100, tags={"model": "v1"}) -``` - -**Note**: The metrics client uses the UDP protocol to push metrics, so if it fails during a metrics push, no exception is thrown. diff --git a/docs/workloads/realtime/container.md b/docs/workloads/realtime/container.md new file mode 100644 index 0000000000..6a6c605d88 --- /dev/null +++ b/docs/workloads/realtime/container.md @@ -0,0 +1,46 @@ +# Container Interface + +## Handling requests + +In order to handle requests to your Realtime API, one of your containers must run a web server which is listening for HTTP requests on the port which is configured in the `pod.port` field of your [API configuration](configuration.md) (default: 8080). + +Subpaths are supported; for example, if your API is named `my-api`, a request to `/my-api` will be routed to the root (`/`) of your web server, and a request to `/my-api/subpatch` will be routed to `/subpath` on your web server. + +## Readiness checks + +It is often important to implement a readiness check for your API. By default, as soon as your web server has bound to the port, it will start receiving traffic. In some cases, the web server may start listening on the port before its workers are ready to handle traffic (e.g. `tiangolo/uvicorn-gunicorn-fastapi` behaves this way). Readiness checks ensure that traffic is not sent into your web server before it's ready to handle them. + +There are three types of readiness checks which are supported: `http_get`, `tcp_socket`, and `exec` (see [API configuration](configuration.md) for usage instructions). A simple and often effective approach is to add a route to your web server (e.g. `/healthz`) which responds with status code 200, and configure your readiness probe accordingly: + +```yaml +readiness_probe: + http_get: + port: 8080 + path: /healthz +``` + +## Multiple containers + +Your API pod can contain multiple containers, only one of which can be listening for requests on the target port (it can be any of the containers). + +The `/mnt` directory is mounted to each container's file system, and is shared across all containers. + +## Using the Cortex CLI or client + +It is possible to use the Cortex CLI or client to interact with your cluster's APIs from within your API containers. All containers will have a CLI configuration file present at `/cortex/client/cli.yaml`, which is configured to connect to the cluster. In addition, the `CORTEX_CLI_CONFIG_DIR` environment variable is set to `/cortex/client` by default. Therefore, no additional configuration is required to use the CLI or Python client (which can be instantiated via `cortex.client()`). + +Note: your Cortex CLI or client must match the version of your cluster (available in the `CORTEX_VERSION` environment variable). + +## Chaining APIs + +It is possible to make requests from any Cortex API type to a Realtime API within a Cortex cluster. All running APIs are accessible at `http://api-:8888/`, where `` is the name of the API you are making a request to. + +For example, if there is a Realtime api named `my-api` running in the cluster, you could make a request to it from a different API by using: + +```python +import requests + +response = requests.post("http://api-my-api:8888/", json={"text": "hello world"}) +``` + +Note that if the API making the request is a Realtime API or Async API, its autoscaling configuration (i.e. `target_in_flight`) should be modified with the understanding that requests will be considered "in-flight" in the first API as the request is being fulfilled by the second API. diff --git a/docs/workloads/realtime/metrics.md b/docs/workloads/realtime/metrics.md index 9343f03925..a38aaee989 100644 --- a/docs/workloads/realtime/metrics.md +++ b/docs/workloads/realtime/metrics.md @@ -32,29 +32,3 @@ The `cortex get API_NAME` command also provides a link to a Grafana dashboard: | p90 Latency | 90th percentile latency, computed over a minute, for an API | Value might not be accurate because the histogram buckets are not dynamically set. | | p50 Latency | 50th percentile latency, computed over a minute, for an API | Value might not be accurate because the histogram buckets are not dynamically set. | | Average Latency | Average latency, computed over a minute, for an API | | - -## Custom user metrics - -It is possible to export custom user metrics by adding the `metrics_client` -argument to the handler constructor. Below there is an example of how to use the metrics client. The implementation is similar to all handler types. - -```python -class Handler: - def __init__(self, config, metrics_client): - self.metrics = metrics_client - - def handle_post(self, payload): - # --- my handler code here --- - result = ... - - # increment a counter with name "my_metric" and tags model:v1 - self.metrics.increment(metric="my_counter", value=1, tags={"model": "v1"}) - - # set the value for a gauge with name "my_gauge" and tags model:v1 - self.metrics.gauge(metric="my_gauge", value=42, tags={"model": "v1"}) - - # set the value for an histogram with name "my_histogram" and tags model:v1 - self.metrics.histogram(metric="my_histogram", value=100, tags={"model": "v1"}) -``` - -**Note**: The metrics client uses the UDP protocol to push metrics, so if it fails during a metrics push, no exception is thrown. diff --git a/docs/workloads/task/container.md b/docs/workloads/task/container.md new file mode 100644 index 0000000000..05bcc3753a --- /dev/null +++ b/docs/workloads/task/container.md @@ -0,0 +1,11 @@ +# Container Interface + +## Multiple containers + +Your Task's pod can contain multiple containers. The `/mnt` directory is mounted to each container's file system, and is shared across all containers. + +## Using the Cortex CLI or client + +It is possible to use the Cortex CLI or client to interact with your cluster's APIs from within your API containers. All containers will have a CLI configuration file present at `/cortex/client/cli.yaml`, which is configured to connect to the cluster. In addition, the `CORTEX_CLI_CONFIG_DIR` environment variable is set to `/cortex/client` by default. Therefore, no additional configuration is required to use the CLI or Python client (which can be instantiated via `cortex.client()`). + +Note: your Cortex CLI or client must match the version of your cluster (available in the `CORTEX_VERSION` environment variable). diff --git a/docs/workloads/task/metrics.md b/docs/workloads/task/metrics.md deleted file mode 100644 index aab2c3c742..0000000000 --- a/docs/workloads/task/metrics.md +++ /dev/null @@ -1,27 +0,0 @@ -## Custom user metrics - -It is possible to export custom user metrics by adding the `metrics_client` -argument to the task definition constructor. Below there is an example of how to use the metrics client. - -Currently, it is only possible to instantiate the metrics client from a class task definition. - -```python -class Task: - def __init__(self, metrics_client): - self.metrics = metrics_client - - def __call__(self, config): - # --- my task code here --- - ... - - # increment a counter with name "my_metric" and tags model:v1 - self.metrics.increment(metric="my_counter", value=1, tags={"model": "v1"}) - - # set the value for a gauge with name "my_gauge" and tags model:v1 - self.metrics.gauge(metric="my_gauge", value=42, tags={"model": "v1"}) - - # set the value for an histogram with name "my_histogram" and tags model:v1 - self.metrics.histogram(metric="my_histogram", value=100, tags={"model": "v1"}) -``` - -**Note**: The metrics client uses the UDP protocol to push metrics, so if it fails during a metrics push, no exception is thrown. diff --git a/pkg/enqueuer/enqueuer.go b/pkg/enqueuer/enqueuer.go index 06c1664c91..2286a22eb4 100644 --- a/pkg/enqueuer/enqueuer.go +++ b/pkg/enqueuer/enqueuer.go @@ -48,7 +48,7 @@ type EnvConfig struct { JobID string } -// FIXME: all these types should be shared with the cortex webserver (from where the payload is submitted) +// FIXME: all these types should be shared with the cortex web server (from where the payload is submitted) type ItemList struct { Items []json.RawMessage `json:"items"` diff --git a/pkg/workloads/k8s.go b/pkg/workloads/k8s.go index 16396c4644..1777ffc1d4 100644 --- a/pkg/workloads/k8s.go +++ b/pkg/workloads/k8s.go @@ -87,12 +87,7 @@ func AsyncGatewayContainer(api spec.API, queueURL string, volumeMounts []kcore.V Ports: []kcore.ContainerPort{ {ContainerPort: consts.ProxyListeningPortInt32}, }, - Env: []kcore.EnvVar{ - { - Name: "CORTEX_LOG_LEVEL", - Value: strings.ToUpper(userconfig.InfoLogLevel.String()), - }, - }, + Env: baseEnvVars, Resources: kcore.ResourceRequirements{ Requests: kcore.ResourceList{ kcore.ResourceCPU: _asyncGatewayCPURequest, @@ -142,12 +137,7 @@ func RealtimeProxyContainer(api spec.API) (kcore.Container, kcore.Volume) { {Name: "admin", ContainerPort: consts.AdminPortInt32}, {ContainerPort: consts.ProxyListeningPortInt32}, }, - Env: []kcore.EnvVar{ - { - Name: "CORTEX_LOG_LEVEL", - Value: strings.ToUpper(userconfig.InfoLogLevel.String()), - }, - }, + Env: baseEnvVars, EnvFrom: baseClusterEnvVars(), VolumeMounts: []kcore.VolumeMount{ ClusterConfigMount(), @@ -302,13 +292,20 @@ func userPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { } } - containerEnvVars := []kcore.EnvVar{} + containerEnvVars := baseEnvVars + + containerEnvVars = append(containerEnvVars, kcore.EnvVar{ + Name: "CORTEX_CLI_CONFIG_DIR", + Value: _clientConfigDir, + }) + if api.Kind != userconfig.TaskAPIKind { containerEnvVars = append(containerEnvVars, kcore.EnvVar{ Name: "CORTEX_PORT", Value: s.Int32(*api.Pod.Port), }) } + for k, v := range container.Env { containerEnvVars = append(containerEnvVars, kcore.EnvVar{ Name: k, @@ -435,19 +432,13 @@ func GenerateNodeAffinities(apiNodeGroups []string) *kcore.Affinity { } } -// func getAsyncAPIEnvVars(api spec.API, queueURL string) []kcore.EnvVar { -// envVars := apiContainerEnvVars(&api) - -// envVars = append(envVars, -// kcore.EnvVar{ -// Name: "CORTEX_QUEUE_URL", -// Value: queueURL, -// }, -// kcore.EnvVar{ -// Name: "CORTEX_ASYNC_WORKLOAD_PATH", -// Value: aws.S3Path(config.ClusterConfig.Bucket, fmt.Sprintf("%s/workloads/%s", config.ClusterConfig.ClusterUID, api.Name)), -// }, -// ) - -// return envVars -// } +var baseEnvVars = []kcore.EnvVar{ + { + Name: "CORTEX_VERSION", + Value: consts.CortexVersion, + }, + { + Name: "CORTEX_LOG_LEVEL", + Value: strings.ToUpper(userconfig.InfoLogLevel.String()), + }, +} From 86620a7aa609d50421231aaaf50c58c9d49eadd5 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Thu, 27 May 2021 16:36:43 -0700 Subject: [PATCH 31/82] Update lint.sh --- build/lint.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build/lint.sh b/build/lint.sh index 2093ef6f4e..7690181a04 100755 --- a/build/lint.sh +++ b/build/lint.sh @@ -84,6 +84,7 @@ output=$(cd "$ROOT" && find . -type f \ ! -path "**/.vscode/*" \ ! -path "**/.idea/*" \ ! -path "**/.history/*" \ +! -path "**/testbin/*" \ ! -path "**/__pycache__/*" \ ! -path "**/.pytest_cache/*" \ ! -path "**/*.egg-info/*" \ @@ -118,6 +119,7 @@ if [ "$is_release_branch" = "true" ]; then ! -path "**/.vscode/*" \ ! -path "**/.idea/*" \ ! -path "**/.history/*" \ + ! -path "**/testbin/*" \ ! -path "**/__pycache__/*" \ ! -path "**/.pytest_cache/*" \ ! -path "**/*.egg-info/*" \ @@ -141,6 +143,7 @@ output=$(cd "$ROOT" && find . -type f \ ! -path "**/.idea/*" \ ! -path "**/.history/*" \ ! -path "**/.vscode/*" \ +! -path "**/testbin/*" \ ! -path "**/__pycache__/*" \ ! -path "**/.pytest_cache/*" \ ! -path "**/*.egg-info/*" \ @@ -164,6 +167,7 @@ output=$(cd "$ROOT" && find . -type f \ ! -path "**/.idea/*" \ ! -path "**/.history/*" \ ! -path "**/.vscode/*" \ +! -path "**/testbin/*" \ ! -path "**/__pycache__/*" \ ! -path "**/.pytest_cache/*" \ ! -path "**/*.egg-info/*" \ @@ -188,6 +192,7 @@ output=$(cd "$ROOT" && find . -type f \ ! -path "**/.vscode/*" \ ! -path "**/.idea/*" \ ! -path "**/.history/*" \ +! -path "**/testbin/*" \ ! -path "**/__pycache__/*" \ ! -path "**/.pytest_cache/*" \ ! -path "**/*.egg-info/*" \ @@ -210,6 +215,7 @@ output=$(cd "$ROOT" && find . -type f \ ! -path "**/.idea/*" \ ! -path "**/.history/*" \ ! -path "**/.vscode/*" \ +! -path "**/testbin/*" \ ! -path "**/__pycache__/*" \ ! -path "**/.pytest_cache/*" \ ! -path "**/*.egg-info/*" \ From a51f94736ccf0e6a177d8911c2e0c99ac84a65f5 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Fri, 28 May 2021 10:22:40 +0100 Subject: [PATCH 32/82] Dequeuer proxy for Async and Batch APIs (#2181) --- .circleci/config.yml | 12 +- build/images.sh | 1 + cmd/dequeuer/main.go | 209 +++++++++++++++++ go.mod | 7 +- go.sum | 32 ++- images/dequeuer/Dockerfile | 28 +++ pkg/async-gateway/endpoint.go | 3 +- pkg/async-gateway/service.go | 54 +++-- pkg/async-gateway/types.go | 33 +-- pkg/config/config.go | 6 +- pkg/dequeuer/async_handler.go | 206 +++++++++++++++++ pkg/dequeuer/async_handler_test.go | 121 ++++++++++ pkg/dequeuer/batch_handler.go | 230 +++++++++++++++++++ pkg/dequeuer/batch_handler_test.go | 106 +++++++++ pkg/dequeuer/dequeuer.go | 212 +++++++++++++++++ pkg/dequeuer/dequeuer_test.go | 357 +++++++++++++++++++++++++++++ pkg/dequeuer/errors.go | 62 +++++ pkg/dequeuer/message_handler.go | 35 +++ pkg/dequeuer/queue_attributes.go | 74 ++++++ pkg/lib/aws/aws.go | 15 +- pkg/lib/configreader/reader.go | 6 +- pkg/types/async/s3_paths.go | 41 ++++ pkg/types/async/status.go | 42 ++++ 23 files changed, 1812 insertions(+), 80 deletions(-) create mode 100644 cmd/dequeuer/main.go create mode 100644 images/dequeuer/Dockerfile create mode 100644 pkg/dequeuer/async_handler.go create mode 100644 pkg/dequeuer/async_handler_test.go create mode 100644 pkg/dequeuer/batch_handler.go create mode 100644 pkg/dequeuer/batch_handler_test.go create mode 100644 pkg/dequeuer/dequeuer.go create mode 100644 pkg/dequeuer/dequeuer_test.go create mode 100644 pkg/dequeuer/errors.go create mode 100644 pkg/dequeuer/message_handler.go create mode 100644 pkg/dequeuer/queue_attributes.go create mode 100644 pkg/types/async/s3_paths.go create mode 100644 pkg/types/async/status.go diff --git a/.circleci/config.yml b/.circleci/config.yml index 748ba82dea..c541a0054f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,8 +20,9 @@ commands: - run: name: Install Go command: | - wget https://dl.google.com/go/go1.14.7.linux-amd64.tar.gz - sudo tar -C /usr/local -xzf go1.14.7.linux-amd64.tar.gz + sudo rm -rf /usr/local/go + wget https://dl.google.com/go/go1.15.12.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf go1.15.12.linux-amd64.tar.gz rm -rf go*.tar.gz echo 'export PATH=$PATH:/usr/local/go/bin' >> $BASH_ENV echo 'export PATH=$PATH:~/go/bin' >> $BASH_ENV @@ -75,18 +76,17 @@ commands: jobs: test: - docker: - - image: circleci/python:3.6 + machine: + image: ubuntu-1604:202104-01 # machine executor necessary to run go integration tests steps: - checkout - - setup_remote_docker - install-go - run: name: Install Linting Tools command: | go get -u -v golang.org/x/lint/golint go get -u -v github.com/kyoh86/looppointer/cmd/looppointer - sudo pip install black aiohttp + pip3 install black aiohttp - run: name: Initialize Credentials command: | diff --git a/build/images.sh b/build/images.sh index b3c2d57551..67257a750e 100644 --- a/build/images.sh +++ b/build/images.sh @@ -24,6 +24,7 @@ dev_images=( "proxy" "async-gateway" "enqueuer" + "dequeuer" ) non_dev_images=( diff --git a/cmd/dequeuer/main.go b/cmd/dequeuer/main.go new file mode 100644 index 0000000000..124f0f465d --- /dev/null +++ b/cmd/dequeuer/main.go @@ -0,0 +1,209 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "os" + "os/signal" + "strconv" + + "github.com/DataDog/datadog-go/statsd" + "github.com/cortexlabs/cortex/pkg/consts" + "github.com/cortexlabs/cortex/pkg/dequeuer" + awslib "github.com/cortexlabs/cortex/pkg/lib/aws" + "github.com/cortexlabs/cortex/pkg/lib/errors" + "github.com/cortexlabs/cortex/pkg/lib/logging" + "github.com/cortexlabs/cortex/pkg/lib/telemetry" + "github.com/cortexlabs/cortex/pkg/types/clusterconfig" + "github.com/cortexlabs/cortex/pkg/types/userconfig" + "go.uber.org/zap" +) + +func main() { + var ( + clusterConfigPath string + clusterUID string + queueURL string + userContainerPort int + apiName string + jobID string + statsdPort int + apiKind string + ) + flag.StringVar(&clusterConfigPath, "cluster-config", "", "cluster config path") + flag.StringVar(&clusterUID, "cluster-uid", "", "cluster unique identifier") + flag.StringVar(&queueURL, "queue", "", "target queue URL from which the api messages will be dequeued") + flag.StringVar(&apiKind, "api-kind", "", fmt.Sprintf("api kind (%s|%s)", userconfig.BatchAPIKind.String(), userconfig.AsyncAPIKind.String())) + flag.StringVar(&apiName, "api-name", "", "api name") + flag.StringVar(&jobID, "job-id", "", "job ID") + flag.IntVar(&userContainerPort, "user-port", 8080, "target port to which the dequeued messages will be sent to") + flag.IntVar(&statsdPort, "statsd-port", 9125, "port for to send udp statsd metrics") + + flag.Parse() + + version := os.Getenv("CORTEX_VERSION") + if version == "" { + version = consts.CortexVersion + } + + hostIP := os.Getenv("HOST_IP") + + log := logging.GetLogger() + defer func() { + _ = log.Sync() + }() + + switch { + case clusterConfigPath == "": + log.Fatal("--cluster-config is a required option") + case queueURL == "": + log.Fatal("--queue is a required option") + case apiName == "": + log.Fatal("--api-name is a required option") + case apiKind == "": + log.Fatal("--api-kind is a required option") + } + + targetURL := "http://127.0.0.1:" + strconv.Itoa(userContainerPort) + + clusterConfig, err := clusterconfig.NewForFile(clusterConfigPath) + if err != nil { + exit(log, err) + } + + awsClient, err := awslib.NewForRegion(clusterConfig.Region) + if err != nil { + exit(log, err, "failed to create aws client") + } + + _, userID, err := awsClient.CheckCredentials() + if err != nil { + exit(log, err) + } + + err = telemetry.Init(telemetry.Config{ + Enabled: clusterConfig.Telemetry, + UserID: userID, + Properties: map[string]string{ + "kind": apiKind, + "image_type": "dequeuer", + }, + Environment: "api", + LogErrors: true, + BackoffMode: telemetry.BackoffDuplicateMessages, + }) + if err != nil { + log.Fatalw("failed to initialize telemetry", "error", err) + } + defer telemetry.Close() + + metricsClient, err := statsd.New(fmt.Sprintf("%s:%d", hostIP, statsdPort)) + if err != nil { + exit(log, err, "unable to initialize metrics client") + } + + var dequeuerConfig dequeuer.SQSDequeuerConfig + var messageHandler dequeuer.MessageHandler + + switch apiKind { + case userconfig.BatchAPIKind.String(): + if jobID == "" { + log.Fatal("--job-id is a required option") + } + + config := dequeuer.BatchMessageHandlerConfig{ + Region: clusterConfig.Region, + APIName: apiName, + JobID: jobID, + QueueURL: queueURL, + TargetURL: targetURL, + } + + messageHandler = dequeuer.NewBatchMessageHandler(config, awsClient, metricsClient, log) + dequeuerConfig = dequeuer.SQSDequeuerConfig{ + Region: clusterConfig.Region, + QueueURL: queueURL, + StopIfNoMessages: true, + } + + case userconfig.AsyncAPIKind.String(): + if clusterUID == "" { + log.Fatal("--cluster-uid is a required option") + } + + config := dequeuer.AsyncMessageHandlerConfig{ + ClusterUID: clusterUID, + Bucket: clusterConfig.Bucket, + APIName: apiName, + TargetURL: targetURL, + } + + messageHandler = dequeuer.NewAsyncMessageHandler(config, awsClient, log) + dequeuerConfig = dequeuer.SQSDequeuerConfig{ + Region: clusterConfig.Region, + QueueURL: queueURL, + StopIfNoMessages: false, + } + default: + exit(log, err, fmt.Sprintf("kind %s is not supported", apiKind)) + } + + sigint := make(chan os.Signal, 1) + signal.Notify(sigint, os.Interrupt) + + sqsDequeuer, err := dequeuer.NewSQSDequeuer(dequeuerConfig, awsClient, log) + if err != nil { + exit(log, err, "failed to create sqs dequeuer") + } + + errCh := make(chan error) + go func() { + log.Info("Starting dequeuer...") + errCh <- sqsDequeuer.Start(messageHandler) + }() + + select { + case err = <-errCh: + exit(log, err, "error during message dequeueing") + case <-sigint: + log.Info("Received TERM signal, handling a graceful shutdown...") + sqsDequeuer.Shutdown() + log.Info("Shutdown complete, exiting...") + } +} + +func exit(log *zap.SugaredLogger, err error, wrapStrs ...string) { + if err == nil { + os.Exit(0) + } + + for _, str := range wrapStrs { + err = errors.Wrap(err, str) + } + + if !errors.IsNoTelemetry(err) { + telemetry.Error(err) + } + + if !errors.IsNoPrint(err) { + log.Error(err) + } + + os.Exit(1) +} diff --git a/go.mod b/go.mod index 80fe04706c..f24687d9f9 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.15 require ( cloud.google.com/go v0.73.0 // indirect - github.com/Microsoft/go-winio v0.4.11 // indirect + github.com/DataDog/datadog-go v4.7.0+incompatible github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/aws/aws-sdk-go v1.36.2 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect @@ -16,7 +16,6 @@ require ( github.com/denormal/go-gitignore v0.0.0-20180930084346-ae8ad1d07817 github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0 - github.com/docker/go-connections v0.4.0 // indirect github.com/fatih/color v1.10.0 github.com/getsentry/sentry-go v0.10.0 github.com/go-logr/logr v0.3.0 @@ -31,7 +30,7 @@ require ( github.com/onsi/ginkgo v1.14.1 github.com/onsi/gomega v1.10.2 github.com/opencontainers/go-digest v1.0.0 - github.com/opencontainers/image-spec v1.0.1 // indirect + github.com/ory/dockertest/v3 v3.6.5 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.7.1 @@ -44,7 +43,7 @@ require ( github.com/stretchr/testify v1.6.1 github.com/ugorji/go/codec v1.2.1 github.com/xlab/treeprint v1.0.0 - github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect + github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c go.uber.org/atomic v1.6.0 go.uber.org/zap v1.15.0 golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect diff --git a/go.sum b/go.sum index 9cba6915bb..69e402fc18 100644 --- a/go.sum +++ b/go.sum @@ -61,10 +61,14 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= +github.com/DataDog/datadog-go v4.7.0+incompatible h1:setZNZoivEjeG87iK0abKZ9XHwHV6z63eAHhwmSzFes= +github.com/DataDog/datadog-go v4.7.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= -github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q= -github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -99,6 +103,8 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc= +github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -114,6 +120,8 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/containerd/containerd v1.4.3 h1:ijQT13JedHSHrQGWFcGEwzcNKrAGIiZ+jSD5QQG07SY= github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6 h1:NmTXa/uVnDyp0TY5MKi197+3HWcnYWfnHGyaFthlnGw= +github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -132,6 +140,8 @@ github.com/cortexlabs/yaml v0.0.0-20200511220111-581aea36a2e4/go.mod h1:nuzR4zMP github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -163,8 +173,6 @@ github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7fo github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/proto v1.9.0 h1:l0QiNT6Qs7Yj0Mb4X6dnWBQer4ebei2BFcgQLbGqUDc= -github.com/emicklei/proto v1.9.0/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -420,6 +428,7 @@ github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -431,6 +440,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2 h1:hRGSmZu7j271trc9sneMrpOW7GN5ngLm8YUZIPzf394= +github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -468,6 +479,8 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -502,10 +515,15 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v1.0.0-rc9 h1:/k06BMULKF5hidyoZymkoDCzdJzltZpz/UU4LguQVtc= +github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/ory/dockertest/v3 v3.6.5 h1:mhNKFeVEHuvaYW+/u+59mLzM/6XXGjpaet/yApgv+yc= +github.com/ory/dockertest/v3 v3.6.5/go.mod h1:iYKQSRlYrt/2s5fJWYdB98kCQG6g/LjBMvzEYii63vg= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= @@ -558,6 +576,7 @@ github.com/shirou/gopsutil v3.20.11+incompatible h1:LJr4ZQK4mPpIV5gOa4jCOKOGb4ty github.com/shirou/gopsutil v3.20.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= @@ -586,6 +605,7 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -734,6 +754,7 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -824,6 +845,7 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1031,6 +1053,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -1041,6 +1064,7 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclp gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/images/dequeuer/Dockerfile b/images/dequeuer/Dockerfile new file mode 100644 index 0000000000..7e4a59cee5 --- /dev/null +++ b/images/dequeuer/Dockerfile @@ -0,0 +1,28 @@ +# Build the manager binary +FROM golang:1.15 as builder + +# Copy the Go Modules manifests +COPY go.mod go.sum /workspace/ +WORKDIR /workspace +RUN go mod download + +COPY pkg/config pkg/config +COPY pkg/consts pkg/consts +COPY pkg/lib pkg/lib +COPY pkg/dequeuer pkg/dequeuer +COPY pkg/types pkg/types +COPY pkg/crds pkg/crds +COPY pkg/workloads pkg/workloads +COPY cmd/dequeuer cmd/dequeuer + +# Build +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o dequeuer ./cmd/dequeuer + +# Use distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM gcr.io/distroless/static:nonroot +WORKDIR / +COPY --from=builder /workspace/dequeuer . +USER nonroot:nonroot + +ENTRYPOINT ["/dequeuer"] diff --git a/pkg/async-gateway/endpoint.go b/pkg/async-gateway/endpoint.go index 736dfb8bfa..4a92c086e6 100644 --- a/pkg/async-gateway/endpoint.go +++ b/pkg/async-gateway/endpoint.go @@ -23,6 +23,7 @@ import ( "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/telemetry" + "github.com/cortexlabs/cortex/pkg/types/async" "github.com/gorilla/mux" "go.uber.org/zap" ) @@ -92,7 +93,7 @@ func (e *Endpoint) GetWorkload(w http.ResponseWriter, r *http.Request) { logErrorWithTelemetry(log, errors.Wrap(err, "failed to get workload")) return } - if res.Status == StatusNotFound { + if res.Status == async.StatusNotFound { respondPlainText(w, http.StatusNotFound, fmt.Sprintf("error: id %s not found", res.ID)) logErrorWithTelemetry(log, errors.ErrorUnexpected(fmt.Sprintf("error: id %s not found", res.ID))) return diff --git a/pkg/async-gateway/service.go b/pkg/async-gateway/service.go index 4e0a25914b..bba7a611e9 100644 --- a/pkg/async-gateway/service.go +++ b/pkg/async-gateway/service.go @@ -22,6 +22,7 @@ import ( "io" "strings" + "github.com/cortexlabs/cortex/pkg/types/async" "go.uber.org/zap" ) @@ -52,10 +53,10 @@ func NewService(clusterUID, apiName string, queue Queue, storage Storage, logger // CreateWorkload enqueues an async workload request and uploads the request payload to S3 func (s *service) CreateWorkload(id string, payload io.Reader, contentType string) (string, error) { - prefix := s.workloadStoragePrefix() + prefix := async.StoragePath(s.clusterUID, s.apiName) log := s.logger.With(zap.String("id", id), zap.String("contentType", contentType)) - payloadPath := fmt.Sprintf("%s/%s/payload", prefix, id) + payloadPath := async.PayloadPath(prefix, id) log.Debug("uploading payload", zap.String("path", payloadPath)) if err := s.storage.Upload(payloadPath, payload, contentType); err != nil { return "", err @@ -66,8 +67,8 @@ func (s *service) CreateWorkload(id string, payload io.Reader, contentType strin return "", err } - statusPath := fmt.Sprintf("%s/%s/status/%s", prefix, id, StatusInQueue) - log.Debug(fmt.Sprintf("setting status to %s", StatusInQueue)) + statusPath := fmt.Sprintf("%s/%s/status/%s", prefix, id, async.StatusInQueue) + log.Debug(fmt.Sprintf("setting status to %s", async.StatusInQueue)) if err := s.storage.Upload(statusPath, strings.NewReader(""), "text/plain"); err != nil { return "", err } @@ -79,21 +80,21 @@ func (s *service) CreateWorkload(id string, payload io.Reader, contentType strin func (s *service) GetWorkload(id string) (GetWorkloadResponse, error) { log := s.logger.With(zap.String("id", id)) - status, err := s.getStatus(id) + st, err := s.getStatus(id) if err != nil { return GetWorkloadResponse{}, err } - if status != StatusCompleted { + if st != async.StatusCompleted { return GetWorkloadResponse{ ID: id, - Status: status, + Status: st, }, nil } // attempt to download user result - prefix := s.workloadStoragePrefix() - resultPath := fmt.Sprintf("%s/%s/result.json", prefix, id) + prefix := async.StoragePath(s.clusterUID, s.apiName) + resultPath := async.ResultPath(prefix, id) log.Debug("downloading user result", zap.String("path", resultPath)) resultBuf, err := s.storage.Download(resultPath) if err != nil { @@ -113,46 +114,43 @@ func (s *service) GetWorkload(id string) (GetWorkloadResponse, error) { return GetWorkloadResponse{ ID: id, - Status: status, + Status: st, Result: &userResponse, Timestamp: ×tamp, }, nil } -func (s *service) getStatus(id string) (Status, error) { - prefix := s.workloadStoragePrefix() +func (s *service) getStatus(id string) (async.Status, error) { + prefix := async.StoragePath(s.clusterUID, s.apiName) log := s.logger.With(zap.String("id", id)) // download workload status - log.Debug("checking status", zap.String("path", fmt.Sprintf("%s/%s/status/*", prefix, id))) - files, err := s.storage.List(fmt.Sprintf("%s/%s/status", prefix, id)) + statusPrefixPath := async.StatusPrefixPath(prefix, id) + log.Debug("checking status", zap.String("path", statusPrefixPath)) + files, err := s.storage.List(statusPrefixPath) if err != nil { return "", err } if len(files) == 0 { - return StatusNotFound, nil + return async.StatusNotFound, nil } // determine request status - status := StatusInQueue + st := async.StatusInQueue for _, file := range files { - fileStatus := Status(file) + fileStatus := async.Status(file) if !fileStatus.Valid() { - status = fileStatus - return "", fmt.Errorf("invalid workload status: %s", status) + st = fileStatus + return "", fmt.Errorf("invalid workload status: %s", st) } - if fileStatus == StatusInProgress { - status = fileStatus + if fileStatus == async.StatusInProgress { + st = fileStatus } - if fileStatus == StatusCompleted || fileStatus == StatusFailed { - status = fileStatus + if fileStatus == async.StatusCompleted || fileStatus == async.StatusFailed { + st = fileStatus break } } - return status, nil -} - -func (s *service) workloadStoragePrefix() string { - return fmt.Sprintf("%s/workloads/%s", s.clusterUID, s.apiName) + return st, nil } diff --git a/pkg/async-gateway/types.go b/pkg/async-gateway/types.go index 5e1c07e687..e32159d653 100644 --- a/pkg/async-gateway/types.go +++ b/pkg/async-gateway/types.go @@ -16,35 +16,14 @@ limitations under the License. package gateway -import "time" +import ( + "time" -// UserResponse represents the user's API response, which has to be JSON serializable -type UserResponse = map[string]interface{} - -// Status is an enum type for workload status -type Status string - -// Different possible workload status -const ( - StatusNotFound Status = "not_found" - StatusFailed Status = "failed" - StatusInProgress Status = "in_progress" - StatusInQueue Status = "in_queue" - StatusCompleted Status = "completed" + "github.com/cortexlabs/cortex/pkg/types/async" ) -func (status Status) String() string { - return string(status) -} - -func (status Status) Valid() bool { - switch status { - case StatusNotFound, StatusFailed, StatusInProgress, StatusInQueue, StatusCompleted: - return true - default: - return false - } -} +// UserResponse represents the user's API response, which has to be JSON serializable +type UserResponse = map[string]interface{} //CreateWorkloadResponse represents the response returned to the user on workload creation type CreateWorkloadResponse struct { @@ -54,7 +33,7 @@ type CreateWorkloadResponse struct { // GetWorkloadResponse represents the workload response that is returned to the user type GetWorkloadResponse struct { ID string `json:"id"` - Status Status `json:"status"` + Status async.Status `json:"status"` Result *UserResponse `json:"result,omitempty"` Timestamp *time.Time `json:"timestamp,omitempty"` } diff --git a/pkg/config/config.go b/pkg/config/config.go index dd88371b58..ea278fa48c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -67,9 +67,9 @@ func getClusterConfigFromConfigMap() (clusterconfig.Config, error) { return clusterconfig.Config{}, err } clusterConfig := clusterconfig.Config{} - errs := cr.ParseYAMLBytes(&clusterConfig, clusterconfig.FullManagedValidation, []byte(configMapData["cluster.yaml"])) - if errors.FirstError(errs...) != nil { - return clusterconfig.Config{}, errors.FirstError(errs...) + err = cr.ParseYAMLBytes(&clusterConfig, clusterconfig.FullManagedValidation, []byte(configMapData["cluster.yaml"])) + if err != nil { + return clusterconfig.Config{}, err } return clusterConfig, nil diff --git a/pkg/dequeuer/async_handler.go b/pkg/dequeuer/async_handler.go new file mode 100644 index 0000000000..a0940a1d0b --- /dev/null +++ b/pkg/dequeuer/async_handler.go @@ -0,0 +1,206 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dequeuer + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/sqs" + awslib "github.com/cortexlabs/cortex/pkg/lib/aws" + "github.com/cortexlabs/cortex/pkg/lib/errors" + "github.com/cortexlabs/cortex/pkg/lib/telemetry" + "github.com/cortexlabs/cortex/pkg/types/async" + "go.uber.org/zap" +) + +const ( + // CortexRequestIDHeader is the header containing the workload request id for the user container + CortexRequestIDHeader = "X-Cortex-Request-ID" +) + +type AsyncMessageHandler struct { + config AsyncMessageHandlerConfig + aws *awslib.Client + log *zap.SugaredLogger + storagePath string + httpClient *http.Client +} + +type AsyncMessageHandlerConfig struct { + ClusterUID string + Bucket string + APIName string + TargetURL string +} + +type userPayload struct { + Body io.ReadCloser + ContentType string +} + +func NewAsyncMessageHandler(config AsyncMessageHandlerConfig, awsClient *awslib.Client, logger *zap.SugaredLogger) *AsyncMessageHandler { + return &AsyncMessageHandler{ + config: config, + aws: awsClient, + log: logger, + storagePath: async.StoragePath(config.ClusterUID, config.APIName), + httpClient: &http.Client{}, + } +} + +func (h *AsyncMessageHandler) Handle(message *sqs.Message) error { + if message == nil { + return errors.ErrorUnexpected("got unexpected nil SQS message") + } + + if message.Body == nil || *message.Body == "" { + return errors.ErrorUnexpected("got unexpected sqs message with empty or nil body") + } + + requestID := *message.Body + err := h.handleMessage(requestID) + if err != nil { + return err + } + return nil +} + +func (h *AsyncMessageHandler) handleMessage(requestID string) error { + h.log.Infow("processing workload", "id", requestID) + + err := h.updateStatus(requestID, async.StatusInProgress) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to update status to %s", async.StatusInProgress)) + } + + payload, err := h.getPayload(requestID) + if err != nil { + updateStatusErr := h.updateStatus(requestID, async.StatusFailed) + if updateStatusErr != nil { + h.log.Errorw("failed to update status after failure to get payload", "id", requestID, "error", updateStatusErr) + } + return errors.Wrap(err, "failed to get payload") + } + defer h.deletePayload(requestID) + + result, err := h.submitRequest(payload, requestID) + if err != nil { + h.log.Errorw("failed to submit request to user container", "id", requestID, "error", err) + updateStatusErr := h.updateStatus(requestID, async.StatusFailed) + if updateStatusErr != nil { + return errors.Wrap(updateStatusErr, fmt.Sprintf("failed to update status to %s", async.StatusFailed)) + } + return nil + } + + if err = h.uploadResult(requestID, result); err != nil { + updateStatusErr := h.updateStatus(requestID, async.StatusFailed) + if updateStatusErr != nil { + h.log.Errorw("failed to update status after failure to upload result", "id", requestID, "error", updateStatusErr) + } + return errors.Wrap(err, "failed to upload result to storage") + } + + if err = h.updateStatus(requestID, async.StatusCompleted); err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to update status to %s", async.StatusCompleted)) + } + + h.log.Infow("workload processing complete", "id", requestID) + + return nil +} + +func (h *AsyncMessageHandler) updateStatus(requestID string, status async.Status) error { + key := async.StatusPath(h.storagePath, requestID, status) + return h.aws.UploadStringToS3("", h.config.Bucket, key) +} + +func (h *AsyncMessageHandler) getPayload(requestID string) (*userPayload, error) { + key := async.PayloadPath(h.storagePath, requestID) + output, err := h.aws.S3().GetObject( + &s3.GetObjectInput{ + Key: aws.String(key), + Bucket: aws.String(h.config.Bucket), + }, + ) + if err != nil { + return nil, errors.WithStack(err) + } + + contentType := "application/octet-stream" + if output.ContentType != nil { + contentType = *output.ContentType + } + + return &userPayload{ + Body: output.Body, + ContentType: contentType, + }, nil +} + +func (h *AsyncMessageHandler) deletePayload(requestID string) { + key := async.PayloadPath(h.storagePath, requestID) + err := h.aws.DeleteS3File(h.config.Bucket, key) + if err != nil { + h.log.Errorw("failed to delete user payload", "error", err) + telemetry.Error(errors.Wrap(err, "failed to delete user payload")) + } +} + +func (h *AsyncMessageHandler) submitRequest(payload *userPayload, requestID string) (interface{}, error) { + req, err := http.NewRequest(http.MethodPost, h.config.TargetURL, payload.Body) + if err != nil { + return nil, errors.WithStack(err) + } + + req.Header.Set("Content-Type", payload.ContentType) + req.Header.Set(CortexRequestIDHeader, requestID) + response, err := h.httpClient.Do(req) + if err != nil { + return nil, ErrorUserContainerNotReachable(err) + } + + defer func() { + _ = response.Body.Close() + }() + + if response.StatusCode != http.StatusOK { + return nil, ErrorUserContainerResponseStatusCode(response.StatusCode) + } + + if !strings.HasPrefix(response.Header.Get("Content-Type"), "application/json") { + return nil, ErrorUserContainerResponseMissingJSONHeader() + } + + var result interface{} + if err = json.NewDecoder(response.Body).Decode(&result); err != nil { + return nil, ErrorUserContainerResponseNotJSONDecodable() + } + + return result, nil +} + +func (h *AsyncMessageHandler) uploadResult(requestID string, result interface{}) error { + key := async.ResultPath(h.storagePath, requestID) + return h.aws.UploadJSONToS3(result, h.config.Bucket, key) +} diff --git a/pkg/dequeuer/async_handler_test.go b/pkg/dequeuer/async_handler_test.go new file mode 100644 index 0000000000..ffe7b96af8 --- /dev/null +++ b/pkg/dequeuer/async_handler_test.go @@ -0,0 +1,121 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dequeuer + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/sqs" + "github.com/cortexlabs/cortex/pkg/lib/errors" + "github.com/cortexlabs/cortex/pkg/lib/random" + "github.com/cortexlabs/cortex/pkg/types/async" + "github.com/stretchr/testify/require" +) + +const ( + _testBucket = "test" +) + +func TestAsyncMessageHandler_Handle(t *testing.T) { + t.Parallel() + + log := newLogger(t) + awsClient := testAWSClient(t) + + requestID := random.String(8) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, requestID, r.Header.Get(CortexRequestIDHeader)) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + + asyncHandler := NewAsyncMessageHandler(AsyncMessageHandlerConfig{ + ClusterUID: "cortex-test", + Bucket: _testBucket, + APIName: "async-test", + TargetURL: server.URL, + }, awsClient, log) + + _, err := awsClient.S3().CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(_testBucket), + }) + require.NoError(t, err) + + err = awsClient.UploadStringToS3("{}", asyncHandler.config.Bucket, fmt.Sprintf("%s/%s/payload", asyncHandler.storagePath, requestID)) + require.NoError(t, err) + + err = asyncHandler.Handle(&sqs.Message{ + Body: aws.String(requestID), + MessageId: aws.String(requestID), + }) + require.NoError(t, err) + + _, err = awsClient.ReadStringFromS3( + _testBucket, + fmt.Sprintf("%s/%s/status/%s", asyncHandler.storagePath, requestID, async.StatusCompleted), + ) + require.NoError(t, err) +} + +func TestAsyncMessageHandler_Handle_Errors(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + message *sqs.Message + expectedError error + }{ + { + name: "nil", + message: nil, + expectedError: errors.ErrorUnexpected("got unexpected nil SQS message"), + }, + { + name: "nil body", + message: &sqs.Message{}, + expectedError: errors.ErrorUnexpected("got unexpected sqs message with empty or nil body"), + }, + { + name: "empty body", + message: &sqs.Message{Body: aws.String("")}, + expectedError: errors.ErrorUnexpected("got unexpected sqs message with empty or nil body"), + }, + } + + log := newLogger(t) + awsClient := testAWSClient(t) + + asyncHandler := NewAsyncMessageHandler(AsyncMessageHandlerConfig{ + ClusterUID: "cortex-test", + Bucket: _testBucket, + APIName: "async-test", + TargetURL: "http://fake.cortex.dev", + }, awsClient, log) + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + err := asyncHandler.Handle(tt.message) + require.EqualError(t, err, tt.expectedError.Error()) + }) + } +} diff --git a/pkg/dequeuer/batch_handler.go b/pkg/dequeuer/batch_handler.go new file mode 100644 index 0000000000..2fc94f6f7c --- /dev/null +++ b/pkg/dequeuer/batch_handler.go @@ -0,0 +1,230 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dequeuer + +import ( + "bytes" + "net/http" + "time" + + "github.com/DataDog/datadog-go/statsd" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" + awslib "github.com/cortexlabs/cortex/pkg/lib/aws" + "github.com/cortexlabs/cortex/pkg/lib/errors" + "github.com/cortexlabs/cortex/pkg/lib/urls" + "github.com/xtgo/uuid" + "go.uber.org/zap" +) + +const ( + // CortexJobIDHeader is the header containing the job id for the user container + CortexJobIDHeader = "X-Cortex-Job-ID" + _jobCompleteMessageDelay = 10 * time.Second +) + +type BatchMessageHandler struct { + config BatchMessageHandlerConfig + jobCompleteMessageDelay time.Duration + tags []string + aws *awslib.Client + metrics statsd.ClientInterface + log *zap.SugaredLogger + httpClient *http.Client +} + +type BatchMessageHandlerConfig struct { + APIName string + JobID string + QueueURL string + Region string + TargetURL string +} + +func NewBatchMessageHandler(config BatchMessageHandlerConfig, awsClient *awslib.Client, statsdClient statsd.ClientInterface, log *zap.SugaredLogger) *BatchMessageHandler { + tags := []string{ + "api_name:" + config.APIName, + "job_id" + config.JobID, + } + + return &BatchMessageHandler{ + config: config, + jobCompleteMessageDelay: _jobCompleteMessageDelay, + tags: tags, + aws: awsClient, + metrics: statsdClient, + log: log, + httpClient: &http.Client{}, + } +} + +func (h *BatchMessageHandler) Handle(message *sqs.Message) error { + if isOnJobCompleteMessage(message) { + err := h.onJobComplete(message) + if err != nil { + return errors.Wrap(err, "failed to handle 'onJobComplete' message") + } + return nil + } + err := h.handleBatch(message) + if err != nil { + return err + } + return nil +} + +func (h *BatchMessageHandler) recordSuccess() error { + err := h.metrics.Incr("cortex_batch_succeeded", h.tags, 1.0) + if err != nil { + return errors.WithStack(err) + } + return nil +} + +func (h *BatchMessageHandler) recordFailure() error { + err := h.metrics.Incr("cortex_batch_failed", h.tags, 1.0) + if err != nil { + return errors.WithStack(err) + } + return nil +} + +func (h *BatchMessageHandler) recordTimePerBatch(elapsedTime time.Duration) error { + err := h.metrics.Histogram("cortex_time_per_batch", elapsedTime.Seconds(), h.tags, 1.0) + if err != nil { + return errors.WithStack(err) + } + return nil +} + +func (h *BatchMessageHandler) submitRequest(messageBody string, isOnJobComplete bool) error { + targetURL := h.config.TargetURL + if isOnJobComplete { + targetURL = urls.Join(targetURL, "/on-job-complete") + } + + req, err := http.NewRequest(http.MethodPost, targetURL, bytes.NewBuffer([]byte(messageBody))) + if err != nil { + return errors.WithStack(err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set(CortexJobIDHeader, h.config.JobID) + response, err := h.httpClient.Do(req) + if err != nil { + return ErrorUserContainerNotReachable(err) + } + + if response.StatusCode == http.StatusNotFound && isOnJobComplete { + return nil + } + + if response.StatusCode != http.StatusOK { + return ErrorUserContainerResponseStatusCode(response.StatusCode) + } + + return nil +} + +func (h *BatchMessageHandler) handleBatch(message *sqs.Message) error { + h.log.Infow("processing batch", "id", *message.MessageId) + + startTime := time.Now() + + err := h.submitRequest(*message.Body, false) + if err != nil { + h.log.Errorw("failed to process batch", "id", *message.MessageId, "error", err) + recordFailureErr := h.recordFailure() + if recordFailureErr != nil { + return errors.Wrap(recordFailureErr, "failed to record failure metric") + } + return nil + } + + endTime := time.Now().Sub(startTime) + + err = h.recordSuccess() + if err != nil { + return errors.Wrap(err, "failed to record success metric") + } + + err = h.recordTimePerBatch(endTime) + if err != nil { + return errors.Wrap(err, "failed to record time per batch") + } + return nil +} + +func (h *BatchMessageHandler) onJobComplete(message *sqs.Message) error { + shouldRunOnJobComplete := false + h.log.Info("received job_complete message") + for true { + queueAttributes, err := GetQueueAttributes(h.aws, h.config.QueueURL) + if err != nil { + return err + } + + totalMessages := queueAttributes.TotalMessages() + + if totalMessages > 1 { + time.Sleep(h.jobCompleteMessageDelay) + h.log.Infow("found other messages in queue, requeuing job_complete message", "id", *message.MessageId) + newMessageID := uuid.NewRandom().String() + if _, err = h.aws.SQS().SendMessage( + &sqs.SendMessageInput{ + QueueUrl: &h.config.QueueURL, + MessageBody: aws.String("job_complete"), + MessageAttributes: map[string]*sqs.MessageAttributeValue{ + "job_complete": { + DataType: aws.String("String"), + StringValue: aws.String("true"), + }, + "api_name": { + DataType: aws.String("String"), + StringValue: aws.String(h.config.APIName), + }, + "job_id": { + DataType: aws.String("String"), + StringValue: aws.String(h.config.JobID), + }, + }, + MessageDeduplicationId: aws.String(newMessageID), + MessageGroupId: aws.String(newMessageID), + }, + ); err != nil { + return errors.WithStack(err) + } + + return nil + } + + if shouldRunOnJobComplete { + h.log.Infow("processing job_complete message", "id", *message.MessageId) + return h.submitRequest(*message.Body, true) + } + shouldRunOnJobComplete = true + + time.Sleep(h.jobCompleteMessageDelay) + } + + return nil +} + +func isOnJobCompleteMessage(message *sqs.Message) bool { + _, found := message.MessageAttributes["job_complete"] + return found +} diff --git a/pkg/dequeuer/batch_handler_test.go b/pkg/dequeuer/batch_handler_test.go new file mode 100644 index 0000000000..49a80a8c67 --- /dev/null +++ b/pkg/dequeuer/batch_handler_test.go @@ -0,0 +1,106 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dequeuer + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/DataDog/datadog-go/statsd" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" + "github.com/stretchr/testify/require" +) + +func TestBatchMessageHandler_Handle(t *testing.T) { + t.Parallel() + awsClient := testAWSClient(t) + + var callCount int + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + callCount++ + w.WriteHeader(http.StatusOK) + }), + ) + + batchHandler := NewBatchMessageHandler(BatchMessageHandlerConfig{ + APIName: "test", + JobID: "12345", + Region: _localStackDefaultRegion, + TargetURL: server.URL, + }, awsClient, &statsd.NoOpClient{}, newLogger(t)) + + err := batchHandler.Handle(&sqs.Message{ + Body: aws.String(""), + MessageId: aws.String("1"), + }) + + require.Equal(t, callCount, 1) + require.NoError(t, err) +} + +func TestBatchMessageHandler_Handle_OnJobComplete(t *testing.T) { + t.Parallel() + awsClient := testAWSClient(t) + queueURL := createQueue(t, awsClient) + + var callCount int + mux := http.NewServeMux() + mux.HandleFunc("/on-job-complete", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + callCount++ + w.WriteHeader(http.StatusOK) + }) + server := httptest.NewServer(mux) + + batchHandler := NewBatchMessageHandler(BatchMessageHandlerConfig{ + APIName: "test", + JobID: "12345", + Region: _localStackDefaultRegion, + TargetURL: server.URL, + QueueURL: queueURL, + }, awsClient, &statsd.NoOpClient{}, newLogger(t)) + + batchHandler.jobCompleteMessageDelay = 0 + + err := batchHandler.Handle(&sqs.Message{ + Body: aws.String("job_complete"), + MessageAttributes: map[string]*sqs.MessageAttributeValue{ + "job_complete": { + DataType: aws.String("String"), + StringValue: aws.String("true"), + }, + "api_name": { + DataType: aws.String("String"), + StringValue: aws.String("test"), + }, + "job_id": { + DataType: aws.String("String"), + StringValue: aws.String("12345"), + }, + }, + MessageId: aws.String("00000"), + }) + + require.NoError(t, err) + require.Equal(t, callCount, 1) +} diff --git a/pkg/dequeuer/dequeuer.go b/pkg/dequeuer/dequeuer.go new file mode 100644 index 0000000000..e448b1cb1c --- /dev/null +++ b/pkg/dequeuer/dequeuer.go @@ -0,0 +1,212 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dequeuer + +import ( + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" + awslib "github.com/cortexlabs/cortex/pkg/lib/aws" + "github.com/cortexlabs/cortex/pkg/lib/errors" + "github.com/cortexlabs/cortex/pkg/lib/telemetry" + "go.uber.org/zap" +) + +var ( + _messageAttributes = []string{"All"} + _waitTime = 10 * time.Second + _visibilityTimeout = 30 * time.Second + _notFoundSleepTime = 10 * time.Second + _renewalPeriod = 10 * time.Second +) + +type SQSDequeuerConfig struct { + Region string + QueueURL string + StopIfNoMessages bool +} + +type SQSDequeuer struct { + aws *awslib.Client + config SQSDequeuerConfig + hasDeadLetterQueue bool + waitTimeSeconds *int64 + visibilityTimeout *int64 + notFoundSleepTime time.Duration + renewalPeriod time.Duration + log *zap.SugaredLogger + done chan struct{} +} + +func NewSQSDequeuer(config SQSDequeuerConfig, awsClient *awslib.Client, logger *zap.SugaredLogger) (*SQSDequeuer, error) { + attr, err := GetQueueAttributes(awsClient, config.QueueURL) + if err != nil { + return nil, err + } + + return &SQSDequeuer{ + aws: awsClient, + config: config, + hasDeadLetterQueue: attr.HasRedrivePolicy, + waitTimeSeconds: aws.Int64(int64(_waitTime.Seconds())), + visibilityTimeout: aws.Int64(int64(_visibilityTimeout.Seconds())), + notFoundSleepTime: _notFoundSleepTime, + renewalPeriod: _renewalPeriod, + log: logger, + done: make(chan struct{}), + }, nil +} + +func (d *SQSDequeuer) ReceiveMessage() (*sqs.Message, error) { + output, err := d.aws.SQS().ReceiveMessage(&sqs.ReceiveMessageInput{ + QueueUrl: aws.String(d.config.QueueURL), + MaxNumberOfMessages: aws.Int64(1), + MessageAttributeNames: aws.StringSlice(_messageAttributes), + VisibilityTimeout: d.visibilityTimeout, + WaitTimeSeconds: d.waitTimeSeconds, + }) + + if err != nil { + return nil, errors.WithStack(err) + } + + if len(output.Messages) == 0 { + return nil, nil + } + + return output.Messages[0], nil +} + +func (d *SQSDequeuer) Start(messageHandler MessageHandler) error { + noMessagesInPreviousIteration := false + +loop: + for { + select { + case <-d.done: + break loop + default: + message, err := d.ReceiveMessage() + if err != nil { + return err + } + + if message == nil { // no message received + queueAttributes, err := GetQueueAttributes(d.aws, d.config.QueueURL) + if err != nil { + return err + } + + if queueAttributes.TotalMessages() == 0 { + if noMessagesInPreviousIteration && d.config.StopIfNoMessages { + d.log.Info("no messages found in queue, exiting ...") + return nil + } + noMessagesInPreviousIteration = true + } + time.Sleep(d.notFoundSleepTime) + continue + } + + noMessagesInPreviousIteration = false + receiptHandle := *message.ReceiptHandle + done := d.StartMessageRenewer(receiptHandle) + err = d.handleMessage(message, messageHandler, done) + if err != nil { + d.log.Error(err) + if !errors.IsNoTelemetry(err) { + telemetry.Error(err) + } + } + } + } + + return nil +} + +func (d *SQSDequeuer) Shutdown() { + d.done <- struct{}{} +} + +func (d *SQSDequeuer) handleMessage(message *sqs.Message, messageHandler MessageHandler, done chan struct{}) error { + messageErr := messageHandler.Handle(message) // handle error later + + done <- struct{}{} + isOnJobComplete := isOnJobCompleteMessage(message) + + if messageErr != nil && d.hasDeadLetterQueue && !isOnJobComplete { + // expire messages when dead letter queue is configured to facilitate redrive policy. + // always delete onJobComplete messages regardless of redrive policy because a new one will + // be added if an onJobComplete message has been consumed prematurely + _, err := d.aws.SQS().ChangeMessageVisibility( + &sqs.ChangeMessageVisibilityInput{ + QueueUrl: &d.config.QueueURL, + ReceiptHandle: message.ReceiptHandle, + VisibilityTimeout: aws.Int64(0), + }, + ) + if err != nil { + return errors.Wrap(err, "failed to change sqs message visibility") + } + return nil + } + + _, err := d.aws.SQS().DeleteMessage( + &sqs.DeleteMessageInput{ + QueueUrl: &d.config.QueueURL, + ReceiptHandle: message.ReceiptHandle, + }, + ) + if err != nil { + return errors.Wrap(err, "failed to delete sqs message") + } + + if messageErr != nil { + return messageErr + } + + return nil +} + +func (d *SQSDequeuer) StartMessageRenewer(receiptHandle string) chan struct{} { + done := make(chan struct{}) + ticker := time.NewTicker(d.renewalPeriod) + startTime := time.Now() + go func() { + defer ticker.Stop() + for true { + select { + case <-done: + return + case tickerTime := <-ticker.C: + newVisibilityTimeout := tickerTime.Sub(startTime) + d.renewalPeriod + _, err := d.aws.SQS().ChangeMessageVisibility( + &sqs.ChangeMessageVisibilityInput{ + QueueUrl: &d.config.QueueURL, + ReceiptHandle: &receiptHandle, + VisibilityTimeout: aws.Int64(int64(newVisibilityTimeout.Seconds())), + }, + ) + if err != nil { + d.log.Errorw("failed to renew message visibility timeout", "error", err) + } + } + } + }() + return done +} diff --git a/pkg/dequeuer/dequeuer_test.go b/pkg/dequeuer/dequeuer_test.go new file mode 100644 index 0000000000..6e13fec3c4 --- /dev/null +++ b/pkg/dequeuer/dequeuer_test.go @@ -0,0 +1,357 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dequeuer + +import ( + "errors" + "fmt" + "log" + "net/http" + "os" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/sqs" + awslib "github.com/cortexlabs/cortex/pkg/lib/aws" + "github.com/cortexlabs/cortex/pkg/lib/random" + "github.com/ory/dockertest/v3" + dc "github.com/ory/dockertest/v3/docker" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +var localStackEndpoint string + +const ( + _localStackDefaultRegion = "us-east-1" +) + +func TestMain(m *testing.M) { + // uses a sensible default on windows (tcp/http) and linux/osx (socket) + log.Println("Starting AWS localstack docker...") + pool, err := dockertest.NewPool("") + if err != nil { + log.Fatalf("Could not connect to docker: %s", err) + } + + options := &dockertest.RunOptions{ + Repository: "localstack/localstack", + Tag: "latest", + PortBindings: map[dc.Port][]dc.PortBinding{ + "4566/tcp": { + {HostPort: "4566"}, + }, + }, + Env: []string{"SERVICES=sqs,s3"}, + } + + resource, err := pool.RunWithOptions(options) + if err != nil { + log.Fatalf("Could not start resource: %s", err) + } + + localStackEndpoint = fmt.Sprintf("localhost:%s", resource.GetPort("4566/tcp")) + + // exponential backoff-retry, because the application in the container might not be ready to accept connections yet + // the minio client does not do service discovery for you (i.e. it does not check if connection can be established), so we have to use the health check + if err := pool.Retry(func() error { + url := fmt.Sprintf("http://%s/health", localStackEndpoint) + resp, err := http.Get(url) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("status code not OK") + } + return nil + }); err != nil { + log.Fatalf("Could not connect to docker: %s", err) + } + + code := m.Run() + + // You can't defer this because os.Exit doesn't care for defer + if err := pool.Purge(resource); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } + + os.Exit(code) +} + +func testAWSClient(t *testing.T) *awslib.Client { + t.Helper() + + sess, err := session.NewSessionWithOptions(session.Options{ + Config: aws.Config{ + Credentials: credentials.NewStaticCredentials("test", "test", ""), + Endpoint: aws.String(localStackEndpoint), + Region: aws.String(_localStackDefaultRegion), // localstack default region + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + }, + }) + require.NoError(t, err) + + client, err := awslib.NewForSession(sess) + require.NoError(t, err) + + return client +} + +func newLogger(t *testing.T) *zap.SugaredLogger { + t.Helper() + + config := zap.NewDevelopmentConfig() + config.Level = zap.NewAtomicLevelAt(zap.ErrorLevel) + logger, err := config.Build() + require.NoError(t, err) + + logr := logger.Sugar() + + return logr +} + +func createQueue(t *testing.T, awsClient *awslib.Client) string { + t.Helper() + + createQueueOutput, err := awsClient.SQS().CreateQueue( + &sqs.CreateQueueInput{ + QueueName: aws.String(fmt.Sprintf("test-%s.fifo", random.Digits(5))), + Attributes: aws.StringMap( + map[string]string{ + sqs.QueueAttributeNameFifoQueue: "true", + sqs.QueueAttributeNameVisibilityTimeout: "60", + }, + ), + }, + ) + require.NoError(t, err) + require.NotNil(t, createQueueOutput.QueueUrl) + require.NotEmpty(t, *createQueueOutput.QueueUrl) + + queueURL := *createQueueOutput.QueueUrl + return queueURL +} + +func TestSQSDequeuer_ReceiveMessage(t *testing.T) { + t.Parallel() + + awsClient := testAWSClient(t) + queueURL := createQueue(t, awsClient) + + messageID := "12345" + messageBody := "blah" + sentMessage, err := awsClient.SQS().SendMessage(&sqs.SendMessageInput{ + MessageBody: aws.String(messageBody), + MessageDeduplicationId: aws.String(messageID), + MessageGroupId: aws.String(messageID), + QueueUrl: aws.String(queueURL), + }) + require.NoError(t, err) + + dq, err := NewSQSDequeuer( + SQSDequeuerConfig{ + Region: _localStackDefaultRegion, + QueueURL: queueURL, + StopIfNoMessages: true, + }, awsClient, newLogger(t), + ) + require.NoError(t, err) + + gotMessage, err := dq.ReceiveMessage() + require.NoError(t, err) + + require.NotNil(t, gotMessage) + require.Equal(t, messageBody, *gotMessage.Body) + require.Equal(t, *sentMessage.MessageId, *gotMessage.MessageId) +} + +func TestSQSDequeuer_StartMessageRenewer(t *testing.T) { + t.Parallel() + + awsClient := testAWSClient(t) + queueURL := createQueue(t, awsClient) + + dq, err := NewSQSDequeuer( + SQSDequeuerConfig{ + Region: _localStackDefaultRegion, + QueueURL: queueURL, + StopIfNoMessages: true, + }, awsClient, newLogger(t), + ) + require.NoError(t, err) + + dq.renewalPeriod = time.Second + dq.visibilityTimeout = aws.Int64(2) + + messageID := "12345" + messageBody := "blah" + _, err = awsClient.SQS().SendMessage(&sqs.SendMessageInput{ + MessageBody: aws.String(messageBody), + MessageDeduplicationId: aws.String(messageID), + MessageGroupId: aws.String(messageID), + QueueUrl: aws.String(queueURL), + }) + require.NoError(t, err) + + message, err := dq.ReceiveMessage() + require.NoError(t, err) + require.NotNil(t, message) + + done := dq.StartMessageRenewer(*message.ReceiptHandle) + defer func() { + done <- struct{}{} + }() + + require.Never(t, func() bool { + msg, err := dq.ReceiveMessage() + require.NoError(t, err) + + return msg != nil + }, time.Second, 10*time.Second) +} + +func TestSQSDequeuerTerminationOnEmptyQueue(t *testing.T) { + t.Parallel() + + awsClient := testAWSClient(t) + queueURL := createQueue(t, awsClient) + + dq, err := NewSQSDequeuer( + SQSDequeuerConfig{ + Region: _localStackDefaultRegion, + QueueURL: queueURL, + StopIfNoMessages: true, + }, awsClient, newLogger(t), + ) + require.NoError(t, err) + + dq.notFoundSleepTime = 0 + dq.waitTimeSeconds = aws.Int64(0) + + messageID := "12345" + messageBody := "blah" + _, err = awsClient.SQS().SendMessage(&sqs.SendMessageInput{ + MessageBody: aws.String(messageBody), + MessageDeduplicationId: aws.String(messageID), + MessageGroupId: aws.String(messageID), + QueueUrl: aws.String(queueURL), + }) + require.NoError(t, err) + + msgHandler := &handleFuncMessageHandler{ + HandleFunc: func(msg *sqs.Message) error { + return nil + }, + } + + errCh := make(chan error, 1) + go func() { + errCh <- dq.Start(msgHandler) + }() + + time.AfterFunc(10*time.Second, func() { errCh <- errors.New("timeout: dequeuer did not finish") }) + + err = <-errCh + require.NoError(t, err) +} + +func TestSQSDequeuer_Shutdown(t *testing.T) { + t.Parallel() + + awsClient := testAWSClient(t) + queueURL := createQueue(t, awsClient) + + dq, err := NewSQSDequeuer( + SQSDequeuerConfig{ + Region: _localStackDefaultRegion, + QueueURL: queueURL, + StopIfNoMessages: true, + }, awsClient, newLogger(t), + ) + require.NoError(t, err) + + dq.notFoundSleepTime = 0 + dq.waitTimeSeconds = aws.Int64(0) + + msgHandler := NewHandleFuncMessageHandler( + func(message *sqs.Message) error { + return nil + }, + ) + + errCh := make(chan error, 1) + go func() { + errCh <- dq.Start(msgHandler) + }() + + time.AfterFunc(5*time.Second, func() { errCh <- errors.New("timeout: dequeuer did not exit") }) + + dq.Shutdown() + + err = <-errCh + require.NoError(t, err) +} + +func TestSQSDequeuer_Start_HandlerError(t *testing.T) { + t.Parallel() + + awsClient := testAWSClient(t) + queueURL := createQueue(t, awsClient) + + dq, err := NewSQSDequeuer( + SQSDequeuerConfig{ + Region: _localStackDefaultRegion, + QueueURL: queueURL, + StopIfNoMessages: true, + }, awsClient, newLogger(t), + ) + require.NoError(t, err) + + dq.waitTimeSeconds = aws.Int64(0) + dq.notFoundSleepTime = 0 + dq.renewalPeriod = time.Second + dq.visibilityTimeout = aws.Int64(1) + + msgHandler := NewHandleFuncMessageHandler( + func(message *sqs.Message) error { + return fmt.Errorf("an error occurred") + }, + ) + + messageID := "12345" + messageBody := "blah" + _, err = awsClient.SQS().SendMessage(&sqs.SendMessageInput{ + MessageBody: aws.String(messageBody), + MessageDeduplicationId: aws.String(messageID), + MessageGroupId: aws.String(messageID), + QueueUrl: aws.String(queueURL), + }) + require.NoError(t, err) + + err = dq.Start(msgHandler) + require.NoError(t, err) + + require.Never(t, func() bool { + msg, err := dq.ReceiveMessage() + require.NoError(t, err) + return msg != nil + }, 5*time.Second, time.Second) +} diff --git a/pkg/dequeuer/errors.go b/pkg/dequeuer/errors.go new file mode 100644 index 0000000000..4c495f88f8 --- /dev/null +++ b/pkg/dequeuer/errors.go @@ -0,0 +1,62 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dequeuer + +import ( + "fmt" + + "github.com/cortexlabs/cortex/pkg/lib/errors" +) + +const ( + ErrUserContainerResponseStatusCode = "dequeuer.user_container_response_status_code" + ErrUserContainerResponseMissingJSONHeader = "dequeuer.user_container_response_missing_json_header" + ErrUserContainerResponseNotJSONDecodable = "dequeuer.user_container_response_not_json_decodable" + ErrUserContainerNotReachable = "dequeuer.user_container_not_reachable" +) + +func ErrorUserContainerResponseStatusCode(statusCode int) error { + return &errors.Error{ + Kind: ErrUserContainerResponseStatusCode, + Message: fmt.Sprintf("invalid response from user container; got status code %d, expected status code 200", statusCode), + NoTelemetry: true, + } +} + +func ErrorUserContainerResponseMissingJSONHeader() error { + return &errors.Error{ + Kind: ErrUserContainerResponseMissingJSONHeader, + Message: "invalid response from user container; response content type header is not 'application/json'", + NoTelemetry: true, + } +} + +func ErrorUserContainerResponseNotJSONDecodable() error { + return &errors.Error{ + Kind: ErrUserContainerResponseNotJSONDecodable, + Message: "invalid response from user container; response is not json decodable", + NoTelemetry: true, + } +} + +func ErrorUserContainerNotReachable(err error) error { + return &errors.Error{ + Kind: ErrUserContainerNotReachable, + Message: fmt.Sprintf("user container not reachable: %v", err), + NoTelemetry: true, + } +} diff --git a/pkg/dequeuer/message_handler.go b/pkg/dequeuer/message_handler.go new file mode 100644 index 0000000000..27e05e9352 --- /dev/null +++ b/pkg/dequeuer/message_handler.go @@ -0,0 +1,35 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dequeuer + +import "github.com/aws/aws-sdk-go/service/sqs" + +type MessageHandler interface { + Handle(*sqs.Message) error +} + +func NewHandleFuncMessageHandler(handleFunc func(*sqs.Message) error) MessageHandler { + return &handleFuncMessageHandler{HandleFunc: handleFunc} +} + +type handleFuncMessageHandler struct { + HandleFunc func(message *sqs.Message) error +} + +func (h *handleFuncMessageHandler) Handle(msg *sqs.Message) error { + return h.HandleFunc(msg) +} diff --git a/pkg/dequeuer/queue_attributes.go b/pkg/dequeuer/queue_attributes.go new file mode 100644 index 0000000000..135f5ceaa0 --- /dev/null +++ b/pkg/dequeuer/queue_attributes.go @@ -0,0 +1,74 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dequeuer + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" + awslib "github.com/cortexlabs/cortex/pkg/lib/aws" + "github.com/cortexlabs/cortex/pkg/lib/errors" + s "github.com/cortexlabs/cortex/pkg/lib/strings" +) + +type QueueAttributes struct { + VisibleMessages int + InvisibleMessages int + HasRedrivePolicy bool +} + +func (attr QueueAttributes) TotalMessages() int { + return attr.VisibleMessages + attr.InvisibleMessages +} + +func GetQueueAttributes(client *awslib.Client, queueURL string) (QueueAttributes, error) { + result, err := client.SQS().GetQueueAttributes( + &sqs.GetQueueAttributesInput{ + QueueUrl: aws.String(queueURL), + AttributeNames: aws.StringSlice([]string{"All"}), + }, + ) + if err != nil { + return QueueAttributes{}, errors.WithStack(err) + } + + attributes := aws.StringValueMap(result.Attributes) + + var visibleCount int + var notVisibleCount int + var hasRedrivePolicy bool + if val, found := attributes["ApproximateNumberOfMessages"]; found { + count, ok := s.ParseInt(val) + if ok { + visibleCount = count + } + } + + if val, found := attributes["ApproximateNumberOfMessagesNotVisible"]; found { + count, ok := s.ParseInt(val) + if ok { + notVisibleCount = count + } + } + + _, hasRedrivePolicy = attributes["RedrivePolicy"] + + return QueueAttributes{ + VisibleMessages: visibleCount, + InvisibleMessages: notVisibleCount, + HasRedrivePolicy: hasRedrivePolicy, + }, nil +} diff --git a/pkg/lib/aws/aws.go b/pkg/lib/aws/aws.go index 669379b17d..1fea886658 100644 --- a/pkg/lib/aws/aws.go +++ b/pkg/lib/aws/aws.go @@ -35,6 +35,17 @@ type Client struct { hashedAccountID *string } +func NewForSession(sess *session.Session) (*Client, error) { + if sess.Config.Region == nil { + return nil, errors.ErrorUnexpected("session config is missing the Region field") + } + + return &Client{ + Region: *sess.Config.Region, + sess: sess, + }, nil +} + func NewFromClientS3Path(s3Path string, awsClient *Client) (*Client, error) { if !awsClient.IsAnonymous { return NewFromS3Path(s3Path) @@ -124,10 +135,6 @@ func New() (*Client, error) { }, nil } -func NewAnonymousClient() (*Client, error) { - return NewAnonymousClientWithRegion("us-east-1") // region is always required -} - func NewAnonymousClientWithRegion(region string) (*Client, error) { sess, err := session.NewSession(&aws.Config{ Credentials: credentials.AnonymousCredentials, diff --git a/pkg/lib/configreader/reader.go b/pkg/lib/configreader/reader.go index b6a990a73b..3e9032eb52 100644 --- a/pkg/lib/configreader/reader.go +++ b/pkg/lib/configreader/reader.go @@ -1038,15 +1038,15 @@ func ParseYAMLFile(dest interface{}, validation *StructValidation, filePath stri return nil } -func ParseYAMLBytes(dest interface{}, validation *StructValidation, data []byte) []error { +func ParseYAMLBytes(dest interface{}, validation *StructValidation, data []byte) error { fileInterface, err := ReadYAMLBytes(data) if err != nil { - return []error{err} + return err } errs := Struct(dest, fileInterface, validation) if errors.HasError(errs) { - return errs + return errors.FirstError(errs...) } return nil diff --git a/pkg/types/async/s3_paths.go b/pkg/types/async/s3_paths.go new file mode 100644 index 0000000000..dab5c7fa45 --- /dev/null +++ b/pkg/types/async/s3_paths.go @@ -0,0 +1,41 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package async + +import ( + "fmt" +) + +func StoragePath(clusterUID, apiName string) string { + return fmt.Sprintf("%s/workloads/%s", clusterUID, apiName) +} + +func PayloadPath(storagePath string, requestID string) string { + return fmt.Sprintf("%s/%s/payload", storagePath, requestID) +} + +func ResultPath(storagePath string, requestID string) string { + return fmt.Sprintf("%s/%s/result.json", storagePath, requestID) +} + +func StatusPrefixPath(storagePath string, requestID string) string { + return fmt.Sprintf("%s/%s/status", storagePath, requestID) +} + +func StatusPath(storagePath string, requestID string, status Status) string { + return fmt.Sprintf("%s/%s", StatusPrefixPath(storagePath, requestID), status) +} diff --git a/pkg/types/async/status.go b/pkg/types/async/status.go new file mode 100644 index 0000000000..c0a95a2cb6 --- /dev/null +++ b/pkg/types/async/status.go @@ -0,0 +1,42 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package async + +// Status is an enum type for workload status +type Status string + +// Different possible workload status +const ( + StatusNotFound Status = "not_found" + StatusFailed Status = "failed" + StatusInProgress Status = "in_progress" + StatusInQueue Status = "in_queue" + StatusCompleted Status = "completed" +) + +func (status Status) String() string { + return string(status) +} + +func (status Status) Valid() bool { + switch status { + case StatusNotFound, StatusFailed, StatusInProgress, StatusInQueue, StatusCompleted: + return true + default: + return false + } +} From c04bd6e8aab6dfc68041402486b541798b435a46 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Fri, 28 May 2021 19:09:04 +0300 Subject: [PATCH 33/82] CaaS - move max_concurrency/max_queue_length fields (#2198) --- pkg/types/spec/validations.go | 47 ++++++++++--------- pkg/types/userconfig/api.go | 42 +++++++---------- pkg/types/userconfig/config_key.go | 18 +++---- pkg/workloads/k8s.go | 4 +- .../task/iris-classifier-trainer/submit.py | 2 +- 5 files changed, 54 insertions(+), 59 deletions(-) diff --git a/pkg/types/spec/validations.go b/pkg/types/spec/validations.go index 99ac629703..e7b12e89ca 100644 --- a/pkg/types/spec/validations.go +++ b/pkg/types/spec/validations.go @@ -180,6 +180,26 @@ func podValidation(kind userconfig.Kind) *cr.StructFieldValidation { }, }, }, + { + StructField: "MaxQueueLength", + Int64Validation: &cr.Int64Validation{ + Default: consts.DefaultMaxQueueLength, + GreaterThan: pointer.Int64(0), + // the proxy can theoretically accept up to 32768 connections, but during testing, + // it has been observed that the number is just slightly lower, so it has been offset by 2678 + LessThanOrEqualTo: pointer.Int64(30000), + }, + }, + { + StructField: "MaxConcurrency", + Int64Validation: &cr.Int64Validation{ + Default: consts.DefaultMaxConcurrency, + GreaterThan: pointer.Int64(0), + // the proxy can theoretically accept up to 32768 connections, but during testing, + // it has been observed that the number is just slightly lower, so it has been offset by 2678 + LessThanOrEqualTo: pointer.Int64(30000), + }, + }, containersValidation(kind), }, }, @@ -474,26 +494,6 @@ func autoscalingValidation() *cr.StructFieldValidation { GreaterThan: pointer.Int32(0), }, }, - { - StructField: "MaxQueueLength", - Int64Validation: &cr.Int64Validation{ - Default: consts.DefaultMaxQueueLength, - GreaterThan: pointer.Int64(0), - // the proxy can theoretically accept up to 32768 connections, but during testing, - // it has been observed that the number is just slightly lower, so it has been offset by 2678 - LessThanOrEqualTo: pointer.Int64(30000), - }, - }, - { - StructField: "MaxConcurrency", - Int64Validation: &cr.Int64Validation{ - Default: consts.DefaultMaxConcurrency, - GreaterThan: pointer.Int64(0), - // the proxy can theoretically accept up to 32768 connections, but during testing, - // it has been observed that the number is just slightly lower, so it has been offset by 2678 - LessThanOrEqualTo: pointer.Int64(30000), - }, - }, { StructField: "TargetInFlight", Float64PtrValidation: &cr.Float64PtrValidation{ @@ -805,13 +805,14 @@ func validateProbe(probe userconfig.Probe, supportsExecProbe bool) error { func validateAutoscaling(api *userconfig.API) error { autoscaling := api.Autoscaling + pod := api.Pod if autoscaling.TargetInFlight == nil { - autoscaling.TargetInFlight = pointer.Float64(float64(autoscaling.MaxConcurrency)) + autoscaling.TargetInFlight = pointer.Float64(float64(pod.MaxConcurrency)) } - if *autoscaling.TargetInFlight > float64(autoscaling.MaxConcurrency)+float64(autoscaling.MaxQueueLength) { - return ErrorTargetInFlightLimitReached(*autoscaling.TargetInFlight, autoscaling.MaxConcurrency, autoscaling.MaxQueueLength) + if *autoscaling.TargetInFlight > float64(pod.MaxConcurrency)+float64(pod.MaxQueueLength) { + return ErrorTargetInFlightLimitReached(*autoscaling.TargetInFlight, pod.MaxConcurrency, pod.MaxQueueLength) } if autoscaling.MinReplicas > autoscaling.MaxReplicas { diff --git a/pkg/types/userconfig/api.go b/pkg/types/userconfig/api.go index e8e8e03644..0560765797 100644 --- a/pkg/types/userconfig/api.go +++ b/pkg/types/userconfig/api.go @@ -44,10 +44,12 @@ type API struct { } type Pod struct { - NodeGroups []string `json:"node_groups" yaml:"node_groups"` - ShmSize *k8s.Quantity `json:"shm_size" yaml:"shm_size"` - Port *int32 `json:"port" yaml:"port"` - Containers []*Container `json:"containers" yaml:"containers"` + NodeGroups []string `json:"node_groups" yaml:"node_groups"` + ShmSize *k8s.Quantity `json:"shm_size" yaml:"shm_size"` + Port *int32 `json:"port" yaml:"port"` + MaxQueueLength int64 `json:"max_queue_length" yaml:"max_queue_length"` + MaxConcurrency int64 `json:"max_concurrency" yaml:"max_concurrency"` + Containers []*Container `json:"containers" yaml:"containers"` } type Container struct { @@ -109,8 +111,6 @@ type Autoscaling struct { MinReplicas int32 `json:"min_replicas" yaml:"min_replicas"` MaxReplicas int32 `json:"max_replicas" yaml:"max_replicas"` InitReplicas int32 `json:"init_replicas" yaml:"init_replicas"` - MaxQueueLength int64 `json:"max_queue_length" yaml:"max_queue_length"` - MaxConcurrency int64 `json:"max_concurrency" yaml:"max_concurrency"` TargetInFlight *float64 `json:"target_in_flight" yaml:"target_in_flight"` Window time.Duration `json:"window" yaml:"window"` DownscaleStabilizationPeriod time.Duration `json:"downscale_stabilization_period" yaml:"downscale_stabilization_period"` @@ -152,6 +152,12 @@ func IdentifyAPI(filePath string, name string, kind Kind, index int) string { // InitReplicas was left out deliberately func (api *API) ToK8sAnnotations() map[string]string { annotations := map[string]string{} + + if api.Pod != nil { + annotations[MaxConcurrencyAnnotationKey] = s.Int64(api.Pod.MaxConcurrency) + annotations[MaxQueueLengthAnnotationKey] = s.Int64(api.Pod.MaxQueueLength) + } + if api.Networking != nil { annotations[EndpointAnnotationKey] = *api.Networking.Endpoint } @@ -159,9 +165,7 @@ func (api *API) ToK8sAnnotations() map[string]string { if api.Autoscaling != nil { annotations[MinReplicasAnnotationKey] = s.Int32(api.Autoscaling.MinReplicas) annotations[MaxReplicasAnnotationKey] = s.Int32(api.Autoscaling.MaxReplicas) - annotations[MaxQueueLengthAnnotationKey] = s.Int64(api.Autoscaling.MaxQueueLength) annotations[TargetInFlightAnnotationKey] = s.Float64(*api.Autoscaling.TargetInFlight) - annotations[MaxConcurrencyAnnotationKey] = s.Int64(api.Autoscaling.MaxConcurrency) annotations[WindowAnnotationKey] = api.Autoscaling.Window.String() annotations[DownscaleStabilizationPeriodAnnotationKey] = api.Autoscaling.DownscaleStabilizationPeriod.String() annotations[UpscaleStabilizationPeriodAnnotationKey] = api.Autoscaling.UpscaleStabilizationPeriod.String() @@ -188,18 +192,6 @@ func AutoscalingFromAnnotations(k8sObj kmeta.Object) (*Autoscaling, error) { } a.MaxReplicas = maxReplicas - maxQueueLength, err := k8s.ParseInt64Annotation(k8sObj, MaxQueueLengthAnnotationKey) - if err != nil { - return nil, err - } - a.MaxQueueLength = maxQueueLength - - maxConcurrency, err := k8s.ParseInt64Annotation(k8sObj, MaxConcurrencyAnnotationKey) - if err != nil { - return nil, err - } - a.MaxConcurrency = maxConcurrency - targetInFlight, err := k8s.ParseFloat64Annotation(k8sObj, TargetInFlightAnnotationKey) if err != nil { return nil, err @@ -309,6 +301,9 @@ func (pod *Pod) UserStr() string { sb.WriteString(fmt.Sprintf("%s: %d\n", PortKey, *pod.Port)) } + sb.WriteString(fmt.Sprintf("%s: %s\n", MaxConcurrencyKey, s.Int64(pod.MaxConcurrency))) + sb.WriteString(fmt.Sprintf("%s: %s\n", MaxQueueLengthKey, s.Int64(pod.MaxQueueLength))) + sb.WriteString(fmt.Sprintf("%s:\n", ContainersKey)) for _, container := range pod.Containers { containerUserStr := s.Indent(container.UserStr(), " ") @@ -465,8 +460,6 @@ func (autoscaling *Autoscaling) UserStr() string { sb.WriteString(fmt.Sprintf("%s: %s\n", MinReplicasKey, s.Int32(autoscaling.MinReplicas))) sb.WriteString(fmt.Sprintf("%s: %s\n", MaxReplicasKey, s.Int32(autoscaling.MaxReplicas))) sb.WriteString(fmt.Sprintf("%s: %s\n", InitReplicasKey, s.Int32(autoscaling.InitReplicas))) - sb.WriteString(fmt.Sprintf("%s: %s\n", MaxQueueLengthKey, s.Int64(autoscaling.MaxQueueLength))) - sb.WriteString(fmt.Sprintf("%s: %s\n", MaxConcurrencyKey, s.Int64(autoscaling.MaxConcurrency))) sb.WriteString(fmt.Sprintf("%s: %s\n", TargetInFlightKey, s.Float64(*autoscaling.TargetInFlight))) sb.WriteString(fmt.Sprintf("%s: %s\n", WindowKey, autoscaling.Window.String())) sb.WriteString(fmt.Sprintf("%s: %s\n", DownscaleStabilizationPeriodKey, autoscaling.DownscaleStabilizationPeriod.String())) @@ -566,6 +559,9 @@ func (api *API) TelemetryEvent() map[string]interface{} { event["pod.port"] = *api.Pod.Port } + event["pod.max_concurrency"] = api.Pod.MaxConcurrency + event["pod.max_queue_length"] = api.Pod.MaxQueueLength + event["pod.containers._len"] = len(api.Pod.Containers) var numReadinessProbes int @@ -607,8 +603,6 @@ func (api *API) TelemetryEvent() map[string]interface{} { event["autoscaling.min_replicas"] = api.Autoscaling.MinReplicas event["autoscaling.max_replicas"] = api.Autoscaling.MaxReplicas event["autoscaling.init_replicas"] = api.Autoscaling.InitReplicas - event["autoscaling.max_queue_length"] = api.Autoscaling.MaxQueueLength - event["autoscaling.max_concurrency"] = api.Autoscaling.MaxConcurrency event["autoscaling.target_in_flight"] = *api.Autoscaling.TargetInFlight event["autoscaling.window"] = api.Autoscaling.Window.Seconds() event["autoscaling.downscale_stabilization_period"] = api.Autoscaling.DownscaleStabilizationPeriod.Seconds() diff --git a/pkg/types/userconfig/config_key.go b/pkg/types/userconfig/config_key.go index b61784f249..fce539daac 100644 --- a/pkg/types/userconfig/config_key.go +++ b/pkg/types/userconfig/config_key.go @@ -31,11 +31,13 @@ const ( ShadowKey = "shadow" // Pod - PodKey = "pod" - NodeGroupsKey = "node_groups" - ShmSizeKey = "shm_size" - PortKey = "port" - ContainersKey = "containers" + PodKey = "pod" + NodeGroupsKey = "node_groups" + ShmSizeKey = "shm_size" + PortKey = "port" + MaxConcurrencyKey = "max_concurrency" + MaxQueueLengthKey = "max_queue_length" + ContainersKey = "containers" // Containers ContainerNameKey = "name" @@ -72,8 +74,6 @@ const ( MinReplicasKey = "min_replicas" MaxReplicasKey = "max_replicas" InitReplicasKey = "init_replicas" - MaxQueueLengthKey = "max_queue_length" - MaxConcurrencyKey = "max_concurrency" TargetInFlightKey = "target_in_flight" WindowKey = "window" DownscaleStabilizationPeriodKey = "downscale_stabilization_period" @@ -89,10 +89,10 @@ const ( // K8s annotation EndpointAnnotationKey = "networking.cortex.dev/endpoint" + MaxConcurrencyAnnotationKey = "pod.cortex.dev/max-concurrency" + MaxQueueLengthAnnotationKey = "pod.cortex.dev/max-queue-length" MinReplicasAnnotationKey = "autoscaling.cortex.dev/min-replicas" MaxReplicasAnnotationKey = "autoscaling.cortex.dev/max-replicas" - MaxQueueLengthAnnotationKey = "autoscaling.cortex.dev/max-queue-length" - MaxConcurrencyAnnotationKey = "autoscaling.cortex.dev/max-concurrency" TargetInFlightAnnotationKey = "autoscaling.cortex.dev/target-in-flight" WindowAnnotationKey = "autoscaling.cortex.dev/window" DownscaleStabilizationPeriodAnnotationKey = "autoscaling.cortex.dev/downscale-stabilization-period" diff --git a/pkg/workloads/k8s.go b/pkg/workloads/k8s.go index 1777ffc1d4..4cae268412 100644 --- a/pkg/workloads/k8s.go +++ b/pkg/workloads/k8s.go @@ -127,9 +127,9 @@ func RealtimeProxyContainer(api spec.API) (kcore.Container, kcore.Volume) { "--user-port", s.Int32(*api.Pod.Port), "--max-concurrency", - s.Int32(int32(api.Autoscaling.MaxConcurrency)), + s.Int32(int32(api.Pod.MaxConcurrency)), "--max-queue-length", - s.Int32(int32(api.Autoscaling.MaxQueueLength)), + s.Int32(int32(api.Pod.MaxQueueLength)), "--cluster-config", consts.DefaultInClusterConfigPath, }, diff --git a/test/apis/task/iris-classifier-trainer/submit.py b/test/apis/task/iris-classifier-trainer/submit.py index 2b63d55d25..d39d8fa9f2 100644 --- a/test/apis/task/iris-classifier-trainer/submit.py +++ b/test/apis/task/iris-classifier-trainer/submit.py @@ -20,7 +20,7 @@ def main(): # get task endpoint cx = cortex.client(env_name) - task_endpoint = cx.get_api("trainer")["endpoint"] + task_endpoint = cx.get_api("iris-classifier-trainer")["endpoint"] # submit job job_spec = {"config": {"dest_s3_dir": dest_s3_dir}} From 89c0665c84447f68f4ed98bcea70ed55050cab8c Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Fri, 28 May 2021 20:47:24 +0300 Subject: [PATCH 34/82] CaaS - cleanup and fixes (#2195) --- .circleci/config.yml | 3 - Makefile | 7 - build/images.sh | 4 - build/test.sh | 10 - images/neuron-rtd/Dockerfile | 26 - images/neuron-rtd/entrypoint.sh | 22 - images/python-handler-cpu/Dockerfile | 64 - images/python-handler-gpu/Dockerfile | 66 - images/python-handler-inf/Dockerfile | 74 - images/tensorflow-handler/Dockerfile | 66 - images/tensorflow-serving-cpu/Dockerfile | 10 - images/tensorflow-serving-cpu/run.sh | 35 - images/tensorflow-serving-gpu/Dockerfile | 12 - images/tensorflow-serving-gpu/run.sh | 35 - images/tensorflow-serving-inf/Dockerfile | 26 - images/tensorflow-serving-inf/run.sh | 43 - .../tensorflow-serving-inf/supervisord.conf | 18 - images/tensorflow-serving-inf/template.conf | 23 - images/test/Dockerfile | 23 - images/test/run.sh | 29 - pkg/types/clusterconfig/cluster_config.go | 11 - python/serve/cortex_internal.requirements.txt | 11 - python/serve/cortex_internal/__init__.py | 13 - python/serve/cortex_internal/consts.py | 16 - python/serve/cortex_internal/lib/__init__.py | 13 - .../serve/cortex_internal/lib/api/__init__.py | 18 - .../cortex_internal/lib/api/async_api.py | 234 --- python/serve/cortex_internal/lib/api/batch.py | 222 --- .../serve/cortex_internal/lib/api/realtime.py | 375 ----- python/serve/cortex_internal/lib/api/task.py | 126 -- python/serve/cortex_internal/lib/api/utils.py | 338 ---- .../cortex_internal/lib/api/validations.py | 177 -- .../cortex_internal/lib/checkers/__init__.py | 13 - .../serve/cortex_internal/lib/checkers/pod.py | 32 - .../cortex_internal/lib/client/__init__.py | 13 - .../cortex_internal/lib/client/python.py | 405 ----- .../cortex_internal/lib/client/tensorflow.py | 515 ------ .../lib/concurrency/__init__.py | 16 - .../cortex_internal/lib/concurrency/files.py | 197 --- .../lib/concurrency/threading.py | 208 --- .../serve/cortex_internal/lib/exceptions.py | 53 - python/serve/cortex_internal/lib/log.py | 74 - python/serve/cortex_internal/lib/metrics.py | 72 - .../cortex_internal/lib/model/__init__.py | 44 - .../serve/cortex_internal/lib/model/cron.py | 1434 ----------------- .../serve/cortex_internal/lib/model/model.py | 588 ------- python/serve/cortex_internal/lib/model/tfs.py | 764 --------- .../serve/cortex_internal/lib/model/tree.py | 526 ------ .../serve/cortex_internal/lib/model/type.py | 169 -- .../cortex_internal/lib/model/validation.py | 529 ------ .../cortex_internal/lib/queue/__init__.py | 13 - python/serve/cortex_internal/lib/queue/sqs.py | 185 --- python/serve/cortex_internal/lib/signals.py | 31 - .../cortex_internal/lib/storage/__init__.py | 16 - .../cortex_internal/lib/storage/local.py | 127 -- .../serve/cortex_internal/lib/storage/s3.py | 250 --- python/serve/cortex_internal/lib/stringify.py | 55 - python/serve/cortex_internal/lib/telemetry.py | 102 -- .../lib/test/dynamic_batching_test.py | 120 -- .../cortex_internal/lib/test/util_test.py | 45 - .../cortex_internal/lib/type/__init__.py | 22 - python/serve/cortex_internal/lib/type/type.py | 59 - python/serve/cortex_internal/lib/util.py | 370 ----- .../serve/cortex_internal/serve/__init__.py | 13 - python/serve/cortex_internal/serve/serve.py | 361 ----- python/serve/cortex_internal/serve/wsgi.py | 17 - python/serve/init/bootloader.sh | 195 --- python/serve/init/export_env_vars.py | 147 -- .../serve/init/install-core-dependencies.sh | 36 - python/serve/init/script.py | 193 --- python/serve/init/templates/finish | 62 - python/serve/init/templates/run | 17 - python/serve/log_config.yaml | 57 - python/serve/nginx.conf.j2 | 169 -- python/serve/poll/readiness.sh | 24 - python/serve/serve.requirements.txt | 8 - python/serve/setup.py | 49 - python/serve/start/async_api.py | 176 -- python/serve/start/batch.py | 267 --- python/serve/start/server.py | 46 - python/serve/start/server_grpc.py | 272 ---- python/serve/start/task.py | 58 - .../image-classifier-resnet50/sample.json | 2 +- test/apis/realtime/sleep/cortex_cpu.yaml | 2 + .../apis/task/iris-classifier-trainer/main.py | 2 +- test/e2e/e2e/tests.py | 26 +- 86 files changed, 24 insertions(+), 11372 deletions(-) delete mode 100644 images/neuron-rtd/Dockerfile delete mode 100644 images/neuron-rtd/entrypoint.sh delete mode 100644 images/python-handler-cpu/Dockerfile delete mode 100644 images/python-handler-gpu/Dockerfile delete mode 100644 images/python-handler-inf/Dockerfile delete mode 100644 images/tensorflow-handler/Dockerfile delete mode 100644 images/tensorflow-serving-cpu/Dockerfile delete mode 100644 images/tensorflow-serving-cpu/run.sh delete mode 100644 images/tensorflow-serving-gpu/Dockerfile delete mode 100644 images/tensorflow-serving-gpu/run.sh delete mode 100644 images/tensorflow-serving-inf/Dockerfile delete mode 100644 images/tensorflow-serving-inf/run.sh delete mode 100644 images/tensorflow-serving-inf/supervisord.conf delete mode 100644 images/tensorflow-serving-inf/template.conf delete mode 100644 images/test/Dockerfile delete mode 100644 images/test/run.sh delete mode 100644 python/serve/cortex_internal.requirements.txt delete mode 100644 python/serve/cortex_internal/__init__.py delete mode 100644 python/serve/cortex_internal/consts.py delete mode 100644 python/serve/cortex_internal/lib/__init__.py delete mode 100644 python/serve/cortex_internal/lib/api/__init__.py delete mode 100644 python/serve/cortex_internal/lib/api/async_api.py delete mode 100644 python/serve/cortex_internal/lib/api/batch.py delete mode 100644 python/serve/cortex_internal/lib/api/realtime.py delete mode 100644 python/serve/cortex_internal/lib/api/task.py delete mode 100644 python/serve/cortex_internal/lib/api/utils.py delete mode 100644 python/serve/cortex_internal/lib/api/validations.py delete mode 100644 python/serve/cortex_internal/lib/checkers/__init__.py delete mode 100644 python/serve/cortex_internal/lib/checkers/pod.py delete mode 100644 python/serve/cortex_internal/lib/client/__init__.py delete mode 100644 python/serve/cortex_internal/lib/client/python.py delete mode 100644 python/serve/cortex_internal/lib/client/tensorflow.py delete mode 100644 python/serve/cortex_internal/lib/concurrency/__init__.py delete mode 100644 python/serve/cortex_internal/lib/concurrency/files.py delete mode 100644 python/serve/cortex_internal/lib/concurrency/threading.py delete mode 100644 python/serve/cortex_internal/lib/exceptions.py delete mode 100644 python/serve/cortex_internal/lib/log.py delete mode 100644 python/serve/cortex_internal/lib/metrics.py delete mode 100644 python/serve/cortex_internal/lib/model/__init__.py delete mode 100644 python/serve/cortex_internal/lib/model/cron.py delete mode 100644 python/serve/cortex_internal/lib/model/model.py delete mode 100644 python/serve/cortex_internal/lib/model/tfs.py delete mode 100644 python/serve/cortex_internal/lib/model/tree.py delete mode 100644 python/serve/cortex_internal/lib/model/type.py delete mode 100644 python/serve/cortex_internal/lib/model/validation.py delete mode 100644 python/serve/cortex_internal/lib/queue/__init__.py delete mode 100644 python/serve/cortex_internal/lib/queue/sqs.py delete mode 100644 python/serve/cortex_internal/lib/signals.py delete mode 100644 python/serve/cortex_internal/lib/storage/__init__.py delete mode 100644 python/serve/cortex_internal/lib/storage/local.py delete mode 100644 python/serve/cortex_internal/lib/storage/s3.py delete mode 100644 python/serve/cortex_internal/lib/stringify.py delete mode 100644 python/serve/cortex_internal/lib/telemetry.py delete mode 100644 python/serve/cortex_internal/lib/test/dynamic_batching_test.py delete mode 100644 python/serve/cortex_internal/lib/test/util_test.py delete mode 100644 python/serve/cortex_internal/lib/type/__init__.py delete mode 100644 python/serve/cortex_internal/lib/type/type.py delete mode 100644 python/serve/cortex_internal/lib/util.py delete mode 100644 python/serve/cortex_internal/serve/__init__.py delete mode 100644 python/serve/cortex_internal/serve/serve.py delete mode 100644 python/serve/cortex_internal/serve/wsgi.py delete mode 100755 python/serve/init/bootloader.sh delete mode 100644 python/serve/init/export_env_vars.py delete mode 100644 python/serve/init/install-core-dependencies.sh delete mode 100644 python/serve/init/script.py delete mode 100644 python/serve/init/templates/finish delete mode 100644 python/serve/init/templates/run delete mode 100644 python/serve/log_config.yaml delete mode 100644 python/serve/nginx.conf.j2 delete mode 100644 python/serve/poll/readiness.sh delete mode 100644 python/serve/serve.requirements.txt delete mode 100644 python/serve/setup.py delete mode 100644 python/serve/start/async_api.py delete mode 100644 python/serve/start/batch.py delete mode 100644 python/serve/start/server.py delete mode 100644 python/serve/start/server_grpc.py delete mode 100644 python/serve/start/task.py diff --git a/.circleci/config.yml b/.circleci/config.yml index c541a0054f..f32ce2ef1a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -111,9 +111,6 @@ jobs: - run: name: Go Tests command: make test-go - - run: - name: Python Tests - command: make test-python build-and-deploy: docker: diff --git a/Makefile b/Makefile index a5b6d7bdbe..798cc3e615 100644 --- a/Makefile +++ b/Makefile @@ -163,15 +163,8 @@ format: ######### test: - @./build/test.sh - -test-go: @./build/test.sh go -test-python: - @./build/test.sh python - - # build test api images # make sure you login with your quay credentials build-and-push-test-images: diff --git a/build/images.sh b/build/images.sh index 67257a750e..ca07560b3b 100644 --- a/build/images.sh +++ b/build/images.sh @@ -28,8 +28,6 @@ dev_images=( ) non_dev_images=( - "tensorflow-serving-cpu" - "tensorflow-serving-gpu" "cluster-autoscaler" "operator" "controller-manager" @@ -46,10 +44,8 @@ non_dev_images=( "kube-rbac-proxy" "grafana" "event-exporter" - "tensorflow-serving-inf" "metrics-server" "inferentia" - "neuron-rtd" "nvidia" "kubexit" ) diff --git a/build/test.sh b/build/test.sh index 0889579340..9f30168ec1 100755 --- a/build/test.sh +++ b/build/test.sh @@ -79,11 +79,6 @@ function run_go_tests() { ) } -function run_python_tests() { - docker build $ROOT -f $ROOT/images/test/Dockerfile -t cortexlabs/test - docker run cortexlabs/test -} - function run_e2e_tests() { if [ "$create_cluster" = "yes" ]; then pytest $ROOT/test/e2e/tests --config "$sub_cmd" @@ -94,11 +89,6 @@ function run_e2e_tests() { if [ "$cmd" = "go" ]; then run_go_tests -elif [ "$cmd" = "python" ]; then - run_python_tests elif [ "$cmd" = "e2e" ]; then run_e2e_tests -else - run_go_tests - run_python_tests fi diff --git a/images/neuron-rtd/Dockerfile b/images/neuron-rtd/Dockerfile deleted file mode 100644 index 4e375586cf..0000000000 --- a/images/neuron-rtd/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# Source: https://github.com/aws/aws-neuron-sdk/blob/master/docs/neuron-container-tools/docker-example/Dockerfile.neuron-rtd -FROM amazonlinux:2 - -RUN echo $'[neuron] \n\ -name=Neuron YUM Repository \n\ -baseurl=https://yum.repos.neuron.amazonaws.com \n\ -enabled=1' > /etc/yum.repos.d/neuron.repo - -RUN rpm --import https://yum.repos.neuron.amazonaws.com/GPG-PUB-KEY-AMAZON-AWS-NEURON.PUB - -RUN yum install -y \ - aws-neuron-tools-1.4.2.0 \ - aws-neuron-runtime-1.4.3.0 \ - procps-ng-3.3.10-26.amzn2.x86_64 \ - gzip \ - tar \ - curl - -ENV PATH="/opt/aws/neuron/bin:${PATH}" - -RUN ln -s /sock/neuron.sock /run/neuron.sock - -COPY images/neuron-rtd/entrypoint.sh ./entrypoint.sh -RUN chmod +x ./entrypoint.sh - -ENTRYPOINT ["./entrypoint.sh"] diff --git a/images/neuron-rtd/entrypoint.sh b/images/neuron-rtd/entrypoint.sh deleted file mode 100644 index 264f1ab480..0000000000 --- a/images/neuron-rtd/entrypoint.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -start_cmd="neuron-rtd -g unix:/sock/neuron.sock --log-console" -if [ -f "/mnt/kubexit" ]; then - start_cmd="/mnt/kubexit ${start_cmd}" -fi - -eval "${start_cmd}" diff --git a/images/python-handler-cpu/Dockerfile b/images/python-handler-cpu/Dockerfile deleted file mode 100644 index f1e7cf751c..0000000000 --- a/images/python-handler-cpu/Dockerfile +++ /dev/null @@ -1,64 +0,0 @@ -FROM ubuntu:18.04 - -RUN apt-get update -qq && apt-get install -y -q \ - build-essential \ - pkg-config \ - software-properties-common \ - curl \ - git \ - unzip \ - zlib1g-dev \ - locales \ - nginx=1.14.* \ - && apt-get clean -qq && rm -rf /var/lib/apt/lists/* - -RUN cd /tmp/ && \ - curl -L --output s6-overlay-amd64-installer "https://github.com/just-containers/s6-overlay/releases/download/v2.1.0.2/s6-overlay-amd64-installer" && \ - cd - && \ - chmod +x /tmp/s6-overlay-amd64-installer && /tmp/s6-overlay-amd64-installer / && rm /tmp/s6-overlay-amd64-installer - -ENV S6_BEHAVIOUR_IF_STAGE2_FAILS 2 - -RUN locale-gen en_US.UTF-8 -ENV LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LC_ALL=en_US.UTF-8 - -ENV PATH=/opt/conda/bin:$PATH -ENV PYTHONVERSION=3.6.9 -ENV CORTEX_IMAGE_TYPE=python-handler-cpu - -# conda needs an untainted base environment to function properly -# that's why a new separate conda environment is created -RUN curl "https://repo.anaconda.com/miniconda/Miniconda3-4.7.12.1-Linux-x86_64.sh" --output ~/miniconda.sh && \ - /bin/bash ~/miniconda.sh -b -p /opt/conda && \ - rm -rf ~/.cache ~/miniconda.sh - -# split the conda installations because the dev boxes have limited memory -RUN /opt/conda/bin/conda create -n env -c conda-forge python=$PYTHONVERSION pip=19.* && \ - /opt/conda/bin/conda clean -a && \ - ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". /opt/conda/etc/profile.d/conda.sh" > ~/.env && \ - echo "conda activate env" >> ~/.env && \ - echo "source ~/.env" >> ~/.bashrc - -ENV BASH_ENV=~/.env -SHELL ["/bin/bash", "-c"] - -COPY python/serve/serve.requirements.txt /src/cortex/serve/serve.requirements.txt -COPY python/serve/cortex_internal.requirements.txt /src/cortex/serve/cortex_internal.requirements.txt -RUN pip install --no-cache-dir \ - -r /src/cortex/serve/serve.requirements.txt \ - -r /src/cortex/serve/cortex_internal.requirements.txt - -COPY python/serve/init/install-core-dependencies.sh /usr/local/cortex/install-core-dependencies.sh -RUN chmod +x /usr/local/cortex/install-core-dependencies.sh - -COPY python/serve/ /src/cortex/serve -COPY python/client/ /src/cortex/client -ENV CORTEX_LOG_CONFIG_FILE /src/cortex/serve/log_config.yaml - -RUN pip install --no-deps /src/cortex/serve/ && \ - pip install /src/cortex/client && \ - rm -r /src/cortex/client -RUN mv /src/cortex/serve/init/bootloader.sh /etc/cont-init.d/bootloader.sh - -ENTRYPOINT ["/init"] diff --git a/images/python-handler-gpu/Dockerfile b/images/python-handler-gpu/Dockerfile deleted file mode 100644 index 7c8ee389b8..0000000000 --- a/images/python-handler-gpu/Dockerfile +++ /dev/null @@ -1,66 +0,0 @@ -ARG CUDA_VERSION=10.2 -ARG CUDNN=8 -FROM nvidia/cuda:$CUDA_VERSION-cudnn$CUDNN-runtime-ubuntu18.04 - -RUN apt-get update -qq && apt-get install -y -q \ - build-essential \ - pkg-config \ - software-properties-common \ - curl \ - git \ - unzip \ - zlib1g-dev \ - locales \ - nginx=1.14.* \ - && apt-get clean -qq && rm -rf /var/lib/apt/lists/* - -RUN cd /tmp/ && \ - curl -L --output s6-overlay-amd64-installer "https://github.com/just-containers/s6-overlay/releases/download/v2.1.0.2/s6-overlay-amd64-installer" && \ - cd - && \ - chmod +x /tmp/s6-overlay-amd64-installer && /tmp/s6-overlay-amd64-installer / && rm /tmp/s6-overlay-amd64-installer - -ENV S6_BEHAVIOUR_IF_STAGE2_FAILS 2 - -RUN locale-gen en_US.UTF-8 -ENV LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LC_ALL=en_US.UTF-8 - -ENV PATH=/opt/conda/bin:$PATH -ENV PYTHONVERSION=3.6.9 -ENV CORTEX_IMAGE_TYPE=python-handler-gpu - -# conda needs an untainted base environment to function properly -# that's why a new separate conda environment is created -RUN curl "https://repo.anaconda.com/miniconda/Miniconda3-4.7.12.1-Linux-x86_64.sh" --output ~/miniconda.sh && \ - /bin/bash ~/miniconda.sh -b -p /opt/conda && \ - rm -rf ~/.cache ~/miniconda.sh - -# split the conda installations because the dev boxes have limited memory -RUN /opt/conda/bin/conda create -n env -c conda-forge python=$PYTHONVERSION pip=19.* && \ - /opt/conda/bin/conda clean -a && \ - ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". /opt/conda/etc/profile.d/conda.sh" > ~/.env && \ - echo "conda activate env" >> ~/.env && \ - echo "source ~/.env" >> ~/.bashrc - -ENV BASH_ENV=~/.env -SHELL ["/bin/bash", "-c"] - -COPY python/serve/serve.requirements.txt /src/cortex/serve/serve.requirements.txt -COPY python/serve/cortex_internal.requirements.txt /src/cortex/serve/cortex_internal.requirements.txt -RUN pip install --no-cache-dir \ - -r /src/cortex/serve/serve.requirements.txt \ - -r /src/cortex/serve/cortex_internal.requirements.txt - -COPY python/serve/init/install-core-dependencies.sh /usr/local/cortex/install-core-dependencies.sh -RUN chmod +x /usr/local/cortex/install-core-dependencies.sh - -COPY python/serve/ /src/cortex/serve -COPY python/client/ /src/cortex/client -ENV CORTEX_LOG_CONFIG_FILE /src/cortex/serve/log_config.yaml - -RUN pip install --no-deps /src/cortex/serve/ && \ - pip install /src/cortex/client && \ - rm -r /src/cortex/client -RUN mv /src/cortex/serve/init/bootloader.sh /etc/cont-init.d/bootloader.sh - -ENTRYPOINT ["/init"] diff --git a/images/python-handler-inf/Dockerfile b/images/python-handler-inf/Dockerfile deleted file mode 100644 index 726c5ab93c..0000000000 --- a/images/python-handler-inf/Dockerfile +++ /dev/null @@ -1,74 +0,0 @@ -FROM ubuntu:18.04 - -RUN apt-get update -qq && apt-get install -y -q \ - wget \ - gnupg && \ - echo "deb https://apt.repos.neuron.amazonaws.com bionic main" >> /etc/apt/sources.list.d/neuron.list && \ - wget -qO - https://apt.repos.neuron.amazonaws.com/GPG-PUB-KEY-AMAZON-AWS-NEURON.PUB | apt-key add - && \ - apt-get update -qq && apt-get install -y -q \ - aws-neuron-tools=1.4.2.0 \ - aws-neuron-runtime=1.4.3.0 && \ - apt-get clean -qq && rm -rf /var/lib/apt/lists/* - -RUN wget -P /tmp/ https://github.com/just-containers/s6-overlay/releases/download/v2.1.0.2/s6-overlay-amd64-installer && \ - chmod +x /tmp/s6-overlay-amd64-installer && /tmp/s6-overlay-amd64-installer / && rm /tmp/s6-overlay-amd64-installer - -ENV S6_BEHAVIOUR_IF_STAGE2_FAILS 2 - -ENV PATH=/opt/aws/neuron/bin/:$PATH - -RUN apt-get update -qq && apt-get install -y -q \ - build-essential \ - pkg-config \ - software-properties-common \ - curl \ - git \ - unzip \ - zlib1g-dev \ - locales \ - nginx=1.14.* \ - && apt-get clean -qq && rm -rf /var/lib/apt/lists/* - -RUN locale-gen en_US.UTF-8 -ENV LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LC_ALL=en_US.UTF-8 - -ENV PATH=/opt/conda/bin:$PATH -ENV PYTHONVERSION=3.6.9 -ENV CORTEX_IMAGE_TYPE=python-handler-inf - -# conda needs an untainted base environment to function properly -# that's why a new separate conda environment is created -RUN curl "https://repo.anaconda.com/miniconda/Miniconda3-4.7.12.1-Linux-x86_64.sh" --output ~/miniconda.sh && \ - /bin/bash ~/miniconda.sh -b -p /opt/conda && \ - rm -rf ~/.cache ~/miniconda.sh - -# split the conda installations because the dev boxes have limited memory -RUN /opt/conda/bin/conda create -n env -c conda-forge python=$PYTHONVERSION pip=19.* && \ - /opt/conda/bin/conda clean -a && \ - ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". /opt/conda/etc/profile.d/conda.sh" > ~/.env && \ - echo "conda activate env" >> ~/.env && \ - echo "source ~/.env" >> ~/.bashrc - -ENV BASH_ENV=~/.env -SHELL ["/bin/bash", "-c"] - -COPY python/serve/serve.requirements.txt /src/cortex/serve/serve.requirements.txt -COPY python/serve/cortex_internal.requirements.txt /src/cortex/serve/cortex_internal.requirements.txt -RUN pip install --no-cache-dir \ - -r /src/cortex/serve/serve.requirements.txt \ - -r /src/cortex/serve/cortex_internal.requirements.txt - -COPY python/serve/init/install-core-dependencies.sh /usr/local/cortex/install-core-dependencies.sh -RUN chmod +x /usr/local/cortex/install-core-dependencies.sh - -COPY python/serve/ /src/cortex/serve -COPY python/client/ /src/cortex/client -ENV CORTEX_LOG_CONFIG_FILE /src/cortex/serve/log_config.yaml - -RUN pip install --no-deps /src/cortex/serve/ && \ - pip install /src/cortex/client && \ - rm -r /src/cortex/client -RUN mv /src/cortex/serve/init/bootloader.sh /etc/cont-init.d/bootloader.sh - -ENTRYPOINT ["/init"] diff --git a/images/tensorflow-handler/Dockerfile b/images/tensorflow-handler/Dockerfile deleted file mode 100644 index 13d9cff3e3..0000000000 --- a/images/tensorflow-handler/Dockerfile +++ /dev/null @@ -1,66 +0,0 @@ -FROM ubuntu:18.04 - -RUN apt-get update -qq && apt-get install -y -q \ - build-essential \ - pkg-config \ - software-properties-common \ - curl \ - git \ - unzip \ - zlib1g-dev \ - locales \ - nginx=1.14.* \ - && apt-get clean -qq && rm -rf /var/lib/apt/lists/* - -RUN cd /tmp/ && \ - curl -L --output s6-overlay-amd64-installer "https://github.com/just-containers/s6-overlay/releases/download/v2.1.0.2/s6-overlay-amd64-installer" && \ - cd - && \ - chmod +x /tmp/s6-overlay-amd64-installer && /tmp/s6-overlay-amd64-installer / && rm /tmp/s6-overlay-amd64-installer - -ENV S6_BEHAVIOUR_IF_STAGE2_FAILS 2 - -RUN locale-gen en_US.UTF-8 -ENV LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LC_ALL=en_US.UTF-8 - -ENV PATH=/opt/conda/bin:$PATH -ENV PYTHONVERSION=3.6.9 -ENV CORTEX_IMAGE_TYPE=tensorflow-handler - -# conda needs an untainted base environment to function properly -# that's why a new separate conda environment is created -RUN curl "https://repo.anaconda.com/miniconda/Miniconda3-4.7.12.1-Linux-x86_64.sh" --output ~/miniconda.sh && \ - /bin/bash ~/miniconda.sh -b -p /opt/conda && \ - rm -rf ~/.cache ~/miniconda.sh - -# split the conda installations because the dev boxes have limited memory -RUN /opt/conda/bin/conda create -n env -c conda-forge python=$PYTHONVERSION pip=19.* && \ - /opt/conda/bin/conda clean -a && \ - ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". /opt/conda/etc/profile.d/conda.sh" > ~/.env && \ - echo "conda activate env" >> ~/.env && \ - echo "source ~/.env" >> ~/.bashrc - -ENV BASH_ENV=~/.env -SHELL ["/bin/bash", "-c"] - -COPY python/serve/serve.requirements.txt /src/cortex/serve/serve.requirements.txt -COPY python/serve/cortex_internal.requirements.txt /src/cortex/serve/cortex_internal.requirements.txt -RUN pip install --no-cache-dir \ - -r /src/cortex/serve/serve.requirements.txt \ - -r /src/cortex/serve/cortex_internal.requirements.txt \ - tensorflow-cpu==2.3.0 \ - tensorflow-serving-api==2.3.0 - -COPY python/serve/init/install-core-dependencies.sh /usr/local/cortex/install-core-dependencies.sh -RUN chmod +x /usr/local/cortex/install-core-dependencies.sh - -COPY python/serve/ /src/cortex/serve -COPY python/client/ /src/cortex/client -ENV CORTEX_LOG_CONFIG_FILE /src/cortex/serve/log_config.yaml - -RUN pip install --no-deps /src/cortex/serve/ && \ - pip install /src/cortex/client && \ - rm -r /src/cortex/client -RUN mv /src/cortex/serve/init/bootloader.sh /etc/cont-init.d/bootloader.sh - -ENTRYPOINT ["/init"] diff --git a/images/tensorflow-serving-cpu/Dockerfile b/images/tensorflow-serving-cpu/Dockerfile deleted file mode 100644 index 396bfec923..0000000000 --- a/images/tensorflow-serving-cpu/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM tensorflow/serving:2.3.0 - -RUN apt-get update -qq && apt-get install -y -q \ - curl \ - && apt-get clean -qq && rm -rf /var/lib/apt/lists/* - -COPY images/tensorflow-serving-cpu/run.sh /src/ -RUN chmod +x /src/run.sh - -ENTRYPOINT ["/src/run.sh"] diff --git a/images/tensorflow-serving-cpu/run.sh b/images/tensorflow-serving-cpu/run.sh deleted file mode 100644 index ddb9b66cfc..0000000000 --- a/images/tensorflow-serving-cpu/run.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -# empty model config list -mkdir /etc/tfs -echo "model_config_list {}" > /etc/tfs/model_config_server.conf - -# configure batching if specified -if [[ -n ${TF_MAX_BATCH_SIZE} && -n ${TF_BATCH_TIMEOUT_MICROS} ]]; then - echo "max_batch_size { value: ${TF_MAX_BATCH_SIZE} }" > /etc/tfs/batch_config.conf - echo "batch_timeout_micros { value: ${TF_BATCH_TIMEOUT_MICROS} }" >> /etc/tfs/batch_config.conf - echo "num_batch_threads { value: ${TF_NUM_BATCHED_THREADS} }" >> /etc/tfs/batch_config.conf -fi - -# launch TFS -if [ -f "/mnt/kubexit" ]; then - /mnt/kubexit tensorflow_model_server "$@" -else - tensorflow_model_server "$@" -fi diff --git a/images/tensorflow-serving-gpu/Dockerfile b/images/tensorflow-serving-gpu/Dockerfile deleted file mode 100644 index 745e29a56a..0000000000 --- a/images/tensorflow-serving-gpu/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM tensorflow/serving:2.3.0-gpu - -RUN apt-get update -qq && apt-get install -y --no-install-recommends -q \ - libnvinfer6=6.0.1-1+cuda10.1 \ - libnvinfer-plugin6=6.0.1-1+cuda10.1 \ - curl \ - && apt-get clean -qq && rm -rf /var/lib/apt/lists/* - -COPY images/tensorflow-serving-gpu/run.sh /src/ -RUN chmod +x /src/run.sh - -ENTRYPOINT ["/src/run.sh"] diff --git a/images/tensorflow-serving-gpu/run.sh b/images/tensorflow-serving-gpu/run.sh deleted file mode 100644 index ddb9b66cfc..0000000000 --- a/images/tensorflow-serving-gpu/run.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -# empty model config list -mkdir /etc/tfs -echo "model_config_list {}" > /etc/tfs/model_config_server.conf - -# configure batching if specified -if [[ -n ${TF_MAX_BATCH_SIZE} && -n ${TF_BATCH_TIMEOUT_MICROS} ]]; then - echo "max_batch_size { value: ${TF_MAX_BATCH_SIZE} }" > /etc/tfs/batch_config.conf - echo "batch_timeout_micros { value: ${TF_BATCH_TIMEOUT_MICROS} }" >> /etc/tfs/batch_config.conf - echo "num_batch_threads { value: ${TF_NUM_BATCHED_THREADS} }" >> /etc/tfs/batch_config.conf -fi - -# launch TFS -if [ -f "/mnt/kubexit" ]; then - /mnt/kubexit tensorflow_model_server "$@" -else - tensorflow_model_server "$@" -fi diff --git a/images/tensorflow-serving-inf/Dockerfile b/images/tensorflow-serving-inf/Dockerfile deleted file mode 100644 index 1a09658144..0000000000 --- a/images/tensorflow-serving-inf/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# Source: https://github.com/aws/aws-neuron-sdk/blob/master/docs/neuron-container-tools/docker-example/Dockerfile.tf-serving -FROM ubuntu:18.04 - -RUN apt-get update -qq && apt-get install -y -q \ - gettext-base \ - supervisor \ - curl \ - wget \ - netcat \ - gnupg && \ - echo "deb https://apt.repos.neuron.amazonaws.com bionic main" >> /etc/apt/sources.list.d/neuron.list && \ - wget -qO - https://apt.repos.neuron.amazonaws.com/GPG-PUB-KEY-AMAZON-AWS-NEURON.PUB | apt-key add - && \ - apt-get update -qq && apt-get install -y -q \ - aws-neuron-tools=1.4.2.0 \ - aws-neuron-runtime=1.4.3.0 \ - tensorflow-model-server-neuron=1.15.0.1.2.2.0 && \ - apt-get clean -qq && rm -rf /var/lib/apt/lists/* - -ENV PATH=/opt/aws/neuron/bin/:$PATH - -COPY images/tensorflow-serving-inf/run.sh /src/ -COPY images/tensorflow-serving-inf/supervisord.conf /tmp/supervisord.conf -COPY images/tensorflow-serving-inf/template.conf /tmp/template.conf -RUN chmod +x /src/run.sh - -ENTRYPOINT ["/src/run.sh"] diff --git a/images/tensorflow-serving-inf/run.sh b/images/tensorflow-serving-inf/run.sh deleted file mode 100644 index fbeed19c05..0000000000 --- a/images/tensorflow-serving-inf/run.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -# empty model config list -mkdir /etc/tfs -echo "model_config_list {}" > /etc/tfs/model_config_server.conf - -# configure batching if specified -if [[ -n ${TF_MAX_BATCH_SIZE} && -n ${TF_BATCH_TIMEOUT_MICROS} ]]; then - echo "max_batch_size { value: ${TF_MAX_BATCH_SIZE} }" > /etc/tfs/batch_config.conf - echo "batch_timeout_micros { value: ${TF_BATCH_TIMEOUT_MICROS} }" >> /etc/tfs/batch_config.conf - echo "num_batch_threads { value: ${TF_NUM_BATCHED_THREADS} }" >> /etc/tfs/batch_config.conf - export TF_EXTRA_CMD_ARGS="--enable_batching=true --batching_parameters_file=/etc/tfs/batch_config.conf" -fi - -start_cmd=tensorflow_model_server_neuron -if [ -f "/mnt/kubexit" ]; then - start_cmd="/mnt/kubexit $start_cmd" -fi - -# spin up multiple process to handle different NCGs -for i in $(seq 1 $TF_PROCESSES); do - echo -e "\n\n" >> /tmp/supervisord.conf - start_cmd=$start_cmd process=$i port=$((CORTEX_TF_BASE_SERVING_PORT+i-1)) envsubst < /tmp/template.conf >> /tmp/supervisord.conf -done - -mv /tmp/supervisord.conf /etc/supervisor/conf.d/supervisord.conf -/usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf diff --git a/images/tensorflow-serving-inf/supervisord.conf b/images/tensorflow-serving-inf/supervisord.conf deleted file mode 100644 index d4dbf02535..0000000000 --- a/images/tensorflow-serving-inf/supervisord.conf +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[supervisord] -nodaemon=true diff --git a/images/tensorflow-serving-inf/template.conf b/images/tensorflow-serving-inf/template.conf deleted file mode 100644 index 2e35145080..0000000000 --- a/images/tensorflow-serving-inf/template.conf +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[program:tensorflow-$process] -command=$start_cmd --port=$port --model_config_file=$TF_EMPTY_MODEL_CONFIG --max_num_load_retries=$TF_MAX_NUM_LOAD_RETRIES --load_retry_interval_micros=$TF_LOAD_RETRY_INTERVAL_MICROS --grpc_channel_arguments=$TF_GRPC_MAX_CONCURRENT_STREAMS $TF_EXTRA_CMD_ARGS -stdout_logfile=/dev/fd/1 -stdout_logfile_maxbytes=0 -redirect_stderr=true -killasgroup=true -stopasgroup=true diff --git a/images/test/Dockerfile b/images/test/Dockerfile deleted file mode 100644 index 552ca19df0..0000000000 --- a/images/test/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM python:3.6 - -COPY python/serve/cortex_internal.requirements.txt /src/cortex/serve/cortex_internal.requirements.txt - -RUN pip install --upgrade pip && \ - pip install --no-cache-dir -r /src/cortex/serve/cortex_internal.requirements.txt && \ - pip install pytest mock && \ - rm -rf /root/.cache/pip* - -COPY python /src/cortex -COPY images/test/run.sh /src/run.sh - -COPY python/serve/log_config.yaml /src/cortex/serve/log_config.yaml -ENV CORTEX_LOG_LEVEL DEBUG -ENV CORTEX_LOG_CONFIG_FILE /src/cortex/serve/log_config.yaml - -RUN pip install --no-deps /src/cortex/serve/ && \ - rm -rf /root/.cache/pip* - -WORKDIR /src/cortex/serve/cortex_internal/ - -ENTRYPOINT ["/bin/bash"] -CMD ["/src/run.sh"] diff --git a/images/test/run.sh b/images/test/run.sh deleted file mode 100644 index 041aa905da..0000000000 --- a/images/test/run.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -err=0 -trap 'err=1' ERR - -function substitute_env_vars() { - file_to_run_substitution=$1 - python -c "from cortex_internal.lib import util; import os; util.expand_environment_vars_on_file('$file_to_run_substitution')" -} - -substitute_env_vars $CORTEX_LOG_CONFIG_FILE -pytest lib/test - -test $err = 0 diff --git a/pkg/types/clusterconfig/cluster_config.go b/pkg/types/clusterconfig/cluster_config.go index ab1e44259e..a60609783e 100644 --- a/pkg/types/clusterconfig/cluster_config.go +++ b/pkg/types/clusterconfig/cluster_config.go @@ -99,7 +99,6 @@ type CoreConfig struct { ImageClusterAutoscaler string `json:"image_cluster_autoscaler" yaml:"image_cluster_autoscaler"` ImageMetricsServer string `json:"image_metrics_server" yaml:"image_metrics_server"` ImageInferentia string `json:"image_inferentia" yaml:"image_inferentia"` - ImageNeuronRTD string `json:"image_neuron_rtd" yaml:"image_neuron_rtd"` ImageNvidia string `json:"image_nvidia" yaml:"image_nvidia"` ImageFluentBit string `json:"image_fluent_bit" yaml:"image_fluent_bit"` ImageIstioProxy string `json:"image_istio_proxy" yaml:"image_istio_proxy"` @@ -387,13 +386,6 @@ var CoreConfigStructFieldValidations = []*cr.StructFieldValidation{ Validator: validateImageVersion, }, }, - { - StructField: "ImageNeuronRTD", - StringValidation: &cr.StringValidation{ - Default: consts.DefaultRegistry() + "/neuron-rtd:" + consts.CortexVersion, - Validator: validateImageVersion, - }, - }, { StructField: "ImageNvidia", StringValidation: &cr.StringValidation{ @@ -1370,9 +1362,6 @@ func (cc *CoreConfig) TelemetryEvent() map[string]interface{} { if !strings.HasPrefix(cc.ImageInferentia, "cortexlabs/") { event["image_inferentia._is_custom"] = true } - if !strings.HasPrefix(cc.ImageNeuronRTD, "cortexlabs/") { - event["image_neuron_rtd._is_custom"] = true - } if !strings.HasPrefix(cc.ImageNvidia, "cortexlabs/") { event["image_nvidia._is_custom"] = true } diff --git a/python/serve/cortex_internal.requirements.txt b/python/serve/cortex_internal.requirements.txt deleted file mode 100644 index 9b03ad49fc..0000000000 --- a/python/serve/cortex_internal.requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -grpcio==1.36.0 -boto3==1.14.53 -datadog==0.39.0 -dill>=0.3.1.1 -msgpack==1.0.0 -jinja2==2.11.3 -fastapi==0.61.1 -pyyaml==5.4 -python-json-logger==2.0.1 -sentry-sdk==0.20.2 -asgiref==3.3.4 diff --git a/python/serve/cortex_internal/__init__.py b/python/serve/cortex_internal/__init__.py deleted file mode 100644 index dcd1d9ae2f..0000000000 --- a/python/serve/cortex_internal/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/python/serve/cortex_internal/consts.py b/python/serve/cortex_internal/consts.py deleted file mode 100644 index eb902461bb..0000000000 --- a/python/serve/cortex_internal/consts.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -SINGLE_MODEL_NAME = "_cortex_default" -INFERENTIA_NEURON_SOCKET = "/sock/neuron.sock" diff --git a/python/serve/cortex_internal/lib/__init__.py b/python/serve/cortex_internal/lib/__init__.py deleted file mode 100644 index dcd1d9ae2f..0000000000 --- a/python/serve/cortex_internal/lib/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/python/serve/cortex_internal/lib/api/__init__.py b/python/serve/cortex_internal/lib/api/__init__.py deleted file mode 100644 index 07c39562e8..0000000000 --- a/python/serve/cortex_internal/lib/api/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from cortex_internal.lib.api.batch import BatchAPI -from cortex_internal.lib.api.realtime import RealtimeAPI -from cortex_internal.lib.api.task import TaskAPI -from cortex_internal.lib.api.utils import get_spec, model_downloader, DynamicBatcher, CortexMetrics diff --git a/python/serve/cortex_internal/lib/api/async_api.py b/python/serve/cortex_internal/lib/api/async_api.py deleted file mode 100644 index 0c2c734946..0000000000 --- a/python/serve/cortex_internal/lib/api/async_api.py +++ /dev/null @@ -1,234 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import imp -import inspect -import json -import os -from copy import deepcopy -from http import HTTPStatus -from typing import Any, Dict, Union - -import datadog -import dill - -from cortex_internal.lib.api.validations import validate_class_impl -from cortex_internal.lib.client.tensorflow import TensorFlowClient -from cortex_internal.lib.exceptions import CortexException, UserException, UserRuntimeException -from cortex_internal.lib.metrics import MetricsClient -from cortex_internal.lib.storage import S3 -from cortex_internal.lib.type import ( - handler_type_from_api_spec, - TensorFlowHandlerType, - TensorFlowNeuronHandlerType, - PythonHandlerType, -) - -ASYNC_PYTHON_HANDLER_VALIDATION = { - "required": [ - { - "name": "__init__", - "required_args": ["self", "config"], - "optional_args": ["metrics_client"], - }, - { - "name": "handle_async", - "required_args": ["self"], - "optional_args": ["payload", "request_id"], - }, - ], -} - -ASYNC_TENSORFLOW_HANDLER_VALIDATION = { - "required": [ - { - "name": "__init__", - "required_args": ["self", "tensorflow_client", "config"], - "optional_args": ["metrics_client"], - }, - { - "name": "handle_async", - "required_args": ["self"], - "optional_args": ["payload", "request_id"], - }, - ], -} - - -class AsyncAPI: - def __init__( - self, - api_spec: Dict[str, Any], - storage: S3, - storage_path: str, - model_dir: str, - statsd_host: str, - statsd_port: int, - lock_dir: str = "/run/cron", - ): - self.api_spec = api_spec - self.storage = storage - self.storage_path = storage_path - self.path = api_spec["handler"]["path"] - self.config = api_spec["handler"].get("config", {}) - self.type = handler_type_from_api_spec(api_spec) - self.model_dir = model_dir - self.lock_dir = lock_dir - - datadog.initialize(statsd_host=statsd_host, statsd_port=statsd_port) - self.__statsd = datadog.statsd - - @property - def statsd(self): - return self.__statsd - - def update_status(self, request_id: str, status: str): - self.storage.put_str("", f"{self.storage_path}/{request_id}/status/{status}") - - def upload_result(self, request_id: str, result: Dict[str, Any]): - if not isinstance(result, dict): - raise UserRuntimeException( - f"user response must be json serializable dictionary, got {type(result)} instead" - ) - - try: - result_json = json.dumps(result) - except Exception: - raise UserRuntimeException("user response is not json serializable") - - self.storage.put_str(result_json, f"{self.storage_path}/{request_id}/result.json") - - def get_payload(self, request_id: str) -> Union[Dict, str, bytes]: - key = f"{self.storage_path}/{request_id}/payload" - obj = self.storage.get_object(key) - status_code = obj["ResponseMetadata"]["HTTPStatusCode"] - if status_code != HTTPStatus.OK: - raise CortexException( - f"failed to retrieve async payload (request_id: {request_id}, status_code: {status_code})" - ) - - content_type: str = obj["ResponseMetadata"]["HTTPHeaders"]["content-type"] - payload_bytes: bytes = obj["Body"].read() - - # decode payload - if content_type.startswith("application/json"): - try: - return json.loads(payload_bytes) - except Exception as err: - raise UserRuntimeException( - f"the uploaded payload, with content-type {content_type}, could not be decoded to JSON" - ) from err - elif content_type.startswith("text/plain"): - try: - return payload_bytes.decode("utf-8") - except Exception as err: - raise UserRuntimeException( - f"the uploaded payload, with content-type {content_type}, could not be decoded to a utf-8 string" - ) from err - else: - return payload_bytes - - def delete_payload(self, request_id: str): - key = f"{self.storage_path}/{request_id}/payload" - self.storage.delete(key) - - def initialize_impl( - self, - project_dir: str, - metrics_client: MetricsClient, - tf_serving_host: str = None, - tf_serving_port: str = None, - ): - handler_impl = self._get_impl(project_dir) - constructor_args = inspect.getfullargspec(handler_impl.__init__).args - config = deepcopy(self.config) - - args = {} - if "config" in constructor_args: - args["config"] = config - if "metrics_client" in constructor_args: - args["metrics_client"] = metrics_client - - if self.type in [TensorFlowHandlerType, TensorFlowNeuronHandlerType]: - tf_serving_address = tf_serving_host + ":" + tf_serving_port - tf_client = TensorFlowClient( - tf_serving_url=tf_serving_address, - api_spec=self.api_spec, - ) - tf_client.sync_models(lock_dir=self.lock_dir) - args["tensorflow_client"] = tf_client - - try: - handler = handler_impl(**args) - except Exception as e: - raise UserRuntimeException(self.path, "__init__", str(e)) from e - - return handler - - def _get_impl(self, project_dir: str): - target_class_name = "Handler" - if self.type in [TensorFlowHandlerType, TensorFlowNeuronHandlerType]: - validations = ASYNC_TENSORFLOW_HANDLER_VALIDATION - elif self.type == PythonHandlerType: - validations = ASYNC_PYTHON_HANDLER_VALIDATION - else: - raise CortexException(f"invalid handler type: {self.type}") - - try: - impl = self._read_impl( - "cortex_async_handler", os.path.join(project_dir, self.path), target_class_name - ) - except CortexException as e: - e.wrap("error in " + self.path) - raise - - try: - validate_class_impl(impl, validations) - except CortexException as e: - e.wrap("error in " + self.path) - raise - - return impl - - @staticmethod - def _read_impl(module_name: str, impl_path: str, target_class_name): - if impl_path.endswith(".pickle"): - try: - with open(impl_path, "rb") as pickle_file: - return dill.load(pickle_file) - except Exception as e: - raise UserException("unable to load pickle", str(e)) from e - - try: - impl = imp.load_source(module_name, impl_path) - except Exception as e: - raise UserException(str(e)) from e - - classes = inspect.getmembers(impl, inspect.isclass) - - handler_class = None - for class_df in classes: - if class_df[0] == target_class_name: - if handler_class is not None: - raise UserException( - f"multiple definitions for {target_class_name} class found; please check " - f"your imports and class definitions and ensure that there is only one " - f"handler class definition" - ) - handler_class = class_df[1] - - if handler_class is None: - raise UserException(f"{target_class_name} class is not defined") - - return handler_class diff --git a/python/serve/cortex_internal/lib/api/batch.py b/python/serve/cortex_internal/lib/api/batch.py deleted file mode 100644 index 4993e335ca..0000000000 --- a/python/serve/cortex_internal/lib/api/batch.py +++ /dev/null @@ -1,222 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import imp -import inspect -import os -from copy import deepcopy -from typing import Any, Dict, Optional - -import dill -from datadog.dogstatsd.base import DogStatsd - -from cortex_internal.lib import util -from cortex_internal.lib.api.utils import CortexMetrics -from cortex_internal.lib.api.validations import ( - are_models_specified, - validate_class_impl, -) -from cortex_internal.lib.client.tensorflow import TensorFlowClient -from cortex_internal.lib.exceptions import CortexException, UserException, UserRuntimeException -from cortex_internal.lib.metrics import MetricsClient -from cortex_internal.lib.model import TFSAPIServingThreadUpdater -from cortex_internal.lib.type import ( - PythonHandlerType, - TensorFlowNeuronHandlerType, - TensorFlowHandlerType, - handler_type_from_api_spec, -) - -PYTHON_CLASS_VALIDATION = { - "required": [ - { - "name": "__init__", - "required_args": ["self", "config"], - "optional_args": ["job_spec", "metrics_client"], - }, - { - "name": "handle_batch", - "required_args": ["self"], - "optional_args": ["payload", "batch_id"], - }, - ], - "optional": [ - {"name": "on_job_complete", "required_args": ["self"]}, - ], -} - -TENSORFLOW_CLASS_VALIDATION = { - "required": [ - { - "name": "__init__", - "required_args": ["self", "config", "tensorflow_client"], - "optional_args": ["job_spec", "metrics_client"], - }, - { - "name": "handle_batch", - "required_args": ["self"], - "optional_args": ["payload", "batch_id"], - }, - ], - "optional": [ - {"name": "on_job_complete", "required_args": ["self"]}, - ], -} - - -class BatchAPI: - """ - Class to validate/load the handler class (Handler). - Also makes the specified models in cortex.yaml available to the handler's implementation. - """ - - def __init__(self, api_spec: dict, statsd_client: DogStatsd, model_dir: str): - """ - Args: - api_spec: API configuration. - model_dir: Where the models are stored on disk. - """ - - self.metrics = CortexMetrics(statsd_client, api_spec) - - self.name = api_spec["name"] - self.type = handler_type_from_api_spec(api_spec) - self.path = api_spec["handler"]["path"] - self.config = api_spec["handler"].get("config", {}) - self.protobuf_path = api_spec["handler"].get("protobuf_path") - self.api_spec = api_spec - - def initialize_client( - self, tf_serving_host: Optional[str] = None, tf_serving_port: Optional[str] = None - ) -> TensorFlowClient: - """ - Initialize client that gives access to models specified in the API spec (cortex.yaml). - Only applies when models are provided in the API spec. - - Args: - tf_serving_host: Host of TF serving server. To be only used when the TensorFlow type is used. - tf_serving_port: Port of TF serving server. To be only used when the TensorFlow type is used. - - Return: - The client for the respective handler type. - """ - - client = None - - if are_models_specified(self.api_spec) and self.type in [ - TensorFlowHandlerType, - TensorFlowNeuronHandlerType, - ]: - tf_serving_address = tf_serving_host + ":" + tf_serving_port - client = TensorFlowClient( - tf_serving_address, - self.api_spec, - ) - TFSAPIServingThreadUpdater(interval=5.0, client=client).start() - - return client - - def initialize_impl( - self, - project_dir: str, - client: TensorFlowClient, - metrics_client: MetricsClient, - job_spec: Optional[Dict[str, Any]] = None, - ): - """ - Initialize handler class as provided by the user. - - job_spec is a dictionary when the "kind" of the API is set to "BatchAPI". Otherwise, it's None. - - Can raise UserRuntimeException/UserException/CortexException. - """ - - # build args - class_impl = self.class_impl(project_dir) - constructor_args = inspect.getfullargspec(class_impl.__init__).args - config = deepcopy(self.config) - args = {} - if job_spec is not None and job_spec.get("config") is not None: - util.merge_dicts_in_place_overwrite(config, job_spec["config"]) - if "config" in constructor_args: - args["config"] = config - if "job_spec" in constructor_args: - args["job_spec"] = job_spec - if "metrics_client" in constructor_args: - args["metrics_client"] = metrics_client - - # initialize handler class - try: - if self.type == PythonHandlerType: - initialized_impl = class_impl(**args) - if self.type in [TensorFlowHandlerType, TensorFlowNeuronHandlerType]: - args["tensorflow_client"] = client - initialized_impl = class_impl(**args) - except Exception as e: - raise UserRuntimeException(self.path, "__init__", str(e)) from e - - return initialized_impl - - def class_impl(self, project_dir): - """Can only raise UserException/CortexException exceptions""" - target_class_name = "Handler" - if self.type in [TensorFlowHandlerType, TensorFlowNeuronHandlerType]: - validations = TENSORFLOW_CLASS_VALIDATION - elif self.type == PythonHandlerType: - validations = PYTHON_CLASS_VALIDATION - else: - raise CortexException(f"invalid handler type: {self.type}") - - try: - handler_class = self._get_class_impl( - "cortex_handler", os.path.join(project_dir, self.path), target_class_name - ) - except Exception as e: - e.wrap("error in " + self.path) - raise - - try: - validate_class_impl(handler_class, validations) - except Exception as e: - e.wrap("error in " + self.path) - raise - return handler_class - - def _get_class_impl(self, module_name, impl_path, target_class_name): - """Can only raise UserException exception""" - if impl_path.endswith(".pickle"): - try: - with open(impl_path, "rb") as pickle_file: - return dill.load(pickle_file) - except Exception as e: - raise UserException("unable to load pickle", str(e)) from e - - try: - impl = imp.load_source(module_name, impl_path) - except Exception as e: - raise UserException(str(e)) from e - - classes = inspect.getmembers(impl, inspect.isclass) - handler_class = None - for class_df in classes: - if class_df[0] == target_class_name: - if handler_class is not None: - raise UserException( - f"multiple definitions for {target_class_name} class found; please check your imports and class definitions and ensure that there is only one handler class definition" - ) - handler_class = class_df[1] - if handler_class is None: - raise UserException(f"{target_class_name} class is not defined") - - return handler_class diff --git a/python/serve/cortex_internal/lib/api/realtime.py b/python/serve/cortex_internal/lib/api/realtime.py deleted file mode 100644 index c5f1baefb5..0000000000 --- a/python/serve/cortex_internal/lib/api/realtime.py +++ /dev/null @@ -1,375 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import imp -import inspect -import os -from copy import deepcopy -from typing import List, Optional, Union, Any - -import dill -from datadog import DogStatsd - -from cortex_internal.lib.api.utils import model_downloader, CortexMetrics -from cortex_internal.lib.api.validations import ( - validate_class_impl, - validate_python_handler_with_models, - validate_handler_with_grpc, - are_models_specified, -) -from cortex_internal.lib.client.python import ModelClient -from cortex_internal.lib.client.tensorflow import TensorFlowClient -from cortex_internal.lib.exceptions import CortexException, UserException, UserRuntimeException -from cortex_internal.lib.model import ( - FileBasedModelsGC, - TFSAPIServingThreadUpdater, - ModelsGC, - ModelTreeUpdater, - ModelsHolder, - ModelsTree, -) -from cortex_internal.lib.type import ( - handler_type_from_api_spec, - PythonHandlerType, - TensorFlowHandlerType, - TensorFlowNeuronHandlerType, -) - -PYTHON_CLASS_VALIDATION = { - "http": { - "required": [ - { - "name": "__init__", - "required_args": ["self", "config"], - "optional_args": ["model_client", "metrics_client"], - }, - ], - "optional": [ - { - "name": [ - "handle_post", - "handle_get", - "handle_put", - "handle_patch", - "handle_delete", - ], - "required_args": ["self"], - "optional_args": ["payload", "query_params", "headers"], - }, - { - "name": "load_model", - "required_args": ["self", "model_path"], - }, - ], - }, - "grpc": { - "required": [ - { - "name": "__init__", - "required_args": ["self", "config", "proto_module_pb2"], - "optional_args": ["model_client", "metrics_client"], - }, - ], - "optional": [ - { - "name": "load_model", - "required_args": ["self", "model_path"], - }, - ], - }, -} - -TENSORFLOW_CLASS_VALIDATION = { - "http": { - "required": [ - { - "name": "__init__", - "required_args": ["self", "config", "tensorflow_client"], - "optional_args": ["metrics_client"], - }, - ], - "optional": [ - { - "name": [ - "handle_post", - "handle_get", - "handle_put", - "handle_patch", - "handle_delete", - ], - "required_args": ["self"], - "optional_args": ["payload", "query_params", "headers"], - }, - ], - }, - "grpc": { - "required": [ - { - "name": "__init__", - "required_args": ["self", "config", "proto_module_pb2", "tensorflow_client"], - "optional_args": ["metrics_client"], - }, - ], - }, -} - - -class RealtimeAPI: - """ - Class to validate/load the handler class (Handler). - Also makes the specified models in cortex.yaml available to the handler's implementation. - """ - - def __init__(self, api_spec: dict, statsd_client: DogStatsd, model_dir: str): - self.api_spec = api_spec - self.model_dir = model_dir - - self.metrics = CortexMetrics(statsd_client, api_spec) - self.type = handler_type_from_api_spec(api_spec) - self.path = api_spec["handler"]["path"] - self.config = api_spec["handler"].get("config", {}) - self.protobuf_path = api_spec["handler"].get("protobuf_path") - - self.crons = [] - if not are_models_specified(self.api_spec): - return - - self.caching_enabled = self._is_model_caching_enabled() - self.multiple_processes = self.api_spec["handler"]["processes_per_replica"] > 1 - - # model caching can only be enabled when processes_per_replica is 1 - # model side-reloading is supported for any number of processes_per_replica - - if self.caching_enabled: - if self.type == PythonHandlerType: - mem_cache_size = self.api_spec["handler"]["multi_model_reloading"]["cache_size"] - disk_cache_size = self.api_spec["handler"]["multi_model_reloading"][ - "disk_cache_size" - ] - else: - mem_cache_size = self.api_spec["handler"]["models"]["cache_size"] - disk_cache_size = self.api_spec["handler"]["models"]["disk_cache_size"] - self.models = ModelsHolder( - self.type, - self.model_dir, - mem_cache_size=mem_cache_size, - disk_cache_size=disk_cache_size, - on_download_callback=model_downloader, - ) - elif not self.caching_enabled and self.type not in [ - TensorFlowHandlerType, - TensorFlowNeuronHandlerType, - ]: - self.models = ModelsHolder(self.type, self.model_dir) - else: - self.models = None - - if self.multiple_processes: - self.models_tree = None - else: - self.models_tree = ModelsTree() - - @property - def python_server_side_batching_enabled(self): - return ( - self.api_spec["handler"].get("server_side_batching") is not None - and self.api_spec["handler"]["type"] == "python" - ) - - def initialize_client( - self, tf_serving_host: Optional[str] = None, tf_serving_port: Optional[str] = None - ) -> Union[ModelClient, TensorFlowClient]: - """ - Initialize client that gives access to models specified in the API spec (cortex.yaml). - Only applies when models are provided in the API spec. - - Args: - tf_serving_host: Host of TF serving server. To be only used when the TensorFlow type is used. - tf_serving_port: Port of TF serving server. To be only used when the TensorFlow type is used. - - Return: - The client for the respective handler type. - """ - - client = None - - if are_models_specified(self.api_spec): - if self.type == PythonHandlerType: - client = ModelClient(self.api_spec, self.models, self.model_dir, self.models_tree) - - if self.type in [TensorFlowHandlerType, TensorFlowNeuronHandlerType]: - tf_serving_address = tf_serving_host + ":" + tf_serving_port - client = TensorFlowClient( - tf_serving_address, - self.api_spec, - self.models, - self.model_dir, - self.models_tree, - ) - if not self.caching_enabled: - cron = TFSAPIServingThreadUpdater(interval=5.0, client=client) - cron.start() - - return client - - def initialize_impl( - self, - project_dir: str, - client: Union[ModelClient, TensorFlowClient], - metrics_client: DogStatsd, - proto_module_pb2: Optional[Any] = None, - rpc_method_names: Optional[List[str]] = None, - ): - """ - Initialize handler class as provided by the user. - - proto_module_pb2 is a module of the compiled proto when grpc is enabled for the "RealtimeAPI" kind. Otherwise, it's None. - rpc_method_names is a non-empty list when grpc is enabled for the "RealtimeAPI" kind. Otherwise, it's None. - - Can raise UserRuntimeException/UserException/CortexException. - """ - - # build args - class_impl = self.class_impl(project_dir, rpc_method_names) - constructor_args = inspect.getfullargspec(class_impl.__init__).args - config = deepcopy(self.config) - args = {} - if "config" in constructor_args: - args["config"] = config - if "metrics_client" in constructor_args: - args["metrics_client"] = metrics_client - if "proto_module_pb2" in constructor_args: - args["proto_module_pb2"] = proto_module_pb2 - - # initialize handler class - try: - if self.type == PythonHandlerType: - if are_models_specified(self.api_spec): - args["model_client"] = client - # set load method to enable the use of the client in the constructor - # setting/getting from self in load_model won't work because self will be set to None - client.set_load_method( - lambda model_path: class_impl.load_model(None, model_path) - ) - initialized_impl = class_impl(**args) - client.set_load_method(initialized_impl.load_model) - else: - initialized_impl = class_impl(**args) - if self.type in [TensorFlowHandlerType, TensorFlowNeuronHandlerType]: - args["tensorflow_client"] = client - initialized_impl = class_impl(**args) - except Exception as e: - raise UserRuntimeException(self.path, "__init__", str(e)) from e - - # initialize the crons if models have been specified and if the API kind is RealtimeAPI - if are_models_specified(self.api_spec) and self.api_spec["kind"] == "RealtimeAPI": - if not self.multiple_processes and self.caching_enabled: - self.crons += [ - ModelTreeUpdater( - interval=10, - api_spec=self.api_spec, - tree=self.models_tree, - ondisk_models_dir=self.model_dir, - ), - ModelsGC( - interval=10, - api_spec=self.api_spec, - models=self.models, - tree=self.models_tree, - ), - ] - - if not self.caching_enabled and self.type == PythonHandlerType: - self.crons += [ - FileBasedModelsGC(interval=10, models=self.models, download_dir=self.model_dir) - ] - - for cron in self.crons: - cron.start() - - return initialized_impl - - def class_impl(self, project_dir: str, rpc_method_names: Optional[List[str]] = None): - """Can only raise UserException/CortexException exceptions""" - target_class_name = "Handler" - if self.type in [TensorFlowHandlerType, TensorFlowNeuronHandlerType]: - validations = TENSORFLOW_CLASS_VALIDATION - elif self.type == PythonHandlerType: - validations = PYTHON_CLASS_VALIDATION - else: - raise CortexException(f"invalid handler type: {self.type}") - - try: - handler_class = self._get_class_impl( - "cortex_handler", os.path.join(project_dir, self.path), target_class_name - ) - except Exception as e: - e.wrap("error in " + self.path) - raise - - try: - validate_class_impl(handler_class, validations) - validate_handler_with_grpc(handler_class, self.api_spec, rpc_method_names) - if self.type == PythonHandlerType: - validate_python_handler_with_models(handler_class, self.api_spec) - except Exception as e: - e.wrap("error in " + self.path) - raise - return handler_class - - def _get_class_impl(self, module_name, impl_path, target_class_name): - """Can only raise UserException exception""" - if impl_path.endswith(".pickle"): - try: - with open(impl_path, "rb") as pickle_file: - return dill.load(pickle_file) - except Exception as e: - raise UserException("unable to load pickle", str(e)) from e - - try: - impl = imp.load_source(module_name, impl_path) - except Exception as e: - raise UserException(str(e)) from e - - classes = inspect.getmembers(impl, inspect.isclass) - handler_class = None - for class_df in classes: - if class_df[0] == target_class_name: - if handler_class is not None: - raise UserException( - f"multiple definitions for {target_class_name} class found; please check your imports and class definitions and ensure that there is only one handler class definition" - ) - handler_class = class_df[1] - if handler_class is None: - raise UserException(f"{target_class_name} class is not defined") - - return handler_class - - def _is_model_caching_enabled(self) -> bool: - """ - Checks if model caching is enabled. - """ - models = None - if self.type != PythonHandlerType and self.api_spec["handler"]["models"]: - models = self.api_spec["handler"]["models"] - if self.type == PythonHandlerType and self.api_spec["handler"]["multi_model_reloading"]: - models = self.api_spec["handler"]["multi_model_reloading"] - - return models and models["cache_size"] and models["disk_cache_size"] - - def __del__(self) -> None: - for cron in self.crons: - cron.stop() - for cron in self.crons: - cron.join() diff --git a/python/serve/cortex_internal/lib/api/task.py b/python/serve/cortex_internal/lib/api/task.py deleted file mode 100644 index 56506366a5..0000000000 --- a/python/serve/cortex_internal/lib/api/task.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import imp -import inspect -import os - -import datadog -import dill - -from cortex_internal.lib.api.validations import validate_class_impl -from cortex_internal.lib.exceptions import CortexException, UserException -from cortex_internal.lib.log import configure_logger -from cortex_internal.lib.metrics import MetricsClient - -logger = configure_logger("cortex", os.environ["CORTEX_LOG_CONFIG_FILE"]) - -TASK_CLASS_VALIDATION = { - "required": [ - { - "name": "__init__", - "required_args": ["self"], - "optional_args": ["metrics_client"], - }, - { - "name": "__call__", - "required_args": ["self", "config"], - "optional_args": [], - }, - ], - "optional": [], -} - - -class TaskAPI: - def __init__(self, api_spec: dict): - """ - Args: - api_spec: API configuration. - """ - - self.path = api_spec["definition"]["path"] - self.config = api_spec["definition"].get("config", {}) - self.api_spec = api_spec - - host_ip = os.environ["HOST_IP"] - datadog.initialize(statsd_host=host_ip, statsd_port=9125) - self.__statsd = datadog.statsd - - @property - def statsd(self): - return self.__statsd - - def get_callable(self, project_dir: str): - impl = self._get_impl(project_dir) - - constructor_args = inspect.getfullargspec(impl.__init__).args - args = {} - if "metrics_client" in constructor_args: - args["metrics_client"] = MetricsClient(self.statsd) - - return impl(**args) - - def _get_impl(self, project_dir: str): - try: - task_callable = self._read_impl( - "cortex_task", os.path.join(project_dir, self.path), "Task" - ) - except CortexException as e: - e.wrap("error in " + self.path) - raise - - try: - self._validate_impl(task_callable) - except CortexException as e: - e.wrap("error in " + self.path) - raise - return task_callable - - @staticmethod - def _read_impl(module_name: str, impl_path: str, target_class_name: str): - if impl_path.endswith(".pickle"): - try: - with open(impl_path, "rb") as pickle_file: - return dill.load(pickle_file) - except Exception as e: - raise UserException("unable to load pickle", str(e)) from e - - try: - impl = imp.load_source(module_name, impl_path) - except Exception as e: - raise UserException(str(e)) from e - - classes = inspect.getmembers(impl, inspect.isclass) - - if len(classes) > 0: - task_class = None - for class_df in classes: - if class_df[0] == target_class_name: - if task_class is not None: - raise UserException( - f"multiple definitions for {target_class_name} class found; please check " - f"your imports and class definitions and ensure that there is only one " - f"task class definition" - ) - task_class = class_df[1] - if task_class is None: - raise UserException(f"{target_class_name} class is not defined") - return task_class - else: - raise UserException("no callable class was provided") - - @staticmethod - def _validate_impl(impl): - validate_class_impl(impl, TASK_CLASS_VALIDATION) diff --git a/python/serve/cortex_internal/lib/api/utils.py b/python/serve/cortex_internal/lib/api/utils.py deleted file mode 100644 index 49190cb5cf..0000000000 --- a/python/serve/cortex_internal/lib/api/utils.py +++ /dev/null @@ -1,338 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime -import glob -import itertools -import json -import os -import shutil -import threading as td -import time -import traceback -from collections import defaultdict -from http import HTTPStatus -from typing import Any, Callable, Dict, List, Optional, Tuple - -from datadog.dogstatsd.base import DogStatsd -from starlette.responses import Response - -from cortex_internal.lib import util -from cortex_internal.lib.exceptions import CortexException, UserRuntimeException -from cortex_internal.lib.log import configure_logger -from cortex_internal.lib.model import validate_model_paths -from cortex_internal.lib.storage import S3 -from cortex_internal.lib.type import HandlerType - -logger = configure_logger("cortex", os.environ["CORTEX_LOG_CONFIG_FILE"]) - - -def _read_json(json_path: str): - with open(json_path) as json_file: - return json.load(json_file) - - -def get_spec( - spec_path: str, - cache_dir: str, - region: str, - spec_name: str = "api_spec.json", -) -> Tuple[S3, dict]: - """ - Args: - spec_path: Path to API spec (i.e. "s3://cortex-dev-0/apis/iris-classifier/api/69b93378fa5c0218-jy1fjtyihu-9fcc10739e7fc8050cefa8ca27ece1ee/master-spec.json"). - cache_dir: Local directory where the API spec gets saved to. - region: Region of the bucket. - spec_name: File name of the spec as it is saved on disk. - """ - - bucket, key = S3.deconstruct_s3_path(spec_path) - storage = S3(bucket=bucket, region=region) - - local_spec_path = os.path.join(cache_dir, spec_name) - if not os.path.isfile(local_spec_path): - storage.download_file(key, local_spec_path) - - return storage, _read_json(local_spec_path) - - -def model_downloader( - handler_type: HandlerType, - bucket_name: str, - model_name: str, - model_version: str, - model_path: str, - temp_dir: str, - model_dir: str, -) -> Optional[datetime.datetime]: - """ - Downloads model to disk. Validates the s3 model path and the downloaded model. - - Args: - handler_type: The handler type as implemented by the API. - bucket_name: Name of the bucket where the model is stored. - model_name: Name of the model. Is part of the model's local path. - model_version: Version of the model. Is part of the model's local path. - model_path: Model prefix of the versioned model. - temp_dir: Where to temporarily store the model for validation. - model_dir: The top directory of where all models are stored locally. - - Returns: - The model's timestamp. None if the model didn't pass the validation, if it doesn't exist or if there are not enough permissions. - """ - - logger.info( - f"downloading from bucket {bucket_name}/{model_path}, model {model_name} of version {model_version}, temporarily to {temp_dir} and then finally to {model_dir}" - ) - - client = S3(bucket_name) - - # validate upstream S3 model - sub_paths, ts = client.search(model_path) - try: - validate_model_paths(sub_paths, handler_type, model_path) - except CortexException: - logger.info(f"failed validating model {model_name} of version {model_version}") - return None - - # download model to temp dir - temp_dest = os.path.join(temp_dir, model_name, model_version) - try: - client.download_dir_contents(model_path, temp_dest) - except CortexException: - logger.info( - f"failed downloading model {model_name} of version {model_version} to temp dir {temp_dest}" - ) - shutil.rmtree(temp_dest) - return None - - # validate model - model_contents = glob.glob(os.path.join(temp_dest, "**"), recursive=True) - model_contents = util.remove_non_empty_directory_paths(model_contents) - try: - validate_model_paths(model_contents, handler_type, temp_dest) - except CortexException: - logger.info( - f"failed validating model {model_name} of version {model_version} from temp dir" - ) - shutil.rmtree(temp_dest) - return None - - # move model to dest dir - model_top_dir = os.path.join(model_dir, model_name) - ondisk_model_version = os.path.join(model_top_dir, model_version) - logger.info( - f"moving model {model_name} of version {model_version} to final dir {ondisk_model_version}" - ) - if os.path.isdir(ondisk_model_version): - shutil.rmtree(ondisk_model_version) - shutil.move(temp_dest, ondisk_model_version) - - return max(ts) - - -class CortexMetrics: - def __init__( - self, - statsd_client: DogStatsd, - api_spec: Dict[str, Any], - ): - self._metric_value_id = api_spec["id"] - self._metric_value_handler_id = api_spec["handler_id"] - self._metric_value_deployment_id = api_spec["deployment_id"] - self._metric_value_name = api_spec["name"] - self.__statsd = statsd_client - - def metric_dimensions_with_id(self): - return [ - {"Name": "api_name", "Value": self._metric_value_name}, - {"Name": "api_id", "Value": self._metric_value_id}, - {"Name": "handler_id", "Value": self._metric_value_handler_id}, - {"Name": "deployment_id", "Value": self._metric_value_deployment_id}, - ] - - def metric_dimensions(self): - return [{"Name": "api_name", "Value": self._metric_value_name}] - - def post_request_metrics(self, status_code, total_time): - total_time_ms = total_time * 1000 - metrics = [ - self.status_code_metric(self.metric_dimensions(), status_code), - self.status_code_metric(self.metric_dimensions_with_id(), status_code), - self.latency_metric(self.metric_dimensions(), total_time_ms), - self.latency_metric(self.metric_dimensions_with_id(), total_time_ms), - ] - self.post_metrics(metrics) - - def post_status_code_request_metrics(self, status_code): - metrics = [ - self.status_code_metric(self.metric_dimensions(), status_code), - self.status_code_metric(self.metric_dimensions_with_id(), status_code), - ] - self.post_metrics(metrics) - - def post_latency_request_metrics(self, total_time): - total_time_ms = total_time * 1000 - metrics = [ - self.latency_metric(self.metric_dimensions(), total_time_ms), - self.latency_metric(self.metric_dimensions_with_id(), total_time_ms), - ] - self.post_metrics(metrics) - - def post_metrics(self, metrics): - try: - if self.__statsd is None: - raise CortexException("statsd client not initialized") # unexpected - - for metric in metrics: - tags = ["{}:{}".format(dim["Name"], dim["Value"]) for dim in metric["Dimensions"]] - if metric.get("Unit") == "Count": - self.__statsd.increment(metric["MetricName"], value=metric["Value"], tags=tags) - else: - self.__statsd.histogram(metric["MetricName"], value=metric["Value"], tags=tags) - except: - logger.warn("failure encountered while publishing metrics", exc_info=True) - - def status_code_metric(self, dimensions, status_code): - status_code_series = int(status_code / 100) - status_code_dimensions = dimensions + [ - {"Name": "response_code", "Value": "{}XX".format(status_code_series)} - ] - return { - "MetricName": "cortex_status_code", - "Dimensions": status_code_dimensions, - "Value": 1, - "Unit": "Count", - } - - def latency_metric(self, dimensions, total_time): - return { - "MetricName": "cortex_latency", - "Dimensions": dimensions, - "Value": total_time, # milliseconds - } - - -class DynamicBatcher: - def __init__( - self, - handler_impl: Callable, - method_name: str, - max_batch_size: int, - batch_interval: int, - test_mode: bool = False, - ): - self.method_name = method_name - self.handler_impl = handler_impl - - self.batch_max_size = max_batch_size - self.batch_interval = batch_interval # measured in seconds - self.test_mode = test_mode # only for unit testing - self._test_batch_lengths = [] # only when unit testing - - self.barrier = td.Barrier(self.batch_max_size + 1) - - self.samples = {} - self.results = {} - td.Thread(target=self._batch_engine, daemon=True).start() - - self.sample_id_generator = itertools.count() - - def _batch_engine(self): - while True: - if len(self.results) > 0: - time.sleep(0.001) - continue - - try: - self.barrier.wait(self.batch_interval) - except td.BrokenBarrierError: - pass - - self.results = {} - sample_ids = self._get_sample_ids(self.batch_max_size) - try: - if self.samples: - batch = self._make_batch(sample_ids) - - results = getattr(self.handler_impl, self.method_name)(**batch) - if not isinstance(results, list): - raise UserRuntimeException( - f"please return a list when using server side batching, got {type(results)}" - ) - - if self.test_mode: - self._test_batch_lengths.append(len(results)) - - self.results = dict(zip(sample_ids, results)) - except Exception as e: - self.results = {sample_id: e for sample_id in sample_ids} - logger.error(traceback.format_exc()) - finally: - for sample_id in sample_ids: - del self.samples[sample_id] - self.barrier.reset() - - def _get_sample_ids(self, max_number: int) -> List[int]: - if len(self.samples) <= max_number: - return list(self.samples.keys()) - return sorted(self.samples)[:max_number] - - def _make_batch(self, sample_ids: List[int]) -> Dict[str, List[Any]]: - batched_samples = defaultdict(list) - for sample_id in sample_ids: - for key, sample in self.samples[sample_id].items(): - batched_samples[key].append(sample) - - return dict(batched_samples) - - def _enqueue_request(self, sample_id: int, **kwargs): - """ - Enqueue sample for batch processing. This is a blocking method. - """ - - self.samples[sample_id] = kwargs - try: - self.barrier.wait() - except td.BrokenBarrierError: - pass - - def process(self, **kwargs): - """ - Queues a request to be batched with other incoming request, waits for the response - and returns the processed result. This is a blocking method. - """ - sample_id = next(self.sample_id_generator) - self._enqueue_request(sample_id, **kwargs) - result = self._get_result(sample_id) - return result - - def _get_result(self, sample_id: int) -> Any: - """ - Return the processed result. This is a blocking method. - """ - while sample_id not in self.results: - time.sleep(0.001) - - result = self.results[sample_id] - del self.results[sample_id] - - if isinstance(result, Exception): - return Response( - content=str(result), - status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - media_type="text/plain", - ) - - return result diff --git a/python/serve/cortex_internal/lib/api/validations.py b/python/serve/cortex_internal/lib/api/validations.py deleted file mode 100644 index 75a5cafd82..0000000000 --- a/python/serve/cortex_internal/lib/api/validations.py +++ /dev/null @@ -1,177 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import inspect -from typing import Dict, List - -from cortex_internal.lib import util -from cortex_internal.lib.exceptions import UserException -from cortex_internal.lib.type import handler_type_from_api_spec, PythonHandlerType - - -def validate_class_impl(impl, impl_req): - for optional_func_signature in impl_req.get("optional", []): - validate_optional_method_args(impl, optional_func_signature) - - for required_func_signature in impl_req.get("required", []): - validate_required_method_args(impl, required_func_signature) - - -def validate_optional_method_args(impl, func_signature): - if getattr(impl, func_signature["name"], None): - validate_required_method_args(impl, func_signature) - - -def validate_required_method_args(impl, func_signature): - target_class_name = impl.__name__ - - fn = getattr(impl, func_signature["name"], None) - if not fn: - raise UserException( - f"class {target_class_name}", - f'required method "{func_signature["name"]}" is not defined', - ) - - if not callable(fn): - raise UserException( - f"class {target_class_name}", - f'"{func_signature["name"]}" is defined, but is not a method', - ) - - required_args = func_signature.get("required_args", []) - optional_args = func_signature.get("optional_args", []) - - argspec = inspect.getfullargspec(fn) - fn_str = f'{func_signature["name"]}({", ".join(argspec.args)})' - - for arg_name in required_args: - if arg_name not in argspec.args: - raise UserException( - f"class {target_class_name}", - f'invalid signature for method "{fn_str}"', - f'"{arg_name}" is a required argument, but was not provided', - ) - - if arg_name == "self": - if argspec.args[0] != "self": - raise UserException( - f"class {target_class_name}", - f'invalid signature for method "{fn_str}"', - f'"self" must be the first argument', - ) - - seen_args = [] - for arg_name in argspec.args: - if arg_name not in required_args and arg_name not in optional_args: - raise UserException( - f"class {target_class_name}", - f'invalid signature for method "{fn_str}"', - f'"{arg_name}" is not a supported argument', - ) - - if arg_name in seen_args: - raise UserException( - f"class {target_class_name}", - f'invalid signature for method "{fn_str}"', - f'"{arg_name}" is duplicated', - ) - - seen_args.append(arg_name) - - -def validate_python_handler_with_models(impl, api_spec): - if not are_models_specified(api_spec): - return - - target_class_name = impl.__name__ - constructor = getattr(impl, "__init__") - constructor_arg_spec = inspect.getfullargspec(constructor) - if "model_client" not in constructor_arg_spec.args: - raise UserException( - f"class {target_class_name}", - f'invalid signature for method "__init__"', - f'"model_client" is a required argument, but was not provided', - f"when the python handler type is used and models are specified in the api spec, " - f'adding the "model_client" argument is required', - ) - - if getattr(impl, "load_model", None) is None: - raise UserException( - f"class {target_class_name}", - f"required method `load_model` is not defined", - f"when the python handler type is used and models are specified in the api spec, " - f"adding the `load_model` method is required", - ) - - -def are_models_specified(api_spec: Dict) -> bool: - """ - Checks if models have been specified in the API spec (cortex.yaml). - - Args: - api_spec: API configuration. - """ - handler_type = handler_type_from_api_spec(api_spec) - - if handler_type == PythonHandlerType and api_spec["handler"]["multi_model_reloading"]: - models = api_spec["handler"]["multi_model_reloading"] - elif handler_type != PythonHandlerType: - models = api_spec["handler"]["models"] - else: - return False - - return models is not None - - -def is_grpc_enabled(api_spec: Dict) -> bool: - """ - Checks if the API has the grpc protocol enabled (cortex.yaml). - - Args: - api_spec: API configuration. - """ - return api_spec["handler"]["protobuf_path"] is not None - - -def validate_handler_with_grpc(impl, api_spec: Dict, rpc_method_names: List[str]): - if not is_grpc_enabled(api_spec): - return - - target_class_name = impl.__name__ - constructor = getattr(impl, "__init__") - constructor_arg_spec = inspect.getfullargspec(constructor) - if "proto_module_pb2" not in constructor_arg_spec.args: - raise UserException( - f"class {target_class_name}", - f"invalid signature for method `__init__`", - f'"proto_module_pb2" is a required argument, but was not provided', - f"when a protobuf is specified in the api spec, then that means the grpc protocol is enabled, " - f'which means that adding the "proto_module_pb2" argument is required', - ) - - for rpc_method_name in rpc_method_names: - if not util.has_method(impl, rpc_method_name): - raise UserException( - f"method {rpc_method_name} hasn't been defined in the Handler class; define one called {rpc_method_name} to match the RPC method from the protobuf definition" - ) - - rpc_handler = getattr(impl, rpc_method_name) - arg_spec = inspect.getfullargspec(rpc_handler).args - disallowed_params = list(set(arg_spec).difference(set(["self", "payload", "context"]))) - if len(disallowed_params) > 0: - raise UserException( - f"class {target_class_name}", - f'invalid signature for method "{rpc_method_name}"', - f'{util.string_plural_with_s("argument", len(disallowed_params))} {util.and_list_with_quotes(disallowed_params)} cannot be used when the grpc protocol is enabled', - ) diff --git a/python/serve/cortex_internal/lib/checkers/__init__.py b/python/serve/cortex_internal/lib/checkers/__init__.py deleted file mode 100644 index dcd1d9ae2f..0000000000 --- a/python/serve/cortex_internal/lib/checkers/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/python/serve/cortex_internal/lib/checkers/pod.py b/python/serve/cortex_internal/lib/checkers/pod.py deleted file mode 100644 index 575f872e8e..0000000000 --- a/python/serve/cortex_internal/lib/checkers/pod.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import stat -import time - -from cortex_internal import consts - - -def neuron_socket_exists(): - if not os.path.exists(consts.INFERENTIA_NEURON_SOCKET): - return False - else: - mode = os.stat(consts.INFERENTIA_NEURON_SOCKET) - return stat.S_ISSOCK(mode.st_mode) - - -def wait_neuron_rtd(): - while not neuron_socket_exists(): - time.sleep(0.1) diff --git a/python/serve/cortex_internal/lib/client/__init__.py b/python/serve/cortex_internal/lib/client/__init__.py deleted file mode 100644 index dcd1d9ae2f..0000000000 --- a/python/serve/cortex_internal/lib/client/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/python/serve/cortex_internal/lib/client/python.py b/python/serve/cortex_internal/lib/client/python.py deleted file mode 100644 index 051d60d812..0000000000 --- a/python/serve/cortex_internal/lib/client/python.py +++ /dev/null @@ -1,405 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import threading as td -from typing import Any, Optional, Callable - -from cortex_internal import consts -from cortex_internal.lib.concurrency import LockedFile -from cortex_internal.lib.exceptions import ( - UserRuntimeException, - WithBreak, -) -from cortex_internal.lib.log import configure_logger -from cortex_internal.lib.model import ( - ModelsHolder, - LockedModel, - ModelsTree, - LockedModelsTree, - get_models_from_api_spec, - find_ondisk_model_info, - find_ondisk_models_with_lock, -) - -logger = configure_logger("cortex", os.environ["CORTEX_LOG_CONFIG_FILE"]) - - -class ModelClient: - def __init__( - self, - api_spec: dict, - models: ModelsHolder, - model_dir: str, - models_tree: Optional[ModelsTree], - lock_dir: Optional[str] = "/run/cron", - load_model_fn: Optional[Callable[[str], Any]] = None, - ): - """ - Setup Python model client. - - Args: - api_spec: API configuration. - - models: Holding all models into memory. - model_dir: Where the models are saved on disk. - - models_tree: A tree of the available models from upstream. - lock_dir: Where the resource locks are found. Only when processes_per_replica > 0 and caching disabled. - load_model_fn: Function to load model into memory. - """ - - self._api_spec = api_spec - self._models = models - self._models_tree = models_tree - self._model_dir = model_dir - self._lock_dir = lock_dir - - self._spec_models = get_models_from_api_spec(api_spec) - - if ( - self._api_spec["handler"]["multi_model_reloading"] - and self._api_spec["handler"]["multi_model_reloading"]["dir"] - ): - self._models_dir = True - else: - self._models_dir = False - self._spec_model_names = self._spec_models.get_field("name") - - self._multiple_processes = self._api_spec["handler"]["processes_per_replica"] > 1 - self._caching_enabled = self._is_model_caching_enabled() - - if callable(load_model_fn): - self._models.set_callback("load", load_model_fn) - - def set_load_method(self, load_model_fn: Callable[[str], Any]) -> None: - self._models.set_callback("load", load_model_fn) - - def get_model(self, model_name: Optional[str] = None, model_version: str = "latest") -> Any: - """ - Retrieve a model for inference. - - Args: - model_name (optional): Name of the model to retrieve (when multiple models are deployed in an API). - When handler.models.paths is specified, model_name should be the name of one of the models listed in the API config. - When handler.models.dir is specified, model_name should be the name of a top-level directory in the models dir. - model_version (string, optional): Version of the model to retrieve. Can be omitted or set to "latest" to select the highest version. - - Returns: - The value that's returned by your handler's load_model() method. - """ - - if model_version != "latest" and not model_version.isnumeric(): - raise UserRuntimeException( - "model_version must be either a parse-able numeric value or 'latest'" - ) - - # when handler:models:path or handler:models:paths is specified - if not self._models_dir: - - # when handler:models:path is provided - if consts.SINGLE_MODEL_NAME in self._spec_model_names: - model_name = consts.SINGLE_MODEL_NAME - model = self._get_model(model_name, model_version) - if model is None: - raise UserRuntimeException( - f"model {model_name} of version {model_version} wasn't found" - ) - return model - - # when handler:models:paths is specified - if model_name is None: - raise UserRuntimeException( - f"model_name was not specified, choose one of the following: {self._spec_model_names}" - ) - - if model_name not in self._spec_model_names: - raise UserRuntimeException( - f"'{model_name}' model wasn't found in the list of available models" - ) - - # when handler:models:dir is specified - if self._models_dir: - if model_name is None: - raise UserRuntimeException("model_name was not specified") - if not self._caching_enabled: - available_models = find_ondisk_models_with_lock(self._lock_dir) - if model_name not in available_models: - raise UserRuntimeException( - f"'{model_name}' model wasn't found in the list of available models" - ) - - model = self._get_model(model_name, model_version) - if model is None: - raise UserRuntimeException( - f"model {model_name} of version {model_version} wasn't found" - ) - return model - - def _get_model(self, model_name: str, model_version: str) -> Any: - """ - Checks if versioned model is on disk, then checks if model is in memory, - and if not, it loads it into memory, and returns the model. - - Args: - model_name: Name of the model, as it's specified in handler:models:paths or in the other case as they are named on disk. - model_version: Version of the model, as it's found on disk. Can also infer the version number from the "latest" tag. - - Exceptions: - RuntimeError: if another thread tried to load the model at the very same time. - - Returns: - The model as returned by self._load_model method. - None if the model wasn't found or if it didn't pass the validation. - """ - - model = None - tag = "" - if model_version == "latest": - tag = model_version - - if not self._caching_enabled: - # determine model version - if tag == "latest": - model_version = self._get_latest_model_version_from_disk(model_name) - model_id = model_name + "-" + model_version - - # grab shared access to versioned model - resource = os.path.join(self._lock_dir, model_id + ".txt") - with LockedFile(resource, "r", reader_lock=True) as f: - - # check model status - file_status = f.read() - if file_status == "" or file_status == "not-available": - raise WithBreak - - current_upstream_ts = int(file_status.split(" ")[1]) - update_model = False - - # grab shared access to models holder and retrieve model - with LockedModel(self._models, "r", model_name, model_version): - status, local_ts = self._models.has_model(model_name, model_version) - if status == "not-available" or ( - status == "in-memory" and local_ts != current_upstream_ts - ): - update_model = True - raise WithBreak - model, _ = self._models.get_model(model_name, model_version, tag) - - # load model into memory and retrieve it - if update_model: - with LockedModel(self._models, "w", model_name, model_version): - status, _ = self._models.has_model(model_name, model_version) - if status == "not-available" or ( - status == "in-memory" and local_ts != current_upstream_ts - ): - if status == "not-available": - logger.info( - f"loading model {model_name} of version {model_version} (thread {td.get_ident()})" - ) - else: - logger.info( - f"reloading model {model_name} of version {model_version} (thread {td.get_ident()})" - ) - try: - self._models.load_model( - model_name, - model_version, - current_upstream_ts, - [tag], - ) - except Exception as e: - raise UserRuntimeException( - f"failed (re-)loading model {model_name} of version {model_version} (thread {td.get_ident()})", - str(e), - ) - model, _ = self._models.get_model(model_name, model_version, tag) - - if not self._multiple_processes and self._caching_enabled: - # determine model version - try: - if tag == "latest": - model_version = self._get_latest_model_version_from_tree( - model_name, self._models_tree.model_info(model_name) - ) - except ValueError: - # if model_name hasn't been found - raise UserRuntimeException( - f"'{model_name}' model of tag latest wasn't found in the list of available models" - ) - - # grab shared access to model tree - available_model = True - with LockedModelsTree(self._models_tree, "r", model_name, model_version): - - # check if the versioned model exists - model_id = model_name + "-" + model_version - if model_id not in self._models_tree: - available_model = False - raise WithBreak - - # retrieve model tree's metadata - upstream_model = self._models_tree[model_id] - current_upstream_ts = int(upstream_model["timestamp"].timestamp()) - - if not available_model: - return None - - # grab shared access to models holder and retrieve model - update_model = False - with LockedModel(self._models, "r", model_name, model_version): - status, local_ts = self._models.has_model(model_name, model_version) - if status in ["not-available", "on-disk"] or ( - status != "not-available" and local_ts != current_upstream_ts - ): - update_model = True - raise WithBreak - model, _ = self._models.get_model(model_name, model_version, tag) - - # download, load into memory the model and retrieve it - if update_model: - # grab exclusive access to models holder - with LockedModel(self._models, "w", model_name, model_version): - - # check model status - status, local_ts = self._models.has_model(model_name, model_version) - - # refresh disk model - if status == "not-available" or ( - status in ["on-disk", "in-memory"] and local_ts != current_upstream_ts - ): - if status == "not-available": - logger.info( - f"model {model_name} of version {model_version} not found locally; continuing with the download..." - ) - elif status == "on-disk": - logger.info( - f"found newer model {model_name} of version {model_version} on the s3 upstream than the one on the disk" - ) - else: - logger.info( - f"found newer model {model_name} of version {model_version} on the s3 upstream than the one loaded into memory" - ) - - # remove model from disk and memory - if status == "on-disk": - logger.info( - f"removing model from disk for model {model_name} of version {model_version}" - ) - self._models.remove_model(model_name, model_version) - if status == "in-memory": - logger.info( - f"removing model from disk and memory for model {model_name} of version {model_version}" - ) - self._models.remove_model(model_name, model_version) - - # download model - logger.info( - f"downloading model {model_name} of version {model_version} from the s3 upstream" - ) - date = self._models.download_model( - upstream_model["bucket"], - model_name, - model_version, - upstream_model["path"], - ) - if not date: - raise WithBreak - current_upstream_ts = int(date.timestamp()) - - # load model - try: - logger.info( - f"loading model {model_name} of version {model_version} into memory" - ) - self._models.load_model( - model_name, - model_version, - current_upstream_ts, - [tag], - ) - except Exception as e: - raise UserRuntimeException( - f"failed (re-)loading model {model_name} of version {model_version} (thread {td.get_ident()})", - str(e), - ) - - # retrieve model - model, _ = self._models.get_model(model_name, model_version, tag) - - return model - - def _get_latest_model_version_from_disk(self, model_name: str) -> str: - """ - Get the highest version for a specific model name. - Must only be used when processes_per_replica > 0 and caching disabled. - """ - versions, timestamps = find_ondisk_model_info(self._lock_dir, model_name) - if len(versions) == 0: - raise UserRuntimeException( - "'{}' model's versions have been removed; add at least a version to the model to resume operations".format( - model_name - ) - ) - return str(max(map(lambda x: int(x), versions))) - - def _get_latest_model_version_from_tree(self, model_name: str, model_info: dict) -> str: - """ - Get the highest version for a specific model name. - Must only be used when processes_per_replica = 1 and caching is enabled. - """ - versions, timestamps = model_info["versions"], model_info["timestamps"] - return str(max(map(lambda x: int(x), versions))) - - def _is_model_caching_enabled(self) -> bool: - """ - Checks if model caching is enabled (models:cache_size and models:disk_cache_size). - """ - return ( - self._api_spec["handler"]["multi_model_reloading"] - and self._api_spec["handler"]["multi_model_reloading"]["cache_size"] - and self._api_spec["handler"]["multi_model_reloading"]["disk_cache_size"] - ) - - @property - def metadata(self) -> dict: - """ - The returned dictionary will be like in the following example: - { - ... - "yolov3": { - "versions": [ - "2", - "1" - ], - "timestamps": [ - 1601668127, - 1601668127 - ] - } - ... - } - """ - if not self._caching_enabled: - return find_ondisk_models_with_lock(self._lock_dir, include_timestamps=True) - else: - models_info = self._models_tree.get_all_models_info() - for model_name in models_info.keys(): - del models_info[model_name]["bucket"] - del models_info[model_name]["model_paths"] - return models_info - - @property - def caching(self) -> bool: - return self._caching_enabled diff --git a/python/serve/cortex_internal/lib/client/tensorflow.py b/python/serve/cortex_internal/lib/client/tensorflow.py deleted file mode 100644 index c02832bf0d..0000000000 --- a/python/serve/cortex_internal/lib/client/tensorflow.py +++ /dev/null @@ -1,515 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import json -import os -import threading as td -from typing import Any, Optional, List - -import grpc - -from cortex_internal import consts -from cortex_internal.lib.exceptions import ( - UserRuntimeException, - UserException, - WithBreak, -) -from cortex_internal.lib.log import configure_logger -from cortex_internal.lib.model import ( - TensorFlowServingAPI, - ModelsHolder, - ModelsTree, - LockedModel, - LockedModelsTree, - get_models_from_api_spec, -) - -logger = configure_logger("cortex", os.environ["CORTEX_LOG_CONFIG_FILE"]) - - -class TensorFlowClient: - def __init__( - self, - tf_serving_url, - api_spec: dict, - models: Optional[ModelsHolder] = None, - model_dir: Optional[str] = None, - models_tree: Optional[ModelsTree] = None, - ): - """ - Setup gRPC connection to TensorFlow Serving container. - - Args: - tf_serving_url: Localhost URL to TF Serving container (i.e. "localhost:9000") - api_spec: API configuration. - - models: Holding all models into memory. Only when processes_per_replica = 1 and caching enabled. - model_dir: Where the models are saved on disk. Only when processes_per_replica = 1 and caching enabled. - models_tree: A tree of the available models from upstream. Only when processes_per_replica = 1 and caching enabled. - """ - - self.tf_serving_url = tf_serving_url - - self._api_spec = api_spec - self._models = models - self._models_tree = models_tree - self._model_dir = model_dir - - self._spec_models = get_models_from_api_spec(api_spec) - - if ( - self._api_spec["handler"]["models"] - and self._api_spec["handler"]["models"]["dir"] is not None - ): - self._models_dir = True - else: - self._models_dir = False - self._spec_model_names = self._spec_models.get_field("name") - - self._multiple_processes = self._api_spec["handler"]["processes_per_replica"] > 1 - self._caching_enabled = self._is_model_caching_enabled() - - if self._models: - self._models.set_callback("load", self._load_model) - self._models.set_callback("remove", self._remove_models) - - self._client = TensorFlowServingAPI(tf_serving_url) - - def sync_models(self, lock_dir: str = "/run/cron"): - resource_models = os.path.join(lock_dir, "models_tfs.json") - - try: - with open(resource_models, "r") as f: - models = json.load(f) - except Exception: - return - - resource_ts = os.path.join(lock_dir, "model_timestamps.json") - try: - with open(resource_ts, "r") as f: - timestamps = json.load(f) - except Exception: - return - - non_intersecting_model_ids = set(models.keys()).symmetric_difference(timestamps.keys()) - for non_intersecting_model_id in non_intersecting_model_ids: - if non_intersecting_model_id in models: - del models[non_intersecting_model_id] - if non_intersecting_model_id in timestamps: - del timestamps[non_intersecting_model_id] - - for model_id in timestamps.keys(): - models[model_id]["timestamp"] = timestamps[model_id] - - self._client.models = models - - def predict( - self, model_input: Any, model_name: Optional[str] = None, model_version: str = "latest" - ) -> dict: - """ - Validate model_input, convert it to a Prediction Proto, and make a request to TensorFlow Serving. - - Args: - model_input: Input to the model. - model_name (optional): Name of the model to retrieve (when multiple models are deployed in an API). - When handler.models.paths is specified, model_name should be the name of one of the models listed in the API config. - When handler.models.dir is specified, model_name should be the name of a top-level directory in the models dir. - model_version (string, optional): Version of the model to retrieve. Can be omitted or set to "latest" to select the highest version. - - Returns: - dict: TensorFlow Serving response converted to a dictionary. - """ - - if model_version != "latest" and not model_version.isnumeric(): - raise UserRuntimeException( - "model_version must be either a parse-able numeric value or 'latest'" - ) - - # when handler:models:path or handler:models:paths is specified - if not self._models_dir: - - # when handler:models:path is provided - if consts.SINGLE_MODEL_NAME in self._spec_model_names: - return self._run_inference(model_input, consts.SINGLE_MODEL_NAME, model_version) - - # when handler:models:paths is specified - if model_name is None: - raise UserRuntimeException( - f"model_name was not specified, choose one of the following: {self._spec_model_names}" - ) - - if model_name not in self._spec_model_names: - raise UserRuntimeException( - f"'{model_name}' model wasn't found in the list of available models" - ) - - # when handler:models:dir is specified - if self._models_dir and model_name is None: - raise UserRuntimeException("model_name was not specified") - - return self._run_inference(model_input, model_name, model_version) - - def _run_inference(self, model_input: Any, model_name: str, model_version: str) -> dict: - """ - When processes_per_replica = 1 and caching enabled, check/load model and make prediction. - When processes_per_replica > 0 and caching disabled, attempt to make prediction regardless. - - Args: - model_input: Input to the model. - model_name: Name of the model, as it's specified in handler:models:paths or in the other case as they are named on disk. - model_version: Version of the model, as it's found on disk. Can also infer the version number from the "latest" version tag. - - Returns: - The prediction. - """ - - model = None - tag = "" - if model_version == "latest": - tag = model_version - - if not self._caching_enabled: - - # determine model version - if tag == "latest": - versions = self._client.poll_available_model_versions(model_name) - if len(versions) == 0: - raise UserException( - f"model '{model_name}' accessed with tag {tag} couldn't be found" - ) - model_version = str(max(map(lambda x: int(x), versions))) - model_id = model_name + "-" + model_version - - return self._client.predict(model_input, model_name, model_version) - - if not self._multiple_processes and self._caching_enabled: - - # determine model version - try: - if tag == "latest": - model_version = self._get_latest_model_version_from_tree( - model_name, self._models_tree.model_info(model_name) - ) - except ValueError: - # if model_name hasn't been found - raise UserRuntimeException( - f"'{model_name}' model of tag {tag} wasn't found in the list of available models" - ) - - # grab shared access to model tree - available_model = True - logger.info(f"grabbing access to model {model_name} of version {model_version}") - with LockedModelsTree(self._models_tree, "r", model_name, model_version): - - # check if the versioned model exists - model_id = model_name + "-" + model_version - if model_id not in self._models_tree: - available_model = False - logger.info(f"model {model_name} of version {model_version} is not available") - raise WithBreak - - # retrieve model tree's metadata - upstream_model = self._models_tree[model_id] - current_upstream_ts = int(upstream_model["timestamp"].timestamp()) - logger.info(f"model {model_name} of version {model_version} is available") - - if not available_model: - if tag == "": - raise UserException( - f"model '{model_name}' of version '{model_version}' couldn't be found" - ) - raise UserException( - f"model '{model_name}' accessed with tag '{tag}' couldn't be found" - ) - - # grab shared access to models holder and retrieve model - update_model = False - prediction = None - tfs_was_unresponsive = False - with LockedModel(self._models, "r", model_name, model_version): - logger.info(f"checking the {model_name} {model_version} status") - status, local_ts = self._models.has_model(model_name, model_version) - if status in ["not-available", "on-disk"] or ( - status != "not-available" and local_ts != current_upstream_ts - ): - logger.info( - f"model {model_name} of version {model_version} is not loaded (with status {status} or different timestamp)" - ) - update_model = True - raise WithBreak - - # run prediction - logger.info(f"run the prediction on model {model_name} of version {model_version}") - self._models.get_model(model_name, model_version, tag) - try: - prediction = self._client.predict(model_input, model_name, model_version) - except grpc.RpcError as e: - # effectively when it got restarted - if len(self._client.poll_available_model_versions(model_name)) > 0: - raise - tfs_was_unresponsive = True - - # remove model from disk and memory references if TFS gets unresponsive - if tfs_was_unresponsive: - with LockedModel(self._models, "w", model_name, model_version): - available_versions = self._client.poll_available_model_versions(model_name) - status, _ = self._models.has_model(model_name, model_version) - if not (status == "in-memory" and model_version not in available_versions): - raise WithBreak - - logger.info( - f"removing model {model_name} of version {model_version} because TFS got unresponsive" - ) - self._models.remove_model(model_name, model_version) - - # download, load into memory the model and retrieve it - if update_model: - # grab exclusive access to models holder - with LockedModel(self._models, "w", model_name, model_version): - - # check model status - status, local_ts = self._models.has_model(model_name, model_version) - - # refresh disk model - if status == "not-available" or ( - status in ["on-disk", "in-memory"] and local_ts != current_upstream_ts - ): - # unload model from TFS - if status == "in-memory": - try: - logger.info( - f"unloading model {model_name} of version {model_version} from TFS" - ) - self._models.unload_model(model_name, model_version) - except Exception: - logger.info( - f"failed unloading model {model_name} of version {model_version} from TFS" - ) - raise - - # remove model from disk and references - if status in ["on-disk", "in-memory"]: - logger.info( - f"removing model references from memory and from disk for model {model_name} of version {model_version}" - ) - self._models.remove_model(model_name, model_version) - - # download model - logger.info( - f"downloading model {model_name} of version {model_version} from the s3 upstream" - ) - date = self._models.download_model( - upstream_model["bucket"], - model_name, - model_version, - upstream_model["path"], - ) - if not date: - raise WithBreak - current_upstream_ts = int(date.timestamp()) - - # load model - try: - logger.info( - f"loading model {model_name} of version {model_version} into memory" - ) - self._models.load_model( - model_name, - model_version, - current_upstream_ts, - [tag], - kwargs={ - "model_name": model_name, - "model_version": model_version, - "signature_key": self._determine_model_signature_key(model_name), - }, - ) - except Exception as e: - raise UserRuntimeException( - f"failed (re-)loading model {model_name} of version {model_version} (thread {td.get_ident()})", - str(e), - ) - - # run prediction - self._models.get_model(model_name, model_version, tag) - prediction = self._client.predict(model_input, model_name, model_version) - - return prediction - - def _load_model( - self, model_path: str, model_name: str, model_version: str, signature_key: Optional[str] - ) -> Any: - """ - Loads model into TFS. - Must only be used when caching enabled. - """ - - try: - model_dir = os.path.split(model_path)[0] - self._client.add_single_model( - model_name, model_version, model_dir, signature_key, timeout=30.0, max_retries=3 - ) - except Exception as e: - self._client.remove_single_model(model_name, model_version) - raise - - return "loaded tensorflow model" - - def _remove_models(self, model_ids: List[str]) -> None: - """ - Remove models from TFS. - Must only be used when caching enabled. - """ - logger.info(f"unloading models with model IDs {model_ids} from TFS") - - models = {} - for model_id in model_ids: - model_name, model_version = model_id.rsplit("-", maxsplit=1) - if model_name not in models: - models[model_name] = [model_version] - else: - models[model_name].append(model_version) - - model_names = [] - model_versions = [] - for model_name, versions in models.items(): - model_names.append(model_name) - model_versions.append(versions) - - self._client.remove_models(model_names, model_versions) - - def _determine_model_signature_key(self, model_name: str) -> Optional[str]: - """ - Determine what's the signature key for a given model from API spec. - """ - if self._models_dir: - return self._api_spec["handler"]["models"]["signature_key"] - return self._spec_models[model_name]["signature_key"] - - def _get_latest_model_version_from_tree(self, model_name: str, model_info: dict) -> str: - """ - Get the highest version for a specific model name. - Must only be used when processes_per_replica = 1 and caching is enabled. - """ - versions, timestamps = model_info["versions"], model_info["timestamps"] - return str(max(map(lambda x: int(x), versions))) - - def _is_model_caching_enabled(self) -> bool: - """ - Checks if model caching is enabled (models:cache_size and models:disk_cache_size). - """ - return ( - self._api_spec["handler"]["models"] - and self._api_spec["handler"]["models"]["cache_size"] is not None - and self._api_spec["handler"]["models"]["disk_cache_size"] is not None - ) - - @property - def metadata(self) -> dict: - """ - When caching is disabled, the returned dictionary will be like in the following example: - { - ... - "image-classifier-inception-1569014553": { - "disk_path": "/mnt/model/image-classifier-inception/1569014553", - "signature_def": { - "predict": { - "inputs": { - "images": { - "name": "images:0", - "dtype": "DT_FLOAT", - "tensorShape": { - "dim": [ - { - "size": "-1" - }, - { - "size": "-1" - }, - { - "size": "-1" - }, - { - "size": "3" - } - ] - } - } - }, - "outputs": { - "classes": { - "name": "module_apply_default/InceptionV3/Logits/SpatialSqueeze:0", - "dtype": "DT_FLOAT", - "tensorShape": { - "dim": [ - { - "size": "-1" - }, - { - "size": "1001" - } - ] - } - } - }, - "methodName": "tensorflow/serving/predict" - } - }, - "signature_key": "predict", - "input_signature": { - "images": { - "shape": [ - -1, - -1, - -1, - 3 - ], - "type": "float32" - } - }, - "timestamp": 1602025473 - } - ... - } - - Or when the caching is enabled, the following represents the kind of returned dictionary: - { - ... - "image-classifier-inception": { - "versions": [ - "1569014553", - "1569014559" - ], - "timestamps": [ - "1601668127", - "1601668120" - ] - } - ... - } - """ - - if not self._caching_enabled: - # the models dictionary has another field for each key entry - # called timestamp inserted by TFSAPIServingThreadUpdater thread - return self._client.models - else: - models_info = self._models_tree.get_all_models_info() - for model_name in models_info.keys(): - del models_info[model_name]["bucket"] - del models_info[model_name]["model_paths"] - return models_info - - @property - def caching(self) -> bool: - return self._caching_enabled diff --git a/python/serve/cortex_internal/lib/concurrency/__init__.py b/python/serve/cortex_internal/lib/concurrency/__init__.py deleted file mode 100644 index 3f7a2e9118..0000000000 --- a/python/serve/cortex_internal/lib/concurrency/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from cortex_internal.lib.concurrency.files import FileLock, LockedFile, get_locked_files -from cortex_internal.lib.concurrency.threading import ReadWriteLock, LockRead, LockWrite diff --git a/python/serve/cortex_internal/lib/concurrency/files.py b/python/serve/cortex_internal/lib/concurrency/files.py deleted file mode 100644 index e5746c883d..0000000000 --- a/python/serve/cortex_internal/lib/concurrency/files.py +++ /dev/null @@ -1,197 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import fcntl -import os -import time -from typing import List - -from cortex_internal.lib.exceptions import CortexException, WithBreak - - -class FileLock: - def __init__(self, lock_file: str, timeout: float = None, reader_lock: bool = False): - """ - Lock for files. Not thread-safe. Instantiate one lock per thread. - - lock_file - File to use as lock. - timeout - If used, a timeout exception will be raised if the lock can't be acquired. Measured in seconds. - reader_lock - When set to true, a shared lock (LOCK_SH) will be used. Otherwise, an exclusive lock (LOCK_EX) is used. - """ - self._lock_file = lock_file - self._file_handle = None - - self.timeout = timeout - self.reader_lock = reader_lock - self._time_loop = 0.001 - - # create lock if it doesn't exist - with open(self._lock_file, "w+") as f: - pass - - def acquire(self): - """ - To acquire the lock to resource. - """ - if self._file_handle: - return - - if not self.timeout: - self._file_handle = open(self._lock_file, "w") - if self.reader_lock: - fcntl.flock(self._file_handle, fcntl.LOCK_SH) - else: - fcntl.flock(self._file_handle, fcntl.LOCK_EX) - else: - start = time.time() - acquired = False - while start + self.timeout >= time.time(): - try: - self._file_handle = open(self._lock_file, "w") - if self.reader_lock: - fcntl.flock(self._file_handle, fcntl.LOCK_SH | fcntl.LOCK_NB) - else: - fcntl.flock(self._file_handle, fcntl.LOCK_EX | fcntl.LOCK_NB) - acquired = True - break - except OSError: - time.sleep(self._time_loop) - - if not acquired: - self._file_handle = None - raise TimeoutError( - "{} ms timeout on acquiring {} lock".format( - int(self.timeout * 1000), self._lock_file - ) - ) - - def release(self): - """ - To release the lock to resource. - """ - if not self._file_handle: - return - - fd = self._file_handle - self._file_handle = None - fcntl.flock(fd, fcntl.LOCK_UN) - fd.close() - - def __enter__(self): - self.acquire() - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.release() - return None - - def __del__(self): - self.release() - return None - - -class LockedFile: - """ - Create a lock-based file. - """ - - def __init__( - self, - filename: str, - mode: str, - timeout: float = None, - reader_lock: bool = False, - create_file_if_not_found: bool = True, - ): - """ - Open file with locked access to it - either with exclusive or shared lock. - - Args: - filename: Name of the file to open. - mode: Open mode for the file - same modes as for the built-in open function. - timeout: If set, it will try to acquire the lock for this amount of seconds. - reader_lock: Whether to use a shared lock or not. - create_file_if_not_found: Creates the file if it doesn't already exist. - """ - self.dir_path, self.basename = os.path.split(filename) - if self.basename == "": - raise CortexException(f"{filename} does not represent a path to file") - if not self.basename.startswith("."): - self.lockname = "." + self.basename - else: - self.lockname = self.basename - - self.filename = filename - self.mode = mode - self.timeout = timeout - self.reader_lock = reader_lock - self.create_file_if_not_found = create_file_if_not_found - - def __enter__(self): - lockfilepath = os.path.join(self.dir_path, self.lockname + ".lock") - self._lock = FileLock(lockfilepath, self.timeout, self.reader_lock) - self._lock.acquire() - try: - self._fd = open(self.filename, self.mode) - return self._fd - except FileNotFoundError: - if not self.create_file_if_not_found: - raise - except Exception as e: - self._lock.release() - raise e - try: - # w write mode - # r read mode - # a append mode - # - # w+ create file if it doesn't exist and open it in (over)write mode - # [it overwrites the file if it already exists] - # r+ open an existing file in read+write mode - # a+ create file if it doesn't exist and open it in append mode - if self.create_file_if_not_found and self.mode not in ["a+", "w+"]: - open(self.filename, "a+").close() - self._fd = open(self.filename, self.mode) - except Exception as e: - self._lock.release() - raise e - return self._fd - - def __exit__(self, exc_type, exc_value, traceback) -> bool: - # sometimes the `__del__` isn't run right away when the context manager exits - self.__del__() - - if exc_value is not None and exc_type is not WithBreak: - return False - return True - - def __del__(self): - if hasattr(self, "_fd"): - self._fd.close() - - if hasattr(self, "_lock"): - self._lock.release() - - -def get_locked_files(lock_dir: str) -> List[str]: - files = [os.path.basename(file) for file in os.listdir(lock_dir)] - locks = [f for f in files if f.endswith(".lock")] - - locked_files = [] - for lock in locks: - locked_file = os.path.splitext(lock)[0] - locked_file = locked_file[1:] # to ignore the added "." - locked_files.append(locked_file) - - return locked_files diff --git a/python/serve/cortex_internal/lib/concurrency/threading.py b/python/serve/cortex_internal/lib/concurrency/threading.py deleted file mode 100644 index 612c347165..0000000000 --- a/python/serve/cortex_internal/lib/concurrency/threading.py +++ /dev/null @@ -1,208 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import threading as td -from typing import Optional - - -class ReadWriteLock: - """ - Locking object allowing for write once, read many operations. - - The lock must not be acquired multiple times in a single thread without paired release calls. - - Can set different priority policies: "r" for read-preferring RW lock allowing for maximum concurrency - or can be set to "w" for write-preferring RW lock to prevent from starving the writer. - """ - - def __init__(self, prefer: str = "r"): - """ - "r" for read-preferring RW lock. - - "w" for write-preferring RW lock. - """ - self._prefer = prefer - self._write_preferred = td.Event() - self._write_preferred.set() - self._read_allowed = td.Condition(td.RLock()) - self._readers = [] - # a single writer is supported despite the fact that this is a list. - self._writers = [] - - def acquire(self, mode: str, timeout: Optional[float] = None) -> bool: - """ - Acquire a lock. - - Args: - mode: "r" for read lock, "w" for write lock. - timeout: How many seconds to wait to acquire the lock. - - Returns: - Whether the mode was valid or not. - """ - if not timeout: - acquire_timeout = -1 - else: - acquire_timeout = timeout - - if mode == "r": - # wait until "w" has been released - if self._prefer == "w": - if not self._write_preferred.wait(timeout): - self._throw_timeout_error(timeout, mode) - - # finish acquiring once all writers have released - if not self._read_allowed.acquire(timeout=acquire_timeout): - self._throw_timeout_error(timeout, mode) - # while loop only relevant when prefer == "r" - # but it's necessary when the preference policy is changed - while len(self._writers) > 0: - if not self._read_allowed.wait(timeout): - self._read_allowed.release() - self._throw_timeout_error(timeout, mode) - - self._readers.append(td.get_ident()) - self._read_allowed.release() - - elif mode == "w": - # stop "r" acquirers from acquiring - if self._prefer == "w": - self._write_preferred.clear() - - # acquire once all readers have released - if not self._read_allowed.acquire(timeout=acquire_timeout): - self._write_preferred.set() - self._throw_timeout_error(timeout, mode) - while len(self._readers) > 0: - if not self._read_allowed.wait(timeout): - self._read_allowed.release() - self._write_preferred.set() - self._throw_timeout_error(timeout, mode) - self._writers.append(td.get_ident()) - else: - return False - - return True - - def release(self, mode: str) -> bool: - """ - Releases a lock. - - Args: - mode: "r" for read lock, "w" for write lock. - - Returns: - Whether the mode was valid or not. - """ - if mode == "r": - # release and let writers acquire - self._read_allowed.acquire() - if not len(self._readers) - 1: - self._read_allowed.notifyAll() - self._readers.remove(td.get_ident()) - self._read_allowed.release() - - elif mode == "w": - # release and let readers acquire - self._writers.remove(td.get_ident()) - # notify all only relevant when prefer == "r" - # but it's necessary when the preference policy is changed - self._read_allowed.notifyAll() - self._read_allowed.release() - - # let "r" acquirers acquire again - if self._prefer == "w": - self._write_preferred.set() - else: - return False - - return True - - def set_preference_policy(self, prefer: str) -> bool: - """ - Change preference policy dynamically. - - When readers have acquired the lock, the policy change is immediate. - When a writer has acquired the lock, the policy change will block until the writer releases the lock. - - Args: - prefer: "r" for read-preferring RW lock, "w" for write-preferring RW lock. - - Returns: - True when the policy has been changed, false otherwise. - """ - if self._prefer == prefer: - return False - - self._read_allowed.acquire() - self._prefer = prefer - self._write_preferred.set() - self._read_allowed.release() - - return True - - def _throw_timeout_error(self, timeout: float, mode: str) -> None: - raise TimeoutError( - "{} ms timeout on acquiring '{}' lock in {} thread".format( - int(timeout * 1000), mode, td.get_ident() - ) - ) - - -class LockRead: - """ - To be used as: - - ```python - rw_lock = ReadWriteLock() - with LockRead(rw_lock): - # code - ``` - """ - - def __init__(self, lock: ReadWriteLock, timeout: Optional[float] = None): - self._lock = lock - self._timeout = timeout - - def __enter__(self): - self._lock.acquire("r", self._timeout) - return self - - def __exit__(self, exc_type, exc_value, traceback): - self._lock.release("r") - return False - - -class LockWrite: - """ - To be used as: - - ```python - rw_lock = ReadWriteLock() - with LockWrite(rw_lock): - # code - ``` - """ - - def __init__(self, lock: ReadWriteLock, timeout: Optional[float] = None): - self._lock = lock - self._timeout = timeout - - def __enter__(self): - self._lock.acquire("w", self._timeout) - return self - - def __exit__(self, exc_type, exc_value, traceback): - self._lock.release("w") - return False diff --git a/python/serve/cortex_internal/lib/exceptions.py b/python/serve/cortex_internal/lib/exceptions.py deleted file mode 100644 index 46c5414cc1..0000000000 --- a/python/serve/cortex_internal/lib/exceptions.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections import deque - - -class WithBreak(Exception): - """ - Gracefully exit with clauses. - """ - - pass - - -class CortexException(Exception): - def __init__(self, *messages): - super().__init__(": ".join(messages)) - self.errors = deque(messages) - - def wrap(self, *messages): - self.errors.extendleft(reversed(messages)) - - def __str__(self): - return self.stringify() - - def __repr__(self): - return self.stringify() - - def stringify(self): - return "error: " + ": ".join(self.errors) - - -class UserException(CortexException): - def __init__(self, *messages): - super().__init__(*messages) - - -class UserRuntimeException(UserException): - def __init__(self, *messages): - msg_list = list(messages) - msg_list.append("runtime exception") - super().__init__(*msg_list) diff --git a/python/serve/cortex_internal/lib/log.py b/python/serve/cortex_internal/lib/log.py deleted file mode 100644 index 04180d99d5..0000000000 --- a/python/serve/cortex_internal/lib/log.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import http -import logging -import logging.config -import threading - -import yaml -from pythonjsonlogger.jsonlogger import JsonFormatter - - -# https://github.com/encode/uvicorn/blob/master/uvicorn/logging.py -class CortexAccessFormatter(JsonFormatter): - def get_path(self, scope): - return scope.get("root_path", "") + scope["path"] - - def get_status_code(self, record): - status_code = record.__dict__["status_code"] - status_and_phrase = status_code - - try: - status_phrase = http.HTTPStatus(status_code).phrase - status_and_phrase = f"{status_code} {status_phrase}" - except: - pass - - return status_and_phrase - - def format(self, record): - if "scope" in record.__dict__: - scope = record.__dict__["scope"] - record.__dict__.update( - { - "method": scope["method"], - "status_code": self.get_status_code(record), - "path": self.get_path(scope), - } - ) - record.__dict__.pop("scope", None) - - return super().format(record) - - -logger = None -_logger_initializer_mutex = threading.Lock() - - -def configure_logger(name: str, config_file: str): - with _logger_initializer_mutex: - global logger - if logger is not None: - return logger - - logger = retrieve_logger(name, config_file) - return logger - - -def retrieve_logger(name: str, config_file: str): - with open(config_file, "r") as f: - config = yaml.safe_load(f.read()) - logging.config.dictConfig(config) - return logging.getLogger(name) diff --git a/python/serve/cortex_internal/lib/metrics.py b/python/serve/cortex_internal/lib/metrics.py deleted file mode 100644 index 4665ebda59..0000000000 --- a/python/serve/cortex_internal/lib/metrics.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Dict - -from datadog import DogStatsd - -from cortex_internal.lib.exceptions import UserException - - -def validate_metric(fn): - def _metric_validation(self, metric: str, value: float, tags: Dict[str, str] = None): - internal_prefixes = ("cortex_", "istio_") - if metric.startswith(internal_prefixes): - raise UserException( - f"Metric name ({metric}) is invalid because it starts with a cortex exclusive prefix.\n" - f"The following are prefixes are exclusive to cortex: {internal_prefixes}." - ) - return fn(self, metric=metric, value=value, tags=tags) - - _metric_validation.__name__ = fn.__name__ - return _metric_validation - - -class MetricsClient: - def __init__(self, statsd_client: DogStatsd): - self.__statsd = statsd_client - - @validate_metric - def gauge(self, metric: str, value: float, tags: Dict[str, str] = None): - """ - Record the value of a gauge. - - Example: - >>> metrics.gauge('active_connections', 1001, tags={"protocol": "http"}) - """ - return self.__statsd.gauge(metric, value=value, tags=transform_tags(tags)) - - @validate_metric - def increment(self, metric: str, value: float = 1, tags: Dict[str, str] = None): - """ - Increment the value of a counter. - - Example: - >>> metrics.increment('model_calls', 1, tags={"model_version": "v1"}) - """ - return self.__statsd.increment(metric, value=value, tags=transform_tags(tags)) - - @validate_metric - def histogram(self, metric: str, value: float, tags: Dict[str, str] = None): - """ - Set the value in a histogram metric - - Example: - >>> metrics.histogram('inference_time_milliseconds', 120, tags={"model_version": "v1"}) - """ - return self.__statsd.histogram(metric, value=value, tags=transform_tags(tags)) - - -def transform_tags(tags: Dict[str, str] = None): - return [f"{key}:{value}" for key, value in tags.items()] if tags else None diff --git a/python/serve/cortex_internal/lib/model/__init__.py b/python/serve/cortex_internal/lib/model/__init__.py deleted file mode 100644 index 7bf6fd0eac..0000000000 --- a/python/serve/cortex_internal/lib/model/__init__.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from cortex_internal.lib.model.model import ( - ModelsHolder, - LockedGlobalModelsGC, - LockedModel, - ids_to_models, -) -from cortex_internal.lib.model.tfs import TensorFlowServingAPI, TensorFlowServingAPIClones -from cortex_internal.lib.model.tree import ( - ModelsTree, - LockedModelsTree, - find_all_s3_models, -) -from cortex_internal.lib.model.type import get_models_from_api_spec, CuratedModelResources -from cortex_internal.lib.model.validation import ( - validate_models_dir_paths, - validate_model_paths, - ModelVersion, -) -from cortex_internal.lib.model.cron import ( - FileBasedModelsTreeUpdater, - FileBasedModelsGC, - find_ondisk_models_with_lock, - find_ondisk_model_ids_with_lock, - find_ondisk_model_info, - TFSModelLoader, - TFSAPIServingThreadUpdater, - find_ondisk_models, - ModelsGC, - ModelTreeUpdater, -) diff --git a/python/serve/cortex_internal/lib/model/cron.py b/python/serve/cortex_internal/lib/model/cron.py deleted file mode 100644 index 4c1130914a..0000000000 --- a/python/serve/cortex_internal/lib/model/cron.py +++ /dev/null @@ -1,1434 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import copy -import datetime -import glob -import json -import multiprocessing as mp -import os -import shutil -import threading as td -import time -from concurrent.futures import ThreadPoolExecutor -from typing import Dict, List, Tuple, Any, Union, Callable, Optional - -import grpc - -from cortex_internal.lib import util -from cortex_internal.lib.client.tensorflow import TensorFlowClient -from cortex_internal.lib.concurrency import LockedFile, get_locked_files -from cortex_internal.lib.exceptions import CortexException, WithBreak -from cortex_internal.lib.log import configure_logger -from cortex_internal.lib.model import ( - find_all_s3_models, - validate_model_paths, - TensorFlowServingAPI, - TensorFlowServingAPIClones, - ModelsHolder, - ids_to_models, - LockedGlobalModelsGC, - LockedModel, - get_models_from_api_spec, - ModelsTree, -) -from cortex_internal.lib.storage import S3 -from cortex_internal.lib.telemetry import get_default_tags, init_sentry -from cortex_internal.lib.type import ( - handler_type_from_api_spec, - PythonHandlerType, - TensorFlowHandlerType, - TensorFlowNeuronHandlerType, -) - -logger = configure_logger("cortex", os.environ["CORTEX_LOG_CONFIG_FILE"]) - - -class AbstractLoopingThread(td.Thread): - """ - Abstract class of the td.Thread class. - - Takes a method and keeps calling it in a loop every certain interval. - """ - - def __init__(self, interval: int, runnable: Callable[[], None]): - td.Thread.__init__(self, daemon=True) - - self._interval = interval - self._runnable = runnable - - if not callable(self._runnable): - raise ValueError("runnable parameter must be a callable method") - - self._event_stopper = td.Event() - self._stopped = False - - def run(self): - """ - td.Thread-specific method. - """ - while not self._event_stopper.is_set(): - self._runnable() - time.sleep(self._interval) - self._stopped = True - - def stop(self, blocking: bool = False): - """ - Stop the thread. - - Args: - blocking: Whether to wait until the thread is stopped or not. - """ - - self._event_stopper.set() - if blocking: - self.join() - - def join(self): - """ - Block until the thread finishes. - """ - - while not self._stopped: - time.sleep(0.001) - - -class FileBasedModelsTreeUpdater(mp.Process): - """ - Monitors the S3 path(s)/dir and continuously updates the file-based tree. - The model paths are validated - the bad paths are ignored. - When a new model is found, it updates the tree and downloads it - likewise when a model is removed. - """ - - def __init__( - self, - interval: int, - api_spec: dict, - download_dir: str, - temp_dir: str = "/tmp/cron", - lock_dir: str = "/run/cron", - ): - """ - Args: - interval: How often to update the models tree. Measured in seconds. - api_spec: Identical copy of pkg.type.spec.api.API. - download_dir: Path to where the models are stored. - temp_dir: Path to where the models are temporarily stored. - lock_dir: Path to where the resource locks are stored. - """ - - mp.Process.__init__(self, daemon=True) - - self._interval = interval - self._api_spec = api_spec - self._download_dir = download_dir - self._temp_dir = temp_dir - self._lock_dir = lock_dir - - self._s3_paths = [] - self._spec_models = get_models_from_api_spec(self._api_spec) - self._s3_model_names = self._spec_models.get_s3_model_names() - for model_name in self._s3_model_names: - self._s3_paths.append(self._spec_models[model_name]["path"]) - - self._handler_type = handler_type_from_api_spec(self._api_spec) - - if ( - self._handler_type == PythonHandlerType - and self._api_spec["handler"]["multi_model_reloading"] - ): - models = self._api_spec["handler"]["multi_model_reloading"] - elif self._handler_type != PythonHandlerType: - models = self._api_spec["handler"]["models"] - else: - models = None - - if models is None: - raise CortexException("no specified model") - - if models["dir"] is not None: - self._is_dir_used = True - self._models_dir = models["dir"] - else: - self._is_dir_used = False - self._models_dir = None - - try: - os.mkdir(self._lock_dir) - except FileExistsError: - pass - - self._ran_once = mp.Event() - self._event_stopper = mp.Event() - self._stopped = mp.Event() - - def run(self): - """ - mp.Process-specific method. - """ - - init_sentry(tags=get_default_tags()) - while not self._event_stopper.is_set(): - self._update_models_tree() - if not self._ran_once.is_set(): - self._ran_once.set() - time.sleep(self._interval) - self._stopped.set() - - def stop(self, blocking: bool = False): - """ - Trigger the process of stopping the process. - - Args: - blocking: Whether to wait until the process is stopped or not. - """ - - self._event_stopper.set() - if blocking: - self.join() - - def join(self): - """ - Block until the process exits. - """ - - while not self._stopped.is_set(): - time.sleep(0.001) - - def ran_once(self) -> bool: - """ - Tells whether the FileBasedModelsTreeUpdater loop has run at least once. - """ - - return self._ran_once.is_set() - - def _update_models_tree(self) -> None: - # get updated/validated paths/versions of the s3 models - ( - model_names, - versions, - model_paths, - sub_paths, - timestamps, - bucket_names, - ) = find_all_s3_models( - self._is_dir_used, - self._models_dir, - self._handler_type, - self._s3_paths, - self._s3_model_names, - ) - - # update models on the local disk if changes have been detected - # a model is updated if its directory tree has changed, if it's not present or if it doesn't exist on the upstream - with ThreadPoolExecutor(max_workers=5) as executor: - futures = [] - for idx, (model_name, bucket_name, bucket_sub_paths) in enumerate( - zip(model_names, bucket_names, sub_paths) - ): - futures += [ - executor.submit( - self._refresh_model, - idx, - model_name, - model_paths[idx], - versions[model_name], - timestamps[idx], - bucket_sub_paths, - bucket_name, - ) - ] - - [future.result() for future in futures] - - # remove models that no longer appear in model_names - for model_name, versions in find_ondisk_models_with_lock(self._lock_dir).items(): - if model_name in model_names or model_name in self._local_model_names: - continue - for ondisk_version in versions: - resource = os.path.join(self._lock_dir, model_name + "-" + ondisk_version + ".txt") - ondisk_model_version_path = os.path.join( - self._download_dir, model_name, ondisk_version - ) - with LockedFile(resource, "w+") as f: - shutil.rmtree(ondisk_model_version_path) - f.write("not-available") - shutil.rmtree(os.path.join(self._download_dir, model_name)) - - logger.debug(f"{self.__class__.__name__} cron heartbeat") - - def _refresh_model( - self, - idx: int, - model_name: str, - model_path: str, - versions: List[str], - timestamps: List[datetime.datetime], - sub_paths: List[str], - bucket_name: str, - ) -> None: - - client = S3(bucket_name) - - ondisk_model_path = os.path.join(self._download_dir, model_name) - for version, model_ts in zip(versions, timestamps): - - # for the lock file - resource = os.path.join(self._lock_dir, model_name + "-" + version + ".txt") - - # check if a model update is mandated - update_model = False - ondisk_model_version_path = os.path.join(ondisk_model_path, version) - if os.path.exists(ondisk_model_version_path): - local_paths = glob.glob( - os.path.join(ondisk_model_version_path, "**"), recursive=True - ) - local_paths = util.remove_non_empty_directory_paths(local_paths) - local_paths = [ - os.path.relpath(local_path, ondisk_model_version_path) - for local_path in local_paths - ] - local_paths = [path for path in local_paths if not path.startswith("../")] - - s3_model_version_path = os.path.join(model_path, version) - s3_paths = [ - os.path.relpath(sub_path, s3_model_version_path) for sub_path in sub_paths - ] - s3_paths = [path for path in s3_paths if not path.startswith("../")] - s3_paths = util.remove_non_empty_directory_paths(s3_paths) - - # update if the paths don't match - if set(local_paths) != set(s3_paths): - update_model = True - - # update if the timestamp is newer - with LockedFile(resource, "r", reader_lock=True) as f: - file_status = f.read() - if file_status == "" or file_status == "not-available": - raise WithBreak - current_model_ts = int(file_status.split(" ")[1]) - if current_model_ts < int(model_ts.timestamp()): - update_model = True - else: - update_model = True - - if update_model: - # download to a temp directory - temp_dest = os.path.join(self._temp_dir, model_name, version) - s3_src = os.path.join(model_path, version) - client.download_dir_contents(s3_src, temp_dest) - - # validate the downloaded model - model_contents = glob.glob(os.path.join(temp_dest, "**"), recursive=True) - model_contents = util.remove_non_empty_directory_paths(model_contents) - try: - validate_model_paths(model_contents, self._handler_type, temp_dest) - passed_validation = True - except CortexException: - passed_validation = False - shutil.rmtree(temp_dest) - - s3_path = S3.construct_s3_path(bucket_name, s3_src) - logger.debug( - f"failed validating model {model_name} of version {version} found at {s3_path} path" - ) - - # move the model to its destination directory - if passed_validation: - with LockedFile(resource, "w+") as f: - if os.path.exists(ondisk_model_version_path): - shutil.rmtree(ondisk_model_version_path) - shutil.move(temp_dest, ondisk_model_version_path) - f.write("available " + str(int(model_ts.timestamp()))) - - # remove the temp model directory if it exists - model_temp_dest = os.path.join(self._temp_dir, model_name) - if os.path.exists(model_temp_dest): - os.rmdir(model_temp_dest) - - # remove model versions if they are not found on the upstream - # except when the model version found on disk is 1 and the number of detected versions on the upstream is 0, - # thus indicating the 1-version on-disk model must be a model that came without a version - if os.path.exists(ondisk_model_path): - ondisk_model_versions = glob.glob(os.path.join(ondisk_model_path, "**")) - ondisk_model_versions = [ - os.path.relpath(path, ondisk_model_path) for path in ondisk_model_versions - ] - for ondisk_version in ondisk_model_versions: - if ondisk_version not in versions and (ondisk_version != "1" or len(versions) > 0): - resource = os.path.join( - self._lock_dir, model_name + "-" + ondisk_version + ".txt" - ) - ondisk_model_version_path = os.path.join(ondisk_model_path, ondisk_version) - with LockedFile(resource, "w+") as f: - shutil.rmtree(ondisk_model_version_path) - f.write("not-available") - - # remove the model directory if there are no models left - if len(glob.glob(os.path.join(ondisk_model_path, "**"))) == 0: - shutil.rmtree(ondisk_model_path) - - # if it's a non-versioned model ModelVersion.NOT_PROVIDED - if len(versions) == 0 and len(sub_paths) > 0: - - # for the lock file - resource = os.path.join(self._lock_dir, model_name + "-" + "1" + ".txt") - model_ts = int(timestamps[0].timestamp()) - - # check if a model update is mandated - update_model = False - ondisk_model_version_path = os.path.join(ondisk_model_path, "1") - if os.path.exists(ondisk_model_version_path): - local_paths = glob.glob( - os.path.join(ondisk_model_version_path, "**"), recursive=True - ) - local_paths = util.remove_non_empty_directory_paths(local_paths) - local_paths = [ - os.path.relpath(local_path, ondisk_model_version_path) - for local_path in local_paths - ] - local_paths = [path for path in local_paths if not path.startswith("../")] - - s3_model_version_path = model_path - s3_paths = [ - os.path.relpath(sub_path, s3_model_version_path) for sub_path in sub_paths - ] - s3_paths = [path for path in s3_paths if not path.startswith("../")] - s3_paths = util.remove_non_empty_directory_paths(s3_paths) - - # update if the paths don't match - if set(local_paths) != set(s3_paths): - update_model = True - - # update if the timestamp is newer - with LockedFile(resource, "r", reader_lock=True) as f: - file_status = f.read() - if file_status == "" or file_status == "not-available": - raise WithBreak() - current_model_ts = int(file_status.split(" ")[1]) - if current_model_ts < model_ts: - update_model = True - else: - update_model = True - - if not update_model: - return - - # download to a temp directory - temp_dest = os.path.join(self._temp_dir, model_name) - client.download_dir_contents(model_path, temp_dest) - - # validate the downloaded model - model_contents = glob.glob(os.path.join(temp_dest, "**"), recursive=True) - model_contents = util.remove_non_empty_directory_paths(model_contents) - try: - validate_model_paths(model_contents, self._handler_type, temp_dest) - passed_validation = True - except CortexException: - passed_validation = False - shutil.rmtree(temp_dest) - - s3_path = S3.construct_s3_path(bucket_name, model_path) - logger.debug( - f"failed validating model {model_name} of version {version} found at {s3_path} path" - ) - - # move the model to its destination directory - if passed_validation: - with LockedFile(resource, "w+") as f: - if os.path.exists(ondisk_model_version_path): - shutil.rmtree(ondisk_model_version_path) - shutil.move(temp_dest, ondisk_model_version_path) - f.write("available " + str(model_ts)) - - -class FileBasedModelsGC(AbstractLoopingThread): - """ - GC for models that no longer exist on disk. To be used with FileBasedModelsTreeUpdater. - - There has to be a FileBasedModelsGC cron for each API process. - - This needs to run on the API process because the FileBasedModelsTreeUpdater process cannot - unload the models from the API process' memory by itself. API process has to rely on this cron to do this periodically. - - This is for the case when the FileBasedModelsTreeUpdater process has removed models from disk and there are still models loaded into the API process' memory. - """ - - def __init__( - self, - interval: int, - models: ModelsHolder, - download_dir: str, - lock_dir: str = "/run/cron", - ): - """ - Args: - interval: How often to run the GC. Measured in seconds. - download_dir: Path to where the models are stored. - lock_dir: Path to where the resource locks are stored. - """ - AbstractLoopingThread.__init__(self, interval, self._run_gc) - - self._models = models - self._download_dir = download_dir - self._lock_dir = lock_dir - - def _run_gc(self): - on_disk_model_ids = find_ondisk_model_ids_with_lock(self._lock_dir) - in_memory_model_ids = self._models.get_model_ids() - - logger.debug(f"{self.__class__.__name__} cron heartbeat") - - for in_memory_id in in_memory_model_ids: - if in_memory_id in on_disk_model_ids: - continue - with LockedModel(self._models, "w", model_id=in_memory_id): - if self._models.has_model_id(in_memory_id)[0] == "in-memory": - model_name, model_version = in_memory_id.rsplit("-", maxsplit=1) - logger.info( - f"removing model {model_name} of version {model_version} from memory as it's no longer present on disk/S3 (thread {td.get_ident()})" - ) - self._models.remove_model_by_id( - in_memory_id, mem=True, disk=False, del_reference=True - ) - - -def find_ondisk_models_with_lock( - lock_dir: str, include_timestamps: bool = False -) -> Union[Dict[str, List[str]], Dict[str, Dict[str, Any]]]: - """ - Returns all available models from the disk. - To be used in conjunction with FileBasedModelsTreeUpdater/FileBasedModelsGC. - - Can be used for Python/TensorFlow clients. - - Args: - lock_dir: Path to where the resource locks are stored. - include_timestamps: Whether to include timestamps for each version of each model. - - Returns: - Dictionary with available model names and their associated versions when include_timestamps is False. - { - "model-A": ["177", "245", "247"], - "model-B": ["1"], - ... - } - - Dictionary with available model names and their associated versions/timestamps when include_timestamps is True. - { - "model-A": { - "versions": ["177", "245", "247"], - "timestamps": [1602198945, 1602198946, 1602198947] - } - "model-B": { - "versions": ["1"], - "timestamps": [1602198567] - }, - ... - } - """ - models = {} - - for locked_file in get_locked_files(lock_dir): - with LockedFile(os.path.join(lock_dir, locked_file), "r", reader_lock=True) as f: - status = f.read() - - if status.startswith("available"): - timestamp = int(status.split(" ")[1]) - _model_name, _model_version = os.path.splitext(locked_file)[0].rsplit("-", maxsplit=1) - if _model_name not in models: - if include_timestamps: - models[_model_name] = {"versions": [_model_version], "timestamps": [timestamp]} - else: - models[_model_name] = [_model_version] - else: - if include_timestamps: - models[_model_name]["versions"] += [_model_version] - models[_model_name]["timestamps"] += [timestamp] - else: - models[_model_name] += [_model_version] - - return models - - -def find_ondisk_model_ids_with_lock(lock_dir: str) -> List[str]: - """ - Returns all available model IDs from the disk. - To be used in conjunction with FileBasedModelsTreeUpdater/FileBasedModelsGC. - - Can be used for Python/TensorFlow clients. - - Args: - lock_dir: Path to where the resource locks are stored. - - Returns: - A list with all model IDs present on disk. - """ - model_ids = [] - - for locked_file in get_locked_files(lock_dir): - with LockedFile(os.path.join(lock_dir, locked_file), "r", reader_lock=True) as f: - status = f.read() - - if status.startswith("available"): - model_id = os.path.splitext(locked_file)[0] - model_ids.append(model_id) - - return model_ids - - -def find_ondisk_model_info(lock_dir: str, model_name: str) -> Tuple[List[str], List[int]]: - """ - Returns all available versions/timestamps of a model from the disk. - To be used in conjunction with FileBasedModelsTreeUpdater/FileBasedModelsGC. - - Can be used for Python/TensorFlow clients. - - Args: - lock_dir: Path to where the resource locks are stored. - model_name: Name of the model as specified in handler:models:paths:name, _cortex_default when handler:models:path is set or the discovered model names when handler:models:dir is used. - - Returns: - 2-element tuple made of a list with the available versions and a list with the corresponding timestamps for each model. Empty when the model is not available. - """ - versions = [] - timestamps = [] - - for locked_file in get_locked_files(lock_dir): - _model_name, _model_version = os.path.splitext(locked_file)[0].rsplit("-", maxsplit=1) - if _model_name != model_name: - continue - - with LockedFile(os.path.join(lock_dir, locked_file), "r", reader_lock=True) as f: - status = f.read() - if not status.startswith("available"): - continue - - current_upstream_ts = int(status.split(" ")[1]) - timestamps.append(current_upstream_ts) - versions.append(_model_version) - - return (versions, timestamps) - - -class TFSModelLoader(mp.Process): - """ - Monitors the S3 path(s)/dir and continuously updates the models on TFS. - The model paths are validated - the bad paths are ignored. - When a new model is found, it updates the tree, downloads it and loads it into memory - likewise when a model is removed. - """ - - def __init__( - self, - interval: int, - api_spec: dict, - tfs_model_dir: str, - download_dir: str, - address: Optional[str] = None, - addresses: Optional[List[str]] = None, - temp_dir: str = "/tmp/cron", - lock_dir: str = "/run/cron", - ): - """ - Args: - interval: How often to update the models tree. Measured in seconds. - api_spec: Identical copy of pkg.type.spec.api.API. - address: An address with the "host:port" format to where TFS is located. - addresses: A list of addresses with the "host:port" format to where the TFS servers are located. - tfs_model_dir: Path to where the models are stored within the TFS container. - download_dir: Path to where the models are stored. - temp_dir: Directory where models are temporarily stored. - lock_dir: Directory in which model timestamps are stored. - """ - - if address and addresses: - raise ValueError("address and addresses arguments cannot be passed in at the same time") - if not address and not addresses: - raise ValueError("must pass in at least one of the two arguments: address or addresses") - - mp.Process.__init__(self, daemon=True) - - self._interval = interval - self._api_spec = api_spec - self._tfs_model_dir = tfs_model_dir - self._download_dir = download_dir - self._temp_dir = temp_dir - self._lock_dir = lock_dir - - if address: - self._tfs_address = address - self._tfs_addresses = None - else: - self._tfs_address = None - self._tfs_addresses = addresses - - self._s3_paths = [] - self._spec_models = get_models_from_api_spec(self._api_spec) - self._s3_model_names = self._spec_models.get_s3_model_names() - for model_name in self._s3_model_names: - self._s3_paths.append(self._spec_models[model_name]["path"]) - - if ( - self._api_spec["handler"]["models"] is not None - and self._api_spec["handler"]["models"]["dir"] is not None - ): - self._is_dir_used = True - self._models_dir = self._api_spec["handler"]["models"]["dir"] - else: - self._is_dir_used = False - self._models_dir = None - - if self._api_spec["handler"]["type"] == "tensorflow": - if self._api_spec["compute"]["inf"] > 0: - self._handler_type = TensorFlowNeuronHandlerType - else: - self._handler_type = TensorFlowHandlerType - else: - raise CortexException( - "'tensorflow' handler type is the only allowed type for this cron" - ) - - self._ran_once = mp.Event() - self._event_stopper = mp.Event() - self._stopped = mp.Event() - - # keeps an old record of the model timestamps - self._old_ts_state = {} - - def run(self): - """ - mp.Process-specific method. - """ - - init_sentry(tags=get_default_tags()) - if self._tfs_address: - self._client = TensorFlowServingAPI(self._tfs_address) - else: - self._client = TensorFlowServingAPIClones(self._tfs_addresses) - - # wait until TFS is responsive - while not self._client.is_tfs_accessible(): - self._reset_when_tfs_unresponsive() - time.sleep(1.0) - - while not self._event_stopper.is_set(): - success = self._update_models() - if success and not self._ran_once.is_set(): - self._ran_once.set() - logger.debug(f"{self.__class__.__name__} cron heartbeat") - time.sleep(self._interval) - self._stopped.set() - - def stop(self, blocking: bool = False): - """ - Trigger the process of stopping the process. - - Args: - blocking: Whether to wait until the process is stopped or not. - """ - - self._event_stopper.set() - if blocking: - self.join() - - def join(self): - """ - Block until the process exits. - """ - - while not self._stopped.is_set(): - time.sleep(0.001) - - def ran_once(self) -> bool: - """ - Tells whether the TFS loader loop has run at least once. - """ - - return self._ran_once.is_set() - - def _update_models(self) -> bool: - # get updated/validated paths/versions of the S3 models - ( - model_names, - versions, - model_paths, - sub_paths, - timestamps, - bucket_names, - ) = find_all_s3_models( - self._is_dir_used, - self._models_dir, - self._handler_type, - self._s3_paths, - self._s3_model_names, - ) - - # update models on the local disk if changes have been detected - # a model is updated if its directory tree has changed, if it's not present or if it doesn't exist on the upstream - with ThreadPoolExecutor(max_workers=5) as executor: - futures = [] - for idx, (model_name, bucket_name, bucket_sub_paths) in enumerate( - zip(model_names, bucket_names, sub_paths) - ): - futures += [ - executor.submit( - self._refresh_model, - idx, - model_name, - model_paths[idx], - versions[model_name], - timestamps[idx], - bucket_sub_paths, - bucket_name, - ) - ] - [future.result() for future in futures] - - # remove models that no longer appear in model_names - for model_name, model_versions in find_ondisk_models(self._download_dir).items(): - if model_name in model_names: - continue - for ondisk_version in model_versions: - ondisk_model_version_path = os.path.join( - self._download_dir, model_name, ondisk_version - ) - shutil.rmtree(ondisk_model_version_path) - shutil.rmtree(os.path.join(self._download_dir, model_name)) - self._client.remove_models([model_name], [model_versions]) - - # check tfs connection - if not self._client.is_tfs_accessible(): - self._reset_when_tfs_unresponsive() - return False - - # remove versioned models from TFS that no longer exist on disk - tfs_model_ids = self._client.get_registered_model_ids() - ondisk_models = find_ondisk_models(self._download_dir) - ondisk_model_ids = [] - for model_name, model_versions in ondisk_models.items(): - for model_version in model_versions: - ondisk_model_ids.append(f"{model_name}-{model_version}") - for tfs_model_id in tfs_model_ids: - if tfs_model_id not in ondisk_model_ids: - try: - model_name, model_version = tfs_model_id.rsplit("-", maxsplit=1) - self._client.remove_single_model(model_name, model_version) - logger.info( - "model '{}' of version '{}' has been unloaded".format( - model_name, model_version - ) - ) - except grpc.RpcError as err: - if err.code() == grpc.StatusCode.UNAVAILABLE: - logger.warning( - "TFS server unresponsive after trying to load model '{}' of version '{}': {}".format( - model_name, model_version, str(err) - ) - ) - self._reset_when_tfs_unresponsive() - return False - - # # update TFS models - current_ts_state = {} - for model_name, model_versions in ondisk_models.items(): - try: - ts = self._update_tfs_model( - model_name, model_versions, timestamps, model_names, versions - ) - except grpc.RpcError: - return False - current_ts_state = {**current_ts_state, **ts} - - # save model timestamp states - for model_id, ts in current_ts_state.items(): - self._old_ts_state[model_id] = ts - - # remove model timestamps that no longer exist - loaded_model_ids = self._client.models.keys() - aux_ts_state = self._old_ts_state.copy() - for model_id in self._old_ts_state.keys(): - if model_id not in loaded_model_ids: - del aux_ts_state[model_id] - self._old_ts_state = aux_ts_state - - # save model timestamp states to disk - # could be cast to a short-lived thread - # required for printing the model stats when cortex getting - resource = os.path.join(self._lock_dir, "model_timestamps.json") - with open(resource, "w") as f: - json.dump(self._old_ts_state, f, indent=2) - - # save model stats for TFS to disk - resource = os.path.join(self._lock_dir, "models_tfs.json") - with open(resource, "w") as f: - json.dump(self._client.models, f, indent=2) - - return True - - def _refresh_model( - self, - idx: int, - model_name: str, - model_path: str, - versions: List[str], - timestamps: List[datetime.datetime], - sub_paths: List[str], - bucket_name: str, - ) -> None: - - client = S3(bucket_name) - - ondisk_model_path = os.path.join(self._download_dir, model_name) - for version, model_ts in zip(versions, timestamps): - - # check if a model update is mandated - update_model = False - ondisk_model_version_path = os.path.join(ondisk_model_path, version) - if os.path.exists(ondisk_model_version_path): - local_paths = glob.glob( - os.path.join(ondisk_model_version_path, "**"), recursive=True - ) - local_paths = util.remove_non_empty_directory_paths(local_paths) - local_paths = [ - os.path.relpath(local_path, ondisk_model_version_path) - for local_path in local_paths - ] - local_paths = [path for path in local_paths if not path.startswith("../")] - - s3_model_version_path = os.path.join(model_path, version) - s3_paths = [ - os.path.relpath(sub_path, s3_model_version_path) for sub_path in sub_paths - ] - s3_paths = [path for path in s3_paths if not path.startswith("../")] - s3_paths = util.remove_non_empty_directory_paths(s3_paths) - - if set(local_paths) != set(s3_paths): - update_model = True - - model_id = f"{model_name}-{version}" - if self._is_this_a_newer_model_id(model_id, int(model_ts.timestamp())): - update_model = True - else: - update_model = True - - if update_model: - # download to a temp directory - temp_dest = os.path.join(self._temp_dir, model_name, version) - s3_src = os.path.join(model_path, version) - client.download_dir_contents(s3_src, temp_dest) - - # validate the downloaded model - model_contents = glob.glob(os.path.join(temp_dest, "**"), recursive=True) - model_contents = util.remove_non_empty_directory_paths(model_contents) - try: - validate_model_paths(model_contents, self._handler_type, temp_dest) - passed_validation = True - except CortexException: - passed_validation = False - shutil.rmtree(temp_dest) - - s3_path = S3.construct_s3_path(bucket_name, model_path) - logger.debug( - f"failed validating model {model_name} of version {version} found at {s3_path} path" - ) - - # move the model to its destination directory - if passed_validation: - if os.path.exists(ondisk_model_version_path): - shutil.rmtree(ondisk_model_version_path) - shutil.move(temp_dest, ondisk_model_version_path) - - # remove the temp model directory if it exists - model_temp_dest = os.path.join(self._temp_dir, model_name) - if os.path.exists(model_temp_dest): - os.rmdir(model_temp_dest) - - # remove model versions if they are not found on the upstream - # except when the model version found on disk is 1 and the number of detected versions on the upstream is 0, - # thus indicating the 1-version on-disk model must be a model that came without a version - if os.path.exists(ondisk_model_path): - ondisk_model_versions = glob.glob(os.path.join(ondisk_model_path, "**")) - ondisk_model_versions = [ - os.path.relpath(path, ondisk_model_path) for path in ondisk_model_versions - ] - for ondisk_version in ondisk_model_versions: - if ondisk_version not in versions and (ondisk_version != "1" or len(versions) > 0): - ondisk_model_version_path = os.path.join(ondisk_model_path, ondisk_version) - shutil.rmtree(ondisk_model_version_path) - - if len(glob.glob(os.path.join(ondisk_model_path, "**"))) == 0: - shutil.rmtree(ondisk_model_path) - - # if it's a non-versioned model ModelVersion.NOT_PROVIDED - if len(versions) == 0 and len(sub_paths) > 0: - - model_ts = timestamps[0] - - # check if a model update is mandated - update_model = False - ondisk_model_version_path = os.path.join(ondisk_model_path, "1") - if os.path.exists(ondisk_model_version_path): - local_paths = glob.glob( - os.path.join(ondisk_model_version_path, "**"), recursive=True - ) - local_paths = util.remove_non_empty_directory_paths(local_paths) - local_paths = [ - os.path.relpath(local_path, ondisk_model_version_path) - for local_path in local_paths - ] - local_paths = [path for path in local_paths if not path.startswith("../")] - - s3_model_version_path = model_path - s3_paths = [ - os.path.relpath(sub_path, s3_model_version_path) for sub_path in sub_paths - ] - s3_paths = [path for path in s3_paths if not path.startswith("../")] - s3_paths = util.remove_non_empty_directory_paths(s3_paths) - - # update if the paths don't match - if set(local_paths) != set(s3_paths): - update_model = True - - model_id = f"{model_name}-1" - if self._is_this_a_newer_model_id(model_id, int(model_ts.timestamp())): - update_model = True - else: - update_model = True - - if not update_model: - return - - # download to a temp directory - temp_dest = os.path.join(self._temp_dir, model_name) - client.download_dir_contents(model_path, temp_dest) - - # validate the downloaded model - model_contents = glob.glob(os.path.join(temp_dest, "**"), recursive=True) - model_contents = util.remove_non_empty_directory_paths(model_contents) - try: - validate_model_paths(model_contents, self._handler_type, temp_dest) - passed_validation = True - except CortexException: - passed_validation = False - shutil.rmtree(temp_dest) - - s3_path = S3.construct_s3_path(bucket_name, model_path) - logger.debug( - f"failed validating model {model_name} of version {version} found at {s3_path} path" - ) - - # move the model to its destination directory - if passed_validation: - if os.path.exists(ondisk_model_version_path): - shutil.rmtree(ondisk_model_version_path) - shutil.move(temp_dest, ondisk_model_version_path) - - def _update_tfs_model( - self, - model_name: str, - model_versions: List[str], - _s3_timestamps: List[List[datetime.datetime]], - _s3_model_names: List[str], - _s3_versions: Dict[str, List[str]], - ) -> Optional[dict]: - """ - Compares the existing models from TFS with those present on disk. - Does the loading/unloading/reloading of models. - - From the _s3_timestamps, _s3_model_names, _s3_versions params, only the fields of the respective model name are used. - """ - - # to prevent overwriting mistakes - s3_timestamps = copy.deepcopy(_s3_timestamps) - s3_model_names = copy.deepcopy(_s3_model_names) - s3_versions = copy.deepcopy(_s3_versions) - - current_ts_state = {} - - # get the right order of model versions with respect to the model ts order - model_timestamps = s3_timestamps[s3_model_names.index(model_name)] - filtered_model_versions = [] - if len(s3_versions[model_name]) == 0: - filtered_model_versions = ["1"] * len(model_timestamps) - else: - for idx in range(len(model_timestamps)): - if s3_versions[model_name][idx] in model_versions: - filtered_model_versions.append(s3_versions[model_name][idx]) - - for model_version, model_ts in zip(filtered_model_versions, model_timestamps): - model_ts = int(model_ts.timestamp()) - - # remove outdated model - model_id = f"{model_name}-{model_version}" - is_model_outdated = False - first_time_load = False - if model_id in self._old_ts_state and self._old_ts_state[model_id] != model_ts: - try: - self._client.remove_single_model(model_name, model_version) - except grpc.RpcError as err: - if err.code() == grpc.StatusCode.UNAVAILABLE: - logger.warning( - "TFS server unresponsive after trying to unload model '{}' of version '{}': {}".format( - model_name, model_version, str(err) - ) - ) - logger.warning("waiting for tensorflow serving") - raise - is_model_outdated = True - elif model_id not in self._old_ts_state: - first_time_load = True - - if not is_model_outdated and not first_time_load: - continue - - # load model - model_disk_path = os.path.join(self._tfs_model_dir, model_name) - try: - self._client.add_single_model( - model_name, - model_version, - model_disk_path, - self._determine_model_signature_key(model_name), - timeout=30.0, - ) - except Exception as e: - try: - self._client.remove_single_model(model_name, model_version) - logger.warning( - "model '{}' of version '{}' couldn't be loaded: {}".format( - model_name, model_version, str(e) - ) - ) - except grpc.RpcError as err: - if err.code() == grpc.StatusCode.UNAVAILABLE: - logger.warning( - "TFS server unresponsive after trying to load model '{}' of version '{}': {}".format( - model_name, model_version, str(err) - ) - ) - self._reset_when_tfs_unresponsive() - raise - - is_model_outdated = False - first_time_load = False - - # save timestamp of loaded model - current_ts_state[model_id] = model_ts - if is_model_outdated: - logger.info( - "model '{}' of version '{}' has been reloaded".format(model_name, model_version) - ) - elif first_time_load: - logger.info( - "model '{}' of version '{}' has been loaded".format(model_name, model_version) - ) - - return current_ts_state - - def _is_this_a_newer_model_id(self, model_id: str, timestamp: int) -> bool: - return model_id in self._old_ts_state and self._old_ts_state[model_id] < timestamp - - def _determine_model_signature_key(self, model_name: str) -> Optional[str]: - if self._models_dir: - signature_key = self._api_spec["handler"]["models"]["signature_key"] - else: - signature_key = self._spec_models[model_name]["signature_key"] - - return signature_key - - def _reset_when_tfs_unresponsive(self): - logger.warning("waiting for tensorflow serving") - - if self._tfs_address: - self._client = TensorFlowServingAPI(self._tfs_address) - else: - self._client = TensorFlowServingAPIClones(self._tfs_addresses) - - resource = os.path.join(self._lock_dir, "models_tfs.json") - with open(resource, "w") as f: - json.dump(self._client.models, f, indent=2) - - -class TFSAPIServingThreadUpdater(AbstractLoopingThread): - """ - When live reloading and the TensorFlow type is used, the serving container - needs to have a way of accessing the models' metadata which is generated using the TFSModelLoader cron. - - This cron runs on each serving process and periodically reads the exported metadata from the TFSModelLoader cron. - This is then fed into each serving process. - """ - - def __init__( - self, - interval: Union[int, float], - client: TensorFlowClient, - lock_dir: str = "/run/cron", - ): - AbstractLoopingThread.__init__(self, interval, self._run_tfs) - - self._client = client - self._lock_dir = lock_dir - - def _run_tfs(self) -> None: - self._client.sync_models(self._lock_dir) - - -def find_ondisk_models(models_dir: str) -> Dict[str, List[str]]: - """ - Returns all available models from the disk. - To be used in conjunction with TFSModelLoader. - - This function should never be used for determining whether a model has to be loaded or not. - Can be used for Python/TensorFlow clients. - - Args: - models_dir: Path to where the models are stored. - - Returns: - Dictionary with available model names and their associated versions. - { - "model-A": [177, 245, 247], - "model-B": [1], - ... - } - """ - - models = {} - model_names = [os.path.basename(file) for file in os.listdir(models_dir)] - - for model_name in model_names: - model_versions = os.listdir(os.path.join(models_dir, model_name)) - models[model_name] = model_versions - - return models - - -class ModelsGC(AbstractLoopingThread): - """ - GC for models loaded into memory and/or stored on disk. - - If the number of models exceeds the cache size, then evict the LRU models. - Also removes models that are no longer present in the model tree. - """ - - def __init__( - self, - interval: int, - api_spec: dict, - models: ModelsHolder, - tree: ModelsTree, - ): - """ - Args: - interval: How often to update the models tree. Measured in seconds. - api_spec: Identical copy of pkg.type.spec.api.API. - models: The object holding all models in memory / on disk. - tree: Model tree representation of the available models on the S3 upstream. - """ - - AbstractLoopingThread.__init__(self, interval, self._run_gc) - - self._api_spec = api_spec - self._models = models - self._tree = tree - - self._spec_models = get_models_from_api_spec(self._api_spec) - - # run the cron every 10 seconds - self._lock_timeout = 10.0 - - self._event_stopper = td.Event() - self._stopped = False - - def _run_gc(self) -> None: - - # are there any models to collect (aka remove) from cache - with LockedGlobalModelsGC(self._models, "r"): - collectible, _, _ = self._models.garbage_collect(dry_run=True) - if not collectible: - self._remove_stale_models() - return - - # try to grab exclusive access to all models with shared access preference - # and if it works, remove excess models from cache - self._models.set_global_preference_policy("r") - with LockedGlobalModelsGC(self._models, "w", self._lock_timeout) as lg: - acquired = lg.acquired - if not acquired: - raise WithBreak - - _, memory_evicted_model_ids, disk_evicted_model_ids = self._models.garbage_collect() - - # otherwise, grab exclusive access to all models with exclusive access preference - # and remove excess models from cache - if not acquired: - self._models.set_global_preference_policy("w") - with LockedGlobalModelsGC(self._models, "w"): - _, memory_evicted_model_ids, disk_evicted_model_ids = self._models.garbage_collect() - self._models.set_global_preference_policy("r") - - memory_evicted_models = ids_to_models(memory_evicted_model_ids) - disk_evicted_models = ids_to_models(disk_evicted_model_ids) - - self._log_removed_models(memory_evicted_models, memory=True) - self._log_removed_models(disk_evicted_models, disk=True) - - self._remove_stale_models() - - def _remove_stale_models(self) -> None: - """ - Remove models that exist locally in-memory and on-disk that no longer appear on S3 - """ - - # get available upstream S3 model IDs - s3_model_names = self._tree.get_model_names() - s3_model_versions = [ - self._tree.model_info(model_name)["versions"] for model_name in s3_model_names - ] - s3_model_ids = [] - for model_name, model_versions in zip(s3_model_names, s3_model_versions): - if len(model_versions) == 0: - continue - for model_version in model_versions: - s3_model_ids.append(f"{model_name}-{model_version}") - - # get model IDs loaded into memory or on disk. - with LockedGlobalModelsGC(self._models, "r"): - present_model_ids = self._models.get_model_ids() - - # remove models that don't exist in the S3 upstream - ghost_model_ids = list(set(present_model_ids) - set(s3_model_ids)) - for model_id in ghost_model_ids: - model_name, model_version = model_id.rsplit("-", maxsplit=1) - with LockedModel(self._models, "w", model_name, model_version): - status, ts = self._models.has_model(model_name, model_version) - if status == "in-memory": - logger.info( - f"unloading stale model {model_name} of version {model_version} using the garbage collector" - ) - self._models.unload_model(model_name, model_version) - if status in ["in-memory", "on-disk"]: - logger.info( - f"removing stale model {model_name} of version {model_version} using the garbage collector" - ) - self._models.remove_model(model_name, model_version) - - def _log_removed_models( - self, models: Dict[str, List[str]], memory: bool = False, disk: bool = False - ) -> None: - """ - Log the removed models from disk/memory. - """ - - if len(models) == 0: - return None - - if len(models) > 1: - message = "models " - else: - message = "model " - - for idx, (model_name, versions) in enumerate(models.items()): - message += f"{model_name} " - if len(versions) == 1: - message += f"(version {versions[0]})" - else: - message += f"(versions {','.join(versions)})" - if idx + 1 < len(models): - message += ", " - else: - if memory: - message += " removed from the memory cache using the garbage collector" - if disk: - message += " removed from the disk cache using the garbage collector" - - logger.info(message) - - -class ModelTreeUpdater(AbstractLoopingThread): - """ - Model tree updater. Updates a local representation of all available models from the S3 upstreams. - """ - - def __init__(self, interval: int, api_spec: dict, tree: ModelsTree, ondisk_models_dir: str): - """ - Args: - interval: How often to update the models tree. Measured in seconds. - api_spec: Identical copy of pkg.type.spec.api.API. - tree: Model tree representation of the available models on S3. - ondisk_models_dir: Where the models are stored on disk. Necessary when local models are used. - """ - - AbstractLoopingThread.__init__(self, interval, self._update_models_tree) - - self._api_spec = api_spec - self._tree = tree - self._ondisk_models_dir = ondisk_models_dir - - self._s3_paths = [] - self._spec_models = get_models_from_api_spec(self._api_spec) - self._s3_model_names = self._spec_models.get_s3_model_names() - for model_name in self._s3_model_names: - self._s3_paths.append(self._spec_models[model_name]["path"]) - - self._handler_type = handler_type_from_api_spec(self._api_spec) - - if ( - self._handler_type == PythonHandlerType - and self._api_spec["handler"]["multi_model_reloading"] - ): - models = self._api_spec["handler"]["multi_model_reloading"] - elif self._handler_type != PythonHandlerType: - models = self._api_spec["handler"]["models"] - else: - models = None - - if models is None: - raise CortexException("no specified model") - - if models and models["dir"] is not None: - self._is_dir_used = True - self._models_dir = models["dir"] - else: - self._is_dir_used = False - self._models_dir = None - - def _update_models_tree(self) -> None: - # get updated/validated paths/versions of the S3 models - ( - model_names, - versions, - model_paths, - sub_paths, - timestamps, - bucket_names, - ) = find_all_s3_models( - self._is_dir_used, - self._models_dir, - self._handler_type, - self._s3_paths, - self._s3_model_names, - ) - - # update model tree - self._tree.update_models( - model_names, - versions, - model_paths, - sub_paths, - timestamps, - bucket_names, - ) - - logger.debug(f"{self.__class__.__name__} cron heartbeat") diff --git a/python/serve/cortex_internal/lib/model/model.py b/python/serve/cortex_internal/lib/model/model.py deleted file mode 100644 index 9354fa6a8e..0000000000 --- a/python/serve/cortex_internal/lib/model/model.py +++ /dev/null @@ -1,588 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime -import os -import shutil -import threading as td -import time -from typing import Dict, List, Any, Tuple, Callable, Optional - -from cortex_internal.lib.concurrency import ReadWriteLock -from cortex_internal.lib.exceptions import WithBreak -from cortex_internal.lib.log import configure_logger -from cortex_internal.lib.type import HandlerType - -logger = configure_logger("cortex", os.environ["CORTEX_LOG_CONFIG_FILE"]) - - -class ModelsHolder: - """ - Class to hold models in memory and references for those on disk. - Can limit the number of models in memory/on-disk based on an LRU policy - by default, it's disabled. - """ - - def __init__( - self, - handler_type: HandlerType, - model_dir: str, - temp_dir: str = "/tmp/cron", - mem_cache_size: int = -1, - disk_cache_size: int = -1, - on_download_callback: Optional[ - Callable[[HandlerType, str, str, str, str, str, str, str], datetime.datetime] - ] = None, - on_load_callback: Optional[Callable[[str], Any]] = None, - on_remove_callback: Optional[Callable[[List[str]], None]] = None, - ): - """ - Args: - handler_type: The handler type. Can be PythonHandler, TensorFlowHandler or TensorFlowNeuronHandler. - model_dir: Where models are saved on disk. - temp_dir: Where models are temporary stored for validation. - mem_cache_size: The size of the cache for in-memory models. For negative values, the cache is disabled. - disk_cache_size: The size of the cache for on-disk models. For negative values, the cache is disabled. - on_download_callback(, , , , , , ): Function to be called for downloading a model to disk. Returns the downloaded model's upstream timestamp, otherwise a negative number is returned. - on_load_callback(, **kwargs): Function to be called when a model is loaded from disk. Returns the actual model. May throw exceptions if it doesn't work. - on_remove_callback(, **kwargs): Function to be called when the GC is called. E.g. for the TensorFlow type, the function would communicate with TFS to unload models. - """ - self._handler_type = handler_type - self._model_dir = model_dir - self._temp_dir = temp_dir - - if mem_cache_size > 0 and disk_cache_size > 0 and mem_cache_size > disk_cache_size: - raise RuntimeError( - f"mem_cache_size ({mem_cache_size}) must be equal or smaller than disk_cache_size ({disk_cache_size})" - ) - - if mem_cache_size == 0 or disk_cache_size == 0: - raise RuntimeError( - "mem_cache_size or disk_cache_size can't be set to 0; must be negative to disable the cache or positive to have it enabled" - ) - - self._mem_cache_size = mem_cache_size - self._disk_cache_size = disk_cache_size - - self._download_callback = on_download_callback - self._load_callback = on_load_callback - self._remove_callback = on_remove_callback - - self._models = {} # maps the model ID to the model that's placed in memory - self._timestamps = {} # maps the model ID to the last access time of the model - self._locks = {} # maps the model ID to the underlying lock for each model - - self._create_lock = ( - td.RLock() - ) # to ensure atomicity when 2 threads are trying to create locks for the same model ID that doesn't exist in self._locks - self._global_lock = ReadWriteLock() - - def set_callback(self, ctype: str, callback: Callable) -> None: - """ - Sets a callback. - - Args: - ctype: "download", "load" or "remove" callback type - see the constructor to mark each one. - callback: The actual callback. - """ - if ctype == "download": - self._download_callback = callback - if ctype == "load": - self._load_callback = callback - if ctype == "remove": - self._remove_callback = callback - - def global_acquire(self, mode: str, timeout: Optional[float] = None) -> None: - """ - Acquire shared/exclusive (R/W) access over all models. - - Use "w" when wanting to acquire exclusive access for the GC (method garbage_collect), or "r" when wanting to grant shared access for any other method to be called (i.e. get_model_ids). - - Args: - mode: "r" for read lock, "w" for write lock. - timeout: How many seconds to wait to acquire the lock. - """ - self._global_lock.acquire(mode, timeout) - - def global_release(self, mode: str) -> None: - """ - Release shared/exclusive (R/W) access over all models. - - Args: - mode: "r" for read lock, "w" for write lock. - """ - self._global_lock.release(mode) - - def model_acquire(self, mode: str, model_name: str, model_version: str) -> None: - """ - Acquire shared/exclusive (R/W) access for a specific model. - - Args: - mode: "r" for read lock, "w" for write lock. - model_name: The name of the model. - model_version: The version of the model. - - When mode is "r", only the following methods can be called: - * has_model - * get_model - - When mode is "w", the methods available for "r" can be called plus the following ones: - * load_model - * download_model - * remove_model - """ - model_id = f"{model_name}-{model_version}" - - if not model_id in self._locks: - lock = ReadWriteLock() - self._create_lock.acquire() - if model_id not in self._locks: - self._locks[model_id] = lock - self._create_lock.release() - - self._locks[model_id].acquire(mode) - - def model_release(self, mode: str, model_name: str, model_version: str) -> None: - """ - Release shared/exclusive (R/W) access for a specific model. - - Args: - mode: "r" for read lock, "w" for write lock. - model_name: The name of the model. - model_version: The version of the model. - """ - model_id = f"{model_name}-{model_version}" - self._locks[model_id].release(mode) - - def set_global_preference_policy(self, prefer: bool) -> bool: - """ - Wrapper of cortex_internal.lib.concurrency.ReadWriteLock.set_preference_policy. - """ - return self._global_lock.set_preference_policy(prefer) - - def get_model_names_by_tag_count(self, tag: str, count: int) -> Tuple[List[str], List[int]]: - """ - Filter model names by the tag count based on the latest recently used model version. - - Locking is already done within the method. - - Args: - tag: Tag as passed on in load_model method. If tag is not found, then the model is not considered. - count: How many appearances a tag has to make for a given model name to be selected. - - Returns: - List of model names that abide by the method's selection rule. - List of timestamps representing the latest upstream timestamp that abide by the method's selection rule. - """ - - models = {} - with LockedGlobalModelsGC(self, "r"): - models_ids = self.get_model_ids() - - for model_id in models_ids: - - model_name, model_version = model_id.rsplit("-", maxsplit=1) - with LockedModel(self, "r", model_name, model_version): - if not self.has_model_id(model_id): - raise WithBreak - - tag_count = self._models[model_id]["metadata"]["consecutive_tag_count"][tag] - ts = self._timestamps[model_id] - - if ( - model_name in models and models[model_name]["timestamp"] < ts - ) or model_name not in models: - models[model_name]["timestamp"] = ts - models[model_name]["count"] = tag_count - - filtered_model_names = [] - filtered_model_ts = [] - for model_name, v in models: - if v["count"] >= count: - filtered_model_names.append(model_name) - filtered_model_ts.append(v["timestamp"]) - - return filtered_model_names, filtered_model_ts - - def has_model(self, model_name: str, model_version: str) -> Tuple[str, int]: - """ - Verifies if a model is loaded into memory / on disk. - - Args: - model_name: The name of the model. - model_version: The version of the model. - - Returns: - "in-memory" and the upstream timestamp of the model when the model is loaded into memory. "in-memory" also implies "on-disk". - "on-disk" and the upstream timestamp of the model when the model is saved to disk. - "not-available" and 0 for the upstream timestamp when the model is not available. - """ - model_id = f"{model_name}-{model_version}" - if model_id in self._models: - if self._models[model_id]["model"] is not None: - return "in-memory", self._models[model_id]["upstream_timestamp"] - else: - return "on-disk", self._models[model_id]["upstream_timestamp"] - return "not-available", 0 - - def has_model_id(self, model_id: str) -> Tuple[str, int]: - """ - Wrapper for has_model method. - """ - model_name, model_version = model_id.rsplit("-", maxsplit=1) - return self.has_model(model_name, model_version) - - def get_model( - self, model_name: str, model_version: str, version_tag: str = "" - ) -> Tuple[Any, int]: - """ - Retrieves a model from memory. - - If the returned model is None, but the upstream timestamp is positive, then it means the model is present on disk. - If the returned model is None and the upstream timestamp is 0, then the model is not present. - If the returned model is not None, then the upstream timestamp will also be positive. - - Args: - model_name: The name of the model. - model_version: The version of the model. - version_tag: The tag associated with the given model. If the tag is present, its count will be increased by one. - - Returns: - The model and the model's upstream timestamp. - """ - model_id = f"{model_name}-{model_version}" - - if model_id in self._models: - self._timestamps[model_id] = time.time() - - if version_tag in self._models[model_id]["metadata"]["consecutive_tag_count"]: - self._models[model_id]["metadata"]["consecutive_tag_count"][version_tag] += 1 - else: - for tag in self._models[model_id]["metadata"]["consecutive_tag_count"]: - self._models[model_id]["metadata"]["consecutive_tag_count"][tag] = 0 - - return self._models[model_id]["model"], self._models[model_id]["upstream_timestamp"] - - return None, 0 - - def load_model( - self, - model_name: str, - model_version: str, - upstream_timestamp: int, - tags: List[str] = [], - kwargs: dict = {}, - ) -> None: - """ - Loads a given model into memory. - It is assumed the model already exists on disk. The model must be downloaded externally or with download_model method. - - Args: - model_name: The name of the model. - model_version: The version of the model. - upstream_timestamp: When was this model last modified on the upstream source (S3 only). - tags: List of tags to initialize the model with. - kwargs: Extra arguments to pass into the loading callback. - - Raises: - RuntimeError if a load callback isn't set. Can also raise exception if the load callback raises. - """ - - if self._load_callback: - model_id = f"{model_name}-{model_version}" - disk_path = os.path.join(self._model_dir, model_name, model_version) - - model = { - "model": self._load_callback(disk_path, **kwargs), - "disk_path": disk_path, - "upstream_timestamp": upstream_timestamp, - "metadata": { - "consecutive_tag_count": {}, - }, - } - if len(tags) > 0: - for tag in tags: - model["metadata"]["consecutive_tag_count"][tag] = 0 - - self._models[model_id] = model - else: - raise RuntimeError( - "a load callback must be provided; use set_callback to set a callback" - ) - - def download_model( - self, - bucket: str, - model_name: str, - model_version: str, - model_path: str, - ) -> datetime.datetime: - """ - Download a model to disk. To be called before load_model method is called. - - To be used when the caching is enabled. - It is assumed that when caching is disabled, an external mechanism is responsible for downloading/removing models to/from disk. - - Args: - bucket: The upstream model's bucket name. - model_name: The name of the model. - model_version: The version of the model. - model_path: Path to the model as discovered in models:dir or specified in models:paths. - - Returns: - Returns the downloaded model's upstream timestamp, otherwise None is returned if it fails. - - Raises: - Exceptions if the download callback raises any. - """ - if self._download_callback: - return self._download_callback( - self._handler_type, - bucket, - model_name, - model_version, - model_path, - self._temp_dir, - self._model_dir, - ) - raise RuntimeError( - "a download callback must be provided; use set_callback to set a callback" - ) - - def unload_model(self, model_name: str, model_version: str, kwargs: dict = {}) -> None: - """ - Unloads a model from memory. If applicable, it gets called before remove_model/remove_model_by_id. - - Args: - model_name: The name of the model. - model_version: The version of the model. - kwargs: Passable arguments to the remove callback. - - Raises: - Exceptions if the remove callback raises any. - """ - - if self._remove_callback: - model_id = f"{model_name}-{model_version}" - self._remove_callback([model_id], **kwargs) - - def remove_model(self, model_name: str, model_version: str) -> None: - """ - Removes a model from memory and disk if it exists. - """ - model_id = f"{model_name}-{model_version}" - self.remove_model_by_id(model_id, True, True) - - def remove_model_by_id( - self, model_id: str, mem: bool, disk: bool, del_reference: bool = False - ) -> None: - """ - Remove a model from this object and/or from disk. - - Args: - model_id: The model ID to remove. - mem: Whether to remove the model from memory or not. - disk: Whether to remove the model from disk or not. - del_reference: Whether to remove the model reference or not. Don't touch this unless you know what you do. - """ - if model_id not in self._models: - return None - - if mem: - # remove model from memory (but keeps it on disk) - self._models[model_id]["model"] = None - - if disk: - disk_path = self._models[model_id]["disk_path"] - shutil.rmtree(disk_path) - - if disk or del_reference: - del self._models[model_id] - del self._timestamps[model_id] - - def garbage_collect( - self, exclude_disk_model_ids: List[str] = [], dry_run: bool = False - ) -> Tuple[bool, List[str], List[str]]: - """ - Removes stale in-memory and on-disk models based on LRU policy. - Also calls the "remove" callback before removing the models from this object. The callback must not raise any exceptions. - - Must be called with a write lock unless dry_run is set to true. - - Args: - exclude_disk_model_ids: Model IDs to exclude from removing from disk. Necessary for locally-provided models. - dry_run: Just test if there are any models to remove. If set to true, this method can then be called with a read lock. - - Returns: - A 3-element tuple. First element tells whether models had to be collected. The 2nd and 3rd elements contain the model IDs that were removed from memory and disk respectively. - """ - collected = False - if self._mem_cache_size <= 0 or self._disk_cache_size <= 0: - return collected - - stale_mem_model_ids = self._lru_model_ids(self._mem_cache_size, filter_in_mem=True) - stale_disk_model_ids = self._lru_model_ids( - self._disk_cache_size - len(exclude_disk_model_ids), filter_in_mem=False - ) - - if self._remove_callback and not dry_run: - self._remove_callback(stale_mem_model_ids) - - # don't delete excluded model IDs from disk - stale_disk_model_ids = list(set(stale_disk_model_ids) - set(exclude_disk_model_ids)) - stale_disk_model_ids = stale_disk_model_ids[ - len(stale_disk_model_ids) - self._disk_cache_size : - ] - - if not dry_run: - logger.info( - f"unloading models {stale_mem_model_ids} from memory using the garbage collector" - ) - logger.info( - f"unloading models {stale_disk_model_ids} from disk using the garbage collector" - ) - for model_id in stale_mem_model_ids: - self.remove_model_by_id(model_id, mem=True, disk=False) - for model_id in stale_disk_model_ids: - self.remove_model_by_id(model_id, mem=False, disk=True) - - if len(stale_mem_model_ids) > 0 or len(stale_disk_model_ids) > 0: - collected = True - - return collected, stale_mem_model_ids, stale_disk_model_ids - - def get_model_ids(self) -> List[str]: - """ - Gets a list of all loaded model IDs (in memory or on disk). - """ - return list(self._models.keys()) - - def _lru_model_ids(self, threshold: int, filter_in_mem: bool) -> List[str]: - """ - Sort model ids by last access and get the model ids with ranks below the specified threshold. - - Args: - threshold: The memory cache size or the disk cache size. - filter_in_mem: In the counting process, set whether to only look at models loaded in memory or not. True for only looking at models loaded in memory and on disk. - - Returns: - A list of stale model IDs. - """ - copied_timestamps = self._timestamps.copy() - timestamps = { - k: v - for k, v in sorted(copied_timestamps.items(), key=lambda item: item[1], reverse=True) - } - model_ids = [] - for counter, model_id in enumerate(timestamps): - # skip models if they are not loaded in memory but on disk - if filter_in_mem and self._models[model_id]["model"] is None: - continue - if counter >= threshold: - model_ids.append(model_id) - - return model_ids - - -def ids_to_models(model_ids: List[str]) -> Dict[str, List[str]]: - """ - Convert model IDs (MODEL_NAME-MODEL_VERSION) to a dictionary with its keys being - the model names and its values being lists of the associated versions for each given model name. - """ - - models = {} - for model_id in model_ids: - model_name, model_version = model_id.rsplit("-", maxsplit=1) - if model_name not in models: - models[model_name] = [model_version] - else: - models[model_name].append(model_version) - return models - - -class LockedGlobalModelsGC: - """ - Applies global exclusive lock (R/W) on the models holder. - - For running the GC for all loaded models (or present on disk). - This is the locking implementation for the stop-the-world GC. - - The context manager can be exited by raising cortex_internal.lib.exceptions.WithBreak. - """ - - def __init__( - self, - models: ModelsHolder, - mode: str = "w", - prefer: str = "r", - timeout: Optional[float] = None, - ): - self._models = models - self._mode = mode - self._timeout = timeout - - def __enter__(self): - self.acquired = True - try: - self._models.global_acquire(self._mode, self._timeout) - except TimeoutError: - self.acquired = False - return self - - def __exit__(self, exc_type, exc_value, traceback) -> bool: - self._models.global_release(self._mode) - - if exc_value is not None and exc_type is not WithBreak: - return False - return True - - -class LockedModel: - """ - For granting shared/exclusive (R/W) access to a model resource (model name + model version). - Also applies global read lock on the models holder. - - The context manager can be exited by raising cortex_internal.lib.exceptions.WithBreak. - """ - - def __init__( - self, - models: ModelsHolder, - mode: str, - model_name: str = "", - model_version: str = "", - model_id: str = "", - ): - """ - mode can be "r" for read or "w" for write. - """ - self._models = models - self._mode = mode - if model_id != "": - self._model_name, self._model_version = model_id.rsplit("-", maxsplit=1) - else: - self._model_name = model_name - self._model_version = model_version - - def __enter__(self): - self._models.global_acquire("r") - self._models.model_acquire(self._mode, self._model_name, self._model_version) - return self - - def __exit__(self, exc_type, exc_value, traceback) -> bool: - self._models.model_release(self._mode, self._model_name, self._model_version) - self._models.global_release("r") - - if exc_value is not None and exc_type is not WithBreak: - return False - return True diff --git a/python/serve/cortex_internal/lib/model/tfs.py b/python/serve/cortex_internal/lib/model/tfs.py deleted file mode 100644 index c4cd450c88..0000000000 --- a/python/serve/cortex_internal/lib/model/tfs.py +++ /dev/null @@ -1,764 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import copy -import os -import time -from typing import Any, Optional, Dict, List, Tuple - -import grpc - -from cortex_internal.lib.exceptions import CortexException, UserException -from cortex_internal.lib.log import configure_logger - -logger = configure_logger("cortex", os.environ["CORTEX_LOG_CONFIG_FILE"]) - - -# TensorFlow types -def _define_types() -> Tuple[Dict[str, Any], Dict[str, str]]: - return ( - { - "DT_FLOAT": tf.float32, - "DT_DOUBLE": tf.float64, - "DT_INT32": tf.int32, - "DT_UINT8": tf.uint8, - "DT_INT16": tf.int16, - "DT_INT8": tf.int8, - "DT_STRING": tf.string, - "DT_COMPLEX64": tf.complex64, - "DT_INT64": tf.int64, - "DT_BOOL": tf.bool, - "DT_QINT8": tf.qint8, - "DT_QUINT8": tf.quint8, - "DT_QINT32": tf.qint32, - "DT_BFLOAT16": tf.bfloat16, - "DT_QINT16": tf.qint16, - "DT_QUINT16": tf.quint16, - "DT_UINT16": tf.uint16, - "DT_COMPLEX128": tf.complex128, - "DT_HALF": tf.float16, - "DT_RESOURCE": tf.resource, - "DT_VARIANT": tf.variant, - "DT_UINT32": tf.uint32, - "DT_UINT64": tf.uint64, - }, - { - "DT_INT32": "intVal", - "DT_INT64": "int64Val", - "DT_FLOAT": "floatVal", - "DT_STRING": "stringVal", - "DT_BOOL": "boolVal", - "DT_DOUBLE": "doubleVal", - "DT_HALF": "halfVal", - "DT_COMPLEX64": "scomplexVal", - "DT_COMPLEX128": "dcomplexVal", - }, - ) - - -# for TensorFlowServingAPI -try: - import tensorflow as tf - from tensorflow_serving.apis import predict_pb2 - from tensorflow_serving.apis import get_model_metadata_pb2 - from tensorflow_serving.apis import prediction_service_pb2_grpc - from tensorflow_serving.apis import model_service_pb2_grpc - from tensorflow_serving.apis import model_management_pb2 - from tensorflow_serving.apis import get_model_status_pb2 - from tensorflow_serving.config import model_server_config_pb2 - from tensorflow_serving.sources.storage_path.file_system_storage_path_source_pb2 import ( - FileSystemStoragePathSourceConfig, - ) - - ServableVersionPolicy = FileSystemStoragePathSourceConfig.ServableVersionPolicy - Specific = FileSystemStoragePathSourceConfig.ServableVersionPolicy.Specific - from google.protobuf import json_format - - tensorflow_dependencies_installed = True - DTYPE_TO_TF_TYPE, DTYPE_TO_VALUE_KEY = _define_types() - predictRequestClass = predict_pb2.PredictRequest - -except ImportError: - tensorflow_dependencies_installed = False - predictRequestClass = Any - - -class TensorFlowServingAPI: - def __init__(self, address: str): - """ - TensorFlow Serving API for loading/unloading/reloading TF models and for running predictions. - - Extra arguments passed to the tensorflow/serving container: - * --max_num_load_retries=0 - * --load_retry_interval_micros=30000000 # 30 seconds - * --grpc_channel_arguments="grpc.max_concurrent_streams=*" when inf == 0, otherwise - * --grpc_channel_arguments="grpc.max_concurrent_streams=" when inf > 0. - - Args: - address: An address with the "host:port" format. - """ - - if not tensorflow_dependencies_installed: - raise NameError("tensorflow_serving_api and tensorflow packages not installed") - - self.address = address - self.models = ( - {} - ) # maps the model ID to the model metadata (signature def, signature key and so on) - - # remove limit for maximum/receive transmission sizes - options = [ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ] - self.channel = grpc.insecure_channel(self.address, options=options) - - self._service = model_service_pb2_grpc.ModelServiceStub(self.channel) - self._pred = prediction_service_pb2_grpc.PredictionServiceStub(self.channel) - - def is_tfs_accessible(self) -> bool: - """ - Tests whether TFS is accessible or not. - """ - request = get_model_status_pb2.GetModelStatusRequest() - request.model_spec.name = "test-model-name" - - try: - self._service.GetModelStatus(request, timeout=10.0) - except grpc.RpcError as err: - if err.code() in [grpc.StatusCode.UNAVAILABLE, grpc.StatusCode.DEADLINE_EXCEEDED]: - return False - return True - - def add_single_model( - self, - model_name: str, - model_version: str, - model_disk_path: str, - signature_key: Optional[str] = None, - timeout: Optional[float] = None, - max_retries: int = 0, - ) -> None: - """ - Wrapper for add_models method. - """ - self.add_models( - [model_name], - [[model_version]], - [model_disk_path], - [signature_key], - timeout=timeout, - max_retries=max_retries, - ) - - def remove_single_model( - self, - model_name: str, - model_version: str, - timeout: Optional[float] = None, - ) -> None: - """ - Wrapper for remove_models method. - """ - self.remove_models([model_name], [[model_version]], timeout) - - def add_models( - self, - model_names: List[str], - model_versions: List[List[str]], - model_disk_paths: List[str], - signature_keys: List[Optional[str]], - skip_if_present: bool = False, - timeout: Optional[float] = None, - max_retries: int = 0, - ) -> None: - """ - Add models to TFS. If they can't be loaded, use remove_models to remove them from TFS. - - Args: - model_names: List of model names to add. - model_versions: List of lists - each element is a list of versions for a given model name. - model_disk_paths: The common model disk path of multiple versioned models of the same model name (i.e. modelA/ for modelA/1 and modelA/2). - skip_if_present: If the models are already loaded, don't make a new request to TFS. - signature_keys: The signature keys as set in cortex_internal.yaml. If an element is set to None, then "predict" key will be assumed. - max_retries: How many times to call ReloadConfig before giving up. - Raises: - grpc.RpcError in case something bad happens while communicating. - StatusCode.DEADLINE_EXCEEDED when timeout is encountered. StatusCode.UNAVAILABLE when the service is unreachable. - cortex_internal.lib.exceptions.CortexException if a non-0 response code is returned (i.e. model couldn't be loaded). - cortex_internal.lib.exceptions.UserException when a model couldn't be validated for the signature def. - """ - - request = model_management_pb2.ReloadConfigRequest() - model_server_config = model_server_config_pb2.ModelServerConfig() - - num_added_models = 0 - for model_name, versions, model_disk_path in zip( - model_names, model_versions, model_disk_paths - ): - for model_version in versions: - versioned_model_disk_path = os.path.join(model_disk_path, model_version) - num_added_models += self._add_model_to_dict( - model_name, model_version, versioned_model_disk_path - ) - - if skip_if_present and num_added_models == 0: - return - - config_list = model_server_config_pb2.ModelConfigList() - current_model_names = self._get_model_names() - for model_name in current_model_names: - versions, model_disk_path = self._get_model_info(model_name) - versions = [int(version) for version in versions] - model_config = config_list.config.add() - model_config.name = model_name - model_config.base_path = model_disk_path - model_config.model_version_policy.CopyFrom( - ServableVersionPolicy(specific=Specific(versions=versions)) - ) - model_config.model_platform = "tensorflow" - - model_server_config.model_config_list.CopyFrom(config_list) - request.config.CopyFrom(model_server_config) - - while max_retries >= 0: - max_retries -= 1 - try: - # to prevent HandleReloadConfigRequest from - # throwing an exception (TFS has some race-condition bug) - time.sleep(0.125) - response = self._service.HandleReloadConfigRequest(request, timeout) - break - except grpc.RpcError as err: - # to prevent HandleReloadConfigRequest from - # throwing another exception on the next run - time.sleep(0.125) - raise - - if not (response and response.status.error_code == 0): - if response: - raise CortexException( - "couldn't load user-requested models {} - failed with error code {}: {}".format( - model_names, response.status.error_code, response.status.error_message - ) - ) - else: - raise CortexException("couldn't load user-requested models") - - # get models metadata - for model_name, versions, signature_key in zip(model_names, model_versions, signature_keys): - for model_version in versions: - self._load_model_signatures(model_name, model_version, signature_key) - - def remove_models( - self, - model_names: List[str], - model_versions: List[List[str]], - timeout: Optional[float] = None, - ) -> None: - """ - Remove models to TFS. - - Args: - model_names: List of model names to add. - model_versions: List of lists - each element is a list of versions for a given model name. - Raises: - grpc.RpcError in case something bad happens while communicating. - StatusCode.DEADLINE_EXCEEDED when timeout is encountered. StatusCode.UNAVAILABLE when the service is unreachable. - cortex_internal.lib.exceptions.CortexException if a non-0 response code is returned (i.e. model couldn't be unloaded). - """ - - request = model_management_pb2.ReloadConfigRequest() - model_server_config = model_server_config_pb2.ModelServerConfig() - - for model_name, versions in zip(model_names, model_versions): - for model_version in versions: - self._remove_model_from_dict(model_name, model_version) - - config_list = model_server_config_pb2.ModelConfigList() - remaining_model_names = self._get_model_names() - for model_name in remaining_model_names: - versions, model_disk_path = self._get_model_info(model_name) - versions = [int(version) for version in versions] - model_config = config_list.config.add() - model_config.name = model_name - model_config.base_path = model_disk_path - model_config.model_version_policy.CopyFrom( - ServableVersionPolicy(specific=Specific(versions=versions)) - ) - model_config.model_platform = "tensorflow" - - model_server_config.model_config_list.CopyFrom(config_list) - request.config.CopyFrom(model_server_config) - - response = self._service.HandleReloadConfigRequest(request, timeout) - - if not (response and response.status.error_code == 0): - if response: - raise CortexException( - "couldn't unload user-requested models {} - failed with error code {}: {}".format( - model_names, response.status.error_code, response.status.error_message - ) - ) - else: - raise CortexException("couldn't unload user-requested models") - - def poll_available_model_versions(self, model_name: str) -> List[str]: - """ - Gets the available model versions from TFS. - - Args: - model_name: The model name to check for versions. - - Returns: - List of the available versions for the given model from TFS. - """ - request = get_model_status_pb2.GetModelStatusRequest() - request.model_spec.name = model_name - - versions = [] - - try: - for model in self._service.GetModelStatus(request).model_version_status: - if model.state == get_model_status_pb2.ModelVersionStatus.AVAILABLE: - versions.append(str(model.version)) - except grpc.RpcError as e: - pass - - return versions - - def get_registered_model_ids(self) -> List[str]: - """ - Get the registered model IDs (doesn't poll the TFS server). - """ - return list(self.models.keys()) - - def predict( - self, model_input: Any, model_name: str, model_version: str, timeout: float = 300.0 - ) -> Any: - """ - Args: - model_input: The input to run the prediction on - as passed by the user. - model_name: Name of the model. - model_version: Version of the model. - timeout: How many seconds to wait for the prediction to run before timing out. - - Raises: - UserException when the model input is not valid or when the model's shape doesn't match that of the input's. - grpc.RpcError in case something bad happens while communicating - should not happen. - - Returns: - The prediction. - """ - - model_id = f"{model_name}-{model_version}" - - signature_def = self.models[model_id]["signature_def"] - signature_key = self.models[model_id]["signature_key"] - input_signatures = self.models[model_id]["input_signatures"] - - # validate model input - for input_name, _ in input_signatures.items(): - if input_name not in model_input: - raise UserException( - "missing key '{}' for model '{}' of version '{}'".format( - input_name, model_name, model_version - ) - ) - - # create prediction request - prediction_request = self._create_prediction_request( - signature_def, signature_key, model_name, model_version, model_input - ) - - # run prediction - response_proto = self._pred.Predict(prediction_request, timeout=timeout) - - # interpret response message - results_dict = json_format.MessageToDict(response_proto) - outputs = results_dict["outputs"] - outputs_simplified = {} - for key in outputs: - value_key = DTYPE_TO_VALUE_KEY[outputs[key]["dtype"]] - outputs_simplified[key] = outputs[key][value_key] - - # return parsed response - return outputs_simplified - - def _remove_model_from_dict(self, model_name: str, model_version: str) -> Tuple[bool, str]: - model_id = f"{model_name}-{model_version}" - try: - model = copy.deepcopy(self.models[model_id]) - del self.models[model_id] - return True, model - except KeyError: - pass - return False, "" - - def _add_model_to_dict(self, model_name: str, model_version: str, model_disk_path: str) -> bool: - model_id = f"{model_name}-{model_version}" - if model_id not in self.models: - self.models[model_id] = { - "disk_path": model_disk_path, - } - return True - return False - - def _load_model_signatures( - self, model_name: str, model_version: str, signature_key: Optional[str] = None - ) -> None: - """ - Queries the signature defs from TFS. - - Args: - model_name: Name of the model. - model_version: Version of the model. - signature_key: Signature key of the model as passed in with handler:signature_key, handler:models:paths:signature_key or handler:models:signature_key. - When set to None, "predict" is the assumed key. - - Raises: - cortex_internal.lib.exceptions.UserException when the signature def can't be validated. - """ - - # create model metadata request - request = get_model_metadata_pb2.GetModelMetadataRequest() - request.model_spec.name = model_name - request.model_spec.version.value = int(model_version) - request.metadata_field.append("signature_def") - - # get signature def - last_idx = 0 - for times in range(100): - try: - resp = self._pred.GetModelMetadata(request) - break - except grpc.RpcError as e: - # it has been observed that it may take a little bit of time - # until a model gets to be accessible with TFS (even though it's already loaded in) - time.sleep(0.3) - last_idx = times - if last_idx == 99: - raise UserException( - "couldn't find model '{}' of version '{}' to extract the signature def".format( - model_name, model_version - ) - ) - - sigAny = resp.metadata["signature_def"] - signature_def_map = get_model_metadata_pb2.SignatureDefMap() - sigAny.Unpack(signature_def_map) - sigmap = json_format.MessageToDict(signature_def_map) - signature_def = sigmap["signatureDef"] - - # extract signature key and input signature - signature_key, input_signatures = self._extract_signatures( - signature_def, signature_key, model_name, model_version - ) - - model_id = f"{model_name}-{model_version}" - self.models[model_id]["signature_def"] = signature_def - self.models[model_id]["signature_key"] = signature_key - self.models[model_id]["input_signatures"] = input_signatures - - def _get_model_names(self) -> List[str]: - return list(set([model_id.rsplit("-", maxsplit=1)[0] for model_id in self.models])) - - def _get_model_info(self, model_name: str) -> Tuple[List[str], str]: - model_disk_path = "" - versions = [] - for model_id in self.models: - _model_name, model_version = model_id.rsplit("-", maxsplit=1) - if _model_name == model_name: - versions.append(model_version) - if model_disk_path == "": - model_disk_path = os.path.dirname(self.models[model_id]["disk_path"]) - - return versions, model_disk_path - - def _extract_signatures( - self, signature_def, signature_key, model_name: str, model_version: str - ): - logger.info( - "signature defs found in model '{}' for version '{}': {}".format( - model_name, model_version, signature_def - ) - ) - - available_keys = list(signature_def.keys()) - if len(available_keys) == 0: - raise UserException( - "unable to find signature defs in model '{}' of version '{}'".format( - model_name, model_version - ) - ) - - if signature_key is None: - if len(available_keys) == 1: - logger.info( - "signature_key was not configured by user, using signature key '{}' for model '{}' of version '{}' (found in the signature def map)".format( - available_keys[0], - model_name, - model_version, - ) - ) - signature_key = available_keys[0] - elif "predict" in signature_def: - logger.info( - "signature_key was not configured by user, using signature key 'predict' for model '{}' of version '{}' (found in the signature def map)".format( - model_name, - model_version, - ) - ) - signature_key = "predict" - else: - raise UserException( - "signature_key was not configured by user, please specify one the following keys '{}' for model '{}' of version '{}' (found in the signature def map)".format( - ", ".join(available_keys), model_name, model_version - ) - ) - else: - if signature_def.get(signature_key) is None: - possibilities_str = "key: '{}'".format(available_keys[0]) - if len(available_keys) > 1: - possibilities_str = "keys: '{}'".format("', '".join(available_keys)) - - raise UserException( - "signature_key '{}' was not found in signature def map for model '{}' of version '{}', but found the following {}".format( - signature_key, model_name, model_version, possibilities_str - ) - ) - - signature_def_val = signature_def.get(signature_key) - - if signature_def_val.get("inputs") is None: - raise UserException( - "unable to find 'inputs' in signature def '{}' for model '{}'".format( - signature_key, model_name - ) - ) - - parsed_signatures = {} - for input_name, input_metadata in signature_def_val["inputs"].items(): - if input_metadata["tensorShape"] == {}: - # a scalar with rank 0 and empty shape - shape = "scalar" - elif input_metadata["tensorShape"].get("unknownRank", False): - # unknown rank and shape - # - # unknownRank is set to True if the model input has no rank - # it may lead to an undefined behavior if unknownRank is only checked for its presence - # so it also gets to be tested against its value - shape = "unknown" - elif input_metadata["tensorShape"].get("dim", None): - # known rank and known/unknown shape - shape = [int(dim["size"]) for dim in input_metadata["tensorShape"]["dim"]] - else: - raise UserException( - "invalid 'tensorShape' specification for input '{}' in signature key '{}' for model '{}'", - input_name, - signature_key, - model_name, - ) - - parsed_signatures[input_name] = { - "shape": shape if type(shape) == list else [shape], - "type": DTYPE_TO_TF_TYPE[input_metadata["dtype"]].name, - } - return signature_key, parsed_signatures - - def _create_prediction_request( - self, - signature_def: dict, - signature_key: str, - model_name: str, - model_version: int, - model_input: Any, - ) -> predictRequestClass: - prediction_request = predict_pb2.PredictRequest() - prediction_request.model_spec.name = model_name - prediction_request.model_spec.version.value = int(model_version) - prediction_request.model_spec.signature_name = signature_key - - for column_name, value in model_input.items(): - if signature_def[signature_key]["inputs"][column_name]["tensorShape"] == {}: - shape = "scalar" - elif signature_def[signature_key]["inputs"][column_name]["tensorShape"].get( - "unknownRank", False - ): - # unknownRank is set to True if the model input has no rank - # it may lead to an undefined behavior if unknownRank is only checked for its presence - # so it also gets to be tested against its value - shape = "unknown" - else: - shape = [] - for dim in signature_def[signature_key]["inputs"][column_name]["tensorShape"][ - "dim" - ]: - shape.append(int(dim["size"])) - - sig_type = signature_def[signature_key]["inputs"][column_name]["dtype"] - - try: - tensor_proto = tf.compat.v1.make_tensor_proto( - value, dtype=DTYPE_TO_TF_TYPE[sig_type] - ) - prediction_request.inputs[column_name].CopyFrom(tensor_proto) - except Exception as e: - if shape == "scalar": - raise UserException( - 'key "{}"'.format(column_name), "expected to be a scalar", str(e) - ) from e - elif shape == "unknown": - raise UserException( - 'key "{}"'.format(column_name), "can be of any rank and shape", str(e) - ) from e - else: - raise UserException( - 'key "{}"'.format(column_name), "expected shape {}".format(shape), str(e) - ) from e - - return prediction_request - - -class TensorFlowServingAPIClones: - """ - TFS API to load/unload models from multiple TFS server clones. Built on top of TensorFlowServingAPI. - """ - - def __init__(self, addresses: List[str]): - """ - Args: - addresses: A list of addresses with the "host:port" format. - """ - - if len(addresses) == 0: - raise ValueError("addresses list must have at least one address") - self._clients = [TensorFlowServingAPI(address) for address in addresses] - - def is_tfs_accessible(self) -> bool: - """ - Tests whether all TFS servers are accessible or not. - """ - return all([client.is_tfs_accessible() for client in self._clients]) - - def add_single_model( - self, - model_name: str, - model_version: str, - model_disk_path: str, - signature_key: Optional[str] = None, - timeout: Optional[float] = None, - max_retries: int = 0, - ) -> None: - """ - Wrapper for add_models method. - """ - for client in self._clients: - client.add_single_model( - model_name, model_version, model_disk_path, signature_key, timeout, max_retries - ) - - def remove_single_model( - self, - model_name: str, - model_version: str, - timeout: Optional[float] = None, - ) -> None: - """ - Wrapper for remove_models method. - """ - for client in self._clients: - client.remove_single_model(model_name, model_version, timeout) - - def add_models( - self, - model_names: List[str], - model_versions: List[List[str]], - model_disk_paths: List[str], - signature_keys: List[Optional[str]], - skip_if_present: bool = False, - timeout: Optional[float] = None, - max_retries: int = 0, - ) -> None: - """ - Add the same models to multiple TFS servers. If they can't be loaded, use remove_models to remove them from TFS. - - Args: - model_names: List of model names to add. - model_versions: List of lists - each element is a list of versions for a given model name. - model_disk_paths: The common model disk path of multiple versioned models of the same model name (i.e. modelA/ for modelA/1 and modelA/2). - skip_if_present: If the models are already loaded, don't make a new request to TFS. - signature_keys: The signature keys as set in cortex_internal.yaml. If an element is set to None, then "predict" key will be assumed. - max_retries: How many times to call ReloadConfig before giving up. - Raises: - grpc.RpcError in case something bad happens while communicating. - StatusCode.DEADLINE_EXCEEDED when timeout is encountered. StatusCode.UNAVAILABLE when the service is unreachable. - cortex_internal.lib.exceptions.CortexException if a non-0 response code is returned (i.e. model couldn't be loaded). - cortex_internal.lib.exceptions.UserException when a model couldn't be validated for the signature def. - """ - for client in self._clients: - client.add_models( - model_names, - model_versions, - model_disk_paths, - signature_keys, - skip_if_present, - timeout, - max_retries, - ) - - def remove_models( - self, - model_names: List[str], - model_versions: List[List[str]], - timeout: Optional[float] = None, - ) -> None: - """ - Remove the same models from multiple TFS servers. - - Args: - model_names: List of model names to add. - model_versions: List of lists - each element is a list of versions for a given model name. - Raises: - grpc.RpcError in case something bad happens while communicating. - StatusCode.DEADLINE_EXCEEDED when timeout is encountered. StatusCode.UNAVAILABLE when the service is unreachable. - cortex_internal.lib.exceptions.CortexException if a non-0 response code is returned (i.e. model couldn't be unloaded). - """ - for client in self._clients: - client.remove_models(model_names, model_versions, timeout) - - def poll_available_model_versions(self, model_name: str) -> List[str]: - """ - Gets the available model versions from TFS. - Since all TFS servers are assumed to have the same models in memory, it makes sense to just poll one. - - Args: - model_name: The model name to check for versions. - - Returns: - List of the available versions for the given model from TFS. - """ - - return self._clients[0].poll_available_model_versions(model_name) - - def get_registered_model_ids(self) -> List[str]: - """ - Get the registered model IDs (doesn't poll the TFS server). - Since all TFS servers are assumed to have the same models in memory, it makes sense to just poll one. - """ - return self._clients[0].get_registered_model_ids() - - @property - def models(self) -> dict: - return self._clients[0].models diff --git a/python/serve/cortex_internal/lib/model/tree.py b/python/serve/cortex_internal/lib/model/tree.py deleted file mode 100644 index 8308a623e3..0000000000 --- a/python/serve/cortex_internal/lib/model/tree.py +++ /dev/null @@ -1,526 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime -import itertools -import os -import threading as td -from typing import List, Dict, Tuple, AbstractSet - -from cortex_internal.lib import util -from cortex_internal.lib.concurrency import ReadWriteLock -from cortex_internal.lib.exceptions import CortexException, WithBreak -from cortex_internal.lib.log import configure_logger -from cortex_internal.lib.model.validation import ( - validate_models_dir_paths, - validate_model_paths, - ModelVersion, -) -from cortex_internal.lib.storage import S3 -from cortex_internal.lib.type import HandlerType - -logger = configure_logger("cortex", os.environ["CORTEX_LOG_CONFIG_FILE"]) - - -class ModelsTree: - """ - Model tree for S3 models. - """ - - def __init__(self): - self.models = {} - self._locks = {} - self._create_lock = td.RLock() - self._removable = set() - - def acquire(self, mode: str, model_name: str, model_version: str) -> None: - """ - Acquire shared/exclusive (R/W) access for a specific model. Use this when multiple threads are used. - - Args: - mode: "r" for read lock, "w" for write lock. - model_name: The name of the model. - model_version: The version of the model. - """ - model_id = f"{model_name}-{model_version}" - - if not model_id in self._locks: - lock = ReadWriteLock() - self._create_lock.acquire() - if model_id not in self._locks: - self._locks[model_id] = lock - self._create_lock.release() - - self._locks[model_id].acquire(mode) - - def release(self, mode: str, model_name: str, model_version: str) -> None: - """ - Release shared/exclusive (R/W) access for a specific model. Use this when multiple threads are used. - - Args: - mode: "r" for read lock, "w" for write lock. - model_name: The name of the model. - model_version: The version of the model. - """ - model_id = f"{model_name}-{model_version}" - self._locks[model_id].release(mode) - - def update_models( - self, - model_names: List[str], - model_versions: Dict[str, List[str]], - model_paths: List[str], - sub_paths: List[List[str]], - timestamps: List[List[datetime.datetime]], - bucket_names: List[str], - ) -> Tuple[AbstractSet[str], AbstractSet[str]]: - """ - Updates the model tree with the latest from the upstream and removes stale models. - - Locking is not required. Locking is already done within the method. - - Args: - model_names: The unique names of the models as discovered in models:dir or specified in models:paths. - model_versions: The detected versions of each model. If the list is empty, then version "1" should be assumed. The dictionary keys represent the models' names. - model_paths: S3 model paths to each model. - sub_paths: A list of filepaths lists for each file of each model. - timestamps: When was each versioned model updated the last time on the upstream. When no versions are passed, a timestamp is still expected. - bucket_names: A list with the bucket_names required for each model. Empty elements if no bucket is used. - - Returns: - The loaded model IDs ("-": , - } - And where "versions" represents the available versions of a model and each "timestamps" element is the corresponding - last-edit time of each versioned model. - """ - - current_model_ids = set() - updated_model_ids = set() - for idx in range(len(model_names)): - model_name = model_names[idx] - - if len(model_versions[model_name]) == 0: - model_id = f"{model_name}-1" - with LockedModelsTree(self, "w", model_name, "1"): - updated = self.update_model( - bucket_names[idx], - model_name, - "1", - model_paths[idx], - sub_paths[idx], - timestamps[idx][0], - True, - ) - current_model_ids.add(model_id) - if updated: - updated_model_ids.add(model_id) - - for v_idx, model_version in enumerate(model_versions[model_name]): - model_id = f"{model_name}-{model_version}" - with LockedModelsTree(self, "w", model_name, model_version): - updated = self.update_model( - bucket_names[idx], - model_name, - model_version, - os.path.join(model_paths[idx], model_version) + "/", - sub_paths[idx], - timestamps[idx][v_idx], - True, - ) - current_model_ids.add(model_id) - if updated: - updated_model_ids.add(model_id) - - old_model_ids = set(self.models.keys()) - current_model_ids - - for old_model_id in old_model_ids: - model_name, model_version = old_model_id.rsplit("-", maxsplit=1) - if old_model_id not in self._removable: - continue - with LockedModelsTree(self, "w", model_name, model_version): - del self.models[old_model_id] - self._removable = self._removable - set([old_model_id]) - - return old_model_ids, updated_model_ids - - def update_model( - self, - bucket: str, - model_name: str, - model_version: str, - model_path: str, - sub_paths: List[str], - timestamp: datetime.datetime, - removable: bool, - ) -> bool: - """ - Updates the model tree with the given model. - - Locking is required. - - Args: - bucket: The bucket on which the model is stored. Empty if there's no bucket. - model_name: The unique name of the model as discovered in models:dir or specified in models:paths. - model_version: A detected version of the model. - model_path: The model path to the versioned model. - sub_paths: A list of filepaths for each file of the model. - timestamp: When was the model path updated the last time. - removable: If update_models method is allowed to remove the model. - - Returns: - True if the model wasn't in the tree or if the timestamp is newer. False otherwise. - """ - - model_id = f"{model_name}-{model_version}" - has_changed = False - if model_id not in self.models: - has_changed = True - elif self.models[model_id]["timestamp"] < timestamp: - has_changed = True - - if has_changed or model_id in self.models: - self.models[model_id] = { - "bucket": bucket, - "path": model_path, - "sub_paths": sub_paths, - "timestamp": timestamp, - } - if removable: - self._removable.add(model_id) - else: - self._removable = self._removable - set([model_id]) - - return has_changed - - def model_info(self, model_name: str) -> dict: - """ - Gets model info about the available versions and model timestamps. - - Locking is not required. - - Returns: - A dict with keys "bucket", "model_paths, "versions" and "timestamps". - "model_paths" contains the s3 prefixes of each versioned model, "versions" represents the available versions of the model, - and each "timestamps" element is the corresponding last-edit time of each versioned model. - - Empty lists are returned if the model is not found. - - Example of returned dictionary for model_name. - ```json - { - "bucket": "bucket-0", - "model_paths": ["modelA/1", "modelA/4", "modelA/7", ...], - "versions": [1,4,7, ...], - "timestamps": [12884999, 12874449, 12344931, ...] - } - ``` - """ - - info = { - "model_paths": [], - "versions": [], - "timestamps": [], - } - - # to ensure atomicity - models = self.models.copy() - for model_id in models: - _model_name, model_version = model_id.rsplit("-", maxsplit=1) - if _model_name == model_name: - if "bucket" not in info: - info["bucket"] = models[model_id]["bucket"] - info["model_paths"] += [os.path.join(models[model_id]["path"], model_version)] - info["versions"] += [model_version] - info["timestamps"] += [models[model_id]["timestamp"]] - - return info - - def get_model_names(self) -> List[str]: - """ - Gets the available model names. - - Locking is not required. - - Returns: - List of all model names. - """ - model_names = set() - - # to ensure atomicity - models = self.models.copy() - for model_id in models: - model_name = model_id.rsplit("-", maxsplit=1)[0] - model_names.add(model_name) - - return list(model_names) - - def get_all_models_info(self) -> dict: - """ - Gets model info about the available versions and model timestamps. - - Locking is not required. - - It's like model_info method, but for all model names. - - Example of returned dictionary. - ```json - { - ... - "modelA": { - "bucket": "bucket-0", - "model_paths": ["modelA/1", "modelA/4", "modelA/7", ...], - "versions": ["1","4","7", ...], - "timestamps": [12884999, 12874449, 12344931, ...] - } - ... - } - ``` - """ - - models_info = {} - # to ensure atomicity - models = self.models.copy() - - # extract model names - model_names = set() - for model_id in models: - model_name = model_id.rsplit("-", maxsplit=1)[0] - model_names.add(model_name) - model_names = list(model_names) - - # build models info dictionary - for model_name in model_names: - model_info = { - "model_paths": [], - "versions": [], - "timestamps": [], - } - for model_id in models: - _model_name, model_version = model_id.rsplit("-", maxsplit=1) - if _model_name == model_name: - if "bucket" not in model_info: - model_info["bucket"] = models[model_id]["bucket"] - model_info["model_paths"] += [ - os.path.join(models[model_id]["path"], model_version) - ] - model_info["versions"] += [model_version] - model_info["timestamps"] += [int(models[model_id]["timestamp"].timestamp())] - - models_info[model_name] = model_info - - return models_info - - def __getitem__(self, model_id: str) -> dict: - """ - Each value of a key (model ID) is a dictionary with the following format: - { - "bucket": , - "path": , - "sub_paths": , - "timestamp": - } - - Locking is required. - """ - return self.models[model_id].copy() - - def __contains__(self, model_id: str) -> bool: - """ - Each value of a key (model ID) is a dictionary with the following format: - { - "bucket": , - "path": , - "sub_paths": , - "timestamp": - } - - Locking is required. - """ - return model_id in self.models - - -class LockedModelsTree: - """ - When acquiring shared/exclusive (R/W) access to a model resource (model name + version). - - Locks just for a specific model. Apply read lock when granting shared access or write lock when it's exclusive access (for adding/removing operations). - - The context manager can be exited by raising cortex_internal.lib.exceptions.WithBreak. - """ - - def __init__(self, tree: ModelsTree, mode: str, model_name: str, model_version: str): - """ - mode can be "r" for read or "w" for write. - """ - self._tree = tree - self._mode = mode - self._model_name = model_name - self._model_version = model_version - - def __enter__(self): - self._tree.acquire(self._mode, self._model_name, self._model_version) - return self - - def __exit__(self, exc_type, exc_value, traceback) -> bool: - self._tree.release(self._mode, self._model_name, self._model_version) - - if exc_value is not None and exc_type is not WithBreak: - return False - return True - - -def find_all_s3_models( - is_dir_used: bool, - models_dir: str, - handler_type: HandlerType, - s3_paths: List[str], - s3_model_names: List[str], -) -> Tuple[ - List[str], - Dict[str, List[str]], - List[str], - List[List[str]], - List[List[datetime.datetime]], - List[str], - List[str], -]: - """ - Get updated information on all models that are currently present on the S3 upstreams. - Information on the available models, versions, last edit times, the subpaths of each model, and so on. - - Args: - is_dir_used: Whether handler:models:dir is used or not. - models_dir: The value of handler:models:dir in case it's present. Ignored when not required. - handler_type: The handler type. - s3_paths: The S3 model paths as they are specified in handler:models:path/handler:models:paths/handler:models:dir is used. Ignored when not required. - s3_model_names: The S3 model names as they are specified in handler:models:paths:name when handler:models:paths is used or the default name of the model when handler:models:path is used. Ignored when not required. - - Returns: The tuple with the following elements: - model_names - a list with the names of the models (i.e. bert, gpt-2, etc) and they are unique - versions - a dictionary with the keys representing the model names and the values being lists of versions that each model has. - For non-versioned model paths ModelVersion.NOT_PROVIDED, the list will be empty. - model_paths - a list with the prefix of each model. - sub_paths - a list of filepaths lists for each file of each model. - timestamps - a list of timestamps lists representing the last edit time of each versioned model. - bucket_names - a list of the bucket names of each model. - """ - - # validate models stored in S3 that were specified with handler:models:dir field - if is_dir_used: - bucket_name, models_path = S3.deconstruct_s3_path(models_dir) - client = S3(bucket_name) - - sub_paths, timestamps = client.search(models_path) - - model_paths, ooa_ids = validate_models_dir_paths(sub_paths, handler_type, models_path) - model_names = [os.path.basename(model_path) for model_path in model_paths] - - model_paths = [ - model_path for model_path in model_paths if os.path.basename(model_path) in model_names - ] - model_paths = [ - model_path + "/" * (not model_path.endswith("/")) for model_path in model_paths - ] - - bucket_names = len(model_paths) * [bucket_name] - sub_paths = len(model_paths) * [sub_paths] - timestamps = len(model_paths) * [timestamps] - - # validate models stored in S3 that were specified with handler:models:paths field - if not is_dir_used: - sub_paths = [] - ooa_ids = [] - model_paths = [] - model_names = [] - timestamps = [] - bucket_names = [] - for idx, path in enumerate(s3_paths): - if S3.is_valid_s3_path(path): - bucket_name, model_path = S3.deconstruct_s3_path(path) - client = S3(bucket_name) - else: - continue - - sb, model_path_ts = client.search(model_path) - try: - ooa_ids.append(validate_model_paths(sb, handler_type, model_path)) - except CortexException: - continue - model_paths.append(model_path) - model_names.append(s3_model_names[idx]) - bucket_names.append(bucket_name) - sub_paths += [sb] - timestamps += [model_path_ts] - - # determine the detected versions for each s3 model - # if the model was not versioned, then leave the version list empty - versions = {} - for model_path, model_name, model_ooa_ids, bucket_sub_paths in zip( - model_paths, model_names, ooa_ids, sub_paths - ): - if ModelVersion.PROVIDED not in model_ooa_ids: - versions[model_name] = [] - continue - - model_sub_paths = [os.path.relpath(sub_path, model_path) for sub_path in bucket_sub_paths] - model_versions_paths = [path for path in model_sub_paths if not path.startswith("../")] - model_versions = [ - util.get_leftmost_part_of_path(model_version_path) - for model_version_path in model_versions_paths - ] - model_versions = list(set(model_versions)) - versions[model_name] = model_versions - - # pick up the max timestamp for each versioned model - aux_timestamps = [] - for model_path, model_name, bucket_sub_paths, sub_path_timestamps in zip( - model_paths, model_names, sub_paths, timestamps - ): - model_ts = [] - if len(versions[model_name]) == 0: - masks = list( - map( - lambda x: x.startswith(model_path + "/" * (model_path[-1] != "/")), - bucket_sub_paths, - ) - ) - model_ts = [max(itertools.compress(sub_path_timestamps, masks))] - - for version in versions[model_name]: - masks = list( - map( - lambda x: x.startswith(os.path.join(model_path, version) + "/"), - bucket_sub_paths, - ) - ) - model_ts.append(max(itertools.compress(sub_path_timestamps, masks))) - - aux_timestamps.append(model_ts) - - timestamps = aux_timestamps # type: List[List[datetime.datetime]] - - # model_names - a list with the names of the models (i.e. bert, gpt-2, etc) and they are unique - # versions - a dictionary with the keys representing the model names and the values being lists of versions that each model has. - # For non-versioned model paths ModelVersion.NOT_PROVIDED, the list will be empty - # model_paths - a list with the prefix of each model - # sub_paths - a list of filepaths lists for each file of each model - # timestamps - a list of timestamps lists representing the last edit time of each versioned model - # bucket_names - names of the buckets - - return model_names, versions, model_paths, sub_paths, timestamps, bucket_names diff --git a/python/serve/cortex_internal/lib/model/type.py b/python/serve/cortex_internal/lib/model/type.py deleted file mode 100644 index 59569bc487..0000000000 --- a/python/serve/cortex_internal/lib/model/type.py +++ /dev/null @@ -1,169 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Optional - -import cortex_internal.consts -from cortex_internal.lib.model import find_all_s3_models -from cortex_internal.lib.type import handler_type_from_api_spec, PythonHandlerType - - -class CuratedModelResources: - def __init__(self, curated_model_resources: List[dict]): - """ - An example of curated_model_resources object: - [ - { - 'path': 's3://cortex-examples/models/tensorflow/transformer/', - 'name': 'modelB', - 'signature_key': None, - 'versions': [1554540232] - }, - ... - ] - """ - self._models = curated_model_resources - - for res in self._models: - if not res["versions"]: - res["versions"] = [] - else: - res["versions"] = [str(version) for version in res["versions"]] - - def get_field(self, field: str) -> List[str]: - """ - Get a list of the values of each models' specified field. - - Args: - field: name, path, signature_key or versions. - - Returns: - A list with the specified value of each model. - """ - return [model[field] for model in self._models] - - def get_versions_for(self, name: str) -> Optional[List[str]]: - """ - Get versions for a given model name. - - Args: - name: Name of the model (_cortex_default for handler:models:path) or handler:models:paths:name. - - Returns: - Versions for a given model. None if the model wasn't found. - """ - versions = [] - model_not_found = True - for i, _ in enumerate(self._models): - if self._models[i]["name"] == name: - versions = self._models[i]["versions"] - model_not_found = False - break - - if model_not_found: - return None - return [str(version) for version in versions] - - def get_s3_model_names(self) -> List[str]: - """ - Get S3 models as specified with handler:models:path or handler:models:paths. - - Returns: - A list of names of all models available from the bucket(s). - """ - return self.get_field("name") - - def __getitem__(self, name: str) -> dict: - """ - Gets the model resource for a given model name. - """ - for model in self._models: - if model["name"] == name: - return model - - raise KeyError(f"model resource {name} does not exit") - - def __contains__(self, name: str) -> bool: - """ - Checks if there's a model resource whose name is the provided one. - """ - try: - self[name] - return True - except KeyError: - return False - - -def get_models_from_api_spec( - api_spec: dict, model_dir: str = "/mnt/model" -) -> CuratedModelResources: - """ - Only effective for models:path, models:paths or for models:dir fields when the dir is a local path. - It does not apply for when models:dir field is set to an S3 model path. - """ - handler_type = handler_type_from_api_spec(api_spec) - - if handler_type == PythonHandlerType and api_spec["handler"]["multi_model_reloading"]: - models_spec = api_spec["handler"]["multi_model_reloading"] - elif handler_type != PythonHandlerType: - models_spec = api_spec["handler"]["models"] - else: - return CuratedModelResources([]) - - if not models_spec["path"] and not models_spec["paths"]: - return CuratedModelResources([]) - - # for models.path - models = [] - if models_spec["path"]: - model = { - "name": cortex_internal.consts.SINGLE_MODEL_NAME, - "path": models_spec["path"], - "signature_key": models_spec["signature_key"], - } - models.append(model) - - # for models.paths - if models_spec["paths"]: - for model in models_spec["paths"]: - models.append( - { - "name": model["name"], - "path": model["path"], - "signature_key": model["signature_key"], - } - ) - - # building model resources for models.path or models.paths - model_resources = [] - for model in models: - model_resource = {} - model_resource["name"] = model["name"] - - if not model["signature_key"]: - model_resource["signature_key"] = models_spec["signature_key"] - else: - model_resource["signature_key"] = model["signature_key"] - - model_resource["path"] = model["path"] - _, versions, _, _, _, _ = find_all_s3_models( - False, "", handler_type, [model_resource["path"]], [model_resource["name"]] - ) - if model_resource["name"] not in versions: - continue - model_resource["versions"] = versions[model_resource["name"]] - - model_resources.append(model_resource) - - return CuratedModelResources(model_resources) diff --git a/python/serve/cortex_internal/lib/model/validation.py b/python/serve/cortex_internal/lib/model/validation.py deleted file mode 100644 index 51a4d2088e..0000000000 --- a/python/serve/cortex_internal/lib/model/validation.py +++ /dev/null @@ -1,529 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import collections -import operator -import os -import uuid -from enum import IntEnum -from fnmatch import fnmatchcase -from typing import List, Any, Tuple - -from cortex_internal.lib import util -from cortex_internal.lib.exceptions import CortexException -from cortex_internal.lib.log import configure_logger -from cortex_internal.lib.type import ( - PythonHandlerType, - TensorFlowHandlerType, - TensorFlowNeuronHandlerType, - HandlerType, -) - -logger = configure_logger("cortex", os.environ["CORTEX_LOG_CONFIG_FILE"]) - - -class TemplatePlaceholder(collections.namedtuple("TemplatePlaceholder", "placeholder priority")): - """ - Placeholder type that denotes an operation, a text placeholder, etc. - - Accessible properties: type, priority. - """ - - def __new__(cls, placeholder: str, priority: int): - return super(cls, TemplatePlaceholder).__new__(cls, "<" + placeholder + ">", priority) - - def __str__(self) -> str: - return str(self.placeholder) - - def __repr__(self) -> str: - return str(self.placeholder) - - @property - def type(self) -> str: - return str(self.placeholder).strip("<>") - - -class GenericPlaceholder( - collections.namedtuple("GenericPlaceholder", "placeholder value priority") -): - """ - Generic placeholder. - - Can hold any value. - Can be of one type only: generic. - - Accessible properties: placeholder, value, type, priority. - """ - - def __new__(cls, value: str): - return super(cls, GenericPlaceholder).__new__(cls, "", value, 0) - - def __eq__(self, other) -> bool: - return isinstance(other, GenericPlaceholder) - - def __hash__(self): - return hash((self.placeholder, self.value)) - - def __str__(self) -> str: - return f"<{self.type}>" + str(self.value) + f"" - - def __repr__(self) -> str: - return f"<{self.type}>" + str(self.value) + f"" - - @property - def type(self) -> str: - return str(self.placeholder).strip("<>") - - -class PlaceholderGroup: - """ - Order-based addition of placeholder types. - Can use AnyPlaceholder, GenericPlaceholder, SinglePlaceholder. - - Accessible properties: parts, type, priority. - """ - - def __init__(self, *args, priority=0): - self.parts = args - self.priority = priority - - def __getitem__(self, index: int): - return self.parts[index] - - def __len__(self) -> int: - return len(self.parts) - - def __str__(self) -> str: - return "" + str(self.parts) + "" - - def __repr__(self) -> str: - return "" + str(self.parts) + "" - - @property - def type(self) -> str: - return str(self.parts) - - -class OneOfAllPlaceholder: - """ - Can be any of the provided alternatives. - - Accessible properties: parts, type, priority, ID. - """ - - def __init__(self, ID: Any = None): - self._placeholder = TemplatePlaceholder("oneofall", priority=-1) - if not ID: - ID = uuid.uuid4().int - self.ID = ID - - def __str__(self) -> str: - return str(self._placeholder) - - def __repr__(self) -> str: - return str(self._placeholder) - - @property - def type(self) -> str: - return str(self._placeholder).strip("<>") - - @property - def priority(self) -> int: - return self._placeholder.priority - - -IntegerPlaceholder = TemplatePlaceholder("integer", priority=1) # the path name must be an integer -SinglePlaceholder = TemplatePlaceholder( - "single", priority=2 -) # can only have a single occurrence of this, but its name can take any form -AnyPlaceholder = TemplatePlaceholder( - "any", priority=4 -) # the path can be any file or any directory (with multiple subdirectories) - - -class ModelVersion(IntEnum): - NOT_PROVIDED = 1 # for models provided without a specific version - PROVIDED = 2 # for models provided with version directories (1, 2, 452, etc). - - -# to be used when handler:models:path, handler:models:paths or handler:models:dir is used -ModelTemplate = { - PythonHandlerType: { - OneOfAllPlaceholder(ModelVersion.PROVIDED): { - IntegerPlaceholder: AnyPlaceholder, - }, - OneOfAllPlaceholder(ModelVersion.NOT_PROVIDED): { - AnyPlaceholder: None, - }, - }, - TensorFlowHandlerType: { - OneOfAllPlaceholder(ModelVersion.PROVIDED): { - IntegerPlaceholder: { - AnyPlaceholder: None, - GenericPlaceholder("saved_model.pb"): None, - GenericPlaceholder("variables"): { - GenericPlaceholder("variables.index"): None, - PlaceholderGroup( - GenericPlaceholder("variables.data-00000-of-"), AnyPlaceholder - ): None, - AnyPlaceholder: None, - }, - }, - }, - OneOfAllPlaceholder(ModelVersion.NOT_PROVIDED): { - AnyPlaceholder: None, - GenericPlaceholder("saved_model.pb"): None, - GenericPlaceholder("variables"): { - GenericPlaceholder("variables.index"): None, - PlaceholderGroup( - GenericPlaceholder("variables.data-00000-of-"), AnyPlaceholder - ): None, - AnyPlaceholder: None, - }, - }, - }, - TensorFlowNeuronHandlerType: { - OneOfAllPlaceholder(ModelVersion.PROVIDED): { - IntegerPlaceholder: { - GenericPlaceholder("saved_model.pb"): None, - AnyPlaceholder: None, - } - }, - OneOfAllPlaceholder(ModelVersion.NOT_PROVIDED): { - GenericPlaceholder("saved_model.pb"): None, - AnyPlaceholder: None, - }, - }, -} - - -def json_model_template_representation(model_template) -> dict: - dct = {} - if model_template is None: - return None - if isinstance(model_template, dict): - if any(isinstance(x, OneOfAllPlaceholder) for x in model_template): - oneofall_placeholder_index = 0 - for key in model_template: - if isinstance(key, OneOfAllPlaceholder): - dct[ - str(key) + f"-{oneofall_placeholder_index}" - ] = json_model_template_representation(model_template[key]) - oneofall_placeholder_index += 1 - else: - dct[str(key)] = json_model_template_representation(model_template[key]) - return dct - else: - return str(model_template) - - -def _single_model_pattern(handler_type: HandlerType) -> dict: - """ - To be used when handler:models:path or handler:models:paths in cortex.yaml is used. - """ - return ModelTemplate[handler_type] - - -def validate_models_dir_paths( - paths: List[str], handler_type: HandlerType, common_prefix: str -) -> Tuple[List[str], List[List[int]]]: - """ - Validates the models paths based on the given handler type. - To be used when handler:models:dir in cortex.yaml is used. - - Args: - paths: A list of all paths for a given s3/local prefix. Must be underneath the common prefix. - handler_type: The handler type. - common_prefix: The common prefix of the directory which holds all models. AKA handler:models:dir. - - Returns: - List with the prefix of each model that's valid. - List with the OneOfAllPlaceholder IDs validated for each valid model. - """ - if len(paths) == 0: - raise CortexException( - f"{handler_type} handler at '{common_prefix}'", "model top path can't be empty" - ) - - rel_paths = [os.path.relpath(top_path, common_prefix) for top_path in paths] - rel_paths = [path for path in rel_paths if not path.startswith("../")] - - model_names = [util.get_leftmost_part_of_path(path) for path in rel_paths] - model_names = list(set(model_names)) - - valid_model_prefixes = [] - ooa_valid_key_ids = [] - for model_name in model_names: - try: - ooa_valid_key_ids.append(validate_model_paths(rel_paths, handler_type, model_name)) - valid_model_prefixes.append(os.path.join(common_prefix, model_name)) - except CortexException as e: - logger.debug(f"failed validating model {model_name}: {str(e)}") - continue - - return valid_model_prefixes, ooa_valid_key_ids - - -def validate_model_paths( - paths: List[str], handler_type: HandlerType, common_prefix: str -) -> List[int]: - """ - To be used when handler:models:path or handler:models:paths in cortex.yaml is used. - - Args: - paths: A list of all paths for a given s3/local prefix. Must be the top directory of a model. - handler_type: Handler type. Can be PythonHandlerType, TensorFlowHandlerType or TensorFlowNeuronHandlerType. - common_prefix: The common prefix of the directory which holds all models. - - Returns: - List of all OneOfAllPlaceholder IDs that had been validated. - - Exception: - CortexException if the paths don't match the model's template. - """ - if len(paths) == 0: - raise CortexException( - f"{handler_type} handler at '{common_prefix}'", "model path can't be empty" - ) - - paths_by_prefix_cache = {} - - def _validate_model_paths(pattern: Any, paths: List[str], common_prefix: str) -> None: - if common_prefix not in paths_by_prefix_cache: - paths_by_prefix_cache[common_prefix] = util.get_paths_with_prefix(paths, common_prefix) - paths = paths_by_prefix_cache[common_prefix] - - rel_paths = [os.path.relpath(path, common_prefix) for path in paths] - rel_paths = [path for path in rel_paths if not path.startswith("../")] - - objects = [util.get_leftmost_part_of_path(path) for path in rel_paths] - objects = list(set(objects)) - visited_objects = len(objects) * [False] - - ooa_valid_key_ids = [] # OneOfAllPlaceholder IDs that are valid - - if pattern is None: - if len(objects) == 1 and objects[0] == ".": - return ooa_valid_key_ids - raise CortexException( - f"{handler_type} handler at '{common_prefix}'", - "template doesn't specify a substructure for the given path", - ) - if not isinstance(pattern, dict): - pattern = {pattern: None} - - keys = list(pattern.keys()) - keys.sort(key=operator.attrgetter("priority")) - - try: - if any(isinstance(x, OneOfAllPlaceholder) for x in keys) and not all( - isinstance(x, OneOfAllPlaceholder) for x in keys - ): - raise CortexException( - f"{handler_type} handler at '{common_prefix}'", - f"{OneOfAllPlaceholder()} is a mutual-exclusive key with all other keys", - ) - elif all(isinstance(x, OneOfAllPlaceholder) for x in keys): - num_keys = len(keys) - num_validation_failures = 0 - - for key_id, key in enumerate(keys): - if key == IntegerPlaceholder: - _validate_integer_placeholder(keys, key_id, objects, visited_objects) - elif key == AnyPlaceholder: - _validate_any_placeholder(keys, key_id, objects, visited_objects) - elif key == SinglePlaceholder: - _validate_single_placeholder(keys, key_id, objects, visited_objects) - elif isinstance(key, GenericPlaceholder): - _validate_generic_placeholder(keys, key_id, objects, visited_objects, key) - elif isinstance(key, PlaceholderGroup): - _validate_group_placeholder(keys, key_id, objects, visited_objects) - elif isinstance(key, OneOfAllPlaceholder): - try: - _validate_model_paths(pattern[key], paths, common_prefix) - ooa_valid_key_ids.append(key.ID) - except CortexException: - num_validation_failures += 1 - else: - raise CortexException("found a non-placeholder object in model template") - - except CortexException as e: - raise CortexException(f"{handler_type} handler at '{common_prefix}'", str(e)) - - if ( - all(isinstance(x, OneOfAllPlaceholder) for x in keys) - and num_validation_failures == num_keys - ): - raise CortexException( - f"couldn't validate for any of the {OneOfAllPlaceholder()} placeholders" - ) - if all(isinstance(x, OneOfAllPlaceholder) for x in keys): - return ooa_valid_key_ids - - unvisited_paths = [] - for idx, visited in enumerate(visited_objects): - if visited is False: - untraced_common_prefix = os.path.join(common_prefix, objects[idx]) - untraced_paths = [os.path.relpath(path, untraced_common_prefix) for path in paths] - untraced_paths = [ - os.path.join(objects[idx], path) - for path in untraced_paths - if not path.startswith("../") - ] - unvisited_paths += untraced_paths - if len(unvisited_paths) > 0: - raise CortexException( - f"{handler_type} handler model at '{common_prefix}'", - "unexpected path(s) for " + str(unvisited_paths), - ) - - new_common_prefixes = [] - sub_patterns = [] - paths_by_prefix = {} - for obj_id, key_id in enumerate(visited_objects): - obj = objects[obj_id] - key = keys[key_id] - if key != AnyPlaceholder: - new_common_prefixes.append(os.path.join(common_prefix, obj)) - sub_patterns.append(pattern[key]) - - if len(new_common_prefixes) > 0: - paths_by_prefix = util.get_paths_by_prefixes(paths, new_common_prefixes) - - aggregated_ooa_valid_key_ids = [] - for sub_pattern, new_common_prefix in zip(sub_patterns, new_common_prefixes): - aggregated_ooa_valid_key_ids += _validate_model_paths( - sub_pattern, paths_by_prefix[new_common_prefix], new_common_prefix - ) - - return aggregated_ooa_valid_key_ids - - pattern = _single_model_pattern(handler_type) - return _validate_model_paths(pattern, paths, common_prefix) - - -def _validate_integer_placeholder( - placeholders: list, key_id: int, objects: List[str], visited: list -) -> None: - appearances = 0 - for idx, obj in enumerate(objects): - if obj.isnumeric() and visited[idx] is False: - visited[idx] = key_id - appearances += 1 - - if appearances > 1 and len(placeholders) > 1: - raise CortexException(f"too many {IntegerPlaceholder} appearances in path") - if appearances == 0: - raise CortexException(f"{IntegerPlaceholder} not found in path") - - -def _validate_any_placeholder( - placeholders: list, - key_id: int, - objects: List[str], - visited: list, -) -> None: - for idx, obj in enumerate(objects): - if visited[idx] is False and obj != ".": - visited[idx] = key_id - - -def _validate_single_placeholder( - placeholders: list, key_id: int, objects: List[str], visited: list -) -> None: - if len(placeholders) > 1 or len(objects) > 1: - raise CortexException(f"only a single {SinglePlaceholder} is allowed per directory") - if len(visited) > 0 and visited[0] is False: - visited[0] = key_id - - -def _validate_generic_placeholder( - placeholders: list, - key_id: int, - objects: List[str], - visited: list, - generical: GenericPlaceholder, -) -> None: - found = False - for idx, obj in enumerate(objects): - if obj == generical.value: - if visited[idx] is False: - visited[idx] = key_id - found = True - return - - if not found: - raise CortexException(f"{generical.type} placeholder for {generical} wasn't found") - - -def _validate_group_placeholder( - placeholders: list, key_id: int, objects: List[str], visited: list -) -> None: - """ - Can use AnyPlaceholder, GenericPlaceholder, SinglePlaceholder. - - The minimum number of placeholders a group must hold is 2. - - The accepted formats are: - - ... AnyPlaceholder, GenericPlaceholder, AnyPlaceholder, ... - - ... SinglePlaceholder, GenericPlaceholder, SinglePlaceholder, ... - - AnyPlaceholder and SinglePlaceholder cannot be mixed together in one group. - """ - - placeholder_group = placeholders[key_id] - - if len(placeholder_group) < 2: - raise CortexException(f"{placeholder_group} must come with at least 2 placeholders") - - for placeholder in placeholder_group: - if placeholder not in [AnyPlaceholder, SinglePlaceholder] and not isinstance( - placeholder, GenericPlaceholder - ): - raise CortexException( - f'{placeholder_group} must have a combination of the following placeholder types: {AnyPlaceholder}, {SinglePlaceholder}, {GenericPlaceholder("").placeholder}' - ) - - if {AnyPlaceholder, SinglePlaceholder}.issubset(set(placeholder_group)): - raise CortexException( - f"{placeholder_group} cannot have a mix of the following placeholder types: {AnyPlaceholder} and {SinglePlaceholder}" - ) - - group_len = len(placeholder_group) - for idx in range(group_len): - if idx + 1 < group_len: - a = placeholder_group[idx] - b = placeholder_group[idx + 1] - if a == b: - raise CortexException( - f'{placeholder_group} cannot accept the same type to be specified consecutively ({AnyPlaceholder}, {SinglePlaceholder} or {GenericPlaceholder("").placeholder})' - ) - - pattern = "" - for placeholder in placeholder_group: - if placeholder in [AnyPlaceholder, SinglePlaceholder]: - pattern += "*" - if isinstance(placeholder, GenericPlaceholder): - pattern += str(placeholder.value) - - num_occurences = 0 - for idx, obj in enumerate(objects): - if visited[idx] is False and fnmatchcase(obj, pattern): - visited[idx] = key_id - num_occurences += 1 - - if SinglePlaceholder in placeholder_group and num_occurences > 1: - raise CortexException( - f"{placeholder_group} must match once (not {num_occurences} times) because {SinglePlaceholder} is present" - ) diff --git a/python/serve/cortex_internal/lib/queue/__init__.py b/python/serve/cortex_internal/lib/queue/__init__.py deleted file mode 100644 index dcd1d9ae2f..0000000000 --- a/python/serve/cortex_internal/lib/queue/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/python/serve/cortex_internal/lib/queue/sqs.py b/python/serve/cortex_internal/lib/queue/sqs.py deleted file mode 100644 index d6a00559b5..0000000000 --- a/python/serve/cortex_internal/lib/queue/sqs.py +++ /dev/null @@ -1,185 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import threading -import time -from typing import Callable, Dict, Any, Optional, Tuple - -import botocore.exceptions - -from cortex_internal.lib.exceptions import UserRuntimeException -from cortex_internal.lib.log import logger as log -from cortex_internal.lib.signals import SignalHandler -from cortex_internal.lib.telemetry import capture_exception - - -class SQSHandler: - def __init__( - self, - sqs_client, - queue_url: str, - dead_letter_queue_url: str = None, - message_wait_time: int = 10, - visibility_timeout: int = 30, - not_found_sleep_time: int = 10, - renewal_period: int = 15, - stop_if_no_messages: bool = False, - ): - self.sqs_client = sqs_client - self.queue_url = queue_url - self.dead_letter_queue_url = dead_letter_queue_url - self.message_wait_time = message_wait_time - self.visibility_timeout = visibility_timeout - self.not_found_sleep_time = not_found_sleep_time - self.renewal_period = renewal_period - self.stop_if_no_messages = stop_if_no_messages - - self.receipt_handle_mutex = threading.Lock() - self.stop_renewal = set() - - def start( - self, - message_fn: Callable[[Dict[str, Any]], None], - message_failure_fn: Callable[[Dict[str, Any]], None], - on_job_complete_fn: Optional[Callable[[Dict[str, Any]], None]] = None, - ): - no_messages_found_in_previous_iteration = False - signal_handler = SignalHandler() - - while not signal_handler.received_signal(): - response = self.sqs_client.receive_message( - QueueUrl=self.queue_url, - MaxNumberOfMessages=1, - WaitTimeSeconds=self.message_wait_time, - VisibilityTimeout=self.visibility_timeout, - MessageAttributeNames=["All"], - ) - - if response.get("Messages") is None or len(response["Messages"]) == 0: - visible_messages, invisible_messages = self._get_total_messages_in_queue() - if visible_messages + invisible_messages == 0: - if no_messages_found_in_previous_iteration and self.stop_if_no_messages: - log.info("no messages left in queue, exiting...") - return - no_messages_found_in_previous_iteration = True - - time.sleep(self.not_found_sleep_time) - continue - - no_messages_found_in_previous_iteration = False - message = response["Messages"][0] - receipt_handle = message["ReceiptHandle"] - - renewer = threading.Thread( - target=self._renew_message_visibility, - args=(receipt_handle,), - daemon=True, - ) - renewer.start() - - if is_on_job_complete(message): - self._on_job_complete(message, on_job_complete_fn) - else: - self._handle_message(message, message_fn, message_failure_fn) - - def _renew_message_visibility(self, receipt_handle: str): - interval = self.renewal_period - new_timeout = self.visibility_timeout - - cur_time = time.time() - while True: - time.sleep((cur_time + interval) - time.time()) - cur_time += interval - new_timeout += interval - - with self.receipt_handle_mutex: - if receipt_handle in self.stop_renewal: - self.stop_renewal.remove(receipt_handle) - break - - try: - self.sqs_client.change_message_visibility( - QueueUrl=self.queue_url, - ReceiptHandle=receipt_handle, - VisibilityTimeout=new_timeout, - ) - except botocore.exceptions.ClientError as err: - if err.response["Error"]["Code"] == "InvalidParameterValue": - # unexpected; this error is thrown when attempting to renew a message that has been deleted - continue - elif err.response["Error"]["Code"] == "AWS.SimpleQueueService.NonExistentQueue": - # there may be a delay between the cron may deleting the queue and this worker stopping - log.info( - "failed to renew message visibility because the queue was not found" - ) - else: - self.stop_renewal.remove(receipt_handle) - raise err - - def _get_total_messages_in_queue(self): - return get_total_messages_in_queue(sqs_client=self.sqs_client, queue_url=self.queue_url) - - def _handle_message(self, message, callback_fn, failure_callback_fn): - receipt_handle = message["ReceiptHandle"] - - try: - callback_fn(message) - except Exception as err: - if not isinstance(err, UserRuntimeException): - capture_exception(err) - - failure_callback_fn(message) - - with self.receipt_handle_mutex: - self.stop_renewal.add(receipt_handle) - if self.dead_letter_queue_url is not None: - self.sqs_client.change_message_visibility( # return message - QueueUrl=self.queue_url, ReceiptHandle=receipt_handle, VisibilityTimeout=0 - ) - else: - self.sqs_client.delete_message( - QueueUrl=self.queue_url, ReceiptHandle=receipt_handle - ) - else: - with self.receipt_handle_mutex: - self.stop_renewal.add(receipt_handle) - self.sqs_client.delete_message( - QueueUrl=self.queue_url, ReceiptHandle=receipt_handle - ) - - def _on_job_complete(self, message, callback_fn): - receipt_handle = message["ReceiptHandle"] - try: - callback_fn(message) - except Exception as err: - raise type(err)("failed to handle on_job_complete") from err - finally: - with self.receipt_handle_mutex: - self.stop_renewal.add(receipt_handle) - self.sqs_client.delete_message( - QueueUrl=self.queue_url, ReceiptHandle=receipt_handle - ) - - -def get_total_messages_in_queue(sqs_client, queue_url: str) -> Tuple[int, int]: - attributes = sqs_client.get_queue_attributes(QueueUrl=queue_url, AttributeNames=["All"])[ - "Attributes" - ] - visible_count = int(attributes.get("ApproximateNumberOfMessages", 0)) - not_visible_count = int(attributes.get("ApproximateNumberOfMessagesNotVisible", 0)) - return visible_count, not_visible_count - - -def is_on_job_complete(message: Dict[str, Any]) -> bool: - return "MessageAttributes" in message and "job_complete" in message["MessageAttributes"] diff --git a/python/serve/cortex_internal/lib/signals.py b/python/serve/cortex_internal/lib/signals.py deleted file mode 100644 index f3ea829d41..0000000000 --- a/python/serve/cortex_internal/lib/signals.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from signal import signal, SIGINT, SIGTERM - -from cortex_internal.lib.log import logger as log - - -class SignalHandler: - def __init__(self): - self.__received_signal = False - signal(SIGINT, self._signal_handler) - signal(SIGTERM, self._signal_handler) - - def _signal_handler(self, sys_signal, _): - log.info(f"handling signal {sys_signal}, exiting gracefully") - self.__received_signal = True - - def received_signal(self): - return self.__received_signal diff --git a/python/serve/cortex_internal/lib/storage/__init__.py b/python/serve/cortex_internal/lib/storage/__init__.py deleted file mode 100644 index 2398fca715..0000000000 --- a/python/serve/cortex_internal/lib/storage/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from cortex_internal.lib.storage.local import LocalStorage -from cortex_internal.lib.storage.s3 import S3 diff --git a/python/serve/cortex_internal/lib/storage/local.py b/python/serve/cortex_internal/lib/storage/local.py deleted file mode 100644 index 86c81dd16b..0000000000 --- a/python/serve/cortex_internal/lib/storage/local.py +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import shutil -import time -from pathlib import Path - -import msgpack - -from cortex_internal.lib import util -from cortex_internal.lib.exceptions import CortexException - - -class LocalStorage(object): - def __init__(self, base_dir): - self.base_dir = base_dir - - def _get_path(self, key): - return Path(os.path.join(self.base_dir, key)) - - def _get_or_create_path(self, key): - p = Path(os.path.join(self.base_dir, key)) - p.parent.mkdir(parents=True, exist_ok=True) - return p - - def _get_path_if_exists(self, key, allow_missing=False, num_retries=0, retry_delay_sec=2): - while True: - try: - return self._get_path_if_exists_single(key, allow_missing=allow_missing) - except: - if num_retries <= 0: - raise - num_retries -= 1 - time.sleep(retry_delay_sec) - - def _get_path_if_exists_single(self, key, allow_missing=False): - p = Path(os.path.join(self.base_dir, key)) - if not p.exists() and allow_missing: - return None - elif not p.exists() and not allow_missing: - raise KeyError(p + " not found in local storage") - return p - - def blob_path(self, key): - return os.path.join(self.base_dir, key) - - def search(self, prefix="", suffix=""): - files = [] - for root, dirs, files in os.walk(self.base_dir, topdown=False): - common_prefix_len = min(len(prefix), len(root)) - if root[:common_prefix_len] != prefix[:common_prefix_len]: - continue - - for name in files: - filename = os.path.join(root, name) - if filename.startswith(prefix) and filename.endswith(suffix): - files.append(filename) - return files - - def _put_str(self, str_val, key): - f = self._get_or_create_path(key) - f.write_text(str_val) - - def put_str(self, str_val, key): - self._put_str(str_val, key) - - def put_json(self, obj, key): - self._put_str(json.dumps(obj), key) - - def get_json(self, key, allow_missing=False, num_retries=0, retry_delay_sec=2): - f = self._get_path_if_exists( - key, - allow_missing=allow_missing, - num_retries=num_retries, - retry_delay_sec=retry_delay_sec, - ) - if f is None: - return None - return json.loads(f.read_text()) - - def put_object(self, body: bytes, key): - f = self._get_or_create_path(key) - f.write_bytes(body) - - def put_msgpack(self, obj, key): - f = self._get_or_create_path(key) - f.write_bytes(msgpack.dumps(obj)) - - def get_msgpack(self, key, allow_missing=False, num_retries=0, retry_delay_sec=2): - f = self._get_path_if_exists( - key, - allow_missing=allow_missing, - num_retries=num_retries, - retry_delay_sec=retry_delay_sec, - ) - if f is None: - return None - return msgpack.loads(f.read_bytes()) - - def upload_file(self, local_path, key): - shutil.copy(local_path, str(self._get_or_create_path(key))) - - def download_file(self, key, local_path): - try: - Path(local_path).parent.mkdir(parents=True, exist_ok=True) - shutil.copy(str(self._get_path(key)), local_path) - except Exception as e: - raise CortexException("file not found", key) from e - - def download_and_unzip(self, key, local_dir): - util.mkdir_p(local_dir) - local_zip = os.path.join(local_dir, "zip.zip") - self.download_file(key, local_zip) - util.extract_zip(local_zip, delete_zip_file=True) diff --git a/python/serve/cortex_internal/lib/storage/s3.py b/python/serve/cortex_internal/lib/storage/s3.py deleted file mode 100644 index f9c642cfc3..0000000000 --- a/python/serve/cortex_internal/lib/storage/s3.py +++ /dev/null @@ -1,250 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime -import json -import os -import time -from typing import List, Tuple - -import boto3 -import botocore -import msgpack - -from cortex_internal.lib import util -from cortex_internal.lib.exceptions import CortexException - - -class S3: - def __init__(self, bucket=None, region=None, client_config={}): - self.bucket = bucket - self.region = region - - if client_config is None: - client_config = {} - - if region is not None: - client_config["region_name"] = region - - session = boto3.Session() - credentials = session.get_credentials() - - # use anonymous client if credentials haven't been detected - if credentials is None: - client_config["config"] = botocore.client.Config(signature_version=botocore.UNSIGNED) - - self.s3 = boto3.client("s3", **client_config) - - @staticmethod - def construct_s3_path(bucket_name: str, prefix: str) -> str: - return f"s3://{bucket_name}/{prefix}" - - @staticmethod - def deconstruct_s3_path(s3_path: str) -> Tuple[str, str]: - path = util.trim_prefix(s3_path, "s3://") - bucket = path.split("/")[0] - key = os.path.join(*path.split("/")[1:]) - return bucket, key - - @staticmethod - def is_valid_s3_path(path: str) -> bool: - if not path.startswith("s3://"): - return False - parts = path[5:].split("/") - if len(parts) < 2: - return False - if parts[0] == "" or parts[1] == "": - return False - return True - - def blob_path(self, key): - return os.path.join("s3://", self.bucket, key) - - def _file_exists(self, key): - try: - self.s3.head_object(Bucket=self.bucket, Key=key) - return True - except botocore.exceptions.ClientError as e: - if e.response["Error"]["Code"] == "404": - return False - else: - raise - - def _is_s3_prefix(self, prefix): - response = self.s3.list_objects_v2(Bucket=self.bucket, Prefix=prefix) - return response["KeyCount"] > 0 - - def _is_s3_dir(self, dir_path): - prefix = util.ensure_suffix(dir_path, "/") - return self._is_s3_prefix(prefix) - - def _get_matching_s3_objects_generator(self, prefix="", suffix="", include_dir_objects=False): - kwargs = {"Bucket": self.bucket, "Prefix": prefix} - - while True: - resp = self.s3.list_objects_v2(**kwargs) - try: - contents = resp["Contents"] - except KeyError: - return - - for obj in contents: - key = obj["Key"] - if ( - key.startswith(prefix) - and key.endswith(suffix) - and (include_dir_objects or not key.endswith("/")) - ): - yield obj - - try: - kwargs["ContinuationToken"] = resp["NextContinuationToken"] - except KeyError: - break - - def _get_matching_s3_keys_generator(self, prefix="", suffix="", include_dir_objects=False): - for obj in self._get_matching_s3_objects_generator(prefix, suffix, include_dir_objects): - yield obj["Key"], obj["LastModified"] - - def put_object(self, body, key): - self.s3.put_object(Bucket=self.bucket, Key=key, Body=body) - - def get_object(self, key): - return self.s3.get_object(Bucket=self.bucket, Key=key) - - def _read_bytes_from_s3( - self, key, allow_missing=False, ext_bucket=None, num_retries=0, retry_delay_sec=2 - ): - while True: - try: - return self._read_bytes_from_s3_single( - key, allow_missing=allow_missing, ext_bucket=ext_bucket - ) - except: - if num_retries <= 0: - raise - num_retries -= 1 - time.sleep(retry_delay_sec) - - def _read_bytes_from_s3_single(self, key, allow_missing=False, ext_bucket=None): - bucket = self.bucket - if ext_bucket is not None: - bucket = ext_bucket - - try: - try: - byte_array = self.s3.get_object(Bucket=bucket, Key=key)["Body"].read() - except self.s3.exceptions.NoSuchKey: - if allow_missing: - return None - raise - except Exception as e: - raise CortexException( - 'key "{}" in bucket "{}" could not be accessed; '.format(key, bucket) - + "it may not exist, or you may not have sufficient permissions" - ) from e - - return byte_array.strip() - - def search( - self, prefix="", suffix="", include_dir_objects=False - ) -> Tuple[List[str], List[datetime.datetime]]: - paths = [] - timestamps = [] - - for key, ts in self._get_matching_s3_keys_generator(prefix, suffix, include_dir_objects): - paths.append(key) - timestamps.append(ts) - - return paths, timestamps - - def put_str(self, str_val, key): - self.put_object(str_val, key) - - def put_json(self, obj, key): - self.put_object(json.dumps(obj), key) - - def get_json(self, key, allow_missing=False, num_retries=0, retry_delay_sec=2): - obj = self._read_bytes_from_s3( - key, - allow_missing=allow_missing, - num_retries=num_retries, - retry_delay_sec=retry_delay_sec, - ) - if obj is None: - return None - return json.loads(obj.decode("utf-8")) - - def put_msgpack(self, obj, key): - self.put_object(msgpack.dumps(obj), key) - - def get_msgpack(self, key, allow_missing=False, num_retries=0, retry_delay_sec=2): - obj = self._read_bytes_from_s3( - key, - allow_missing=allow_missing, - num_retries=num_retries, - retry_delay_sec=retry_delay_sec, - ) - if obj == None: - return None - return msgpack.loads(obj, raw=False) - - def upload_file(self, local_path, key): - self.s3.upload_file(local_path, self.bucket, key) - - def download_file_to_dir(self, key, local_dir_path): - filename = os.path.basename(key) - return self.download_file(key, os.path.join(local_dir_path, filename)) - - def download_file(self, key, local_path): - util.mkdir_p(os.path.dirname(local_path)) - try: - self.s3.download_file(self.bucket, key, local_path) - return local_path - except Exception as e: - raise CortexException( - 'key "{}" in bucket "{}" could not be accessed; '.format(key, self.bucket) - + "it may not exist, or you may not have sufficient permissions" - ) from e - - def download_dir(self, prefix, local_dir): - dir_name = util.trim_suffix(prefix, "/").split("/")[-1] - return self.download_dir_contents(prefix, os.path.join(local_dir, dir_name)) - - def download_dir_contents(self, prefix, local_dir): - util.mkdir_p(local_dir) - prefix = util.ensure_suffix(prefix, "/") - for key, _ in self._get_matching_s3_keys_generator(prefix, include_dir_objects=True): - rel_path = util.trim_prefix(key, prefix) - local_dest_path = os.path.join(local_dir, rel_path) - - if not local_dest_path.endswith("/"): - self.download_file(key, local_dest_path) - else: - util.mkdir_p(os.path.dirname(local_dest_path)) - - def download_and_unzip(self, key, local_dir): - util.mkdir_p(local_dir) - local_zip = os.path.join(local_dir, "zip.zip") - self.download_file(key, local_zip) - util.extract_zip(local_zip, delete_zip_file=True) - - def download(self, prefix, local_dir): - if self._is_s3_dir(prefix): - self.download_dir(prefix, local_dir) - else: - self.download_file_to_dir(prefix, local_dir) - - def delete(self, key): - self.s3.delete_object(Bucket=self.bucket, Key=key) diff --git a/python/serve/cortex_internal/lib/stringify.py b/python/serve/cortex_internal/lib/stringify.py deleted file mode 100644 index aae88a5fef..0000000000 --- a/python/serve/cortex_internal/lib/stringify.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections.abc import Iterable - - -def truncate(item, max_elements=10, max_str_len=500): - if isinstance(item, str): - s = item - if max_str_len > 3 and len(s) > max_str_len: - s = s[: max_str_len - 3] + "..." - return '"{}"'.format(s) - - if isinstance(item, dict): - count = 0 - item_strs = [] - for key in item: - if max_elements > 0 and count >= max_elements: - item_strs.append("...") - break - - key_str = truncate(key, max_elements, max_str_len) - val_str = truncate(item[key], max_elements, max_str_len) - item_strs.append("{}: {}".format(key_str, val_str)) - count += 1 - - return "{" + ", ".join(item_strs) + "}" - - if isinstance(item, Iterable) and hasattr(item, "__getitem__"): - item_strs = [] - - for element in item[:max_elements]: - item_strs.append(truncate(element, max_elements, max_str_len)) - - if max_elements > 0 and len(item) > max_elements: - item_strs.append("...") - - return "[" + ", ".join(item_strs) + "]" - - # Fallback - s = str(item) - if max_str_len > 3 and len(s) > max_str_len: - s = s[: max_str_len - 3] + "..." - return s diff --git a/python/serve/cortex_internal/lib/telemetry.py b/python/serve/cortex_internal/lib/telemetry.py deleted file mode 100644 index 52bff60ad1..0000000000 --- a/python/serve/cortex_internal/lib/telemetry.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from typing import Optional - -import sentry_sdk -from sentry_sdk.integrations.logging import LoggingIntegration -from sentry_sdk.integrations.threading import ThreadingIntegration - -import cortex_internal.lib.exceptions as cexp - - -def get_default_tags(): - vars = { - "kind": "CORTEX_KIND", - "image_type": "CORTEX_IMAGE_TYPE", - } - - tags = {} - for target_tag, env_var in vars.items(): - value = os.getenv(env_var) - if value and value != "": - tags[target_tag] = value - - return tags - - -def init_sentry( - dsn: str = "", - environment: str = "", - release: str = "", - tags: dict = {}, - disabled: Optional[bool] = None, -): - """ - Initialize sentry. If no arguments are passed in, the following env vars will be used instead. - 1. dsn -> CORTEX_TELEMETRY_SENTRY_DSN - 2. environment -> CORTEX_TELEMETRY_SENTRY_ENVIRONMENT - 3. release -> CORTEX_VERSION - 4. disabled -> CORTEX_TELEMETRY_DISABLE - - In addition to that, the user ID tag is added to every reported event. - """ - - if ( - disabled is True - or os.getenv("CORTEX_TELEMETRY_DISABLE", "").lower() == "true" - or os.getenv("CORTEX_DEBUGGING", "true").lower() == "true" - ): - return - - if dsn == "": - dsn = os.environ["CORTEX_TELEMETRY_SENTRY_DSN"] - - if environment == "": - environment = os.environ["CORTEX_TELEMETRY_SENTRY_ENVIRONMENT"] - - if release == "": - release = os.environ["CORTEX_VERSION"] - - user_id = os.environ["CORTEX_TELEMETRY_SENTRY_USER_ID"] - - sentry_sdk.init( - dsn=dsn, - environment=environment, - release=release, - ignore_errors=[cexp.UserRuntimeException], - attach_stacktrace=True, - integrations=[ - LoggingIntegration(event_level=None, level=None), - ThreadingIntegration(propagate_hub=True), - ], - ) - sentry_sdk.set_user({"id": user_id}) - - for k, v in tags.items(): - sentry_sdk.set_tag(k, v) - - -def capture_exception(err: Exception, level: str = "error", extra_tags: dict = {}): - """ - Capture an exception. Optionally set the log level of the event. Extra tags accepted. - """ - - with sentry_sdk.push_scope() as scope: - scope.set_level(level) - for k, v in extra_tags: - scope.set_tag(k, v) - sentry_sdk.capture_exception(err, scope) - sentry_sdk.flush() diff --git a/python/serve/cortex_internal/lib/test/dynamic_batching_test.py b/python/serve/cortex_internal/lib/test/dynamic_batching_test.py deleted file mode 100644 index 0cba92b705..0000000000 --- a/python/serve/cortex_internal/lib/test/dynamic_batching_test.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import itertools -import threading as td -import time - -from cortex_internal.lib.api.utils import DynamicBatcher - - -class Handler: - def handle_post(self, payload): - time.sleep(0.2) - return payload - - -def test_dynamic_batching_while_hitting_max_batch_size(): - max_batch_size = 32 - dynamic_batcher = DynamicBatcher( - Handler(), - method_name="handle_post", - max_batch_size=max_batch_size, - batch_interval=0.1, - test_mode=True, - ) - counter = itertools.count(1) - event = td.Event() - global_list = [] - - def submitter(): - while not event.is_set(): - global_list.append(dynamic_batcher.process(payload=next(counter))) - time.sleep(0.1) - - running_threads = [] - for _ in range(128): - thread = td.Thread(target=submitter, daemon=True) - thread.start() - running_threads.append(thread) - - time.sleep(60) - event.set() - - # if this fails, then the submitter threads are getting stuck - for thread in running_threads: - thread.join(3.0) - if thread.is_alive(): - raise TimeoutError("thread", thread.getName(), "got stuck") - - sum1 = int(len(global_list) * (len(global_list) + 1) / 2) - sum2 = sum(global_list) - assert sum1 == sum2 - - # get the last 80% of batch lengths - # we ignore the first 20% because it may take some time for all threads to start making requests - batch_lengths = dynamic_batcher._test_batch_lengths - batch_lengths = batch_lengths[int(len(batch_lengths) * 0.2) :] - - # verify that the batch size is always equal to the max batch size - assert len(set(batch_lengths)) == 1 - assert max_batch_size in batch_lengths - - -def test_dynamic_batching_while_hitting_max_interval(): - max_batch_size = 32 - dynamic_batcher = DynamicBatcher( - Handler(), - method_name="handle_post", - max_batch_size=max_batch_size, - batch_interval=1.0, - test_mode=True, - ) - counter = itertools.count(1) - event = td.Event() - global_list = [] - - def submitter(): - while not event.is_set(): - global_list.append(dynamic_batcher.process(payload=next(counter))) - time.sleep(0.1) - - running_threads = [] - for _ in range(2): - thread = td.Thread(target=submitter, daemon=True) - thread.start() - running_threads.append(thread) - - time.sleep(30) - event.set() - - # if this fails, then the submitter threads are getting stuck - for thread in running_threads: - thread.join(3.0) - if thread.is_alive(): - raise TimeoutError("thread", thread.getName(), "got stuck") - - sum1 = int(len(global_list) * (len(global_list) + 1) / 2) - sum2 = sum(global_list) - assert sum1 == sum2 - - # get the last 80% of batch lengths - # we ignore the first 20% because it may take some time for all threads to start making requests - batch_lengths = dynamic_batcher._test_batch_lengths - batch_lengths = batch_lengths[int(len(batch_lengths) * 0.2) :] - - # verify that the batch size is always equal to the number of running threads - assert len(set(batch_lengths)) == 1 - assert len(running_threads) in batch_lengths diff --git a/python/serve/cortex_internal/lib/test/util_test.py b/python/serve/cortex_internal/lib/test/util_test.py deleted file mode 100644 index 4e870558ec..0000000000 --- a/python/serve/cortex_internal/lib/test/util_test.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from copy import deepcopy - -from cortex_internal.lib import util - - -def test_merge_dicts(): - dict1 = {"k1": "v1", "k2": "v2", "k3": {"k1": "v1", "k2": "v2"}} - dict2 = {"k1": "V1", "k4": "V4", "k3": {"k1": "V1", "k4": "V4"}} - - expected1 = {"k1": "V1", "k2": "v2", "k4": "V4", "k3": {"k1": "V1", "k2": "v2", "k4": "V4"}} - expected2 = {"k1": "v1", "k2": "v2", "k4": "V4", "k3": {"k1": "v1", "k2": "v2", "k4": "V4"}} - - merged = util.merge_dicts_overwrite(dict1, dict2) - assert expected1 == merged - assert dict1 != expected1 - assert dict2 != expected1 - - merged = util.merge_dicts_no_overwrite(dict1, dict2) - assert expected2 == merged - assert dict1 != expected2 - assert dict2 != expected2 - - dict1_copy = deepcopy(dict1) - util.merge_dicts_in_place_overwrite(dict1_copy, dict2) - assert expected1 == dict1_copy - assert dict1 != dict1_copy - - dict1_copy = deepcopy(dict1) - util.merge_dicts_in_place_no_overwrite(dict1_copy, dict2) - assert expected2 == dict1_copy - assert dict1 != dict1_copy diff --git a/python/serve/cortex_internal/lib/type/__init__.py b/python/serve/cortex_internal/lib/type/__init__.py deleted file mode 100644 index 6829453d90..0000000000 --- a/python/serve/cortex_internal/lib/type/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from cortex_internal.lib.type.type import ( - PythonHandlerType, - TensorFlowHandlerType, - TensorFlowNeuronHandlerType, - HandlerType, - handler_type_from_string, - handler_type_from_api_spec, -) diff --git a/python/serve/cortex_internal/lib/type/type.py b/python/serve/cortex_internal/lib/type/type.py deleted file mode 100644 index 2aa8675957..0000000000 --- a/python/serve/cortex_internal/lib/type/type.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import collections - - -class HandlerType(collections.namedtuple("HandlerType", "type")): - def __str__(self) -> str: - return str(self.type) - - def __repr__(self) -> str: - return str(self.type) - - -PythonHandlerType = HandlerType("python") - -TensorFlowHandlerType = HandlerType("tensorflow") -TensorFlowNeuronHandlerType = HandlerType("tensorflow-neuron") - - -def handler_type_from_string(handler_type: str) -> HandlerType: - """ - Get handler type from string. - - Args: - handler_type: "python", "tensorflow" or "tensorflow-neuron" - - Raises: - ValueError if handler_type does not hold the right value. - """ - handler_types = [ - PythonHandlerType, - TensorFlowHandlerType, - TensorFlowNeuronHandlerType, - ] - for candidate in handler_types: - if str(candidate) == handler_type: - return candidate - raise ValueError("handler_type can only be 'python', 'tensorflow' or 'tensorflow-neuron'") - - -def handler_type_from_api_spec(api_spec: dict) -> HandlerType: - """ - Get handler type from API spec. - """ - if api_spec["compute"]["inf"] > 0 and api_spec["handler"]["type"] == str(TensorFlowHandlerType): - return handler_type_from_string("tensorflow-neuron") - return handler_type_from_string(api_spec["handler"]["type"]) diff --git a/python/serve/cortex_internal/lib/util.py b/python/serve/cortex_internal/lib/util.py deleted file mode 100644 index b043cd56fa..0000000000 --- a/python/serve/cortex_internal/lib/util.py +++ /dev/null @@ -1,370 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import collections -import os -import pathlib -import shutil -import zipfile -from copy import deepcopy -from typing import List, Dict, Optional - - -def has_method(object, method: str): - return callable(getattr(object, method, None)) - - -def expand_environment_vars_on_file(in_file: str, out_file: Optional[str] = None): - if out_file is None: - out_file = in_file - with open(in_file, "r") as f: - data = f.read() - with open(out_file, "w") as f: - f.write(os.path.expandvars(data)) - - -def extract_zip(zip_path, dest_dir=None, delete_zip_file=False): - if dest_dir is None: - dest_dir = os.path.dirname(zip_path) - - zip_ref = zipfile.ZipFile(zip_path, "r") - zip_ref.extractall(dest_dir) - zip_ref.close() - - if delete_zip_file: - rm_file(zip_path) - - -def mkdir_p(dir_path): - pathlib.Path(dir_path).mkdir(parents=True, exist_ok=True) - - -def rm_dir(dir_path): - if os.path.isdir(dir_path): - shutil.rmtree(dir_path) - return True - return False - - -def rm_file(path): - if os.path.isfile(path): - os.remove(path) - return True - return False - - -def trim_prefix(string, prefix): - if string.startswith(prefix): - return string[len(prefix) :] - return string - - -def ensure_prefix(string, prefix): - if string.startswith(prefix): - return string - return prefix + string - - -def trim_suffix(string, suffix): - if string.endswith(suffix): - return string[: -len(suffix)] - return string - - -def ensure_suffix(string, suffix): - if string.endswith(suffix): - return string - return string + suffix - - -def get_paths_with_prefix(paths: List[str], prefix: str) -> List[str]: - return list(filter(lambda path: path.startswith(prefix), paths)) - - -def get_paths_by_prefixes(paths: List[str], prefixes: List[str]) -> Dict[str, List[str]]: - paths_by_prefix = {} - for path in paths: - for prefix in prefixes: - if not path.startswith(prefix): - continue - if prefix not in paths_by_prefix: - paths_by_prefix[prefix] = [path] - else: - paths_by_prefix[prefix].append(path) - return paths_by_prefix - - -def get_leftmost_part_of_path(path: str) -> str: - """ - Gets the leftmost part of a path. - - If a path looks like - models/tensorflow/iris/15559399 - - Then this function will return - models - """ - if path == "." or path == "./": - return "." - return pathlib.PurePath(path).parts[0] - - -def remove_non_empty_directory_paths(paths: List[str]) -> List[str]: - """ - Eliminates dir paths from the tree that are not empty. - - If paths looks like: - models/tensorflow/ - models/tensorflow/iris/1569001258 - models/tensorflow/iris/1569001258/saved_model.pb - - Then after calling this function, it will look like: - models/tensorflow/iris/1569001258/saved_model.pb - """ - - leading_slash_paths_mask = [path.startswith("/") for path in paths] - all_paths_start_with_leading_slash = all(leading_slash_paths_mask) - some_paths_start_with_leading_slash = any(leading_slash_paths_mask) - - if not all_paths_start_with_leading_slash and some_paths_start_with_leading_slash: - raise ValueError("can only either pass in absolute paths or relative paths") - - path_map = {} - split_paths = [list(filter(lambda x: x != "", path.split("/"))) for path in paths] - - for split_path in split_paths: - composed_path = "" - split_path_length = len(split_path) - for depth, path_level in enumerate(split_path): - if composed_path != "": - composed_path += "/" - composed_path += path_level - if composed_path not in path_map: - path_map[composed_path] = 1 - if depth < split_path_length - 1: - path_map[composed_path] += 1 - else: - path_map[composed_path] += 1 - - file_paths = [] - for file_path, appearances in path_map.items(): - if appearances == 1: - file_paths.append(all_paths_start_with_leading_slash * "/" + file_path) - - return file_paths - - -def merge_dicts_in_place_overwrite(*dicts): - """Merge dicts, right into left, with overwriting. First dict is updated in place""" - dicts = list(dicts) - target = dicts.pop(0) - for d in dicts: - merge_two_dicts_in_place_overwrite(target, d) - return target - - -def merge_dicts_in_place_no_overwrite(*dicts): - """Merge dicts, right into left, without overwriting. First dict is updated in place""" - dicts = list(dicts) - target = dicts.pop(0) - for d in dicts: - merge_two_dicts_in_place_no_overwrite(target, d) - return target - - -def merge_dicts_overwrite(*dicts): - """Merge dicts, right into left, with overwriting. A new dict is created, original ones not modified.""" - result = {} - for d in dicts: - result = merge_two_dicts_overwrite(result, d) - return result - - -def merge_dicts_no_overwrite(*dicts): - """Merge dicts, right into left, without overwriting. A new dict is created, original ones not modified.""" - result = {} - for d in dicts: - result = merge_two_dicts_no_overwrite(result, d) - return result - - -def merge_two_dicts_in_place_overwrite(x, y): - """Merge y into x, with overwriting. x is updated in place""" - if x is None: - x = {} - - if y is None: - y = {} - - for k, v in y.items(): - if k in x and isinstance(x[k], dict) and isinstance(y[k], collections.Mapping): - merge_dicts_in_place_overwrite(x[k], y[k]) - else: - x[k] = y[k] - return x - - -def merge_two_dicts_in_place_no_overwrite(x, y): - """Merge y into x, without overwriting. x is updated in place""" - for k, v in y.items(): - if k in x and isinstance(x[k], dict) and isinstance(y[k], collections.Mapping): - merge_dicts_in_place_no_overwrite(x[k], y[k]) - else: - if k not in x: - x[k] = y[k] - return x - - -def merge_two_dicts_overwrite(x, y): - """Merge y into x, with overwriting. A new dict is created, original ones not modified.""" - x = deepcopy(x) - return merge_dicts_in_place_overwrite(x, y) - - -def merge_two_dicts_no_overwrite(x, y): - """Merge y into x, without overwriting. A new dict is created, original ones not modified.""" - y = deepcopy(y) - return merge_dicts_in_place_overwrite(y, x) - - -def is_bool(var): - return isinstance(var, bool) - - -def is_float(var): - return isinstance(var, float) - - -def is_int(var): - return isinstance(var, int) and not isinstance(var, bool) - - -def is_str(var): - return isinstance(var, str) - - -def is_dict(var): - return isinstance(var, dict) - - -def is_list(var): - return isinstance(var, list) - - -def is_tuple(var): - return isinstance(var, tuple) - - -def is_float_or_int(var): - return is_int(var) or is_float(var) - - -def is_int_list(var): - if not is_list(var): - return False - for item in var: - if not is_int(item): - return False - return True - - -def is_float_list(var): - if not is_list(var): - return False - for item in var: - if not is_float(item): - return False - return True - - -def is_str_list(var): - if not is_list(var): - return False - for item in var: - if not is_str(item): - return False - return True - - -def is_bool_list(var): - if not is_list(var): - return False - for item in var: - if not is_bool(item): - return False - return True - - -def is_float_or_int_list(var): - if not is_list(var): - return False - for item in var: - if not is_float_or_int(item): - return False - return True - - -def and_list_with_quotes(values: List) -> str: - """ - Converts a list like ["a", "b", "c"] to '"a", "b" and "c"'". - """ - string = "" - - if len(values) == 1: - string = '"' + values[0] + '"' - elif len(values) > 1: - for val in values[:-2]: - string += '"' + val + '", ' - string += '"' + values[-2] + '" and "' + values[-1] + '"' - - return string - - -def or_list_with_quotes(values: List) -> str: - """ - Converts a list like ["a", "b", "c"] to '"a", "b" or "c"'. - """ - string = "" - - if len(values) == 1: - string = '"' + values[0] + '"' - elif len(values) > 1: - for val in values[:-2]: - string += '"' + val + '", ' - string += '"' + values[-2] + '" or "' + values[-1] + '"' - - return string - - -def string_plural_with_s(string: str, count: int) -> str: - """ - Pluralize the word with an "s" character if the count is greater than 1. - """ - if count > 1: - string += "s" - return string - - -def render_jinja_template(jinja_template_file: str, context: dict) -> str: - from jinja2 import Environment, FileSystemLoader - - template_path = pathlib.Path(jinja_template_file) - - env = Environment(loader=FileSystemLoader(str(template_path.parent))) - env.trim_blocks = True - env.lstrip_blocks = True - env.rstrip_blocks = True - - template = env.get_template(str(template_path.name)) - return template.render(**context) diff --git a/python/serve/cortex_internal/serve/__init__.py b/python/serve/cortex_internal/serve/__init__.py deleted file mode 100644 index dcd1d9ae2f..0000000000 --- a/python/serve/cortex_internal/serve/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/python/serve/cortex_internal/serve/serve.py b/python/serve/cortex_internal/serve/serve.py deleted file mode 100644 index 51fcf0454b..0000000000 --- a/python/serve/cortex_internal/serve/serve.py +++ /dev/null @@ -1,361 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -import inspect -import json -import os -import re -import signal -import sys -import threading -import time -import uuid -from concurrent.futures import ThreadPoolExecutor -from typing import Any, Dict - -import datadog -from asgiref.sync import async_to_sync -from fastapi import FastAPI -from fastapi.exceptions import RequestValidationError -from starlette.exceptions import HTTPException as StarletteHTTPException -from starlette.requests import Request -from starlette.responses import JSONResponse, PlainTextResponse, Response - -from cortex_internal.lib import util -from cortex_internal.lib.api import DynamicBatcher, RealtimeAPI -from cortex_internal.lib.concurrency import FileLock, LockedFile -from cortex_internal.lib.exceptions import UserException, UserRuntimeException -from cortex_internal.lib.log import configure_logger -from cortex_internal.lib.metrics import MetricsClient -from cortex_internal.lib.telemetry import capture_exception, get_default_tags, init_sentry - -init_sentry(tags=get_default_tags()) -logger = configure_logger("cortex", os.environ["CORTEX_LOG_CONFIG_FILE"]) - -NANOSECONDS_IN_SECOND = 1e9 - - -request_thread_pool = ThreadPoolExecutor(max_workers=int(os.environ["CORTEX_THREADS_PER_PROCESS"])) -loop = asyncio.get_event_loop() -loop.set_default_executor(request_thread_pool) - -app = FastAPI() - -local_cache: Dict[str, Any] = { - "api": None, - "handler_impl": None, - "dynamic_batcher": None, - "api_route": None, - "client": None, -} - - -@app.on_event("startup") -def startup(): - open(f"/mnt/workspace/proc-{os.getpid()}-ready.txt", "a").close() - - -@app.on_event("shutdown") -def shutdown(): - try: - os.remove("/mnt/workspace/api_readiness.txt") - except FileNotFoundError: - pass - - try: - os.remove(f"/mnt/workspace/proc-{os.getpid()}-ready.txt") - except FileNotFoundError: - pass - - -def is_allowed_request(request): - return ( - request.url.path == local_cache["api_route"] - and request.method.lower() in local_cache["handle_fn_args"] - ) - - -@app.exception_handler(StarletteHTTPException) -async def http_exception_handler(request, e): - response = Response(content=str(e.detail), status_code=e.status_code) - return response - - -@app.exception_handler(RequestValidationError) -async def validation_exception_handler(request, e): - response = Response(content=str(e), status_code=400) - return response - - -@app.exception_handler(Exception) -async def uncaught_exception_handler(request, e): - response = Response(content="internal server error", status_code=500) - return response - - -@app.middleware("http") -async def register_request(request: Request, call_next): - request.state.start_time = time.time() - - file_id = None - response = None - try: - if is_allowed_request(request): - if "x-request-id" in request.headers: - request_id = request.headers["x-request-id"] - else: - request_id = uuid.uuid1() - file_id = f"/mnt/requests/{request_id}" - open(file_id, "a").close() - - response = await call_next(request) - finally: - if file_id is not None: - try: - os.remove(file_id) - except FileNotFoundError: - pass - - if is_allowed_request(request): - status_code = 500 - if response is not None: - status_code = response.status_code - api: RealtimeAPI = local_cache["api"] - api.metrics.post_request_metrics(status_code, time.time() - request.state.start_time) - - return response - - -@app.middleware("http") -async def parse_payload(request: Request, call_next): - if not is_allowed_request(request): - return await call_next(request) - - verb = request.method.lower() - if ( - verb in local_cache["handle_fn_args"] - and "payload" not in local_cache["handle_fn_args"][verb] - ): - return await call_next(request) - - content_type = request.headers.get("content-type", "").lower() - - if content_type.startswith("text/plain"): - try: - charset = "utf-8" - matches = re.findall(r"charset=(\S+)", content_type) - if len(matches) > 0: - charset = matches[-1].rstrip(";") - body = await request.body() - request.state.payload = body.decode(charset) - except Exception as e: - return PlainTextResponse(content=str(e), status_code=400) - elif content_type.startswith("multipart/form") or content_type.startswith( - "application/x-www-form-urlencoded" - ): - try: - request.state.payload = await request.form() - except Exception as e: - return PlainTextResponse(content=str(e), status_code=400) - elif content_type.startswith("application/json"): - try: - request.state.payload = await request.json() - except json.JSONDecodeError as e: - return JSONResponse(content={"error": str(e)}, status_code=400) - else: - request.state.payload = await request.body() - - return await call_next(request) - - -def handle(request: Request): - if async_to_sync(request.is_disconnected)(): - return Response(status_code=499, content="disconnected client") - - verb = request.method.lower() - handle_fn_args = local_cache["handle_fn_args"] - if verb not in handle_fn_args: - return Response(status_code=405, content="method not implemented") - - handler_impl = local_cache["handler_impl"] - dynamic_batcher = None - if verb == "post": - dynamic_batcher: DynamicBatcher = local_cache["dynamic_batcher"] - kwargs = build_handler_kwargs(request) - - if dynamic_batcher: - result = dynamic_batcher.process(**kwargs) - else: - result = getattr(handler_impl, f"handle_{verb}")(**kwargs) - - callback = None - if isinstance(result, tuple) and len(result) == 2 and callable(result[1]): - callback = result[1] - result = result[0] - - if isinstance(result, bytes): - response = Response(content=result, media_type="application/octet-stream") - elif isinstance(result, str): - response = Response(content=result, media_type="text/plain") - elif isinstance(result, Response): - response = result - else: - try: - json_string = json.dumps(result) - except Exception as e: - raise UserRuntimeException( - str(e), - "please return an object that is JSON serializable (including its nested fields), a bytes object, " - "a string, or a `starlette.response.Response` object", - ) from e - response = Response(content=json_string, media_type="application/json") - - if callback is not None: - request_thread_pool.submit(callback) - - return response - - -def build_handler_kwargs(request: Request): - kwargs = {} - verb = request.method.lower() - - if "payload" in local_cache["handle_fn_args"][verb]: - kwargs["payload"] = request.state.payload - if "headers" in local_cache["handle_fn_args"][verb]: - kwargs["headers"] = request.headers - if "query_params" in local_cache["handle_fn_args"][verb]: - kwargs["query_params"] = request.query_params - - return kwargs - - -def get_summary(): - response = {} - - if hasattr(local_cache["client"], "metadata"): - client = local_cache["client"] - response = { - "model_metadata": client.metadata, - } - - return response - - -# this exists so that the user's __init__() can be executed by the request thread pool, which helps -# to avoid errors that occur when the user's __init__() function must be called by the same thread -# which executes handle_() methods. This only avoids errors if threads_per_worker == 1 -def start(): - future = request_thread_pool.submit(start_fn) - return future.result() - - -def start_fn(): - project_dir = os.environ["CORTEX_PROJECT_DIR"] - spec_path = os.environ["CORTEX_API_SPEC"] - model_dir = os.getenv("CORTEX_MODEL_DIR") - host_ip = os.environ["HOST_IP"] - tf_serving_port = os.getenv("CORTEX_TF_BASE_SERVING_PORT", "9000") - tf_serving_host = os.getenv("CORTEX_TF_SERVING_HOST", "localhost") - - try: - has_multiple_servers = os.getenv("CORTEX_MULTIPLE_TF_SERVERS") - if has_multiple_servers: - with LockedFile("/run/used_ports.json", "r+") as f: - used_ports = json.load(f) - for port in used_ports.keys(): - if not used_ports[port]: - tf_serving_port = port - used_ports[port] = True - break - f.seek(0) - json.dump(used_ports, f) - f.truncate() - - datadog.initialize(statsd_host=host_ip, statsd_port=9125) - statsd_client = datadog.statsd - - with open(spec_path) as json_file: - api_spec = json.load(json_file) - api = RealtimeAPI(api_spec, statsd_client, model_dir) - - client = api.initialize_client( - tf_serving_host=tf_serving_host, tf_serving_port=tf_serving_port - ) - - with FileLock("/run/init_stagger.lock"): - logger.info("loading the handler from {}".format(api.path)) - handler_impl = api.initialize_impl( - project_dir=project_dir, client=client, metrics_client=MetricsClient(statsd_client) - ) - - # crons only stop if an unhandled exception occurs - def check_if_crons_have_failed(): - while True: - for cron in api.crons: - if not cron.is_alive(): - os.kill(os.getpid(), signal.SIGQUIT) - time.sleep(1) - - threading.Thread(target=check_if_crons_have_failed, daemon=True).start() - - local_cache["api"] = api - local_cache["client"] = client - local_cache["handler_impl"] = handler_impl - - local_cache["handle_fn_args"] = {} - for verb in ["post", "get", "put", "patch", "delete"]: - if util.has_method(handler_impl, f"handle_{verb}"): - local_cache["handle_fn_args"][verb] = inspect.getfullargspec( - getattr(handler_impl, f"handle_{verb}") - ).args - if len(local_cache["handle_fn_args"]) == 0: - raise UserException( - "no user-defined `handle_` method found in handler class; define at least one verb handler (`handle_post`, `handle_get`, `handle_put`, `handle_patch`, `handle_delete`)" - ) - - if api.python_server_side_batching_enabled: - dynamic_batching_config = api.api_spec["handler"]["server_side_batching"] - - if "post" in local_cache["handle_fn_args"]: - local_cache["dynamic_batcher"] = DynamicBatcher( - handler_impl, - method_name=f"handle_post", - max_batch_size=dynamic_batching_config["max_batch_size"], - batch_interval=dynamic_batching_config["batch_interval"] - / NANOSECONDS_IN_SECOND, # convert nanoseconds to seconds - ) - else: - raise UserException( - "dynamic batcher has been enabled, but no `handle_post` method could be found in the `Handler` class" - ) - - local_cache["api_route"] = "/" - local_cache["info_route"] = "/info" - - except Exception as err: - if not isinstance(err, UserRuntimeException): - capture_exception(err) - logger.exception("failed to start api") - sys.exit(1) - - app.add_api_route( - local_cache["api_route"], - handle, - methods=[verb.upper() for verb in local_cache["handle_fn_args"]], - ) - app.add_api_route(local_cache["info_route"], get_summary, methods=["GET"]) - - return app diff --git a/python/serve/cortex_internal/serve/wsgi.py b/python/serve/cortex_internal/serve/wsgi.py deleted file mode 100644 index 0d5fde4737..0000000000 --- a/python/serve/cortex_internal/serve/wsgi.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from cortex_internal.serve.serve import start - -app = start() diff --git a/python/serve/init/bootloader.sh b/python/serve/init/bootloader.sh deleted file mode 100755 index a391333290..0000000000 --- a/python/serve/init/bootloader.sh +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/with-contenv bash - -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -# CORTEX_VERSION -export EXPECTED_CORTEX_VERSION=master - -if [ "$CORTEX_VERSION" != "$EXPECTED_CORTEX_VERSION" ]; then - echo "error: your Cortex operator version ($CORTEX_VERSION) doesn't match your handler image version ($EXPECTED_CORTEX_VERSION); please update your handler image by modifying the \`image\` field in your API configuration file (e.g. cortex.yaml) and re-running \`cortex deploy\`, or update your cluster by following the instructions at https://docs.cortex.dev/" - exit 1 -fi - -export CORTEX_DEBUGGING=${CORTEX_DEBUGGING:-"true"} - -eval $(/opt/conda/envs/env/bin/python /src/cortex/serve/init/export_env_vars.py $CORTEX_API_SPEC) - -function substitute_env_vars() { - file_to_run_substitution=$1 - /opt/conda/envs/env/bin/python -c "from cortex_internal.lib import util; import os; util.expand_environment_vars_on_file('$file_to_run_substitution')" -} - -# configure log level for python scripts§ -substitute_env_vars $CORTEX_LOG_CONFIG_FILE - -mkdir -p /mnt/workspace -mkdir -p /mnt/requests - -cd /mnt/project - -# if the container restarted, ensure that it is not perceived as ready -rm -rf /mnt/workspace/api_readiness.txt -rm -rf /mnt/workspace/init_script_run.txt -rm -rf /mnt/workspace/proc-*-ready.txt - -if [ "$CORTEX_KIND" == "RealtimeAPI" ]; then - sysctl -w net.core.somaxconn="65535" >/dev/null - sysctl -w net.ipv4.ip_local_port_range="15000 64000" >/dev/null - sysctl -w net.ipv4.tcp_fin_timeout=30 >/dev/null -fi - -# to export user-specified environment files -source_env_file_cmd="if [ -f \"/mnt/project/.env\" ]; then set -a; source /mnt/project/.env; set +a; fi" - -function install_deps() { - eval $source_env_file_cmd - - # execute script if present in project's directory - if [ -f "/mnt/project/${CORTEX_DEPENDENCIES_SHELL}" ]; then - bash -e "/mnt/project/${CORTEX_DEPENDENCIES_SHELL}" - fi - - # install from conda-packages.txt - if [ -f "/mnt/project/${CORTEX_DEPENDENCIES_CONDA}" ]; then - # look for packages in defaults and then conda-forge to improve chances of finding the package (specifically for python reinstalls) - conda config --append channels conda-forge - conda install -y --file "/mnt/project/${CORTEX_DEPENDENCIES_CONDA}" - fi - - # install pip packages - if [ -f "/mnt/project/${CORTEX_DEPENDENCIES_PIP}" ]; then - pip --no-cache-dir install -r "/mnt/project/${CORTEX_DEPENDENCIES_PIP}" - fi - - # install core cortex dependencies if required - /usr/local/cortex/install-core-dependencies.sh -} - -# install user dependencies -if [ "$CORTEX_LOG_LEVEL" = "DEBUG" ] || [ "$CORTEX_LOG_LEVEL" = "INFO" ]; then - install_deps -# if log level is set to warning/error -else - # buffer install_deps stdout/stderr to a file - tempf=$(mktemp) - set +e - ( - set -e - install_deps - ) > $tempf 2>&1 - set -e - - # if there was an error while running install_deps - # print the stdout/stderr and exit - exit_code=$? - if [ $exit_code -ne 0 ]; then - cat $tempf - exit $exit_code - fi - rm $tempf -fi - -# only terminate pod if this process exits with non-zero exit code -create_s6_service() { - export SERVICE_NAME=$1 - export COMMAND_TO_RUN=$2 - - dest_dir="/etc/services.d/$SERVICE_NAME" - mkdir $dest_dir - - dest_script="$dest_dir/run" - cp /src/cortex/serve/init/templates/run $dest_script - substitute_env_vars $dest_script - chmod +x $dest_script - - dest_script="$dest_dir/finish" - cp /src/cortex/serve/init/templates/finish $dest_script - substitute_env_vars $dest_script - chmod +x $dest_script - - unset SERVICE_NAME - unset COMMAND_TO_RUN -} - -# only terminate pod if this process exits with non-zero exit code -create_s6_service_from_file() { - export SERVICE_NAME=$1 - runnable=$2 - - dest_dir="/etc/services.d/$SERVICE_NAME" - mkdir $dest_dir - - cp $runnable $dest_dir/run - chmod +x $dest_dir/run - - dest_script="$dest_dir/finish" - cp /src/cortex/serve/init/templates/finish $dest_script - substitute_env_vars $dest_script - chmod +x $dest_script - - unset SERVICE_NAME -} - -# prepare webserver -if [ "$CORTEX_KIND" = "RealtimeAPI" ]; then - if [ $CORTEX_SERVING_PROTOCOL = "http" ]; then - mkdir /run/servers - fi - - if [ $CORTEX_SERVING_PROTOCOL = "grpc" ]; then - /opt/conda/envs/env/bin/python -m grpc_tools.protoc --proto_path=$CORTEX_PROJECT_DIR --python_out=$CORTEX_PYTHON_PATH --grpc_python_out=$CORTEX_PYTHON_PATH $CORTEX_PROTOBUF_FILE - fi - - # prepare servers - for i in $(seq 1 $CORTEX_PROCESSES_PER_REPLICA); do - # prepare uvicorn workers - if [ $CORTEX_SERVING_PROTOCOL = "http" ]; then - create_s6_service "uvicorn-$((i-1))" "cd /mnt/project && $source_env_file_cmd && PYTHONUNBUFFERED=TRUE PYTHONPATH=$PYTHONPATH:$CORTEX_PYTHON_PATH exec /opt/conda/envs/env/bin/python /src/cortex/serve/start/server.py /run/servers/proc-$((i-1)).sock" - fi - - # prepare grpc workers - if [ $CORTEX_SERVING_PROTOCOL = "grpc" ]; then - create_s6_service "grpc-$((i-1))" "cd /mnt/project && $source_env_file_cmd && PYTHONUNBUFFERED=TRUE PYTHONPATH=$PYTHONPATH:$CORTEX_PYTHON_PATH exec /opt/conda/envs/env/bin/python /src/cortex/serve/start/server_grpc.py localhost:$((i-1+20000))" - fi - done - - # generate nginx conf - /opt/conda/envs/env/bin/python -c 'from cortex_internal.lib import util; import os; generated = util.render_jinja_template("/src/cortex/serve/nginx.conf.j2", os.environ); print(generated);' > /run/nginx.conf - - create_s6_service "py_init" "cd /mnt/project && exec /opt/conda/envs/env/bin/python /src/cortex/serve/init/script.py" - create_s6_service "nginx" "exec nginx -c /run/nginx.conf" - create_s6_service_from_file "api_readiness" "/src/cortex/serve/poll/readiness.sh" - -elif [ "$CORTEX_KIND" = "BatchAPI" ]; then - start_cmd="/opt/conda/envs/env/bin/python /src/cortex/serve/start/batch.py" - if [ -f "/mnt/kubexit" ]; then - start_cmd="/mnt/kubexit ${start_cmd}" - fi - - create_s6_service "py_init" "cd /mnt/project && exec /opt/conda/envs/env/bin/python /src/cortex/serve/init/script.py" - create_s6_service "batch" "cd /mnt/project && $source_env_file_cmd && PYTHONUNBUFFERED=TRUE PYTHONPATH=$PYTHONPATH:$CORTEX_PYTHON_PATH exec ${start_cmd}" -elif [ "$CORTEX_KIND" = "AsyncAPI" ]; then - create_s6_service "py_init" "cd /mnt/project && exec /opt/conda/envs/env/bin/python /src/cortex/serve/init/script.py" - create_s6_service "async" "cd /mnt/project && $source_env_file_cmd && PYTHONUNBUFFERED=TRUE PYTHONPATH=$PYTHONPATH:$CORTEX_PYTHON_PATH exec /opt/conda/envs/env/bin/python /src/cortex/serve/start/async_api.py" -elif [ "$CORTEX_KIND" = "TaskAPI" ]; then - start_cmd="/opt/conda/envs/env/bin/python /src/cortex/serve/start/task.py" - if [ -f "/mnt/kubexit" ]; then - start_cmd="/mnt/kubexit ${start_cmd}" - fi - - create_s6_service "task" "cd /mnt/project && $source_env_file_cmd && PYTHONUNBUFFERED=TRUE PYTHONPATH=$PYTHONPATH:$CORTEX_PYTHON_PATH exec ${start_cmd}" -fi diff --git a/python/serve/init/export_env_vars.py b/python/serve/init/export_env_vars.py deleted file mode 100644 index adfeeb380d..0000000000 --- a/python/serve/init/export_env_vars.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import sys -from pathlib import Path - -NEURON_CORES_PER_INF = 4 - - -def extract_from_handler(api_kind: str, handler_config: dict, compute_config: dict) -> dict: - handler_type = handler_config["type"].lower() - - env_vars = { - "CORTEX_LOG_LEVEL": handler_config["log_level"].upper(), - "CORTEX_PROCESSES_PER_REPLICA": handler_config["processes_per_replica"], - "CORTEX_THREADS_PER_PROCESS": handler_config["threads_per_process"], - "CORTEX_DEPENDENCIES_PIP": handler_config["dependencies"]["pip"], - "CORTEX_DEPENDENCIES_CONDA": handler_config["dependencies"]["conda"], - "CORTEX_DEPENDENCIES_SHELL": handler_config["dependencies"]["shell"], - } - - if handler_config.get("python_path") is not None: - env_vars["CORTEX_PYTHON_PATH"] = os.path.normpath( - os.path.join("/mnt", "project", handler_config["python_path"]) - ) - - if api_kind == "RealtimeAPI": - if handler_config.get("protobuf_path") is not None: - env_vars["CORTEX_SERVING_PROTOCOL"] = "grpc" - env_vars["CORTEX_PROTOBUF_FILE"] = os.path.join( - "/mnt", "project", handler_config["protobuf_path"] - ) - else: - env_vars["CORTEX_SERVING_PROTOCOL"] = "http" - - if handler_type == "tensorflow": - env_vars["CORTEX_TF_BASE_SERVING_PORT"] = "9000" - env_vars["CORTEX_TF_SERVING_HOST"] = "localhost" - - if compute_config.get("inf", 0) > 0: - if handler_type == "python": - env_vars["NEURON_RTD_ADDRESS"] = "unix:/sock/neuron.sock" - env_vars["NEURONCORE_GROUP_SIZES"] = int( - compute_config["inf"] - * NEURON_CORES_PER_INF - / handler_config.get("processes_per_replica", 1) - ) - - if handler_type == "tensorflow": - env_vars["CORTEX_MULTIPLE_TF_SERVERS"] = "yes" - env_vars["CORTEX_ACTIVE_NEURON"] = "yes" - - return env_vars - - -def extract_from_task_definition(definition_config: dict, compute_config: dict) -> dict: - env_vars = { - "CORTEX_LOG_LEVEL": definition_config["log_level"].upper(), - "CORTEX_DEPENDENCIES_PIP": definition_config["dependencies"]["pip"], - "CORTEX_DEPENDENCIES_CONDA": definition_config["dependencies"]["conda"], - "CORTEX_DEPENDENCIES_SHELL": definition_config["dependencies"]["shell"], - } - - if definition_config.get("python_path") is not None: - env_vars["CORTEX_PYTHON_PATH"] = os.path.normpath( - os.path.join("/mnt", "project", definition_config["python_path"]) - ) - - if compute_config.get("inf", 0) > 0: - env_vars["NEURON_RTD_ADDRESS"] = "unix:/sock/neuron.sock" - env_vars["NEURONCORE_GROUP_SIZES"] = int(compute_config["inf"] * NEURON_CORES_PER_INF) - - return env_vars - - -def extract_from_autoscaling(autoscaling_config: dict): - return {"CORTEX_MAX_REPLICA_CONCURRENCY": autoscaling_config["max_replica_concurrency"]} - - -def set_env_vars_for_s6(env_vars: dict): - s6_env_base = "/var/run/s6/container_environment" - - Path(s6_env_base).mkdir(parents=True, exist_ok=True) - - for k, v in env_vars.items(): - if v is not None: - Path(f"{s6_env_base}/{k}").write_text(str(v)) - - -def print_env_var_exports(env_vars: dict): - for k, v in env_vars.items(): - if v is not None: - print(f"export {k}='{v}'") - - -def main(api_spec_path: str): - with open(api_spec_path, "r") as f: - api_config = json.load(f) - - api_kind = api_config["kind"] - env_vars = { - "CORTEX_SERVING_PORT": 8888, - "CORTEX_CACHE_DIR": "/mnt/cache", - "CORTEX_PROJECT_DIR": "/mnt/project", - "CORTEX_CLI_CONFIG_DIR": "/mnt/client", - "CORTEX_MODEL_DIR": "/mnt/model", - "CORTEX_LOG_CONFIG_FILE": "/src/cortex/serve/log_config.yaml", - "CORTEX_PYTHON_PATH": "/mnt/project", - "HOST_IP": os.environ.get("HOST_IP", "localhost"), - "CORTEX_KIND": api_kind, - } - - handler_config = api_config.get("handler", None) - compute_config = api_config.get("compute", None) - definition_config = api_config.get("definition", None) - autoscaling_config = api_config.get("autoscaling", None) - - if handler_config is not None: - env_vars.update(extract_from_handler(api_kind, handler_config, compute_config)) - - if definition_config is not None: - env_vars.update(extract_from_task_definition(definition_config, compute_config)) - - if autoscaling_config is not None: - env_vars.update(extract_from_autoscaling(autoscaling_config)) - - set_env_vars_for_s6(env_vars) - print_env_var_exports(env_vars) - - -if __name__ == "__main__": - main(sys.argv[1]) diff --git a/python/serve/init/install-core-dependencies.sh b/python/serve/init/install-core-dependencies.sh deleted file mode 100644 index 6cf89a0b89..0000000000 --- a/python/serve/init/install-core-dependencies.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -function module_exists() { - module_name=$1 - python -c "import importlib.util, sys; loader = importlib.util.find_spec('$module_name'); sys.exit(1) if loader is None else sys.exit(0)" -} - -if ! module_exists "cortex_internal"; then - pip install --no-cache-dir -U \ - -r /src/cortex/serve/serve.requirements.txt \ - /src/cortex/serve/ -fi - -if [ "${CORTEX_IMAGE_TYPE}" = "tensorflow-handler" ]; then - if ! module_exists "tensorflow" || ! module_exists "tensorflow_serving"; then - pip install --no-cache-dir -U \ - tensorflow-cpu==2.3.0 \ - tensorflow-serving-api==2.3.0 - fi -fi diff --git a/python/serve/init/script.py b/python/serve/init/script.py deleted file mode 100644 index f3c0f2a227..0000000000 --- a/python/serve/init/script.py +++ /dev/null @@ -1,193 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import sys -import time - -from cortex_internal.lib.log import configure_logger -from cortex_internal.lib.telemetry import get_default_tags, init_sentry - -init_sentry(tags=get_default_tags()) -logger = configure_logger("cortex", os.environ["CORTEX_LOG_CONFIG_FILE"]) - -from cortex_internal.lib.type import ( - handler_type_from_api_spec, - PythonHandlerType, - TensorFlowHandlerType, - TensorFlowNeuronHandlerType, -) -from cortex_internal.lib.model import ( - FileBasedModelsTreeUpdater, # only when num workers > 1 - TFSModelLoader, -) -from cortex_internal.lib.checkers.pod import wait_neuron_rtd - - -def prepare_tfs_servers_api(api_spec: dict, model_dir: str) -> TFSModelLoader: - # get TFS address-specific details - tf_serving_host = os.getenv("CORTEX_TF_SERVING_HOST", "localhost") - tf_base_serving_port = int(os.getenv("CORTEX_TF_BASE_SERVING_PORT", "9000")) - - # determine if multiple TF processes are required - num_processes = 1 - has_multiple_tf_servers = os.getenv("CORTEX_MULTIPLE_TF_SERVERS") - if has_multiple_tf_servers: - num_processes = int(os.environ["CORTEX_PROCESSES_PER_REPLICA"]) - - # initialize models for each TF process - addresses = [] - for w in range(int(num_processes)): - addresses.append(f"{tf_serving_host}:{tf_base_serving_port+w}") - - if len(addresses) == 1: - return TFSModelLoader( - interval=10, - api_spec=api_spec, - address=addresses[0], - tfs_model_dir=model_dir, - download_dir=model_dir, - ) - return TFSModelLoader( - interval=10, - api_spec=api_spec, - addresses=addresses, - tfs_model_dir=model_dir, - download_dir=model_dir, - ) - - -def are_models_specified(api_spec: dict) -> bool: - """ - Checks if models have been specified in the API spec (cortex.yaml). - - Args: - api_spec: API configuration. - """ - handler_type = handler_type_from_api_spec(api_spec) - - if handler_type == PythonHandlerType and api_spec["handler"]["multi_model_reloading"]: - models = api_spec["handler"]["multi_model_reloading"] - elif handler_type != PythonHandlerType: - models = api_spec["handler"]["models"] - else: - return False - - return models is not None - - -def is_model_caching_enabled(api_spec: dir) -> bool: - handler_type = handler_type_from_api_spec(api_spec) - - if handler_type == PythonHandlerType and api_spec["handler"]["multi_model_reloading"]: - models = api_spec["handler"]["multi_model_reloading"] - elif handler_type != PythonHandlerType: - models = api_spec["handler"]["models"] - else: - return False - - return models and models["cache_size"] and models["disk_cache_size"] - - -def main(): - # wait until neuron-rtd sidecar is ready - uses_inferentia = os.getenv("CORTEX_ACTIVE_NEURON") - if uses_inferentia: - wait_neuron_rtd() - - # strictly for Inferentia - has_multiple_tf_servers = os.getenv("CORTEX_MULTIPLE_TF_SERVERS") - num_processes = int(os.environ["CORTEX_PROCESSES_PER_REPLICA"]) - if has_multiple_tf_servers: - base_serving_port = int(os.environ["CORTEX_TF_BASE_SERVING_PORT"]) - used_ports = {} - for w in range(int(num_processes)): - used_ports[str(base_serving_port + w)] = False - with open("/run/used_ports.json", "w+") as f: - json.dump(used_ports, f) - - # get API spec - spec_path = os.environ["CORTEX_API_SPEC"] - cache_dir = os.getenv("CORTEX_CACHE_DIR") - region = os.getenv("AWS_DEFAULT_REGION") # when it's deployed to AWS - - with open(spec_path) as json_file: - api_spec = json.load(json_file) - - handler_type = handler_type_from_api_spec(api_spec) - caching_enabled = is_model_caching_enabled(api_spec) - model_dir = os.getenv("CORTEX_MODEL_DIR") - - # start live-reloading when model caching not enabled > 1 - cron = None - if not caching_enabled: - # create cron dirs if they don't exist - os.makedirs("/run/cron", exist_ok=True) - os.makedirs("/tmp/cron", exist_ok=True) - - # prepare crons - if handler_type == PythonHandlerType and are_models_specified(api_spec): - cron = FileBasedModelsTreeUpdater( - interval=10, - api_spec=api_spec, - download_dir=model_dir, - ) - cron.start() - elif handler_type == TensorFlowHandlerType: - tf_serving_port = os.getenv("CORTEX_TF_BASE_SERVING_PORT", "9000") - tf_serving_host = os.getenv("CORTEX_TF_SERVING_HOST", "localhost") - cron = TFSModelLoader( - interval=10, - api_spec=api_spec, - address=f"{tf_serving_host}:{tf_serving_port}", - tfs_model_dir=model_dir, - download_dir=model_dir, - ) - cron.start() - elif handler_type == TensorFlowNeuronHandlerType: - cron = prepare_tfs_servers_api(api_spec, model_dir) - cron.start() - - # wait until the cron finishes its first pass - if cron: - while cron.is_alive() and not cron.ran_once(): - time.sleep(0.25) - - # disable live reloading when the BatchAPI kind is used - # disable live reloading for the TF type when Inferentia is used and when multiple processes are used (num procs > 1) - if api_spec["kind"] != "RealtimeAPI" or ( - handler_type == TensorFlowNeuronHandlerType - and has_multiple_tf_servers - and num_processes > 1 - ): - cron.stop() - - # to syncronize with the other serving processes - open("/mnt/workspace/init_script_run.txt", "a").close() - - # don't exit the script if the cron is running - while cron and cron.is_alive(): - time.sleep(0.25) - - # exit if cron has exited with errors - if cron and isinstance(cron.exitcode, int) and cron.exitcode != 0: - # if it was killed by a catchable signal - if cron.exitcode < 0: - sys.exit(-cron.exitcode) - sys.exit(cron.exitcode) - - -if __name__ == "__main__": - main() diff --git a/python/serve/init/templates/finish b/python/serve/init/templates/finish deleted file mode 100644 index c19935292b..0000000000 --- a/python/serve/init/templates/finish +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/execlineb -S2 - -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# good pages to read about s6-overlay -# https://wiki.gentoo.org/wiki/S6#Process_supervision -# https://skarnet.org/software/s6/s6-svscanctl.html -# http://skarnet.org/software/s6/s6-svc.html -# http://skarnet.org/software/s6/servicedir.html - -# good pages to read about execline -# http://www.troubleshooters.com/linux/execline.htm -# https://danyspin97.org/blog/getting-started-with-execline-scripting/ -# https://wiki.tcl-lang.org/page/execline - -define exit_code ${1} -define signal ${2} -define sigterm 15 - -# when process receives a non-catchable signal (i.e. KILL/9) -# s6 sets the exit code to >= 256 and expects the user to inspect the signal value instead -ifelse { s6-test ${exit_code} -gt 255 } { - if -n { s6-test ${signal} -eq ${sigterm} } - backtick -n new_exit_code { s6-expr 128 + ${signal} } - importas -ui new_exit_code new_exit_code - foreground { s6-echo "[finish-manager] service ${SERVICE_NAME} received signal value ${signal}, exiting with ${new_exit_code} exit code" } - foreground { redirfd -w 1 /var/run/s6/env-stage3/S6_STAGE2_EXITED s6-echo -n -- ${new_exit_code} } - s6-svscanctl -t /var/run/s6/services -} - -# if we receive an exit code between 0 and 255, then exit accordingly with the given value -ifelse { s6-test ${exit_code} -ne 0 } { - foreground { s6-echo "[finish-manager] service ${SERVICE_NAME} exiting with exit code ${exit_code}" } - foreground { redirfd -w 1 /var/run/s6/env-stage3/S6_STAGE2_EXITED s6-echo -n -- ${exit_code} } - s6-svscanctl -t /var/run/s6/services -} - -# otherwise stop the service -if { s6-test ${exit_code} -eq 0 } -foreground { s6-echo "[finish-manager] service ${SERVICE_NAME} exiting with exit code 0" } -foreground { s6-svc -O /var/run/s6/services/${SERVICE_NAME} } -foreground { s6-rmrf /etc/services.d/${SERVICE_NAME} } - -# stop the supervisor when all services have stopped successfully -pipeline { s6-ls /etc/services.d/ } -backtick -n NUM_RUNNING_SERVICES { wc -l } -importas -ui NUM_RUNNING_SERVICES NUM_RUNNING_SERVICES -if { s6-test ${NUM_RUNNING_SERVICES} -eq 0 } -foreground { s6-echo "[finish-manager] all container services have finished; stopping supervisor" } -s6-svscanctl -h /var/run/s6/services diff --git a/python/serve/init/templates/run b/python/serve/init/templates/run deleted file mode 100644 index 04440defb6..0000000000 --- a/python/serve/init/templates/run +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/with-contenv bash - -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -${COMMAND_TO_RUN} diff --git a/python/serve/log_config.yaml b/python/serve/log_config.yaml deleted file mode 100644 index d3abc5b6b1..0000000000 --- a/python/serve/log_config.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# https://github.com/encode/uvicorn/pull/515/files -# https://github.com/encode/uvicorn/issues/511 -# https://github.com/encode/uvicorn/issues/491 - -version: 1 -disable_existing_loggers: False -formatters: - default: - "()": pythonjsonlogger.jsonlogger.JsonFormatter - format: "%(asctime)s %(levelname)s %(message)s %(process)d" - uvicorn_access: - "()": cortex_internal.lib.log.CortexAccessFormatter - format: "%(asctime)s %(levelname)s %(message)s %(process)d" -handlers: - default: - formatter: default - class: logging.StreamHandler - stream: ext://sys.stdout - uvicorn_access: - formatter: uvicorn_access - class: logging.StreamHandler - stream: ext://sys.stdout -loggers: - cortex: - level: $CORTEX_LOG_LEVEL - handlers: - - default - propagate: no - uvicorn: - level: $CORTEX_LOG_LEVEL - handlers: - - uvicorn_access - propagate: no - uvicorn.error: - level: $CORTEX_LOG_LEVEL - handlers: - - uvicorn_access - propagate: no - uvicorn.access: - level: $CORTEX_LOG_LEVEL - handlers: - - uvicorn_access - propagate: no diff --git a/python/serve/nginx.conf.j2 b/python/serve/nginx.conf.j2 deleted file mode 100644 index 1f1d689055..0000000000 --- a/python/serve/nginx.conf.j2 +++ /dev/null @@ -1,169 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# good articles to read -# https://hub.packtpub.com/fine-tune-nginx-configufine-tune-nginx-configurationfine-tune-nginx-configurationratio/ -# https://www.nginx.com/blog/tuning-nginx/ -# https://www.digitalocean.com/community/tutorials/understanding-nginx-http-proxying-load-balancing-buffering-and-caching -# https://serverfault.com/a/788703 -# https://stackoverflow.com/questions/59846238/guide-on-how-to-use-regex-in-nginx-location-block-section - -daemon off; -# maximum number of open files per worker -worker_rlimit_nofile 65535; -worker_processes 1; - -thread_pool pool threads={{ CORTEX_PROCESSES_PER_REPLICA | int }}; - -events { - # max num requests = (worker_processes * worker_connections ) / 2 for reverse proxy - # max num requests is also limited by the number of socket connections available on the system (~64k) - worker_connections 65535; - - # The multi_accept flag enables an NGINX worker to accept as many connections as possible when it - # gets the notification of a new connection. The purpose of this flag is to accept all connections - # in the listen queue at once. If the directive is disabled, a worker process will accept connections one by one. - multi_accept off; - - # An efficient method of processing connections available on Linux 2.6+. The method is similar to the FreeBSD kqueue. - use epoll; -} - -http { - # don't enforce a max body size - client_max_body_size 0; - - # send headers in one piece, it is better than sending them one by one - tcp_nopush on; - - # don't buffer data sent, good for small data bursts in real time - tcp_nodelay on; - - # to limit concurrent requests - limit_conn_zone 1 zone=inflights:1m; - - # to distribute load - aio threads=pool; - - upstream servers { - # load balancing policy - least_conn; - - {% for i in range(CORTEX_PROCESSES_PER_REPLICA | int) %} - {% if CORTEX_SERVING_PROTOCOL == 'http' %} - server unix:/run/servers/proc-{{ i }}.sock; - {% endif %} - {% if CORTEX_SERVING_PROTOCOL == 'grpc' %} - server localhost:{{ i + 20000 }}; - {% endif %} - {% endfor %} - } - - {% if CORTEX_SERVING_PROTOCOL == 'grpc' %} - server { - listen {{ CORTEX_SERVING_PORT | int }} http2; - default_type application/grpc; - underscores_in_headers on; - - grpc_read_timeout 3600s; - - location /nginx_status { - stub_status on; - allow 127.0.0.1; - deny all; - } - - location / { - limit_conn inflights {{ CORTEX_MAX_REPLICA_CONCURRENCY | int }}; - - grpc_set_header Upgrade $http_upgrade; - grpc_set_header Connection "Upgrade"; - grpc_set_header Connection keep-alive; - grpc_set_header Host $host:$server_port; - grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - grpc_set_header X-Forwarded-Proto $scheme; - - grpc_pass grpc://servers; - } - } - {% endif %} - - {% if CORTEX_SERVING_PROTOCOL == 'http' %} - server { - listen {{ CORTEX_SERVING_PORT | int }}; - underscores_in_headers on; - - # how much time an inference can take - proxy_read_timeout 3600s; - - location /nginx_status { - stub_status on; - allow 127.0.0.1; - deny all; - } - - location /info { - limit_conn inflights {{ CORTEX_MAX_REPLICA_CONCURRENCY | int }}; - - proxy_set_header 'HOST' $host; - proxy_set_header 'X-Forwarded-For' $proxy_add_x_forwarded_for; - proxy_set_header 'X-Real-IP' $remote_addr; - proxy_set_header 'X-Forwarded-Proto' $scheme; - - proxy_redirect off; - proxy_buffering off; - - proxy_pass http://servers; - } - - location / { - deny all; - } - - location = / { - # CORS (inspired by https://enable-cors.org/server_nginx.html) - - # for all CRUD methods - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'POST, GET, PUT, PATCH, DELETE, OPTIONS'; - add_header 'Access-Control-Allow-Headers' '*'; - add_header 'Access-Control-Allow-Credentials' 'true'; - add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; - - if ($request_method = 'OPTIONS') { - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'POST, GET, PUT, PATCH, DELETE, OPTIONS'; - add_header 'Access-Control-Allow-Headers' '*'; - add_header 'Access-Control-Allow-Credentials' 'true'; - add_header 'Access-Control-Max-Age' 1728000; - add_header 'Content-Type' 'text/plain; charset=utf-8'; - add_header 'Content-Length' 0; - return 204; - } - - limit_conn inflights {{ CORTEX_MAX_REPLICA_CONCURRENCY | int }}; - - proxy_set_header 'HOST' $host; - proxy_set_header 'X-Forwarded-For' $proxy_add_x_forwarded_for; - proxy_set_header 'X-Real-IP' $remote_addr; - proxy_set_header 'X-Forwarded-Proto' $scheme; - - proxy_redirect off; - proxy_buffering off; - - proxy_pass http://servers; - } - } - {% endif %} -} diff --git a/python/serve/poll/readiness.sh b/python/serve/poll/readiness.sh deleted file mode 100644 index 5f41121210..0000000000 --- a/python/serve/poll/readiness.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/with-contenv sh - -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -while true; do - procs_ready="$(ls /mnt/workspace/proc-*-ready.txt 2>/dev/null | wc -l)" - if [ "$CORTEX_PROCESSES_PER_REPLICA" = "$procs_ready" ] && curl --silent "localhost:$CORTEX_SERVING_PORT/nginx_status" --output /dev/null; then - touch /mnt/workspace/api_readiness.txt - break - fi - sleep 1 -done diff --git a/python/serve/serve.requirements.txt b/python/serve/serve.requirements.txt deleted file mode 100644 index eb80cbd963..0000000000 --- a/python/serve/serve.requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -grpcio==1.36.0 -grpcio-tools==1.36.0 -grpcio-reflection==1.36.0 -python-multipart==0.0.5 -requests==2.24.0 -uvicorn==0.11.8 -uvloop==0.14.0 -pyyaml==5.4 diff --git a/python/serve/setup.py b/python/serve/setup.py deleted file mode 100644 index e804a98740..0000000000 --- a/python/serve/setup.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pathlib - -import pkg_resources -from setuptools import setup, find_packages - -with pathlib.Path("cortex_internal.requirements.txt").open() as requirements_txt: - install_requires = [ - str(requirement) for requirement in pkg_resources.parse_requirements(requirements_txt) - ] - -setup( - name="cortex-internal", - version="master", # CORTEX_VERSION - description="Internal package for Cortex containers", - author="cortex.dev", - author_email="dev@cortex.dev", - license="Apache License 2.0", - url="https:/github.com/cortexlabs/cortex", - setup_requires=(["setuptools", "wheel"]), - packages=find_packages(), - install_requires=install_requires, - python_requires=">=3.6", - classifiers=[ - "Operating System :: MacOS", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.6", - "Intended Audience :: Developers", - ], - project_urls={ - "Bug Reports": "https://github.com/cortexlabs/cortex/issues", - "Chat with us": "https://gitter.im/cortexlabs/cortex", - "Documentation": "https://docs.cortex.dev", - "Source Code": "https://github.com/cortexlabs/cortex", - }, -) diff --git a/python/serve/start/async_api.py b/python/serve/start/async_api.py deleted file mode 100644 index 754b981408..0000000000 --- a/python/serve/start/async_api.py +++ /dev/null @@ -1,176 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import inspect -import json -import os -import pathlib -import sys -import time -from typing import Dict, Any - -import boto3 - -from cortex_internal.lib.api.async_api import AsyncAPI -from cortex_internal.lib.exceptions import UserRuntimeException -from cortex_internal.lib.log import configure_logger -from cortex_internal.lib.metrics import MetricsClient -from cortex_internal.lib.queue.sqs import SQSHandler -from cortex_internal.lib.storage import S3 -from cortex_internal.lib.telemetry import init_sentry, get_default_tags, capture_exception - -init_sentry(tags=get_default_tags()) -log = configure_logger("cortex", os.environ["CORTEX_LOG_CONFIG_FILE"]) - -SQS_POLL_WAIT_TIME = 10 # seconds -MESSAGE_NOT_FOUND_SLEEP = 10 # seconds -INITIAL_MESSAGE_VISIBILITY = 30 # seconds -MESSAGE_RENEWAL_PERIOD = 15 # seconds -JOB_COMPLETE_MESSAGE_RENEWAL = 10 # seconds - -local_cache: Dict[str, Any] = { - "api": None, - "handler_impl": None, - "handle_async_fn_args": None, - "sqs_client": None, - "storage_client": None, -} - - -def handle_workload(message): - api: AsyncAPI = local_cache["api"] - handler_impl = local_cache["handler_impl"] - - request_id = message["Body"] - log.info(f"processing workload...", extra={"id": request_id}) - - api.update_status(request_id, "in_progress") - payload = api.get_payload(request_id) - - try: - result = handler_impl.handle_async(**build_handle_async_args(payload, request_id)) - except Exception as err: - raise UserRuntimeException from err - - log.debug("uploading result", extra={"id": request_id}) - api.upload_result(request_id, result) - - log.debug("updating status to completed", extra={"id": request_id}) - api.update_status(request_id, "completed") - - log.debug("deleting payload from s3") - api.delete_payload(request_id=request_id) - - log.info("workload processing complete", extra={"id": request_id}) - - -def handle_workload_failure(message): - api: AsyncAPI = local_cache["api"] - request_id = message["Body"] - - log.error("failed to process workload", exc_info=True, extra={"id": request_id}) - api.update_status(request_id, "failed") - - log.debug("deleting payload from s3") - api.delete_payload(request_id=request_id) - - -def build_handle_async_args(payload, request_id): - args = {} - if "payload" in local_cache["handle_async_fn_args"]: - args["payload"] = payload - if "request_id" in local_cache["handle_async_fn_args"]: - args["request_id"] = request_id - return args - - -def main(): - while not pathlib.Path("/mnt/workspace/init_script_run.txt").is_file(): - time.sleep(0.2) - - model_dir = os.getenv("CORTEX_MODEL_DIR") - api_spec_path = os.environ["CORTEX_API_SPEC"] - workload_path = os.environ["CORTEX_ASYNC_WORKLOAD_PATH"] - project_dir = os.environ["CORTEX_PROJECT_DIR"] - readiness_file = os.getenv("CORTEX_READINESS_FILE", "/mnt/workspace/api_readiness.txt") - region = os.getenv("AWS_DEFAULT_REGION") - queue_url = os.environ["CORTEX_QUEUE_URL"] - statsd_host = os.getenv("HOST_IP") - statsd_port = os.getenv("CORTEX_STATSD_PORT", "9125") - tf_serving_host = os.getenv("CORTEX_TF_SERVING_HOST") - tf_serving_port = os.getenv("CORTEX_TF_BASE_SERVING_PORT") - - bucket, key = S3.deconstruct_s3_path(workload_path) - storage = S3(bucket=bucket, region=region) - - with open(api_spec_path) as json_file: - api_spec = json.load(json_file) - - sqs_client = boto3.client("sqs", region_name=region) - api = AsyncAPI( - api_spec=api_spec, - storage=storage, - storage_path=key, - statsd_host=statsd_host, - statsd_port=int(statsd_port), - model_dir=model_dir, - ) - - try: - log.info(f"loading the handler from {api.path}") - metrics_client = MetricsClient(api.statsd) - handler_impl = api.initialize_impl( - project_dir, - metrics_client, - tf_serving_host=tf_serving_host, - tf_serving_port=tf_serving_port, - ) - except UserRuntimeException as err: - err.wrap(f"failed to initialize handler implementation") - log.error(str(err), exc_info=True) - sys.exit(1) - except Exception as err: - capture_exception(err) - log.error(f"failed to initialize handler implementation", exc_info=True) - sys.exit(1) - - local_cache["api"] = api - local_cache["handler_impl"] = handler_impl - local_cache["sqs_client"] = sqs_client - local_cache["storage_client"] = storage - local_cache["handle_async_fn_args"] = inspect.getfullargspec(handler_impl.handle_async).args - - open(readiness_file, "a").close() - - log.info("polling for workloads...") - try: - sqs_handler = SQSHandler( - sqs_client=sqs_client, - queue_url=queue_url, - renewal_period=MESSAGE_RENEWAL_PERIOD, - visibility_timeout=INITIAL_MESSAGE_VISIBILITY, - not_found_sleep_time=MESSAGE_NOT_FOUND_SLEEP, - message_wait_time=SQS_POLL_WAIT_TIME, - ) - sqs_handler.start(message_fn=handle_workload, message_failure_fn=handle_workload_failure) - except UserRuntimeException as err: - log.error(str(err), exc_info=True) - sys.exit(1) - except Exception as err: - capture_exception(err) - log.error(str(err), exc_info=True) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/python/serve/start/batch.py b/python/serve/start/batch.py deleted file mode 100644 index f5a60d5e68..0000000000 --- a/python/serve/start/batch.py +++ /dev/null @@ -1,267 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import inspect -import json -import os -import pathlib -import sys -import threading -import time -import uuid -from typing import Dict, Any - -import boto3 -import datadog - -from cortex_internal.lib.api import BatchAPI -from cortex_internal.lib.concurrency import LockedFile -from cortex_internal.lib.exceptions import UserRuntimeException -from cortex_internal.lib.log import configure_logger -from cortex_internal.lib.metrics import MetricsClient -from cortex_internal.lib.queue.sqs import SQSHandler, get_total_messages_in_queue -from cortex_internal.lib.telemetry import get_default_tags, init_sentry, capture_exception - -init_sentry(tags=get_default_tags()) -log = configure_logger("cortex", os.environ["CORTEX_LOG_CONFIG_FILE"]) - -SQS_POLL_WAIT_TIME = 10 # seconds -MESSAGE_NOT_FOUND_SLEEP = 10 # seconds -INITIAL_MESSAGE_VISIBILITY = 30 # seconds -MESSAGE_RENEWAL_PERIOD = 15 # seconds -JOB_COMPLETE_MESSAGE_RENEWAL = 10 # seconds - -local_cache: Dict[str, Any] = { - "api": None, - "job_spec": None, - "handler_impl": None, - "sqs_client": None, -} - -receipt_handle_mutex = threading.Lock() -stop_renewal = set() - - -def dimensions(): - return [ - {"Name": "api_name", "Value": local_cache["api"].name}, - {"Name": "job_id", "Value": local_cache["job_spec"]["job_id"]}, - ] - - -def success_counter_metric(): - return { - "MetricName": "cortex_batch_succeeded", - "Dimensions": dimensions(), - "Unit": "Count", - "Value": 1, - } - - -def failed_counter_metric(): - return { - "MetricName": "cortex_batch_failed", - "Dimensions": dimensions(), - "Unit": "Count", - "Value": 1, - } - - -def time_per_batch_metric(total_time_seconds): - return { - "MetricName": "cortex_time_per_batch", - "Dimensions": dimensions(), - "Value": total_time_seconds, - } - - -def build_handle_batch_args(payload, batch_id): - args = {} - - if "payload" in local_cache["handle_batch_fn_args"]: - args["payload"] = payload - if "headers" in local_cache["handle_batch_fn_args"]: - args["headers"] = None - if "query_params" in local_cache["handle_batch_fn_args"]: - args["query_params"] = None - if "batch_id" in local_cache["handle_batch_fn_args"]: - args["batch_id"] = batch_id - return args - - -def handle_batch_message(message): - handler_impl = local_cache["handler_impl"] - api: BatchAPI = local_cache["api"] - - log.info(f"processing batch {message['MessageId']}") - - start_time = time.time() - payload = json.loads(message["Body"]) - batch_id = message["MessageId"] - - try: - handler_impl.handle_batch(**build_handle_batch_args(payload, batch_id)) - except Exception as err: - raise UserRuntimeException from err - - api.metrics.post_metrics( - [success_counter_metric(), time_per_batch_metric(time.time() - start_time)] - ) - - -def handle_batch_failure(message): - api: BatchAPI = local_cache["api"] - - api.metrics.post_metrics([failed_counter_metric()]) - log.exception(f"failed processing batch {message['MessageId']}") - - -def on_job_complete(message): - job_spec = local_cache["job_spec"] - handler_impl = local_cache["handler_impl"] - sqs_client = local_cache["sqs_client"] - queue_url = job_spec["sqs_url"] - - should_run_on_job_complete = False - - while True: - visible_messages, invisible_messages = get_total_messages_in_queue( - sqs_client=sqs_client, queue_url=queue_url - ) - total_messages = visible_messages + invisible_messages - if total_messages > 1: - new_message_id = uuid.uuid4() - time.sleep(JOB_COMPLETE_MESSAGE_RENEWAL) - sqs_client.send_message( - QueueUrl=queue_url, - MessageBody='"job_complete"', - MessageAttributes={ - "job_complete": {"StringValue": "true", "DataType": "String"}, - "api_name": {"StringValue": job_spec["api_name"], "DataType": "String"}, - "job_id": {"StringValue": job_spec["job_id"], "DataType": "String"}, - }, - MessageDeduplicationId=str(new_message_id), - MessageGroupId=str(new_message_id), - ) - break - else: - if should_run_on_job_complete: - if getattr(handler_impl, "on_job_complete", None): - log.info("executing on_job_complete") - try: - handler_impl.on_job_complete() - except Exception as err: - raise UserRuntimeException from err - break - should_run_on_job_complete = True - - time.sleep(10) # verify that the queue is empty one more time - - -def start(): - while not pathlib.Path("/mnt/workspace/init_script_run.txt").is_file(): - time.sleep(0.2) - - api_spec_path = os.environ["CORTEX_API_SPEC"] - job_spec_path = os.environ["CORTEX_JOB_SPEC"] - project_dir = os.environ["CORTEX_PROJECT_DIR"] - - model_dir = os.getenv("CORTEX_MODEL_DIR") - host_ip = os.environ["HOST_IP"] - tf_serving_port = os.getenv("CORTEX_TF_BASE_SERVING_PORT", "9000") - tf_serving_host = os.getenv("CORTEX_TF_SERVING_HOST", "localhost") - - region = os.getenv("AWS_DEFAULT_REGION") - - has_multiple_servers = os.getenv("CORTEX_MULTIPLE_TF_SERVERS") - if has_multiple_servers: - with LockedFile("/run/used_ports.json", "r+") as f: - used_ports = json.load(f) - for port in used_ports.keys(): - if not used_ports[port]: - tf_serving_port = port - used_ports[port] = True - break - f.seek(0) - json.dump(used_ports, f) - f.truncate() - - datadog.initialize(statsd_host=host_ip, statsd_port=9125) - statsd_client = datadog.statsd - - with open(api_spec_path) as json_file: - api_spec = json.load(json_file) - api = BatchAPI(api_spec, statsd_client, model_dir) - - with open(job_spec_path) as json_file: - job_spec = json.load(json_file) - - sqs_client = boto3.client("sqs", region_name=region) - - client = api.initialize_client(tf_serving_host=tf_serving_host, tf_serving_port=tf_serving_port) - - try: - log.info("loading the handler from {}".format(api.path)) - handler_impl = api.initialize_impl( - project_dir=project_dir, - client=client, - metrics_client=MetricsClient(statsd_client), - job_spec=job_spec, - ) - except UserRuntimeException as err: - err.wrap(f"failed to start job {job_spec['job_id']}") - log.error(str(err), exc_info=True) - sys.exit(1) - except Exception as err: - capture_exception(err) - log.error(f"failed to start job {job_spec['job_id']}", exc_info=True) - sys.exit(1) - - local_cache["api"] = api - local_cache["job_spec"] = job_spec - local_cache["handler_impl"] = handler_impl - local_cache["handle_batch_fn_args"] = inspect.getfullargspec(handler_impl.handle_batch).args - local_cache["sqs_client"] = sqs_client - - open("/mnt/workspace/api_readiness.txt", "a").close() - - log.info("polling for batches...") - try: - sqs_handler = SQSHandler( - sqs_client=sqs_client, - queue_url=job_spec["sqs_url"], - renewal_period=MESSAGE_RENEWAL_PERIOD, - visibility_timeout=INITIAL_MESSAGE_VISIBILITY, - not_found_sleep_time=MESSAGE_NOT_FOUND_SLEEP, - message_wait_time=SQS_POLL_WAIT_TIME, - dead_letter_queue_url=job_spec.get("sqs_dead_letter_queue"), - stop_if_no_messages=True, - ) - sqs_handler.start( - message_fn=handle_batch_message, - message_failure_fn=handle_batch_failure, - on_job_complete_fn=on_job_complete, - ) - except UserRuntimeException as err: - err.wrap(f"failed to run job {job_spec['job_id']}") - log.error(str(err), exc_info=True) - sys.exit(1) - except Exception as err: - capture_exception(err) - log.error(f"failed to run job {job_spec['job_id']}", exc_info=True) - sys.exit(1) - - -if __name__ == "__main__": - start() diff --git a/python/serve/start/server.py b/python/serve/start/server.py deleted file mode 100644 index c44aed5597..0000000000 --- a/python/serve/start/server.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import pathlib -import sys -import time - -import uvicorn -import yaml - -from cortex_internal.lib.telemetry import get_default_tags, init_sentry - - -def main(): - uds = sys.argv[1] - - with open(os.environ["CORTEX_LOG_CONFIG_FILE"], "r") as f: - log_config = yaml.load(f, yaml.FullLoader) - - while not pathlib.Path("/mnt/workspace/init_script_run.txt").is_file(): - time.sleep(0.2) - - uvicorn.run( - "cortex_internal.serve.wsgi:app", - uds=uds, - forwarded_allow_ips="*", - proxy_headers=True, - log_config=log_config, - ) - - -if __name__ == "__main__": - init_sentry(tags=get_default_tags()) - main() diff --git a/python/serve/start/server_grpc.py b/python/serve/start/server_grpc.py deleted file mode 100644 index cfa7fcbd01..0000000000 --- a/python/serve/start/server_grpc.py +++ /dev/null @@ -1,272 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import importlib -import inspect -import json -import os -import pathlib -import signal -import sys -import threading -import time -import traceback -import uuid -from concurrent import futures -from typing import Callable, Dict, Any, List - -import datadog -import grpc -from grpc_reflection.v1alpha import reflection - -from cortex_internal.lib.api import RealtimeAPI -from cortex_internal.lib.concurrency import FileLock, LockedFile -from cortex_internal.lib.exceptions import UserRuntimeException -from cortex_internal.lib.log import configure_logger -from cortex_internal.lib.metrics import MetricsClient -from cortex_internal.lib.telemetry import capture_exception, get_default_tags, init_sentry - -NANOSECONDS_IN_SECOND = 1e9 - - -class ThreadPoolExecutorWithRequestMonitor: - def __init__(self, post_latency_metrics_fn: Callable[[int, float], None], *args, **kwargs): - self._post_latency_metrics_fn = post_latency_metrics_fn - self._thread_pool_executor = futures.ThreadPoolExecutor(*args, **kwargs) - - def submit(self, fn, *args, **kwargs): - request_id = uuid.uuid1() - file_id = f"/mnt/requests/{request_id}" - open(file_id, "a").close() - - start_time = time.time() - - def wrapper_fn(*args, **kwargs): - try: - result = fn(*args, **kwargs) - except: - raise - finally: - try: - os.remove(file_id) - except FileNotFoundError: - pass - self._post_latency_metrics_fn(time.time() - start_time) - - return result - - self._thread_pool_executor.submit(wrapper_fn, *args, **kwargs) - - def map(self, *args, **kwargs): - return self._thread_pool_executor.map(*args, **kwargs) - - def shutdown(self, *args, **kwargs): - return self._thread_pool_executor.shutdown(*args, **kwargs) - - -def get_service_name_from_module(module_proto_pb2_grpc) -> Any: - classes = inspect.getmembers(module_proto_pb2_grpc, inspect.isclass) - for class_name, _ in classes: - if class_name.endswith("Servicer"): - return class_name[: -len("Servicer")] - # this line will never be reached because we're guaranteed to have one servicer class in the module - - -def get_servicer_from_module(module_proto_pb2_grpc) -> Any: - classes = inspect.getmembers(module_proto_pb2_grpc, inspect.isclass) - for class_name, module_class in classes: - if class_name.endswith("Servicer"): - return module_class - # this line will never be reached because we're guaranteed to have one servicer class in the module - - -def get_servicer_to_server_from_module(module_proto_pb2_grpc) -> Any: - functions = inspect.getmembers(module_proto_pb2_grpc, inspect.isfunction) - for function_name, function in functions: - if function_name.endswith("_to_server"): - return function - # this line will never be reached because we're guaranteed to have one servicer adder in the module - - -def get_rpc_methods_from_servicer(servicer: Any) -> List[str]: - rpc_names = [] - for (rpc_name, rpc_method) in inspect.getmembers(servicer, inspect.isfunction): - rpc_names.append(rpc_name) - return rpc_names - - -def build_method_kwargs(method_fn_args, payload, context) -> Dict[str, Any]: - method_kwargs = {} - if "payload" in method_fn_args: - method_kwargs["payload"] = payload - if "context" in method_fn_args: - method_kwargs["context"] = context - return method_kwargs - - -def construct_handler_servicer_class(ServicerClass: Any, handler_impl: Any) -> Any: - class HandlerServicer(ServicerClass): - def __init__(self, handler_impl: Any, api: RealtimeAPI): - self.handler_impl = handler_impl - self.api = api - - rpc_names = get_rpc_methods_from_servicer(ServicerClass) - for rpc_name in rpc_names: - arg_spec = inspect.getfullargspec(getattr(handler_impl, rpc_name)).args - - def _rpc_method(self: HandlerServicer, payload, context): - try: - kwargs = build_method_kwargs(arg_spec, payload, context) - response = getattr(self.handler_impl, rpc_name)(**kwargs) - self.api.metrics.post_status_code_request_metrics(200) - except Exception: - logger.error(traceback.format_exc()) - self.api.metrics.post_status_code_request_metrics(500) - context.abort(grpc.StatusCode.INTERNAL, "internal server error") - return response - - setattr(HandlerServicer, rpc_name, _rpc_method) - - return HandlerServicer - - -def init(): - project_dir = os.environ["CORTEX_PROJECT_DIR"] - spec_path = os.environ["CORTEX_API_SPEC"] - - model_dir = os.getenv("CORTEX_MODEL_DIR") - cache_dir = os.getenv("CORTEX_CACHE_DIR") - region = os.getenv("AWS_DEFAULT_REGION") - - host_ip = os.environ["HOST_IP"] - tf_serving_port = os.getenv("CORTEX_TF_BASE_SERVING_PORT", "9000") - tf_serving_host = os.getenv("CORTEX_TF_SERVING_HOST", "localhost") - - has_multiple_servers = os.getenv("CORTEX_MULTIPLE_TF_SERVERS") - if has_multiple_servers: - with LockedFile("/run/used_ports.json", "r+") as f: - used_ports = json.load(f) - for port in used_ports.keys(): - if not used_ports[port]: - tf_serving_port = port - used_ports[port] = True - break - f.seek(0) - json.dump(used_ports, f) - f.truncate() - - datadog.initialize(statsd_host=host_ip, statsd_port=9125) - statsd_client = datadog.statsd - - with open(spec_path) as json_file: - api_spec = json.load(json_file) - api = RealtimeAPI(api_spec, statsd_client, model_dir) - - config: Dict[str, Any] = { - "api": None, - "client": None, - "handler_impl": None, - "module_proto_pb2_grpc": None, - } - - proto_without_ext = pathlib.Path(api.protobuf_path).stem - module_proto_pb2 = importlib.import_module(proto_without_ext + "_pb2") - module_proto_pb2_grpc = importlib.import_module(proto_without_ext + "_pb2_grpc") - - client = api.initialize_client(tf_serving_host=tf_serving_host, tf_serving_port=tf_serving_port) - - ServicerClass = get_servicer_from_module(module_proto_pb2_grpc) - rpc_names = get_rpc_methods_from_servicer(ServicerClass) - - with FileLock("/run/init_stagger.lock"): - logger.info("loading the handler from {}".format(api.path)) - handler_impl = api.initialize_impl( - project_dir=project_dir, - client=client, - metrics_client=MetricsClient(statsd_client), - proto_module_pb2=module_proto_pb2, - rpc_method_names=rpc_names, - ) - - # crons only stop if an unhandled exception occurs - def check_if_crons_have_failed(): - while True: - for cron in api.crons: - if not cron.is_alive(): - os.kill(os.getpid(), signal.SIGQUIT) - time.sleep(1) - - threading.Thread(target=check_if_crons_have_failed, daemon=True).start() - - HandlerServicer = construct_handler_servicer_class(ServicerClass, handler_impl) - - config["api"] = api - config["client"] = client - config["handler_impl"] = handler_impl - config["module_proto_pb2"] = module_proto_pb2 - config["module_proto_pb2_grpc"] = module_proto_pb2_grpc - config["handler_servicer"] = HandlerServicer - - return config - - -def main(): - address = sys.argv[1] - threads_per_process = int(os.environ["CORTEX_THREADS_PER_PROCESS"]) - - try: - config = init() - except Exception as err: - if not isinstance(err, UserRuntimeException): - capture_exception(err) - logger.exception("failed to start api") - sys.exit(1) - - module_proto_pb2 = config["module_proto_pb2"] - module_proto_pb2_grpc = config["module_proto_pb2_grpc"] - HandlerServicer = config["handler_servicer"] - - api: RealtimeAPI = config["api"] - handler_impl = config["handler_impl"] - - server = grpc.server( - ThreadPoolExecutorWithRequestMonitor( - post_latency_metrics_fn=api.metrics.post_latency_request_metrics, - max_workers=threads_per_process, - ), - options=[("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1)], - ) - - add_HandlerServicer_to_server = get_servicer_to_server_from_module(module_proto_pb2_grpc) - add_HandlerServicer_to_server(HandlerServicer(handler_impl, api), server) - - service_name = get_service_name_from_module(module_proto_pb2_grpc) - SERVICE_NAMES = ( - module_proto_pb2.DESCRIPTOR.services_by_name[service_name].full_name, - reflection.SERVICE_NAME, - ) - reflection.enable_server_reflection(SERVICE_NAMES, server) - - server.add_insecure_port(address) - server.start() - - time.sleep(5.0) - open(f"/mnt/workspace/proc-{os.getpid()}-ready.txt", "a").close() - server.wait_for_termination() - - -if __name__ == "__main__": - init_sentry(tags=get_default_tags()) - logger = configure_logger("cortex", os.environ["CORTEX_LOG_CONFIG_FILE"]) - main() diff --git a/python/serve/start/task.py b/python/serve/start/task.py deleted file mode 100644 index 2db0e4319c..0000000000 --- a/python/serve/start/task.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import sys -from copy import deepcopy - -from cortex_internal.lib import util -from cortex_internal.lib.api import TaskAPI -from cortex_internal.lib.log import configure_logger -from cortex_internal.lib.telemetry import get_default_tags, init_sentry - -init_sentry(tags=get_default_tags()) -logger = configure_logger("cortex", os.environ["CORTEX_LOG_CONFIG_FILE"]) - - -def start(): - project_dir = os.environ["CORTEX_PROJECT_DIR"] - api_spec_path = os.environ["CORTEX_API_SPEC"] - task_spec_path = os.environ["CORTEX_TASK_SPEC"] - - with open(api_spec_path) as json_file: - api_spec = json.load(json_file) - - with open(task_spec_path) as json_file: - task_spec = json.load(json_file) - - logger.info("loading the task definition from {}".format(api_spec["definition"]["path"])) - task_api = TaskAPI(api_spec) - - logger.info("executing the task definition from {}".format(api_spec["definition"]["path"])) - callable_fn = task_api.get_callable(project_dir) - - config = deepcopy(api_spec["definition"]["config"]) - if task_spec is not None and task_spec.get("config") is not None: - util.merge_dicts_in_place_overwrite(config, task_spec["config"]) - - try: - callable_fn(config) - except Exception as err: - logger.error(f"failed to run task {task_spec['job_id']}", exc_info=True) - sys.exit(1) - - -if __name__ == "__main__": - start() diff --git a/test/apis/realtime/image-classifier-resnet50/sample.json b/test/apis/realtime/image-classifier-resnet50/sample.json index 36c70ef46c..a767675ce4 100644 --- a/test/apis/realtime/image-classifier-resnet50/sample.json +++ b/test/apis/realtime/image-classifier-resnet50/sample.json @@ -1 +1 @@ -"{\"instances\" : [{\"b64\": \"/9j/4AAQSkZJRgABAQAASABIAAD/4QBYRXhpZgAATU0AKgAAAAgAAgESAAMAAAABAAEAAIdpAAQAAAABAAAAJgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAB8qADAAQAAAABAAAC0AAAAAD/7QA4UGhvdG9zaG9wIDMuMAA4QklNBAQAAAAAAAA4QklNBCUAAAAAABDUHYzZjwCyBOmACZjs+EJ+/8AAEQgC0AHyAwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMFBQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkqNDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/EAB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMRBAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/bAEMAAwMDAwMDBQMDBQgFBQUICggICAgKDQoKCgoKDRANDQ0NDQ0QEBAQEBAQEBMTExMTExYWFhYWGRkZGRkZGRkZGf/bAEMBBAQEBgYGCwYGCxoSDxIaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGv/dAAQAIP/aAAwDAQACEQMRAD8A/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA//Q/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAphkQHBPWs7Vr5LG2aZ2CADqelfM3iT4xi0vXtIiQAcHaCxz9elceKxsKHxHVh8JOt8J9VLIjZ2nOKfkV8i6V8b4knRL2TAPXcMY/Kvb/AA/4+0fVYg8UwZ26DPSs6GZUqul7F1sDVp6tHpdFU7W9huVzG24eo6VcrvTT1RxtWCiiimIKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAoopuD3oAdkVl32pR2ti94CPlBYA9wOuPwHFc/4v15NHsRzhp38gexbBB/Lj8a+aPiH8W4reF47YM7iR2WNTk7DwFbHQDnP1rzsZmFOgmnud2FwM6zVloeqal8Wb20AkhgjdWJBGGyp4ODz6Zrn5fjjqILeTbwELgc56/8AfX6V8Z6h4y1fW5XK+YkR9Rhc+u3/ABrn5by+d1zcyZHXIxn8hXzM85xDekrH0MMopJaxPvzSfjeJ5fL1KKNOM5QHqO3U8+le36Vrdhq8CTWkqy7gCdhyBkZxn2zX4/XOtX1vOWW4kVsjGCAK+hPg38aDp1/FpWqShI5mAB7474PGM4ruwWcT5kqzuu5x4zKYqPNTWp+itFZul6pbarbLc2Z3Ie46frjNaVfURkmro+daadmFFFFMQUUUUAFFFFABRRRQAUUUUAf/0f1SooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA4/xtDI/h68aIciNjnr2r8utQv531CRrmYxMWPJYknnoF7nH1xX6y6pBHPYzRydGUjn3Ffkx4306TSfEt3DJIUjikYOTgMQW4A47181nsWnGZ9Fkck+aJZj1CRiY7aJ5PlyzyyYzjuccL+Jq9p2q6pA3m2l2I3B4CMxH8h+dYlnkxCSVSsQ5CHvju3cn0/lW2keoIu52FujDqw+Zh6hBjj/eP4GvmozbZ9BOmtj3LwZ8aNS0t1steAZV4DKePxr6s8MePNH123VracSuwyQO31r85vsqy5gjk3Njuo499qrx+JzWhomuax4Vu0ubGYFMglfu5/4CTXq4XNa1F2lqjy8TllOqrx0Z+pCOHUMO9Pr53+HnxbsNZjWC/kw+Bwa99gv7S4UNFIG3dPxr6vDYunWjzQZ8zXw06UuWSLdFFFdRzhRRRQAUUUUAFFFFABRRRQAUUUUAFFFGaACikzmloAa7pGpdzgDqTXLa14ostNRkDZfkfQ/5IrD8ba69hauiHBfIX6pn+or5V8ZeOJrW22pIXuZR8q9SCTliR+tePmOZ+x9yO56eCy91tXsT/FD4jy3M32dJMTh2AABbGM44Hfnqa+ZWubi9uDNdF23MfmYDBPf5T0xV6WC5vZ5JxcebI3UyDGS3Tv07ZqKJ7QpJBe+bbmNgkgJyFbOMjPI/zmvjqs51ZOTZ9hRoRpQ5USh/Kb94+z0dfusPTjpWZe3kYXEcy7v9ruPqOP0FW5rKaydWeQz2sp271OQM8ZYdRn6cGubuLNhMd4G8NnrjnoQQeDms+SyNVqZdyp37ojkEHK9SOf5Vzhml068W4jJVkbPHGD6iupLRxn7m3HYjke1c5evBNHjvmrpysxTjc/TL4AePm8QeHbeEy5dBiRCAACOMhgO/v+dfT6uGAI71+SHwO1+5stYexkuxbFhuUkkKSp54APX0NfqV4V1JdT0uGYAkgAFuxPtnkfSvr8nxTnD2T6Hx+a4b2dTmXU6eiiivbPICiiigAooooAKKKKACiiigD//S/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigCC5iE0JQnFfmv+0J4fl07xS9zApWOUhg20s2f5Amv0vPSvl79ojwi2qaA2oR5DQDPy9ffHvXk5xQ56N10PVyiv7Ouk+p8RaFKZSGUDcvCknIGO/px+prrBHa3MwMjltzcsG+8fQcZ/wA+p48z0+5fTfOtlOZIuCc9MkKFX/PauwtdRt4Ll45kCBBtCgbpAmdqr14Z/wA+TniviIxaZ9pNX2N+NJL2V47MMtsh+YJ8qlvfux9yfoO9Wl0y2CmVNgGcEhdxOOvPP88UttN9uzPOSYrcfJbxjChv9oggMfY8Dqa0IbD7SRc34aTLAJFEOOO+TyRnjI6npXbGHMjkk7GGI7vT5vtGns6MDleAP0zXq/hv4p6lb+RFNKUkiK5DcZ2nIOPQ+1ctDopkcrdxxwq3IQ8yH6ryR/wIk+wqld6GmP3TrGQQV4BK/iD39KpRnT1g7GM1CppJH2L4T+KNnf7LW/YRyM4BJPGDx1/WvXbXULW8j82Bww27vwr80rXUr/S5fKmw4HI7AjvjPevYNA8e6lBEpilYRbSrZ7Dtn6GvVwudSh7tZXPIxOUp+9TPtqivDdL+J4ljTzhyuC3uB1x/Ot6++JWmQGRYX3so+THfJ6/lXsxzTDOPNznlPA1k+XlPVKK8Yt/ilar5T3KkCQHcPQr0H41txfEzRzaLM5+dskr6c/4VdPMMPP4ZEywVaO8T0yivMW+KGjZUxqSGIHX1qhf/ABU0y2nMYIKhSMj+/jj8Kc8ww8VdzQRwdZuyieu0V4Jc/F+1D7IFLDGfyOKzZvjI0pWRIsIeMZ6H/wCtXLLOsIvtm8crxD+yfRpIHBPWjINfM6/FG/vWhKpho95Y5659PTAqW1+JWsW0i+eodYwfYc5Az+FT/beH8ynlVc+k6QjNfPTfFe+a4K+ViIALx16/4V0Nj8VLctJ9tjKc/KB2Hv8AWrhnGFk7cxnLLa8Vex7KBiqOo6hbabavdXLbVQE1wl58R9Hghyr5YxhsDrls4H4da8N8b+P7i6hlW4YqnJRRz2AyfypYrNqNOD5HdlYfLqtSSUlZGZ8SviFDcRsYBtCOzxjOT8x6/QE8V81Sy3erXr3quxZWxgLgbu+Wz3PrV67nuNdvJWY7Vc5ZsYCrxhSzFRnn7q+tdlpuhTmBIuYLhMKjkAowPOGA6D3zXyM3Urz5p7s+upU4UIcqONWynnjYMuZAM/NwSAc9cDp7ipbyx+2Wkd9kTb8RS9x8vC7vRsYB9cda7e50OaH968DQujZUlSQrezDsagbTZpnOQrrMMSJ1U+uVPOPQg5FbwoeRLrLdHm8dkIYJbG53NbnBIccqp4PPp/LrXIalbPZ3DWty2UJIVsg5HY5Feo39lLHGEVpIymcB1J+XoMZ+8OOevFcNr1g1xBHKEQZG3IUEe3XGQR+NTUpWNYVNTz28MxbfuJA7/TiuPmkKuyKeSTj2rq7u3uY2KBc44+XpWDHbEy7nxjmudRsbPU6bwHdG38TWrruBzggduMEmv1C+DF/LcWjISHC4B24GPr3/ADr8r/D8LQ65bvuGwMDk4GB6/j2r9K/gzaTiU3UROw8Ntxlc9m/pXrZTJrEK3Y8TN4r2ep9Q0Ui7sfNyaWvsT5IKKKKACiiigAooooAKKKKAP//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigArm/FmkR6zodxZv1ZDg/4V0lNdd6MvqMVFSCnFxfUqEnGSkj8f/F2hS+HNcubMsEkaXIB5IAPUDux7ZrNtWkH+qbY0kgz/E7ueg49SfwAr6T/AGivBH9mXY1vywFcEs69c+nqa+dvDTQXN5bxheAS20nbt/vM30A/OvgsTRlCq0z9AwtZVaKmj0S3vLWxaHQ5ys05BeTcV2KiDJkdR1GegPYZPpWzavJf3X7p5pJJMFtmRJkj5Qo528dAMBRySSad4e8OWc1vPeXETAXZDyuW+fDEFRkA47bVAJ79a9M0v+z9PtfskSLawAliu3YxyernJJJ68kk9zXXSo3V5M5q1VR2V2c/a6HevIdyeUH6gEuxx/fbOAB2A789aZc6SY02i4YOpxmPA5/HNdW+rW8qljF/o4bHHG4kdgOWJ9P6VgXN9eSJhLTyEH3Y1KbiPQ9Pxxj8a1lGFvdOVTm3qcldaSxVvMJcDoXbJz9SBWXZXkukuAGDd/LOAduMHBXIIre1OPWGbJjVARyGdePbHeuD1a0ZFErSYxzsXJ5Hc9h9RXBVp9UdlN3VmdvbawJleS1PlvHnfHnOR2K9609N12OZgCPmJPH5153o199kuEMhyoBHI5I6j8Aenaqs0503UmhaQssjZXnpk7v8A9Vc3mVKNtD3KW8gnRuh24P6cn9awLi/lSXybT5s+/Q1TsUM9ivk/MXbj8OT+FaenaIrYuI2yAH3HPJOf6VnOHOKKUdytObhnSKFjgthsfn+FP8iWFkE55UYck/3uldlBFaQHeuCxJI47471z+oRG4eaViFVSBnHH/wCvrzU1KSUUy4TV7WOVUSQCUoCzb8H/AHTip4Z1Nx5bkKWclfQL1/QVfk8hbBVkz5s4Ue55Y/4Uy5tWEalX8oAFR77yeOPQCsFS6o3c11NbTtTtvLkkiZCisB1HDNyf8KfcXMUmYGbndycjB5yao+H7SEWRjH3gxY+vJ4HuaTUYoIvk4aSVhs/LJNdCbsYNLmNl7qKGJHj4XHGevPGapLcPLI6xEbR8xP8AeOM4/Ws77OLpke4yIyOcjHAOf1qzDdQSXQsLfDHPJ9Mdf0q4tt6ktJEmta6ukwiOMLJcnacP0QHA5xXmk+vRzFri/Ly7i2BnGQoyff0/DjioPF9/LNqsrQuFEQAweCQMgH8OTXnWoXi2phiacR7V643MQPuqB2GeSc8nsTSd5Ssb06aUbnsGmSSSItxdSLHGRjyiAx555Y4x9AK3YruRz5cdrgx9PMlXaR6ADJ5+grx3TtWluLtLSJpGj2/KOUwM8n5DuP4kn1Ir0drY3BEH2m3tQw/1cyI+Rj2yxJ9ya9CjHQ5q2j1OhXW2tI2W/KwAHOULTAD68kfyqI+J7KchRcx3IYk7RlHx9M9q5G802SHMGnOINgYnyZUDAdN2HC4H+6vHrXkWvapqKnypLiKeTJB2ljKdvcruBb6iutRdrr+v69TKNNSPoz+3NGv8wzE5OCBsJAx7jpz61gXvh/Tpo3Nmcbzltvzqx/2kJyD6EV8yNrF6wVDcSW/ozAx5/wCA9fx/OrVvql1A/wA7NvxjdnaxH1HUfnSkpdTWOHtrFnaeI/D89shkjkBzn7vLH/H8ea4J7dbaMgnB756/WtSTVJ4fmmlabfx8xz/kiueur0yEq/8AEOorhnG7OuN0rMvaO5l1OIICQrLkgDPXtX6l/BuK/ist0ixNGRgOOXHsSD2+lfm58P8ATln1qxVxyzMVXqXKgnGPUYr9TPhppj2OkRMzhgw3Kdmxh6qw45B9RXp5PC9e66Hh51P3LHqlFFFfWHyoUUUUAFFFFABRRRQAUUUUAf/U/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA83+J3hBPFvhqezUDzVUsp75HSvzh0zRprHxSumyr5cibotxUEljnJGRwBk5Jr9ZWUMpU9DXxD8evAw0vUW17TP3P2gYZunzH0Pb/wCvXz2d4e1q8fmfQZLi7N0X12POb/xvo2kXC6dYzbvJHGw/Jk9SXwcsxznsPWuevPGFvdsIJtszkjCCXaozjoGA/Mj6ZrzO1DbxBOxjRWB4xufHbkdOO/15NdJo+leJ9Vui5/dKAW3EAIvP3iT8zEA8KPxwK8SNSU9UfRewhBXZ041vUpZozY2kS2iZVXaRgm0dcAAPKxPXbhc9TXULq18BstoFVtvL9CPXAJ2gZ4yWPPSsx47LS2MWLjUrhFziMMQCP78uOT6IgAH6muJfFtwoWxs0sYF4KFtr8c/OcnH4kn8a6InPJJ7IsTm38wGSHa8g5JVdzfTHzY/AA1mTW7pCIleSRGPIZiu38+g9K1LWC581luctITk4c4YdOCeMenHPtTrmwulbcibQPlYpHgj+7yTyB34rOqnYSZwxRAmE+c9VOfTkr2zXUnRxrFxBeoMIzKzDqPlAOM/h+lXLbRYpmzcLhkOCQMZHXP41uSkROYoV2L1BH94Zz+vNeXVnym8feNG3uYNLtTASDtPy/jWnp2qCbCR8bwTjtnArza/mmBKseuR9R/8AWp2g6hcy35hGVVAQSeg61zKtJs1dFWbO81XUngkBhPrkY5ABxx+lJqF5K9qlsowZdvPf5jgViTKJpHM5wWT5eeucevYVrpIvmJLs3CMgDPP3ec/hVqV3YmySRjW7qdXs4JwzKu8/98L/APXq/NcRy3cUZbKld+PTOaclmtnJNkbnV2MfHODjI9qyrSOZdR+2yYC54HYg8H8qPhuh76o20uUsrgup3ALnA9c9/wADWet+txqEczfdQAjPoetYOtXpt7mNs4GMH3wcf04pLOVrmRJ8AbwFyOePvfn2qOdrYpQT1Z2mryzSWx8oHZIOo+nFcppV5FpP2m7vOJEUlWbrkgcV0lzdPFagDO1FLEfgAP5153rcAv4JZnk2JGhLhOCSeAg+tbueqMEtLHmWt3893qJjjcgznIUDcSc5UAe55yeABUR0hUjI1GF3zwsioRIT/sYzwO5OM9qZdaVeQOivCZbuddwWMnESMcBSw/iYDoOceldhpPhVrC3e41mMWkBA3BZGWSUsOR1DBR+v05r0KdLRMcqtkc/bamNIjWxtlmSCUnMsICuD6qzEkkd8rjsOma0P7WdZ49M1icTIqlw32hCxQ9CyNGdwPvyM1pS6Hf3iS6hYyqlo67VGI0TYv91eWIXuWbnrXOP4VttUVPmAgiG9jA2ckdxJwD14AyPYV1xgk9TGUkzvdP1Tw00iaV9vktcHPBUoB2wRkD2+7TtX8C2OrRC7s75Z8qSwOwMew3K0bkfUH8689/4Rm5uLOTTVaG/hGfLS5BhuIixzkSxg8Z/vDHeoYtI8W+Hbdbq8nlEFuxAZVacbCuWVigb046D8a3gu39fqZtdmZV14V1HTrloEI6FsR4lcD1OX3gfVMVjtaGzVpZJBKD0KqU/MN/StDVdZ8RWNmTfPb6jazcxMr7Jx7eW4Xkd8AE9a4aXV/td0oVTCwAypGD+NOdzopyfU6B4JWAdzkFQ351SSMySIHIALZ/AV1+nXUVpYxSSxrKOQc44H41mazZzG8NxbHdDtGwk889vY15dSerSOhHpXwaistb+LemQt8lpZW8i84wXZTnuB37mv1Y0WC3tYEhtSSrYIznIUDGTnnntmvzA/Zq0wf8JHqWsMq5tIQi7hn5pGx+PAxj3r9MvDVwJrfibzmJ+dlA5I498KOg/HHFfRZLFKnfqfK53L97yrojrKKKK908MKKKKACiiigAooooAKKKKAP//V/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAK8b+Nmiw6x4Ou0lG4hDt9jXslch42s3vdBuIkA+4SfpiuXGw5qE15HRhJ8taMvM/M3w5pVtBE1/rJZvJ4ZQpy/PyhfX8P5V0Fxr8d5GYEjZWX/V2sJB5HTzJCcE/wCyOB7VX8R2V7bWT2gDD5yAFbaCDy2cdP8ACvNrtDaQhbEhHJ+Z1Hf+6uece/U+1fBRqSi7H6AqaqK53hn1eaF4YpILEZBMSSh5WbOPmbDhRnk4Vjx8oHWrNtbLczeUZJ9WnQ/6qFGggAzjAbLSN/tO5rzBIbnUn+wm7MbjmZtwwB3XAIOAPTqeOgr6X+Hfhq2to47u2mVkyMtyXkx03N3Yf/Wx3r1MOubY5MTJU1dnR6J4T/0NDqlqLfI/1ceWVc9QW6tn1ySadfeH7axZltgie0WM7fqRn8K9MWa3iGRhAOuRgj3OOv5VzeszQGNWjQHPQriuzFKEaZ5FKrOU9TzEwyjdDJ8yqT+HuPp3FQG3eQHI6de45HWti6LS/v0BHPJPb61T89YSUbAyOCOf84r5mtFN3PZpt2OM1aJvJZh1Ukfh2NP8PKx8ySRQCxA9+oyao6vdCa58tMk87ucdOePwrpdKgNnaAznceCG7HP8ASuOK1OmWkR968ZkMZ+Xym2luvy4J/nTlnFtKNykEl8egz09ua57ULlpdQFuPus4BI5OScHPoKtIbi8iljAOYuFA9evHf1qkm3oTolqb32oTTWzZw0ikEf7QIPP1qtfI9vGFQFsKWH45IGPxpLVo474goQSVb6ZGD/wCPU/W3jVGlZsED5cHp3FW/MjrZHmt9ema4EUvEhJAHbJ6fqDXY6Y2YUAOVb5go65HX8K4XUvKub1rqFtz5JwO3A4H410miXJa3Dn5cJkL3z6VLWxq9jrr25RN6hs8bfXjI/WuUhs/tEjXtwB9nViFT+8x/ngUs140hECnLs3bng5q2NrQwo+T5QBKg9c54+projHqzmbsUkjaO5a4tTvu7gqMjkxqOMIF9up4rrLTSpLmNWuGfc5O0OFI/Uc+5zgdBz0qWJSHdPMDGeXcrxwBgD8+n09q6SwuBLsuCvlr0T+8wHoTyBXfRmc1S/Qzp/DEUtu1xf3DyTKQI0hXa68cBWGCPyrgPEWi3VjCUsZ3jveHEarnzQOPnAzggf3fvHrnmvclmhwGuQHC9IxyXJ7HP/wBYV5d4xb+1LecpMXbOEgsEyPlPCNJuTdz15Ar0lytaHPGcr2Z4fp+s6nsktLpEuIELYN0vkFW4GEZXRcnPYqT6VvweKYbRZ7LxNo84gVUJmhUncT0AAJB7DGeMHr1rkNe8B+I9de2lvYpWkhZjKGCyO5GNigoNg6c88VDDpniLSk/sy9jmSNidgh3s1u548wN8oJx2J55JxxVtNf1/X5G6SZJr7eF/EF3KtlfzQ26H5zDM2BIv8DwzAj5Rx8revXNefNZ6le3git5/tMY+6NzDGPUOFwPpmqOv6b4ntbhIDPIVA2RRxOWURk/fZ/VvQ5JPWruk2N/aWbwvM5dyCzbiQB6c1MnZXOinE0JGnhBgJOQe/tXo3hm0ku4Htb0gIvzKeB06/rwM1wGkaPPPqii5dnjjBJQcZOeM8c8V7Vodt5LF3G6V+igcIteZiLJnQ37p1PwKsZbHUNetHwcGE7RzySf5Dgmv0T8HuY9MjRsDPAAAHA47dq+KPhpo8lq2pXcW7N1NGuR97CDn8MnrX274TsWtNPiLDkqOTnP0yf6D8TX0uUp8iZ8jm81Ks3/Wx1tFFFe0eQFFFFABRRRQAUUUUAFFFFAH/9b9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigArN1gE6ZcADOUNaVMlQSRtGejDFTNXi0OLs0z88vGVlc/aZUAQRyk7lPcH615RqOjyGDChgwPynHIx0zzz9a+r/H3hwW95NEqfeywJxjB9civD7/7TZrsVRDg/eYjGB1I718BVpuM2mffYTEKUE0ct4N+E19rFyl/ffJDncMFhyPY4yP5V9P2OmQaPbpAIlBUDnuQOnP/ANavC9E8SalARFFfROwzgyPhf/QT/jXfQa9qzw7tQVSOzRnK/h616FCtCENFqceMjVqT956HT3WpurMqsR7Hkfga527upZPnBJ6npVmZopFE44DDr15rEu3Cjfnp+ePauLFVm9bhQppFGa5kQtkfK/A56Vi38rpGVjGCuCe9SzSmRiScZ/DNZ91cMVBJ+YHaccH/ACa8mc+Y9GMbHEXErJfgOdpkJBPp2Ga9EtHd9PRZOrDcc84H/wCuvOdTtZ7md2QYCr25OPX8D60nh7xLPcf8S28wOMehOKmETSeqOg0h5H8RtDL8+1xJgf3CwBJ/OvSYdMe1vLdI03ASukh9QjHj8M5rndDhgi8Sw3T8R3KyWjccBiMrz/wEYr3HStOWZPLbq+JQf9vaAx/EivZwGEVSNzzcXieRnid3pstlrkk3mfuWYoR/dPpz+B/GuU8U3TNB6ZHr6V6F46dbXU5geEnCHPTDL0P9Pwrz3UtNudQs/LY4ffvHsrZx+o/WvNxELVWkdlGV4qTOTsrYzwQmOTa27nHOQxH8q6jyYEt3uIyBGgCrjj5geSf89ax4YJIE2YP7pOe3OT/LpUP2nMUaxL8jMWAHPAJz+FZJO5pJ3NolVVJARuZCp9RkY/PrWjaQbcM5wEQkY7EjjPuaxrhJ2hDRrgF+P93j+dbTyC3eEEgAHzGHqccCuqN2kYSLVvIZBlvkXCrgjOMgA/jitV7hkCSxnduAC5+uOf8ACsCBWnMSFuZCWznso5PsOa0fMY+SsRG4jIHXqeP0NbRRjIuyTkxuZwWydiqM5dj16f5x6VesoNPhVVvz9ocEnltsYPZQq4zgfhWXEhXYCcKg69yT7e9UrmS6t5d6IT2T6nk9a6YTcdjJxvoegPf26lS3k2qAYCooU4zxjqf0Fc1repaS0DxmYIeeMAtyepDAj865yLzZiFQh5DycEkD1J/8Ar8V5/wCInmgcwxzF5Dn5t2FXngLxyffpW3tW9yYUlc89+IPmxyhhcquTu2vguw6A4AJHtnGfSuU8PsLkEzyZwdp962da0eW4AjjnSVi/ziMlsE+pbGD6muet7SfRrloG+bzeVx7d/pTc7xstzvp6aM9U064toPkiABAH1J7Zr0PSrOTyxc3TMo4bkYGR+PWvNvCemSzyCeToSG3YBP4civbfszqYY1UmMkZyOw7V5zjedhVqtlZHtXw40hp7aETzmNHbeRk7j+A/rX13aQQwQokK7VAGM8nH1r5y8C6c8t8hi2QqiAAsM7R32jp+J+g5NfScQ2xqASeOp619tgKajTsj4rFzcptj6KKK7zkCiiigAooooAKKKKACiiigD//X/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAPKfiDp0NzEZM4dOlfKHivTmZXEIChucjk5/Gvr7xrcIITgD5e9eBXNtBcS/v1LDnv/AEr5HMYp1pJH0mXTcaabPn/RPCV3dXzNcR7UXHTaCQfUY/8Ar16la6YumL5MUxkjxnnqB6HqDXXeVEkey1QKMcN7/WsKfadyP949RXmzXKen7RzepWkdVUiPkeg6evSs2SP7WV2dR07H/wCvVi4gkHMeRnv9arWl3HFNtvwSgP8AnHFcsnzOzNYqyujEu7G6hzuXjpz6ensRXPXSSRgrjGBnvyM17W9lp2p2vmwsA6jB75x2OK881pIU3RNwQMH29/pSxGF5FzJ6F0K/O7MoaXYx3Nv5pXJIK9DjB9a8k1G2GneLolRDsZwv4k//AFq+idDt/L0/KL0z+R615zrOird649xHzLBJEyDvtLAZ/AmiNPReY4z95o6uKwk/cQIuZYphIo9SQT+lfTFjZbY45gu1imcemR/SuTsvDcb6nBdYG2PqMZHzL/8AXr1CO2VIxH/dGPwr6vLMG6cW5HzuPxSnZI+cfFWmLPrUcc3zKcqdwyOexrnLnSzbIsMg5RcZHoOnP1Fe3a9pGZvOKkrnt2/ya4PxDCFjYxjDFe/9K8fGYVwlKTPSw+J5oxijwnUdkczxIu3JOfoxzUWm6W0ske/O1R2HQY5znvz+vtVi5KRXCMVIIyST04zzXe+HrMXarwAOO2R+NeZShzTsejUfLC5CmmgwRxui713O2TkjP3R+Vc7PDFLcOWj3IoHDcLx1JPU5xx7V3OtXttaZsrI7mA5GOWPcn1/lXCCK8lmlMrhjkBUXJA+uOv0reslB2iY0m5K7IFZpZGf5VVvvED+HsBWnE6uJGjXHXpycYx1p5sblgzyFE42/Pldv0Azz60FZLYeXbOX3/edRjP8Au9h7miMhyQIRDtHKhRk45JPHAqtLatesvnDG4846Adl+vrWgkiWq8KC/dmO7HsoFTRN55DsvA5Abj8hWikZNHPXljLCBb2rAxkcgZ+b6nso9PxrDuvCd5qakxys+7KkopX6jcckD3HJ9q9CnEPE0cSs7nuSefZafG91G224+RjjIPX8AKnnaY13PKn+HtpC6iNN+zqAensM+vfvW7aeEoY49pgQ5xuyM/wAxXp0ckbMFEeAPoD9eelWGtzjMakg9iarfW4nVZwlhoFnAxMW1JFz2B5+hrbjsZVnUyNnBHTp+lSzsUzwAyk85B/wqG3vvOuY7flixA4/+tSh8SIm21c+o/h5DboTLkyucHaoyAAOCT0Htk17WpyAa8t8D2XlqsTsWWJRhc8A9yQPy5r1OvusOrQR8hWd5MKKKK3MgooooAKKKKACiiigAooooA//Q/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKa5+U/0p1Nf7pHrQwPIvGkB+zs0ORg5OTzXkJaJE8x8jPYn9a9h8YoUh/dsx7c9PevEJpI8iEpsKnORXx2NdqzPpMGr00X5HMdqHDAHtz1z71zN0yZzL+Z4/Wr107Ngswb0A5rIlkAmUuCM9Aozn8zXmVp3dj0aUeo6GESMZfMDBeuWwB75I/lSXdudm5tkg7YGT+dOutesNLtztT7RNgEKCAB6lm+6oFeL+Ivj3oNlcyQxtDdzxn5vKVpEX6udqfln2pqMbcq1fkVzS3PdbKRIbc7pSoboCp25/L+tec+I5gbxRy3bjkfpXzndftPRxT7LiJSvIzCsikj3JO38hiuj0n4xeDvFUyQtcGGRgB8wxg1rXw9XkXuu3oKjUipN3PrLw9AF0yNV+YngZ9DU+l+E2u9Zhu3Xd5TgPnuByPzPFc94U1QwwLaTkbVI2kdCuOCD+Ne+aBGrW4mGCWPUdwK78Dho1Gr9DixWIlTu11Ny1tFjxgY4H51rEYFMjUdae5ABHoK+nSsj55u7MXUY1eMqehrxbxLhQcjgZB/CvXdRuQiNXh3izUFjZmBPPbqTntXh5tZxPWy6/MfP3iS/EFywzhcE57ACu48K6ysOmhwTGAPmY8du3px+NeY6+yWxn1G5OYUIK7j0HXHPB54r5v8AEnxR1R7trDSpCQn8QJ2qT1wP6mvCo4ac5e4e7Uqx5bM+1J9b0yS5Ms1/HFnKhWcA7h1ODk/nV7StX0GZALa7EjgckFd2fYEg498V+b0fjrxa12sFjI00ruMAKDlugA/wrpP+Fn+KtLums9bsofPhO1w0YRlI90wf1rteWVl7yVzk+uU/hufopE+mToxEm/Pdcbvw3Lj9TVG9u9Hj2xrJI5XtINwyP9wgV8Y2Xxe/drJfQSxsoBDwyFW+pz8rfQj8a9c8N/EfUvFkQV4vMQHIm3YbA6cco30IrCVGaTvE1hJN6M9be/lllyJMjsuCn6d66K0UvGC7Mm7nIAHH868vj1u1j+a4wpHLYUJU6+OLSBglpl29Sefzx1rzlLleh0yg2j2W1gtLL98AzOc43N/QVJK9tg7l2N3APLfXH8s15Gnji7aQbtoJHAyCR7nFbEHiKW9GHcIWHRVOfxOMfzrdVU9DB05J3OnS/EUhiBGOe3f8P8avxXSsDucrnuAST9BjAriWX5PO3YXH/LPPftzjn8atW+pRsnlCXjcQATnIHHP/AOupUmnqOUU0dVcp5illJB7bufzrK0G1kGsxyEkhTxsGD+tRtLbRqv2gk7uyOo/qa6Lw5brJqUY6AHIBJJA/CunDrmqRsc9V2gz6t8EbxbKGQru+bGPvH3PfH4CvRq4XwlZxwQiT5tzgdeM/hgH88/hXdV91S+FHyU9wooorQgKKKKACiiigAooooAKKKKAP/9H9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAqKVgqnNS1RvCQhpPYaPMvGE37rhTg8DsK+etUlMMjuCcLzkAnmvoDxax8n5evc9/oK8F1kGMSSK21hkgDknNfH5qrVHY+ky5+7Y8r1DXL2SVxZZ3DgK2efyBx+NQofFd+BGJYokyAxDHdj64/pUltaveXTLGzAk8vnHPsAea9d0pEjVWKscDBbA/XjivGoUFOWp7NSqoLRHlvijwrP8A8I0yzMzs+Nw5A54/zmvhPVrA2Wi3c7KN6yuv0IcqfyAGK/VbXtIj1TQrlLXhvLJXB6lecHqa/PnxxoYF5caPGnli93Tw9MOxP7xf94da9dxVBW6bnHCbrRfc+W9Y1PRbm10+LSjOZfs4a9aZEQC5LNlYdrNmMJtwzYYtngACqV3rdrNo9jpsFjDBc2sk0kl4hfz5vM27EfLbQsYX5QoBySSTVbWdIm0i+e2uFIwTjjH4VVt4TcSLBCuWbgDHNfUxrRac4bNfgeH7Np2e6P0G/Zh8ay+K9Pk8Ma45e7sFEsMjHl4iQMH1Kn9K++/CrNDD9mfPydPoTxX5P/BWZvCnxJ8MxEjN27wSgH+CRD1/ECv1k0oYKOo68V5uD5XNyhsb4xNQXMdyjAZx6VUuptqE5qVG+WsTVpmSAgd816knZHlxV2cPr2pPGvB5NeR6tDLqFyRuwqjn26/4V3mqt506555rH1S0WGylMY2syHn265z9ea8WvT57tnsUHyWSPir47619hthptp8rPmRgP7qD+XpXzHptqtrpM2oOvmsE3nPGSeTzXs3xavDqniy4UEyIuIx6Y6kD615RpEJkt59LkP7xASAehTtiudR5KN13VzumnzW8jzSHUJYrmO8G1pI3DgMoK/Kc4Kngj1HcVY1vXNR8Rarc61qjK1xdSNK+xFiQFyWIVEAVVGeFUAAcCnanpU+lXBEiHYeVbHY1TgilvJlgt1MjscADmvejVi4Xi9NzxnT967Wp6N4UtZtSsH/eiIFWUk8ZXByCfQ19ifBbwzBF4Tt5TFucRiU7x0LuSMjPPH5V8++EfD0sKw6JEP8ASZ1BkI5Ecf8AExHqei+pPtX6N+APCsei+G1S5hbdJ8zKMfKB90fUAV4Ln7VyUdm/wPUS9nFSZwkvhC3u5WkMeR1BGCgz6Z//AF1xureDdPthuhiecrnCkhFH5DNen63fxqzERkKp6ucDr0HTNcjJrGZvsu1W3Y3KcHj0AGa8ao4KVkd0JTtc8ourMRNssYVjY5zsO4D23HFZUV5fWcxWQtuHT5gT19icV7zfaakluWFkhBGdxHA9MDOa8p1p49PhkkW02c5O05J/76/pSdEuNZPQ6uwvWmjRpR5hA53nJH41biMD5SEqSpyVPHXtgf415dpHia3mfyYIPJz1YMOD7g8A/Su9h1iNgInnWTH96Rdw/LNZvTRiaOhY3NvGXhQomMcbf55zXY+CLmd9Sg4KZ7g5z+Irzldc0+NgId0znjBZiP0BFen+CtUN1qEZiVYsdTg9fxArqwlnVic2IuqbPsTwpBdiDzHhMUbfxEjc34dh9ea7ivPvCchlAWSYOV/hXJ/E+leg191T+E+QnuFFFFaEhRRRQAUUUUAFFFFABRRRQB//0v1SooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACoZoxIuKmooBHnviawtorKWaUc9vrXzn4g3xK46s2Tg4HPv3r601ezjuIw0kfmFfuj0z/WvnbxhpgSZlZPlYfd/wAa+Zzmm17yPcy2or2Z4DpVxGdUeyaWISMdwVVzge7cAfmc1ozXN5pF6YjIyK2MED5SPqKzpNIgsdZW4s1CEn5mAzj+f6YrotTto71BJICCvdh/9fvXz0oNw03R7vMlLyZ0ejeKnKm01FswNwGZgMH8eo9a8e+JHgCHURJdQJ9ps5X3hocmSGTu8Z7/AExzW1dWk0CEwKsnfHXP4VvaV4guLVRbztuTHzRvjb9ARjFaUsW2lGp06h7LlblDr0PirxF4VaVjDqFmNRU8CaEgOf8AejcrtP0JFYeneHLTThjStCnaUnBeYxxqD6El+BX6Fz2fgjVjuvbUI0hxnaCD+IGKuwfDrwJMyz2llE7j5i5Ab68dM1301zLlg9Oyb/I55ezT5pJ/cfDHgnwlqw8Z6d4jvmR/s0yuwj+4o6ABj1xnsOa/UXQZhNYxyDuq9K+XfFOmm21QfZbdUSE4wvAAH06/hX1R4aiEGj2qydREufrjn8q9LK6rnUd1a2hzZrCMaUXHqdQvMYx1xWBrOfL2ryK0ROVYqvPFZl5i4jIfo1e1PVHiQVnc8uuIZftrOQSAQM/WszxbdTQaTK6DjB5HOPpXX6jGEO7bjHX1rhtdt5L3TJIZG+RyowBggE/0/WvMrxtFpHpUmnOLZ8P+IPDM+satcy7uGcbRz1x2NcrdfDrUopvtjC4t3QfJMib1Ge5wMlfUYr6DsNLkXxHdaXKdht5sgHjj6fka+j9HtojpyNfxgov3tuDn9OteZSqTb5EexiHBK7Vz8210XVArWGo21vqHP31doz+TIDW5oXw91a6n8vRtOW2aT+PmVh7rwoB9zn6V+gV1Po1smYbVbl1ySCgDAE8HGMn69qoPr1vbWYWygigYnnK7frjI5NYVJ04XX+f5bGMUpaqL/A8v8B/Dix8JL9t1cqkmd7b23M7gdXbufQDAHau68S/EG28kWOlkxbcgnzC2c9RtX5fzNc/qcySsJ76XMg6AuCPqQv8AKsBIbSJPPt4DcEgYbb8uT2BbmvOq42esYM6VQi2pTIb7Ubq5jEkvzSDuw5x7AZ/MmptEtY7MC4chmc5Cqfm/EVQFvd3VyxkOxF6jAOOenr+dWr+5i06MzK6wxqPvTBgTn0xWNODfvMc5fZR15mSYGWdTx93c24fp0/OuQ16zt5o2a8yARxiQjj22jj8TVDS/EUd4+UkQM2MZUnOO4yMYrpp4ru8GQx5HHOAffAwRXbGSasc3K4s8UvdDWc+dZsIgTnEyqc+nzAg8/SrVho2oqMm3THTOQBj1BzXpF3pdykS+c25vYlwB6kHmmSzW9vAkUywXB/uSRl+fbjioku5upvoZljoMi7ZLqTYAMBQRn9OtezeCrZ7e5TycjaO/615rpkFu0vmrbR2xY56dvrgY/HNe3eG1MKggbzj1H9K3wME6qZy4ubUGj27wvrrC+WzVERRj7uMlvfuTXs6klQTXiHhKWOG7yyrvbAyw5H+78pGT6mvbYyCgI6e9faUG2tT5SokmPooorczCiiigAooooAKKKKACiiigD//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAI5V3IR3Irxjxnp0bltmWYjOa9rIzXMaxpC3VvI7tl3HX0HtXn5jh3Vp2R14SryTufEes2Est2FDnZnpjjjtUes6jp1hCiXVwVAHOR0Hpn/AArvPF2kz2F08XK9SCK8E8RW8kiMJGCDoDyTkemOc/TH1r4uUnFOJ9ZTSnaR0X9oW0A3xuXB5UxnqPfioIb61uX25ckHJG0nv2JAFcTp93qj2eLqFo4Y/uyNjc/bkc/1+tdboiRXimSAYQfebG0k+xwM1xuMuax06JXOu0/THusRwOAp6BnzjPsB1H1r2Xw94afSrUyR8u464JH4lu35D2rM8F6fYReXIRMzOMBucYPuGIr2eCzjZf3mTgY5Oa+qyzL0o+0lueDj8a78i2PLJ/Dh1K4V7lck8qMdh3P1Pau8hiSIR2yfKFHStmeFI4zjgjpXOfaAt2hA7f5FezSoRpXa6nnTrSq2T6G5thjQ4A9v6VkXTxmPeuOmc/Wpb2VlXCYySOPY1iXNztUkEHcfm7Be38q0lImEOpyWrSckAkfz/nXN+V9ttDEVww5/Kn6/eiPLyHksOO+O2fQVT0557q4SS3kGwfe759sDv9K85zvOx6Cg1C50vh/wJp9tC018guJ3eRw7gbgJCCV3d8dB3xW7NoqxM4t1+VhjB4+tb1ioSNS2V6fnWuio3TJ+tdP1eFrJHE8RO+rPD77w9eQXHmQLuLD+LnPGM4PPHf1rzzU4VhkxeReU6EjLIXUjPHfAz9BX1NeRRsCEC5HUEc15D4nlTzNlzbouPlEgYbwPUgbTj868LHZfGKcos9XCY1ydmjwm/klhQzRQMwOdoSNdpwepPWuZfxDdTstsYwijqCPmXHoWIH4ium1qaztGB84sjEgHC5/DKgiuTuLVp22wRGUdc7tpGeeg6/nXz86bjLU9hSTRv/bJbaEH7SuOuFAc59toGK8+11b7VJTPCAXTplXf8SckA/pXW2WmXBA89iR/dXgjPsf/AK9dFaaQQplSPJHOR8vH4E/pXRTTMHJJniNvdahYF47mEKc8tna36Cu60vWdRt1T908i5xw+7j24/pXeTabHIux7VAPVuSPwbIrKXwuZXzEvkr0ygUNj/eHP60Sg90UqkXuE1ydQVAeQTn/aX8QMit/TdJgVFe7m3dcCT5m/PH9auaT4TtbZ/PJdjnJJxn8a6wW6uBHERj0OK1hSk9zGdVLSJmWemws4+zKeec7q9B0WEwyKjgD6c1iWsFvEvzIQ3+fStvTnKSBuQB2r1MJTUZJnnYio5Kx32m3z2eoKs0m1X4AGBn8e34V7/p063NokqABSOMcivmWZBI8cx6gjI45r3nwzqUl1ZxiX5R0HGBx2HTOPYYHrX0OHlq0zxa0TrqKKK7DmCiiigAooooAKKKKACiiigD//1P1SooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACijpyaj81T9z5vpzQBJUcqb1Ipu6Y/dUL9T/Qf40hSYj5pAPoP8c0mNHkfjnw1LcQNdRgNIK+atR8OW89yJZ0JYnlR/WvuG6to54yjSySZ6gf/AFhXinijQGs7g3UUDbepJC8fzr5nMcvjGftY7dT3MDjXy+zZ4M+jozYkzsThU7n8O31NWrewRZFaCIjtjOE/kK71zZcyrD5hPoq8/iQKz3vNJhP/ACD8sDnIfbz9Oc/lXN9Xho7nV7eW1juvCumrHbiWYgluuzAH6V6IsqRKB0FefeFrp75SII5ERf7xXA/EAGu5XMQEcq7s+2a+jwyXIuU8bENubuWpgWTPSuLnZINRVXYbSCfxNdSZxKWReNvWvDPiJ4hufD2rRRDG25GYy3+z1FViKihHmZpgqDq1PZxPQr3UIWYDdgk8DPXFYl5eQxQku4AH3vXjjrXk58cGQZlGGTpj1PWuW1nxhcyxeXEcJzwOp/8ArV588bHc92nk9XRMteJ9Ua71FDbEKm4/e4HzDGTz/OvUvCdnYQRBrYqWflmBzyB1z/gK+LvEHxEnSX7LprxXN2x4UfPtwOWOz09K+qfgzb3Nv4UtW1An7RIN77jk5IyefXP0x0rDDTbnfuLMKPs6fL2PdI2JQZ4HWq8+rR28ywMj/OeMLkfy4qit+8kzQwI+U6tt4P0J61JFNrRkYzIqx/wkcnHvnvXq8x4HL3NJ7pGXjPPrXKaxBbX8DxnI4wGTBI9xnNWjpl3fD/Trggbs7UAXIznHU1g63Y6nEc6dcYRR9w/Ln8a5MQ3yu6NqSSejPNNW0vVbJ3CXTXELdFZeMehU8Vy7WkjPtNnsHfAwPy5FdPc3lyNxuZYgwJ48xzz68cViGYufkeDPfnP8xXzdRRvorHrxlKwyKwES5ZGAPOMFR+gpzTKM+VG4PuDUkMNzMR+8gAGeyd/wrZt9NuOu1MjnKhMH8qhR7Dcu5nW2JF/eRjjqc8/rWvHDC46KfXjB/MVOsMqDcVwfoMfhUbPMGG5VQ9N23GfrxWsVYhyuNKjA8uQg+mOv5VIkQdMrnI74qSOIOf3hGT3FX1gfdlX/AMa2jEzlIzhJIMCVmA+hre02SEMOSfqaqbvL+WQBj7Zq3Es24OB8p9RXVSVncxm7o7Zo1nth0yBXongeDAy6KCeN2WLN7AHov0A+teb6ZcsqeXMCVP6V3Hh/XrfSrr7LP9x/utnHJ7EngfXrXs0mrqTPNqReqR7TRVe2m86MOOnr0z9ParFeicIUUUUAFFFFABRRRQAUUUUAf//V/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigBCQKQ7j04p2AKKAGeWv8AF831pdwHAGfpS7c9aWgYzDt32/qaaIh1PJ9+alqBpiWMcI3MOueg+v8AhSduoIkJVFyxAArC1a1+327xJGcMOrfKP6k/lW0kIB3uS7ep7fQdqSSUHcqY+Xqx6ConHmVpFRlZ3R88y6Z/Z9xJB5atzxkkj8uKoTWtwp3RmKL0wij9SP8AGvTNd0txIblUwGPU8Mf8B7VxU8e0kdTXluioLlPSVXm1JdCe+iRlMoK+u3H+Gfyrak1S7hUgRbj6+v4Vx7PcQsohJDdgP5nsBV6LUruMbCPNbHJHU1rTq2XKROF3c3v7dBYpNCyYGc//AKq8k+Jnhm08Z6dhXMdxFny2BIK56ke/pXXXPibYVEsDBDxwOCa53VNesJi1ugdGQZYnj/PH4Ac1OIqRlBps1wynCanFHxmkuteGrWbStWleaS2P7uST7zJnGCe+PWuR1zX5NX0maxWUxmQbXKnBI7j8a978bJo2twrFJukaQlUYAjb7k8cCvmiLTLDS/FUcV7KZLZmJG7uU5BNeGopyufZUcdenaS1PUfh18ONNtbJLqeIok+Nx7EA5x6g5HXNfWelXrxR+d5LCNR0UAuw/+vXgtprUjuHtYWKxxqVGVCtnnp0H6V3Np4yvliRI4DuYgbEC4Hbk5/lWlOryy5pHi4pSqHqsfiDxDdxILHTtqscFpHwVA9VI5NWZW8Ru4InWNOpHUflXmv8AwlHiKdlaxljhA+8r8tj2we1XHvtZuSv22+2oRjKgEc9+1dX1uLR57oNPZHoBXVJjie9TylGdqjaSfqSax7m80u0Yu11uc/7W7n0xxXFTwWkWTd3TTKnUhiSPwrOub2NxsSITQcfP/EPr3/Aiuari/IuNE0bu4srqfM8Ko3Y7SufzK/oTWfPZ2G7OGjPX723j/gagf+PUkM9vGuxHMat64Cn+aH8QKtq6oAjDaGPBX5Mn6cofwxXA5c250JWIY7SWPDRSfIe7pkf99LuFaEcN1gFQHHrG2f5H+lLDZlPmhOc/3fkcfh0P4GtJGdMNJhwOpxhh9e4qoxFKRVW08wgSnBPcirrWpiTdksvcZ/l2NTfasx/u2ORyVb5hVJrlFb5Djn8PyrZKKM7tiHaBmL8j2/GhAr8kEZPQ9KWUoy+bGwYfTkexqh9p2/Iw3c8EcY/DpTvYRr+fEo8tMbvftWlZOXx5nJrnhH5uGKhsfhVqK4aNsFCuPxropy1uZTid5BIQAQoWr88giEd5tG5SDnk59q5OzvpZBtAb8Oa7CyMVxAYXZskcgivTpy5lY45qz1PbPCurrq2nrIiFNvB9Pw/wrqK8I8H6kdH1c2LysI5zgf3dw6ce9e6qwdQw716NCfNHXc4asLS0HUUUVsZBRRRQAUUUUAFFFFAH/9b9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAoopCM8HpQBGwaQ7VO1e57n6U9VVFCqMAU6msSTtH40hkcjFsgHao6n+gqP5VUPJ8qj7q/wD1vWkZgSoQbsfdUd/c+1DqsSm4mOWHf69lHvUsoo34M0ewqSW6IPvH6nsK8d1YixuDEcb888cD6V7OEMcL3F1+7BGW57emf85rw7xfqUElypX5MHoPT0rixklGPMzqwsXKXKilLOoTbGMs361kTXT2wKxncT99vX2HsP1NSefuGd2MelUZW5yOQK8+dRvY7lCxmaneX7jELhdvLHGT/uj+tecarquqQOZHUMmCPlHJrvrmRzgY9cdhz3rFmjjdSjLkH2rgrTk3udVKy6HgOsz6zdwzSRQiJpF2lu+D2UV51c+CdQeaK5vXDXEozjrjjpn1GOfSvqqXSo5c7F5Jqj/Z0MDtLMu9lGBx0H90VnGdjrjVPnHSdG1y3uJEs5CXBCqh+6wxlRz6811VrJrke6aIjkD5WH+fpXoj6couwuBkLuz/ALdTNZJNllGOc8+//wBfrUTqXLc0zjYZdaxHM6gB+GHQ5z1/KurspblY1jP3R69j6/SrkcDQoUl+7nH0I6VnXOqRwqQq4J4weP1FYuXYzbNln8va4b5W6Z52nuvuPT2qi17bl2MJ8uSPqB/Me1csNVe7kZW/1UgGO+G7H8DTLUXPnb3Pzp07/h/ntUO5J18U8c5ZWG3d1I6fXH+Fa1t5tuQmeG5x1U4rmoy6tvg+63+SK1ILiYy7GGOMAduKcUJs66O4idv+eZHUdVIPXg1YaaSEgSZZBx6jHseufxrmnRjgqSCR19D7+1XIZpyOchuOR7eorVSZm0XC+9iYGzjt/hUWS3z55HFJEke7cp2P1xVpBC52PhvQjgj/ABpqIFZXYNhBtJ64rShthLgnj3Ax+YpVsWjA3nK9iCCKvoiqo2sHz+YreEO5nKXYlhthF941rxW3ngDBOe9QWtszYOc/pW3BDsIHeu+lA5akhINOSP7oPHatKImDDBSv41LGCRgnP1qfZhcHIFd0YpbHNJ33I5EimdLnJDKeq9a928P6gb2wjMhJbHUjGR9eQf8AOQK8LQRjMY6n1PFeg+C9QVZWsmXY3UAHr749fXH4iuik7S9TCqrxPUaKB0orsOUKKKKACiiigAooooA//9f9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooADVWZ2yIIxl25PoB6mppH2DIGSeAPU01EESlnPzNyx/z2pPsNDo4xGOuSep9apGWMyNczsFjiyFz0z3P9B+Nc34k8WW+jQMF3PIeBhTx6nPtXzP4s+J+r3ebG1l8kvgKAv3B+fWvPxeYUaCtI7cNgqlbVHuHirxnaws0TS7Y4xnaCBk9s/4V8+3WsNrGoNdE/JuwB1yfXA615Fd6xf3twz+Y+1iAoJC+wzk9+pP1rq9Gv4XQOju0ScEqMZ+nTOf0/Svl8Vmcq8lfRH0GHwCox8z1uwjijjEW1pJX+bLdAB3OeM/n9KvtFztUdB26fmawdF+23mREggSQ7nLdWC8AZPQevb0Ga63y1jwu7e3oBgf48V6mH96CaOKtpIwJLPqcVnS2y8jH5V1jx7ztH5VDJaYHA5q5UbmSmcXLAVXEYAz3NZ0tqSp3HiuzltEU5K7jWNdwuynoormnRsaxqHE3SxRnccZFQx27SKZApAPTtWs9mJZxHGC5PftXQLphWIBuPSsI0WzV1LHBzwHaQe45/wAa4XWIhGA5OdxIA+nWvXr2w2qSDxXnGuWbSTC1jYKz4LH+6o6/nWcoa2LjK5SstKJhQ4wT0+lbdvpJIG0YwME/TpXY6fou+OOVRhWAAz6f/XrSmsBbHbjKk801h3a7JdRbHGafppTNu68A5H49a05rEK6PGvCnk100lhmQTJxjg4rQSyQxgN1atY0OhDqnLpY+egHQ9anEAtyEmQD0btW7Havb53DoP61PJapdQYPKtVqiS6hmNYQ3CAgDNKNFglXLEq3qKnhtLmzwo+ZfQ1rxM7KGIrWNNPdESk+jMAaVJBxFcZHcEVehsn3AsM+4wK3YlDkbhiriWidfu1tGguhlKozPigRCOTWjGoKgKc49RUiRFM7SM1MmBw4x9OK6YRsZNlmGMkDB5q0Y5FXpuHoDg1HDk/dbcP1q6FLDDHB/KuiK0MJGU4YHK7gB2atTTpFjuY2mJUZGGB5U9iKhljfHzZYeoNQxMw4PGOmapB0PfrKfzoVJIY46qeKuVz3h68S5skyVLAAAqc8fjyPpXQ13Rd1c4mrMKKKKYgooooAKKKKAP//Q/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooqrcXlvaoXmcKB6nFJtLcaTexapGZUUs5wB1JrzHXviZpOkq3luGI9AWyfoBXh+vfGnVb9HisIQir13KT+YJrzMTm+Go6OV35HfQy2vV2VkfS2o+KNI01Gmup1UqOBkcD/GvD/FXxngBez04cgffJ4H0x1rwG/8Q6lq05uLoZ2jOMlAT24H51nwuCw/djnPIyR6/ebNfOYvP6tT3aWiPcw+TU4e9U1Oj1DXr3UZPt158yIM4PUntgH8K465u2kEr3DyFnYqNuOvU+/tV6a5txH5kqlt5zjPYdPzzmsLUb22gIjSInYdoA4AY9T6V406jk9WepCCWyM2Qrcb1hBfGVXryz9Tx7Aiuo0aWZFS3x0+6vRj7nsBXMKzGzVSAqszMwHUjA6nsK3vD7xQzM0Z3SMcZHRe2fr6VKKlsexaLqUwQ2cIIUY3lfvu3pu7D6c/SuutBc3WSHCx5424xj0XH6nmuDsLJUaMNlA3ylR1Oe3HPPf19a9HilihiSCABCgxjoEHfJHf/wDUK+iwMpNWm9jxsUkneK3NeC1VFAxz70+SFB93k+tEO5UAf5e54/n7+1TMwAGOvpXuRSseTK9zGnthjJ61iz2bSfKB/hXViAk7n6mq9wiouAMfzrOVJPcqM7HNW1gsR+Ufj61cls3K8DNbNtbNjzHH4VNLEzdeB6UKirA6mpwl7p/loWOXduFUdz7VwmtaUtnCsAG+5u3VCfQE817Q1sFDXMvHYZ7CuYi00XetRTSDhDkVzVMOrqxtTq23NK3sFht1UjGwAfSm3tsiWzDGWxxn1rqGhVR83T0rPmtvOJZunat5UrKyMVUu7mJBZgICef8AOaYkQW4eMjiM5B9mreWDbGo6Fev0FUJUAnbjnA/WodOyRalcrMmWw2BtP5ioo7dYgVHCnt71flRmUnutKIgycjg8fSly6hfQorHuO09quJDGORz9KcIjuJPBFTLEcF1OR3FUoktiLEpGTz9P60pG0ZGRj8aa4OcZwaZuZDg8eo7VexJZQ7scA4q6qJ3z+FU4wfoTV2OQ4wwx+taxfciRMiRjBP5irOExnPHrUajseM+lSHK+/wChrVGbK7oDnaA30ODVOMAOQDz6E1bkIIJBGfcVm/vDKMqM9ip/pSbGj0Dwrqht5vs8oAQnB3Njk9COMc9DzzXqgOa8a8PJMmoBWyC/r0b2I6H/ADg17BAgSIKo2jsPT2rqot21OaqknoS0UUVsZBRRRQAUUUUAf//R/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACmPJHEpeRgoHUmsfWdfsNEt2nu3AwOB618x+Nfirf6p5lpYr5MI7jPzV5mOzWjhlZ6y7HdhMBUrvTbuexeKfijoOjBreKYSSDrt5r5o8SfEnXtcmkj092WIepwfy61xrW97cM11cJ52exPr7mswxmGfYgRPZTz+OetfHYzNcRX0k7LyPqMLltGjtqy9CLy5YyXMjeZ6nJ+mBmnXFpjJkDFR0y/PT0q7DGJkxg78YzuwPxwavwrbRZV9uS3Uct/XFeeo6bna3ZnN+RJHEAm2IH5mySWPp+lEFsDG0mNxGR3Ix3+g9TW3L++y4jOGJ5f07Yz/hVKSM7WEzgonqcjI9ccfhSUNbj5uhRnd2VQo2+WvRMcn/erjbrbkzSssarkFiSx9Sff866m7lcIdiMcqAM/KB6/h+Nci0aSF2275cfKF5Vcep6D6CmwSHGI3Wz+GCMk+WcjHTBf1b0BNXtIuo0kIU7Qp4749/rnp7/SqcqPgvIwVd3zFccsewx1J/SqVr5oljjjb5Q2QvU8dzn/AD6VSYmj6S0ScR2P2ll/ePwOcnGOpP8An8a2NMvLqWQRpgKDkDHft16n0rgNAvWuLVUkwiqOSTknPcn0z1I+grprS6hhm+0EkAdz94A/3ey7vXsPevWw9bWOuh5tWlvoekfaGgAhyWYdhzg/zJ7f54uRkx7WuPvt91B/WuMgvpDulXCJEMsx+4mexPVm9h+PpXRWAPmYQEuBlyx59h7Dua96jX5tjyqtKy1N7qOeT3pIrXzH8yUfQU63AlYqhyBWmgUERiu+KvqcUtCMQoo6VHJEuNxFaOwd6gkTNaNGdzBuIjJy3QdB70y1sSkwbHOOa2vIy3ParEaAAn1qFBN3K57KyMtoNxIIzWfcMY8ZHFdDInU/lWPcW/mgpnjOamcdNBxfcz92XaMDoM59qaLPcd/8Qq8lu6Mrt34qcJj5ug71nyX3L5uxkSRkZGODUcce1ijDjofoRxWsyK/H51D5Q6N1X+lLlHzFFojkEdB0P9DUghIO5O/UVaVMNg9DzUwXAx6UKInIymjRhtXgjsf6VEsJU5XtWmYBISfTv7VKIVPXgjvRyXDmKMZJGCuQaspCD8yHj0zzT/JZCQg+o/qKN5X5iMr3x1qkrbibvsOVWA55H608uQNrDingq+Np696hfcowfzFWQU5WUg7QRj/PSswXLb9u5XA9cZFWrqcpk7d304P+FZTyQu4mkj2jPBweP6j+VZSlqaxR6P4c+d0aRsoCDkE8duR6V69GNqgV4v4entjMgDbS3Gc8H+ley2+fKXJzxXfRehx1tyeiiitjEKKKKACiiigD/9L9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAriPFPjjSvDsRRpFknP8ACG+79az/AB34zg0KxeC3IaZhj2H/ANevknVNTv8AV7kM6Ha3TjP4/WvnM2zr2T9jQ1fV9j2cuyz2v7ypsdF4m8TXviC4eZWwh6Dt/wDXrjxbys3zISAeDj/CplgW3JaWRiwxxn1/lTvtZjbaIjyPvHpXyE5OUuab1Pp6cVFcsEV57G4kQebnBzgMP554FUzYK75j4Zf8/Sqt/qEpl8rpgdeh/PkVY09S0XmeTuychiMk596zTTexrZpFlNMypZnMzr/tAD/CpY7GdJDIWWLGcbckir6Rkk7js9hjGPqxwKoXUyBfLRTI2T1fj9MA1o4pamd29ChlhON++bHO4kk59AOBUTshnUC3Z5emAMbR7iqa3Lgnc6x5PVMbhj/PrWvbIJyWZHbsC7bQfXnv+FELNFS0Od1Zd8YEo8sHnbgMf0P86wigUrDAC2QNzOcAH0CL1/H9a7DUbezjzJIFGOOrHHscn9BiuNuJZ7htkA8iJBgt32k9AB0z6Dk9zS2Y1sZjxSzXAi8whE3fe5b3OewH/wBarVrttr8IyiQt8scXJLMO745Cjr7/AErRKLbJvVQjYBO/kADpn1PsKyrGSRNQ807laRtpZx8/94sfQkc+1VEJHeWU7wzNHcAeYTlx1yMcL6D3rqbebzWMrD92gLqnZn/vH2WuMkASdZB845Az0Yn+fuTXRwXWYE89tmOScE7iOnHoOo7cVUJO5lJaHc6VK9xNClx8zDLIpwPm7yOOw/u/4129k32pdlkc2zH5nHWQ57e1eTvfbx/Z8LFJrsjz5By6oeFQH+8Rx+fpmvXdJdbYx21suNgCDP3UVcfmfU9M8V9Bl81LQ8jGQtqdZGv2ZBbxD5z+g9T71owxqg45J6msuGRTIVjJxnk9yR1P/wBatJJB/qxwF619DCx4syxj0pCnc0BxjI79KQuMhRWtzKxGUydtEhCY9qsDHWqsnzHPpSY0MkJxjvVPBaQ46VeAHSmiPBz61L1GUGySB+NI3y+4JyKshVZmI7GoJOAcfwmoKKO3DFenWnkg49e5/lTXDMSV+opURs4btUFDcgkL0owQ3XkfqKkeIHGOpJpwUN97jAp2AIwN3y8VIwXqOD6UnTgilLbxtxzVIlkTqGHHIH5g1ErqSQetSHI5H5VVkAchkJ3e3X8qljQ+RAvK8A1SkeSPljkepqXzWYfKfriovm+8mSO+P6j/AAqG+xaRVkYMpypx1yvI/KsiSIEF4JDGTnjPH5VsMARlBjPpx+orOvFmEZZDz9A2azki4mjpc6uO6uvccV79o00stojO24FQQSMH/wCvXzNpl4qzBWbBY7fYEfWvonwnPHPpMZQ/MvDD0NdeEncwxMbK509FFFdpxBRRRQAUUUUAf//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigArl/FHiW18O2LXEzLuI+UE9/pV3XddsNBspLu+lEYVSR3r4k8W+L7zxRqcmCWTPAJ7dif8K8POM0WHj7On8T/AAPUy3L3XlzS+FEniPxJLr109xLymTySAB9KwYb4sdiJkdyeuPxrO4+UzsPb0AHXA7D3rTgMKxh1xjj5cZz747/yr4fmbd5M+uUFFWS0NJo5powbceXGvBYABR+I4qg2mRiPzbiVkx1zgZ/rV5LiV8NyFQ98YH4dKytSvk25D7snqSBgfjVSUbXFHm2Mq5Fqv3jnHTnH8+taMDGYBidyjgbnIT8h/wDXrDQwSXSKxDhjwqLliP8APvXbWUFrDs3Y2ngLgM3P0JwKmmm2XN2QsEdsIt7RcjPJG0cHsCSf0rEvruMErtVQT/EefyP+FdzKhEe23iRFxwAOBn/PfmuZvbJlTaQuWPoP51rVg0rIypzu9TlLVfOuBIFRu67hwM988fov411SDYE2yKzN1252jHv3/OobXSbYSEySqMH5gMH9MY/PFbUMUJPmwBpPSV+B+A6fgOKdOOmo6kjltTtB5LlW+Zz8zHrjrxx0964wKHukhtznbk7VGW56sx7fj+Ar03VdPlnQuANgPLkkk8flXmV/JLF/olsdqEktjgE+5HJ/PjvWdRWlsXTd0VrphGCARLMCQBnhTkAe3A6CseeNpr6HyiW28swJwf8AZHY5PU/StG1s4mVsjOB8zDgENyQB2z3PJIqxIuJFRywOcKFBOF9SP0AJGT146i8hs6mzXcE+0Zc4HA6D/ZHrnuatCOe5lM0mdxbYBngEckfQDg/lU0I2ugUbEQBVycnheWPvzz6Us8saiOyQ/O2AAOe+efbPp1qktLGXU1fDiifUUnIyuSy+vPGT6E9B7fWvX4LlZHaK0O1Y8B5QMgHqFT1xnJ9WxXjenLHA0kMb7dp3O2eSPqOnp711theTW0W24OyMMSFHGWOcZ+g5Nelg63s9GcmJpc+p65pu5VMmMKq8kc4Hb6k960IrhFQb/lJ5I/z6VzVpfxiyRJX3STp5jgcEID8qgds55+uK2bdSz/apgFQkdeOnT8B1/D2r6alUTirHhVIWbubMUkjAyuNuecHsKsJwCx7Vh296t3KJfuxgblH+yP4z/T86sS38aYij+Z2BYD8M7j7V0KorXMJQd7GtvBJx3/pTJHVBj0qhbykRbmOWbGCe59celSRFXdpCc5OBVc1yHEshiy7iKfIdv4YpjSBRgdqqtKWUu3fHHtTuKw1jsVmH8RqIyDbgDkkflRPKqw575/Wos8YIxgVDZSRYhVc8c44qVo8AnHSooSFBPbBP5VYLDaSeOtUiWVyAOvSmOB1B5pk0yquCeCKqeeSwH8VS5IpJlsPkevtTCw/h6e/amFgenDVF5nzfz/8A1UmwsK7lTkDj+VQlwwJHOKSSZVqjLh8unbrt6j6iolItInlUsNykn19RVYybcEHPcEcYoVpWG+JskenWk3LMN5O1uhPSouUkKbnzFOQC3TPr9ayZpwARICh/T65qe5SWP5miyR3H/wBbism4u4dm2XKqffB/A1nOdty4x7Ebv5c4fuec9z9cdf1r3fwCUezMgIycjKk/kR0+hr5ze4jZtsE2/oQOjcfoa95+Gt2TC8Hl43c54z/n+VaYOd52JxMfcPWaKKK9c8sKKKKACiiigD//1P1SooooAKKKKACiiigAooooAKKKKACiiigAqC4nit4WlncRooyWJwAPXNPlljgjaWVgqKMknoBXyX8V/iDJeTS2FlKWt0yFUHhj0zgYz+NedmWYwwlPmereyOzBYOWInyrY5v4neOBq+rHTLKTz4g3XkL9fevN4woXbjbzz/tH3/wAKq6ZZTXI+07QsshPzOedvfA6DFakgit8kMGIwAwHA9ev86/PalSVSbqT6n2tKlGnFQj0GCJ9yMo+6OAe/19vatCB7VWMkwZwOw6k/lwPQVyj3xhk2QqZJJOdzDP8AOtC2v7mOEtNJub+6nQfjjr9BUQavoaSTOlaXz0xEghTHHGW/M9K5S83zTPFb89ieBgd8kZ4/WrzzXSxia+bYp+7CAN7E+pPT271mRaq8JVZrf5c8RoAQfXPBzj15rSWpES9p+m20lwH2+aSPuoSB6dckmvQ9Mso7cgGOG0D/AHfl8yQ/QAnmud0m5M/zSM1onoACcegxgA/rXbabKxO63ilKk/eLIg9TgjJP511Yanqn/X+ZhXm7W/r/ACNKa1WO3BkWWRSON+I1/pXFX6Pknaqgduo/Wu1lBmLeWgZmxjLF2AHuK5fUrEoSZsDnnHHP61ri46aGeHlrqYUUzg+QoQ4+YAYAHueBWxHIk8mLu4ds9ewx6ADt+nvVKKFUJWEcn/ZJx79q1rOxiE4e6GXYbiM5OB3bsK5qSkb1LFTVI0uItqgsgIwvb0/z3NcLqGmQq77tzseQTwAOnAHH9PavWpRCwAhiDLjA4647nP8A9asOezklLfZxg4JkfPIA9D/hVVqV3cmlVsrHlaW00CyNCBu/vMcbT7Z/njNVrKB5J5BayMzhvmKjJ49z+lbuqWy6a73DuDkdevHooH6mjTFk+zmRotkLnJA6yE92Pb2Fc67M3fcmNvLDbRPM/QYYZJGeu0HuP7x78Uy5doCGU4lGW47DtVq7nSNVjOJjGOeyj0VR9fzrN02P7Y7ySKNjOxc9SQvbd6MevsMCmmr2Ia0udHo1ukUQ8+TdJI4bHUe31J/zxmuxihju5P3TBtuQMnhc8lm9yBxXBpI0c+wOB/IZxk59cdvoPWvSNJsiLMQvkCQ5+bjK/wB5v89K7aGr5bHNW0XMRWrTxyrISSJGDFh1KqcKPZQefeukuNd88nI/chgqqT97bxk+2f0FU7wWywkwg4cdT7cAH+eKyEjDPukBYrztx0GOPpnrnsK7FUqU/cizm5Yz95o7aO/URGGNd807ZIPAA64+n9Kj04ZRmllLKxO+TvIerH2UdB7VgfvUiAwFONueg3HqPwHWte2XESoH4GAM8c+p/wAK7oVnJq/Q5J00k7HSi6Z9qRL878Aeg/8ArD+tX/OSJc8bYzx7k9K5kSrAi4yA2eT1IHU/jWfJc3F5KinKjeWx+GBmuv6zy77nN7C5082oqkIkByd2wf7TDqfoDViGYuxAPAA59BjNc0sMl1KigYjiLfQAdTWm7MEkVeDJgfRT1P5YrWFVvVmcqaWiJSzzSqDwuQfwq68iqxz0HP4AVSZuQg4IHP4/4Uk0qDnOdxx+FUpWJcbloXBS2dm6rj8z1p3nko4Y/d4/Mf41itdJG5VjlTgAevc1QnvjuMinK4Ofp7VPt0h+yOiYLNGI88/4io4gUYbhlen0rDg1RAEd2wcfnjity1uoZVAyCcU41Iy2FKDiWZBwADn0P0qpJMSwDdexp8kioD3U/wA/8ayppt5AUg81cpWJjEkeQsenTt/WjGRkcEfp+NSqoIy3U9D6fjUUjPEQ2M1HmV5AAW5Iww7jjNRyMXyG4cd+n59jUu+J1DJ39DVaZ2C7vvr0NDGZ1xNMg2n5R6jlf8/jWJPdW8wZZGBK9z8w/wCBd1+tatw3lbp4MOvcY6ewwePxrjtSuEu13wKFk5zkd/8AeGCP+BCuOtOx0U43K93LHDMqsuwnldp3D8j/AEr6H+FZZ4WZSNuM7SORnuPY18nbszCPJGD91s5H4HtX2D8MbM2+mJIVChlA4ORnr07GtstvKdyMdpCx6vRRRXvnihRRRQAUUUUAf//V/VKiiigAooooAKKKKACiiigAooooAKKCcVRv7pLW0knkfywo69efYetTKSim2NK7scD8RvEMdhpU2nwMDcOuSD0Vff3PavjO5DXV4XX5znJY9hXpnj3X4ry+Nra52gksWOWJ75rgIdyoEjPLHJ4r88zPFvE13J7I+yy7DKjS82CWowfMJ257jlj7+w9KwtVmVmJVuE4OBhR/ia3p51C+WDhh6HoP5Cse8SPYrL0XgADA/Adz715stdD0Y9zjUju7qbYVCKD0IOT9ea3XkMMaRI5WV+gXBx74AGTUsoeCNSx3buoX5QPfn+v5VRub2O32x26s8jDO8dh+X60r9EUacFooQSzRH/enfGD6n39qvqoK/Iy7W7r8xI/3R/U1kxzmFUZ4cZxt3AFvqAentnFbNlZXZ+afJMnO3IwB+FapENm7psd4oJVdg7Fhlj7egruNPg1FmWS8xEpIOXbIwPpXKW0H2cDdIqEH7qDcfxxwK63TnuQd22QDozuy5Gen3uB+Wa9DC6OzOKu7rQ3J1iMZd23Bhnp1+ijgfrXJ3lsWJkVfl9TwOa6tp7KAMrXDyyMOcHIz/vAD9K5TUWmkIMabQejHr+Ga6cXa1zHD3uY2145MGTYf9kY/KtbelhAsYGHc5yRzk9yT39MA+1Yq5SQsGBxwWHcn0PJrV0uCdpfNQEksSZGOMY64znGPWuCje9kdlTa7NJIQGSa/YgMCQhPp/sjOPqcmor6RXi8uBfkBzycfi39B19avratNJznYeWfucehPb/P0r3youLaBguTgnr+tdU4NRZzRkrnj2vPtm2YUk8sTnHX27ewp1le3KhI3YiTaZArcEJ03t/dX09eg9m+JGto5z5fDZPJG5sZ4IXpz6msjTFAmkeSJnJKuEI3NIw4VpOzbf4F+6OpzXlq3Mz0Ps3NW6lR0UIScg/OeOvUgfyqjbyyXDP8AZgyQWwwenXPAPr6n8q27izmlZ57puGB4zjGTyc+tUYrR5JVghP7odQBgAdTj0HqepqVfmHpY7mzs0ljinRR8pAAPGT1J98dq6eyE16v2gMy2+7AOfmfb1I9F4x9K422lmkKE/LEq4H8PA6k/Xkn0AxXplu8cemx7U4kwEUdSn07Z4HP6V7OFpqVzy8RNozmlVsmZPkJHlr+mT9T+lTyXHkKhiXdJI2WLenqfYDp+FTSwxoXnuyAU6KP4fQD3Oaw55JHnyUwRwF9O5/8Ar1rNunuZx940EuY1uxO/711+SCMdAc9QP5k966S3eFBsOHlbsPfjk/h2/wAa4GO5a3L4UtI/DOf5fl2HA+tb2m3MjI0mcg/eYHBJ/ur6ADjPQCtMNXV7E1qTtc27hlEyoxDsxIZh047D2FRwARysWPKqWYn+HJ4z7k9vSssalHHKwTBAA3N/Cg7KvqavytnyrVx88rbio6/ifYdK6YyjJ3MHFrQ14JGkjwnCuRj/AHBx+bNmr7N5Nu079Wxj6DpWatyqcnBxgceo4AHsP6GsbVtWkkim+baiZC+2B1/PmuqVWMI3Zzqm5SsX5dR8uN5V+Y8n3JPA/rVFr9pGdiTj7qjucdT+NcpFeyXN3HDED5MYUMP9rbwPwGPzNT3Bk8wJFyvVj6+34nrXnvEuWx1exUdzVEsu8SEH584z+VSkqIGiA5zjHuetLaIfsyzzZ3McAn070WrRsdo5JYE/yNaRX4mTZQj0q5kYKx4Q5H9a2o7KW0G8E5U5rWhlh8tmHPWlkYzAKvrj866YUYxV0YyqN7kIlaVDnPv/AIirMcMUgEg6t1x3I7/WlRFi6jkdakJjHzIcA10xXcyfkTBABjOaqt8pKHlfTqRSGZlPTg96Z56FsP1HT2/Gq5kJJkDwgBnXp3wcj6/55qi0kUQ8wnA9c8VduZBGdwYr7j/PNc3qE8gRpvLKnu0XRh7jpWFSSjsaQTZNctA585Hw7dcAEke471zl/wCWqsWIViOu3bn1yOQa5DUtVgik2TqYXB4LLtBP1U4NZa6+88ZimQFccFCTx9Tz+BrzamJTdmdsKLRbeOOfUoCrDaXAODxn19q+5vBlsIdHhVWJwBkNg/qOtfn7p8xfXEJBeLIwUIJ//WK+/PAhm/sSIyAqCOAR6ccEZGP5V7GWdjhzDZHcUUUV7J5AUUUUAFFFFAH/1v1SooooAKKKKACiiigAooooAKKKKAEIryz4oayumaQEyFZ84OefoP6mvSr29ttPtmurt9kadT/Qe9fGfxM8Y3HiHUHkiXbBb7ggPQKO59zXh55jI0qDpp+8z1Mrwzq1VK2iOEleB55L68O4t1Hp6AemamF0ot3mlATrgf41gWazT7JSpkZ+V3HAX3NWblI2iEpzjooH8XqfpXwvMz6/l6Fa6kiWMSEjHXc3Cgn27j61k/aJLwiGDdIzY+bGT9R6/wBKrTNczXBd9rZ6L6A9zWpZ7IlYZIQk5K/JuPueTj2BqEaNFpbVyhjchWPqQ2B6kf8A6vpWPd3kiSeQn3vpzn1P/wBc1XuNXlMn2S0G1RyNi8D8f69agWOdtimRd7dEHX6k8CnzXBRtuT29zEj+Y4adv4mYjb+A/qa7S0lkMcRR/J3dY40zKfrkYH61woL2jGV5QwHIAAAX6Dp9c10ejSSXLtKx3I5GQqE59B1yfzxVQepM0el6SbVY96AKqcfvCSc9Dwvf8a6gSQvtdIBxwCRtUfRRnr71y1k1yUUpG0Y6buARj6A4/Ct5NsW4PMAw5bqzj8zgV61GVlY86qrs12u4o9yQqpYew4H5cfjXK6jeSysWk+c9ieg/PGalnvjErRxRsN38TEAnNc7cISxMzYZf4eWPPr1x/Os8TWclZFUaSTuyncPICPLJJJwPXn07/lXU6XblYjNdSZhTkgnClh2OOuPQcZ61z1rE7S7ijFh2/iOffnA+nNdTBHPdyCW+A8qPHA+WNR6Z749Kyw0Nb9TWvLSxoxzSTIbm5XZABiJO7HscfyzWReSs2ZAOTngdB7D29TWvc3Cld7H5UAHA7dgB6n8653UbyRUYY2MB84HVF7AnsT6Ct8RNJWuY0otu9jzfUfMF1IW+aQ/OPRewwO5+vStLS/IhjYsd7knJY9SOMKOpA9fWs+9jR75Zp8hA28qM/NjoD3wPzNbCRF5Nyx5eUABQOQnUZ9AOpFefHV3R2y2sJeM9zjaCyLj2FZxE4jMUL73c8gdFGffHbk9810N5gcOwHXAXHygdTjpnHGT0rOEEsifuBsGMnAJ4/vHPUnoPepcHcFJWMyC68p40dyURtpPUuxOAB69PoBXoOi6yby2WcMFklO2MDkRoOB9WOD/M8V5lPaSrMcMd2SBn+AHgnP0/M1v6M8to6YUoFGEUjpnAGffHQds5NdGGruDsZVqSkj1W22uQ7NiOIE5OT9Tk9WJ7/lVO+Rty4B3uQqoBzj3NMaeSRktkf5wdzMOg44/+tU7vDEmVPlxL95+5HoD7160mpKx5yTTuY9xaTRSBCCGJ2oi8n8/5n61ZVLiGBbQKQwGWx79gP5Vu6dbsxM8KFGYZLt1Vew9QT/L3NbkdnHbqZIh88hA3t0/AfyFOlhL+8hVMRbQ5e1toNPi+03n3oxvWP0PYt6n+Z59K0LaZrWEXt4S1zMP3aD7w3cD8z/nilu7cNOI1b5Ac7j3Pc8/kPz7VjTyBkeZTjnJY8t0wAPw6f/XrRv2e3Qz+PU0Li6LSpYQsvmRgs5HQf/qrImYXqmyGGQKXZvUe59/6Vz8EspaSaQ7FlGCc87SeBn3/AJVuWCNb2MjyHL3ICqPQHP64rBVXUfkaOCgi3a2iCItH1Dlif1q0pTgMByCT9OgH4mq5vEjXy1/jyOOgUcH8TWZJKxjAHLE7ifbt/jWt4wWhi7yeptXMzSCO3U8DA49e9VS7WUSugxuIY59zgf402FsKhbqcGo5/NuI4oz3G7/vnpVt316k26G5bTxeWWQ8Nz+nFMi1ZmuHUfd4IPr7VnCMjYIjjKge2arrhJtvG0g5/r+NU6slYnkTOplvHnUPCfnHH1FVLXVI5TtfhumPUj+tc5NeGzlUZzFMOG9GHY1KphmQy7uTjd7HsT/jV/WG2T7JJHYLfREbE+Y91Pf6VRvL2ONcEfKc8ntXHSyTWsg3HhvuntUUmqzPmFzz0wx4P0Yf1pvFNqzF7HXQ1pNVSJC6uJkHOAfmUVkN4gtWBe3bYw4Kt3z79KxpkFxJvwUcenIP8qiuNPSZcg4fHOPlb/wCvWDqyextGEepXvtdtGR1eNm6/d5x9UbI/EcfSuammVQLjdhW6EKB+Py8fhVu90h4o/NUhWXkNjDY/rXGSmcHyXcbs/eAAz7Gud3clc6Eklod34Js9OvdVzcEFWb/dI9wfUdcd+3Nff/h6yistPjSBmKMq8Mc4OOoPoa+Hfhr4fvbzW7Z7GULKG5BGCQPzBFfeenQCC1ji2hdo6DpX0uWx0bPFzCWqRfooor1DzQooooAKKKKAP//X/VKiiigAooooAKKKKACiiigAooqnqFytpZyXD4CoCSW6Af1+lTKSim2NK7seIfFjxHJBALcv5UXIAB5c+v0r5Sc/2gzahetts4/uL/FKw6Z9h+prvviH4hOu6rJKXymdo4xwOOBXJJFBNJE33ghwq9Bnv/8Arr84x2I9vXlUvc+3wND2VFREsgWjae4Tg8HsMf3cfz9TTdR3SKCi+WG+7nk/gB+grWuLwISjISF6Dp+H9Sa52aRppnmuHLMw2fL0GeSAOw7Dua5GlsdavuYc9rcy4itxlW++R0z9e59uAO9W0svLjLSOHb0HOMerdB/KtuO1eWBYbdFVDxycE/gO3vV1dGhto1NwfOkbOFx8qj6dM/Wj2T3Y/aLY4iXfMuIIjj+HB2qR67jyR78ZqqiiPdKxHHBK8L/30eT9c/hXY3kMYG3diMnBA5LH0z6fp71nx2kEf72T92qHCrgZ/XpUcrvZFqWhg2+nJP8A6Tdx+ZHxjzDtXHYBOTj64z6V1emRkoGct6YxtH0XPIH5U2KREYlysRUfKoIZiT3Jx19eKlhe6k+feuwngtuJwfQYz+NCdnoJ6nW217PHEqIqh2OFznj0Ax1P6VuWM8iQGSZ1iVsbj0JJ6ALyxrJ00KCqru7bn+7ge3Uk1uSG3WIICsaAZ4wpP1J6fqT616lFWXNc4Ku9rFSa73MSkZcjOSflH9SP51z9zdzyuI7YKg9cd++B/wDXrXuBDJGCh2qBgdPzP/1zWU62yHAOR3PQH+p/zzWFZyelzWmkugtpckSGL77gYJ4O3146D3JrsrHddp5znaqjK57gd8Ht79TXI2sSykuzeXAnLM2ACB2A4AFdRa3TX6iOzj/dH+I8Z2+nr7D8cCt8K+jMq/kW5EH2Yyg5ZjkYAzn1HtXJXjSRt5eOhyecjPqSepr0SW0McH7wheP++cD+f9TXn+sRtFG0pXoMKDnH19zRjabjZsnDTT0PO9YkleTNsTuUgk4xtUn+Zx1POBXX2jrNZx4cRjIUEDJJHU4HU/oK4yd5I5HtY873+aR2P3PTkdz2A+tbmjXBugLSBfKigQZPUgHnkn+LA6du9cVJnZUWhtP5EgcRfMyjaSBwD2X3Pc44HfNPiiEcJ8sjL/ebOT+Z6n+VRxSRfZmMI226sQ2PvSE/wg9ceuOTSW264uPtCKQFO0eik9FUe3860nZSsZK9iyLHIWGJMySEHHUnHbnsO5NSz6e0GHfLHPBHAwep/PpWnZRP5Zhjb945O4ryce59P0ropLNfljPMjDGCc7V9T05rqp4bmjcwnWcWc9Z/NwQRCvUDguew/H9K6eyt7e9cXkwUxRDCZ+5kd1HcD17npVSa2hiVlwAgyDz1x1A+veo4tVTzI7KBDLcSsAqjhFC8kn0Ue1dlGKg+WZy1G5K8TqYZYolM92NqHlIx1Oe7e57DtUcl1b5+037bdnQDAVQOgHqfp9KzZ7iEgAyKVRhlUHVvc/zrCmuNy/a3IJ3fIT0A7ED+v5etdk6/IjnjS5jYvrtHcrIfLXALf7K44GOpPtXL3lzBP80g8u1Q5255YdyT79B7fhWXcaqqK8u7zHc9evzN3J9T+grKfzLmNY3YkDLMPTP82P6Z9q8ytieZ6HZToWL9u51B8sAAz/IMdcf4dAK2JrrdOI41+UHCjvxx/M1Db3DW+Z1RVaNSsX91MYG4/wAgPasCXU3RAIQS4DOCep2tx+ZNKDtEUldm7aypKZZclgAwX8Op/E1ppBmP5upxk/lXN2EzwWsKT/fbCnt0yxx+JArqY2b/AFOPuxoPxHJ/wropWa1MJ6MnX/VxepHP51a8j59g6Lx+eRVFMCdd3GP5VpW8wZC5PpXTGzMZFecbLdcdQP5f5zWPJMjSSleAc+5BH+FX9Rn2qyp2IIxWHHMkjEg4Y+vtWNWWti4LS5LFgqYLrHlSdGHQH1Hp71WYTWjlVOdvB+lXJkj2lwSufvr/AF/z25rEu2dR945XofasZaFrUmkvxgxvjb3HpjuKrSXKAYlXPo4/kaotcK/zPjf0PofQ57GqsjsrbofuE4I9D9KnnZXKaguo4wHVxn+7nB/D/Cl/tOwvgbecmOQcBtuMfWssfZn+8SCfamyx2RGXAZh68H8D1q1JhYz9UaeKN0Ys8f8AeXJBA9Qf6GuLjuPOuzBwQDwTnP455r0m3SMoRbz4LfwtyP1rPs9Nd9SPmRo27PQrkn1HQ1rCKbuDlpY9l+DXmm6J8rzI8gA7gQG7Yz0b05GelfXMX3ehH1rw34T6F/Z8LzMWV5eDkDa4H3WHHUchh9DXui9K+owUWqSueBi5XqOw6iiius5QooooAKKKKAP/0P1SooooAKKKKACiiigAooooAK8k+K+tPY6T9jjbb5vLegA9fqeAK9Zd1jQuxwAMmvkH4neJF1rWXsoW/dQt8xz1I9P6V4ueYn2eHcE9ZaHpZXQdSsn0R5XDEbt3cnapPLH19BWrFZmBo8DbxlR32+v4+9Z1vI10+6VTHDGcBVHXnp9T1J/CtG7uXAMiLyRwcEhQO1fDxStdn1zvsYWoyRRJvI+d8hQT2Hc1iDb5qlyxLfKicZbPU+2fXH0qO4e5uZ23sdqdAD2HTJ/pVrT0EbGTcWkfPz9So9j/AC9fpWSd2bWsjrUc2y759qM2FKry2R0XPapLy7f/AI94+XUZPPOT249uvpT7dQjRhPvL6/eJ7/T3NUrlmjbEimMscYUZPXoP6mt5uy0MUrszRsO4AGaX8cJ7f4k/QVTuYZJcJE5e4cY6AhM/XCgn1NK90tu+PLO92+SJR949ief896s6bDwZJQJp+vllsopP95gACfYCsormNG7ai6bpTR4e6myP4v8Alo5P+9wPy4rSmZEHlwRMkeOSx6j3ParUM8jB2hdEA43ffbJ7D/AVHeWSyKZplYAfxMR19eSBn8Kpx090nm11KcGpW8JI+YyAcKmWCn19/wAeBW5ZSPqDeZJHlV/ikPyqfXA44rkY4o4v3qDKnkdBu+vrUqfbp33Tsqxrnhhx9MHj9CadKo1o0E4Jq6OwvpklAijke5ZeNkY2x/8AAm6fhyT6VmOsm7E0aqB0AGMn37/nU9lJbxoFLSXEzdgdq5/22+6o/U9BVqeSExlY8FVyrMDgZ/uqR/QH0rqlHmXMYRfLoUGV5SomfCKeExgH/gPUk+9dxozKAqwgvKMgjsi9/YZ+uTXGpbp5gUyrbLwTg5kIPUeorpH1FYLZLLTozEmcAEHdIfUDOSPyHv1NbYW0XzMzr+8uVHUieJ0MEJzI5ySTk+5PoPT9K5rVkVlzKSFPy56Zx2A61q6OIYbYyy/vHfkhMHJPAyRx/QdqoayplLqxwdo4XIGD0APX/Hk9K6sSnKnc5qTSnY8g1C6SKdo7VNu05z6E+/8AePcnoBxRo8YLtHcDCcE88sTjH06Z+gFbWsWcdjE+EDP3Xoo471zGmvNaTSS6gTGZQCU4LkHgDHbeeMdcDHrXjRVnZnqN3jdHXLOkirEOY0yeOmfr6f55phuwkDTTZC/diVeM5POPc+tZdxcQRMJr5jtQZWBepz3b+np7k0Wn2q6uxf3UYATKwxE4UNjqT64/IcdabkSonf6PeGBlghVVdFyVPPPq2e3p6mus3RWkDuWLSkjOP7zdvXPtXD6LbSBhsPznlnxgFj7df89q60RRx+XHEd7ZIjz6/wAUje/YDrXp4Wo+Q4MRFcxkXMsrFsv83IJ7L6//AK6fZFlJNqvloeGb+JvRfUknk/lW1Par5YVPmZflwOgJPOSO/sOlRC12DOduzjeRwueoQd29TVqnJSIc48tjMuLuC3jCIgRFyBuOScnnPuTXHanqSySGAPlgOSeka/QdT6Dqe+BWxqbbInmhXZuB8sHsOm8/0/SuIttKnvW8yZsqTu92+vp7dvrXHWqu/KdNKCtcvxXGVV7cbyPlXA3ZJ7Dtn+83rwK27m4W0hEVuge4KgADnLNxyfrz9BUNtBcF8wrtWNcIF4VR9e5xyT0rd0zSohKzu2doLSueigD/AAq6NJt2RFSaRkyWs12qWaEiCMAtjqzcd/oP1rUGlJh7+cdcH2AHOB7dq6W3todu8jb5zBR+PP8ALH51Xvb6OOLy0G4K+MZ7AZOfxr0FQUY3kcUqrbsjiol83UZEI4gx+f8AFXXsQrMCRuBA/HG4/lXNmAWrGYHOSQ3uWGf61K85aWd0PzIMgepY/N/h+FZwfKOSubsVvKEE754BH+fwqhbzlbeRSc7QCPwrW0u+ju4pLLO1mHyg9MkZH4Hp7E1y0rGFSGU+XKvB6445FaTaSUkZq7bTK1xdurMRnswx6/8A6qmjKyjzVAAPzY+tczBcyB/JkG7b8p9cA8flW9aiRBuHTv8A41xqd2bNWRff5kwGPHr1x6fhXN3cjKXjyQcA/QjjI9Qe4rWvJhBtkPAJwSOx7f8A1q5bULxGbcSA69e34g1U2KJW+1fN5brg4P4j2P8ASrcRcjZgkHgg+n171ioGmYlgAc8EdD71tWpmhA3KSPUc1ES2WI7OZe/X8aklsnkXBchvQ9KvwSiRQc81caQOgQgMB271vGKaM22cva2VxCzsdgI5BIOD+NaNmZJrgeYysD0PQD0PTNaBkjEbFAeuCcjj8f8AECqdoqxys4dtp6g9Pr/9euqnFInmufSPwr1m+mZtLnl3Bfuq3PA9D3+uele9r0rxr4aaN5WlebMDvLBlLegPY+o7g+1eyJ92vpsKmqaueDiWnN2HUUUV0nOFFFFABRRRQB//0f1SooooAKKKKACiiigAooqnqFwbWzkmVSzAHAHc0AeS/EjxnJp9rNZ2UoRBhHZfvZPVQfp6fU4r5QvLszMSSQZicHPT3z/n861fiB4guNR1yVnbKQkqqj7o9cVzFraNfFVlysaqCzZ+6o7Accnv6V8DnOI9piWl0PscsoKnQTfU17ONZjHawgtGo+Zzxx359zVzUS8lv5EZ+VxxxwFHcfXoo/GmWim8k+XEcCAgD68Zpbx2MJRAS8hC7h/CMfzx/OvMXwnf1OQleK1IJ+Ux8c9Ccd/XHU+pxWhp9u8WLiXJkfBjD4JIz94j1OeB2piWttJMnngnB4Uc59AT6nvXXWcKp/pNwBtOAo4JZufzH8zSpwvqVOehJbxGGI9cnl5DyeOuP5DH1NZGpSHmNdyMBzkcge4/xrqriOWMhM7ZpOeeNq+v+ArBuUUny4gTHzliPmkOeT7Ln861qQstTKnK7OLMDeW8odkB43McO2OoUDt61Kl1KAsFp8zYAZuh2/0H61auYJJZGI3YHykjjOP4R7fSorayRcxldqnjAJH16Vx3aOrQ37C5kjZlC5EYyXPA49McADtn8q1kuXu8yRR7woyDtGFz7k9fSsa3tLoFVEgSPg7QCT7cf1rdmsrifYkjbUX+Hdg59cD5R+OTW9NyaMJ2TOWvjFGN9yfKBPGWyze/Hb2rOkuJCAI0IU8cnLY7jpgfzrsJ7Lyj5sUkaMOM9W/MnrXMXluxnKSzE9ehCjJ9TjP5VMo2LjK4y0W6d/tFwCIlyFG4genHYe7cn0rprfUBEWzCTsGBJwiRr1wnp9eprBtvkkCF0IU44GQo92Of0q1PeQ4AEgk9MqQPw4J/KtKc+XqTOPN0NZrmaf5bZPLDnICcf99OwyT9M49q1LdoYQIY1a4uHHT7qkdySSW2n1PWsaOO6QiTCu+35VJyRnr8vv3P8qYZ5UkAjUSscBneT5cjrkLgH8TXVCp1ZhKPRHo8d1b2iCCL95IxBZ/4dwHYDAIQdO2T3qhctOEaRHBlYjLnnDnsM9wMf/WArnY7+eaQ28D7fMYGSXAUH0RM+vUnHA9K0SiX2YIpP9HgwksycBjnLqhPOexP4Cu7n542RycnK7syriG3SP5j8395uTnufc15pqlwI73Fqryt0B6HPTJPbjJ/ya9DvJDc3jwx/JDCPmxxgdhnt05PX8a5DULS3CtN8wIA3AdW54AHbJ4HevNqQtsd1OXcq21tGzxyzttDsPl7se+Py49K7CDEtxF+62KgyFyMAD6/nz+tYDJHBHFeSN1Unt8qjjI9uw9akhklv5TAquUfkID8xGeGkb37AY4rnirOzNZbXR6BaahExzF8+T26nsM+3oByTW5aFnm6jzOjNyQo/ugDjjv71iaXZRwqIY8s7cOy9AO6r6e7eldrDZBVESYVQOdvyjHYfT/Jr2cPSbSPLrTSbK8bLM0iL8sacHGAfx+vXAqLULiHyVVOpOPTKjsPTJ696aZ433eRjykOOOAxPue2OSaw55FlleVjlD93Ax8vTIz0BP51rUqcsbIyhG8tTA1V4pAZZ3AReWycgnoAo7+gH6VRikYgI0OE5Y7unH971Y+g4Axn0qldNLeXf7vJSM5G0cE9sZ7Due/auz03So8RrMTJO5Bcg52gdF9M55wOAeSeBXnUKbqTbR3VZKEbMLW2ubtWlkULsOQG6bj0z9OuO/5VYvENtZR6bacCZsyMerH39s8kfSt+YJH/AKPapiKBTubr+A9WJ/KsS1ia7nZn6IpQe2PmP54xXr+y5Vyrc81zvqyxDO8stv5fMaTMFPqF7/iRx7ViyxhHkicniMZP+0Tmunkt0hiiEY5JOPrjoKzpYftE8uOCXP5ADAoqQdrExkrmbJEZVLHlZFP1DoeP5Cs64UQ7ZVBO8duue9dTFbeQ2YxlT0/4FVDUIRDumCDyznPbafX25rKdPS5SnrY465nlhc3Fq5DKMj+nWoLq/e+QzqRGZjv44CydTx2B/wAamv2EM4mxjaMMOzDrnHt3/OsC4ge3dxDzDN8yZ6euP6VySbWhsknqWYzK0hM2cg9T1H410lq0oQA88ZU96xrCNpRvIw46+49a2yuyIkcKDhh02k8g/j2pQXUJvoZepzqkbMccjGCOMehHpn8q4WWQyP5bghTypB5HqPcV0mtXDSRnGHzwynr9RXFWbyhzESWGejc/lSm9QitDqrJNoC5BrooQF5/yawrRcDArWWbbjdwaIBJF54kxmPg+3Iqk8jxtsmOM9CKcJwDt6Z/Wonl3EJIA4PY8H8K6ImbJfMHlt5jK47FRhufb/A0unNEuY3zsY9eR+RHf+dZsUUTSbYZGX+8hOf068fpXqWheCZZ0inkJMbkE4O5WX1VgTyO4NdtGnKWxEpxivePojwEWj0aC3EizIqjDA54x9AcexGR06V6IvSuS8N6bFptqlvEchBXWr0r6ekrRSPn6rvJtC0UUVoZhRRRQAUUUUAf/0v1SooooAKKKKACiikBoAWuP8c+ILTw94fubq4+ZmQqijuTXXswRS7cAV8zfG7xC5gjsIuDJwq9Tjux9PYVz4qsqVKVR9DfD0nUqKCPle/vWvtTZiCzSMWx2/GuktYRFGlk5Yyzctk87f/risG2gS2mluJV8xoyAFyAAx5JJ9fQV0lnLI0MmoS8yHgDHc8KP6n2Ffms5OcnKW7PuklGKii5cOVjIh+XadqgYxuPH546VKZY7ezMikbgu1e+49yPWsFboGTaxJI4X8eCePUd6vrMbiRZHj3RxHaoz1yeB9KUZDcS3CFESyzrjI5HTgds+nrioBqDxXAeQAOwBjU9Qo7+gyeB6CqOp3bwAIG3zO3Poo9MdPpWVYXAuLmSRCCzMQCWyBt6k+yjj6/jTUtbAo6XZ6Bb7ERrq7b963zOSOQSOuAfThRV1bNpJFkkG6QgBIVzu/wCBY6Y9P61zum3KvO00LkBeQzHnPeQj/wBB9Pwrozq8EMWzb5UZ+bv5sg6DOPmC4+n866lKLV5GElJP3SebThsHmhccYUHj6DHpUUejIR5r7trc7Y1GCPqThRU9jN9pIk28cYQjk46cdABVu4LqdxOTnOCcn8ugqnGDXNYjmkna5QQyW6F44gFPH49ueM/yrPngvbxyZHKqP7uefx/wrWkkiEiiSUPJgZI6Y9qlN1byhlWViEGG24GP8B+prJxUnZsvma1SObXSZPMLLgN0LHnr6ZqnLoJdjufdjjPH6Zrt0lswoQLkAHvlmPqT6UoOX3eXz6noPYCl9Xi9Lh7aSOEHhtAn15J/+sOtL/Y5iUbIzx0yelejfZlnUeccBfbA/Cs+fS7EhvOTCg59M+9a/Uo9CfrT6nGRpd2oJUtGx67QAv4kDJ+lVgss5XzXL5YZLqVAHbHYnPrXWtZW8oZIfMZvQE4A9SenFZz20w3IqkMMfNjOMdBik6DiwVVMzn0mVGUQRkhSxZskHnvuPQU/+04tPgisYm8pYyQi4LHdwSxx3Azj3xWluug3kD7pIXc/AAbsqKD1NU7y2iuVEafM+ck4HY8DA6fnWi93WJN76SKxlW30/ZCAXdsbm5+Y9OnGQOe+OvWuei3tFGJFLIGZ+vzOegwPX+VQ3FzexzC0SMCLIVnXcwAz8xHHXAwoHU1Umu3+3FrYbCSI1B/hxyQR2AHU+2KG+axSVjoBp8uppL5qjzdyqFAG2MDgBccEgcDsK6W3sY7WKO0tkBMjckHJb3dvT2Hauc8OXUqyRqFIVizru5Zs8bz9e34V391IplisrfYspA3yDog7gDueM9h+FKnTi9QnN7GzZJDFEUc7iv3seg6DjufTsKszySyRqk7bIyMt2P0z7CoIhBBGJERiCCY0zlmJPBOOme1QX5nRlg489wBs7IPc/TrmvSu1A4GryIrm481fKhjLRqAAq9y3RfbPc9cVjaiHjUxjEspOCR9xpO4x/dQfyrbdks7ZDGSTkhSerE/eb8uB6CuV1KVijEfLGqdfb09+etcuKlaGptQjeWhgx7llJuCBEhySvGSPQ9z9Onau2sbmQRBmxHkfKO/1C+gH4k9eK8ut7xZ7wBBuMXJzn5fy6cf4D1rq4dR8sfZ45A8xzJIzcBF6gsew7geg+grDB1LG2Jhc7WWZrXTyZhzKScHkhVBbn+Z9Sa1PD9u72yPMMNMCdvoME/yxXF6fcvqzbiT5bncS393+EfiFyf1rv0nENr9oj/hXao75cZz+Ve1QkpPm6I8yrFpcvUbcGNntnyAu4n8zVGQpbt5yjgtnPoR/+qlmZWmhgUfL8uP6fzP5VUtJftAls5fvoSV98cEVUppuxCjoTSqYbh4cjZIPkJ6YbkfUetYlxdXMRkDp8ycbG7/Q/wD6waS8uWMaR5wQTtPoR1H9RWZqd2H09jKN8ZXa/OGVv4WH8vbFc8pp3sWo9zzjVtagmmjaNGhkUlW2nKnHRh+tTWd0JgIpeVPb0PqK52QR3N0ZRw/cdN3v9a6Cxg2YGMgH6GvNcm2dVtDrbFDAB3XPB9P/AK1WNQmWPaR/GNp9CBUFozRxAxncoHQ1k3crNI0XQKNyfzre9omdrsxNScncrdV4NY9rEWYMeoOD/wDXrQuHdi0q8n+YH/1qkgVPMZk4Dc+3NYPUtFiNzEwJFX1kBXL4OOtV3RWDoOoPT69qzpJ2jYSA4I59iKa0HY2WMU0nkhgkn8Of4vb6+lZs9zNA54wO6E4H1B9ayL+SObKplNp+Rwc7f9k+3pTvOmv4fs10w81R8r98/X+frWkX0FY39Cv2utURJEEjA4+YDdt/r+dfc3hPQkt9KhkUKCR8wA/mPUfrX50+EtYu9M8SkyRf8e55XOAR6iv0T8DeJpta02KR4Aq4HK8fp0/I19Fljjszysw5t0dlBbiJiq1oAYpAMEn1p1e2eS2FFFFAgooooAKKKKAP/9P9UqKKKACiiigBGPFNHWhjTlFAFTULiK1s5bmf7kalj+FfAfxB1+41vXLnUOQM7UGOg/zivrz4j67JpujTlFAjVSCzdCxHAA7/AMq+E7ktcSl5mxvJdmPfvj2zXzPEeJtTjRXU9/JKF5Oo+gqw5hiidtrOMtn8yfqT1rppIoY7WO3jOcfNj+p/nXKWTrfzmXd/rMqMDoAeceg4/Oupjie8uSsYIjVQFzycHrmvkI7n0kjHthIDNNcdByxPp0wAK1EnIt/MIB8z5gFHGOgAH8hVqdEZhBHnyVJLseASOcn8eB+PepX/ANTlRsVcDn/a749T2x0qoxtcTdzgNZuvIjKkkySHaxHUdiR79lH41zkd7JEzR2KABUAOeTn+FeOvqavakBJfCAne3IO7ooHr7mrOh6VJcRrcyKdhO5COM9uAP0NZxXY2ubelx3MURMjkAY3HoS56jP49AOK63TolmmI2GSXILHsMdMnoFHpVI6ftQRyEb8HGScIO4Udz6n14rrLC2ihtltoSBHgOzYyW/DuPTPGfWrhGTlqZTkkjSW8jgVg77io+b+n0HoPSueu9QklJEC4X6457kkce2K0roQrEIU+VTz6fUn1J7+9ZBga6JAUlegA7+vtgdzWlWcn7qM6cVuyk05RfKU5ycsRxk+gI5xWna28skQByijBA7fUg4yfrVq1sUjOM8+wz+H/6q3I4fL+ZkIxjGRz/APrpQot6sc6iWiKkVsY4yxOB6nqfb/69W4ZCq4AOTwOwH+NE2xzhOq+nAAqJCm7BO4jr/hW0dHoYvUuu0zjKybE/U/5/KrUWnGch5GBA654Az+pqmG2MGAJY++MVKsxLbd+eeABwP6mumFRX94wlHTQ6SEoV8lACv5Dj0FRXEAkYE4RU6E+vr/hVOKT5sSuNvQ9s4qF7/wA9ysAwpH1IA7D0r0FVi42Zy+zaehlT6Wwn3RYJY/w/5/Wsm6W1gdllQggHaT+Wf144rsZJP3TDbhjkj1wB/WqqRxzkRLErepPOAeT+XGKwlRjfQ0VR9ThNQ0aC6wtlGAoYAsCfu9jx3rzoadJa3l0LxgLaTo7nAVSfmUD3zxyc17y6RJI8cciqqK25R2BzzXEa54dnvLYSIm5guQSMncjBh+n86xqUuXVG0Kt9Gc7JNNFOXTFvEoT5h2UjhB/ujH8q6rSLy3c7xhIA+WZj1J4C89fXr/SvOru+MlmIMM8rMXynOEGOv+8x7elWNLmQyImoEiPAEcWcu5YjJI5CjsT36etYuVp3Rvy3jZntWj3ltfTztYOZQjFTKB1YcMFP91emRgZ4FaEsSyyGJCFXq5B4AHPJ9+pH59qxoLmNbFbS22oCOFU4yM47D1zjP19hkzahJPdCyiICAEyHJ2qB1ye+P58V2SrxjFI5FScm2X729E06wQN8ozlzxtUdcfU8Vw+t3yysIYc+VnCjoBjoW9enAqa81iK2t5mgDFjlVz3AHAHuep9OlcRrcsz2kUMrkbAGnKnB+b7qg/3mP5CvMr1uY7qVHlFOrKqvFZjIQk7hwN36Akfz+lXtNV54I7dyFRiZJjnmSRui+4UVRtod0Rto1WNYiEwvKhiMke+3ufU11EFvEsas/KxDcwAxx6E+/fHPaoptp2RU0rHciBEMemxkDKBS3f7o3tj36fQe9aGoawbi8MNqpCBnIx3JwB+HT8qxLWC5WNr+5Y+Y4YDt15Yn+Q9qktrabyludvQH8yf/AK9eoqsrWR57gt2WbzUJDMm07fLVSSPVEIX8Oc1lSX06FpEciUNuz7nn+f8AOp7q2ky8bckjae3+eBVDymhILkHcoGPXFQ5yb1DlikE96lyzGI/J98Z6gE8j/gJzVC8d7m1fZwcEHHPI7gdx6j8RWNcOLaUO3yIw+9z8pJ4P09aItQeIEgg4POP7w/kf0PWlGbb1FKPY5xICH3jBxzxzkV1dlh0GOGXg9/pTI7RZibi3GAxJxVu3tzu2jgiklqSy8zeUmE439u2fT/CsyYZlGeCBx/WtOeMyIEPDH+fY/j0qjcEsgkxyMcfzB/nWrRCOWlO07Dwe2aWBieP9lf1p2oKDMjg/5P8A+qktmDsX/hxj8qytqWWLmXaflP8ArBwffNY4Du6xn3GPUf8A1jVlFdkIl5A+b/HFakMAlf5QAcbh9euKSTZT0MaO0mD84PTPoR2NWLu0RLcSxg/StlDGAOMFePwP/wBeq13cCNmtZ0PI3Kw/iH+I/wA8dOmEEZOTNfwNo2hajchtYZombC+aB93/AHh1wPUV9oeA9Ht9EtPs1vOky5O0owKsp7qf5j/J+LPB9z511EI1WYCTbhiVDc9MjkV91eG7e1+zo8ds9szANgsHDAjqGHDfU8172Weh5ePVjsKKKDXtHlBRRRQAUUUUAFFFFAH/1P1SooooAKKKRulADepokcRxliQMetA61nazcw2mmz3NwcJGhY/hQB8sfHDxKtxcxaLasWSP53x3avAEUqgEgzJcj7oH3U/yK6fX74axqt3qdwCY1PA/kBXOQCbzzdHHI9Op6Ko9Bnp9K/Os1xHtcTKXQ+3y+j7OhFGxYW+LtbdYx8g6DvxhF/mTXaW1ulvAYBnzJMea/wDtHog9gOTWPoURt4nuTlpHyqnHfH7x/wAOg/GtdrlhIVgA4BBPYZ4yPUnkVz0kkrm1Rtuxk3SJ5qW0RO0nnA6he/PesjV74RRLFFIcDcR6n1Iz+Wfritq53NKQv7xlBXP19+1cFrd1H9tdVIY7tvHQ4HIA9BWNR20RrBXMOy05pJXku1BkuvmEZzlI+o3ehbqc9uBXfWhg3xiEnCDluhY+w7e3tWPpdheToYSfKeY5dtvJ/wBlc8n/AD0FdjpdrbQ5SD5mBxnGcEck++PyFEU3sOTSJ4bQs3mzjDSkKiDrtHQAdh+vU1p3VzHZps+8R1UdiOmfp6dqfFGE3SgfP15OD+J7fQVi6g7b2CAEk4+X29Pxq5e6tDJe8yhPeTSuEBAZhyT0UZ+vX2qytyY1WFgfQK3U+5/wxxWN9naBjO5HHOSen0q1YnNwrrxu6seW5+vbFc0Zu50OKsd7YlEtw9y+1nPReuB0A9zVq4AkO6X5c/dj74qtYlDGsyDcxOF3D5j6YXGPp+dX5kkiUqyDPQ5OT9M/zr1knyHnS0kY7s54j6evqfQChZRGmFGB6+tPkBdhHH7ZJ44FU36bj6kKP6/4Vxu61OhW6lgSbQ2OWb0q35626iPP7wjn2J/rWQd0SbjjceFBqZESNA8rbmyBwO5pRmwlFG9CiSMN5zxwKv2sI3FkAAXv24rm7OcCTk4znH4delasWoSSAbCQo6HpwOB+td1KtHqctSm+hvPOgBG0MxHf17VnX5aLMYf92U+nzGoAftBIAJQfxHpjBoS3eV1edwwGTj1Ax1z7V2+0ckc3Kosx/KDXe5pAEAAAJxuwOGY/WnfbJHstlxgSqVQ4J5LHOR7mr93oyyFQ2c4BJHBwRg5rNNoUJWQgYI9s5xjH5Vg+aLtY1XLI831iyXQr5r12xaPFkgdAOAo498VkgC0DTR7TNIeTzwW447ZwDj0HNem61YJqGjvaFR8q7CMY3Acj8f8ACvIJ/NtYorS5x52ZMMBk+ikL7D1rCcUtEdEHfU37XUVjk8iKUStn5VUZJxxyegH9B787j3EvkmHG9nwrbBkfTPfHc965XQ7SVkE+RHaxAhd5DM79C8jdCeuFHH8q9MtraMoiMegySRt7dh1+prn9jNo1dWKZwcsM0b+dMRhMhUUZycdPw71lPYzsRdTryzEoDyTI3G4+/Yeg4HevVjZRS/NCMjGN2OMe3rVOfQpkjLFCG24HGCq+3pn86x+r1FsjT28Op5g0Lxtb2Q5yT8o4475I7nufc4r0HR9PkvJYlnQCPjCjhQByT+Ap9hoMKzSTy/My8N7Z6IPc8V3EViYYCqDEknyAfTk/0FdmEw0pO7OfEV0lZBfwRuqwxfdC/Me2Rycew7/lWi1uFjgt0HBwcd8Djn8qgMsVvvR13BEA/wCAxjcx/ElR+NWbe6Zrwsw+baFHPfODj6Yr2FCN/U8tydjLu7dvNZh/ECx+vIrm7vDiMY68DHr0/mK6x5fJn8sgkjDHPp1rKvrdCpEYz8hcD15OcfhWU6Sd2hxn3OPmtlcESL8wyCvqO+Pf2rn7vS2RC9t8w9OzL2x3BHpXYSkHJYb84IYDJ9v/ANVZ5mSTfAevUj09x6g/5wa5/Zo1U2Z+hygSfZz/AMtPuj1Pp9cVrPb7H82M4Knn6HkH8e9c0++OQoGz/FGeuGHI59D0rSOpO8YnJ2sPvZ9Cev4HqPxoTVrMT3L8rkuCOGH6H/A1i3M/Lc8Nkj6jtVma7jJBGAcfl7fT0/Cudvb2JQN3RiPwPQUSYJFG6nLE5OM4wfekhBWBgeC3A+orKabzVz/zzYD+ecVsRI0jR56qcmsupZrQRhoVRhknPP1x/wDXqeGF45M+nf2/+tToomyFPp+grQWMnlumc/gf/r1rFENlN4OTkc8gjuQf88Vy+qNJcRfZN33TlW7o319DXYyFS2yU4I6H+VcpfSMJf3i/d7dCD9R69a1WhKO4+HPh5pZ4vnyCw8zBw3JxuGfQ1956ZaNb2kQfAkAAfHCsRxux2J618b/DaNZdVt5I8ZXGVzgsp64zkfnX2nbkeUAoxjseD/h+XFfQ5ZFcjZ5OPleSRaooor1DzgooooAKKKKACiiigD//1f1SooooAKa1OqNjigBVry34s6kbTw3JCr7d/wB76elemeaFPNfOHxv1pY5YNNTJ3fMw9SOg/OubF1VToym+iOjDUnOrGJ82XrLFahpu5yV/ln6020tJruQZ+RV5yOxx7d8cDjvUl6pMyyuu5YiT9ZO/1A6fWtqytjp8KichpZmDKvXaO5/E1+cW5pOTPuE7KyNgZtLYIgZndR8oGDtzwo9Af5CpIEJCgDcQTuYcAtj+QGcVn3jzF5JlkDPgxR9ueN7/AJYUfjUjyLZ2oRGX92m12PAyBliPT0rRPqRYjuZwd0UJ2b3JAUfdwMDP071yY01BcmVySVwoJ4GO/PqOp9zir0+rKEzD94puOATjJ+Ue+euPSs8yz3UYe4U4TGyMDGT6n6Ht6msZtM1imbqeSQtvb7meZcE8Agdznn0/zjFdbCIkUW1qqrFEAmTnk9cDucck1wltJ9mDK0oaSTLOeNoHufQen5da6Oyu/wB0FjAXJyvHJB6sQO59KuE1YicOp0O8sCWIQDkcdDjjP8+KpvblV84DzC4+UYP4E/XtWdbyqeJd0yL8xAOFwegz/k10clzGYV8wqCc/KDwMDoT7Dr+VaxSktTJ3i9DnWsHZA0sak9gemc8fh/OpLawCEzNtIHB9Sc9O9WVmW4YzEYEhwD329MDHTPPPoPerUssCxRlHWIbQFwOBk4GB7cmpVOD94tzlsdVpMcfmLIx+aMdM8KB157k9z/kak8KyR4bCE84/ur/PmuUg1aKGF1tvljiIUEkZwo5Y+/p2Ga0ba6N9cKASFzudz1OACT7DsK9anOHLyI8+cJc3MTtpayfu0b5EGW9ye1Z02nNGPMc4RM8e/oK7GEbVEaqOcE47ZOcn8BWTdSjUJQkQCwofvew6nHrj+dOrhoJX6ip1pX8jh5IHch8cnp1wBnNNEZyFPPPH19a66SwR8sB+7Xt3JHr7VDLYpaRtLJ/rCceyj0zXnSwclqzrWIWxgGEqSQuSwx6YHens+0NHG27nBwMDP+f5VEzPOSqk89f6VdtrdjKI4gPlwST9ayjBt2RcpJK7LcIkDCMErgct3+gHpW5a+Woy3GB37/Wqa2uZjKzcbSP8/ia0444wxV+COT+HavUowaZw1JJof5jStuT5STg44I7/AKVz+owyRglQMFsgZ5B649xxW3KpWPeM5OOB6nGDVe+t4zYEXIwyYJ+jZH4VvUp8yaMoS5WjmHuEW1klJwyuFH+6QCD+GMVwnibT1EX2pc/JkZHdTg9uvX8q7bUGLjyOC23A/wBrB7/571QWGC8hudNuh86qpQ9zjKn8wRXnSTb5WdkXZcyPI4NQNrMlywVFt8qC3KoT3wOrt2A5xxnrXoVhqH2pVS6J2cM+eNxPQE9z6gdK8e8W2N/b3EUVuwTYSTv+6h9WxyQPTvx2rf8ADlwCiKztPLINxd/vBPUjsXP3VHb2rTVpWG0e7JqsEAzGQWwNuP0/D6VYmuWkQBHBduASc5J5Y4HpXnXnLBliDJK5x16exP8AMCqr69NajzUBeRfugD5VHrz3z3/KpliraMlUL6o9TR7ezjijRsuCWJb+AEZLn/aPX8aR7pVn8tDtSJfmz2UcufYlj/TtXlMHieKLcZ8mbGVj64zjBOe5PPPQVvpeieAWfnAIcNdSk5yV5Kj2HT65raGKi1ZETw7W51QYyBrqbHztmT02g7yo+p2j8K0o5cGKUjJjUZ+pJP8AhXKSa9DqV0NK0seYsP3s/wAUr4x+XP8AnNb9oXkgSNAT5khYt1yFyB/jW8JXehhOLS1NS5KedyMZKgk+hGf0zWZcoY5sDgxybfbBHFbc6rNeKWHyuCg+vT+YrOuCJIVlcfNIgV/95Bj+VdM1uc6exzE8Ij+UDKc8dx3x/hXF38JQfaICQU+brge/Pv8Al616DOhyDnjgEj1I+U/Q/oa4rVphA8jJ93B3pjt3IH8//wBVclSJtFmHJcIZME4zjGe27kfqKsK6Sgg9R1Hsev5Hke1cqshMnkEjHVGHIx1GPyyPTpWhBLnK9COQfT/61c9zSw27n8oBQ3Tgf0/UVz1/c+bIsYOQB29+RVvVZxl4xwd2Qaw4w9xOO2cD6ev61LRSNbTY2kjUn+I5z68YrsdPt/lMjDAJOKyra18lAFH3eAPb1rp7R9sXIx0wPr/k1cYgy2kONzegx+ozVmJM/Kf4ePwIp7KI4zu7/KfqcVWeYrIhBxuG38eK12M3qMlEcqmMj5gOPUjv+INcTqZlVVdPni6Er1A9/ofXpWzq98bZlnIJVz1U8g9Dj36GuR1CacS742EiMM8HHXr/APXHak5Iaiz6S+DEcFxcrJIvmIpAPGduf85/CvsSNdqhc5wK+RPgNbTQ3IkliYRzr8rg8Z/usOhB7e/evr4da+oy5fujwsa/3gtFFFd5xhRRRQAUUUUAFFFFAH//1v1SooooADVdzU5qEjNA0ZdxvZ1Re5r43+KmsR3/AI0mgR/N8giIEdAV6/lX1b4t1OTSdFvLyDiRUwh9Ce/4V8BJO02ozTM3JDZPU5bj86+fz+u4UVTXU9nJ6PNUc30NCRfNkXeu1FywB646rn8OTSw3qvMX+8ygE55HOcA/Tn8K5q7v55JHtI2O5s7sY4wevtj+ddlpmneXZeeY/nf5Uzxub/ADGfavjI3ex9Q7Jal6EhVE7jDRgYyPXkAe5PJ9AK5bVbzzSLbAKjkg5IznofXHX3OK6Zl224jwJJDwuf4mPBOPaqC2dnbbkuvmnds4HJwOQCR+Zq2m0kiU9blS0s5TEZ2By7ZCnP3iOB9AP1rOv1WAFbYF5B8u/OAMdST6gH+g55rsod08YSMgZByx45I5x7Ac5qlcWaNtULsRULAE9FB6kepNTKnpoNTszl4wtvbZmG/glFHRmHTPrjqauxFzmEFpJXKhznk5PCj69+nFMnidlyvWTAGP4RjilguoYsKAV3kksOCxOAcH0CjH8qiNky3qdAk5EaRbRtJ654wByc9TjoAKtyzqFZZBufLAY6Kq+g9s/ia5ZtR37REdqYU7sYAH8KqPp1PvV6G5nM0ixLltoLueQik7jge3Sqc29CeWw+e9uUCbAfmHyp/EAeMnPt+VQG9fzA8XzM0h2kDCJhQAefQVQvP9FtVSQtvmQ9OXIJzj9Rknr79KzL17vIiGUjVAu0DgEnJ57+5qLtbl2TO1iuUYQ2tucKr7mbrnnr79sev0rrdN1KPZ9jgP7mMEysx+ZznJGe1eQWlvfMrtDnfIoHmMcKoPUKOOg/8Ar12emxTpbpbI4CKRtVRwTnr79uTXRQrO+hlVpq2p6jLqEnklZHw8oG9h2LZyB7jgVLYy29vZM8SgeXuCb/4mPVm9h2HfFc1FbyyxI7El1J69Pr/npU5LyARLl9pCgLzluw/+vXpRryvexwunG1rm2dVAgKQ5+UHDHqWbqT7/AKCsvUZ3bZCSBtUFs+p/r/KoZoHZ3VTtWP7xzxgcY/E1QuJvPkB24Qc89x7moq1ZWsx04RvdGnaxtcr56oEjOFjz0LtnH1wAWPtVm2nt8stq26OIDLf32PTn3PSsf+0Zm0/ywuC25Y/UAjBP5VNprpZwurbdiOrMW/vLyB74ohUimrDlB2dzq/kWF5CQMFQCe+DgHHu2aQSqzrDHglmBPPY8/wCNcuL1JLcRS5QyEM5b72wZIH/1qjtb9o5zJ/E4KAemR8v6Vr9ZjdGToM6uO8jmSKVcETHBHTA6kn+VPupeMSYPmfkMDIH5VhCSFYWOPlAYD8v61RlvLweTIpIiUsT65/yK2WISWpk6V3oRXsc0Eu9VLRIRx3Hf/P5Vm38VygF3Zt+8jZWP0U8HHt0b6V11xJa3Ba5YABMCVewyN2f89Olc7KS6O8BIO3I56qzdPwrKtTT2ZpTk0efeMLCPUtLuNVRAxAzg9QU7H8q818Ms0ql4GZWlOXkPDY9F/uqB+J4r2i4T7MZCU/dy5WQEZAz3x6fyzXiWtTyaTqbadFE1tEAMyDI75AReS7H1GAPrWVJ3TizaXdHoj3iRx+VbsoYfLz1H19D/AJNYt7LcSkQQt5ca4y2cM5Hv1IH5VQhmuZFVreHyxx97AWMe5/vevXmtm20ia5kLvk5PJOcfmf1NcVZNvQ3ptJamRFZzSzYtkDtGdxbGVBHTjufTPGeea3bK1ugGWbfkkHJP4hf6101paxwxbLYHpnIGBx6Z6D3x9K0RC0KeacZwQFXls92J7eg/WiNBilW7EGhWc1vJNLF8rSNhSDymeD9WOTn0H416PYSwwZYHcvRD7kH8MccewrzaCVYoWmd8x4UDacAAnoPx49WJrQ+3CEQ2ksoZl3ySspyAwjPyj/dzj8a9HDT5EclaLmeiSt/oyyOCHiVX/Niao3bK0M0AyGQhk+vOf0qpLdC7W9dCVjG2NQfSMDJ/GoNUulY+dGflkAHP/XPNd0pqxx8juUZGDgM5IV8qw9GXn/64rjNfEnAb/WgbkYcZ9wR69CP/ANVdW10LmzMK/wCtfBH++oyP++hkfUVyl5cQzRspJ2scjjO1uh4/mP61zzaaNIo8zllEExEiFCT0x0J9D0wfb8qu292q+YSe24Z/M1X1RiZTC2OvI7Y9s9K566uGtpvLz/D09ewP58VzpGhqXLreSbx0OVP+NadhbgOZX4xyfqOlZFgFReeg5APqa0hOq/KxwOp9lHrVD1OqjnTaMfxevpV23uVkdN3CsR+AriTqG8sxyEXgDuSeP0FbunSljHn+EZP480ORXKb818JVC55wx/HNUJ5/PdihwGyR7EHP9aqqGwhHpn6nNV7wrBJlchTnGOxHFZOoWoGbfXsnzWspyj/MCPUe/oR+VZxleaIwyY3fwtjg46Ejsfcfypt8sm5ZegHHHTP9Pas64dIxGYiY2yMAZ9eeO2etFN800E1aJ+g/wg0K40zw/byMNpZRuRuV5AIIPb9a9wHIzjFcF8OruSfwrYNKyyYjUFl6jA/iHY+9d9X3GHilTSifKV23N3CkHNKaQdK3MRaKKKACiiigAooooA//1/1SooooAYTTadjNBGFOaBnhHxx1VrDw21rG21ZDlznknso/rXxg1xJp2mGcAfabkkrnny1PAbHrjkfnXvHx/wBbe51+00KIfu4hu25+8fc9h6+1fO9zcJNflpXLwqAWPYkfyB9PQV8Zn9bmrqC6H1GUU+WlzPqWNDtna5UPgSSHccjJUfwj6mvVpZ4ra38lGBIHl8/Tnp6dSfavPNH8yBmvSAGZSUyfuqemffHPtx3rRaWXy9jjaihVYA9R94qPrkZrwVLl2PXkr7kt3rEdtjyk3u5CRgnHA64+p/rUOnXLXJd3beSRuI/vMcsfZcDA9etcld3b3Fw/lgliAcAYAXoAD2J6k9667RrVoIIpWyUDBpDxyegA9R247A0J9ymtDtlnWO3DOAmW4HUknGc5/ICqJLCRpJyP3nLE/wAIA2qPzJqja3cd1qNsX/gaSVh14Rfkz+eT7mpZbuGByZEMiQhQqn+OTgj68nJHtitr30MbWZBNZgyuu0tsBA4+9k9Pof5CsR7VWlCA7jHkEnnp1zjgD2rqW/132CQ77mdcORxhn5cY68dKzlsJX85IGChyVD44GTxgdyOBzyfpWM6ZpGZk29vPdTLDChRCxZAerEdyccDp+H1ropWEDG3hbzJGHmSuOgA4AA9BwAO55NV5JIrSMx2/II2MSeXIH3c9vc9u3NWtNUCaTzHBlnG6RgOEQcAD0AHIpwp23FKdzPkjedTPtJ2ZIz94tnAH5+tNi0+UDCvkKQgZuckY3EDpgdB6mtuCW1mtmMI2RLn8FDf+hEn8z7VSM8mTg4YEAADhck4H5kAD2q3TiSpsox2ksl03XyIMrnA+YgZIHocnoK6uGOKxRFkLNK6gOOm0HovsO57msxw1nIHK7zDtBzzl3bKqPXnk464x0rMfVLmVnhQkNkK5OSSxyTz64H5mhOMA1kdJcalIVkMjeWnAIGOFHQe5YkYH+TswXxskG3AZQ2B3GAc8+p9f6V51FIzOrzsqlWDhScgFeh+gPJ/AVblvELRyGTbGpwQeyjk5xzlj1+oHrVQxLV2EqCeh3MlysNpGHbLSHlAPvORz+Cj+dULmQed5e8DqpOeODg49TngVzBv7ieRro5VdpEaD7/P8R9CevsBk9qt2qp59vNP8xR0AXsMf0HNN17oSpWN+2iCiTJ4QYG7JPvj/ABNVpZCw8vGQ5+cduPesyG8ODGxOWlHfAJAbOfpnP1+lS3NwnmLGMhQ29l/3xgZ/LJoc48ocruXXOZhcynIGFC98dPwFIkZNyEJ+UEAEeqgg/lUNu+yzM7H5o5CeehUghj+HatGOMH95H0jYkZ913Y+uRinFJ7EyYs1yY4ljGcyHn2CdMfzp5laSM4AwOWHvinSRq8zvngEY/wCBDPH65qjbPslltmPB4U/7eOv41V3ciysXfOW5SYKdjgE89c/56VTW6UwmL7syKQB2f/8AXUrnDjnG9Dg/SsogFjGemQfp7/nWilJbk2RqMEv7YkZLgc+vHr+HGa8/8a6Y13owvLSXyZrc7fMx8wU8YOf1/PNdUolsbhLtXG09cnGB6Z9+2atFEvFeaDarn5Xzna6443DkZ9+4rSMk/Uhq3oeGafC/2mKNZ2YRc4GQDjqc98nqRgdsmvRbMRnCTO0jAdAfkHru5/MZNec3Wl2WjavNEYZFnkbLLywUH/aJwAe2P1rsLWURJm7YliMKi4VVH48/pTlGzC522AY9u9lV+cZAaQ9hx0H41Vu7hncW1viOCPgkDqB/CvesOPUoU2oWwyjc5BJ2jsP/AK351iXmswxusikbE5AX7qqPfuSetZzmkrFRgy5q0txHGJYcq8cilVb7oK/dYj/YHI98Vk+HtQNvPDDeP5gVmMpP3cswO0Hjg9D7VQuNUdylpKTuciRyTngckD3HT/8AVWZO7xQqIVyzfIEPUmQ55H0JJ/AVjzO90bJaWZ6dp/iR5ILq5ly8jAr04LsQ5OPTofYGrn9ryTW8QZtwU5/EI2P0rzlp0hEwictlGjQ/3pSBuY/ifyGK07KVktWZmwscarj/AGgME/ka0+sPYh0kdQ99LHiROsWPxKc/qKx9UmKyvJCf9YN4HZhjkj39aoRXLbd8bff+cDryDz+YqK7ZY4yuSsed6Ec7fb6A1SqGbgjnLu9hkJD5BHKnqMHg/ke1cfLdmW/aOXrtx7HHU1b8QSzKJHYbCw+cDoT2YYrjtOvZbs+Y3EjfLz2rshHS5zdT022n/dCRuW71HLcMW2k4Xgn+n/1vzqDTYfNGF5X7o98dfzNbz6aZVJA/z3/wFZTvc2jZGXAXYgyYC9cfXn+QA/Gu0sl+cjuFBP1PJrn4rQ+YiuMAf0rq7GLAkbB52/zrJ6lstCFsKf7vFQ3dr5pKjvmumt7UMWDAY2qM0klgwY8dKHB2I5zj/sBeIxuu4Dhx3x2P/wBeud1GxEclq0fAjlQliOi56n1HrXpM8ZjKug57H19jXMaxcwWl9YBEyss444wMnpzXRh6dpJmNSbaZ+jXhSwt7fQrLywmREuGjPHIz19PY11NcF4DETaLH5TEhAFwccDqOhI6V3or7SlblVj5ip8TGsT0FOoorQgKKKKACiiigAooooA//0P1SooooAKrXtzFZ2kt3OwVIlLMT6CrNeHfHbxK2g+EpESXy2lBAA6k/0AqZyUYuTKhFykoo+LPiB4nl8QeNLzUpn3BiwXB6L0/lXNCdfKjXIDO2QB2Lcc/hz9K5eH7XfTlpPkErYLY6YOSf5VqySg3CxKPLCtgAd8cD8z/KvzvG1Pa1XM+3w1P2cFHsdwt0rWcbSchpMYJ5PPAHtxk+1Xp2e33SSkZAO845LuATgewrm7RlkuolBIitw7HsBjj+Z/Gtn7WWkJRdxDbUB5O9gAM/Qcn0xXGnobPczIjbLcorxlMk8E/xejY9O/oSRW3ealIGt7aBsmRjyOFCgYBGPTOBWNFFEsvmyYZmZsY5J56H3ycmqLXGdTjjhXccmJD/ALowSO2F6D3peha8zqLS9RpN8Z2Ql3iQDkkABdxPoOg/+tViS8D3CPjAjcHHXBI4HPoozn1Nc8R9kkgQnc5IbA+6mMYB9uhNTQoZZhCuWHzscHnAGM4HVjk/SmmJnT6VcPMW1O5yxI+X9e/5kmtG71Ke1AS3GBgFmA6Z7L6nqSeg/DFc/fzG3tYLW3AUxKFYHkBiw446gAfia43Vtakk1BbSGT91GhLdySTySfp/PFXzdBKNzsre4tzcwrt3gMQuD0Q7ecnux78cD0FR3msPAt3HFzPP930G5tqL+eSfYCsDS5fs8YuDlpZZG4zyS+cZ9gqfrgVo6UJb7ULmbZgA/KW9Bk5/X9fak5N7D5UtzYsbk2emJYRnLtJEu7/ZQ8n2zg/hV571BOZICdgkaU9hgfKuffAP0HNcvtMiXH2diQHUE923nPHoNo/WtWOyjEH2QNmRzCq54DBgpkz9cEfQ1UW3uS0jsCyJvknJULjaO+SpJJ96IIIluIncAKwMxXA7jcP8D7D3FZ+s3cD3EqdVKsxxnB3BT27cgCsObUZJdSzHtaONAqbgeqoO2cdc03JISi2jqZLONIVuJvlllIYDoQvXHsCSST7H2rLOmGaRLdkCR56kYxkjHHrjn6mpkkuLx4xdZ2KPm2nGWxnH0HTHrV9HkjKPMm+UsehyqySE5/CNM+2feqVOMkS5NMY0UUe1DGM+WSMHnaMd+5b196iljmaIuiYwVDNnG3OTxUEerCa5DodxJK7iflCsQCQBjJABA56kmqM3iBHvI7RT5ah8nsAu04+uOT9TQ1DYacjTmVUEZdNhyQqevXk/XPI+lRsXkjjuJeXI+YdFGD3Hes3V7qe4vrKNCyjylL9j8z5IA/vNxk9hW/qEcEaLA65YF3wvOAoAUH1JPQfnWcqd22ilKyQi3kf2RpOsS4Ue5PX8wM1b0++Ml3LA3y7hyR/f9/xFYUpit7gaI0gNyQp25yQ56E+mM1GrmG4cSEBZcLn0JySfwI4+tJOSYNJo6+S8hFu0+eARj229f51VmlaLy7xR8n3sjoCvOP6VzsF2JdNadASjFgV65wo4/L+talpdAQC0mOBK7AkjI3EEqfbpg1spX0Zk422Ogu4ooo4LiEAoPMI9GEg4H4HisQykFH/iUAk9yp/qD1o0m5a7sFguQQS6oy9vnXg47YYD86vJFHNHhRtdc5/E4P5E10O72MdtGT200N7C0SqGJ4x0B9QQensfWube1ezm8y3YgfdZTkEew7Z9qttDd2su6Pg+nrVuJhcEiT/WgZ2n+MegPqO3ftzSTUtHoxNcu2xxXjDThdWa6js882ylgwGSnHJx3x6H8DXkdvrl5cqFt3Aj45U8uR3PevosSQpMQBtLDH1/x/KvDtc0K2sdZla2McHmsTtQYJJ7+34HHpWsk3GwoPXUgWSaaIi4Jbn5ix2k8dAvp7mpGQPAqHnd6Hnj+XoK0rLRmMYZRgL05wo/Crn9jR7PNlDbUOFXHX3PTiuKUWdKkjh2kEkzNy7kY+U8AAjgH6/nV5b+OCK6l3bp3JHm54jyAoI/vEjp+dbsujSCM8bd3IHfHb86ypdA3XG5iWwMhQO/r7AetKLtuW2mW7gRyXgt41YRp5b/APAiMgfkMn8K2XikjBDcJLMxOfRlP6c020sJ2EcZfJyWdsevYfQCusTTvtMoLDCLg8dBk4xWnI3sZOSRhw6a5IyeoxnoAcZ/nV6ezZ8LjDMMqT0yR0/GurjsBGqk87k/Vev6VNPYEjC4II4Hr36+vpW1OizCdU+bvExnsdyFcBgec8A/3SO49K4TSPJWbzZMglvlQd/9709h/KvUfilaLFZkyOsco5Qn7sgHbgEgjv8AmO9eF6bdtCFJ+8xyr5GMexGc16VOPuq5yN6n0Lo8qR4jkByOo7fSu5ilikjCxpg968l8LTs4RZTkHoB29zXtNhGsiK2PkH60vZofMyibMNMvAHety0s8ruA69PpViaBBKsan5sZY/wBK1LeE4GemKj2SuPnZPDbhlZh69ParMsX7vdjoeauQqqBeOox+dQyOTGSpAZB+YFU4pEXbOXvVEQJGWA53YyMe4/nXmHiKC7l1nTJ7Z2VFnTzI15Df3SOD/SvSr+5CHkFS2dp9SPf1rz6/uEn1WxQxNkyg+ZEMY543YOefp1opu0kOS0P0l8ESWtxodvcQxiJ2RQ6jPUDrzzg12Ncv4OkSbQLWZTksgz65xzn+ddRX11L4EfOVfjYUUUVoZhRRRQAUUUUAFFFFAH//0f1SooooAa7BELHsM1+d/wAfPGr6z4m/seBibe1yzH+83b8B2r7116RlsZDvEaKCXJ9B2/Gvyu8b39vqnjS68pi0e85b+8c/1P6V5Wb1HHDtLqellcFKsm+hWtXjVo++O/b1P+fetaK2SYhsDcRvGemeQCfYZJrAtmaadGYck8enP/1s/nWtMzQQvsJ3uNue+M8Zr4SW59d0LNsRHuQPu83aQehCjnt69fyqaCaSCeRo1JkT7gbqWfjPHYcmsn7SBckwHBRgp9SVHGcfhWhHIliJr24P7x2bap/uouM/QDJ+posIa17HbRzzZLCF1iHr0y38jUtlZi41Se43FYrUCNRkAAjGSxPH3j19qwriaV7a1wuQoEhUf3+ijj1Y5+lGrahJaeHfsdjjzJpI42fOSxzhj+DE496IrUpnS395A8qxW/zLwMjku7Y2j6AAsT34JqxozCzt7m/mOVJ8vP8As9Sq+pPy5+prjZ5yT5YYgRy43DgdAMZ/2VAHuTXQzSmVdO0kfKqymR1z1IYEZ+pIqoq7Jehtagq2oMpfbID82MnDP2HuAPwrlU0+NJmLg5UF29yx+X8B2H0rU1rUIr5pmVv3dqyN3wWlUkn1PyhQM+tPZXuEMeQqSgHI45Bwx/ADNEo9UCl0C3t33xkEFXyFIGAONoA9TnOPWuuFsILW98k9I/KTHdyCCfw5NILO2fYsK4SKSNFPbaqEc/nioZL623/Z5B5ccAaVx03YB3A9gNxHHfmjSIXbMuS9huJ7extcAuY2l54xtx19l/x71uW95A08N0wAYqJE5HAC4UfiMn8RXExNu1CW4ZQpO9FXP3UwFXOPXGSfatu5cLClwxEawZVyenGTgD1wAB+FLm0G4khujLumY7Q0m0n1C9APQZ4Fbnh2w22wuXjUyIcKDzlupGT2Ud65yG3SdYrdv3cUCeY2WIVe+W7kKo/Emuim8QiOx3WgECuQIh0YRbQWfnoTnj25NOnFfFIVR/ZRrX11FAtyI3KeWQpkB+bAGW2D1B4Huc1WkmuJfMikPlgKECJ/AhA+QHuxY/Mx5OKbZ28V3YwJL+8MjGZ3P8UmcKB7Dv7CtRrdY9vIbYBhuox90cepJY/lW129EZaI5q+W00q03R8MoDKo7/X/AD04rmWjVpGuA2S3B7EEjJ698D9a27yYz3QjgG5pSfmbnbjHUfSoW06C3gnZ2LrGwV27l3+Yge+MZ/KsOVLVGil3MqfU7hLqC8kQmQtGx9Bx0H4DjtyPaug0nUJFV4bt9kkUSPnOfnwDJnPHB4HvXOTW7yyPcXHQ48lV/vKDk+nB4H5+lIlsB5Ue3ejxvuYchUQbv1YEZPX8aIKSKk0zes76OTXQkC+ZcJvJc4JC8Pk565zxn0rVvpbWOeG25MxBkIY8krDkqT6gHn3xXE2X/Et1Ge7blzEIxz/G4VyB9F/QVorEi3UNyGIdrUIGPVWkYHP1OefYV0X01Rk0bWniWGB1lGYbeUpxxk4BYkezAj6U+Kdpnls3JDofNUDnKruHX8SPwqCG5iksntmf5ZSW2+uOSfrt5NZVrPLJJNqVk20uZRtA+7tdXAAPHBP61FlZMfkdlYmOVkkjfi4UMvruTr/MGti3nlR2kbknGfr/AHh9R1rjtKuoHjubVV+zyAm5jx90K+DuUdge4/hOexFbegXjanpUjZ8m4gBVx1B2nKn8q1jHXQznsdmMXcQnYDcMh1759R/Os69tCuJYmIOePcjn8DVmzmItwxG18fgf/wBVX5AwifzADk546Eeo967VTUlqcTk4vQ42ZmmUtt+Ykkr79x7eo965fVRb3CeckoRo+WDg7l57j+Jc/lXeS2yyszQkFiM4PGR7/wCIrzzxOEClpFMbjgNggq2O+Oo9uhqYxa3K5kyrbyoApeUbc8eWDz9STn9BWst5Z8yZ34Hc9fw615O+sXlmTuEaK5x5o5T8Sc/TAwfUVs6detOi7JlfPBb7mPrx+nWsZuxvGNz0QOJ5PkVt2MklflXtx6YqUWaRjZMfLZz90AE8ep/X2+tcsk7EeRJPtRiOWJDN745yPTNdRaW+3JHTuTyx/wD11EbMJJofaiOOdSq52ZyegAHXPv8A44roLBbeQyMeDOQMf3QoJ/PGM1h3UQRPIiOS3b174H45P4VBayXVrdbTwNxkYnqA/B/SrjNReqJcbrQ6qzaO4td+SBuyufQ/Kf51N5iMfIz85LbNw4Yjqvsc5xWBBcSLEtpzhXGD7nIH4etaDL5kDNKfldt4P91ud35EHPtW0ZpmU4nnnxJ0T+0NGuHKkoqksDnKH+9x1HqD9eor4itb6W31l7aXywFbaHGBwPoQCT6Yr9GLx5ZrZ4F+aZVwAeQw7g+4/UV8HeLdO0+TxBcRW8c1nJG53ROqmPIPODwQD2P54ruoy0szmmrHqnhi4UbdnzF+2c/mew9a9v0icuidwOfqa+bvC96LVAiHDHgk/McfU5zntXvmg3ieSpkbJb05x7Z7n1o6jO4Qt5gYnLHOT710toEMXv0rmYdsuGU4ArXtZdqbl9P51LdgsbZJdNydCeRVK6Yqu6Pl0HHoR6Gn7iqK/Qd/w71BLOjofLP7wZxnv7fUfrWMmXFHF6jPtaSMqXhk54PIPYj0I6VxOs6Y1xf2ctqFLxzKfnO0k8dD0z9RXpNyY5laSNVD56fwn1+lcRqXmXM0FjHIqM8ihCWCkMDkDnrmoop86KqS90/RvwI27w5a7i28IMhsZB+o7V2dcn4LSVfD9qLmIwzKihx68da6w9K+zpfAj5ip8TEXpS0i9KWtCAooooAKKKKACiiigD//0v1SpG4BpaZKcRsfagDzP4i6pBpugXV5csFijQliemPp61+XcV2dR1e+1PaUjlclR/dXsPqRX2F+0b4rH2KLw3A/+vbMgB6qvavk6CCOO1YLgs5BwPzr5fPcWm1RXQ+iyjDtRdRmxY+UybeFI79hnr/hRqEq/ZnmVwFR8KSeuByfoOT9MetYAmNqArPwxHf+f61jeLdX+x6dDZr/AKwhnbHbd/kV81GDbse42Wbe5bzY51J27Wlz0ycBUH5ksaveW9/eSRSPnYPKBz8oLkFm/wCAiuUtbnyYo5JWztGFGdwbYM8/ienrTrTVGEQUcuwO8f7Z469uo/KtHBvYEzck1YXV1JKj5t7dnkC57RINn0GR3681k2089wIbE8rGYArdw7LuJ/EmsaAtboUOcSsFf18tF3H8xXQI/l7Jx8rTGS4Y46fKRHx6LxT5bBc1kZoYnSX5lbcR6LtJbk9Mkgk4rW0S5lWGS8lCs6xnb2wQck+5JIA+lZkNutzbHByzxsVPqCQrZ9sc+2TV95Y7VBIr7k8oSMByuA44H5k1CXQG7m1Zxq9w8IPzvKGYdz5MeAM+gOfzrQkuI/N+xKd0ShwD2+UrnPqSSfzrk7bUthW5jYiV45WlPTglRx6HkfhWva34tra4uJSG8uRNgbsqgZJ/4ESSKUkxI27fxFbyaemXILR4QHuyEhuPdhj8a4uzu7u+vZI3XfHIC8/spwVUe5PGff2qt4gAg1RZIZBHErM0Yz0V+qnnqD1p+h3kSRXLMdspjLDjum5v6jFJx3aNIvS522k2UjsLu7df3yY+Xpk7l49Bnv8AX2rpY7M305R8GKCTpj5flznPv0Jz9KxdPu3SK0W4yrlFJ4GMKDhc9M8kn6VbbWo/3KwKWW4YLtA6+azck98459Mj1p8q6GbbbK8qxXLywqS8SmSMr1BaNVzkDk8uBj1qhNb3VxqP2Bn3HBaR8AgNIfkUZ6gAZIxjp2rZ+1W6Myq3loTJukxhvmIJxj3LN74FW9Gt5dUu/tAh8mDeuQ3+tcLwqk9FGAMgcnuaI67Dlpqy60674ULbQAwxn+HoB9TzUV/qM3mBYxhArEDudufm/M4H0rQFtCLvzcZRd2COnHDHPoBwPc+1VUEZtZdWnOEYlIwByw6AD25zTcXFXIvzM5SW7fSza7TlicMO5PBOT+B/OqMl9MYIrN5CEhLtJIOhd1JY/wDAASP96m3cjteR3Z5RSRtBzg5I/kKuNYpbWqxXvMjNkj1Zzu2j6/yqL2jc0S1MdtQeWZ7qJNi4XA7IsascDtxwPc1clvbZL6CwkO3fbFpMtwFt4m2qPqQXPviqN0qLOlrbHLSMrk9eFy+OPr0+lZ1xayPercnKmENCPcurE/zxTjUG4GnaXCXG++lUCV2dVJ7B0WIMeewz+Aqnf6s4v47ja3lWjxSbXPJA8xGBA7NkN+NIllOyRKhO0Bdy56lDtP4nqfY1A2npNfxyTSfJMvmFjx8q/L09F71cat9EQ4pFB/FCQa2t0TmKQSMmOhZB5YPPQFR9KnPihdH1mO0i3fZ7gtMJABx5o8wfz21Cnhh7drm0nTe9uzKjYyAh52t6DOSD6HNSajpyafLFJqMXnCHy4HXjIDLlGHr1I/Cr5r6JC0Oo03W7K/WKJphDdqdqnoCrLkf8B5/MYrttE3Iwa3O12jZSPcHkfUEdK8ts/DsDnYrgLKSIZN3GCM4PoD+hq7aX2seHrqO51COQAsY5uhxJ/C/HYjr7j3q42vczl2R9AWdzujVJUAdS2cdyP5VorNjCKdw5wO+Pb3x0riNJ1cX1tHcnBZgQ5H98DqPY10FvMkn7gkAj7p9h/hXdCRwzjqRTuPvRkr7dseo9K57VTbXlttvMusnAdOcHHRh/hWvcSbZN0q7gThsdj/eH+etcvqQ8qQ7SEL9Tj5W9MgfdPuKYkeXXOjQx3heznG5uCwfIdRx8w9e3OeamsNKAUmSYMp6BGJGT9BTdVmMV2Xb5MnP3TgH13Lg89/et7QrwzgsJlx36Fh+LDdz7VlVp31NqdRo3bLT0tYwxYFvX73+fwzXQ20VwVCdEXgdO/c4qK3MWP3Sl5D69B7k1pweY2CwVQOeTx/ke1ZRhYtyuI8cdpt/5aMwIZj1wOw9KR1jYvI+GG0Ej3IyFH0x+lWHg3ZZk+VcDP64xVJnYHzScOBwOuFPBz6kmnJISGK0RjmLEAKQAewIBA/Dv+FRpfNZzRxFt+5nkK9iQPnx+h965W7vCjpbnJVhk4PocAfTqPxqlJdSDVLZyfLDuQP8AeBOD7Dt9KwVW2xr7M7o6pDcRp5a4IAAPXcPT345XP0r5q+M3hZpL1dbsGVknBZl3gNuHXhsZHrglh6GvfNOCyItxJGTFgElf4SOuPp/KqXirRNK8S6TcaNqkRaN18yKZGwSQOGBHRh3HX0rtoVWndnLVh0R8leFjj5rpGAQ8Andk/wDAe1e9eHpnnkTA4GMA9AP8a8gtNAbRbtre6kynARgCwkH++OvuDzXqmhzC3ZUyO24qOnooz3rsbT2MD2O0mXZheff+ZrZicIMgZ6AVxtpcB1VTxz0Hb/69dJZ/vThTjAz+NZSZSRfutQjjDFThh0B6GuevNUB+eDIzw6N1Vh2+nv2rVurVbiPyzgcYFY8lgdrEjLgYPcNx0PofQ1xVJSudEErGVcamzqZmyG/hZeuff1/EVzEl8mrajaKYhcAyqSUHTB6Fex9D0zWpqsU9pCbiHIUY+Vxxj/636VzmhW0kvjnTryx4LOpZVOOfY/41eGm3NJk1orlbR+pngsxt4etXhkMibAATnI46EHuK6usbQhB/ZsMkKeWHUEj3x7cVsZ+bFfcQVoo+Un8TFHSiijIqyQooHSigAooooAKKKKAP/9P9Uqq3zbLOZ+mFJq1WRr84ttFvJyMhImOPXApMaPzF+L2pzXnjGbzmJCdMemf85rzqXUBaxGXglS2APU0eLtZOoeJry7LbwHIHcYFcZdXfnk28WcKPmPqc9q+Exq568mfZYVctGKNSKSS5ZJbhslwAR+n51y2uXLXV6sjnKOyoM9wvzH8OK1ZLtYrctn1P0APHNYcJM88UzkMbdtuMZAAUkk/57VlFa3NmTzXM6L9kVczMNgyfu8bifrmt4RrBG1wSSHDvyeCfvk/gM1gI8cc0d8hD7D8xJ4JJJJPtkfjXQRt5+nxrL/HCwHQ4dwsfH6k+1OWiBFZMXNlLPOf3jbYlHQfOWzx9F/Wutgt1ubSdVPyrAGX/AGlWMqB9c1h26w3tx5MOQqYkBAzngKMZ69atS6iIXuGhUqQse1en316H6ECsHd6Is14d9gUgZg32fCY/2W2hzVKZZJbK4tl272jZG6922L+ozS27mRBdDLSThiS2OpAP9PyrI1C6ngjZEHzMoyR3yc5/E0kndDNm0lVyyyybVYR4VV3Eq2CSf0/KrzHz0l05fnWVVQufXIzx3OTXPRSMs1kD8rZAYHsvPBx6L+tdfdQrarAbNcyXCvlge4zhuenzHFTJPoVoUbqKTUYmIO8tIERvVSwUNjr1zmrUq+Vq0kKDYrqYuw+WX5RjHfpVmWwEbwshKqFyP++vu/gKzrmVxdy3yHcETeOORtbYmD2wcEDvQrX0Jvobeoa2I18lcERyLAORypIDMPrnAo0q9AS1vJG3C3QsPQswOP1ArktWjW3iBcAtGsarjHDjPOP724856YrZhmWGCJZ0VvLJkKk8DjgH0Axlj7YqHqrRNIpLVnY+H7aQx/2pfA/u0PlqTkAE4Vvd3JwB9T0r03Qre5CMj/uzHw5H3slTgD0Pf2+tcTokiu4ku222tgw27+styR97HpECMDH3iB2re1PxImn2UiQt++Q4X18xgfT06nNbRtFXZjO8nZF24kingNhZDCHcHbH8Kj7oz3J4H6964/XtRW5vILC1bMVquQqj70hyOvoP1NQ201/cWoijBXzfm3E8hen5H9fxrXt7BrYi3sU3Tycl/wCLHfk9Pr2rmnVlN2RrGEY6so6fpbxpEswwSRnPXHtRrZS92vEoI4RO3J+Tdn/dB/ya6me1t9Ph33MmXfnPbHqvfHbJ615vqWpCaby7RcwQKWyRhQTwB6kk/pR7OV7C9otxiX9vZa7bk5lW2LZf+HkYAP1I/Ks7U9RUXe2D5i0weNR6JkE5PryazRY31y7PduNvLAdBzxwPoTyaz7y7ZtSYWeWKjYG5xznOB7ZPJ+taKlshc/U6nT79WjIuAInilYnB6oMY/EdD7Yp11fW1i0V26iVMvbyoBygckhh+HB9a5fymhXy1YlFDE45LFzk/pjFWAsl1HOrghlKFieQu3g/XGce5FappENNlvRL+50a8NtfMZ7WeJV8wgnMeONw647eoqbxM7XCxwWh3hCV2Z3AqvIwe4Ixj0rOaAz2W1ly1uTHnPJjbp+IYfrWdPFeQ232mMMstowfj+JM7c/kcH6CplK70LUbbl2z1O5s5jcoBNaSkGSPuFI5A+ma9BsZYtQguLJ33q4XY5OcgD5c+9eaMFZ3YL8kygtjsT6egNdTprvpm4qTJE4GfXB/zilGre1yZwPRdGtWhRVU8SAblJx0GP5VsTyusBuBkSQnD49OxqlpTOpVh86EAnPUZ5z9Ks6g01tE00ZyrfLz1UnsfUGvQprS6OCo9R0t+l1CtxD82eGx2I9q5q71KEo0FywVSPlb6eo68Hr6VjSXr2lwQCE83GAeFPtnsfesLV9RlQNHeREg9HC8n8OQfcjrXRFXMmzC8QajNYy+TcRiWPkjnkr/eRx1x6HNO8P39tLN5Ts21cEDcBjPcZ5I/GuD1K9jlDRQuCgOdnPykdwDyM96zdK8RT2E4EbM7xnA3HsfqD3rSUNBKWp9X6O6XalwRj1Jzg/TpXRAlgP3iqMYyvJ+mfU+1ebeFdaW/VfPjXnAyWzz7gY/WvSLXy5NvlKqkd8/4f41ycupvcvGRXIt4eNh5P8/x7Vk3cayeZkggjPp1IwK1JNqxFd3AGT259K5LVr5lHlQ/Kq8uxz0P+eBWdSSS1Lpxu9DndRt4iBOTjYRjrxz1/M1mTKpkZsZxIMBuxAwcfnVs3quzgEhY0z9STgcdO9YwvHlFwYSciMHJ7kEgt9cYzXC0tzrSZ2djetaSOY2O1SSQvI2nk8ewOSPT6VfubiVAU2b4/vLkfKc84BHQ9fx9q4e0uZgI5weUO5+fXgkf7px+FdXGeEaCUb4+sZ6YPf02n34raE+hjONtTyrxfoKxXyavYwxrDNgOMnG/OckDgH0PFVbKRbd0EkgkbtsOV/Pua9d1ezj1DTZtjbJcFk29yozgZ4OOoHWvJLWyjvALpwN7DG8DYT67l6AjvivToybjqcU42bPQNJlWUoEPXv8AzruLZ9jAD2ya8x0w7ZlZcqq8fh613kcjSIET75APtn/9VEnYSR1cbJcJvUDzEGPY5rFvXuIV8+LGGGCvuO1WrKfCsHGM/wAhRqMZKOMbt/PpyOo/EVyzd9jaKs9TjdR1F2QzRIZIDw4HJj9eO4rn/DsVvD4u0+YOcbwQyjkjPtjJFXbxJbF9okOwk5OOqnsfpVHw8qw+PrJLfAB2bk59eCO3NZU5e8mFVWi0fqvo/OmwNuD5QHcBjPHWrFw5jKsKi0sqdPgKdNg6jHb0qxcR+ZGQOor79fCfJvclVty5pOajtzlMHtUhHNUIfRRRQIKKKKACiiigD//U/VKvO/i1qLaT8NfEGoocGCzlYH3xXoleQfH0kfBvxSR/z4yfzFZ1XaEn5F0leaR+R8ty0ss8khyXk+ZvyOPxNZ0LtFtixmQsTn0+XI/n+lYFzqDmWKPI65Hrk5/+tU8N0JSJmbhF5zxyBtxmvh2ru7Ps00lYdqFy5U26N8p3MT6gcD8OtQLcGO0ZtxDdzjkkkf061KHjuNsmMgDaR/suCP0JGaxLV5CZBIS2FXH1LA5PtTS0BsnivH+zgclpUBX2IPp/Ku9tJwbGKLIX94kYbHOeWJOexNcZDbwsZPKziJQqgf3Vdef1re06YXLjzgcRO0qgHGQi7Rj8aU0rBF6mvbmZLZFVS7yDA46bpFPGParIWRRNdxgne5PP90rkA/StVzFZ3drclduXGVz0UyADPpUN+/2dYooxtVy5b1ARgOfqAa5tb6G2ljbifc1pHE2SJZGOMD5UXCj8cnP0p4sVlHmTgMI4trY6/ujxx7g4+tZlrtj1GBN4KqyHg8DcDn8iK32uI4r3epULLA+4Y67cbv5VNgZnadaC4ypCmQKyrkd9uMfmQas2t4WuhHKwCBBuBOQGXaCPqzsSfQVEr/Z2eInO+aNx6jzcEj2wRgfWktWt4o5rqQHYzSsp4wcyfp93FOwrs3SZnaS0243F85PTORnH1X9azikiziGIbpZtuEHZf7x/E4H4ntV6W5iivZZQu0qDu56855/GqtgWea9vWbZPIzIhPAQ7eST6KvQfT1rOK6lXKD2yXBlu8HMPRe7Ox2oSPx4Hqa6+3tLe2htA43PJ8xjOAo2EAFvXBAPPfntVzTrC3ggWYR5OVZN3HEecOx7ckn16VB50t0d0eI4cZaZ+mOo49CTwo68E1LfYerNNNWkZEZUDbFEkeeDliTuOe5J3ewq7pmiTXzebdKJNmHZj0LEc59uAMdags9PLSB5ydijzCpA3tg43v6Adh6/jXZTW0klukfmG2Rh8qj7yg9WPq3ueB2pcrm/ITlylISRRE2tmPOuScE4woA7n0x2Fb8cX2OHfJ88koy24cuT0B9FHZe/8oYINO0wJ5KgEYCRk5I7739SeuPSuZ1jxMqhltjl2Jw7dz65/w6VqlGCuZ6ydiHWZIFR57iQFySSzHOcDsOgHpXDtOkQSGMGR3O47z0GPvEenpnr6VVvNSkvrsbnE5UYB6Rrj09frjH1po84/LYANITueRucehOf0H4msnUN40ieV4bdXSV2kmuCNzOcEL14XsMc+p+lV1shcrJJHiCDd87fdLL/BEo7DHLHqamgs5EbAJeVieW5Yk8l2z69v/wBVb+leFrm8YtKT5YJz689QPc9zS9q2U4RS1M+LS47kAQ58tQeV/iJ6tn25rp7Hww8kRgWPAZQSo6gZyAT2Pc13Wm+HpHWOKNAAhHAH5ZNej2vh62tIBA+C2Mt7nqSfr0FdNHB1KvoclXFRhseNjwrEcELiNRnH949vw9KyG8PvvZ3X5CCpHXKnr+Ve5XFjDzF0XHb+Z9sVnXNmokJiTauMc9Rgbj/hXQ8DYxWLufOraC1rKN4ym1s+69R+taFppA2CIfeJ43dMk525+vSvT302Oa4BXiN41U8dCfX8aYdJ+yfuZo96sQwI5zg9vcYrCODd9TSWJ6GPbWstqiSEbdgwp9j2I9P8in394Gt2C9CMPH3Hr+XUGt2aRY7c+W26PJOeuPUMO2K4y4kt5/nuNqgnC84yQDgj2/8A1V304cuhyTlzamLqFqHi8qZ1KHlXIypU/wB7vj17g153rE95pcR8+Iz2hyrRk7sEdCp6jjv3r0W4kNrF5hbzM8qGGM54wwz0/WvIfEt19qQ3Vk6hj8rJ82Vx1XjnHcccV0wRi2zzPWbuFroXNpIxQ8gH7yexB649azYpg8gkjTc/8SnGGB9ORTLx3LbJGUr2PQ59uKoLaNKRuGCOjY4/L/OK3toZ31PdfB94IWRbWOONu4RIy5/FmJH4CvoPSr6ZolaRdo9B6+7cZ/Cvj7w3BdBU5XOdy7wcbRnnAwSc9MntX0BoWoTK0UbsWJxztbn2BPb6VxVYWdzqhK6PWCkzrufKlugI5OO+OwFcdq9o5/1mfLJyRnBb/Ae/5V0MGqhiyq+EX7zDkgf09gOamkmt57YySjABPLdz2H0FcVanzLQ6aU3E8seGfZckHfI5AHoB/T0qGK3dEMEbY3KVPHYL/ia7iXT4PNZCwAwHY9Mgep9z/KiysIZEa5YD5lZ8ew46/SuT2bvY6PaKxy8VsSIpJQVJAGPXs36VryG3jlt5IQU4IUnkbckFGHdSQfp2rZ+zn5VH/PYEZ9MHANU5rPLrGV2iMuyg9wSSR+WDVqNkQ5XZLFDItpMjRllIBKA5OR02H1H8Pr0rg2v7G5M0kbKxJIPGCSPXOMN6g9/evQVjENtucHf/AB55yh6Njvjow/EV4p4mSex1czQcSORz13D2PRh2zwR0I7120HbQ5KivqdlpjwsS78DAAFdvaQvEolPJwT9Djj9K80055JTGZeApBI9T2r0/T5fPwgO1exPfHGfzraaIRupCJF2qORyfpVhXUw7cElevsRVOW5eI+YvPHT168fSmm7QvHLG2C/4cjqDXJKSTNlFtGFrVisTtNgPE/wAxzxj1B/oa8+Se1svENvOMgxMGUcH6hT1+v516zMgu9275WyTtbnOeM5rktY8NPdQO1uA04Gfu9x2ot1iRJdz9GPA+u22ueH7W5t3MnyAHI5Bx0NdlX55/Cb4vap4bePRNSLJbg7cDBxjjo3TH0r7w0HXLTXbFLu1fepHXj+nFfX5fjoV4K2587isNKlLyNZEKOfQ1NRRXoHIFFFFABRRRQAUUUUAf/9X9Uq8b/aDbb8F/FTeli/8AMV7JXjP7QpA+C3iokZH2J/8A0Jayr/w5ejNaH8SPqj8UsCQrLIfmUA/iSRip8bLfyhhi6g8fUnP51Ru38t/KTAKqf++c8fnVtd7SxIBwF3Z/2ck18a1pc+uuR29xNAquvXZlh7delWIpRFYGZBwqsCMdR24HYd/bmrgtle1N+gyNw+UdcY5A/TApjaf5ZXY/7iTmNhxtJ5APqCOtTdFalewjuokkmQn5TxnoykrnB/CtXTrpV1eFlb92QUfP4ZFXk024miTzLcxNCpIAzjk8nA459OhrGn05/PLw5R0JGMHn2/D1pOSlcdmjsYpYr3TILwvvH2grIQOQFYn9c9PaujlW3nvD90O4QbegG8vjI9Mr+deSaffLpbm2mJ8mQZYA8Z6E/hXbvq9rcR+bnfIVEcmP4thZsj3OeD+FRODQ4yudXeMtteRiMAEL3wORkjH4YP1qJpViuVWTc4wVB7ASD5vxxVbSryPVmtpdwd1lABYfeGNvPueDUrzi/tUGQsmGbcO7RZX+WK57NaM0Kj30iDznJLufvAcZRWK/pinm5OyLTnyy8NJtHqN5/NjwKz2YXVq4UZY4Ck56E7SR9eKcJZo7tDAuZpnGAOTgKvT36VdgTOwKyraz3cimS4uCcAYyOQQMdskfgDzXSaTFIfLmm24KlwM/u+T8zt3OSOPUDsKyra1iESQuwuFi4ZMnaWPPzFevPYdQOa0w5urd443Hlg5lcDnC8HGOFHYKDwcDrnHNJ32LSNqa4W9Tz3DGCU+XEucM+eCwA6A+vYZNWYLCOe8hMhyqkCBF5GQOSFGee+egA9ScULC0mvZkvZGG2TKxogx+7HBwD64wSeMZ9a72wFzGGby0abZkAfcRe248Z9amKuwbsixFbQQEGbCow3FM5Z2HTd7DtnjNc9f6xcTXAgsUM0jn5mXn6AHv7f5Nbi6feXqO17JuQ85xgN6HHH4cAeg71UurMW8vl24AB4GBzz1J/wD1VUlLoRFrqc/9pngWUykPI24MYwX6dVT15+8x69uK5S8Fxejy3QgHjYME8diegHsK9JGiLy052IqjgsRwPXp+VPRNPjP7mJVCg/M3T9OT61m6cpPU1VRLZHnWn+GbuWZfMGQecdh9a7W18PMh+zRDJH3sjoTxkj19P8K6KxmtlbI+Zx0JHAJ/uj+prrrCS3tcNKBtJzxxz7+taU8OpPVkVK8ktjE0zwT5Cq8ikE8lm659fqa66LSIodsMfCL1an3GvWyg7+/YnAH9a5e+8SJIfLj+c9hnAPue4/nXdy0KW2px3rVNz0OK40+zj+VgdvQDnJ96z5NZWY+UgJd+MdT789gPWvN5dZ3rs44HRDxUqakI0I4HHCjAX8T1NU8x6RWgvqS3bPQjNbpEsSuGcnc2Oe/c1TnvYmDRnJ3H/Dqa4GXVJDkLJ7nbwPpmqEsl9McE4UfhWU8fJ9Co4SK3Z293qNtGdqEFj06ADH865+fVVjk2EmUEk4XsT1xWG6+UdrSFn7nqfpjtVSdoo8Ce4K54CqOSPr/hXNLGT6HRHDw6k9xLNcv8+1dx+5nOf+Ajr+NS2+l2hYXT7rmZOuQNoIHGB0znv0HvUUKQKwZSF9MAkk+5NbkYZ49rvtxyBwf06fnRTru+rCdJWsjnbrToLuLM581yNohT7gP0GWJPqTXnOtfDzxBqkjl4QIW4WMLl8EdCAR17biSO1e8W6nAKOeOpOOf6Vu2ckFqdxXczdSzdz6mu+niPOxxTpeR8qaL8ANXu5Wn1XFpGMbUzuP0OOp/r7c16Db/A1rpUtVZY4cAMAPuqOdoI5JY9eg/SveZNajQ7FCn1OcD8+v5Cta01NXA+6sajnA2j9eTXdCtGT1kc0qUktjxuL4HWX2mCeZlEEPzGJBjewGF3NnoO/b0ro7L4QEv5z3HkhioO3OAi8iNM4wO2a9Q/t21UgMwZh0Hf8AOaB4hkQb3URgev+Fb3ofakZWrdEc7F8NrK2hWKGRlAOdoAyx7ZJ6AVg6v4I1wCQ2Xluq/6pCcRr7k8s5z24Fd6/iyBT1OT0zirdtrsVyQQoH15P61MoYWekWUpYiOrR4jN4RvLWHydTnEhUjcFGNxPOSc9KqRhVkcOAEhUcZGBgbjnHHGRxX0JKthcp/pWHJ4wB2rAl0DSbkNHDGFUggjGcA9a554JL4WbQxT+0jxW1xHCgky7Fywz/ECev5VZlSM3L2znEgODgjqqgEjPrmu81bw2Qmy2cABSucc47fl6Vx82iXE088sCkb2LbyeVyAD16dK5J0nDc6I1FLUwNRDhGlU+Y6jeFHXvnb69OR7e9eRfEPTL24toNX0xC7j/AFkQIOR1yAepx6c17e2mXLq8dz8qE5Djs3UEZ9/0rjvF+nvd6JLFZQEycNHngsQckexz0rKLd7oqSVjz7wdcNf2+Y2yMZAJzj25r0uySeGXDLhX4/P8A+vXE+ErN7Z1ab/XPyVIGRnn5sd8/j617Db2nmxZcZ9a73HmV0cvNysekalFY/dyAR1wG/wAKnTTlGQw8xcg5A9uD9Oa07ezCgbujZB9eO9b9npUkaCUEg4xx3Fc0sO5PYtV7I56DTAi72jyg564P+cfnWtb6bFcA8Bh78EfiK7uzsIgRsXAPr1rYaOKGMhVHFdlHAdWznqYq58teLfCLWeoC9tElR+zA7xntjGK9H8BeNPE3hZzbsxlRSN4PIAIJxknrivQJdOguJlaQYwc8U5tA014fJCYBbJPcnjP54xURwNSFV1KMrEyrxlHlmrnpWm/FuzneGK6h2hiA7dME4HA5J616np2q2WrQmezYsgOMkEV8oTeG5Ul8+ybYR6/pj0re8PazeeH7qNbmVtmegJwc9eK9Wjj60HautO5wVMNBq9M+o6KzNL1ODU7VZ4WzkVp17UZJq6PPatowooopiCiiigD/1v1Srxz9oLj4L+KiBkixc4+hBr2OvL/jVaLf/CnxLaMM+ZYyjHA5xx1rOsr05LyZpRdqkX5n4fW/lvdLLL8yr27ggZP1Hep7e4WJ0k++A3U9MZzj6dawZfPW9ltYx843A/8AAsD+ldHaxJCEjkAIjHz55ycZx+Zr42pHl0Z9dF3JFfz5HhhGFmBxj+FwN6n9BWojefaNCNskUpyQQdyE8gkDPGc8de4qhZ2Qe7FypLLHkAD0xW6lmj+VDEm1Ao3Y7+hz7Vi2tC0SadPqMUgtJJQyx/wBjkbufXkd+n4V22mCAkiSM3Ibgo4BAI9GIzx61hraQXMsHk5WZMqWH4A59a2BEkcmyMk7BknHUDgcjnJzWFWz1RcXbRlhtD0K/JTULUiM4OxJBkn1BOCPyrnZ/AGjeZ52l6jLCWHCSx7h6jDLj+VdfAkMgLs3mIFB5IJDdx0zUrFJCHiBOe5JA7Vkqko7S/r5lcqfQ5G18Gazpd3Hdx3UE8W8OVVip+XB6MPr3qWWwu7HVRI8OLfngNuJDFt3T/ezXfW9iZl3zTHccfcJ/XrWkdP06Ebxh5TxluT6cU3Xn9odo9DyeHSdb8tVaB23EEEj+FenA6V02m6Bd3eoJJEjMseN7DIy2OEBIHy55bua7FruNX8yQbm6Y+n9KedanCBY3Ck8A9Mf59qTrtiUexBcaLKqrbuCqkY2qwRQc9vTPcmpbexjRIracgQxsT5a/KjEdPcgc/jVEyfaD5c8o685yRXRWNva5Qt8zZySf8Kwu3sabbnQaZYqkKQxEIzAZYLwB6DP5V2ELW0Uaw26F1U9CcjPUliepNcNJqHkE7NpwMcNk/jzTv7VncEOfoq9fzrZTsjJxvqdlc3sa53EMzdk6+wrLkv1gH3PLLAcdT7+/wCPFcjLrEyt5UeUPY5/wrImkklBxOArehLN9KmVRvYqMF1N6810SziBGL7cZwPu+1RRJd3Tgs2xRxwMEfnWZCFUbly+3uy8/hWlHtX19ueTWDTb1NVJLY6i3MNquAwPbAI/VqdNqT/xFWx0VTx+J71ySXcAO1gMD8cmrS38Eg8uOFmZhj0H51abWxD13NiWa6u1DMQF7Y4x9arpbzZx8oHODjqfw5rMIULhgR7E8D6D1q4sicKXKj39u2KVrjv2NWGMKB8vTj5jgA/QVYWO3iHmTMu7/PassOF+dn2r2z1/AU19RtoiRbx4duNz9fwFV6CNQzRb1bGMdA3r9BVSaUuhZnwPZSf04zVSO6IBctuYdh/XGKryXdxdsVMhKjsSF/QVEikicTRoCqxsxzyWJBP4DpVhYc/vpESIn2x+p5rKlaS3BcThAPTB4/U1mnUo92ZZDMc8E9PyrNRbKudMRHIco+BnqoPb61rRzY+RTwefTmuNi1GSYgIMk/7PC+/Na9vjcGZ/mPGTWqViZHWwSDfkjfj8qu795ywXHYDnH9K5+3uY412liR7nithbyJQGJA710RMJJ3NSINkFVUe5GavMHcgTMX9c/KKwlv4ACzMTjnIzmpP7UjXBjJx/nrzWimkZ8rZuc4IjTBxg46VQl39T87f7RrMn1CaZQMkp9MVX+2vs2qAAevGTUTqI0jBmv8ztiUggdhWlb7jINik+3QVz8M2Puvitm3llTBGR74p0pK9yaiZ1kMT4ElywQdQBTZbyJAY43z2wOlYnnM6hnfOR3OP5VA0oiyHIVR/dGc/jXofWbKyOP2OupqNcN/rJnUAeo/pWJfatbEeWqbyP73SoJZIuGKs/sScfpWXLP2ZAg6cdf0rmqYh2NoUVco3N09xIHmJ29Aq/KPz4rMuWuLpCkZ27uMjqR+PNX55ogMGJn7DIP8qqmSeRsEGNe+BjP9a43Wex0qmjMs/D8UMomkcK3HCjHPue5rt7GGKE5B4HU9Kz7aNEUGNcY7t1/CrouEAwRlfbmuqlWkupz1Ka6HTWos5G3cKRzjsK3I5FTJWQdOB6ivPGuyn3F2g+poGsSRLsTDdhurtp4xLRnLPD3PVYb2MDAOW9+gq7HJHIclgTXi8OszRuFkfZnoByTXoGj6q8kfKkZ9RjNdtDFqbsYVcM46nSSxjdlTzU1uw6MaqyXJY7UGTjJqSF8oG6HvxxXVfXQ5nF2L7EYOKxdSiW4iKggHGcj1rSdg6lT0PXPFcbqOqraylFk46gdf1oqTVtRRi+h6F8L9anjv30qc5x0PPSvoavl34ZW8t34me8j+4ByexzX1EK9DAfwjhxNufQKKKK7TnCiiigD//X/VKuQ8f6Z/bPgvV9LyV+0W0i5HXpmuvqG4hW4geFxlXBBH1pSV00OLs7n4A6lpwsvEN9A4xIJSvPXjPJHXvVqCZZP3MSYXGGx3yDyfYYr3H9pzwNJ4G8fPeWybLbUMshAwAW4NeGW9s9pAtx91iMqOvOeBz6ivj8dTcarTPrMLUUqakjZ0nTpAxkAyvJZugBAzgew5rYHlvEwUjywkZcng4PJHsSOB9aoreqBNAgPlsTgE43LjYpP86rtfItvIjrhskBTwSwwV/HA6VyWbZvfQ6OG4TTwbfjdJtdTjhWBzz9ScGpbeeZ53VGG4/KRkY69B9a5qFQtv5spLMV+XPTgir5litysiAhjt69ee34VnKOpSZ0LMkUxtZQzdNuOmDgg4610cVuBB8xLKoJ+XjBrmXn8yHzIhhhjPfp/wDrq8t8YAGlIjfoR1BFQ43RV3c6aO+8tNsYHAHf171VuLxZQ23IbHXoM/5Fc5JJaeZINgBZQcgHngHp6Y9OlVVuITMPsxbAPOG4/X0qOW4zVa7lUjfISBg9OxqaO4jeTzcbT27k1T81Ng4UZ7E5+nT1pSkojWQgBccruHT881PIUpG9HfAv8iKoXuSTmpVuJT8rSAhv7prmG+0OoSKNRk565yP8KmiWQRn7VlVXqVPH0470vZhzHQx6nDbEnkdvUVeTVJZFV49wQewwf/11yi3ljHhVXhh/FjJFLJqSqmI3VcdB2/Ok4Bc64XoZ8yKQSTnIH6VIt3bQjIP3fTA/+vXCJqTNyVJ9cEf061YGoMjfMDk8cYx+gzS9kx3R38N9CQCZME9Tgk/nUjanbRAbQGHr3rikuJHwJHAJ5/8A15NS/b1hTfwcd6OUR2a6nG4yi49SQBikF3CxwhJyeey5/ma4OTUVY7mcAEZGCP8AGlS83/efjHqM80nF7jPRPMjAzv59sf8A16WG7tlP7xgceuTXCR3RZf3Qb645/WpV2kBpGOO45FS7oDuzqNqcsWC546c1Vl1G2hDFFyT0PGa4xrqONti9O1PSZCPl+VuvJGam9xnSvqyldhGB3xms6410QKVjXGfQdKz3fgbiGqi6zyHjhT69P0qlFMLkr65JM7eWjOT9MUxL2/3F1Qfj2/OodgQ4Y/MPSpY3QY3NjPbg0+XsPmLi6hdqCFySeflx/wDrq3DqV3ECX+UD1z3qAW/AZhuB6YIBpj28VwAIXBkB6NhsD6ip0KOis/EAjX55GP8Ad2LnP410MOvKyhihbjqxAFcFbRzwyhJpYzjtgqRWoJNp4dsY5xhhSvqJo7FNTVmJfYR6A/8A1qurrCkeWsDMO3GP5DNcdFJKy/6MRx0IVc8/72BU0l1eRcs0iknuVQZHrxg/nRcVjrptTm+UGPA6/e5/KrC30zfcjCAj+IiuPN1qBUtFIip33gH9RTre8Df8fLRPj+6x+lQyrHcR3B7kJtrVgutxUPnB6NXnkWr2aS7bhmxnjcrEfz/lWxDqFpMP9HZsdwoxn86uLa1IkrneC5k2hI2xg/Tj61SknmQ5YEA8jvWAl/BkRsyoexk5/rUxvHWM4kVsHtjj861U2zHlsWzPMQww34n+lZk7XjtySqnsWx+maZJeyAAiSPB/vc/qKozaiYifLn2n0HNS9S0TSRXGcMWcjryB/I1ArPFjcEUn/aOayJ9RkUbmkaQZ5UDms06lAmCylD7kfyqNizv4bg52uQTWgLlCu0/L+GR/OvObfXF3EBRjOck10Ca5A0eFaJT7kg1pGbInFHQyMjNtLY49OazpWAPy7vr2rObU1wFUJIT3XJ5/Gq5vJpSRJiMLk5Jxn0Ap8zZnYeshSUskjBie3+Sa2rLWZ7Z8yM2c8ev65rAaQsudzAew4/oKbvA+XnH+fQYq4VGtglFPc9as/ElvjPlszE9WfJ/GurtdajLkS/LkDvxzXz7BqHkTB0/x4rqNOuNS1Fxb2sbZkPPHbHAr1sNipy0POxFKMdT03UfEtrCTAP3jP8uBjj61zuleH9R8S3iiKErEWwWxzzXp/gv4UO6C81TguQ3PX8q9/wBN0XT9KiEVpEFx7V7FHBTqe9U0R5VTEqOkDE8I+F7fw9YLEq/P3Peuxoor2IxUVZHA227sKKKKoQUUUUAf/9D9UqKKKAPnn9oj4Uw/EjwkWt0zfWJ8yL1bHVfxr8mfFVje6JEbG8iZJ4iTgg5O1gv5cgV+9bAMpU96+Zviv8APD/j2GSaNBbXjDAdR755/GvLzDA+1anHdHpYHGezThLY/JqCYSWKo3zTI5Vj6gjgfgazDetJqCWrnd5hYpn+HCZBz+le1fEL4E+LPhvGXmjNzaqeJUGc5Bzmvne5tb4TRXKIwMZGTj2wa8T2Eoyakj2Y1YyinFndQ322KJD/BuBHXryPzxTGuLiRhbt8zOQ6joc+lc3pmpKE/0kFXQ4II64/zmrEVys80ckb5aMlg305IP9K53Bpu6Nk9NDsrfU4oZWKEKdpYAZPAPTHr7VrRahbupud2Y2IBZSCFz1yOo59RXmsupGNwMbQQuJPQj1/HvVoX4RpJG3bm5zjG4H1I61Dplcx3+54iAx3AsCCxIyPY017v7OC5KsoPJ749+K89fXWiKkDaoPKqSB+PX9K0INfNxlh5ZUfwhiGx9TU+yktWiudHVf2las48qUKT74B/Gte2myMRzEP1ySCvv24riIr63Zy0bKsigBlkOAR9eKb/AGtFHmGF1icHgbgQefUVEoN6IpSXU7WUynEhl74ABP8A7LU0eo2JRUuA+TwME446mvOl8RxxTNs+fP3ieuavf8JBp6hsAoXGQEUde+eafs5W1QcyPQBc2wLISVA5yeAR6ZbOaadRsY8fZ3B56bg3Hvj+Veet4p3bfP8AMeLjIJ5H0yTSW/iGzdtq26uOTlmIP5YIpOi+qFzpHof9rSA5MZfkngHgfhUT6tJCChhYAdCwOOffiuIutTec4hLc+j8D8gKgiuyx2zXGW6D5iRSVKw+c9FjmaUBt+09gDgAe+etWfnXAllOScfKw/lXn0WoR28nliVdx7gn+taX9oJtCySFj2OeB+FTKmxqR1StaqxKFsdATg9OtPaexOTuxtPb/AOt0rlXuFlVkTDE9BnrUAE2Vbyh0weDx9fSjl8xnapqD5Bi+YDv0/LPWpDdSTLtCdfVsGubs5TjMjjI471ea/YfK649Mc1MoAmaQWbhpdpK9+OB+FT+dOPmRxtPXoOn05rHa+i7ptI7H+dVzqUSy7GXgdSTxmlyeQXOnS6hT5SM5Hfn+dRTakqrtBA9Oa546jbsTFCp3MBznIGaoy3F1uIVPlwME9PzoVNBc6Zb63cYDfMcA5IAFSC+to3G9iM9cc4/IVgxXgMYLOmO6HOT9DiqrXwk3LHblj65/wFPkC51s98GTdG7H0HP+FRxX1xHh88n14P4HFc4t/LCuHmEeR0bqAPoahfWE3ZFyoJ4KhSf1o9n5Dudn/beonBLgYOPm6/1FSnWJh805Td6jGfxxXHQ6orPsYsxzjdWit3agExn5/TOefyqPZLsPmOmXXkBPJJ6DDZp8ut3ZAUlhkj+70/Ec1zpnkmwOwHHHf1qXzJIRtRGwR1PPP5CmqC7Cc/M1I9buoVMkJA2/3cH/AL6UH9RUkOuXskmFSN1b0wck+hIFcxJvkOZTGGx/HHz14w4pht3ucg7o89dihl/kDVfVm+hPt0up6BHfqoG9fLbGSudp49MjFaVlrthGAfOYHvgq2Pwry7yxb/u3LMB35x+I5qazaWR1jtI5N3YKN/P8/wA6f1GT2RLxUV1PZI/EUAO77a21uoKD8+DUp8T2A5EzSf7WzI/ka5XSvBvjzVF82y0maRGGQ5h2gn6k13OnfCD4l3i4a3MB9WwP5dvrVxy6p2/r7jCWNprdmTJ4hsZEMIbzR1yu0Y/A4rOkvPtGBZY3H1cKRj0r06P9njx+xBkmjIPrj8jxWsf2d/GyDe0ylem09PqK2WVVukTJ5hS7ngMguA2XkZGPZmyP0rJkYQvv8yMA8nLMT/8ArzXr+pfs8/EVZG8kRyIxzggdPpjBxXAX3wJ+Kcc5SLTiRnIZTjp70/7NrdUWsfSf2jn49StY5Pn4IPVGx/OurtdZteiStk9ScHFSx/Az4o3EqLDp+MgYD+/XkV7l4U/Zu8QPGr66UiY8sE5qo5VVl9kynj6a+0ePLd+fzHK2T6qDVi3t9Qdyke2XIz93JODX2FpnwH0q2UC5O84xXY6X8JtCsJA5UEDtiuuGR1Hucsszj0Pie20LXLqQi3tnOeflXH867HS/hZ4k1Jl86GRPqCK+6LPQdLsVCwQKMe1aqxxr91QK7qeQ0lrNnPPNqj+FHy1oHwRkQA3vy+tez6D8P9J0bayxgsOenevQaK9OjgqVL4UcNXE1KnxMRVCgKowBS0UV1nOFFFFABRRRQAUUUUAf/9H9UqKKKADg0wpn3oIpvzDpQBkar4e0vWrZ7PUYVmifqrDIry68+Avw5uIXh/s2Nd/cCvaPMI608MGqZQi90XGclsz8/wDxj+xZYanetNot2YImOdp7fSvFNe/Y58aaD582iSC7UrwOhz61+tuKjKVzywdJq1jojjaq1ufgb4g+Hvjrw3dtDeafLHtznKEj+VcM5urYtDfRtGDx3ytf0N3ejaXff8flrHLn+8oNeaeIPgV8MvEhZtQ0iIM3UoNtcs8rh9lnVHM39pH4QvdeTIyqdwHI3D+tU3vJGIKKAtfs9c/sffCOdy32Z1B7A9K5PUf2H/hrcvusppoAeozms1lzRr/aUD8ivtby/I+5x79q0YU81PLjBTnsOtfrRb/sSfDmJQr3EzevNb1v+x78OLZQkbSDHc8mn/Z7sH9oQPx9XTXceWqn8Rmta20O9YgQKxPptzX68W/7JHw6iffNJPIPTgV19j+z14E00AWlvnH98Amj+z5Pdk/2jHofj5D4e1GXbuteFAGSOv8AKmy+G5Oph2HPPHFftAfgv4ObBaziDAYyFAzVab4HeC5wQ9qv5VKyrXcf9prsfjbF4cuQCqoTnBPbn8qVvDl87hljIx6f/qr9hW+AHgY9LfGfSq8n7Png1j8qbah5U+jGszXY/ISfw7fMf9W3H1/woi0e+i5EbFvfp+NfruP2evBw6x5pp/Z68HdoRS/sp2sx/wBpo/Ia5tdQziCEqW5bv+RqktzqNtmNkbjGNwyB785r9err9nrww6lYYgPwrj7/APZg0e5B8squfaollTtoi45nHqfmPDeXdwC0uAx5yoH+TWtbi6kbLy8dgy5BP5ivvGb9kaFixjuAAfQVAP2Tnh4ScEVxzyytsonRHMaXVnxBKbhV3SDn24rlby/u0lO2F9g6hiMfgetfoev7LNx0eYEVK37Ktuy4lkBJ9qUMrrdYjlmVLoz8zpNQuXbZAjBE/PP14qzBrd+oXaMMp4LqOM/Wv0kX9k3TmPzMAK04P2SdDXhm49MV0/2bO1uUy/tGHc/M8yavMm8DoeNi4xSm71/zQ8wkHTlSVyB9MV+pUH7KnhmPAJOB2rbh/Zn8IwjBjz9aSyur2QnmVPuflVHqGoPhWWZzg8k9j9akt7C8mG8QM3OSST+FfrFD+zt4Lh5EA9627T4K+DrPG22Q49s01lE+rsS80h0R+V+n+Etfv2UWVg5z15NezeHfgx4wv1UpamLOM5NfpDpngXw9aYMUCjHTiuzt9Ms7dQIowMe1dVPKIfaZz1M0n9lHwhpX7OeuSov2namRzzXd2H7M1vsC3cufWvsIIo6CnV2wy+jHock8dVl1PlyD9mHwtjFw5I9hW/Zfs2fDy25lhaT8cV9CUV0KhTW0TB1pvdniEf7Pnw3jO4Wjk+7mu40r4deENGAFhp8SEd9oz+ddvRVeyh2Jc5dyrHZW0ShI4woHYCpfKjHapaTAqrIm435R0FOAFGBRimIQqh6gUbE9BUbA9qAGzSGSbFHQU6kHSlpiCiiigAooooAKKKKACiiigAooooAKKKKACiiigD//0v1SooooAKTFLRQAwqDTdvcVLRigCMFhT91LgUYFABkUdaTFGKAA7u1MLMO1P5paAGKxPan0UUAFFFFABRRRQAUUUUAFFFFABRRRQA3mkwafRQMZg1GYsnmp6KLBcgEIFOCYqWilYLkew00x5qaiiwXK/kg0eQvpViiiyC5EsSjoKlAxRRTEFGaKMUAJmlzSYNGDQMWikwaUUAFFFFAgooooATFGKWigdwooooEFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAH//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA//Z\"}]}" +{"instances": [{"b64": "/9j/4AAQSkZJRgABAQAASABIAAD/4QBYRXhpZgAATU0AKgAAAAgAAgESAAMAAAABAAEAAIdpAAQAAAABAAAAJgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAB8qADAAQAAAABAAAC0AAAAAD/7QA4UGhvdG9zaG9wIDMuMAA4QklNBAQAAAAAAAA4QklNBCUAAAAAABDUHYzZjwCyBOmACZjs+EJ+/8AAEQgC0AHyAwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMFBQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkqNDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/EAB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMRBAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/bAEMAAwMDAwMDBQMDBQgFBQUICggICAgKDQoKCgoKDRANDQ0NDQ0QEBAQEBAQEBMTExMTExYWFhYWGRkZGRkZGRkZGf/bAEMBBAQEBgYGCwYGCxoSDxIaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGv/dAAQAIP/aAAwDAQACEQMRAD8A/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA//Q/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAphkQHBPWs7Vr5LG2aZ2CADqelfM3iT4xi0vXtIiQAcHaCxz9elceKxsKHxHVh8JOt8J9VLIjZ2nOKfkV8i6V8b4knRL2TAPXcMY/Kvb/AA/4+0fVYg8UwZ26DPSs6GZUqul7F1sDVp6tHpdFU7W9huVzG24eo6VcrvTT1RxtWCiiimIKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAoopuD3oAdkVl32pR2ti94CPlBYA9wOuPwHFc/4v15NHsRzhp38gexbBB/Lj8a+aPiH8W4reF47YM7iR2WNTk7DwFbHQDnP1rzsZmFOgmnud2FwM6zVloeqal8Wb20AkhgjdWJBGGyp4ODz6Zrn5fjjqILeTbwELgc56/8AfX6V8Z6h4y1fW5XK+YkR9Rhc+u3/ABrn5by+d1zcyZHXIxn8hXzM85xDekrH0MMopJaxPvzSfjeJ5fL1KKNOM5QHqO3U8+le36Vrdhq8CTWkqy7gCdhyBkZxn2zX4/XOtX1vOWW4kVsjGCAK+hPg38aDp1/FpWqShI5mAB7474PGM4ruwWcT5kqzuu5x4zKYqPNTWp+itFZul6pbarbLc2Z3Ie46frjNaVfURkmro+daadmFFFFMQUUUUAFFFFABRRRQAUUUUAf/0f1SooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA4/xtDI/h68aIciNjnr2r8utQv531CRrmYxMWPJYknnoF7nH1xX6y6pBHPYzRydGUjn3Ffkx4306TSfEt3DJIUjikYOTgMQW4A47181nsWnGZ9Fkck+aJZj1CRiY7aJ5PlyzyyYzjuccL+Jq9p2q6pA3m2l2I3B4CMxH8h+dYlnkxCSVSsQ5CHvju3cn0/lW2keoIu52FujDqw+Zh6hBjj/eP4GvmozbZ9BOmtj3LwZ8aNS0t1steAZV4DKePxr6s8MePNH123VracSuwyQO31r85vsqy5gjk3Njuo499qrx+JzWhomuax4Vu0ubGYFMglfu5/4CTXq4XNa1F2lqjy8TllOqrx0Z+pCOHUMO9Pr53+HnxbsNZjWC/kw+Bwa99gv7S4UNFIG3dPxr6vDYunWjzQZ8zXw06UuWSLdFFFdRzhRRRQAUUUUAFFFFABRRRQAUUUUAFFFGaACikzmloAa7pGpdzgDqTXLa14ostNRkDZfkfQ/5IrD8ba69hauiHBfIX6pn+or5V8ZeOJrW22pIXuZR8q9SCTliR+tePmOZ+x9yO56eCy91tXsT/FD4jy3M32dJMTh2AABbGM44Hfnqa+ZWubi9uDNdF23MfmYDBPf5T0xV6WC5vZ5JxcebI3UyDGS3Tv07ZqKJ7QpJBe+bbmNgkgJyFbOMjPI/zmvjqs51ZOTZ9hRoRpQ5USh/Kb94+z0dfusPTjpWZe3kYXEcy7v9ruPqOP0FW5rKaydWeQz2sp271OQM8ZYdRn6cGubuLNhMd4G8NnrjnoQQeDms+SyNVqZdyp37ojkEHK9SOf5Vzhml068W4jJVkbPHGD6iupLRxn7m3HYjke1c5evBNHjvmrpysxTjc/TL4AePm8QeHbeEy5dBiRCAACOMhgO/v+dfT6uGAI71+SHwO1+5stYexkuxbFhuUkkKSp54APX0NfqV4V1JdT0uGYAkgAFuxPtnkfSvr8nxTnD2T6Hx+a4b2dTmXU6eiiivbPICiiigAooooAKKKKACiiigD//S/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigCC5iE0JQnFfmv+0J4fl07xS9zApWOUhg20s2f5Amv0vPSvl79ojwi2qaA2oR5DQDPy9ffHvXk5xQ56N10PVyiv7Ouk+p8RaFKZSGUDcvCknIGO/px+prrBHa3MwMjltzcsG+8fQcZ/wA+p48z0+5fTfOtlOZIuCc9MkKFX/PauwtdRt4Ll45kCBBtCgbpAmdqr14Z/wA+TniviIxaZ9pNX2N+NJL2V47MMtsh+YJ8qlvfux9yfoO9Wl0y2CmVNgGcEhdxOOvPP88UttN9uzPOSYrcfJbxjChv9oggMfY8Dqa0IbD7SRc34aTLAJFEOOO+TyRnjI6npXbGHMjkk7GGI7vT5vtGns6MDleAP0zXq/hv4p6lb+RFNKUkiK5DcZ2nIOPQ+1ctDopkcrdxxwq3IQ8yH6ryR/wIk+wqld6GmP3TrGQQV4BK/iD39KpRnT1g7GM1CppJH2L4T+KNnf7LW/YRyM4BJPGDx1/WvXbXULW8j82Bww27vwr80rXUr/S5fKmw4HI7AjvjPevYNA8e6lBEpilYRbSrZ7Dtn6GvVwudSh7tZXPIxOUp+9TPtqivDdL+J4ljTzhyuC3uB1x/Ot6++JWmQGRYX3so+THfJ6/lXsxzTDOPNznlPA1k+XlPVKK8Yt/ilar5T3KkCQHcPQr0H41txfEzRzaLM5+dskr6c/4VdPMMPP4ZEywVaO8T0yivMW+KGjZUxqSGIHX1qhf/ABU0y2nMYIKhSMj+/jj8Kc8ww8VdzQRwdZuyieu0V4Jc/F+1D7IFLDGfyOKzZvjI0pWRIsIeMZ6H/wCtXLLOsIvtm8crxD+yfRpIHBPWjINfM6/FG/vWhKpho95Y5659PTAqW1+JWsW0i+eodYwfYc5Az+FT/beH8ynlVc+k6QjNfPTfFe+a4K+ViIALx16/4V0Nj8VLctJ9tjKc/KB2Hv8AWrhnGFk7cxnLLa8Vex7KBiqOo6hbabavdXLbVQE1wl58R9Hghyr5YxhsDrls4H4da8N8b+P7i6hlW4YqnJRRz2AyfypYrNqNOD5HdlYfLqtSSUlZGZ8SviFDcRsYBtCOzxjOT8x6/QE8V81Sy3erXr3quxZWxgLgbu+Wz3PrV67nuNdvJWY7Vc5ZsYCrxhSzFRnn7q+tdlpuhTmBIuYLhMKjkAowPOGA6D3zXyM3Urz5p7s+upU4UIcqONWynnjYMuZAM/NwSAc9cDp7ipbyx+2Wkd9kTb8RS9x8vC7vRsYB9cda7e50OaH968DQujZUlSQrezDsagbTZpnOQrrMMSJ1U+uVPOPQg5FbwoeRLrLdHm8dkIYJbG53NbnBIccqp4PPp/LrXIalbPZ3DWty2UJIVsg5HY5Feo39lLHGEVpIymcB1J+XoMZ+8OOevFcNr1g1xBHKEQZG3IUEe3XGQR+NTUpWNYVNTz28MxbfuJA7/TiuPmkKuyKeSTj2rq7u3uY2KBc44+XpWDHbEy7nxjmudRsbPU6bwHdG38TWrruBzggduMEmv1C+DF/LcWjISHC4B24GPr3/ADr8r/D8LQ65bvuGwMDk4GB6/j2r9K/gzaTiU3UROw8Ntxlc9m/pXrZTJrEK3Y8TN4r2ep9Q0Ui7sfNyaWvsT5IKKKKACiiigAooooAKKKKAP//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigArm/FmkR6zodxZv1ZDg/4V0lNdd6MvqMVFSCnFxfUqEnGSkj8f/F2hS+HNcubMsEkaXIB5IAPUDux7ZrNtWkH+qbY0kgz/E7ueg49SfwAr6T/AGivBH9mXY1vywFcEs69c+nqa+dvDTQXN5bxheAS20nbt/vM30A/OvgsTRlCq0z9AwtZVaKmj0S3vLWxaHQ5ys05BeTcV2KiDJkdR1GegPYZPpWzavJf3X7p5pJJMFtmRJkj5Qo528dAMBRySSad4e8OWc1vPeXETAXZDyuW+fDEFRkA47bVAJ79a9M0v+z9PtfskSLawAliu3YxyernJJJ68kk9zXXSo3V5M5q1VR2V2c/a6HevIdyeUH6gEuxx/fbOAB2A789aZc6SY02i4YOpxmPA5/HNdW+rW8qljF/o4bHHG4kdgOWJ9P6VgXN9eSJhLTyEH3Y1KbiPQ9Pxxj8a1lGFvdOVTm3qcldaSxVvMJcDoXbJz9SBWXZXkukuAGDd/LOAduMHBXIIre1OPWGbJjVARyGdePbHeuD1a0ZFErSYxzsXJ5Hc9h9RXBVp9UdlN3VmdvbawJleS1PlvHnfHnOR2K9609N12OZgCPmJPH5153o199kuEMhyoBHI5I6j8Aenaqs0503UmhaQssjZXnpk7v8A9Vc3mVKNtD3KW8gnRuh24P6cn9awLi/lSXybT5s+/Q1TsUM9ivk/MXbj8OT+FaenaIrYuI2yAH3HPJOf6VnOHOKKUdytObhnSKFjgthsfn+FP8iWFkE55UYck/3uldlBFaQHeuCxJI47471z+oRG4eaViFVSBnHH/wCvrzU1KSUUy4TV7WOVUSQCUoCzb8H/AHTip4Z1Nx5bkKWclfQL1/QVfk8hbBVkz5s4Ue55Y/4Uy5tWEalX8oAFR77yeOPQCsFS6o3c11NbTtTtvLkkiZCisB1HDNyf8KfcXMUmYGbndycjB5yao+H7SEWRjH3gxY+vJ4HuaTUYoIvk4aSVhs/LJNdCbsYNLmNl7qKGJHj4XHGevPGapLcPLI6xEbR8xP8AeOM4/Ws77OLpke4yIyOcjHAOf1qzDdQSXQsLfDHPJ9Mdf0q4tt6ktJEmta6ukwiOMLJcnacP0QHA5xXmk+vRzFri/Ly7i2BnGQoyff0/DjioPF9/LNqsrQuFEQAweCQMgH8OTXnWoXi2phiacR7V643MQPuqB2GeSc8nsTSd5Ssb06aUbnsGmSSSItxdSLHGRjyiAx555Y4x9AK3YruRz5cdrgx9PMlXaR6ADJ5+grx3TtWluLtLSJpGj2/KOUwM8n5DuP4kn1Ir0drY3BEH2m3tQw/1cyI+Rj2yxJ9ya9CjHQ5q2j1OhXW2tI2W/KwAHOULTAD68kfyqI+J7KchRcx3IYk7RlHx9M9q5G802SHMGnOINgYnyZUDAdN2HC4H+6vHrXkWvapqKnypLiKeTJB2ljKdvcruBb6iutRdrr+v69TKNNSPoz+3NGv8wzE5OCBsJAx7jpz61gXvh/Tpo3Nmcbzltvzqx/2kJyD6EV8yNrF6wVDcSW/ozAx5/wCA9fx/OrVvql1A/wA7NvxjdnaxH1HUfnSkpdTWOHtrFnaeI/D89shkjkBzn7vLH/H8ea4J7dbaMgnB756/WtSTVJ4fmmlabfx8xz/kiueur0yEq/8AEOorhnG7OuN0rMvaO5l1OIICQrLkgDPXtX6l/BuK/ist0ixNGRgOOXHsSD2+lfm58P8ATln1qxVxyzMVXqXKgnGPUYr9TPhppj2OkRMzhgw3Kdmxh6qw45B9RXp5PC9e66Hh51P3LHqlFFFfWHyoUUUUAFFFFABRRRQAUUUUAf/U/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA83+J3hBPFvhqezUDzVUsp75HSvzh0zRprHxSumyr5cibotxUEljnJGRwBk5Jr9ZWUMpU9DXxD8evAw0vUW17TP3P2gYZunzH0Pb/wCvXz2d4e1q8fmfQZLi7N0X12POb/xvo2kXC6dYzbvJHGw/Jk9SXwcsxznsPWuevPGFvdsIJtszkjCCXaozjoGA/Mj6ZrzO1DbxBOxjRWB4xufHbkdOO/15NdJo+leJ9Vui5/dKAW3EAIvP3iT8zEA8KPxwK8SNSU9UfRewhBXZ041vUpZozY2kS2iZVXaRgm0dcAAPKxPXbhc9TXULq18BstoFVtvL9CPXAJ2gZ4yWPPSsx47LS2MWLjUrhFziMMQCP78uOT6IgAH6muJfFtwoWxs0sYF4KFtr8c/OcnH4kn8a6InPJJ7IsTm38wGSHa8g5JVdzfTHzY/AA1mTW7pCIleSRGPIZiu38+g9K1LWC581luctITk4c4YdOCeMenHPtTrmwulbcibQPlYpHgj+7yTyB34rOqnYSZwxRAmE+c9VOfTkr2zXUnRxrFxBeoMIzKzDqPlAOM/h+lXLbRYpmzcLhkOCQMZHXP41uSkROYoV2L1BH94Zz+vNeXVnym8feNG3uYNLtTASDtPy/jWnp2qCbCR8bwTjtnArza/mmBKseuR9R/8AWp2g6hcy35hGVVAQSeg61zKtJs1dFWbO81XUngkBhPrkY5ABxx+lJqF5K9qlsowZdvPf5jgViTKJpHM5wWT5eeucevYVrpIvmJLs3CMgDPP3ec/hVqV3YmySRjW7qdXs4JwzKu8/98L/APXq/NcRy3cUZbKld+PTOaclmtnJNkbnV2MfHODjI9qyrSOZdR+2yYC54HYg8H8qPhuh76o20uUsrgup3ALnA9c9/wADWet+txqEczfdQAjPoetYOtXpt7mNs4GMH3wcf04pLOVrmRJ8AbwFyOePvfn2qOdrYpQT1Z2mryzSWx8oHZIOo+nFcppV5FpP2m7vOJEUlWbrkgcV0lzdPFagDO1FLEfgAP5153rcAv4JZnk2JGhLhOCSeAg+tbueqMEtLHmWt3893qJjjcgznIUDcSc5UAe55yeABUR0hUjI1GF3zwsioRIT/sYzwO5OM9qZdaVeQOivCZbuddwWMnESMcBSw/iYDoOceldhpPhVrC3e41mMWkBA3BZGWSUsOR1DBR+v05r0KdLRMcqtkc/bamNIjWxtlmSCUnMsICuD6qzEkkd8rjsOma0P7WdZ49M1icTIqlw32hCxQ9CyNGdwPvyM1pS6Hf3iS6hYyqlo67VGI0TYv91eWIXuWbnrXOP4VttUVPmAgiG9jA2ckdxJwD14AyPYV1xgk9TGUkzvdP1Tw00iaV9vktcHPBUoB2wRkD2+7TtX8C2OrRC7s75Z8qSwOwMew3K0bkfUH8689/4Rm5uLOTTVaG/hGfLS5BhuIixzkSxg8Z/vDHeoYtI8W+Hbdbq8nlEFuxAZVacbCuWVigb046D8a3gu39fqZtdmZV14V1HTrloEI6FsR4lcD1OX3gfVMVjtaGzVpZJBKD0KqU/MN/StDVdZ8RWNmTfPb6jazcxMr7Jx7eW4Xkd8AE9a4aXV/td0oVTCwAypGD+NOdzopyfU6B4JWAdzkFQ351SSMySIHIALZ/AV1+nXUVpYxSSxrKOQc44H41mazZzG8NxbHdDtGwk889vY15dSerSOhHpXwaistb+LemQt8lpZW8i84wXZTnuB37mv1Y0WC3tYEhtSSrYIznIUDGTnnntmvzA/Zq0wf8JHqWsMq5tIQi7hn5pGx+PAxj3r9MvDVwJrfibzmJ+dlA5I498KOg/HHFfRZLFKnfqfK53L97yrojrKKKK908MKKKKACiiigAooooAKKKKAP//V/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAK8b+Nmiw6x4Ou0lG4hDt9jXslch42s3vdBuIkA+4SfpiuXGw5qE15HRhJ8taMvM/M3w5pVtBE1/rJZvJ4ZQpy/PyhfX8P5V0Fxr8d5GYEjZWX/V2sJB5HTzJCcE/wCyOB7VX8R2V7bWT2gDD5yAFbaCDy2cdP8ACvNrtDaQhbEhHJ+Z1Hf+6uece/U+1fBRqSi7H6AqaqK53hn1eaF4YpILEZBMSSh5WbOPmbDhRnk4Vjx8oHWrNtbLczeUZJ9WnQ/6qFGggAzjAbLSN/tO5rzBIbnUn+wm7MbjmZtwwB3XAIOAPTqeOgr6X+Hfhq2to47u2mVkyMtyXkx03N3Yf/Wx3r1MOubY5MTJU1dnR6J4T/0NDqlqLfI/1ceWVc9QW6tn1ySadfeH7axZltgie0WM7fqRn8K9MWa3iGRhAOuRgj3OOv5VzeszQGNWjQHPQriuzFKEaZ5FKrOU9TzEwyjdDJ8yqT+HuPp3FQG3eQHI6de45HWti6LS/v0BHPJPb61T89YSUbAyOCOf84r5mtFN3PZpt2OM1aJvJZh1Ukfh2NP8PKx8ySRQCxA9+oyao6vdCa58tMk87ucdOePwrpdKgNnaAznceCG7HP8ASuOK1OmWkR968ZkMZ+Xym2luvy4J/nTlnFtKNykEl8egz09ua57ULlpdQFuPus4BI5OScHPoKtIbi8iljAOYuFA9evHf1qkm3oTolqb32oTTWzZw0ikEf7QIPP1qtfI9vGFQFsKWH45IGPxpLVo474goQSVb6ZGD/wCPU/W3jVGlZsED5cHp3FW/MjrZHmt9ema4EUvEhJAHbJ6fqDXY6Y2YUAOVb5go65HX8K4XUvKub1rqFtz5JwO3A4H410miXJa3Dn5cJkL3z6VLWxq9jrr25RN6hs8bfXjI/WuUhs/tEjXtwB9nViFT+8x/ngUs140hECnLs3bng5q2NrQwo+T5QBKg9c54+projHqzmbsUkjaO5a4tTvu7gqMjkxqOMIF9up4rrLTSpLmNWuGfc5O0OFI/Uc+5zgdBz0qWJSHdPMDGeXcrxwBgD8+n09q6SwuBLsuCvlr0T+8wHoTyBXfRmc1S/Qzp/DEUtu1xf3DyTKQI0hXa68cBWGCPyrgPEWi3VjCUsZ3jveHEarnzQOPnAzggf3fvHrnmvclmhwGuQHC9IxyXJ7HP/wBYV5d4xb+1LecpMXbOEgsEyPlPCNJuTdz15Ar0lytaHPGcr2Z4fp+s6nsktLpEuIELYN0vkFW4GEZXRcnPYqT6VvweKYbRZ7LxNo84gVUJmhUncT0AAJB7DGeMHr1rkNe8B+I9de2lvYpWkhZjKGCyO5GNigoNg6c88VDDpniLSk/sy9jmSNidgh3s1u548wN8oJx2J55JxxVtNf1/X5G6SZJr7eF/EF3KtlfzQ26H5zDM2BIv8DwzAj5Rx8revXNefNZ6le3git5/tMY+6NzDGPUOFwPpmqOv6b4ntbhIDPIVA2RRxOWURk/fZ/VvQ5JPWruk2N/aWbwvM5dyCzbiQB6c1MnZXOinE0JGnhBgJOQe/tXo3hm0ku4Htb0gIvzKeB06/rwM1wGkaPPPqii5dnjjBJQcZOeM8c8V7Vodt5LF3G6V+igcIteZiLJnQ37p1PwKsZbHUNetHwcGE7RzySf5Dgmv0T8HuY9MjRsDPAAAHA47dq+KPhpo8lq2pXcW7N1NGuR97CDn8MnrX274TsWtNPiLDkqOTnP0yf6D8TX0uUp8iZ8jm81Ks3/Wx1tFFFe0eQFFFFABRRRQAUUUUAFFFFAH/9b9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigArN1gE6ZcADOUNaVMlQSRtGejDFTNXi0OLs0z88vGVlc/aZUAQRyk7lPcH615RqOjyGDChgwPynHIx0zzz9a+r/H3hwW95NEqfeywJxjB9civD7/7TZrsVRDg/eYjGB1I718BVpuM2mffYTEKUE0ct4N+E19rFyl/ffJDncMFhyPY4yP5V9P2OmQaPbpAIlBUDnuQOnP/ANavC9E8SalARFFfROwzgyPhf/QT/jXfQa9qzw7tQVSOzRnK/h616FCtCENFqceMjVqT956HT3WpurMqsR7Hkfga527upZPnBJ6npVmZopFE44DDr15rEu3Cjfnp+ePauLFVm9bhQppFGa5kQtkfK/A56Vi38rpGVjGCuCe9SzSmRiScZ/DNZ91cMVBJ+YHaccH/ACa8mc+Y9GMbHEXErJfgOdpkJBPp2Ga9EtHd9PRZOrDcc84H/wCuvOdTtZ7md2QYCr25OPX8D60nh7xLPcf8S28wOMehOKmETSeqOg0h5H8RtDL8+1xJgf3CwBJ/OvSYdMe1vLdI03ASukh9QjHj8M5rndDhgi8Sw3T8R3KyWjccBiMrz/wEYr3HStOWZPLbq+JQf9vaAx/EivZwGEVSNzzcXieRnid3pstlrkk3mfuWYoR/dPpz+B/GuU8U3TNB6ZHr6V6F46dbXU5geEnCHPTDL0P9Pwrz3UtNudQs/LY4ffvHsrZx+o/WvNxELVWkdlGV4qTOTsrYzwQmOTa27nHOQxH8q6jyYEt3uIyBGgCrjj5geSf89ax4YJIE2YP7pOe3OT/LpUP2nMUaxL8jMWAHPAJz+FZJO5pJ3NolVVJARuZCp9RkY/PrWjaQbcM5wEQkY7EjjPuaxrhJ2hDRrgF+P93j+dbTyC3eEEgAHzGHqccCuqN2kYSLVvIZBlvkXCrgjOMgA/jitV7hkCSxnduAC5+uOf8ACsCBWnMSFuZCWznso5PsOa0fMY+SsRG4jIHXqeP0NbRRjIuyTkxuZwWydiqM5dj16f5x6VesoNPhVVvz9ocEnltsYPZQq4zgfhWXEhXYCcKg69yT7e9UrmS6t5d6IT2T6nk9a6YTcdjJxvoegPf26lS3k2qAYCooU4zxjqf0Fc1repaS0DxmYIeeMAtyepDAj865yLzZiFQh5DycEkD1J/8Ar8V5/wCInmgcwxzF5Dn5t2FXngLxyffpW3tW9yYUlc89+IPmxyhhcquTu2vguw6A4AJHtnGfSuU8PsLkEzyZwdp962da0eW4AjjnSVi/ziMlsE+pbGD6muet7SfRrloG+bzeVx7d/pTc7xstzvp6aM9U064toPkiABAH1J7Zr0PSrOTyxc3TMo4bkYGR+PWvNvCemSzyCeToSG3YBP4civbfszqYY1UmMkZyOw7V5zjedhVqtlZHtXw40hp7aETzmNHbeRk7j+A/rX13aQQwQokK7VAGM8nH1r5y8C6c8t8hi2QqiAAsM7R32jp+J+g5NfScQ2xqASeOp619tgKajTsj4rFzcptj6KKK7zkCiiigAooooAKKKKACiiigD//X/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAPKfiDp0NzEZM4dOlfKHivTmZXEIChucjk5/Gvr7xrcIITgD5e9eBXNtBcS/v1LDnv/AEr5HMYp1pJH0mXTcaabPn/RPCV3dXzNcR7UXHTaCQfUY/8Ar16la6YumL5MUxkjxnnqB6HqDXXeVEkey1QKMcN7/WsKfadyP949RXmzXKen7RzepWkdVUiPkeg6evSs2SP7WV2dR07H/wCvVi4gkHMeRnv9arWl3HFNtvwSgP8AnHFcsnzOzNYqyujEu7G6hzuXjpz6ensRXPXSSRgrjGBnvyM17W9lp2p2vmwsA6jB75x2OK881pIU3RNwQMH29/pSxGF5FzJ6F0K/O7MoaXYx3Nv5pXJIK9DjB9a8k1G2GneLolRDsZwv4k//AFq+idDt/L0/KL0z+R615zrOird649xHzLBJEyDvtLAZ/AmiNPReY4z95o6uKwk/cQIuZYphIo9SQT+lfTFjZbY45gu1imcemR/SuTsvDcb6nBdYG2PqMZHzL/8AXr1CO2VIxH/dGPwr6vLMG6cW5HzuPxSnZI+cfFWmLPrUcc3zKcqdwyOexrnLnSzbIsMg5RcZHoOnP1Fe3a9pGZvOKkrnt2/ya4PxDCFjYxjDFe/9K8fGYVwlKTPSw+J5oxijwnUdkczxIu3JOfoxzUWm6W0ske/O1R2HQY5znvz+vtVi5KRXCMVIIyST04zzXe+HrMXarwAOO2R+NeZShzTsejUfLC5CmmgwRxui713O2TkjP3R+Vc7PDFLcOWj3IoHDcLx1JPU5xx7V3OtXttaZsrI7mA5GOWPcn1/lXCCK8lmlMrhjkBUXJA+uOv0reslB2iY0m5K7IFZpZGf5VVvvED+HsBWnE6uJGjXHXpycYx1p5sblgzyFE42/Pldv0Azz60FZLYeXbOX3/edRjP8Au9h7miMhyQIRDtHKhRk45JPHAqtLatesvnDG4846Adl+vrWgkiWq8KC/dmO7HsoFTRN55DsvA5Abj8hWikZNHPXljLCBb2rAxkcgZ+b6nso9PxrDuvCd5qakxys+7KkopX6jcckD3HJ9q9CnEPE0cSs7nuSefZafG91G224+RjjIPX8AKnnaY13PKn+HtpC6iNN+zqAensM+vfvW7aeEoY49pgQ5xuyM/wAxXp0ckbMFEeAPoD9eelWGtzjMakg9iarfW4nVZwlhoFnAxMW1JFz2B5+hrbjsZVnUyNnBHTp+lSzsUzwAyk85B/wqG3vvOuY7flixA4/+tSh8SIm21c+o/h5DboTLkyucHaoyAAOCT0Htk17WpyAa8t8D2XlqsTsWWJRhc8A9yQPy5r1OvusOrQR8hWd5MKKKK3MgooooAKKKKACiiigAooooA//Q/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKa5+U/0p1Nf7pHrQwPIvGkB+zs0ORg5OTzXkJaJE8x8jPYn9a9h8YoUh/dsx7c9PevEJpI8iEpsKnORXx2NdqzPpMGr00X5HMdqHDAHtz1z71zN0yZzL+Z4/Wr107Ngswb0A5rIlkAmUuCM9Aozn8zXmVp3dj0aUeo6GESMZfMDBeuWwB75I/lSXdudm5tkg7YGT+dOutesNLtztT7RNgEKCAB6lm+6oFeL+Ivj3oNlcyQxtDdzxn5vKVpEX6udqfln2pqMbcq1fkVzS3PdbKRIbc7pSoboCp25/L+tec+I5gbxRy3bjkfpXzndftPRxT7LiJSvIzCsikj3JO38hiuj0n4xeDvFUyQtcGGRgB8wxg1rXw9XkXuu3oKjUipN3PrLw9AF0yNV+YngZ9DU+l+E2u9Zhu3Xd5TgPnuByPzPFc94U1QwwLaTkbVI2kdCuOCD+Ne+aBGrW4mGCWPUdwK78Dho1Gr9DixWIlTu11Ny1tFjxgY4H51rEYFMjUdae5ABHoK+nSsj55u7MXUY1eMqehrxbxLhQcjgZB/CvXdRuQiNXh3izUFjZmBPPbqTntXh5tZxPWy6/MfP3iS/EFywzhcE57ACu48K6ysOmhwTGAPmY8du3px+NeY6+yWxn1G5OYUIK7j0HXHPB54r5v8AEnxR1R7trDSpCQn8QJ2qT1wP6mvCo4ac5e4e7Uqx5bM+1J9b0yS5Ms1/HFnKhWcA7h1ODk/nV7StX0GZALa7EjgckFd2fYEg498V+b0fjrxa12sFjI00ruMAKDlugA/wrpP+Fn+KtLums9bsofPhO1w0YRlI90wf1rteWVl7yVzk+uU/hufopE+mToxEm/Pdcbvw3Lj9TVG9u9Hj2xrJI5XtINwyP9wgV8Y2Xxe/drJfQSxsoBDwyFW+pz8rfQj8a9c8N/EfUvFkQV4vMQHIm3YbA6cco30IrCVGaTvE1hJN6M9be/lllyJMjsuCn6d66K0UvGC7Mm7nIAHH868vj1u1j+a4wpHLYUJU6+OLSBglpl29Sefzx1rzlLleh0yg2j2W1gtLL98AzOc43N/QVJK9tg7l2N3APLfXH8s15Gnji7aQbtoJHAyCR7nFbEHiKW9GHcIWHRVOfxOMfzrdVU9DB05J3OnS/EUhiBGOe3f8P8avxXSsDucrnuAST9BjAriWX5PO3YXH/LPPftzjn8atW+pRsnlCXjcQATnIHHP/AOupUmnqOUU0dVcp5illJB7bufzrK0G1kGsxyEkhTxsGD+tRtLbRqv2gk7uyOo/qa6Lw5brJqUY6AHIBJJA/CunDrmqRsc9V2gz6t8EbxbKGQru+bGPvH3PfH4CvRq4XwlZxwQiT5tzgdeM/hgH88/hXdV91S+FHyU9wooorQgKKKKACiiigAooooAKKKKAP/9H9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAqKVgqnNS1RvCQhpPYaPMvGE37rhTg8DsK+etUlMMjuCcLzkAnmvoDxax8n5evc9/oK8F1kGMSSK21hkgDknNfH5qrVHY+ky5+7Y8r1DXL2SVxZZ3DgK2efyBx+NQofFd+BGJYokyAxDHdj64/pUltaveXTLGzAk8vnHPsAea9d0pEjVWKscDBbA/XjivGoUFOWp7NSqoLRHlvijwrP8A8I0yzMzs+Nw5A54/zmvhPVrA2Wi3c7KN6yuv0IcqfyAGK/VbXtIj1TQrlLXhvLJXB6lecHqa/PnxxoYF5caPGnli93Tw9MOxP7xf94da9dxVBW6bnHCbrRfc+W9Y1PRbm10+LSjOZfs4a9aZEQC5LNlYdrNmMJtwzYYtngACqV3rdrNo9jpsFjDBc2sk0kl4hfz5vM27EfLbQsYX5QoBySSTVbWdIm0i+e2uFIwTjjH4VVt4TcSLBCuWbgDHNfUxrRac4bNfgeH7Np2e6P0G/Zh8ay+K9Pk8Ma45e7sFEsMjHl4iQMH1Kn9K++/CrNDD9mfPydPoTxX5P/BWZvCnxJ8MxEjN27wSgH+CRD1/ECv1k0oYKOo68V5uD5XNyhsb4xNQXMdyjAZx6VUuptqE5qVG+WsTVpmSAgd816knZHlxV2cPr2pPGvB5NeR6tDLqFyRuwqjn26/4V3mqt506555rH1S0WGylMY2syHn265z9ea8WvT57tnsUHyWSPir47619hthptp8rPmRgP7qD+XpXzHptqtrpM2oOvmsE3nPGSeTzXs3xavDqniy4UEyIuIx6Y6kD615RpEJkt59LkP7xASAehTtiudR5KN13VzumnzW8jzSHUJYrmO8G1pI3DgMoK/Kc4Kngj1HcVY1vXNR8Rarc61qjK1xdSNK+xFiQFyWIVEAVVGeFUAAcCnanpU+lXBEiHYeVbHY1TgilvJlgt1MjscADmvejVi4Xi9NzxnT967Wp6N4UtZtSsH/eiIFWUk8ZXByCfQ19ifBbwzBF4Tt5TFucRiU7x0LuSMjPPH5V8++EfD0sKw6JEP8ASZ1BkI5Ecf8AExHqei+pPtX6N+APCsei+G1S5hbdJ8zKMfKB90fUAV4Ln7VyUdm/wPUS9nFSZwkvhC3u5WkMeR1BGCgz6Z//AF1xureDdPthuhiecrnCkhFH5DNen63fxqzERkKp6ucDr0HTNcjJrGZvsu1W3Y3KcHj0AGa8ao4KVkd0JTtc8ourMRNssYVjY5zsO4D23HFZUV5fWcxWQtuHT5gT19icV7zfaakluWFkhBGdxHA9MDOa8p1p49PhkkW02c5O05J/76/pSdEuNZPQ6uwvWmjRpR5hA53nJH41biMD5SEqSpyVPHXtgf415dpHia3mfyYIPJz1YMOD7g8A/Su9h1iNgInnWTH96Rdw/LNZvTRiaOhY3NvGXhQomMcbf55zXY+CLmd9Sg4KZ7g5z+Irzldc0+NgId0znjBZiP0BFen+CtUN1qEZiVYsdTg9fxArqwlnVic2IuqbPsTwpBdiDzHhMUbfxEjc34dh9ea7ivPvCchlAWSYOV/hXJ/E+leg191T+E+QnuFFFFaEhRRRQAUUUUAFFFFABRRRQB//0v1SooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACoZoxIuKmooBHnviawtorKWaUc9vrXzn4g3xK46s2Tg4HPv3r601ezjuIw0kfmFfuj0z/WvnbxhpgSZlZPlYfd/wAa+Zzmm17yPcy2or2Z4DpVxGdUeyaWISMdwVVzge7cAfmc1ozXN5pF6YjIyK2MED5SPqKzpNIgsdZW4s1CEn5mAzj+f6YrotTto71BJICCvdh/9fvXz0oNw03R7vMlLyZ0ejeKnKm01FswNwGZgMH8eo9a8e+JHgCHURJdQJ9ps5X3hocmSGTu8Z7/AExzW1dWk0CEwKsnfHXP4VvaV4guLVRbztuTHzRvjb9ARjFaUsW2lGp06h7LlblDr0PirxF4VaVjDqFmNRU8CaEgOf8AejcrtP0JFYeneHLTThjStCnaUnBeYxxqD6El+BX6Fz2fgjVjuvbUI0hxnaCD+IGKuwfDrwJMyz2llE7j5i5Ab68dM1301zLlg9Oyb/I55ezT5pJ/cfDHgnwlqw8Z6d4jvmR/s0yuwj+4o6ABj1xnsOa/UXQZhNYxyDuq9K+XfFOmm21QfZbdUSE4wvAAH06/hX1R4aiEGj2qydREufrjn8q9LK6rnUd1a2hzZrCMaUXHqdQvMYx1xWBrOfL2ryK0ROVYqvPFZl5i4jIfo1e1PVHiQVnc8uuIZftrOQSAQM/WszxbdTQaTK6DjB5HOPpXX6jGEO7bjHX1rhtdt5L3TJIZG+RyowBggE/0/WvMrxtFpHpUmnOLZ8P+IPDM+satcy7uGcbRz1x2NcrdfDrUopvtjC4t3QfJMib1Ge5wMlfUYr6DsNLkXxHdaXKdht5sgHjj6fka+j9HtojpyNfxgov3tuDn9OteZSqTb5EexiHBK7Vz8210XVArWGo21vqHP31doz+TIDW5oXw91a6n8vRtOW2aT+PmVh7rwoB9zn6V+gV1Po1smYbVbl1ySCgDAE8HGMn69qoPr1vbWYWygigYnnK7frjI5NYVJ04XX+f5bGMUpaqL/A8v8B/Dix8JL9t1cqkmd7b23M7gdXbufQDAHau68S/EG28kWOlkxbcgnzC2c9RtX5fzNc/qcySsJ76XMg6AuCPqQv8AKsBIbSJPPt4DcEgYbb8uT2BbmvOq42esYM6VQi2pTIb7Ubq5jEkvzSDuw5x7AZ/MmptEtY7MC4chmc5Cqfm/EVQFvd3VyxkOxF6jAOOenr+dWr+5i06MzK6wxqPvTBgTn0xWNODfvMc5fZR15mSYGWdTx93c24fp0/OuQ16zt5o2a8yARxiQjj22jj8TVDS/EUd4+UkQM2MZUnOO4yMYrpp4ru8GQx5HHOAffAwRXbGSasc3K4s8UvdDWc+dZsIgTnEyqc+nzAg8/SrVho2oqMm3THTOQBj1BzXpF3pdykS+c25vYlwB6kHmmSzW9vAkUywXB/uSRl+fbjioku5upvoZljoMi7ZLqTYAMBQRn9OtezeCrZ7e5TycjaO/615rpkFu0vmrbR2xY56dvrgY/HNe3eG1MKggbzj1H9K3wME6qZy4ubUGj27wvrrC+WzVERRj7uMlvfuTXs6klQTXiHhKWOG7yyrvbAyw5H+78pGT6mvbYyCgI6e9faUG2tT5SokmPooorczCiiigAooooAKKKKACiiigD//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAI5V3IR3Irxjxnp0bltmWYjOa9rIzXMaxpC3VvI7tl3HX0HtXn5jh3Vp2R14SryTufEes2Est2FDnZnpjjjtUes6jp1hCiXVwVAHOR0Hpn/AArvPF2kz2F08XK9SCK8E8RW8kiMJGCDoDyTkemOc/TH1r4uUnFOJ9ZTSnaR0X9oW0A3xuXB5UxnqPfioIb61uX25ckHJG0nv2JAFcTp93qj2eLqFo4Y/uyNjc/bkc/1+tdboiRXimSAYQfebG0k+xwM1xuMuax06JXOu0/THusRwOAp6BnzjPsB1H1r2Xw94afSrUyR8u464JH4lu35D2rM8F6fYReXIRMzOMBucYPuGIr2eCzjZf3mTgY5Oa+qyzL0o+0lueDj8a78i2PLJ/Dh1K4V7lck8qMdh3P1Pau8hiSIR2yfKFHStmeFI4zjgjpXOfaAt2hA7f5FezSoRpXa6nnTrSq2T6G5thjQ4A9v6VkXTxmPeuOmc/Wpb2VlXCYySOPY1iXNztUkEHcfm7Be38q0lImEOpyWrSckAkfz/nXN+V9ttDEVww5/Kn6/eiPLyHksOO+O2fQVT0557q4SS3kGwfe759sDv9K85zvOx6Cg1C50vh/wJp9tC018guJ3eRw7gbgJCCV3d8dB3xW7NoqxM4t1+VhjB4+tb1ioSNS2V6fnWuio3TJ+tdP1eFrJHE8RO+rPD77w9eQXHmQLuLD+LnPGM4PPHf1rzzU4VhkxeReU6EjLIXUjPHfAz9BX1NeRRsCEC5HUEc15D4nlTzNlzbouPlEgYbwPUgbTj868LHZfGKcos9XCY1ydmjwm/klhQzRQMwOdoSNdpwepPWuZfxDdTstsYwijqCPmXHoWIH4ium1qaztGB84sjEgHC5/DKgiuTuLVp22wRGUdc7tpGeeg6/nXz86bjLU9hSTRv/bJbaEH7SuOuFAc59toGK8+11b7VJTPCAXTplXf8SckA/pXW2WmXBA89iR/dXgjPsf/AK9dFaaQQplSPJHOR8vH4E/pXRTTMHJJniNvdahYF47mEKc8tna36Cu60vWdRt1T908i5xw+7j24/pXeTabHIux7VAPVuSPwbIrKXwuZXzEvkr0ygUNj/eHP60Sg90UqkXuE1ydQVAeQTn/aX8QMit/TdJgVFe7m3dcCT5m/PH9auaT4TtbZ/PJdjnJJxn8a6wW6uBHERj0OK1hSk9zGdVLSJmWemws4+zKeec7q9B0WEwyKjgD6c1iWsFvEvzIQ3+fStvTnKSBuQB2r1MJTUZJnnYio5Kx32m3z2eoKs0m1X4AGBn8e34V7/p063NokqABSOMcivmWZBI8cx6gjI45r3nwzqUl1ZxiX5R0HGBx2HTOPYYHrX0OHlq0zxa0TrqKKK7DmCiiigAooooAKKKKACiiigD//1P1SooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACijpyaj81T9z5vpzQBJUcqb1Ipu6Y/dUL9T/Qf40hSYj5pAPoP8c0mNHkfjnw1LcQNdRgNIK+atR8OW89yJZ0JYnlR/WvuG6to54yjSySZ6gf/AFhXinijQGs7g3UUDbepJC8fzr5nMcvjGftY7dT3MDjXy+zZ4M+jozYkzsThU7n8O31NWrewRZFaCIjtjOE/kK71zZcyrD5hPoq8/iQKz3vNJhP/ACD8sDnIfbz9Oc/lXN9Xho7nV7eW1juvCumrHbiWYgluuzAH6V6IsqRKB0FefeFrp75SII5ERf7xXA/EAGu5XMQEcq7s+2a+jwyXIuU8bENubuWpgWTPSuLnZINRVXYbSCfxNdSZxKWReNvWvDPiJ4hufD2rRRDG25GYy3+z1FViKihHmZpgqDq1PZxPQr3UIWYDdgk8DPXFYl5eQxQku4AH3vXjjrXk58cGQZlGGTpj1PWuW1nxhcyxeXEcJzwOp/8ArV588bHc92nk9XRMteJ9Ua71FDbEKm4/e4HzDGTz/OvUvCdnYQRBrYqWflmBzyB1z/gK+LvEHxEnSX7LprxXN2x4UfPtwOWOz09K+qfgzb3Nv4UtW1An7RIN77jk5IyefXP0x0rDDTbnfuLMKPs6fL2PdI2JQZ4HWq8+rR28ywMj/OeMLkfy4qit+8kzQwI+U6tt4P0J61JFNrRkYzIqx/wkcnHvnvXq8x4HL3NJ7pGXjPPrXKaxBbX8DxnI4wGTBI9xnNWjpl3fD/Trggbs7UAXIznHU1g63Y6nEc6dcYRR9w/Ln8a5MQ3yu6NqSSejPNNW0vVbJ3CXTXELdFZeMehU8Vy7WkjPtNnsHfAwPy5FdPc3lyNxuZYgwJ48xzz68cViGYufkeDPfnP8xXzdRRvorHrxlKwyKwES5ZGAPOMFR+gpzTKM+VG4PuDUkMNzMR+8gAGeyd/wrZt9NuOu1MjnKhMH8qhR7Dcu5nW2JF/eRjjqc8/rWvHDC46KfXjB/MVOsMqDcVwfoMfhUbPMGG5VQ9N23GfrxWsVYhyuNKjA8uQg+mOv5VIkQdMrnI74qSOIOf3hGT3FX1gfdlX/AMa2jEzlIzhJIMCVmA+hre02SEMOSfqaqbvL+WQBj7Zq3Es24OB8p9RXVSVncxm7o7Zo1nth0yBXongeDAy6KCeN2WLN7AHov0A+teb6ZcsqeXMCVP6V3Hh/XrfSrr7LP9x/utnHJ7EngfXrXs0mrqTPNqReqR7TRVe2m86MOOnr0z9ParFeicIUUUUAFFFFABRRRQAUUUUAf//V/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigBCQKQ7j04p2AKKAGeWv8AF831pdwHAGfpS7c9aWgYzDt32/qaaIh1PJ9+alqBpiWMcI3MOueg+v8AhSduoIkJVFyxAArC1a1+327xJGcMOrfKP6k/lW0kIB3uS7ep7fQdqSSUHcqY+Xqx6ConHmVpFRlZ3R88y6Z/Z9xJB5atzxkkj8uKoTWtwp3RmKL0wij9SP8AGvTNd0txIblUwGPU8Mf8B7VxU8e0kdTXluioLlPSVXm1JdCe+iRlMoK+u3H+Gfyrak1S7hUgRbj6+v4Vx7PcQsohJDdgP5nsBV6LUruMbCPNbHJHU1rTq2XKROF3c3v7dBYpNCyYGc//AKq8k+Jnhm08Z6dhXMdxFny2BIK56ke/pXXXPibYVEsDBDxwOCa53VNesJi1ugdGQZYnj/PH4Ac1OIqRlBps1wynCanFHxmkuteGrWbStWleaS2P7uST7zJnGCe+PWuR1zX5NX0maxWUxmQbXKnBI7j8a978bJo2twrFJukaQlUYAjb7k8cCvmiLTLDS/FUcV7KZLZmJG7uU5BNeGopyufZUcdenaS1PUfh18ONNtbJLqeIok+Nx7EA5x6g5HXNfWelXrxR+d5LCNR0UAuw/+vXgtprUjuHtYWKxxqVGVCtnnp0H6V3Np4yvliRI4DuYgbEC4Hbk5/lWlOryy5pHi4pSqHqsfiDxDdxILHTtqscFpHwVA9VI5NWZW8Ru4InWNOpHUflXmv8AwlHiKdlaxljhA+8r8tj2we1XHvtZuSv22+2oRjKgEc9+1dX1uLR57oNPZHoBXVJjie9TylGdqjaSfqSax7m80u0Yu11uc/7W7n0xxXFTwWkWTd3TTKnUhiSPwrOub2NxsSITQcfP/EPr3/Aiuari/IuNE0bu4srqfM8Ko3Y7SufzK/oTWfPZ2G7OGjPX723j/gagf+PUkM9vGuxHMat64Cn+aH8QKtq6oAjDaGPBX5Mn6cofwxXA5c250JWIY7SWPDRSfIe7pkf99LuFaEcN1gFQHHrG2f5H+lLDZlPmhOc/3fkcfh0P4GtJGdMNJhwOpxhh9e4qoxFKRVW08wgSnBPcirrWpiTdksvcZ/l2NTfasx/u2ORyVb5hVJrlFb5Djn8PyrZKKM7tiHaBmL8j2/GhAr8kEZPQ9KWUoy+bGwYfTkexqh9p2/Iw3c8EcY/DpTvYRr+fEo8tMbvftWlZOXx5nJrnhH5uGKhsfhVqK4aNsFCuPxropy1uZTid5BIQAQoWr88giEd5tG5SDnk59q5OzvpZBtAb8Oa7CyMVxAYXZskcgivTpy5lY45qz1PbPCurrq2nrIiFNvB9Pw/wrqK8I8H6kdH1c2LysI5zgf3dw6ce9e6qwdQw716NCfNHXc4asLS0HUUUVsZBRRRQAUUUUAFFFFAH/9b9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAoopCM8HpQBGwaQ7VO1e57n6U9VVFCqMAU6msSTtH40hkcjFsgHao6n+gqP5VUPJ8qj7q/wD1vWkZgSoQbsfdUd/c+1DqsSm4mOWHf69lHvUsoo34M0ewqSW6IPvH6nsK8d1YixuDEcb888cD6V7OEMcL3F1+7BGW57emf85rw7xfqUElypX5MHoPT0rixklGPMzqwsXKXKilLOoTbGMs361kTXT2wKxncT99vX2HsP1NSefuGd2MelUZW5yOQK8+dRvY7lCxmaneX7jELhdvLHGT/uj+tecarquqQOZHUMmCPlHJrvrmRzgY9cdhz3rFmjjdSjLkH2rgrTk3udVKy6HgOsz6zdwzSRQiJpF2lu+D2UV51c+CdQeaK5vXDXEozjrjjpn1GOfSvqqXSo5c7F5Jqj/Z0MDtLMu9lGBx0H90VnGdjrjVPnHSdG1y3uJEs5CXBCqh+6wxlRz6811VrJrke6aIjkD5WH+fpXoj6couwuBkLuz/ALdTNZJNllGOc8+//wBfrUTqXLc0zjYZdaxHM6gB+GHQ5z1/KurspblY1jP3R69j6/SrkcDQoUl+7nH0I6VnXOqRwqQq4J4weP1FYuXYzbNln8va4b5W6Z52nuvuPT2qi17bl2MJ8uSPqB/Me1csNVe7kZW/1UgGO+G7H8DTLUXPnb3Pzp07/h/ntUO5J18U8c5ZWG3d1I6fXH+Fa1t5tuQmeG5x1U4rmoy6tvg+63+SK1ILiYy7GGOMAduKcUJs66O4idv+eZHUdVIPXg1YaaSEgSZZBx6jHseufxrmnRjgqSCR19D7+1XIZpyOchuOR7eorVSZm0XC+9iYGzjt/hUWS3z55HFJEke7cp2P1xVpBC52PhvQjgj/ABpqIFZXYNhBtJ64rShthLgnj3Ax+YpVsWjA3nK9iCCKvoiqo2sHz+YreEO5nKXYlhthF941rxW3ngDBOe9QWtszYOc/pW3BDsIHeu+lA5akhINOSP7oPHatKImDDBSv41LGCRgnP1qfZhcHIFd0YpbHNJ33I5EimdLnJDKeq9a928P6gb2wjMhJbHUjGR9eQf8AOQK8LQRjMY6n1PFeg+C9QVZWsmXY3UAHr749fXH4iuik7S9TCqrxPUaKB0orsOUKKKKACiiigAooooA//9f9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooADVWZ2yIIxl25PoB6mppH2DIGSeAPU01EESlnPzNyx/z2pPsNDo4xGOuSep9apGWMyNczsFjiyFz0z3P9B+Nc34k8WW+jQMF3PIeBhTx6nPtXzP4s+J+r3ebG1l8kvgKAv3B+fWvPxeYUaCtI7cNgqlbVHuHirxnaws0TS7Y4xnaCBk9s/4V8+3WsNrGoNdE/JuwB1yfXA615Fd6xf3twz+Y+1iAoJC+wzk9+pP1rq9Gv4XQOju0ScEqMZ+nTOf0/Svl8Vmcq8lfRH0GHwCox8z1uwjijjEW1pJX+bLdAB3OeM/n9KvtFztUdB26fmawdF+23mREggSQ7nLdWC8AZPQevb0Ga63y1jwu7e3oBgf48V6mH96CaOKtpIwJLPqcVnS2y8jH5V1jx7ztH5VDJaYHA5q5UbmSmcXLAVXEYAz3NZ0tqSp3HiuzltEU5K7jWNdwuynoormnRsaxqHE3SxRnccZFQx27SKZApAPTtWs9mJZxHGC5PftXQLphWIBuPSsI0WzV1LHBzwHaQe45/wAa4XWIhGA5OdxIA+nWvXr2w2qSDxXnGuWbSTC1jYKz4LH+6o6/nWcoa2LjK5SstKJhQ4wT0+lbdvpJIG0YwME/TpXY6fou+OOVRhWAAz6f/XrSmsBbHbjKk801h3a7JdRbHGafppTNu68A5H49a05rEK6PGvCnk100lhmQTJxjg4rQSyQxgN1atY0OhDqnLpY+egHQ9anEAtyEmQD0btW7Havb53DoP61PJapdQYPKtVqiS6hmNYQ3CAgDNKNFglXLEq3qKnhtLmzwo+ZfQ1rxM7KGIrWNNPdESk+jMAaVJBxFcZHcEVehsn3AsM+4wK3YlDkbhiriWidfu1tGguhlKozPigRCOTWjGoKgKc49RUiRFM7SM1MmBw4x9OK6YRsZNlmGMkDB5q0Y5FXpuHoDg1HDk/dbcP1q6FLDDHB/KuiK0MJGU4YHK7gB2atTTpFjuY2mJUZGGB5U9iKhljfHzZYeoNQxMw4PGOmapB0PfrKfzoVJIY46qeKuVz3h68S5skyVLAAAqc8fjyPpXQ13Rd1c4mrMKKKKYgooooAKKKKAP//Q/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooqrcXlvaoXmcKB6nFJtLcaTexapGZUUs5wB1JrzHXviZpOkq3luGI9AWyfoBXh+vfGnVb9HisIQir13KT+YJrzMTm+Go6OV35HfQy2vV2VkfS2o+KNI01Gmup1UqOBkcD/GvD/FXxngBez04cgffJ4H0x1rwG/8Q6lq05uLoZ2jOMlAT24H51nwuCw/djnPIyR6/ebNfOYvP6tT3aWiPcw+TU4e9U1Oj1DXr3UZPt158yIM4PUntgH8K465u2kEr3DyFnYqNuOvU+/tV6a5txH5kqlt5zjPYdPzzmsLUb22gIjSInYdoA4AY9T6V406jk9WepCCWyM2Qrcb1hBfGVXryz9Tx7Aiuo0aWZFS3x0+6vRj7nsBXMKzGzVSAqszMwHUjA6nsK3vD7xQzM0Z3SMcZHRe2fr6VKKlsexaLqUwQ2cIIUY3lfvu3pu7D6c/SuutBc3WSHCx5424xj0XH6nmuDsLJUaMNlA3ylR1Oe3HPPf19a9HilihiSCABCgxjoEHfJHf/wDUK+iwMpNWm9jxsUkneK3NeC1VFAxz70+SFB93k+tEO5UAf5e54/n7+1TMwAGOvpXuRSseTK9zGnthjJ61iz2bSfKB/hXViAk7n6mq9wiouAMfzrOVJPcqM7HNW1gsR+Ufj61cls3K8DNbNtbNjzHH4VNLEzdeB6UKirA6mpwl7p/loWOXduFUdz7VwmtaUtnCsAG+5u3VCfQE817Q1sFDXMvHYZ7CuYi00XetRTSDhDkVzVMOrqxtTq23NK3sFht1UjGwAfSm3tsiWzDGWxxn1rqGhVR83T0rPmtvOJZunat5UrKyMVUu7mJBZgICef8AOaYkQW4eMjiM5B9mreWDbGo6Fev0FUJUAnbjnA/WodOyRalcrMmWw2BtP5ioo7dYgVHCnt71flRmUnutKIgycjg8fSly6hfQorHuO09quJDGORz9KcIjuJPBFTLEcF1OR3FUoktiLEpGTz9P60pG0ZGRj8aa4OcZwaZuZDg8eo7VexJZQ7scA4q6qJ3z+FU4wfoTV2OQ4wwx+taxfciRMiRjBP5irOExnPHrUajseM+lSHK+/wChrVGbK7oDnaA30ODVOMAOQDz6E1bkIIJBGfcVm/vDKMqM9ip/pSbGj0Dwrqht5vs8oAQnB3Njk9COMc9DzzXqgOa8a8PJMmoBWyC/r0b2I6H/ADg17BAgSIKo2jsPT2rqot21OaqknoS0UUVsZBRRRQAUUUUAf//R/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACmPJHEpeRgoHUmsfWdfsNEt2nu3AwOB618x+Nfirf6p5lpYr5MI7jPzV5mOzWjhlZ6y7HdhMBUrvTbuexeKfijoOjBreKYSSDrt5r5o8SfEnXtcmkj092WIepwfy61xrW97cM11cJ52exPr7mswxmGfYgRPZTz+OetfHYzNcRX0k7LyPqMLltGjtqy9CLy5YyXMjeZ6nJ+mBmnXFpjJkDFR0y/PT0q7DGJkxg78YzuwPxwavwrbRZV9uS3Uct/XFeeo6bna3ZnN+RJHEAm2IH5mySWPp+lEFsDG0mNxGR3Ix3+g9TW3L++y4jOGJ5f07Yz/hVKSM7WEzgonqcjI9ccfhSUNbj5uhRnd2VQo2+WvRMcn/erjbrbkzSssarkFiSx9Sff866m7lcIdiMcqAM/KB6/h+Nci0aSF2275cfKF5Vcep6D6CmwSHGI3Wz+GCMk+WcjHTBf1b0BNXtIuo0kIU7Qp4749/rnp7/SqcqPgvIwVd3zFccsewx1J/SqVr5oljjjb5Q2QvU8dzn/AD6VSYmj6S0ScR2P2ll/ePwOcnGOpP8An8a2NMvLqWQRpgKDkDHft16n0rgNAvWuLVUkwiqOSTknPcn0z1I+grprS6hhm+0EkAdz94A/3ey7vXsPevWw9bWOuh5tWlvoekfaGgAhyWYdhzg/zJ7f54uRkx7WuPvt91B/WuMgvpDulXCJEMsx+4mexPVm9h+PpXRWAPmYQEuBlyx59h7Dua96jX5tjyqtKy1N7qOeT3pIrXzH8yUfQU63AlYqhyBWmgUERiu+KvqcUtCMQoo6VHJEuNxFaOwd6gkTNaNGdzBuIjJy3QdB70y1sSkwbHOOa2vIy3ParEaAAn1qFBN3K57KyMtoNxIIzWfcMY8ZHFdDInU/lWPcW/mgpnjOamcdNBxfcz92XaMDoM59qaLPcd/8Qq8lu6Mrt34qcJj5ug71nyX3L5uxkSRkZGODUcce1ijDjofoRxWsyK/H51D5Q6N1X+lLlHzFFojkEdB0P9DUghIO5O/UVaVMNg9DzUwXAx6UKInIymjRhtXgjsf6VEsJU5XtWmYBISfTv7VKIVPXgjvRyXDmKMZJGCuQaspCD8yHj0zzT/JZCQg+o/qKN5X5iMr3x1qkrbibvsOVWA55H608uQNrDingq+Np696hfcowfzFWQU5WUg7QRj/PSswXLb9u5XA9cZFWrqcpk7d304P+FZTyQu4mkj2jPBweP6j+VZSlqaxR6P4c+d0aRsoCDkE8duR6V69GNqgV4v4entjMgDbS3Gc8H+ley2+fKXJzxXfRehx1tyeiiitjEKKKKACiiigD/9L9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAriPFPjjSvDsRRpFknP8ACG+79az/AB34zg0KxeC3IaZhj2H/ANevknVNTv8AV7kM6Ha3TjP4/WvnM2zr2T9jQ1fV9j2cuyz2v7ypsdF4m8TXviC4eZWwh6Dt/wDXrjxbys3zISAeDj/CplgW3JaWRiwxxn1/lTvtZjbaIjyPvHpXyE5OUuab1Pp6cVFcsEV57G4kQebnBzgMP554FUzYK75j4Zf8/Sqt/qEpl8rpgdeh/PkVY09S0XmeTuychiMk596zTTexrZpFlNMypZnMzr/tAD/CpY7GdJDIWWLGcbckir6Rkk7js9hjGPqxwKoXUyBfLRTI2T1fj9MA1o4pamd29ChlhON++bHO4kk59AOBUTshnUC3Z5emAMbR7iqa3Lgnc6x5PVMbhj/PrWvbIJyWZHbsC7bQfXnv+FELNFS0Od1Zd8YEo8sHnbgMf0P86wigUrDAC2QNzOcAH0CL1/H9a7DUbezjzJIFGOOrHHscn9BiuNuJZ7htkA8iJBgt32k9AB0z6Dk9zS2Y1sZjxSzXAi8whE3fe5b3OewH/wBarVrttr8IyiQt8scXJLMO745Cjr7/AErRKLbJvVQjYBO/kADpn1PsKyrGSRNQ807laRtpZx8/94sfQkc+1VEJHeWU7wzNHcAeYTlx1yMcL6D3rqbebzWMrD92gLqnZn/vH2WuMkASdZB845Az0Yn+fuTXRwXWYE89tmOScE7iOnHoOo7cVUJO5lJaHc6VK9xNClx8zDLIpwPm7yOOw/u/4129k32pdlkc2zH5nHWQ57e1eTvfbx/Z8LFJrsjz5By6oeFQH+8Rx+fpmvXdJdbYx21suNgCDP3UVcfmfU9M8V9Bl81LQ8jGQtqdZGv2ZBbxD5z+g9T71owxqg45J6msuGRTIVjJxnk9yR1P/wBatJJB/qxwF619DCx4syxj0pCnc0BxjI79KQuMhRWtzKxGUydtEhCY9qsDHWqsnzHPpSY0MkJxjvVPBaQ46VeAHSmiPBz61L1GUGySB+NI3y+4JyKshVZmI7GoJOAcfwmoKKO3DFenWnkg49e5/lTXDMSV+opURs4btUFDcgkL0owQ3XkfqKkeIHGOpJpwUN97jAp2AIwN3y8VIwXqOD6UnTgilLbxtxzVIlkTqGHHIH5g1ErqSQetSHI5H5VVkAchkJ3e3X8qljQ+RAvK8A1SkeSPljkepqXzWYfKfriovm+8mSO+P6j/AAqG+xaRVkYMpypx1yvI/KsiSIEF4JDGTnjPH5VsMARlBjPpx+orOvFmEZZDz9A2azki4mjpc6uO6uvccV79o00stojO24FQQSMH/wCvXzNpl4qzBWbBY7fYEfWvonwnPHPpMZQ/MvDD0NdeEncwxMbK509FFFdpxBRRRQAUUUUAf//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigArl/FHiW18O2LXEzLuI+UE9/pV3XddsNBspLu+lEYVSR3r4k8W+L7zxRqcmCWTPAJ7dif8K8POM0WHj7On8T/AAPUy3L3XlzS+FEniPxJLr109xLymTySAB9KwYb4sdiJkdyeuPxrO4+UzsPb0AHXA7D3rTgMKxh1xjj5cZz747/yr4fmbd5M+uUFFWS0NJo5powbceXGvBYABR+I4qg2mRiPzbiVkx1zgZ/rV5LiV8NyFQ98YH4dKytSvk25D7snqSBgfjVSUbXFHm2Mq5Fqv3jnHTnH8+taMDGYBidyjgbnIT8h/wDXrDQwSXSKxDhjwqLliP8APvXbWUFrDs3Y2ngLgM3P0JwKmmm2XN2QsEdsIt7RcjPJG0cHsCSf0rEvruMErtVQT/EefyP+FdzKhEe23iRFxwAOBn/PfmuZvbJlTaQuWPoP51rVg0rIypzu9TlLVfOuBIFRu67hwM988fov411SDYE2yKzN1252jHv3/OobXSbYSEySqMH5gMH9MY/PFbUMUJPmwBpPSV+B+A6fgOKdOOmo6kjltTtB5LlW+Zz8zHrjrxx0964wKHukhtznbk7VGW56sx7fj+Ar03VdPlnQuANgPLkkk8flXmV/JLF/olsdqEktjgE+5HJ/PjvWdRWlsXTd0VrphGCARLMCQBnhTkAe3A6CseeNpr6HyiW28swJwf8AZHY5PU/StG1s4mVsjOB8zDgENyQB2z3PJIqxIuJFRywOcKFBOF9SP0AJGT146i8hs6mzXcE+0Zc4HA6D/ZHrnuatCOe5lM0mdxbYBngEckfQDg/lU0I2ugUbEQBVycnheWPvzz6Us8saiOyQ/O2AAOe+efbPp1qktLGXU1fDiifUUnIyuSy+vPGT6E9B7fWvX4LlZHaK0O1Y8B5QMgHqFT1xnJ9WxXjenLHA0kMb7dp3O2eSPqOnp711theTW0W24OyMMSFHGWOcZ+g5Nelg63s9GcmJpc+p65pu5VMmMKq8kc4Hb6k960IrhFQb/lJ5I/z6VzVpfxiyRJX3STp5jgcEID8qgds55+uK2bdSz/apgFQkdeOnT8B1/D2r6alUTirHhVIWbubMUkjAyuNuecHsKsJwCx7Vh296t3KJfuxgblH+yP4z/T86sS38aYij+Z2BYD8M7j7V0KorXMJQd7GtvBJx3/pTJHVBj0qhbykRbmOWbGCe59celSRFXdpCc5OBVc1yHEshiy7iKfIdv4YpjSBRgdqqtKWUu3fHHtTuKw1jsVmH8RqIyDbgDkkflRPKqw575/Wos8YIxgVDZSRYhVc8c44qVo8AnHSooSFBPbBP5VYLDaSeOtUiWVyAOvSmOB1B5pk0yquCeCKqeeSwH8VS5IpJlsPkevtTCw/h6e/amFgenDVF5nzfz/8A1UmwsK7lTkDj+VQlwwJHOKSSZVqjLh8unbrt6j6iolItInlUsNykn19RVYybcEHPcEcYoVpWG+JskenWk3LMN5O1uhPSouUkKbnzFOQC3TPr9ayZpwARICh/T65qe5SWP5miyR3H/wBbism4u4dm2XKqffB/A1nOdty4x7Ebv5c4fuec9z9cdf1r3fwCUezMgIycjKk/kR0+hr5ze4jZtsE2/oQOjcfoa95+Gt2TC8Hl43c54z/n+VaYOd52JxMfcPWaKKK9c8sKKKKACiiigD//1P1SooooAKKKKACiiigAooooAKKKKACiiigAqC4nit4WlncRooyWJwAPXNPlljgjaWVgqKMknoBXyX8V/iDJeTS2FlKWt0yFUHhj0zgYz+NedmWYwwlPmereyOzBYOWInyrY5v4neOBq+rHTLKTz4g3XkL9fevN4woXbjbzz/tH3/wAKq6ZZTXI+07QsshPzOedvfA6DFakgit8kMGIwAwHA9ev86/PalSVSbqT6n2tKlGnFQj0GCJ9yMo+6OAe/19vatCB7VWMkwZwOw6k/lwPQVyj3xhk2QqZJJOdzDP8AOtC2v7mOEtNJub+6nQfjjr9BUQavoaSTOlaXz0xEghTHHGW/M9K5S83zTPFb89ieBgd8kZ4/WrzzXSxia+bYp+7CAN7E+pPT271mRaq8JVZrf5c8RoAQfXPBzj15rSWpES9p+m20lwH2+aSPuoSB6dckmvQ9Mso7cgGOG0D/AHfl8yQ/QAnmud0m5M/zSM1onoACcegxgA/rXbabKxO63ilKk/eLIg9TgjJP511Yanqn/X+ZhXm7W/r/ACNKa1WO3BkWWRSON+I1/pXFX6Pknaqgduo/Wu1lBmLeWgZmxjLF2AHuK5fUrEoSZsDnnHHP61ri46aGeHlrqYUUzg+QoQ4+YAYAHueBWxHIk8mLu4ds9ewx6ADt+nvVKKFUJWEcn/ZJx79q1rOxiE4e6GXYbiM5OB3bsK5qSkb1LFTVI0uItqgsgIwvb0/z3NcLqGmQq77tzseQTwAOnAHH9PavWpRCwAhiDLjA4647nP8A9asOezklLfZxg4JkfPIA9D/hVVqV3cmlVsrHlaW00CyNCBu/vMcbT7Z/njNVrKB5J5BayMzhvmKjJ49z+lbuqWy6a73DuDkdevHooH6mjTFk+zmRotkLnJA6yE92Pb2Fc67M3fcmNvLDbRPM/QYYZJGeu0HuP7x78Uy5doCGU4lGW47DtVq7nSNVjOJjGOeyj0VR9fzrN02P7Y7ySKNjOxc9SQvbd6MevsMCmmr2Ia0udHo1ukUQ8+TdJI4bHUe31J/zxmuxihju5P3TBtuQMnhc8lm9yBxXBpI0c+wOB/IZxk59cdvoPWvSNJsiLMQvkCQ5+bjK/wB5v89K7aGr5bHNW0XMRWrTxyrISSJGDFh1KqcKPZQefeukuNd88nI/chgqqT97bxk+2f0FU7wWywkwg4cdT7cAH+eKyEjDPukBYrztx0GOPpnrnsK7FUqU/cizm5Yz95o7aO/URGGNd807ZIPAA64+n9Kj04ZRmllLKxO+TvIerH2UdB7VgfvUiAwFONueg3HqPwHWte2XESoH4GAM8c+p/wAK7oVnJq/Q5J00k7HSi6Z9qRL878Aeg/8ArD+tX/OSJc8bYzx7k9K5kSrAi4yA2eT1IHU/jWfJc3F5KinKjeWx+GBmuv6zy77nN7C5082oqkIkByd2wf7TDqfoDViGYuxAPAA59BjNc0sMl1KigYjiLfQAdTWm7MEkVeDJgfRT1P5YrWFVvVmcqaWiJSzzSqDwuQfwq68iqxz0HP4AVSZuQg4IHP4/4Uk0qDnOdxx+FUpWJcbloXBS2dm6rj8z1p3nko4Y/d4/Mf41itdJG5VjlTgAevc1QnvjuMinK4Ofp7VPt0h+yOiYLNGI88/4io4gUYbhlen0rDg1RAEd2wcfnjity1uoZVAyCcU41Iy2FKDiWZBwADn0P0qpJMSwDdexp8kioD3U/wA/8ayppt5AUg81cpWJjEkeQsenTt/WjGRkcEfp+NSqoIy3U9D6fjUUjPEQ2M1HmV5AAW5Iww7jjNRyMXyG4cd+n59jUu+J1DJ39DVaZ2C7vvr0NDGZ1xNMg2n5R6jlf8/jWJPdW8wZZGBK9z8w/wCBd1+tatw3lbp4MOvcY6ewwePxrjtSuEu13wKFk5zkd/8AeGCP+BCuOtOx0U43K93LHDMqsuwnldp3D8j/AEr6H+FZZ4WZSNuM7SORnuPY18nbszCPJGD91s5H4HtX2D8MbM2+mJIVChlA4ORnr07GtstvKdyMdpCx6vRRRXvnihRRRQAUUUUAf//V/VKiiigAooooAKKKKACiiigAooooAKKCcVRv7pLW0knkfywo69efYetTKSim2NK7scD8RvEMdhpU2nwMDcOuSD0Vff3PavjO5DXV4XX5znJY9hXpnj3X4ry+Nra52gksWOWJ75rgIdyoEjPLHJ4r88zPFvE13J7I+yy7DKjS82CWowfMJ257jlj7+w9KwtVmVmJVuE4OBhR/ia3p51C+WDhh6HoP5Cse8SPYrL0XgADA/Adz715stdD0Y9zjUju7qbYVCKD0IOT9ea3XkMMaRI5WV+gXBx74AGTUsoeCNSx3buoX5QPfn+v5VRub2O32x26s8jDO8dh+X60r9EUacFooQSzRH/enfGD6n39qvqoK/Iy7W7r8xI/3R/U1kxzmFUZ4cZxt3AFvqAentnFbNlZXZ+afJMnO3IwB+FapENm7psd4oJVdg7Fhlj7egruNPg1FmWS8xEpIOXbIwPpXKW0H2cDdIqEH7qDcfxxwK63TnuQd22QDozuy5Gen3uB+Wa9DC6OzOKu7rQ3J1iMZd23Bhnp1+ijgfrXJ3lsWJkVfl9TwOa6tp7KAMrXDyyMOcHIz/vAD9K5TUWmkIMabQejHr+Ga6cXa1zHD3uY2145MGTYf9kY/KtbelhAsYGHc5yRzk9yT39MA+1Yq5SQsGBxwWHcn0PJrV0uCdpfNQEksSZGOMY64znGPWuCje9kdlTa7NJIQGSa/YgMCQhPp/sjOPqcmor6RXi8uBfkBzycfi39B19avratNJznYeWfucehPb/P0r3youLaBguTgnr+tdU4NRZzRkrnj2vPtm2YUk8sTnHX27ewp1le3KhI3YiTaZArcEJ03t/dX09eg9m+JGto5z5fDZPJG5sZ4IXpz6msjTFAmkeSJnJKuEI3NIw4VpOzbf4F+6OpzXlq3Mz0Ps3NW6lR0UIScg/OeOvUgfyqjbyyXDP8AZgyQWwwenXPAPr6n8q27izmlZ57puGB4zjGTyc+tUYrR5JVghP7odQBgAdTj0HqepqVfmHpY7mzs0ljinRR8pAAPGT1J98dq6eyE16v2gMy2+7AOfmfb1I9F4x9K422lmkKE/LEq4H8PA6k/Xkn0AxXplu8cemx7U4kwEUdSn07Z4HP6V7OFpqVzy8RNozmlVsmZPkJHlr+mT9T+lTyXHkKhiXdJI2WLenqfYDp+FTSwxoXnuyAU6KP4fQD3Oaw55JHnyUwRwF9O5/8Ar1rNunuZx940EuY1uxO/711+SCMdAc9QP5k966S3eFBsOHlbsPfjk/h2/wAa4GO5a3L4UtI/DOf5fl2HA+tb2m3MjI0mcg/eYHBJ/ur6ADjPQCtMNXV7E1qTtc27hlEyoxDsxIZh047D2FRwARysWPKqWYn+HJ4z7k9vSssalHHKwTBAA3N/Cg7KvqavytnyrVx88rbio6/ifYdK6YyjJ3MHFrQ14JGkjwnCuRj/AHBx+bNmr7N5Nu079Wxj6DpWatyqcnBxgceo4AHsP6GsbVtWkkim+baiZC+2B1/PmuqVWMI3Zzqm5SsX5dR8uN5V+Y8n3JPA/rVFr9pGdiTj7qjucdT+NcpFeyXN3HDED5MYUMP9rbwPwGPzNT3Bk8wJFyvVj6+34nrXnvEuWx1exUdzVEsu8SEH584z+VSkqIGiA5zjHuetLaIfsyzzZ3McAn070WrRsdo5JYE/yNaRX4mTZQj0q5kYKx4Q5H9a2o7KW0G8E5U5rWhlh8tmHPWlkYzAKvrj866YUYxV0YyqN7kIlaVDnPv/AIirMcMUgEg6t1x3I7/WlRFi6jkdakJjHzIcA10xXcyfkTBABjOaqt8pKHlfTqRSGZlPTg96Z56FsP1HT2/Gq5kJJkDwgBnXp3wcj6/55qi0kUQ8wnA9c8VduZBGdwYr7j/PNc3qE8gRpvLKnu0XRh7jpWFSSjsaQTZNctA585Hw7dcAEke471zl/wCWqsWIViOu3bn1yOQa5DUtVgik2TqYXB4LLtBP1U4NZa6+88ZimQFccFCTx9Tz+BrzamJTdmdsKLRbeOOfUoCrDaXAODxn19q+5vBlsIdHhVWJwBkNg/qOtfn7p8xfXEJBeLIwUIJ//WK+/PAhm/sSIyAqCOAR6ccEZGP5V7GWdjhzDZHcUUUV7J5AUUUUAFFFFAH/1v1SooooAKKKKACiiigAooooAKKKKAEIryz4oayumaQEyFZ84OefoP6mvSr29ttPtmurt9kadT/Qe9fGfxM8Y3HiHUHkiXbBb7ggPQKO59zXh55jI0qDpp+8z1Mrwzq1VK2iOEleB55L68O4t1Hp6AemamF0ot3mlATrgf41gWazT7JSpkZ+V3HAX3NWblI2iEpzjooH8XqfpXwvMz6/l6Fa6kiWMSEjHXc3Cgn27j61k/aJLwiGDdIzY+bGT9R6/wBKrTNczXBd9rZ6L6A9zWpZ7IlYZIQk5K/JuPueTj2BqEaNFpbVyhjchWPqQ2B6kf8A6vpWPd3kiSeQn3vpzn1P/wBc1XuNXlMn2S0G1RyNi8D8f69agWOdtimRd7dEHX6k8CnzXBRtuT29zEj+Y4adv4mYjb+A/qa7S0lkMcRR/J3dY40zKfrkYH61woL2jGV5QwHIAAAX6Dp9c10ejSSXLtKx3I5GQqE59B1yfzxVQepM0el6SbVY96AKqcfvCSc9Dwvf8a6gSQvtdIBxwCRtUfRRnr71y1k1yUUpG0Y6buARj6A4/Ct5NsW4PMAw5bqzj8zgV61GVlY86qrs12u4o9yQqpYew4H5cfjXK6jeSysWk+c9ieg/PGalnvjErRxRsN38TEAnNc7cISxMzYZf4eWPPr1x/Os8TWclZFUaSTuyncPICPLJJJwPXn07/lXU6XblYjNdSZhTkgnClh2OOuPQcZ61z1rE7S7ijFh2/iOffnA+nNdTBHPdyCW+A8qPHA+WNR6Z749Kyw0Nb9TWvLSxoxzSTIbm5XZABiJO7HscfyzWReSs2ZAOTngdB7D29TWvc3Cld7H5UAHA7dgB6n8653UbyRUYY2MB84HVF7AnsT6Ct8RNJWuY0otu9jzfUfMF1IW+aQ/OPRewwO5+vStLS/IhjYsd7knJY9SOMKOpA9fWs+9jR75Zp8hA28qM/NjoD3wPzNbCRF5Nyx5eUABQOQnUZ9AOpFefHV3R2y2sJeM9zjaCyLj2FZxE4jMUL73c8gdFGffHbk9810N5gcOwHXAXHygdTjpnHGT0rOEEsifuBsGMnAJ4/vHPUnoPepcHcFJWMyC68p40dyURtpPUuxOAB69PoBXoOi6yby2WcMFklO2MDkRoOB9WOD/M8V5lPaSrMcMd2SBn+AHgnP0/M1v6M8to6YUoFGEUjpnAGffHQds5NdGGruDsZVqSkj1W22uQ7NiOIE5OT9Tk9WJ7/lVO+Rty4B3uQqoBzj3NMaeSRktkf5wdzMOg44/+tU7vDEmVPlxL95+5HoD7160mpKx5yTTuY9xaTRSBCCGJ2oi8n8/5n61ZVLiGBbQKQwGWx79gP5Vu6dbsxM8KFGYZLt1Vew9QT/L3NbkdnHbqZIh88hA3t0/AfyFOlhL+8hVMRbQ5e1toNPi+03n3oxvWP0PYt6n+Z59K0LaZrWEXt4S1zMP3aD7w3cD8z/nilu7cNOI1b5Ac7j3Pc8/kPz7VjTyBkeZTjnJY8t0wAPw6f/XrRv2e3Qz+PU0Li6LSpYQsvmRgs5HQf/qrImYXqmyGGQKXZvUe59/6Vz8EspaSaQ7FlGCc87SeBn3/AJVuWCNb2MjyHL3ICqPQHP64rBVXUfkaOCgi3a2iCItH1Dlif1q0pTgMByCT9OgH4mq5vEjXy1/jyOOgUcH8TWZJKxjAHLE7ifbt/jWt4wWhi7yeptXMzSCO3U8DA49e9VS7WUSugxuIY59zgf402FsKhbqcGo5/NuI4oz3G7/vnpVt316k26G5bTxeWWQ8Nz+nFMi1ZmuHUfd4IPr7VnCMjYIjjKge2arrhJtvG0g5/r+NU6slYnkTOplvHnUPCfnHH1FVLXVI5TtfhumPUj+tc5NeGzlUZzFMOG9GHY1KphmQy7uTjd7HsT/jV/WG2T7JJHYLfREbE+Y91Pf6VRvL2ONcEfKc8ntXHSyTWsg3HhvuntUUmqzPmFzz0wx4P0Yf1pvFNqzF7HXQ1pNVSJC6uJkHOAfmUVkN4gtWBe3bYw4Kt3z79KxpkFxJvwUcenIP8qiuNPSZcg4fHOPlb/wCvWDqyextGEepXvtdtGR1eNm6/d5x9UbI/EcfSuammVQLjdhW6EKB+Py8fhVu90h4o/NUhWXkNjDY/rXGSmcHyXcbs/eAAz7Gud3clc6Eklod34Js9OvdVzcEFWb/dI9wfUdcd+3Nff/h6yistPjSBmKMq8Mc4OOoPoa+Hfhr4fvbzW7Z7GULKG5BGCQPzBFfeenQCC1ji2hdo6DpX0uWx0bPFzCWqRfooor1DzQooooAKKKKAP//X/VKiiigAooooAKKKKACiiigAooqnqFytpZyXD4CoCSW6Af1+lTKSim2NK7seIfFjxHJBALcv5UXIAB5c+v0r5Sc/2gzahetts4/uL/FKw6Z9h+prvviH4hOu6rJKXymdo4xwOOBXJJFBNJE33ghwq9Bnv/8Arr84x2I9vXlUvc+3wND2VFREsgWjae4Tg8HsMf3cfz9TTdR3SKCi+WG+7nk/gB+grWuLwISjISF6Dp+H9Sa52aRppnmuHLMw2fL0GeSAOw7Dua5GlsdavuYc9rcy4itxlW++R0z9e59uAO9W0svLjLSOHb0HOMerdB/KtuO1eWBYbdFVDxycE/gO3vV1dGhto1NwfOkbOFx8qj6dM/Wj2T3Y/aLY4iXfMuIIjj+HB2qR67jyR78ZqqiiPdKxHHBK8L/30eT9c/hXY3kMYG3diMnBA5LH0z6fp71nx2kEf72T92qHCrgZ/XpUcrvZFqWhg2+nJP8A6Tdx+ZHxjzDtXHYBOTj64z6V1emRkoGct6YxtH0XPIH5U2KREYlysRUfKoIZiT3Jx19eKlhe6k+feuwngtuJwfQYz+NCdnoJ6nW217PHEqIqh2OFznj0Ax1P6VuWM8iQGSZ1iVsbj0JJ6ALyxrJ00KCqru7bn+7ge3Uk1uSG3WIICsaAZ4wpP1J6fqT616lFWXNc4Ku9rFSa73MSkZcjOSflH9SP51z9zdzyuI7YKg9cd++B/wDXrXuBDJGCh2qBgdPzP/1zWU62yHAOR3PQH+p/zzWFZyelzWmkugtpckSGL77gYJ4O3146D3JrsrHddp5znaqjK57gd8Ht79TXI2sSykuzeXAnLM2ACB2A4AFdRa3TX6iOzj/dH+I8Z2+nr7D8cCt8K+jMq/kW5EH2Yyg5ZjkYAzn1HtXJXjSRt5eOhyecjPqSepr0SW0McH7wheP++cD+f9TXn+sRtFG0pXoMKDnH19zRjabjZsnDTT0PO9YkleTNsTuUgk4xtUn+Zx1POBXX2jrNZx4cRjIUEDJJHU4HU/oK4yd5I5HtY873+aR2P3PTkdz2A+tbmjXBugLSBfKigQZPUgHnkn+LA6du9cVJnZUWhtP5EgcRfMyjaSBwD2X3Pc44HfNPiiEcJ8sjL/ebOT+Z6n+VRxSRfZmMI226sQ2PvSE/wg9ceuOTSW264uPtCKQFO0eik9FUe3860nZSsZK9iyLHIWGJMySEHHUnHbnsO5NSz6e0GHfLHPBHAwep/PpWnZRP5Zhjb945O4ryce59P0ropLNfljPMjDGCc7V9T05rqp4bmjcwnWcWc9Z/NwQRCvUDguew/H9K6eyt7e9cXkwUxRDCZ+5kd1HcD17npVSa2hiVlwAgyDz1x1A+veo4tVTzI7KBDLcSsAqjhFC8kn0Ue1dlGKg+WZy1G5K8TqYZYolM92NqHlIx1Oe7e57DtUcl1b5+037bdnQDAVQOgHqfp9KzZ7iEgAyKVRhlUHVvc/zrCmuNy/a3IJ3fIT0A7ED+v5etdk6/IjnjS5jYvrtHcrIfLXALf7K44GOpPtXL3lzBP80g8u1Q5255YdyT79B7fhWXcaqqK8u7zHc9evzN3J9T+grKfzLmNY3YkDLMPTP82P6Z9q8ytieZ6HZToWL9u51B8sAAz/IMdcf4dAK2JrrdOI41+UHCjvxx/M1Db3DW+Z1RVaNSsX91MYG4/wAgPasCXU3RAIQS4DOCep2tx+ZNKDtEUldm7aypKZZclgAwX8Op/E1ppBmP5upxk/lXN2EzwWsKT/fbCnt0yxx+JArqY2b/AFOPuxoPxHJ/wropWa1MJ6MnX/VxepHP51a8j59g6Lx+eRVFMCdd3GP5VpW8wZC5PpXTGzMZFecbLdcdQP5f5zWPJMjSSleAc+5BH+FX9Rn2qyp2IIxWHHMkjEg4Y+vtWNWWti4LS5LFgqYLrHlSdGHQH1Hp71WYTWjlVOdvB+lXJkj2lwSufvr/AF/z25rEu2dR945XofasZaFrUmkvxgxvjb3HpjuKrSXKAYlXPo4/kaotcK/zPjf0PofQ57GqsjsrbofuE4I9D9KnnZXKaguo4wHVxn+7nB/D/Cl/tOwvgbecmOQcBtuMfWssfZn+8SCfamyx2RGXAZh68H8D1q1JhYz9UaeKN0Ys8f8AeXJBA9Qf6GuLjuPOuzBwQDwTnP455r0m3SMoRbz4LfwtyP1rPs9Nd9SPmRo27PQrkn1HQ1rCKbuDlpY9l+DXmm6J8rzI8gA7gQG7Yz0b05GelfXMX3ehH1rw34T6F/Z8LzMWV5eDkDa4H3WHHUchh9DXui9K+owUWqSueBi5XqOw6iiius5QooooAKKKKAP/0P1SooooAKKKKACiiigAooooAK8k+K+tPY6T9jjbb5vLegA9fqeAK9Zd1jQuxwAMmvkH4neJF1rWXsoW/dQt8xz1I9P6V4ueYn2eHcE9ZaHpZXQdSsn0R5XDEbt3cnapPLH19BWrFZmBo8DbxlR32+v4+9Z1vI10+6VTHDGcBVHXnp9T1J/CtG7uXAMiLyRwcEhQO1fDxStdn1zvsYWoyRRJvI+d8hQT2Hc1iDb5qlyxLfKicZbPU+2fXH0qO4e5uZ23sdqdAD2HTJ/pVrT0EbGTcWkfPz9So9j/AC9fpWSd2bWsjrUc2y759qM2FKry2R0XPapLy7f/AI94+XUZPPOT249uvpT7dQjRhPvL6/eJ7/T3NUrlmjbEimMscYUZPXoP6mt5uy0MUrszRsO4AGaX8cJ7f4k/QVTuYZJcJE5e4cY6AhM/XCgn1NK90tu+PLO92+SJR949ief896s6bDwZJQJp+vllsopP95gACfYCsormNG7ai6bpTR4e6myP4v8Alo5P+9wPy4rSmZEHlwRMkeOSx6j3ParUM8jB2hdEA43ffbJ7D/AVHeWSyKZplYAfxMR19eSBn8Kpx090nm11KcGpW8JI+YyAcKmWCn19/wAeBW5ZSPqDeZJHlV/ikPyqfXA44rkY4o4v3qDKnkdBu+vrUqfbp33Tsqxrnhhx9MHj9CadKo1o0E4Jq6OwvpklAijke5ZeNkY2x/8AAm6fhyT6VmOsm7E0aqB0AGMn37/nU9lJbxoFLSXEzdgdq5/22+6o/U9BVqeSExlY8FVyrMDgZ/uqR/QH0rqlHmXMYRfLoUGV5SomfCKeExgH/gPUk+9dxozKAqwgvKMgjsi9/YZ+uTXGpbp5gUyrbLwTg5kIPUeorpH1FYLZLLTozEmcAEHdIfUDOSPyHv1NbYW0XzMzr+8uVHUieJ0MEJzI5ySTk+5PoPT9K5rVkVlzKSFPy56Zx2A61q6OIYbYyy/vHfkhMHJPAyRx/QdqoayplLqxwdo4XIGD0APX/Hk9K6sSnKnc5qTSnY8g1C6SKdo7VNu05z6E+/8AePcnoBxRo8YLtHcDCcE88sTjH06Z+gFbWsWcdjE+EDP3Xoo471zGmvNaTSS6gTGZQCU4LkHgDHbeeMdcDHrXjRVnZnqN3jdHXLOkirEOY0yeOmfr6f55phuwkDTTZC/diVeM5POPc+tZdxcQRMJr5jtQZWBepz3b+np7k0Wn2q6uxf3UYATKwxE4UNjqT64/IcdabkSonf6PeGBlghVVdFyVPPPq2e3p6mus3RWkDuWLSkjOP7zdvXPtXD6LbSBhsPznlnxgFj7df89q60RRx+XHEd7ZIjz6/wAUje/YDrXp4Wo+Q4MRFcxkXMsrFsv83IJ7L6//AK6fZFlJNqvloeGb+JvRfUknk/lW1Par5YVPmZflwOgJPOSO/sOlRC12DOduzjeRwueoQd29TVqnJSIc48tjMuLuC3jCIgRFyBuOScnnPuTXHanqSySGAPlgOSeka/QdT6Dqe+BWxqbbInmhXZuB8sHsOm8/0/SuIttKnvW8yZsqTu92+vp7dvrXHWqu/KdNKCtcvxXGVV7cbyPlXA3ZJ7Dtn+83rwK27m4W0hEVuge4KgADnLNxyfrz9BUNtBcF8wrtWNcIF4VR9e5xyT0rd0zSohKzu2doLSueigD/AAq6NJt2RFSaRkyWs12qWaEiCMAtjqzcd/oP1rUGlJh7+cdcH2AHOB7dq6W3todu8jb5zBR+PP8ALH51Xvb6OOLy0G4K+MZ7AZOfxr0FQUY3kcUqrbsjiol83UZEI4gx+f8AFXXsQrMCRuBA/HG4/lXNmAWrGYHOSQ3uWGf61K85aWd0PzIMgepY/N/h+FZwfKOSubsVvKEE754BH+fwqhbzlbeRSc7QCPwrW0u+ju4pLLO1mHyg9MkZH4Hp7E1y0rGFSGU+XKvB6445FaTaSUkZq7bTK1xdurMRnswx6/8A6qmjKyjzVAAPzY+tczBcyB/JkG7b8p9cA8flW9aiRBuHTv8A41xqd2bNWRff5kwGPHr1x6fhXN3cjKXjyQcA/QjjI9Qe4rWvJhBtkPAJwSOx7f8A1q5bULxGbcSA69e34g1U2KJW+1fN5brg4P4j2P8ASrcRcjZgkHgg+n171ioGmYlgAc8EdD71tWpmhA3KSPUc1ES2WI7OZe/X8aklsnkXBchvQ9KvwSiRQc81caQOgQgMB271vGKaM22cva2VxCzsdgI5BIOD+NaNmZJrgeYysD0PQD0PTNaBkjEbFAeuCcjj8f8AECqdoqxys4dtp6g9Pr/9euqnFInmufSPwr1m+mZtLnl3Bfuq3PA9D3+uele9r0rxr4aaN5WlebMDvLBlLegPY+o7g+1eyJ92vpsKmqaueDiWnN2HUUUV0nOFFFFABRRRQB//0f1SooooAKKKKACiiigAooqnqFwbWzkmVSzAHAHc0AeS/EjxnJp9rNZ2UoRBhHZfvZPVQfp6fU4r5QvLszMSSQZicHPT3z/n861fiB4guNR1yVnbKQkqqj7o9cVzFraNfFVlysaqCzZ+6o7Accnv6V8DnOI9piWl0PscsoKnQTfU17ONZjHawgtGo+Zzxx359zVzUS8lv5EZ+VxxxwFHcfXoo/GmWim8k+XEcCAgD68Zpbx2MJRAS8hC7h/CMfzx/OvMXwnf1OQleK1IJ+Ux8c9Ccd/XHU+pxWhp9u8WLiXJkfBjD4JIz94j1OeB2piWttJMnngnB4Uc59AT6nvXXWcKp/pNwBtOAo4JZufzH8zSpwvqVOehJbxGGI9cnl5DyeOuP5DH1NZGpSHmNdyMBzkcge4/xrqriOWMhM7ZpOeeNq+v+ArBuUUny4gTHzliPmkOeT7Ln861qQstTKnK7OLMDeW8odkB43McO2OoUDt61Kl1KAsFp8zYAZuh2/0H61auYJJZGI3YHykjjOP4R7fSorayRcxldqnjAJH16Vx3aOrQ37C5kjZlC5EYyXPA49McADtn8q1kuXu8yRR7woyDtGFz7k9fSsa3tLoFVEgSPg7QCT7cf1rdmsrifYkjbUX+Hdg59cD5R+OTW9NyaMJ2TOWvjFGN9yfKBPGWyze/Hb2rOkuJCAI0IU8cnLY7jpgfzrsJ7Lyj5sUkaMOM9W/MnrXMXluxnKSzE9ehCjJ9TjP5VMo2LjK4y0W6d/tFwCIlyFG4genHYe7cn0rprfUBEWzCTsGBJwiRr1wnp9eprBtvkkCF0IU44GQo92Of0q1PeQ4AEgk9MqQPw4J/KtKc+XqTOPN0NZrmaf5bZPLDnICcf99OwyT9M49q1LdoYQIY1a4uHHT7qkdySSW2n1PWsaOO6QiTCu+35VJyRnr8vv3P8qYZ5UkAjUSscBneT5cjrkLgH8TXVCp1ZhKPRHo8d1b2iCCL95IxBZ/4dwHYDAIQdO2T3qhctOEaRHBlYjLnnDnsM9wMf/WArnY7+eaQ28D7fMYGSXAUH0RM+vUnHA9K0SiX2YIpP9HgwksycBjnLqhPOexP4Cu7n542RycnK7syriG3SP5j8395uTnufc15pqlwI73Fqryt0B6HPTJPbjJ/ya9DvJDc3jwx/JDCPmxxgdhnt05PX8a5DULS3CtN8wIA3AdW54AHbJ4HevNqQtsd1OXcq21tGzxyzttDsPl7se+Py49K7CDEtxF+62KgyFyMAD6/nz+tYDJHBHFeSN1Unt8qjjI9uw9akhklv5TAquUfkID8xGeGkb37AY4rnirOzNZbXR6BaahExzF8+T26nsM+3oByTW5aFnm6jzOjNyQo/ugDjjv71iaXZRwqIY8s7cOy9AO6r6e7eldrDZBVESYVQOdvyjHYfT/Jr2cPSbSPLrTSbK8bLM0iL8sacHGAfx+vXAqLULiHyVVOpOPTKjsPTJ696aZ433eRjykOOOAxPue2OSaw55FlleVjlD93Ax8vTIz0BP51rUqcsbIyhG8tTA1V4pAZZ3AReWycgnoAo7+gH6VRikYgI0OE5Y7unH971Y+g4Axn0qldNLeXf7vJSM5G0cE9sZ7Due/auz03So8RrMTJO5Bcg52gdF9M55wOAeSeBXnUKbqTbR3VZKEbMLW2ubtWlkULsOQG6bj0z9OuO/5VYvENtZR6bacCZsyMerH39s8kfSt+YJH/AKPapiKBTubr+A9WJ/KsS1ia7nZn6IpQe2PmP54xXr+y5Vyrc81zvqyxDO8stv5fMaTMFPqF7/iRx7ViyxhHkicniMZP+0Tmunkt0hiiEY5JOPrjoKzpYftE8uOCXP5ADAoqQdrExkrmbJEZVLHlZFP1DoeP5Cs64UQ7ZVBO8duue9dTFbeQ2YxlT0/4FVDUIRDumCDyznPbafX25rKdPS5SnrY465nlhc3Fq5DKMj+nWoLq/e+QzqRGZjv44CydTx2B/wAamv2EM4mxjaMMOzDrnHt3/OsC4ge3dxDzDN8yZ6euP6VySbWhsknqWYzK0hM2cg9T1H410lq0oQA88ZU96xrCNpRvIw46+49a2yuyIkcKDhh02k8g/j2pQXUJvoZepzqkbMccjGCOMehHpn8q4WWQyP5bghTypB5HqPcV0mtXDSRnGHzwynr9RXFWbyhzESWGejc/lSm9QitDqrJNoC5BrooQF5/yawrRcDArWWbbjdwaIBJF54kxmPg+3Iqk8jxtsmOM9CKcJwDt6Z/Wonl3EJIA4PY8H8K6ImbJfMHlt5jK47FRhufb/A0unNEuY3zsY9eR+RHf+dZsUUTSbYZGX+8hOf068fpXqWheCZZ0inkJMbkE4O5WX1VgTyO4NdtGnKWxEpxivePojwEWj0aC3EizIqjDA54x9AcexGR06V6IvSuS8N6bFptqlvEchBXWr0r6ekrRSPn6rvJtC0UUVoZhRRRQAUUUUAf/0v1SooooAKKKKACiikBoAWuP8c+ILTw94fubq4+ZmQqijuTXXswRS7cAV8zfG7xC5gjsIuDJwq9Tjux9PYVz4qsqVKVR9DfD0nUqKCPle/vWvtTZiCzSMWx2/GuktYRFGlk5Yyzctk87f/risG2gS2mluJV8xoyAFyAAx5JJ9fQV0lnLI0MmoS8yHgDHc8KP6n2Ffms5OcnKW7PuklGKii5cOVjIh+XadqgYxuPH546VKZY7ezMikbgu1e+49yPWsFboGTaxJI4X8eCePUd6vrMbiRZHj3RxHaoz1yeB9KUZDcS3CFESyzrjI5HTgds+nrioBqDxXAeQAOwBjU9Qo7+gyeB6CqOp3bwAIG3zO3Poo9MdPpWVYXAuLmSRCCzMQCWyBt6k+yjj6/jTUtbAo6XZ6Bb7ERrq7b963zOSOQSOuAfThRV1bNpJFkkG6QgBIVzu/wCBY6Y9P61zum3KvO00LkBeQzHnPeQj/wBB9Pwrozq8EMWzb5UZ+bv5sg6DOPmC4+n866lKLV5GElJP3SebThsHmhccYUHj6DHpUUejIR5r7trc7Y1GCPqThRU9jN9pIk28cYQjk46cdABVu4LqdxOTnOCcn8ugqnGDXNYjmkna5QQyW6F44gFPH49ueM/yrPngvbxyZHKqP7uefx/wrWkkiEiiSUPJgZI6Y9qlN1byhlWViEGG24GP8B+prJxUnZsvma1SObXSZPMLLgN0LHnr6ZqnLoJdjufdjjPH6Zrt0lswoQLkAHvlmPqT6UoOX3eXz6noPYCl9Xi9Lh7aSOEHhtAn15J/+sOtL/Y5iUbIzx0yelejfZlnUeccBfbA/Cs+fS7EhvOTCg59M+9a/Uo9CfrT6nGRpd2oJUtGx67QAv4kDJ+lVgss5XzXL5YZLqVAHbHYnPrXWtZW8oZIfMZvQE4A9SenFZz20w3IqkMMfNjOMdBik6DiwVVMzn0mVGUQRkhSxZskHnvuPQU/+04tPgisYm8pYyQi4LHdwSxx3Azj3xWluug3kD7pIXc/AAbsqKD1NU7y2iuVEafM+ck4HY8DA6fnWi93WJN76SKxlW30/ZCAXdsbm5+Y9OnGQOe+OvWuei3tFGJFLIGZ+vzOegwPX+VQ3FzexzC0SMCLIVnXcwAz8xHHXAwoHU1Umu3+3FrYbCSI1B/hxyQR2AHU+2KG+axSVjoBp8uppL5qjzdyqFAG2MDgBccEgcDsK6W3sY7WKO0tkBMjckHJb3dvT2Hauc8OXUqyRqFIVizru5Zs8bz9e34V391IplisrfYspA3yDog7gDueM9h+FKnTi9QnN7GzZJDFEUc7iv3seg6DjufTsKszySyRqk7bIyMt2P0z7CoIhBBGJERiCCY0zlmJPBOOme1QX5nRlg489wBs7IPc/TrmvSu1A4GryIrm481fKhjLRqAAq9y3RfbPc9cVjaiHjUxjEspOCR9xpO4x/dQfyrbdks7ZDGSTkhSerE/eb8uB6CuV1KVijEfLGqdfb09+etcuKlaGptQjeWhgx7llJuCBEhySvGSPQ9z9Onau2sbmQRBmxHkfKO/1C+gH4k9eK8ut7xZ7wBBuMXJzn5fy6cf4D1rq4dR8sfZ45A8xzJIzcBF6gsew7geg+grDB1LG2Jhc7WWZrXTyZhzKScHkhVBbn+Z9Sa1PD9u72yPMMNMCdvoME/yxXF6fcvqzbiT5bncS393+EfiFyf1rv0nENr9oj/hXao75cZz+Ve1QkpPm6I8yrFpcvUbcGNntnyAu4n8zVGQpbt5yjgtnPoR/+qlmZWmhgUfL8uP6fzP5VUtJftAls5fvoSV98cEVUppuxCjoTSqYbh4cjZIPkJ6YbkfUetYlxdXMRkDp8ycbG7/Q/wD6waS8uWMaR5wQTtPoR1H9RWZqd2H09jKN8ZXa/OGVv4WH8vbFc8pp3sWo9zzjVtagmmjaNGhkUlW2nKnHRh+tTWd0JgIpeVPb0PqK52QR3N0ZRw/cdN3v9a6Cxg2YGMgH6GvNcm2dVtDrbFDAB3XPB9P/AK1WNQmWPaR/GNp9CBUFozRxAxncoHQ1k3crNI0XQKNyfzre9omdrsxNScncrdV4NY9rEWYMeoOD/wDXrQuHdi0q8n+YH/1qkgVPMZk4Dc+3NYPUtFiNzEwJFX1kBXL4OOtV3RWDoOoPT69qzpJ2jYSA4I59iKa0HY2WMU0nkhgkn8Of4vb6+lZs9zNA54wO6E4H1B9ayL+SObKplNp+Rwc7f9k+3pTvOmv4fs10w81R8r98/X+frWkX0FY39Cv2utURJEEjA4+YDdt/r+dfc3hPQkt9KhkUKCR8wA/mPUfrX50+EtYu9M8SkyRf8e55XOAR6iv0T8DeJpta02KR4Aq4HK8fp0/I19Fljjszysw5t0dlBbiJiq1oAYpAMEn1p1e2eS2FFFFAgooooAKKKKAP/9P9UqKKKACiiigBGPFNHWhjTlFAFTULiK1s5bmf7kalj+FfAfxB1+41vXLnUOQM7UGOg/zivrz4j67JpujTlFAjVSCzdCxHAA7/AMq+E7ktcSl5mxvJdmPfvj2zXzPEeJtTjRXU9/JKF5Oo+gqw5hiidtrOMtn8yfqT1rppIoY7WO3jOcfNj+p/nXKWTrfzmXd/rMqMDoAeceg4/Oupjie8uSsYIjVQFzycHrmvkI7n0kjHthIDNNcdByxPp0wAK1EnIt/MIB8z5gFHGOgAH8hVqdEZhBHnyVJLseASOcn8eB+PepX/ANTlRsVcDn/a749T2x0qoxtcTdzgNZuvIjKkkySHaxHUdiR79lH41zkd7JEzR2KABUAOeTn+FeOvqavakBJfCAne3IO7ooHr7mrOh6VJcRrcyKdhO5COM9uAP0NZxXY2ubelx3MURMjkAY3HoS56jP49AOK63TolmmI2GSXILHsMdMnoFHpVI6ftQRyEb8HGScIO4Udz6n14rrLC2ihtltoSBHgOzYyW/DuPTPGfWrhGTlqZTkkjSW8jgVg77io+b+n0HoPSueu9QklJEC4X6457kkce2K0roQrEIU+VTz6fUn1J7+9ZBga6JAUlegA7+vtgdzWlWcn7qM6cVuyk05RfKU5ycsRxk+gI5xWna28skQByijBA7fUg4yfrVq1sUjOM8+wz+H/6q3I4fL+ZkIxjGRz/APrpQot6sc6iWiKkVsY4yxOB6nqfb/69W4ZCq4AOTwOwH+NE2xzhOq+nAAqJCm7BO4jr/hW0dHoYvUuu0zjKybE/U/5/KrUWnGch5GBA654Az+pqmG2MGAJY++MVKsxLbd+eeABwP6mumFRX94wlHTQ6SEoV8lACv5Dj0FRXEAkYE4RU6E+vr/hVOKT5sSuNvQ9s4qF7/wA9ysAwpH1IA7D0r0FVi42Zy+zaehlT6Wwn3RYJY/w/5/Wsm6W1gdllQggHaT+Wf144rsZJP3TDbhjkj1wB/WqqRxzkRLErepPOAeT+XGKwlRjfQ0VR9ThNQ0aC6wtlGAoYAsCfu9jx3rzoadJa3l0LxgLaTo7nAVSfmUD3zxyc17y6RJI8cciqqK25R2BzzXEa54dnvLYSIm5guQSMncjBh+n86xqUuXVG0Kt9Gc7JNNFOXTFvEoT5h2UjhB/ujH8q6rSLy3c7xhIA+WZj1J4C89fXr/SvOru+MlmIMM8rMXynOEGOv+8x7elWNLmQyImoEiPAEcWcu5YjJI5CjsT36etYuVp3Rvy3jZntWj3ltfTztYOZQjFTKB1YcMFP91emRgZ4FaEsSyyGJCFXq5B4AHPJ9+pH59qxoLmNbFbS22oCOFU4yM47D1zjP19hkzahJPdCyiICAEyHJ2qB1ye+P58V2SrxjFI5FScm2X729E06wQN8ozlzxtUdcfU8Vw+t3yysIYc+VnCjoBjoW9enAqa81iK2t5mgDFjlVz3AHAHuep9OlcRrcsz2kUMrkbAGnKnB+b7qg/3mP5CvMr1uY7qVHlFOrKqvFZjIQk7hwN36Akfz+lXtNV54I7dyFRiZJjnmSRui+4UVRtod0Rto1WNYiEwvKhiMke+3ufU11EFvEsas/KxDcwAxx6E+/fHPaoptp2RU0rHciBEMemxkDKBS3f7o3tj36fQe9aGoawbi8MNqpCBnIx3JwB+HT8qxLWC5WNr+5Y+Y4YDt15Yn+Q9qktrabyludvQH8yf/AK9eoqsrWR57gt2WbzUJDMm07fLVSSPVEIX8Oc1lSX06FpEciUNuz7nn+f8AOp7q2ky8bckjae3+eBVDymhILkHcoGPXFQ5yb1DlikE96lyzGI/J98Z6gE8j/gJzVC8d7m1fZwcEHHPI7gdx6j8RWNcOLaUO3yIw+9z8pJ4P09aItQeIEgg4POP7w/kf0PWlGbb1FKPY5xICH3jBxzxzkV1dlh0GOGXg9/pTI7RZibi3GAxJxVu3tzu2jgiklqSy8zeUmE439u2fT/CsyYZlGeCBx/WtOeMyIEPDH+fY/j0qjcEsgkxyMcfzB/nWrRCOWlO07Dwe2aWBieP9lf1p2oKDMjg/5P8A+qktmDsX/hxj8qytqWWLmXaflP8ArBwffNY4Du6xn3GPUf8A1jVlFdkIl5A+b/HFakMAlf5QAcbh9euKSTZT0MaO0mD84PTPoR2NWLu0RLcSxg/StlDGAOMFePwP/wBeq13cCNmtZ0PI3Kw/iH+I/wA8dOmEEZOTNfwNo2hajchtYZombC+aB93/AHh1wPUV9oeA9Ht9EtPs1vOky5O0owKsp7qf5j/J+LPB9z511EI1WYCTbhiVDc9MjkV91eG7e1+zo8ds9szANgsHDAjqGHDfU8172Weh5ePVjsKKKDXtHlBRRRQAUUUUAFFFFAH/1P1SooooAKKKRulADepokcRxliQMetA61nazcw2mmz3NwcJGhY/hQB8sfHDxKtxcxaLasWSP53x3avAEUqgEgzJcj7oH3U/yK6fX74axqt3qdwCY1PA/kBXOQCbzzdHHI9Op6Ko9Bnp9K/Os1xHtcTKXQ+3y+j7OhFGxYW+LtbdYx8g6DvxhF/mTXaW1ulvAYBnzJMea/wDtHog9gOTWPoURt4nuTlpHyqnHfH7x/wAOg/GtdrlhIVgA4BBPYZ4yPUnkVz0kkrm1Rtuxk3SJ5qW0RO0nnA6he/PesjV74RRLFFIcDcR6n1Iz+Wfritq53NKQv7xlBXP19+1cFrd1H9tdVIY7tvHQ4HIA9BWNR20RrBXMOy05pJXku1BkuvmEZzlI+o3ehbqc9uBXfWhg3xiEnCDluhY+w7e3tWPpdheToYSfKeY5dtvJ/wBlc8n/AD0FdjpdrbQ5SD5mBxnGcEck++PyFEU3sOTSJ4bQs3mzjDSkKiDrtHQAdh+vU1p3VzHZps+8R1UdiOmfp6dqfFGE3SgfP15OD+J7fQVi6g7b2CAEk4+X29Pxq5e6tDJe8yhPeTSuEBAZhyT0UZ+vX2qytyY1WFgfQK3U+5/wxxWN9naBjO5HHOSen0q1YnNwrrxu6seW5+vbFc0Zu50OKsd7YlEtw9y+1nPReuB0A9zVq4AkO6X5c/dj74qtYlDGsyDcxOF3D5j6YXGPp+dX5kkiUqyDPQ5OT9M/zr1knyHnS0kY7s54j6evqfQChZRGmFGB6+tPkBdhHH7ZJ44FU36bj6kKP6/4Vxu61OhW6lgSbQ2OWb0q35626iPP7wjn2J/rWQd0SbjjceFBqZESNA8rbmyBwO5pRmwlFG9CiSMN5zxwKv2sI3FkAAXv24rm7OcCTk4znH4delasWoSSAbCQo6HpwOB+td1KtHqctSm+hvPOgBG0MxHf17VnX5aLMYf92U+nzGoAftBIAJQfxHpjBoS3eV1edwwGTj1Ax1z7V2+0ckc3Kosx/KDXe5pAEAAAJxuwOGY/WnfbJHstlxgSqVQ4J5LHOR7mr93oyyFQ2c4BJHBwRg5rNNoUJWQgYI9s5xjH5Vg+aLtY1XLI831iyXQr5r12xaPFkgdAOAo498VkgC0DTR7TNIeTzwW447ZwDj0HNem61YJqGjvaFR8q7CMY3Acj8f8ACvIJ/NtYorS5x52ZMMBk+ikL7D1rCcUtEdEHfU37XUVjk8iKUStn5VUZJxxyegH9B787j3EvkmHG9nwrbBkfTPfHc965XQ7SVkE+RHaxAhd5DM79C8jdCeuFHH8q9MtraMoiMegySRt7dh1+prn9jNo1dWKZwcsM0b+dMRhMhUUZycdPw71lPYzsRdTryzEoDyTI3G4+/Yeg4HevVjZRS/NCMjGN2OMe3rVOfQpkjLFCG24HGCq+3pn86x+r1FsjT28Op5g0Lxtb2Q5yT8o4475I7nufc4r0HR9PkvJYlnQCPjCjhQByT+Ap9hoMKzSTy/My8N7Z6IPc8V3EViYYCqDEknyAfTk/0FdmEw0pO7OfEV0lZBfwRuqwxfdC/Me2Rycew7/lWi1uFjgt0HBwcd8Djn8qgMsVvvR13BEA/wCAxjcx/ElR+NWbe6Zrwsw+baFHPfODj6Yr2FCN/U8tydjLu7dvNZh/ECx+vIrm7vDiMY68DHr0/mK6x5fJn8sgkjDHPp1rKvrdCpEYz8hcD15OcfhWU6Sd2hxn3OPmtlcESL8wyCvqO+Pf2rn7vS2RC9t8w9OzL2x3BHpXYSkHJYb84IYDJ9v/ANVZ5mSTfAevUj09x6g/5wa5/Zo1U2Z+hygSfZz/AMtPuj1Pp9cVrPb7H82M4Knn6HkH8e9c0++OQoGz/FGeuGHI59D0rSOpO8YnJ2sPvZ9Cev4HqPxoTVrMT3L8rkuCOGH6H/A1i3M/Lc8Nkj6jtVma7jJBGAcfl7fT0/Cudvb2JQN3RiPwPQUSYJFG6nLE5OM4wfekhBWBgeC3A+orKabzVz/zzYD+ecVsRI0jR56qcmsupZrQRhoVRhknPP1x/wDXqeGF45M+nf2/+tToomyFPp+grQWMnlumc/gf/r1rFENlN4OTkc8gjuQf88Vy+qNJcRfZN33TlW7o319DXYyFS2yU4I6H+VcpfSMJf3i/d7dCD9R69a1WhKO4+HPh5pZ4vnyCw8zBw3JxuGfQ1956ZaNb2kQfAkAAfHCsRxux2J618b/DaNZdVt5I8ZXGVzgsp64zkfnX2nbkeUAoxjseD/h+XFfQ5ZFcjZ5OPleSRaooor1DzgooooAKKKKACiiigD//1f1SooooAKa1OqNjigBVry34s6kbTw3JCr7d/wB76elemeaFPNfOHxv1pY5YNNTJ3fMw9SOg/OubF1VToym+iOjDUnOrGJ82XrLFahpu5yV/ln6020tJruQZ+RV5yOxx7d8cDjvUl6pMyyuu5YiT9ZO/1A6fWtqytjp8KichpZmDKvXaO5/E1+cW5pOTPuE7KyNgZtLYIgZndR8oGDtzwo9Af5CpIEJCgDcQTuYcAtj+QGcVn3jzF5JlkDPgxR9ueN7/AJYUfjUjyLZ2oRGX92m12PAyBliPT0rRPqRYjuZwd0UJ2b3JAUfdwMDP071yY01BcmVySVwoJ4GO/PqOp9zir0+rKEzD94puOATjJ+Ue+euPSs8yz3UYe4U4TGyMDGT6n6Ht6msZtM1imbqeSQtvb7meZcE8Agdznn0/zjFdbCIkUW1qqrFEAmTnk9cDucck1wltJ9mDK0oaSTLOeNoHufQen5da6Oyu/wB0FjAXJyvHJB6sQO59KuE1YicOp0O8sCWIQDkcdDjjP8+KpvblV84DzC4+UYP4E/XtWdbyqeJd0yL8xAOFwegz/k10clzGYV8wqCc/KDwMDoT7Dr+VaxSktTJ3i9DnWsHZA0sak9gemc8fh/OpLawCEzNtIHB9Sc9O9WVmW4YzEYEhwD329MDHTPPPoPerUssCxRlHWIbQFwOBk4GB7cmpVOD94tzlsdVpMcfmLIx+aMdM8KB157k9z/kak8KyR4bCE84/ur/PmuUg1aKGF1tvljiIUEkZwo5Y+/p2Ga0ba6N9cKASFzudz1OACT7DsK9anOHLyI8+cJc3MTtpayfu0b5EGW9ye1Z02nNGPMc4RM8e/oK7GEbVEaqOcE47ZOcn8BWTdSjUJQkQCwofvew6nHrj+dOrhoJX6ip1pX8jh5IHch8cnp1wBnNNEZyFPPPH19a66SwR8sB+7Xt3JHr7VDLYpaRtLJ/rCceyj0zXnSwclqzrWIWxgGEqSQuSwx6YHens+0NHG27nBwMDP+f5VEzPOSqk89f6VdtrdjKI4gPlwST9ayjBt2RcpJK7LcIkDCMErgct3+gHpW5a+Woy3GB37/Wqa2uZjKzcbSP8/ia0444wxV+COT+HavUowaZw1JJof5jStuT5STg44I7/AKVz+owyRglQMFsgZ5B649xxW3KpWPeM5OOB6nGDVe+t4zYEXIwyYJ+jZH4VvUp8yaMoS5WjmHuEW1klJwyuFH+6QCD+GMVwnibT1EX2pc/JkZHdTg9uvX8q7bUGLjyOC23A/wBrB7/571QWGC8hudNuh86qpQ9zjKn8wRXnSTb5WdkXZcyPI4NQNrMlywVFt8qC3KoT3wOrt2A5xxnrXoVhqH2pVS6J2cM+eNxPQE9z6gdK8e8W2N/b3EUVuwTYSTv+6h9WxyQPTvx2rf8ADlwCiKztPLINxd/vBPUjsXP3VHb2rTVpWG0e7JqsEAzGQWwNuP0/D6VYmuWkQBHBduASc5J5Y4HpXnXnLBliDJK5x16exP8AMCqr69NajzUBeRfugD5VHrz3z3/KpliraMlUL6o9TR7ezjijRsuCWJb+AEZLn/aPX8aR7pVn8tDtSJfmz2UcufYlj/TtXlMHieKLcZ8mbGVj64zjBOe5PPPQVvpeieAWfnAIcNdSk5yV5Kj2HT65raGKi1ZETw7W51QYyBrqbHztmT02g7yo+p2j8K0o5cGKUjJjUZ+pJP8AhXKSa9DqV0NK0seYsP3s/wAUr4x+XP8AnNb9oXkgSNAT5khYt1yFyB/jW8JXehhOLS1NS5KedyMZKgk+hGf0zWZcoY5sDgxybfbBHFbc6rNeKWHyuCg+vT+YrOuCJIVlcfNIgV/95Bj+VdM1uc6exzE8Ij+UDKc8dx3x/hXF38JQfaICQU+brge/Pv8Al616DOhyDnjgEj1I+U/Q/oa4rVphA8jJ93B3pjt3IH8//wBVclSJtFmHJcIZME4zjGe27kfqKsK6Sgg9R1Hsev5Hke1cqshMnkEjHVGHIx1GPyyPTpWhBLnK9COQfT/61c9zSw27n8oBQ3Tgf0/UVz1/c+bIsYOQB29+RVvVZxl4xwd2Qaw4w9xOO2cD6ev61LRSNbTY2kjUn+I5z68YrsdPt/lMjDAJOKyra18lAFH3eAPb1rp7R9sXIx0wPr/k1cYgy2kONzegx+ozVmJM/Kf4ePwIp7KI4zu7/KfqcVWeYrIhBxuG38eK12M3qMlEcqmMj5gOPUjv+INcTqZlVVdPni6Er1A9/ofXpWzq98bZlnIJVz1U8g9Dj36GuR1CacS742EiMM8HHXr/APXHak5Iaiz6S+DEcFxcrJIvmIpAPGduf85/CvsSNdqhc5wK+RPgNbTQ3IkliYRzr8rg8Z/usOhB7e/evr4da+oy5fujwsa/3gtFFFd5xhRRRQAUUUUAFFFFAH//1v1SooooADVdzU5qEjNA0ZdxvZ1Re5r43+KmsR3/AI0mgR/N8giIEdAV6/lX1b4t1OTSdFvLyDiRUwh9Ce/4V8BJO02ozTM3JDZPU5bj86+fz+u4UVTXU9nJ6PNUc30NCRfNkXeu1FywB646rn8OTSw3qvMX+8ygE55HOcA/Tn8K5q7v55JHtI2O5s7sY4wevtj+ddlpmneXZeeY/nf5Uzxub/ADGfavjI3ex9Q7Jal6EhVE7jDRgYyPXkAe5PJ9AK5bVbzzSLbAKjkg5IznofXHX3OK6Zl224jwJJDwuf4mPBOPaqC2dnbbkuvmnds4HJwOQCR+Zq2m0kiU9blS0s5TEZ2By7ZCnP3iOB9AP1rOv1WAFbYF5B8u/OAMdST6gH+g55rsod08YSMgZByx45I5x7Ac5qlcWaNtULsRULAE9FB6kepNTKnpoNTszl4wtvbZmG/glFHRmHTPrjqauxFzmEFpJXKhznk5PCj69+nFMnidlyvWTAGP4RjilguoYsKAV3kksOCxOAcH0CjH8qiNky3qdAk5EaRbRtJ654wByc9TjoAKtyzqFZZBufLAY6Kq+g9s/ia5ZtR37REdqYU7sYAH8KqPp1PvV6G5nM0ixLltoLueQik7jge3Sqc29CeWw+e9uUCbAfmHyp/EAeMnPt+VQG9fzA8XzM0h2kDCJhQAefQVQvP9FtVSQtvmQ9OXIJzj9Rknr79KzL17vIiGUjVAu0DgEnJ57+5qLtbl2TO1iuUYQ2tucKr7mbrnnr79sev0rrdN1KPZ9jgP7mMEysx+ZznJGe1eQWlvfMrtDnfIoHmMcKoPUKOOg/8Ar12emxTpbpbI4CKRtVRwTnr79uTXRQrO+hlVpq2p6jLqEnklZHw8oG9h2LZyB7jgVLYy29vZM8SgeXuCb/4mPVm9h2HfFc1FbyyxI7El1J69Pr/npU5LyARLl9pCgLzluw/+vXpRryvexwunG1rm2dVAgKQ5+UHDHqWbqT7/AKCsvUZ3bZCSBtUFs+p/r/KoZoHZ3VTtWP7xzxgcY/E1QuJvPkB24Qc89x7moq1ZWsx04RvdGnaxtcr56oEjOFjz0LtnH1wAWPtVm2nt8stq26OIDLf32PTn3PSsf+0Zm0/ywuC25Y/UAjBP5VNprpZwurbdiOrMW/vLyB74ohUimrDlB2dzq/kWF5CQMFQCe+DgHHu2aQSqzrDHglmBPPY8/wCNcuL1JLcRS5QyEM5b72wZIH/1qjtb9o5zJ/E4KAemR8v6Vr9ZjdGToM6uO8jmSKVcETHBHTA6kn+VPupeMSYPmfkMDIH5VhCSFYWOPlAYD8v61RlvLweTIpIiUsT65/yK2WISWpk6V3oRXsc0Eu9VLRIRx3Hf/P5Vm38VygF3Zt+8jZWP0U8HHt0b6V11xJa3Ba5YABMCVewyN2f89Olc7KS6O8BIO3I56qzdPwrKtTT2ZpTk0efeMLCPUtLuNVRAxAzg9QU7H8q818Ms0ql4GZWlOXkPDY9F/uqB+J4r2i4T7MZCU/dy5WQEZAz3x6fyzXiWtTyaTqbadFE1tEAMyDI75AReS7H1GAPrWVJ3TizaXdHoj3iRx+VbsoYfLz1H19D/AJNYt7LcSkQQt5ca4y2cM5Hv1IH5VQhmuZFVreHyxx97AWMe5/vevXmtm20ia5kLvk5PJOcfmf1NcVZNvQ3ptJamRFZzSzYtkDtGdxbGVBHTjufTPGeea3bK1ugGWbfkkHJP4hf6101paxwxbLYHpnIGBx6Z6D3x9K0RC0KeacZwQFXls92J7eg/WiNBilW7EGhWc1vJNLF8rSNhSDymeD9WOTn0H416PYSwwZYHcvRD7kH8MccewrzaCVYoWmd8x4UDacAAnoPx49WJrQ+3CEQ2ksoZl3ySspyAwjPyj/dzj8a9HDT5EclaLmeiSt/oyyOCHiVX/Niao3bK0M0AyGQhk+vOf0qpLdC7W9dCVjG2NQfSMDJ/GoNUulY+dGflkAHP/XPNd0pqxx8juUZGDgM5IV8qw9GXn/64rjNfEnAb/WgbkYcZ9wR69CP/ANVdW10LmzMK/wCtfBH++oyP++hkfUVyl5cQzRspJ2scjjO1uh4/mP61zzaaNIo8zllEExEiFCT0x0J9D0wfb8qu292q+YSe24Z/M1X1RiZTC2OvI7Y9s9K566uGtpvLz/D09ewP58VzpGhqXLreSbx0OVP+NadhbgOZX4xyfqOlZFgFReeg5APqa0hOq/KxwOp9lHrVD1OqjnTaMfxevpV23uVkdN3CsR+AriTqG8sxyEXgDuSeP0FbunSljHn+EZP480ORXKb818JVC55wx/HNUJ5/PdihwGyR7EHP9aqqGwhHpn6nNV7wrBJlchTnGOxHFZOoWoGbfXsnzWspyj/MCPUe/oR+VZxleaIwyY3fwtjg46Ejsfcfypt8sm5ZegHHHTP9Pas64dIxGYiY2yMAZ9eeO2etFN800E1aJ+g/wg0K40zw/byMNpZRuRuV5AIIPb9a9wHIzjFcF8OruSfwrYNKyyYjUFl6jA/iHY+9d9X3GHilTSifKV23N3CkHNKaQdK3MRaKKKACiiigAooooA//1/1SooooAYTTadjNBGFOaBnhHxx1VrDw21rG21ZDlznknso/rXxg1xJp2mGcAfabkkrnny1PAbHrjkfnXvHx/wBbe51+00KIfu4hu25+8fc9h6+1fO9zcJNflpXLwqAWPYkfyB9PQV8Zn9bmrqC6H1GUU+WlzPqWNDtna5UPgSSHccjJUfwj6mvVpZ4ra38lGBIHl8/Tnp6dSfavPNH8yBmvSAGZSUyfuqemffHPtx3rRaWXy9jjaihVYA9R94qPrkZrwVLl2PXkr7kt3rEdtjyk3u5CRgnHA64+p/rUOnXLXJd3beSRuI/vMcsfZcDA9etcld3b3Fw/lgliAcAYAXoAD2J6k9667RrVoIIpWyUDBpDxyegA9R247A0J9ymtDtlnWO3DOAmW4HUknGc5/ICqJLCRpJyP3nLE/wAIA2qPzJqja3cd1qNsX/gaSVh14Rfkz+eT7mpZbuGByZEMiQhQqn+OTgj68nJHtitr30MbWZBNZgyuu0tsBA4+9k9Pof5CsR7VWlCA7jHkEnnp1zjgD2rqW/132CQ77mdcORxhn5cY68dKzlsJX85IGChyVD44GTxgdyOBzyfpWM6ZpGZk29vPdTLDChRCxZAerEdyccDp+H1ropWEDG3hbzJGHmSuOgA4AA9BwAO55NV5JIrSMx2/II2MSeXIH3c9vc9u3NWtNUCaTzHBlnG6RgOEQcAD0AHIpwp23FKdzPkjedTPtJ2ZIz94tnAH5+tNi0+UDCvkKQgZuckY3EDpgdB6mtuCW1mtmMI2RLn8FDf+hEn8z7VSM8mTg4YEAADhck4H5kAD2q3TiSpsox2ksl03XyIMrnA+YgZIHocnoK6uGOKxRFkLNK6gOOm0HovsO57msxw1nIHK7zDtBzzl3bKqPXnk464x0rMfVLmVnhQkNkK5OSSxyTz64H5mhOMA1kdJcalIVkMjeWnAIGOFHQe5YkYH+TswXxskG3AZQ2B3GAc8+p9f6V51FIzOrzsqlWDhScgFeh+gPJ/AVblvELRyGTbGpwQeyjk5xzlj1+oHrVQxLV2EqCeh3MlysNpGHbLSHlAPvORz+Cj+dULmQed5e8DqpOeODg49TngVzBv7ieRro5VdpEaD7/P8R9CevsBk9qt2qp59vNP8xR0AXsMf0HNN17oSpWN+2iCiTJ4QYG7JPvj/ABNVpZCw8vGQ5+cduPesyG8ODGxOWlHfAJAbOfpnP1+lS3NwnmLGMhQ29l/3xgZ/LJoc48ocruXXOZhcynIGFC98dPwFIkZNyEJ+UEAEeqgg/lUNu+yzM7H5o5CeehUghj+HatGOMH95H0jYkZ913Y+uRinFJ7EyYs1yY4ljGcyHn2CdMfzp5laSM4AwOWHvinSRq8zvngEY/wCBDPH65qjbPslltmPB4U/7eOv41V3ciysXfOW5SYKdjgE89c/56VTW6UwmL7syKQB2f/8AXUrnDjnG9Dg/SsogFjGemQfp7/nWilJbk2RqMEv7YkZLgc+vHr+HGa8/8a6Y13owvLSXyZrc7fMx8wU8YOf1/PNdUolsbhLtXG09cnGB6Z9+2atFEvFeaDarn5Xzna6443DkZ9+4rSMk/Uhq3oeGafC/2mKNZ2YRc4GQDjqc98nqRgdsmvRbMRnCTO0jAdAfkHru5/MZNec3Wl2WjavNEYZFnkbLLywUH/aJwAe2P1rsLWURJm7YliMKi4VVH48/pTlGzC522AY9u9lV+cZAaQ9hx0H41Vu7hncW1viOCPgkDqB/CvesOPUoU2oWwyjc5BJ2jsP/AK351iXmswxusikbE5AX7qqPfuSetZzmkrFRgy5q0txHGJYcq8cilVb7oK/dYj/YHI98Vk+HtQNvPDDeP5gVmMpP3cswO0Hjg9D7VQuNUdylpKTuciRyTngckD3HT/8AVWZO7xQqIVyzfIEPUmQ55H0JJ/AVjzO90bJaWZ6dp/iR5ILq5ly8jAr04LsQ5OPTofYGrn9ryTW8QZtwU5/EI2P0rzlp0hEwictlGjQ/3pSBuY/ifyGK07KVktWZmwscarj/AGgME/ka0+sPYh0kdQ99LHiROsWPxKc/qKx9UmKyvJCf9YN4HZhjkj39aoRXLbd8bff+cDryDz+YqK7ZY4yuSsed6Ec7fb6A1SqGbgjnLu9hkJD5BHKnqMHg/ke1cfLdmW/aOXrtx7HHU1b8QSzKJHYbCw+cDoT2YYrjtOvZbs+Y3EjfLz2rshHS5zdT022n/dCRuW71HLcMW2k4Xgn+n/1vzqDTYfNGF5X7o98dfzNbz6aZVJA/z3/wFZTvc2jZGXAXYgyYC9cfXn+QA/Gu0sl+cjuFBP1PJrn4rQ+YiuMAf0rq7GLAkbB52/zrJ6lstCFsKf7vFQ3dr5pKjvmumt7UMWDAY2qM0klgwY8dKHB2I5zj/sBeIxuu4Dhx3x2P/wBeud1GxEclq0fAjlQliOi56n1HrXpM8ZjKug57H19jXMaxcwWl9YBEyss444wMnpzXRh6dpJmNSbaZ+jXhSwt7fQrLywmREuGjPHIz19PY11NcF4DETaLH5TEhAFwccDqOhI6V3or7SlblVj5ip8TGsT0FOoorQgKKKKACiiigAooooA//0P1SooooAKrXtzFZ2kt3OwVIlLMT6CrNeHfHbxK2g+EpESXy2lBAA6k/0AqZyUYuTKhFykoo+LPiB4nl8QeNLzUpn3BiwXB6L0/lXNCdfKjXIDO2QB2Lcc/hz9K5eH7XfTlpPkErYLY6YOSf5VqySg3CxKPLCtgAd8cD8z/KvzvG1Pa1XM+3w1P2cFHsdwt0rWcbSchpMYJ5PPAHtxk+1Xp2e33SSkZAO845LuATgewrm7RlkuolBIitw7HsBjj+Z/Gtn7WWkJRdxDbUB5O9gAM/Qcn0xXGnobPczIjbLcorxlMk8E/xejY9O/oSRW3ealIGt7aBsmRjyOFCgYBGPTOBWNFFEsvmyYZmZsY5J56H3ycmqLXGdTjjhXccmJD/ALowSO2F6D3peha8zqLS9RpN8Z2Ql3iQDkkABdxPoOg/+tViS8D3CPjAjcHHXBI4HPoozn1Nc8R9kkgQnc5IbA+6mMYB9uhNTQoZZhCuWHzscHnAGM4HVjk/SmmJnT6VcPMW1O5yxI+X9e/5kmtG71Ke1AS3GBgFmA6Z7L6nqSeg/DFc/fzG3tYLW3AUxKFYHkBiw446gAfia43Vtakk1BbSGT91GhLdySTySfp/PFXzdBKNzsre4tzcwrt3gMQuD0Q7ecnux78cD0FR3msPAt3HFzPP930G5tqL+eSfYCsDS5fs8YuDlpZZG4zyS+cZ9gqfrgVo6UJb7ULmbZgA/KW9Bk5/X9fak5N7D5UtzYsbk2emJYRnLtJEu7/ZQ8n2zg/hV571BOZICdgkaU9hgfKuffAP0HNcvtMiXH2diQHUE923nPHoNo/WtWOyjEH2QNmRzCq54DBgpkz9cEfQ1UW3uS0jsCyJvknJULjaO+SpJJ96IIIluIncAKwMxXA7jcP8D7D3FZ+s3cD3EqdVKsxxnB3BT27cgCsObUZJdSzHtaONAqbgeqoO2cdc03JISi2jqZLONIVuJvlllIYDoQvXHsCSST7H2rLOmGaRLdkCR56kYxkjHHrjn6mpkkuLx4xdZ2KPm2nGWxnH0HTHrV9HkjKPMm+UsehyqySE5/CNM+2feqVOMkS5NMY0UUe1DGM+WSMHnaMd+5b196iljmaIuiYwVDNnG3OTxUEerCa5DodxJK7iflCsQCQBjJABA56kmqM3iBHvI7RT5ah8nsAu04+uOT9TQ1DYacjTmVUEZdNhyQqevXk/XPI+lRsXkjjuJeXI+YdFGD3Hes3V7qe4vrKNCyjylL9j8z5IA/vNxk9hW/qEcEaLA65YF3wvOAoAUH1JPQfnWcqd22ilKyQi3kf2RpOsS4Ue5PX8wM1b0++Ml3LA3y7hyR/f9/xFYUpit7gaI0gNyQp25yQ56E+mM1GrmG4cSEBZcLn0JySfwI4+tJOSYNJo6+S8hFu0+eARj229f51VmlaLy7xR8n3sjoCvOP6VzsF2JdNadASjFgV65wo4/L+talpdAQC0mOBK7AkjI3EEqfbpg1spX0Zk422Ogu4ooo4LiEAoPMI9GEg4H4HisQykFH/iUAk9yp/qD1o0m5a7sFguQQS6oy9vnXg47YYD86vJFHNHhRtdc5/E4P5E10O72MdtGT200N7C0SqGJ4x0B9QQensfWube1ezm8y3YgfdZTkEew7Z9qttDd2su6Pg+nrVuJhcEiT/WgZ2n+MegPqO3ftzSTUtHoxNcu2xxXjDThdWa6js882ylgwGSnHJx3x6H8DXkdvrl5cqFt3Aj45U8uR3PevosSQpMQBtLDH1/x/KvDtc0K2sdZla2McHmsTtQYJJ7+34HHpWsk3GwoPXUgWSaaIi4Jbn5ix2k8dAvp7mpGQPAqHnd6Hnj+XoK0rLRmMYZRgL05wo/Crn9jR7PNlDbUOFXHX3PTiuKUWdKkjh2kEkzNy7kY+U8AAjgH6/nV5b+OCK6l3bp3JHm54jyAoI/vEjp+dbsujSCM8bd3IHfHb86ypdA3XG5iWwMhQO/r7AetKLtuW2mW7gRyXgt41YRp5b/APAiMgfkMn8K2XikjBDcJLMxOfRlP6c020sJ2EcZfJyWdsevYfQCusTTvtMoLDCLg8dBk4xWnI3sZOSRhw6a5IyeoxnoAcZ/nV6ezZ8LjDMMqT0yR0/GurjsBGqk87k/Vev6VNPYEjC4II4Hr36+vpW1OizCdU+bvExnsdyFcBgec8A/3SO49K4TSPJWbzZMglvlQd/9709h/KvUfilaLFZkyOsco5Qn7sgHbgEgjv8AmO9eF6bdtCFJ+8xyr5GMexGc16VOPuq5yN6n0Lo8qR4jkByOo7fSu5ilikjCxpg968l8LTs4RZTkHoB29zXtNhGsiK2PkH60vZofMyibMNMvAHety0s8ruA69PpViaBBKsan5sZY/wBK1LeE4GemKj2SuPnZPDbhlZh69ParMsX7vdjoeauQqqBeOox+dQyOTGSpAZB+YFU4pEXbOXvVEQJGWA53YyMe4/nXmHiKC7l1nTJ7Z2VFnTzI15Df3SOD/SvSr+5CHkFS2dp9SPf1rz6/uEn1WxQxNkyg+ZEMY543YOefp1opu0kOS0P0l8ESWtxodvcQxiJ2RQ6jPUDrzzg12Ncv4OkSbQLWZTksgz65xzn+ddRX11L4EfOVfjYUUUVoZhRRRQAUUUUAFFFFAH//0f1SooooAa7BELHsM1+d/wAfPGr6z4m/seBibe1yzH+83b8B2r7116RlsZDvEaKCXJ9B2/Gvyu8b39vqnjS68pi0e85b+8c/1P6V5Wb1HHDtLqellcFKsm+hWtXjVo++O/b1P+fetaK2SYhsDcRvGemeQCfYZJrAtmaadGYck8enP/1s/nWtMzQQvsJ3uNue+M8Zr4SW59d0LNsRHuQPu83aQehCjnt69fyqaCaSCeRo1JkT7gbqWfjPHYcmsn7SBckwHBRgp9SVHGcfhWhHIliJr24P7x2bap/uouM/QDJ+posIa17HbRzzZLCF1iHr0y38jUtlZi41Se43FYrUCNRkAAjGSxPH3j19qwriaV7a1wuQoEhUf3+ijj1Y5+lGrahJaeHfsdjjzJpI42fOSxzhj+DE496IrUpnS395A8qxW/zLwMjku7Y2j6AAsT34JqxozCzt7m/mOVJ8vP8As9Sq+pPy5+prjZ5yT5YYgRy43DgdAMZ/2VAHuTXQzSmVdO0kfKqymR1z1IYEZ+pIqoq7Jehtagq2oMpfbID82MnDP2HuAPwrlU0+NJmLg5UF29yx+X8B2H0rU1rUIr5pmVv3dqyN3wWlUkn1PyhQM+tPZXuEMeQqSgHI45Bwx/ADNEo9UCl0C3t33xkEFXyFIGAONoA9TnOPWuuFsILW98k9I/KTHdyCCfw5NILO2fYsK4SKSNFPbaqEc/nioZL623/Z5B5ccAaVx03YB3A9gNxHHfmjSIXbMuS9huJ7extcAuY2l54xtx19l/x71uW95A08N0wAYqJE5HAC4UfiMn8RXExNu1CW4ZQpO9FXP3UwFXOPXGSfatu5cLClwxEawZVyenGTgD1wAB+FLm0G4khujLumY7Q0m0n1C9APQZ4Fbnh2w22wuXjUyIcKDzlupGT2Ud65yG3SdYrdv3cUCeY2WIVe+W7kKo/Emuim8QiOx3WgECuQIh0YRbQWfnoTnj25NOnFfFIVR/ZRrX11FAtyI3KeWQpkB+bAGW2D1B4Huc1WkmuJfMikPlgKECJ/AhA+QHuxY/Mx5OKbZ28V3YwJL+8MjGZ3P8UmcKB7Dv7CtRrdY9vIbYBhuox90cepJY/lW129EZaI5q+W00q03R8MoDKo7/X/AD04rmWjVpGuA2S3B7EEjJ698D9a27yYz3QjgG5pSfmbnbjHUfSoW06C3gnZ2LrGwV27l3+Yge+MZ/KsOVLVGil3MqfU7hLqC8kQmQtGx9Bx0H4DjtyPaug0nUJFV4bt9kkUSPnOfnwDJnPHB4HvXOTW7yyPcXHQ48lV/vKDk+nB4H5+lIlsB5Ue3ejxvuYchUQbv1YEZPX8aIKSKk0zes76OTXQkC+ZcJvJc4JC8Pk565zxn0rVvpbWOeG25MxBkIY8krDkqT6gHn3xXE2X/Et1Ge7blzEIxz/G4VyB9F/QVorEi3UNyGIdrUIGPVWkYHP1OefYV0X01Rk0bWniWGB1lGYbeUpxxk4BYkezAj6U+Kdpnls3JDofNUDnKruHX8SPwqCG5iksntmf5ZSW2+uOSfrt5NZVrPLJJNqVk20uZRtA+7tdXAAPHBP61FlZMfkdlYmOVkkjfi4UMvruTr/MGti3nlR2kbknGfr/AHh9R1rjtKuoHjubVV+zyAm5jx90K+DuUdge4/hOexFbegXjanpUjZ8m4gBVx1B2nKn8q1jHXQznsdmMXcQnYDcMh1759R/Os69tCuJYmIOePcjn8DVmzmItwxG18fgf/wBVX5AwifzADk546Eeo967VTUlqcTk4vQ42ZmmUtt+Ykkr79x7eo965fVRb3CeckoRo+WDg7l57j+Jc/lXeS2yyszQkFiM4PGR7/wCIrzzxOEClpFMbjgNggq2O+Oo9uhqYxa3K5kyrbyoApeUbc8eWDz9STn9BWst5Z8yZ34Hc9fw615O+sXlmTuEaK5x5o5T8Sc/TAwfUVs6detOi7JlfPBb7mPrx+nWsZuxvGNz0QOJ5PkVt2MklflXtx6YqUWaRjZMfLZz90AE8ep/X2+tcsk7EeRJPtRiOWJDN745yPTNdRaW+3JHTuTyx/wD11EbMJJofaiOOdSq52ZyegAHXPv8A44roLBbeQyMeDOQMf3QoJ/PGM1h3UQRPIiOS3b174H45P4VBayXVrdbTwNxkYnqA/B/SrjNReqJcbrQ6qzaO4td+SBuyufQ/Kf51N5iMfIz85LbNw4Yjqvsc5xWBBcSLEtpzhXGD7nIH4etaDL5kDNKfldt4P91ud35EHPtW0ZpmU4nnnxJ0T+0NGuHKkoqksDnKH+9x1HqD9eor4itb6W31l7aXywFbaHGBwPoQCT6Yr9GLx5ZrZ4F+aZVwAeQw7g+4/UV8HeLdO0+TxBcRW8c1nJG53ROqmPIPODwQD2P54ruoy0szmmrHqnhi4UbdnzF+2c/mew9a9v0icuidwOfqa+bvC96LVAiHDHgk/McfU5zntXvmg3ieSpkbJb05x7Z7n1o6jO4Qt5gYnLHOT710toEMXv0rmYdsuGU4ArXtZdqbl9P51LdgsbZJdNydCeRVK6Yqu6Pl0HHoR6Gn7iqK/Qd/w71BLOjofLP7wZxnv7fUfrWMmXFHF6jPtaSMqXhk54PIPYj0I6VxOs6Y1xf2ctqFLxzKfnO0k8dD0z9RXpNyY5laSNVD56fwn1+lcRqXmXM0FjHIqM8ihCWCkMDkDnrmoop86KqS90/RvwI27w5a7i28IMhsZB+o7V2dcn4LSVfD9qLmIwzKihx68da6w9K+zpfAj5ip8TEXpS0i9KWtCAooooAKKKKACiiigD//0v1SpG4BpaZKcRsfagDzP4i6pBpugXV5csFijQliemPp61+XcV2dR1e+1PaUjlclR/dXsPqRX2F+0b4rH2KLw3A/+vbMgB6qvavk6CCOO1YLgs5BwPzr5fPcWm1RXQ+iyjDtRdRmxY+UybeFI79hnr/hRqEq/ZnmVwFR8KSeuByfoOT9MetYAmNqArPwxHf+f61jeLdX+x6dDZr/AKwhnbHbd/kV81GDbse42Wbe5bzY51J27Wlz0ycBUH5ksaveW9/eSRSPnYPKBz8oLkFm/wCAiuUtbnyYo5JWztGFGdwbYM8/ienrTrTVGEQUcuwO8f7Z469uo/KtHBvYEzck1YXV1JKj5t7dnkC57RINn0GR3681k2089wIbE8rGYArdw7LuJ/EmsaAtboUOcSsFf18tF3H8xXQI/l7Jx8rTGS4Y46fKRHx6LxT5bBc1kZoYnSX5lbcR6LtJbk9Mkgk4rW0S5lWGS8lCs6xnb2wQck+5JIA+lZkNutzbHByzxsVPqCQrZ9sc+2TV95Y7VBIr7k8oSMByuA44H5k1CXQG7m1Zxq9w8IPzvKGYdz5MeAM+gOfzrQkuI/N+xKd0ShwD2+UrnPqSSfzrk7bUthW5jYiV45WlPTglRx6HkfhWva34tra4uJSG8uRNgbsqgZJ/4ESSKUkxI27fxFbyaemXILR4QHuyEhuPdhj8a4uzu7u+vZI3XfHIC8/spwVUe5PGff2qt4gAg1RZIZBHErM0Yz0V+qnnqD1p+h3kSRXLMdspjLDjum5v6jFJx3aNIvS522k2UjsLu7df3yY+Xpk7l49Bnv8AX2rpY7M305R8GKCTpj5flznPv0Jz9KxdPu3SK0W4yrlFJ4GMKDhc9M8kn6VbbWo/3KwKWW4YLtA6+azck98459Mj1p8q6GbbbK8qxXLywqS8SmSMr1BaNVzkDk8uBj1qhNb3VxqP2Bn3HBaR8AgNIfkUZ6gAZIxjp2rZ+1W6Myq3loTJukxhvmIJxj3LN74FW9Gt5dUu/tAh8mDeuQ3+tcLwqk9FGAMgcnuaI67Dlpqy60674ULbQAwxn+HoB9TzUV/qM3mBYxhArEDudufm/M4H0rQFtCLvzcZRd2COnHDHPoBwPc+1VUEZtZdWnOEYlIwByw6AD25zTcXFXIvzM5SW7fSza7TlicMO5PBOT+B/OqMl9MYIrN5CEhLtJIOhd1JY/wDAASP96m3cjteR3Z5RSRtBzg5I/kKuNYpbWqxXvMjNkj1Zzu2j6/yqL2jc0S1MdtQeWZ7qJNi4XA7IsascDtxwPc1clvbZL6CwkO3fbFpMtwFt4m2qPqQXPviqN0qLOlrbHLSMrk9eFy+OPr0+lZ1xayPercnKmENCPcurE/zxTjUG4GnaXCXG++lUCV2dVJ7B0WIMeewz+Aqnf6s4v47ja3lWjxSbXPJA8xGBA7NkN+NIllOyRKhO0Bdy56lDtP4nqfY1A2npNfxyTSfJMvmFjx8q/L09F71cat9EQ4pFB/FCQa2t0TmKQSMmOhZB5YPPQFR9KnPihdH1mO0i3fZ7gtMJABx5o8wfz21Cnhh7drm0nTe9uzKjYyAh52t6DOSD6HNSajpyafLFJqMXnCHy4HXjIDLlGHr1I/Cr5r6JC0Oo03W7K/WKJphDdqdqnoCrLkf8B5/MYrttE3Iwa3O12jZSPcHkfUEdK8ts/DsDnYrgLKSIZN3GCM4PoD+hq7aX2seHrqO51COQAsY5uhxJ/C/HYjr7j3q42vczl2R9AWdzujVJUAdS2cdyP5VorNjCKdw5wO+Pb3x0riNJ1cX1tHcnBZgQ5H98DqPY10FvMkn7gkAj7p9h/hXdCRwzjqRTuPvRkr7dseo9K57VTbXlttvMusnAdOcHHRh/hWvcSbZN0q7gThsdj/eH+etcvqQ8qQ7SEL9Tj5W9MgfdPuKYkeXXOjQx3heznG5uCwfIdRx8w9e3OeamsNKAUmSYMp6BGJGT9BTdVmMV2Xb5MnP3TgH13Lg89/et7QrwzgsJlx36Fh+LDdz7VlVp31NqdRo3bLT0tYwxYFvX73+fwzXQ20VwVCdEXgdO/c4qK3MWP3Sl5D69B7k1pweY2CwVQOeTx/ke1ZRhYtyuI8cdpt/5aMwIZj1wOw9KR1jYvI+GG0Ej3IyFH0x+lWHg3ZZk+VcDP64xVJnYHzScOBwOuFPBz6kmnJISGK0RjmLEAKQAewIBA/Dv+FRpfNZzRxFt+5nkK9iQPnx+h965W7vCjpbnJVhk4PocAfTqPxqlJdSDVLZyfLDuQP8AeBOD7Dt9KwVW2xr7M7o6pDcRp5a4IAAPXcPT345XP0r5q+M3hZpL1dbsGVknBZl3gNuHXhsZHrglh6GvfNOCyItxJGTFgElf4SOuPp/KqXirRNK8S6TcaNqkRaN18yKZGwSQOGBHRh3HX0rtoVWndnLVh0R8leFjj5rpGAQ8Andk/wDAe1e9eHpnnkTA4GMA9AP8a8gtNAbRbtre6kynARgCwkH++OvuDzXqmhzC3ZUyO24qOnooz3rsbT2MD2O0mXZheff+ZrZicIMgZ6AVxtpcB1VTxz0Hb/69dJZ/vThTjAz+NZSZSRfutQjjDFThh0B6GuevNUB+eDIzw6N1Vh2+nv2rVurVbiPyzgcYFY8lgdrEjLgYPcNx0PofQ1xVJSudEErGVcamzqZmyG/hZeuff1/EVzEl8mrajaKYhcAyqSUHTB6Fex9D0zWpqsU9pCbiHIUY+Vxxj/636VzmhW0kvjnTryx4LOpZVOOfY/41eGm3NJk1orlbR+pngsxt4etXhkMibAATnI46EHuK6usbQhB/ZsMkKeWHUEj3x7cVsZ+bFfcQVoo+Un8TFHSiijIqyQooHSigAooooAKKKKAP/9P9Uqq3zbLOZ+mFJq1WRr84ttFvJyMhImOPXApMaPzF+L2pzXnjGbzmJCdMemf85rzqXUBaxGXglS2APU0eLtZOoeJry7LbwHIHcYFcZdXfnk28WcKPmPqc9q+Exq568mfZYVctGKNSKSS5ZJbhslwAR+n51y2uXLXV6sjnKOyoM9wvzH8OK1ZLtYrctn1P0APHNYcJM88UzkMbdtuMZAAUkk/57VlFa3NmTzXM6L9kVczMNgyfu8bifrmt4RrBG1wSSHDvyeCfvk/gM1gI8cc0d8hD7D8xJ4JJJJPtkfjXQRt5+nxrL/HCwHQ4dwsfH6k+1OWiBFZMXNlLPOf3jbYlHQfOWzx9F/Wutgt1ubSdVPyrAGX/AGlWMqB9c1h26w3tx5MOQqYkBAzngKMZ69atS6iIXuGhUqQse1en316H6ECsHd6Is14d9gUgZg32fCY/2W2hzVKZZJbK4tl272jZG6922L+ozS27mRBdDLSThiS2OpAP9PyrI1C6ngjZEHzMoyR3yc5/E0kndDNm0lVyyyybVYR4VV3Eq2CSf0/KrzHz0l05fnWVVQufXIzx3OTXPRSMs1kD8rZAYHsvPBx6L+tdfdQrarAbNcyXCvlge4zhuenzHFTJPoVoUbqKTUYmIO8tIERvVSwUNjr1zmrUq+Vq0kKDYrqYuw+WX5RjHfpVmWwEbwshKqFyP++vu/gKzrmVxdy3yHcETeOORtbYmD2wcEDvQrX0Jvobeoa2I18lcERyLAORypIDMPrnAo0q9AS1vJG3C3QsPQswOP1ArktWjW3iBcAtGsarjHDjPOP724856YrZhmWGCJZ0VvLJkKk8DjgH0Axlj7YqHqrRNIpLVnY+H7aQx/2pfA/u0PlqTkAE4Vvd3JwB9T0r03Qre5CMj/uzHw5H3slTgD0Pf2+tcTokiu4ku222tgw27+styR97HpECMDH3iB2re1PxImn2UiQt++Q4X18xgfT06nNbRtFXZjO8nZF24kingNhZDCHcHbH8Kj7oz3J4H6964/XtRW5vILC1bMVquQqj70hyOvoP1NQ201/cWoijBXzfm3E8hen5H9fxrXt7BrYi3sU3Tycl/wCLHfk9Pr2rmnVlN2RrGEY6so6fpbxpEswwSRnPXHtRrZS92vEoI4RO3J+Tdn/dB/ya6me1t9Ph33MmXfnPbHqvfHbJ615vqWpCaby7RcwQKWyRhQTwB6kk/pR7OV7C9otxiX9vZa7bk5lW2LZf+HkYAP1I/Ks7U9RUXe2D5i0weNR6JkE5PryazRY31y7PduNvLAdBzxwPoTyaz7y7ZtSYWeWKjYG5xznOB7ZPJ+taKlshc/U6nT79WjIuAInilYnB6oMY/EdD7Yp11fW1i0V26iVMvbyoBygckhh+HB9a5fymhXy1YlFDE45LFzk/pjFWAsl1HOrghlKFieQu3g/XGce5FappENNlvRL+50a8NtfMZ7WeJV8wgnMeONw647eoqbxM7XCxwWh3hCV2Z3AqvIwe4Ixj0rOaAz2W1ly1uTHnPJjbp+IYfrWdPFeQ232mMMstowfj+JM7c/kcH6CplK70LUbbl2z1O5s5jcoBNaSkGSPuFI5A+ma9BsZYtQguLJ33q4XY5OcgD5c+9eaMFZ3YL8kygtjsT6egNdTprvpm4qTJE4GfXB/zilGre1yZwPRdGtWhRVU8SAblJx0GP5VsTyusBuBkSQnD49OxqlpTOpVh86EAnPUZ5z9Ks6g01tE00ZyrfLz1UnsfUGvQprS6OCo9R0t+l1CtxD82eGx2I9q5q71KEo0FywVSPlb6eo68Hr6VjSXr2lwQCE83GAeFPtnsfesLV9RlQNHeREg9HC8n8OQfcjrXRFXMmzC8QajNYy+TcRiWPkjnkr/eRx1x6HNO8P39tLN5Ts21cEDcBjPcZ5I/GuD1K9jlDRQuCgOdnPykdwDyM96zdK8RT2E4EbM7xnA3HsfqD3rSUNBKWp9X6O6XalwRj1Jzg/TpXRAlgP3iqMYyvJ+mfU+1ebeFdaW/VfPjXnAyWzz7gY/WvSLXy5NvlKqkd8/4f41ycupvcvGRXIt4eNh5P8/x7Vk3cayeZkggjPp1IwK1JNqxFd3AGT259K5LVr5lHlQ/Kq8uxz0P+eBWdSSS1Lpxu9DndRt4iBOTjYRjrxz1/M1mTKpkZsZxIMBuxAwcfnVs3quzgEhY0z9STgcdO9YwvHlFwYSciMHJ7kEgt9cYzXC0tzrSZ2djetaSOY2O1SSQvI2nk8ewOSPT6VfubiVAU2b4/vLkfKc84BHQ9fx9q4e0uZgI5weUO5+fXgkf7px+FdXGeEaCUb4+sZ6YPf02n34raE+hjONtTyrxfoKxXyavYwxrDNgOMnG/OckDgH0PFVbKRbd0EkgkbtsOV/Pua9d1ezj1DTZtjbJcFk29yozgZ4OOoHWvJLWyjvALpwN7DG8DYT67l6AjvivToybjqcU42bPQNJlWUoEPXv8AzruLZ9jAD2ya8x0w7ZlZcqq8fh613kcjSIET75APtn/9VEnYSR1cbJcJvUDzEGPY5rFvXuIV8+LGGGCvuO1WrKfCsHGM/wAhRqMZKOMbt/PpyOo/EVyzd9jaKs9TjdR1F2QzRIZIDw4HJj9eO4rn/DsVvD4u0+YOcbwQyjkjPtjJFXbxJbF9okOwk5OOqnsfpVHw8qw+PrJLfAB2bk59eCO3NZU5e8mFVWi0fqvo/OmwNuD5QHcBjPHWrFw5jKsKi0sqdPgKdNg6jHb0qxcR+ZGQOor79fCfJvclVty5pOajtzlMHtUhHNUIfRRRQIKKKKACiiigD//U/VKvO/i1qLaT8NfEGoocGCzlYH3xXoleQfH0kfBvxSR/z4yfzFZ1XaEn5F0leaR+R8ty0ss8khyXk+ZvyOPxNZ0LtFtixmQsTn0+XI/n+lYFzqDmWKPI65Hrk5/+tU8N0JSJmbhF5zxyBtxmvh2ru7Ps00lYdqFy5U26N8p3MT6gcD8OtQLcGO0ZtxDdzjkkkf061KHjuNsmMgDaR/suCP0JGaxLV5CZBIS2FXH1LA5PtTS0BsnivH+zgclpUBX2IPp/Ku9tJwbGKLIX94kYbHOeWJOexNcZDbwsZPKziJQqgf3Vdef1re06YXLjzgcRO0qgHGQi7Rj8aU0rBF6mvbmZLZFVS7yDA46bpFPGParIWRRNdxgne5PP90rkA/StVzFZ3drclduXGVz0UyADPpUN+/2dYooxtVy5b1ARgOfqAa5tb6G2ljbifc1pHE2SJZGOMD5UXCj8cnP0p4sVlHmTgMI4trY6/ujxx7g4+tZlrtj1GBN4KqyHg8DcDn8iK32uI4r3epULLA+4Y67cbv5VNgZnadaC4ypCmQKyrkd9uMfmQas2t4WuhHKwCBBuBOQGXaCPqzsSfQVEr/Z2eInO+aNx6jzcEj2wRgfWktWt4o5rqQHYzSsp4wcyfp93FOwrs3SZnaS0243F85PTORnH1X9azikiziGIbpZtuEHZf7x/E4H4ntV6W5iivZZQu0qDu56855/GqtgWea9vWbZPIzIhPAQ7eST6KvQfT1rOK6lXKD2yXBlu8HMPRe7Ox2oSPx4Hqa6+3tLe2htA43PJ8xjOAo2EAFvXBAPPfntVzTrC3ggWYR5OVZN3HEecOx7ckn16VB50t0d0eI4cZaZ+mOo49CTwo68E1LfYerNNNWkZEZUDbFEkeeDliTuOe5J3ewq7pmiTXzebdKJNmHZj0LEc59uAMdags9PLSB5ydijzCpA3tg43v6Adh6/jXZTW0klukfmG2Rh8qj7yg9WPq3ueB2pcrm/ITlylISRRE2tmPOuScE4woA7n0x2Fb8cX2OHfJ88koy24cuT0B9FHZe/8oYINO0wJ5KgEYCRk5I7739SeuPSuZ1jxMqhltjl2Jw7dz65/w6VqlGCuZ6ydiHWZIFR57iQFySSzHOcDsOgHpXDtOkQSGMGR3O47z0GPvEenpnr6VVvNSkvrsbnE5UYB6Rrj09frjH1po84/LYANITueRucehOf0H4msnUN40ieV4bdXSV2kmuCNzOcEL14XsMc+p+lV1shcrJJHiCDd87fdLL/BEo7DHLHqamgs5EbAJeVieW5Yk8l2z69v/wBVb+leFrm8YtKT5YJz689QPc9zS9q2U4RS1M+LS47kAQ58tQeV/iJ6tn25rp7Hww8kRgWPAZQSo6gZyAT2Pc13Wm+HpHWOKNAAhHAH5ZNej2vh62tIBA+C2Mt7nqSfr0FdNHB1KvoclXFRhseNjwrEcELiNRnH949vw9KyG8PvvZ3X5CCpHXKnr+Ve5XFjDzF0XHb+Z9sVnXNmokJiTauMc9Rgbj/hXQ8DYxWLufOraC1rKN4ym1s+69R+taFppA2CIfeJ43dMk525+vSvT302Oa4BXiN41U8dCfX8aYdJ+yfuZo96sQwI5zg9vcYrCODd9TSWJ6GPbWstqiSEbdgwp9j2I9P8in394Gt2C9CMPH3Hr+XUGt2aRY7c+W26PJOeuPUMO2K4y4kt5/nuNqgnC84yQDgj2/8A1V304cuhyTlzamLqFqHi8qZ1KHlXIypU/wB7vj17g153rE95pcR8+Iz2hyrRk7sEdCp6jjv3r0W4kNrF5hbzM8qGGM54wwz0/WvIfEt19qQ3Vk6hj8rJ82Vx1XjnHcccV0wRi2zzPWbuFroXNpIxQ8gH7yexB649azYpg8gkjTc/8SnGGB9ORTLx3LbJGUr2PQ59uKoLaNKRuGCOjY4/L/OK3toZ31PdfB94IWRbWOONu4RIy5/FmJH4CvoPSr6ZolaRdo9B6+7cZ/Cvj7w3BdBU5XOdy7wcbRnnAwSc9MntX0BoWoTK0UbsWJxztbn2BPb6VxVYWdzqhK6PWCkzrufKlugI5OO+OwFcdq9o5/1mfLJyRnBb/Ae/5V0MGqhiyq+EX7zDkgf09gOamkmt57YySjABPLdz2H0FcVanzLQ6aU3E8seGfZckHfI5AHoB/T0qGK3dEMEbY3KVPHYL/ia7iXT4PNZCwAwHY9Mgep9z/KiysIZEa5YD5lZ8ew46/SuT2bvY6PaKxy8VsSIpJQVJAGPXs36VryG3jlt5IQU4IUnkbckFGHdSQfp2rZ+zn5VH/PYEZ9MHANU5rPLrGV2iMuyg9wSSR+WDVqNkQ5XZLFDItpMjRllIBKA5OR02H1H8Pr0rg2v7G5M0kbKxJIPGCSPXOMN6g9/evQVjENtucHf/AB55yh6Njvjow/EV4p4mSex1czQcSORz13D2PRh2zwR0I7120HbQ5KivqdlpjwsS78DAAFdvaQvEolPJwT9Djj9K80055JTGZeApBI9T2r0/T5fPwgO1exPfHGfzraaIRupCJF2qORyfpVhXUw7cElevsRVOW5eI+YvPHT168fSmm7QvHLG2C/4cjqDXJKSTNlFtGFrVisTtNgPE/wAxzxj1B/oa8+Se1svENvOMgxMGUcH6hT1+v516zMgu9275WyTtbnOeM5rktY8NPdQO1uA04Gfu9x2ot1iRJdz9GPA+u22ueH7W5t3MnyAHI5Bx0NdlX55/Cb4vap4bePRNSLJbg7cDBxjjo3TH0r7w0HXLTXbFLu1fepHXj+nFfX5fjoV4K2587isNKlLyNZEKOfQ1NRRXoHIFFFFABRRRQAUUUUAf/9X9Uq8b/aDbb8F/FTeli/8AMV7JXjP7QpA+C3iokZH2J/8A0Jayr/w5ejNaH8SPqj8UsCQrLIfmUA/iSRip8bLfyhhi6g8fUnP51Ru38t/KTAKqf++c8fnVtd7SxIBwF3Z/2ck18a1pc+uuR29xNAquvXZlh7delWIpRFYGZBwqsCMdR24HYd/bmrgtle1N+gyNw+UdcY5A/TApjaf5ZXY/7iTmNhxtJ5APqCOtTdFalewjuokkmQn5TxnoykrnB/CtXTrpV1eFlb92QUfP4ZFXk024miTzLcxNCpIAzjk8nA459OhrGn05/PLw5R0JGMHn2/D1pOSlcdmjsYpYr3TILwvvH2grIQOQFYn9c9PaujlW3nvD90O4QbegG8vjI9Mr+deSaffLpbm2mJ8mQZYA8Z6E/hXbvq9rcR+bnfIVEcmP4thZsj3OeD+FRODQ4yudXeMtteRiMAEL3wORkjH4YP1qJpViuVWTc4wVB7ASD5vxxVbSryPVmtpdwd1lABYfeGNvPueDUrzi/tUGQsmGbcO7RZX+WK57NaM0Kj30iDznJLufvAcZRWK/pinm5OyLTnyy8NJtHqN5/NjwKz2YXVq4UZY4Ck56E7SR9eKcJZo7tDAuZpnGAOTgKvT36VdgTOwKyraz3cimS4uCcAYyOQQMdskfgDzXSaTFIfLmm24KlwM/u+T8zt3OSOPUDsKyra1iESQuwuFi4ZMnaWPPzFevPYdQOa0w5urd443Hlg5lcDnC8HGOFHYKDwcDrnHNJ32LSNqa4W9Tz3DGCU+XEucM+eCwA6A+vYZNWYLCOe8hMhyqkCBF5GQOSFGee+egA9ScULC0mvZkvZGG2TKxogx+7HBwD64wSeMZ9a72wFzGGby0abZkAfcRe248Z9amKuwbsixFbQQEGbCow3FM5Z2HTd7DtnjNc9f6xcTXAgsUM0jn5mXn6AHv7f5Nbi6feXqO17JuQ85xgN6HHH4cAeg71UurMW8vl24AB4GBzz1J/wD1VUlLoRFrqc/9pngWUykPI24MYwX6dVT15+8x69uK5S8Fxejy3QgHjYME8diegHsK9JGiLy052IqjgsRwPXp+VPRNPjP7mJVCg/M3T9OT61m6cpPU1VRLZHnWn+GbuWZfMGQecdh9a7W18PMh+zRDJH3sjoTxkj19P8K6KxmtlbI+Zx0JHAJ/uj+prrrCS3tcNKBtJzxxz7+taU8OpPVkVK8ktjE0zwT5Cq8ikE8lm659fqa66LSIodsMfCL1an3GvWyg7+/YnAH9a5e+8SJIfLj+c9hnAPue4/nXdy0KW2px3rVNz0OK40+zj+VgdvQDnJ96z5NZWY+UgJd+MdT789gPWvN5dZ3rs44HRDxUqakI0I4HHCjAX8T1NU8x6RWgvqS3bPQjNbpEsSuGcnc2Oe/c1TnvYmDRnJ3H/Dqa4GXVJDkLJ7nbwPpmqEsl9McE4UfhWU8fJ9Co4SK3Z293qNtGdqEFj06ADH865+fVVjk2EmUEk4XsT1xWG6+UdrSFn7nqfpjtVSdoo8Ce4K54CqOSPr/hXNLGT6HRHDw6k9xLNcv8+1dx+5nOf+Ajr+NS2+l2hYXT7rmZOuQNoIHGB0znv0HvUUKQKwZSF9MAkk+5NbkYZ49rvtxyBwf06fnRTru+rCdJWsjnbrToLuLM581yNohT7gP0GWJPqTXnOtfDzxBqkjl4QIW4WMLl8EdCAR17biSO1e8W6nAKOeOpOOf6Vu2ckFqdxXczdSzdz6mu+niPOxxTpeR8qaL8ANXu5Wn1XFpGMbUzuP0OOp/r7c16Db/A1rpUtVZY4cAMAPuqOdoI5JY9eg/SveZNajQ7FCn1OcD8+v5Cta01NXA+6sajnA2j9eTXdCtGT1kc0qUktjxuL4HWX2mCeZlEEPzGJBjewGF3NnoO/b0ro7L4QEv5z3HkhioO3OAi8iNM4wO2a9Q/t21UgMwZh0Hf8AOaB4hkQb3URgev+Fb3ofakZWrdEc7F8NrK2hWKGRlAOdoAyx7ZJ6AVg6v4I1wCQ2Xluq/6pCcRr7k8s5z24Fd6/iyBT1OT0zirdtrsVyQQoH15P61MoYWekWUpYiOrR4jN4RvLWHydTnEhUjcFGNxPOSc9KqRhVkcOAEhUcZGBgbjnHHGRxX0JKthcp/pWHJ4wB2rAl0DSbkNHDGFUggjGcA9a554JL4WbQxT+0jxW1xHCgky7Fywz/ECev5VZlSM3L2znEgODgjqqgEjPrmu81bw2Qmy2cABSucc47fl6Vx82iXE088sCkb2LbyeVyAD16dK5J0nDc6I1FLUwNRDhGlU+Y6jeFHXvnb69OR7e9eRfEPTL24toNX0xC7j/AFkQIOR1yAepx6c17e2mXLq8dz8qE5Djs3UEZ9/0rjvF+nvd6JLFZQEycNHngsQckexz0rKLd7oqSVjz7wdcNf2+Y2yMZAJzj25r0uySeGXDLhX4/P8A+vXE+ErN7Z1ab/XPyVIGRnn5sd8/j617Db2nmxZcZ9a73HmV0cvNysekalFY/dyAR1wG/wAKnTTlGQw8xcg5A9uD9Oa07ezCgbujZB9eO9b9npUkaCUEg4xx3Fc0sO5PYtV7I56DTAi72jyg564P+cfnWtb6bFcA8Bh78EfiK7uzsIgRsXAPr1rYaOKGMhVHFdlHAdWznqYq58teLfCLWeoC9tElR+zA7xntjGK9H8BeNPE3hZzbsxlRSN4PIAIJxknrivQJdOguJlaQYwc8U5tA014fJCYBbJPcnjP54xURwNSFV1KMrEyrxlHlmrnpWm/FuzneGK6h2hiA7dME4HA5J616np2q2WrQmezYsgOMkEV8oTeG5Ul8+ybYR6/pj0re8PazeeH7qNbmVtmegJwc9eK9Wjj60HautO5wVMNBq9M+o6KzNL1ODU7VZ4WzkVp17UZJq6PPatowooopiCiiigD/1v1Srxz9oLj4L+KiBkixc4+hBr2OvL/jVaLf/CnxLaMM+ZYyjHA5xx1rOsr05LyZpRdqkX5n4fW/lvdLLL8yr27ggZP1Hep7e4WJ0k++A3U9MZzj6dawZfPW9ltYx843A/8AAsD+ldHaxJCEjkAIjHz55ycZx+Zr42pHl0Z9dF3JFfz5HhhGFmBxj+FwN6n9BWojefaNCNskUpyQQdyE8gkDPGc8de4qhZ2Qe7FypLLHkAD0xW6lmj+VDEm1Ao3Y7+hz7Vi2tC0SadPqMUgtJJQyx/wBjkbufXkd+n4V22mCAkiSM3Ibgo4BAI9GIzx61hraQXMsHk5WZMqWH4A59a2BEkcmyMk7BknHUDgcjnJzWFWz1RcXbRlhtD0K/JTULUiM4OxJBkn1BOCPyrnZ/AGjeZ52l6jLCWHCSx7h6jDLj+VdfAkMgLs3mIFB5IJDdx0zUrFJCHiBOe5JA7Vkqko7S/r5lcqfQ5G18Gazpd3Hdx3UE8W8OVVip+XB6MPr3qWWwu7HVRI8OLfngNuJDFt3T/ezXfW9iZl3zTHccfcJ/XrWkdP06Ebxh5TxluT6cU3Xn9odo9DyeHSdb8tVaB23EEEj+FenA6V02m6Bd3eoJJEjMseN7DIy2OEBIHy55bua7FruNX8yQbm6Y+n9KedanCBY3Ck8A9Mf59qTrtiUexBcaLKqrbuCqkY2qwRQc9vTPcmpbexjRIracgQxsT5a/KjEdPcgc/jVEyfaD5c8o685yRXRWNva5Qt8zZySf8Kwu3sabbnQaZYqkKQxEIzAZYLwB6DP5V2ELW0Uaw26F1U9CcjPUliepNcNJqHkE7NpwMcNk/jzTv7VncEOfoq9fzrZTsjJxvqdlc3sa53EMzdk6+wrLkv1gH3PLLAcdT7+/wCPFcjLrEyt5UeUPY5/wrImkklBxOArehLN9KmVRvYqMF1N6810SziBGL7cZwPu+1RRJd3Tgs2xRxwMEfnWZCFUbly+3uy8/hWlHtX19ueTWDTb1NVJLY6i3MNquAwPbAI/VqdNqT/xFWx0VTx+J71ySXcAO1gMD8cmrS38Eg8uOFmZhj0H51abWxD13NiWa6u1DMQF7Y4x9arpbzZx8oHODjqfw5rMIULhgR7E8D6D1q4sicKXKj39u2KVrjv2NWGMKB8vTj5jgA/QVYWO3iHmTMu7/PassOF+dn2r2z1/AU19RtoiRbx4duNz9fwFV6CNQzRb1bGMdA3r9BVSaUuhZnwPZSf04zVSO6IBctuYdh/XGKryXdxdsVMhKjsSF/QVEikicTRoCqxsxzyWJBP4DpVhYc/vpESIn2x+p5rKlaS3BcThAPTB4/U1mnUo92ZZDMc8E9PyrNRbKudMRHIco+BnqoPb61rRzY+RTwefTmuNi1GSYgIMk/7PC+/Na9vjcGZ/mPGTWqViZHWwSDfkjfj8qu795ywXHYDnH9K5+3uY412liR7nithbyJQGJA710RMJJ3NSINkFVUe5GavMHcgTMX9c/KKwlv4ACzMTjnIzmpP7UjXBjJx/nrzWimkZ8rZuc4IjTBxg46VQl39T87f7RrMn1CaZQMkp9MVX+2vs2qAAevGTUTqI0jBmv8ztiUggdhWlb7jINik+3QVz8M2Puvitm3llTBGR74p0pK9yaiZ1kMT4ElywQdQBTZbyJAY43z2wOlYnnM6hnfOR3OP5VA0oiyHIVR/dGc/jXofWbKyOP2OupqNcN/rJnUAeo/pWJfatbEeWqbyP73SoJZIuGKs/sScfpWXLP2ZAg6cdf0rmqYh2NoUVco3N09xIHmJ29Aq/KPz4rMuWuLpCkZ27uMjqR+PNX55ogMGJn7DIP8qqmSeRsEGNe+BjP9a43Wex0qmjMs/D8UMomkcK3HCjHPue5rt7GGKE5B4HU9Kz7aNEUGNcY7t1/CrouEAwRlfbmuqlWkupz1Ka6HTWos5G3cKRzjsK3I5FTJWQdOB6ivPGuyn3F2g+poGsSRLsTDdhurtp4xLRnLPD3PVYb2MDAOW9+gq7HJHIclgTXi8OszRuFkfZnoByTXoGj6q8kfKkZ9RjNdtDFqbsYVcM46nSSxjdlTzU1uw6MaqyXJY7UGTjJqSF8oG6HvxxXVfXQ5nF2L7EYOKxdSiW4iKggHGcj1rSdg6lT0PXPFcbqOqraylFk46gdf1oqTVtRRi+h6F8L9anjv30qc5x0PPSvoavl34ZW8t34me8j+4ByexzX1EK9DAfwjhxNufQKKKK7TnCiiigD//X/VKuQ8f6Z/bPgvV9LyV+0W0i5HXpmuvqG4hW4geFxlXBBH1pSV00OLs7n4A6lpwsvEN9A4xIJSvPXjPJHXvVqCZZP3MSYXGGx3yDyfYYr3H9pzwNJ4G8fPeWybLbUMshAwAW4NeGW9s9pAtx91iMqOvOeBz6ivj8dTcarTPrMLUUqakjZ0nTpAxkAyvJZugBAzgew5rYHlvEwUjywkZcng4PJHsSOB9aoreqBNAgPlsTgE43LjYpP86rtfItvIjrhskBTwSwwV/HA6VyWbZvfQ6OG4TTwbfjdJtdTjhWBzz9ScGpbeeZ53VGG4/KRkY69B9a5qFQtv5spLMV+XPTgir5litysiAhjt69ee34VnKOpSZ0LMkUxtZQzdNuOmDgg4610cVuBB8xLKoJ+XjBrmXn8yHzIhhhjPfp/wDrq8t8YAGlIjfoR1BFQ43RV3c6aO+8tNsYHAHf171VuLxZQ23IbHXoM/5Fc5JJaeZINgBZQcgHngHp6Y9OlVVuITMPsxbAPOG4/X0qOW4zVa7lUjfISBg9OxqaO4jeTzcbT27k1T81Ng4UZ7E5+nT1pSkojWQgBccruHT881PIUpG9HfAv8iKoXuSTmpVuJT8rSAhv7prmG+0OoSKNRk565yP8KmiWQRn7VlVXqVPH0470vZhzHQx6nDbEnkdvUVeTVJZFV49wQewwf/11yi3ljHhVXhh/FjJFLJqSqmI3VcdB2/Ok4Bc64XoZ8yKQSTnIH6VIt3bQjIP3fTA/+vXCJqTNyVJ9cEf061YGoMjfMDk8cYx+gzS9kx3R38N9CQCZME9Tgk/nUjanbRAbQGHr3rikuJHwJHAJ5/8A15NS/b1hTfwcd6OUR2a6nG4yi49SQBikF3CxwhJyeey5/ma4OTUVY7mcAEZGCP8AGlS83/efjHqM80nF7jPRPMjAzv59sf8A16WG7tlP7xgceuTXCR3RZf3Qb645/WpV2kBpGOO45FS7oDuzqNqcsWC546c1Vl1G2hDFFyT0PGa4xrqONti9O1PSZCPl+VuvJGam9xnSvqyldhGB3xms6410QKVjXGfQdKz3fgbiGqi6zyHjhT69P0qlFMLkr65JM7eWjOT9MUxL2/3F1Qfj2/OodgQ4Y/MPSpY3QY3NjPbg0+XsPmLi6hdqCFySeflx/wDrq3DqV3ECX+UD1z3qAW/AZhuB6YIBpj28VwAIXBkB6NhsD6ip0KOis/EAjX55GP8Ad2LnP410MOvKyhihbjqxAFcFbRzwyhJpYzjtgqRWoJNp4dsY5xhhSvqJo7FNTVmJfYR6A/8A1qurrCkeWsDMO3GP5DNcdFJKy/6MRx0IVc8/72BU0l1eRcs0iknuVQZHrxg/nRcVjrptTm+UGPA6/e5/KrC30zfcjCAj+IiuPN1qBUtFIip33gH9RTre8Df8fLRPj+6x+lQyrHcR3B7kJtrVgutxUPnB6NXnkWr2aS7bhmxnjcrEfz/lWxDqFpMP9HZsdwoxn86uLa1IkrneC5k2hI2xg/Tj61SknmQ5YEA8jvWAl/BkRsyoexk5/rUxvHWM4kVsHtjj861U2zHlsWzPMQww34n+lZk7XjtySqnsWx+maZJeyAAiSPB/vc/qKozaiYifLn2n0HNS9S0TSRXGcMWcjryB/I1ArPFjcEUn/aOayJ9RkUbmkaQZ5UDms06lAmCylD7kfyqNizv4bg52uQTWgLlCu0/L+GR/OvObfXF3EBRjOck10Ca5A0eFaJT7kg1pGbInFHQyMjNtLY49OazpWAPy7vr2rObU1wFUJIT3XJ5/Gq5vJpSRJiMLk5Jxn0Ap8zZnYeshSUskjBie3+Sa2rLWZ7Z8yM2c8ev65rAaQsudzAew4/oKbvA+XnH+fQYq4VGtglFPc9as/ElvjPlszE9WfJ/GurtdajLkS/LkDvxzXz7BqHkTB0/x4rqNOuNS1Fxb2sbZkPPHbHAr1sNipy0POxFKMdT03UfEtrCTAP3jP8uBjj61zuleH9R8S3iiKErEWwWxzzXp/gv4UO6C81TguQ3PX8q9/wBN0XT9KiEVpEFx7V7FHBTqe9U0R5VTEqOkDE8I+F7fw9YLEq/P3Peuxoor2IxUVZHA227sKKKKoQUUUUAf/9D9UqKKKAPnn9oj4Uw/EjwkWt0zfWJ8yL1bHVfxr8mfFVje6JEbG8iZJ4iTgg5O1gv5cgV+9bAMpU96+Zviv8APD/j2GSaNBbXjDAdR755/GvLzDA+1anHdHpYHGezThLY/JqCYSWKo3zTI5Vj6gjgfgazDetJqCWrnd5hYpn+HCZBz+le1fEL4E+LPhvGXmjNzaqeJUGc5Bzmvne5tb4TRXKIwMZGTj2wa8T2Eoyakj2Y1YyinFndQ322KJD/BuBHXryPzxTGuLiRhbt8zOQ6joc+lc3pmpKE/0kFXQ4II64/zmrEVys80ckb5aMlg305IP9K53Bpu6Nk9NDsrfU4oZWKEKdpYAZPAPTHr7VrRahbupud2Y2IBZSCFz1yOo59RXmsupGNwMbQQuJPQj1/HvVoX4RpJG3bm5zjG4H1I61Dplcx3+54iAx3AsCCxIyPY017v7OC5KsoPJ749+K89fXWiKkDaoPKqSB+PX9K0INfNxlh5ZUfwhiGx9TU+yktWiudHVf2las48qUKT74B/Gte2myMRzEP1ySCvv24riIr63Zy0bKsigBlkOAR9eKb/AGtFHmGF1icHgbgQefUVEoN6IpSXU7WUynEhl74ABP8A7LU0eo2JRUuA+TwME446mvOl8RxxTNs+fP3ieuavf8JBp6hsAoXGQEUde+eafs5W1QcyPQBc2wLISVA5yeAR6ZbOaadRsY8fZ3B56bg3Hvj+Veet4p3bfP8AMeLjIJ5H0yTSW/iGzdtq26uOTlmIP5YIpOi+qFzpHof9rSA5MZfkngHgfhUT6tJCChhYAdCwOOffiuIutTec4hLc+j8D8gKgiuyx2zXGW6D5iRSVKw+c9FjmaUBt+09gDgAe+etWfnXAllOScfKw/lXn0WoR28nliVdx7gn+taX9oJtCySFj2OeB+FTKmxqR1StaqxKFsdATg9OtPaexOTuxtPb/AOt0rlXuFlVkTDE9BnrUAE2Vbyh0weDx9fSjl8xnapqD5Bi+YDv0/LPWpDdSTLtCdfVsGubs5TjMjjI471ea/YfK649Mc1MoAmaQWbhpdpK9+OB+FT+dOPmRxtPXoOn05rHa+i7ptI7H+dVzqUSy7GXgdSTxmlyeQXOnS6hT5SM5Hfn+dRTakqrtBA9Oa546jbsTFCp3MBznIGaoy3F1uIVPlwME9PzoVNBc6Zb63cYDfMcA5IAFSC+to3G9iM9cc4/IVgxXgMYLOmO6HOT9DiqrXwk3LHblj65/wFPkC51s98GTdG7H0HP+FRxX1xHh88n14P4HFc4t/LCuHmEeR0bqAPoahfWE3ZFyoJ4KhSf1o9n5Dudn/beonBLgYOPm6/1FSnWJh805Td6jGfxxXHQ6orPsYsxzjdWit3agExn5/TOefyqPZLsPmOmXXkBPJJ6DDZp8ut3ZAUlhkj+70/Ec1zpnkmwOwHHHf1qXzJIRtRGwR1PPP5CmqC7Cc/M1I9buoVMkJA2/3cH/AL6UH9RUkOuXskmFSN1b0wck+hIFcxJvkOZTGGx/HHz14w4pht3ucg7o89dihl/kDVfVm+hPt0up6BHfqoG9fLbGSudp49MjFaVlrthGAfOYHvgq2Pwry7yxb/u3LMB35x+I5qazaWR1jtI5N3YKN/P8/wA6f1GT2RLxUV1PZI/EUAO77a21uoKD8+DUp8T2A5EzSf7WzI/ka5XSvBvjzVF82y0maRGGQ5h2gn6k13OnfCD4l3i4a3MB9WwP5dvrVxy6p2/r7jCWNprdmTJ4hsZEMIbzR1yu0Y/A4rOkvPtGBZY3H1cKRj0r06P9njx+xBkmjIPrj8jxWsf2d/GyDe0ylem09PqK2WVVukTJ5hS7ngMguA2XkZGPZmyP0rJkYQvv8yMA8nLMT/8ArzXr+pfs8/EVZG8kRyIxzggdPpjBxXAX3wJ+Kcc5SLTiRnIZTjp70/7NrdUWsfSf2jn49StY5Pn4IPVGx/OurtdZteiStk9ScHFSx/Az4o3EqLDp+MgYD+/XkV7l4U/Zu8QPGr66UiY8sE5qo5VVl9kynj6a+0ePLd+fzHK2T6qDVi3t9Qdyke2XIz93JODX2FpnwH0q2UC5O84xXY6X8JtCsJA5UEDtiuuGR1Hucsszj0Pie20LXLqQi3tnOeflXH867HS/hZ4k1Jl86GRPqCK+6LPQdLsVCwQKMe1aqxxr91QK7qeQ0lrNnPPNqj+FHy1oHwRkQA3vy+tez6D8P9J0bayxgsOenevQaK9OjgqVL4UcNXE1KnxMRVCgKowBS0UV1nOFFFFABRRRQAUUUUAf/9H9UqKKKADg0wpn3oIpvzDpQBkar4e0vWrZ7PUYVmifqrDIry68+Avw5uIXh/s2Nd/cCvaPMI608MGqZQi90XGclsz8/wDxj+xZYanetNot2YImOdp7fSvFNe/Y58aaD582iSC7UrwOhz61+tuKjKVzywdJq1jojjaq1ufgb4g+Hvjrw3dtDeafLHtznKEj+VcM5urYtDfRtGDx3ytf0N3ejaXff8flrHLn+8oNeaeIPgV8MvEhZtQ0iIM3UoNtcs8rh9lnVHM39pH4QvdeTIyqdwHI3D+tU3vJGIKKAtfs9c/sffCOdy32Z1B7A9K5PUf2H/hrcvusppoAeozms1lzRr/aUD8ivtby/I+5x79q0YU81PLjBTnsOtfrRb/sSfDmJQr3EzevNb1v+x78OLZQkbSDHc8mn/Z7sH9oQPx9XTXceWqn8Rmta20O9YgQKxPptzX68W/7JHw6iffNJPIPTgV19j+z14E00AWlvnH98Amj+z5Pdk/2jHofj5D4e1GXbuteFAGSOv8AKmy+G5Oph2HPPHFftAfgv4ObBaziDAYyFAzVab4HeC5wQ9qv5VKyrXcf9prsfjbF4cuQCqoTnBPbn8qVvDl87hljIx6f/qr9hW+AHgY9LfGfSq8n7Png1j8qbah5U+jGszXY/ISfw7fMf9W3H1/woi0e+i5EbFvfp+NfruP2evBw6x5pp/Z68HdoRS/sp2sx/wBpo/Ia5tdQziCEqW5bv+RqktzqNtmNkbjGNwyB785r9err9nrww6lYYgPwrj7/APZg0e5B8squfaollTtoi45nHqfmPDeXdwC0uAx5yoH+TWtbi6kbLy8dgy5BP5ivvGb9kaFixjuAAfQVAP2Tnh4ScEVxzyytsonRHMaXVnxBKbhV3SDn24rlby/u0lO2F9g6hiMfgetfoev7LNx0eYEVK37Ktuy4lkBJ9qUMrrdYjlmVLoz8zpNQuXbZAjBE/PP14qzBrd+oXaMMp4LqOM/Wv0kX9k3TmPzMAK04P2SdDXhm49MV0/2bO1uUy/tGHc/M8yavMm8DoeNi4xSm71/zQ8wkHTlSVyB9MV+pUH7KnhmPAJOB2rbh/Zn8IwjBjz9aSyur2QnmVPuflVHqGoPhWWZzg8k9j9akt7C8mG8QM3OSST+FfrFD+zt4Lh5EA9627T4K+DrPG22Q49s01lE+rsS80h0R+V+n+Etfv2UWVg5z15NezeHfgx4wv1UpamLOM5NfpDpngXw9aYMUCjHTiuzt9Ms7dQIowMe1dVPKIfaZz1M0n9lHwhpX7OeuSov2namRzzXd2H7M1vsC3cufWvsIIo6CnV2wy+jHock8dVl1PlyD9mHwtjFw5I9hW/Zfs2fDy25lhaT8cV9CUV0KhTW0TB1pvdniEf7Pnw3jO4Wjk+7mu40r4deENGAFhp8SEd9oz+ddvRVeyh2Jc5dyrHZW0ShI4woHYCpfKjHapaTAqrIm435R0FOAFGBRimIQqh6gUbE9BUbA9qAGzSGSbFHQU6kHSlpiCiiigAooooAKKKKACiiigAooooAKKKKACiiigD//0v1SooooAKTFLRQAwqDTdvcVLRigCMFhT91LgUYFABkUdaTFGKAA7u1MLMO1P5paAGKxPan0UUAFFFFABRRRQAUUUUAFFFFABRRRQA3mkwafRQMZg1GYsnmp6KLBcgEIFOCYqWilYLkew00x5qaiiwXK/kg0eQvpViiiyC5EsSjoKlAxRRTEFGaKMUAJmlzSYNGDQMWikwaUUAFFFFAgooooATFGKWigdwooooEFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAH//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA//Z"}]} diff --git a/test/apis/realtime/sleep/cortex_cpu.yaml b/test/apis/realtime/sleep/cortex_cpu.yaml index ee64740962..150168bd5e 100644 --- a/test/apis/realtime/sleep/cortex_cpu.yaml +++ b/test/apis/realtime/sleep/cortex_cpu.yaml @@ -14,3 +14,5 @@ mem: 128M autoscaling: max_concurrency: 1 + target_in_flight: 1 + max_queue_length: 128 diff --git a/test/apis/task/iris-classifier-trainer/main.py b/test/apis/task/iris-classifier-trainer/main.py index 440bd1c61e..99ce8ec1d8 100644 --- a/test/apis/task/iris-classifier-trainer/main.py +++ b/test/apis/task/iris-classifier-trainer/main.py @@ -14,7 +14,7 @@ def main(): config = job_spec["config"] job_id = job_spec["job_id"] s3_path = None - if "dest_s3_dir" in config: + if config is not None and "dest_s3_dir" in config: s3_path = config["dest_s3_dir"] # Train the model diff --git a/test/e2e/e2e/tests.py b/test/e2e/e2e/tests.py index 28b2c45f26..b1df5c95e2 100644 --- a/test/e2e/e2e/tests.py +++ b/test/e2e/e2e/tests.py @@ -546,6 +546,18 @@ def test_load_realtime( client=client, api_names=[api_name], timeout=deploy_timeout ), f"api {api_name} not ready" + network_stats = client.get_api(api_name)["metrics"]["network_stats"] + offset_2xx = network_stats["code_2xx"] + offset_4xx = network_stats["code_4xx"] + offset_5xx = network_stats["code_4xx"] + + if offset_2xx is None: + offset_2xx = 0 + if offset_4xx is None: + offset_4xx = 0 + if offset_5xx is None: + offset_5xx = 0 + # give the APIs some time to prevent getting high latency spikes in the beginning time.sleep(5) @@ -583,14 +595,14 @@ def test_load_realtime( network_stats = api_info["metrics"]["network_stats"] assert ( - network_stats["code_4xx"] == 0 - ), f"detected 4xx response codes ({network_stats['code_4xx']}) in cortex get" + network_stats["code_4xx"] - offset_4xx == 0 + ), f"detected 4xx response codes ({network_stats['code_4xx'] - offset_4xx}) in cortex get" assert ( network_stats["code_5xx"] == 0 - ), f"detected 5xx response codes ({network_stats['code_5xx']}) in cortex get" + ), f"detected 5xx response codes ({network_stats['code_5xx'] - offset_5xx}) in cortex get" printer( - f"min RTT: {current_min_rtt} | max RTT: {current_max_rtt} | avg RTT: {current_avg_rtt} | requests: {network_stats['code_2xx']} (out of {total_requests})" + f"min RTT: {current_min_rtt} | max RTT: {current_max_rtt} | avg RTT: {current_avg_rtt} | requests: {network_stats['code_2xx']-offset_2xx} (out of {total_requests-offset})" ) # check if the requesting threads are still healthy @@ -600,9 +612,11 @@ def test_load_realtime( # don't stress the CPU too hard time.sleep(1) - printer("verifying number of processed requests using the client") + printer( + f"verifying number of processed requests ({total_requests}, with an offset of {offset_2xx}) using the client" + ) assert api_requests( - client, api_name, total_requests, timeout=status_code_timeout + client, api_name, total_requests + offset_2xx, timeout=status_code_timeout ), f"the number of 2xx response codes for api {api_name} doesn't match the expected number {total_requests}" except: From 54ef26bc2507f27eb69cf5bf406d09b58b90750f Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 28 May 2021 11:00:33 -0700 Subject: [PATCH 35/82] Update test status code offsets --- test/e2e/e2e/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/e2e/tests.py b/test/e2e/e2e/tests.py index b1df5c95e2..36d26e4f8a 100644 --- a/test/e2e/e2e/tests.py +++ b/test/e2e/e2e/tests.py @@ -549,7 +549,7 @@ def test_load_realtime( network_stats = client.get_api(api_name)["metrics"]["network_stats"] offset_2xx = network_stats["code_2xx"] offset_4xx = network_stats["code_4xx"] - offset_5xx = network_stats["code_4xx"] + offset_5xx = network_stats["code_5xx"] if offset_2xx is None: offset_2xx = 0 @@ -598,7 +598,7 @@ def test_load_realtime( network_stats["code_4xx"] - offset_4xx == 0 ), f"detected 4xx response codes ({network_stats['code_4xx'] - offset_4xx}) in cortex get" assert ( - network_stats["code_5xx"] == 0 + network_stats["code_5xx"] - offset_5xx == 0 ), f"detected 5xx response codes ({network_stats['code_5xx'] - offset_5xx}) in cortex get" printer( From 42747e9aa161fb9e45009351308258e3a28d5a85 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Fri, 28 May 2021 21:08:37 +0300 Subject: [PATCH 36/82] Remove offset value from print --- test/e2e/e2e/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/e2e/tests.py b/test/e2e/e2e/tests.py index 36d26e4f8a..0f560de23c 100644 --- a/test/e2e/e2e/tests.py +++ b/test/e2e/e2e/tests.py @@ -602,7 +602,7 @@ def test_load_realtime( ), f"detected 5xx response codes ({network_stats['code_5xx'] - offset_5xx}) in cortex get" printer( - f"min RTT: {current_min_rtt} | max RTT: {current_max_rtt} | avg RTT: {current_avg_rtt} | requests: {network_stats['code_2xx']-offset_2xx} (out of {total_requests-offset})" + f"min RTT: {current_min_rtt} | max RTT: {current_max_rtt} | avg RTT: {current_avg_rtt} | requests: {network_stats['code_2xx']-offset_2xx} (out of {total_requests})" ) # check if the requesting threads are still healthy From 66038e26023f9a75e80844c81134adc1a36c0cfa Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 28 May 2021 11:24:32 -0700 Subject: [PATCH 37/82] Update versions.md --- dev/versions.md | 82 ------------------------------------------------- 1 file changed, 82 deletions(-) diff --git a/dev/versions.md b/dev/versions.md index 4ccb0cdba2..e57c6465a0 100644 --- a/dev/versions.md +++ b/dev/versions.md @@ -108,38 +108,6 @@ see https://github.com/moby/moby/issues/39302#issuecomment-639687466_ 1. `go mod tidy` 1. Check that the diff in `go.mod` is reasonable -### request-monitor - -1. `cd request-monitor/` -1. `rm -rf go.mod go.sum && go mod init && go clean -modcache` -1. `go mod tidy` -1. Check that the diff in `go.mod` is reasonable - -## Python - -The same Python version should be used throughout Cortex (e.g. search for `3.6` and update all accordingly). - -It's probably safest to use the minor version of Python that you get when you -run `apt-get install python3` ([currently that's what TensorFlow's Docker image does](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile)) -, or what you get by default in Google CoLab. In theory, it should be safe to use the lowest of the maximum supported -python versions in our pip dependencies (e.g. [tensorflow](https://pypi.org/project/tensorflow) -, [Keras](https://pypi.org/project/Keras), -, [pandas](https://pypi.org/project/pandas), [scikit-learn](https://pypi.org/project/scikit-learn) -, [scipy](https://pypi.org/project/scipy), [torch](https://pypi.org/project/torch) -, [xgboost](https://pypi.org/project/xgboost)) - -## TensorFlow / TensorFlow Serving - -1. Find the latest release on [GitHub](https://github.com/tensorflow/tensorflow/releases) -1. Search the codebase for the current minor TensorFlow version (e.g. `2.3`) and update versions as appropriate -1. Update the version for libnvinfer in `images/tensorflow-serving-gpu/Dockerfile` dockerfile as appropriate (https://www.tensorflow.org/install/gpu) - -Note: it's ok if example training notebooks aren't upgraded, as long as the exported model still works - -## CUDA/cuDNN - -1. Search the codebase for the previous CUDA version and `cudnn`. It might be nice to use the version of CUDA which does not require a special pip command when installing pytorch. - ## Nvidia device plugin 1. Update the version in `images/nvidia/Dockerfile` ([releases](https://github.com/NVIDIA/k8s-device-plugin/releases) @@ -167,52 +135,6 @@ Note: it's ok if example training notebooks aren't upgraded, as long as the expo 1. Update the links at the top of the file to the URL you copied from 1. Check that your diff is reasonable (and put back any of our modifications) -## Neuron - -1. `docker run --rm -it amazonlinux:2` -1. Run the `echo $'[neuron] ...' > /etc/yum.repos.d/neuron.repo` command - from [Dockerfile.neuron-rtd](https://github.com/aws/aws-neuron-sdk/blob/master/docs/neuron-container-tools/docker-example/Dockerfile.neuron-rtd) (it needs to be updated to work properly with the new lines) - * e.g. `echo $'[neuron] \nname=Neuron YUM Repository \nbaseurl=https://yum.repos.neuron.amazonaws.com \nenabled=1' > /etc/yum.repos.d/neuron.repo` -1. Run `yum info aws-neuron-tools`, `yum info aws-neuron-runtime`, and `yum info procps-ng` to check the versions - that were installed, and use those versions in `images/neuron-rtd/Dockerfile` -1. Check if there are any updates - to [Dockerfile.neuron-rtd](https://github.com/aws/aws-neuron-sdk/blob/master/docs/neuron-container-tools/docker-example/Dockerfile.neuron-rtd) - which should be brought in to `images/neuron-rtd/Dockerfile` -1. Set the version of `aws-neuron-tools` and `aws-neuron-runtime` in `images/python-handler-inf/Dockerfile` - and `images/tensorflow-serving-inf/Dockerfile` -1. Run `docker run --rm -it ubuntu:18.04` -1. Run the first `RUN` command used in `images/tensorflow-serving-inf/Dockerfile`, having omitted the version specified - for `tensorflow-model-server-neuron` and the cleanup line at the end -1. Run `apt-cache policy tensorflow-model-server-neuron` to find the version that was installed, and update it - in `images/tensorflow-serving-inf/Dockerfile` -1. Check if there are any updates - to [Dockerfile.tf-serving](https://github.com/aws/aws-neuron-sdk/blob/master/docs/neuron-container-tools/docker-example/Dockerfile.tf-serving) - which should be brought in to `images/tensorflow-serving-inf/Dockerfile` -1. Take a deep breath, cross your fingers, rebuild all images, and confirm that the Inferentia examples work. You may need to change the versions of `neuron-cc`, `tensorflow-neuron`, and/or `torch-neuron` in `requirements.txt` files: - 1. Run `docker run --rm -it ubuntu:18.04` - 1. Run `apt-get update && apt-get install -y curl python3.6 python3.6-distutils` (change the python version if necessary) - 1. Run `curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3.6 get-pip.py && pip install --upgrade pip` (change the python version if necessary) - 1. Run `pip install --extra-index-url https://pip.repos.neuron.amazonaws.com neuron-cc tensorflow-neuron torch-neuron` - 1. Run `pip list` to show the versions of all installed dependencies - -## Python packages - -1. Update versions in `python/serve/*requirements.txt` - -## S6-overlay supervisor - -1. Locate the `s6-overlay` installation in `images/python-handler-*/Dockerfile` and `images/tensorflow-handler/Dockerfile`. -1. Update the version in each serving image with the newer one in https://github.com/just-containers/s6-overlay. - -## Nginx - -1. Run a base image of ubuntu that matches the version tag used for the serving images. The running command - is `docker run -it --rm ` -1. Run `apt update && apt-cache policy nginx`. Notice the latest minor version of nginx (e.g. `1.14`) -1. Locate the `nginx` package in `images/python-handler-*/Dockerfile` and `images/tensorflow-handler/Dockerfile`. -1. Update the version for all `nginx` appearances using the minor version from step 2 and add an asterisk at the end to - denote any version (e.g. `1.14.*`) - ## Istio 1. Find the latest [release](https://istio.io/latest/news/releases) and check the release notes (here are @@ -337,10 +259,6 @@ supported () 1. refresh shell 1. `kubectl version` -## Ubuntu base images - -1. Search the codebase for `ubuntu` and update versions as appropriate - ## Alpine base images 1. Find the latest release on [Dockerhub](https://hub.docker.com/_/alpine) From e027242fa2b0eaaffeb331e0c93a99ba6a349806 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 28 May 2021 11:31:12 -0700 Subject: [PATCH 38/82] Update CONTRIBUTING.md --- CONTRIBUTING.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b6eed659d8..f7f73e4af3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -169,7 +169,7 @@ node_groups: Add this to your bash profile (e.g. `~/.bash_profile`, `~/.profile` or `~/.bashrc`), replacing the placeholders accordingly: ```bash -# set the default image for APIs +# set the default image registry export CORTEX_DEV_DEFAULT_IMAGE_REGISTRY=".dkr.ecr..amazonaws.com/cortexlabs" # redirect analytics and error reporting to our dev environment @@ -209,7 +209,7 @@ Here is the typical full dev workflow which covers most cases: 1. `make cluster-up` (creates a cluster using `dev/config/cluster.yaml`) 2. `make devstart` (deletes the in-cluster operator, builds the CLI, and starts the operator locally; file changes will trigger the CLI and operator to re-build) 3. Make your changes -4. `make images-dev` (only necessary if API images or the manager are modified) +4. `make images-dev` (only necessary if changes were made outside of the operator and CLI) 5. Test your changes e.g. via `cortex deploy` (and repeat steps 3 and 4 as necessary) 6. `make cluster-down` (deletes your cluster) @@ -224,6 +224,4 @@ If you are only modifying the CLI, `make cli-watch` will build the CLI and re-bu If you are only modifying the operator, `make operator-local` will build and start the operator locally, and build/restart it when files are changed. -If you are modifying code in the API images (i.e. any of the Python serving code), `make images-dev` may build more images than you need during testing. For example, if you are only testing using the `python-handler-cpu` image, you can run `./dev/registry.sh update-single python-handler-cpu`. - See `Makefile` for additional dev commands. From fd550015599995b1905dbe096d8734322bef6d72 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 28 May 2021 11:32:23 -0700 Subject: [PATCH 39/82] Remove dev/load --- dev/load/cortex.yaml | 30 --------------------- dev/load/handler.py | 62 -------------------------------------------- 2 files changed, 92 deletions(-) delete mode 100644 dev/load/cortex.yaml delete mode 100644 dev/load/handler.py diff --git a/dev/load/cortex.yaml b/dev/load/cortex.yaml deleted file mode 100644 index 66f0bc87b7..0000000000 --- a/dev/load/cortex.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: load - kind: RealtimeAPI - handler: - type: python - path: handler.py - log_level: debug - config: - endpoint: http://a76d3b3343f1a43e8a5dd1f556689215-14bf49c6f95638fb.elb.us-west-2.amazonaws.com/iris-classifier - data: { "sepal_length": 2.2, "sepal_width": 3.6, "petal_length": 1.4, "petal_width": 3.3 } - sleep: 0.8 - num_requests: 1000 - autoscaling: - min_replicas: 50 - max_replicas: 50 - compute: - cpu: 1.3 diff --git a/dev/load/handler.py b/dev/load/handler.py deleted file mode 100644 index 43257dd80f..0000000000 --- a/dev/load/handler.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import requests -import time -from cortex_internal.lib.log import logger as cortex_logger - - -class Handler: - def __init__(self, config): - num_success = 0 - num_fail = 0 - for i in range(config["num_requests"]): - if i > 0: - time.sleep(config["sleep"]) - try: - # response = requests.get(config["endpoint"]) - response = requests.post(config["endpoint"], json=config["data"]) - except Exception as e: - num_fail += 1 - cortex_logger.error( - e, - extra={ - "error": True, - "request_number": i, - }, - ) - continue - if response.status_code == 200: - num_success += 1 - cortex_logger.info( - "successful request", extra={"request_success": True, "request_number": i} - ) - else: - num_fail += 1 - cortex_logger.error( - response.text, - extra={ - "error": True, - "code": response.status_code, - "request_number": i, - }, - ) - - cortex_logger.warn( - "FINISHED", - extra={"finished": True, "num_success": num_success, "num_fail": num_fail}, - ) - - def handle_post(self, payload): - return "ok" From 992e6d9a451fa2fad4eab7b2216f3543ffb03056 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 28 May 2021 11:46:51 -0700 Subject: [PATCH 40/82] Misc changes --- dev/versions.md | 2 +- docs/workloads/batch/jobs.md | 2 +- docs/workloads/realtime/autoscaling.md | 2 +- manager/manifests/grafana/grafana-dashboard-realtime.yaml | 4 +--- pkg/lib/regex/regex_test.go | 2 +- pkg/types/spec/utils.go | 2 -- 6 files changed, 5 insertions(+), 9 deletions(-) diff --git a/dev/versions.md b/dev/versions.md index e57c6465a0..01173ef192 100644 --- a/dev/versions.md +++ b/dev/versions.md @@ -118,7 +118,7 @@ see https://github.com/moby/moby/issues/39302#issuecomment-639687466_ 1. Update the link at the top of the file to the URL you copied from 1. Check that your diff is reasonable (and put back any of our modifications, e.g. the image path, rolling update strategy, resource requests, tolerations, node selector, priority class, etc) -1. Confirm GPUs work for PyTorch, TensorFlow, and ONNX models +1. Confirm GPUs work ## Inferentia device plugin diff --git a/docs/workloads/batch/jobs.md b/docs/workloads/batch/jobs.md index aa2f34418a..fd0977fa65 100644 --- a/docs/workloads/batch/jobs.md +++ b/docs/workloads/batch/jobs.md @@ -202,7 +202,7 @@ RESPONSE: }, "worker_counts": { # worker counts are only available while a job is running "pending": , # number of workers that are waiting for compute resources to be provisioned - "initializing": , # number of workers that are initializing (downloading images or running your handler's init function) + "initializing": , # number of workers that are initializing "running": , # number of workers that are actively working on batches from the queue "succeeded": , # number of workers that have completed after verifying that the queue is empty "failed": , # number of workers that have failed diff --git a/docs/workloads/realtime/autoscaling.md b/docs/workloads/realtime/autoscaling.md index c3a7cfbaca..089e3581c4 100644 --- a/docs/workloads/realtime/autoscaling.md +++ b/docs/workloads/realtime/autoscaling.md @@ -12,7 +12,7 @@ In addition to the autoscaling configuration options (described below), there ar
-**`max_queue_length`** (default: 100): The maximum number of requests which will be queued by the replica (beyond `max_concurrency`) before requests are rejected with HTTP error code 503. For long-running APIs, decreasing `max_replica_concurrency` and configuring the client to retry when it receives 503 responses will improve queue fairness accross replicas by preventing requests from sitting in long queues. +**`max_queue_length`** (default: 100): The maximum number of requests which will be queued by the replica (beyond `max_concurrency`) before requests are rejected with HTTP error code 503. For long-running APIs, decreasing `max_queue_length` and configuring the client to retry when it receives 503 responses will improve queue fairness accross replicas by preventing requests from sitting in long queues.
diff --git a/manager/manifests/grafana/grafana-dashboard-realtime.yaml b/manager/manifests/grafana/grafana-dashboard-realtime.yaml index 6e1fc360d9..39e1db6421 100644 --- a/manager/manifests/grafana/grafana-dashboard-realtime.yaml +++ b/manager/manifests/grafana/grafana-dashboard-realtime.yaml @@ -964,9 +964,7 @@ data: } }, { - "aliasColors": { - "iris-classifier": "light-green" - }, + "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, diff --git a/pkg/lib/regex/regex_test.go b/pkg/lib/regex/regex_test.go index 382718b5e7..bd5ce79687 100644 --- a/pkg/lib/regex/regex_test.go +++ b/pkg/lib/regex/regex_test.go @@ -596,7 +596,7 @@ func TestValidDockerImage(t *testing.T) { match: false, // Support this as valid? }, { - input: "680880929103.dkr.ecr.eu-central-1.amazonaws.com/cortexlabs/python-handler-cpu:latest", + input: "680880929103.dkr.ecr.eu-central-1.amazonaws.com/cortexlabs/async-gateway:latest", match: true, }, } diff --git a/pkg/types/spec/utils.go b/pkg/types/spec/utils.go index 7f609f956a..e328813f39 100644 --- a/pkg/types/spec/utils.go +++ b/pkg/types/spec/utils.go @@ -24,8 +24,6 @@ import ( "github.com/cortexlabs/cortex/pkg/types/userconfig" ) -type modelValidator func(paths []string, prefix string, versionedPrefix *string) error - func FindDuplicateNames(apis []userconfig.API) []userconfig.API { names := make(map[string][]userconfig.API) From 7508c76525fd096ad48ba6fb020b43abd5c6e101 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 28 May 2021 14:11:45 -0700 Subject: [PATCH 41/82] Remove support for max_concurrency and max_queue_length for Async APIs --- docs/workloads/async/autoscaling.md | 16 ++----- docs/workloads/async/configuration.md | 3 +- pkg/types/spec/validations.go | 64 ++++++++++++++++----------- pkg/types/userconfig/api.go | 12 ++--- 4 files changed, 51 insertions(+), 44 deletions(-) diff --git a/docs/workloads/async/autoscaling.md b/docs/workloads/async/autoscaling.md index c83c537154..3ac46beb1c 100644 --- a/docs/workloads/async/autoscaling.md +++ b/docs/workloads/async/autoscaling.md @@ -4,14 +4,6 @@ Cortex auto-scales AsyncAPIs on a per-API basis based on your configuration. ## Autoscaling replicas -### Relevant pod configuration - -In addition to the autoscaling configuration options (described below), there is one field in the pod configuration which are relevant to replica autoscaling: - -**`max_concurrency`** (default: 1): The maximum number of requests that will be concurrently sent into the container by Cortex. If your web server is designed to handle multiple concurrent requests, increasing `max_concurrency` will increase the throughput of a replica (and result in fewer total replicas for a given load). - -
- ### Autoscaling configuration **`min_replicas`**: The lower bound on how many replicas can be running for an API. @@ -22,13 +14,13 @@ In addition to the autoscaling configuration options (described below), there is
-**`target_in_flight`** (default: `max_concurrency` in the pod configuration): This is the desired number of in-flight requests per replica, and is the metric which the autoscaler uses to make scaling decisions. The number of in-flight requests is simply how many requests have been submitted and are not yet finished being processed. Therefore, this number includes requests which are actively being processed as well as requests which are waiting in the queue. +**`target_in_flight`** (default: 1): This is the desired number of in-flight requests per replica, and is the metric which the autoscaler uses to make scaling decisions. The number of in-flight requests is simply how many requests have been submitted and are not yet finished being processed. Therefore, this number includes requests which are actively being processed as well as requests which are waiting in the queue. The autoscaler uses this formula to determine the number of desired replicas: `desired replicas = total in-flight requests / target_in_flight` -For example, setting `target_in_flight` to `max_concurrency` (the default) causes the cluster to adjust the number of replicas so that on average, there are no requests waiting in the queue. +For example, setting `target_in_flight` to 1 (the default) causes the cluster to adjust the number of replicas so that on average, there are no requests waiting in the queue.
@@ -66,9 +58,9 @@ Cortex spins up and down instances based on the aggregate resource requests of a ## Overprovisioning -The default value for `target_in_flight` is `max_concurrency`, which behaves well in many situations (see above for an explanation of how `target_in_flight` affects autoscaling). However, if your application is sensitive to spikes in traffic or if creating new replicas takes too long (see below), you may find it helpful to maintain extra capacity to handle the increased traffic while new replicas are being created. This can be accomplished by setting `target_in_flight` to a lower value relative to the expected replica's concurrency. The smaller `target_in_flight` is, the more unused capacity your API will have, and the more room it will have to handle sudden increased load. The increased request rate will still trigger the autoscaler, and your API will stabilize again (maintaining the overprovisioned capacity). +The default value for `target_in_flight` is 1, which behaves well in many situations (see above for an explanation of how `target_in_flight` affects autoscaling). However, if your application is sensitive to spikes in traffic or if creating new replicas takes too long (see below), you may find it helpful to maintain extra capacity to handle the increased traffic while new replicas are being created. This can be accomplished by setting `target_in_flight` to a lower value. The smaller `target_in_flight` is, the more unused capacity your API will have, and the more room it will have to handle sudden increased load. The increased request rate will still trigger the autoscaler, and your API will stabilize again (maintaining the overprovisioned capacity). -For example, if you've determined that each replica in your API can handle 2 concurrent requests, you would typically set `target_in_flight` to 2. In a scenario where your API is receiving 8 concurrent requests on average, the autoscaler would maintain 4 live replicas (8/2 = 4). If you wanted to overprovision by 25%, you could set `target_in_flight` to 1.6, causing the autoscaler maintain 5 live replicas (8/1.6 = 5). +For example, if you wanted to overprovision by 25%, you could set `target_in_flight` to 0.8. If your API has an average of 4 concurrent requests, the autoscaler would maintain 5 live replicas (4/0.8 = 5). ## Autoscaling responsiveness diff --git a/docs/workloads/async/configuration.md b/docs/workloads/async/configuration.md index 01641bc7c1..3b975b1413 100644 --- a/docs/workloads/async/configuration.md +++ b/docs/workloads/async/configuration.md @@ -5,7 +5,6 @@ kind: AsyncAPI # must be "AsyncAPI" for async APIs (required) pod: # pod configuration (required) port: # port to which requests will be sent (default: 8080; exported as $CORTEX_PORT) - max_concurrency: # maximum number of requests that will be concurrently sent into the container (default: 1) containers: # configurations for the containers to run (at least one constainer must be provided) - name: # name of the container (required) image: # docker image to use for the container (required) @@ -46,7 +45,7 @@ min_replicas: # minimum number of replicas (default: 1) max_replicas: # maximum number of replicas (default: 100) init_replicas: # initial number of replicas (default: ) - target_in_flight: # desired number of in-flight requests per replica (including requests actively being processed as well as queued), which the autoscaler tries to maintain (default: ) + target_in_flight: # desired number of in-flight requests per replica (including requests actively being processed as well as queued), which the autoscaler tries to maintain (default: 1) window: # duration over which to average the API's in-flight requests per replica (default: 60s) downscale_stabilization_period: # the API will not scale below the highest recommendation made during this period (default: 5m) upscale_stabilization_period: # the API will not scale above the lowest recommendation made during this period (default: 1m) diff --git a/pkg/types/spec/validations.go b/pkg/types/spec/validations.go index e7b12e89ca..706dbe039e 100644 --- a/pkg/types/spec/validations.go +++ b/pkg/types/spec/validations.go @@ -141,7 +141,7 @@ func multiAPIsValidation() *cr.StructFieldValidation { } func podValidation(kind userconfig.Kind) *cr.StructFieldValidation { - return &cr.StructFieldValidation{ + validation := &cr.StructFieldValidation{ StructField: "Pod", StructValidation: &cr.StructValidation{ StructFieldValidations: []*cr.StructFieldValidation{ @@ -180,30 +180,37 @@ func podValidation(kind userconfig.Kind) *cr.StructFieldValidation { }, }, }, - { - StructField: "MaxQueueLength", - Int64Validation: &cr.Int64Validation{ - Default: consts.DefaultMaxQueueLength, - GreaterThan: pointer.Int64(0), - // the proxy can theoretically accept up to 32768 connections, but during testing, - // it has been observed that the number is just slightly lower, so it has been offset by 2678 - LessThanOrEqualTo: pointer.Int64(30000), - }, - }, - { - StructField: "MaxConcurrency", - Int64Validation: &cr.Int64Validation{ - Default: consts.DefaultMaxConcurrency, - GreaterThan: pointer.Int64(0), - // the proxy can theoretically accept up to 32768 connections, but during testing, - // it has been observed that the number is just slightly lower, so it has been offset by 2678 - LessThanOrEqualTo: pointer.Int64(30000), - }, - }, containersValidation(kind), }, }, } + + if kind == userconfig.RealtimeAPIKind { + validation.StructValidation.StructFieldValidations = append(validation.StructValidation.StructFieldValidations, + &cr.StructFieldValidation{ + StructField: "MaxQueueLength", + Int64Validation: &cr.Int64Validation{ + Default: consts.DefaultMaxQueueLength, + GreaterThan: pointer.Int64(0), + // the proxy can theoretically accept up to 32768 connections, but during testing, + // it has been observed that the number is just slightly lower, so it has been offset by 2678 + LessThanOrEqualTo: pointer.Int64(30000), + }, + }, + &cr.StructFieldValidation{ + StructField: "MaxConcurrency", + Int64Validation: &cr.Int64Validation{ + Default: consts.DefaultMaxConcurrency, + GreaterThan: pointer.Int64(0), + // the proxy can theoretically accept up to 32768 connections, but during testing, + // it has been observed that the number is just slightly lower, so it has been offset by 2678 + LessThanOrEqualTo: pointer.Int64(30000), + }, + }, + ) + } + + return validation } func containersValidation(kind userconfig.Kind) *cr.StructFieldValidation { @@ -807,12 +814,19 @@ func validateAutoscaling(api *userconfig.API) error { autoscaling := api.Autoscaling pod := api.Pod - if autoscaling.TargetInFlight == nil { - autoscaling.TargetInFlight = pointer.Float64(float64(pod.MaxConcurrency)) + if api.Kind == userconfig.RealtimeAPIKind { + if autoscaling.TargetInFlight == nil { + autoscaling.TargetInFlight = pointer.Float64(float64(pod.MaxConcurrency)) + } + if *autoscaling.TargetInFlight > float64(pod.MaxConcurrency)+float64(pod.MaxQueueLength) { + return ErrorTargetInFlightLimitReached(*autoscaling.TargetInFlight, pod.MaxConcurrency, pod.MaxQueueLength) + } } - if *autoscaling.TargetInFlight > float64(pod.MaxConcurrency)+float64(pod.MaxQueueLength) { - return ErrorTargetInFlightLimitReached(*autoscaling.TargetInFlight, pod.MaxConcurrency, pod.MaxQueueLength) + if api.Kind == userconfig.AsyncAPIKind { + if autoscaling.TargetInFlight == nil { + autoscaling.TargetInFlight = pointer.Float64(1) + } } if autoscaling.MinReplicas > autoscaling.MaxReplicas { diff --git a/pkg/types/userconfig/api.go b/pkg/types/userconfig/api.go index 0560765797..d4434c4ca7 100644 --- a/pkg/types/userconfig/api.go +++ b/pkg/types/userconfig/api.go @@ -153,7 +153,7 @@ func IdentifyAPI(filePath string, name string, kind Kind, index int) string { func (api *API) ToK8sAnnotations() map[string]string { annotations := map[string]string{} - if api.Pod != nil { + if api.Pod != nil && api.Kind == RealtimeAPIKind { annotations[MaxConcurrencyAnnotationKey] = s.Int64(api.Pod.MaxConcurrency) annotations[MaxQueueLengthAnnotationKey] = s.Int64(api.Pod.MaxQueueLength) } @@ -257,7 +257,7 @@ func (api *API) UserStr() string { if api.Pod != nil { sb.WriteString(fmt.Sprintf("%s:\n", PodKey)) - sb.WriteString(s.Indent(api.Pod.UserStr(), " ")) + sb.WriteString(s.Indent(api.Pod.UserStr(api.Kind), " ")) } if api.Networking != nil { @@ -286,7 +286,7 @@ func (trafficSplit *TrafficSplit) UserStr() string { return sb.String() } -func (pod *Pod) UserStr() string { +func (pod *Pod) UserStr(kind Kind) string { var sb strings.Builder if pod.ShmSize != nil { @@ -301,8 +301,10 @@ func (pod *Pod) UserStr() string { sb.WriteString(fmt.Sprintf("%s: %d\n", PortKey, *pod.Port)) } - sb.WriteString(fmt.Sprintf("%s: %s\n", MaxConcurrencyKey, s.Int64(pod.MaxConcurrency))) - sb.WriteString(fmt.Sprintf("%s: %s\n", MaxQueueLengthKey, s.Int64(pod.MaxQueueLength))) + if kind == RealtimeAPIKind { + sb.WriteString(fmt.Sprintf("%s: %s\n", MaxConcurrencyKey, s.Int64(pod.MaxConcurrency))) + sb.WriteString(fmt.Sprintf("%s: %s\n", MaxQueueLengthKey, s.Int64(pod.MaxQueueLength))) + } sb.WriteString(fmt.Sprintf("%s:\n", ContainersKey)) for _, container := range pod.Containers { From 72578d64492dac1b6de5b471babec5a41941049e Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Sat, 29 May 2021 00:32:41 +0300 Subject: [PATCH 42/82] Move the max_concurrency/max_queue_length fields for the test APIs (#2203) --- test/apis/async/text-generator/cortex_cpu.yaml | 3 +-- test/apis/async/text-generator/cortex_gpu.yaml | 3 +-- test/apis/realtime/hello-world/cortex_cpu.yaml | 3 +-- .../realtime/image-classifier-resnet50/cortex_cpu.yaml | 3 +-- .../realtime/image-classifier-resnet50/cortex_gpu.yaml | 3 +-- test/apis/realtime/prime-generator/cortex_cpu.yaml | 3 +-- test/apis/realtime/sleep/cortex_cpu.yaml | 4 ++-- test/apis/realtime/text-generator/cortex_cpu.yaml | 3 +-- test/apis/realtime/text-generator/cortex_gpu.yaml | 3 +-- test/apis/trafficsplitter/hello-world/cortex_cpu.yaml | 9 +++------ 10 files changed, 13 insertions(+), 24 deletions(-) diff --git a/test/apis/async/text-generator/cortex_cpu.yaml b/test/apis/async/text-generator/cortex_cpu.yaml index 6d6332a910..0863c54775 100644 --- a/test/apis/async/text-generator/cortex_cpu.yaml +++ b/test/apis/async/text-generator/cortex_cpu.yaml @@ -2,6 +2,7 @@ kind: AsyncAPI pod: port: 9000 + max_concurrency: 1 containers: - name: api image: quay.io/cortexlabs-test/async-text-generator-cpu:latest @@ -12,5 +13,3 @@ compute: cpu: 1 mem: 2.5G - autoscaling: - max_concurrency: 1 diff --git a/test/apis/async/text-generator/cortex_gpu.yaml b/test/apis/async/text-generator/cortex_gpu.yaml index 3b814daf93..3d0ed0c94c 100644 --- a/test/apis/async/text-generator/cortex_gpu.yaml +++ b/test/apis/async/text-generator/cortex_gpu.yaml @@ -2,6 +2,7 @@ kind: AsyncAPI pod: port: 9000 + max_concurrency: 1 containers: - name: api image: quay.io/cortexlabs-test/async-text-generator-gpu:latest @@ -15,5 +16,3 @@ cpu: 1 gpu: 1 mem: 512M - autoscaling: - max_concurrency: 1 diff --git a/test/apis/realtime/hello-world/cortex_cpu.yaml b/test/apis/realtime/hello-world/cortex_cpu.yaml index 071ba16b88..b913aeef3d 100644 --- a/test/apis/realtime/hello-world/cortex_cpu.yaml +++ b/test/apis/realtime/hello-world/cortex_cpu.yaml @@ -2,6 +2,7 @@ kind: RealtimeAPI pod: port: 9000 + max_concurrency: 1 containers: - name: api image: quay.io/cortexlabs-test/realtime-hello-world-cpu:latest @@ -12,5 +13,3 @@ compute: cpu: 200m mem: 128M - autoscaling: - max_concurrency: 1 diff --git a/test/apis/realtime/image-classifier-resnet50/cortex_cpu.yaml b/test/apis/realtime/image-classifier-resnet50/cortex_cpu.yaml index cb8db93493..8e74412630 100644 --- a/test/apis/realtime/image-classifier-resnet50/cortex_cpu.yaml +++ b/test/apis/realtime/image-classifier-resnet50/cortex_cpu.yaml @@ -2,6 +2,7 @@ kind: RealtimeAPI pod: port: 8501 + max_concurrency: 8 containers: - name: api image: quay.io/cortexlabs-test/realtime-image-classifier-resnet50-cpu:latest @@ -11,5 +12,3 @@ compute: cpu: 1 mem: 2G - autoscaling: - max_concurrency: 8 diff --git a/test/apis/realtime/image-classifier-resnet50/cortex_gpu.yaml b/test/apis/realtime/image-classifier-resnet50/cortex_gpu.yaml index bc840bea3c..2fb05f4aa0 100644 --- a/test/apis/realtime/image-classifier-resnet50/cortex_gpu.yaml +++ b/test/apis/realtime/image-classifier-resnet50/cortex_gpu.yaml @@ -2,6 +2,7 @@ kind: RealtimeAPI pod: port: 8501 + max_concurrency: 8 containers: - name: api image: quay.io/cortexlabs-test/realtime-image-classifier-resnet50-gpu:latest @@ -12,5 +13,3 @@ cpu: 200m gpu: 1 mem: 512Mi - autoscaling: - max_concurrency: 8 diff --git a/test/apis/realtime/prime-generator/cortex_cpu.yaml b/test/apis/realtime/prime-generator/cortex_cpu.yaml index ba5ea380f5..b5ffaf96c8 100644 --- a/test/apis/realtime/prime-generator/cortex_cpu.yaml +++ b/test/apis/realtime/prime-generator/cortex_cpu.yaml @@ -2,6 +2,7 @@ kind: RealtimeAPI pod: port: 9000 + max_concurrency: 1 containers: - name: api image: quay.io/cortexlabs-test/realtime-prime-generator-cpu:latest @@ -12,5 +13,3 @@ compute: cpu: 200m mem: 128M - autoscaling: - max_concurrency: 1 diff --git a/test/apis/realtime/sleep/cortex_cpu.yaml b/test/apis/realtime/sleep/cortex_cpu.yaml index 150168bd5e..cae1a97b39 100644 --- a/test/apis/realtime/sleep/cortex_cpu.yaml +++ b/test/apis/realtime/sleep/cortex_cpu.yaml @@ -2,6 +2,8 @@ kind: RealtimeAPI pod: port: 9000 + max_concurrency: 1 + max_queue_length: 128 containers: - name: api image: quay.io/cortexlabs-test/realtime-sleep-cpu:latest @@ -13,6 +15,4 @@ cpu: 200m mem: 128M autoscaling: - max_concurrency: 1 target_in_flight: 1 - max_queue_length: 128 diff --git a/test/apis/realtime/text-generator/cortex_cpu.yaml b/test/apis/realtime/text-generator/cortex_cpu.yaml index 3e502c40a1..ffea3ec1fb 100644 --- a/test/apis/realtime/text-generator/cortex_cpu.yaml +++ b/test/apis/realtime/text-generator/cortex_cpu.yaml @@ -2,6 +2,7 @@ kind: RealtimeAPI pod: port: 9000 + max_concurrency: 1 containers: - name: api image: quay.io/cortexlabs-test/realtime-text-generator-cpu:latest @@ -12,5 +13,3 @@ compute: cpu: 1 mem: 2.5G - autoscaling: - max_concurrency: 1 diff --git a/test/apis/realtime/text-generator/cortex_gpu.yaml b/test/apis/realtime/text-generator/cortex_gpu.yaml index 1db64d7477..11a70b57a9 100644 --- a/test/apis/realtime/text-generator/cortex_gpu.yaml +++ b/test/apis/realtime/text-generator/cortex_gpu.yaml @@ -2,6 +2,7 @@ kind: RealtimeAPI pod: port: 9000 + max_concurrency: 1 containers: - name: api image: quay.io/cortexlabs-test/realtime-text-generator-gpu:latest @@ -15,5 +16,3 @@ cpu: 1 gpu: 1 mem: 512M - autoscaling: - max_concurrency: 1 diff --git a/test/apis/trafficsplitter/hello-world/cortex_cpu.yaml b/test/apis/trafficsplitter/hello-world/cortex_cpu.yaml index 8d18685e83..73fec8f126 100644 --- a/test/apis/trafficsplitter/hello-world/cortex_cpu.yaml +++ b/test/apis/trafficsplitter/hello-world/cortex_cpu.yaml @@ -2,6 +2,7 @@ kind: RealtimeAPI pod: port: 9000 + max_concurrency: 1 containers: - name: api image: quay.io/cortexlabs-test/realtime-hello-world-cpu:latest @@ -14,13 +15,12 @@ compute: cpu: 200m mem: 128M - autoscaling: - max_concurrency: 1 - name: hello-world-b kind: RealtimeAPI pod: port: 9000 + max_concurrency: 1 containers: - name: api image: quay.io/cortexlabs-test/realtime-hello-world-cpu:latest @@ -33,13 +33,12 @@ compute: cpu: 200m mem: 128M - autoscaling: - max_concurrency: 1 - name: hello-world-shadow kind: RealtimeAPI pod: port: 9000 + max_concurrency: 1 containers: - name: api image: quay.io/cortexlabs-test/realtime-hello-world-cpu:latest @@ -52,8 +51,6 @@ compute: cpu: 200m mem: 128M - autoscaling: - max_concurrency: 1 - name: hello-world kind: TrafficSplitter From 85138e907a492848ce1b5e9d4e320790656b3c79 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Sat, 29 May 2021 00:42:11 +0300 Subject: [PATCH 43/82] CaaS - move the shm field to the compute section (#2202) --- pkg/types/spec/errors.go | 8 ++-- pkg/types/spec/validations.go | 28 ++++++------- pkg/types/userconfig/api.go | 64 +++++++++++------------------- pkg/types/userconfig/config_key.go | 2 +- pkg/workloads/helpers.go | 8 ++-- pkg/workloads/k8s.go | 13 +++--- 6 files changed, 52 insertions(+), 71 deletions(-) diff --git a/pkg/types/spec/errors.go b/pkg/types/spec/errors.go index 5a1f84012e..fcb6643fa9 100644 --- a/pkg/types/spec/errors.go +++ b/pkg/types/spec/errors.go @@ -48,7 +48,7 @@ const ( ErrInvalidSurgeOrUnavailable = "spec.invalid_surge_or_unavailable" ErrSurgeAndUnavailableBothZero = "spec.surge_and_unavailable_both_zero" - ErrShmSizeCannotExceedMem = "spec.shm_size_cannot_exceed_mem" + ErrShmCannotExceedMem = "spec.shm_cannot_exceed_mem" ErrFieldMustBeSpecifiedForKind = "spec.field_must_be_specified_for_kind" ErrFieldIsNotSupportedForKind = "spec.field_is_not_supported_for_kind" @@ -208,10 +208,10 @@ func ErrorSurgeAndUnavailableBothZero() error { }) } -func ErrorShmSizeCannotExceedMem(shmSize k8s.Quantity, mem k8s.Quantity) error { +func ErrorShmCannotExceedMem(shm k8s.Quantity, mem k8s.Quantity) error { return errors.WithStack(&errors.Error{ - Kind: ErrShmSizeCannotExceedMem, - Message: fmt.Sprintf("shm_size (%s) cannot exceed total compute mem (%s)", shmSize.UserString, mem.UserString), + Kind: ErrShmCannotExceedMem, + Message: fmt.Sprintf("%s (%s) cannot exceed total compute %s (%s)", userconfig.ShmKey, shm.UserString, userconfig.MemKey, mem.UserString), }) } diff --git a/pkg/types/spec/validations.go b/pkg/types/spec/validations.go index 706dbe039e..1bea4949ed 100644 --- a/pkg/types/spec/validations.go +++ b/pkg/types/spec/validations.go @@ -145,15 +145,6 @@ func podValidation(kind userconfig.Kind) *cr.StructFieldValidation { StructField: "Pod", StructValidation: &cr.StructValidation{ StructFieldValidations: []*cr.StructFieldValidation{ - { - StructField: "ShmSize", - StringPtrValidation: &cr.StringPtrValidation{ - Required: false, - Default: nil, - AllowExplicitNull: true, - }, - Parser: k8s.QuantityParser(&k8s.QuantityValidation{}), - }, { StructField: "NodeGroups", StringListValidation: &cr.StringListValidation{ @@ -470,6 +461,14 @@ func computeValidation() *cr.StructFieldValidation { GreaterThanOrEqualTo: pointer.Int64(0), }, }, + { + StructField: "Shm", + StringPtrValidation: &cr.StringPtrValidation{ + Default: nil, + AllowExplicitNull: true, + }, + Parser: k8s.QuantityParser(&k8s.QuantityValidation{}), + }, }, }, } @@ -713,12 +712,6 @@ func validatePod( containers := api.Pod.Containers totalCompute := userconfig.GetTotalComputeFromContainers(containers) - if api.Pod.ShmSize != nil { - if totalCompute.Mem != nil && api.Pod.ShmSize.Cmp(totalCompute.Mem.Quantity) > 0 { - return ErrorShmSizeCannotExceedMem(*api.Pod.ShmSize, *totalCompute.Mem) - } - } - if api.Pod.Port != nil && api.Kind == userconfig.TaskAPIKind { return ErrorFieldIsNotSupportedForKind(userconfig.PortKey, api.Kind) } @@ -782,6 +775,11 @@ func validateContainers( } } + compute := container.Compute + if compute.Shm != nil && compute.Mem != nil && compute.Shm.Cmp(compute.Mem.Quantity) > 0 { + return errors.Wrap(ErrorShmCannotExceedMem(*compute.Shm, *compute.Mem), s.Index(i), userconfig.ComputeKey) + } + } return nil diff --git a/pkg/types/userconfig/api.go b/pkg/types/userconfig/api.go index d4434c4ca7..af7fe552cc 100644 --- a/pkg/types/userconfig/api.go +++ b/pkg/types/userconfig/api.go @@ -44,12 +44,11 @@ type API struct { } type Pod struct { - NodeGroups []string `json:"node_groups" yaml:"node_groups"` - ShmSize *k8s.Quantity `json:"shm_size" yaml:"shm_size"` - Port *int32 `json:"port" yaml:"port"` - MaxQueueLength int64 `json:"max_queue_length" yaml:"max_queue_length"` - MaxConcurrency int64 `json:"max_concurrency" yaml:"max_concurrency"` - Containers []*Container `json:"containers" yaml:"containers"` + NodeGroups []string `json:"node_groups" yaml:"node_groups"` + Port *int32 `json:"port" yaml:"port"` + MaxQueueLength int64 `json:"max_queue_length" yaml:"max_queue_length"` + MaxConcurrency int64 `json:"max_concurrency" yaml:"max_concurrency"` + Containers []*Container `json:"containers" yaml:"containers"` } type Container struct { @@ -105,6 +104,7 @@ type Compute struct { Mem *k8s.Quantity `json:"mem" yaml:"mem"` GPU int64 `json:"gpu" yaml:"gpu"` Inf int64 `json:"inf" yaml:"inf"` + Shm *k8s.Quantity `json:"shm" yaml:"shm"` } type Autoscaling struct { @@ -289,9 +289,6 @@ func (trafficSplit *TrafficSplit) UserStr() string { func (pod *Pod) UserStr(kind Kind) string { var sb strings.Builder - if pod.ShmSize != nil { - sb.WriteString(fmt.Sprintf("%s: %s\n", ShmSizeKey, pod.ShmSize.UserString)) - } if pod.NodeGroups == nil { sb.WriteString(fmt.Sprintf("%s: null\n", NodeGroupsKey)) } else { @@ -426,35 +423,12 @@ func (compute *Compute) UserStr() string { } else { sb.WriteString(fmt.Sprintf("%s: %s\n", MemKey, compute.Mem.UserString)) } - return sb.String() -} - -func (compute Compute) Equals(c2 *Compute) bool { - if c2 == nil { - return false - } - - if compute.CPU == nil && c2.CPU != nil || compute.CPU != nil && c2.CPU == nil { - return false - } - - if compute.CPU != nil && c2.CPU != nil && !compute.CPU.Equal(*c2.CPU) { - return false - } - - if compute.Mem == nil && c2.Mem != nil || compute.Mem != nil && c2.Mem == nil { - return false - } - - if compute.Mem != nil && c2.Mem != nil && !compute.Mem.Equal(*c2.Mem) { - return false - } - - if compute.GPU != c2.GPU { - return false + if compute.Shm == nil { + sb.WriteString(fmt.Sprintf("%s: null # not configured\n", ShmKey)) + } else { + sb.WriteString(fmt.Sprintf("%s: %s\n", ShmKey, compute.Shm.UserString)) } - - return true + return sb.String() } func (autoscaling *Autoscaling) UserStr() string { @@ -515,6 +489,15 @@ func GetTotalComputeFromContainers(containers []*Container) Compute { } } + if container.Compute.Shm != nil { + newShmQuantity := k8s.NewMilliQuantity(container.Compute.Shm.ToDec().MilliValue()) + if compute.Shm == nil { + compute.Shm = newShmQuantity + } else if newShmQuantity != nil { + compute.Shm.AddQty(*newShmQuantity) + } + } + compute.GPU += container.Compute.GPU compute.Inf += container.Compute.Inf } @@ -552,9 +535,6 @@ func (api *API) TelemetryEvent() map[string]interface{} { if api.Pod != nil { event["pod._is_defined"] = true - if api.Pod.ShmSize != nil { - event["pod.shm_size"] = api.Pod.ShmSize.String() - } event["pod.node_groups._is_defined"] = len(api.Pod.NodeGroups) > 0 event["pod.node_groups._len"] = len(api.Pod.NodeGroups) if api.Pod.Port != nil { @@ -590,6 +570,10 @@ func (api *API) TelemetryEvent() map[string]interface{} { event["pod.containers.compute.mem._is_defined"] = true event["pod.containers.compute.mem"] = totalCompute.Mem.Value() } + if totalCompute.Shm != nil { + event["pod.containers.compute.shm._is_defined"] = true + event["pod.containers.compute.shm"] = totalCompute.Shm.Value() + } event["pod.containers.compute.gpu"] = totalCompute.GPU event["pod.containers.compute.inf"] = totalCompute.Inf } diff --git a/pkg/types/userconfig/config_key.go b/pkg/types/userconfig/config_key.go index fce539daac..826e144b05 100644 --- a/pkg/types/userconfig/config_key.go +++ b/pkg/types/userconfig/config_key.go @@ -33,7 +33,6 @@ const ( // Pod PodKey = "pod" NodeGroupsKey = "node_groups" - ShmSizeKey = "shm_size" PortKey = "port" MaxConcurrencyKey = "max_concurrency" MaxQueueLengthKey = "max_queue_length" @@ -66,6 +65,7 @@ const ( MemKey = "mem" GPUKey = "gpu" InfKey = "inf" + ShmKey = "shm" // Networking EndpointKey = "endpoint" diff --git a/pkg/workloads/helpers.go b/pkg/workloads/helpers.go index e6f4301e7a..505c9c101b 100644 --- a/pkg/workloads/helpers.go +++ b/pkg/workloads/helpers.go @@ -194,9 +194,9 @@ func ClusterConfigVolume() kcore.Volume { } } -func ShmVolume(q resource.Quantity) kcore.Volume { +func ShmVolume(q resource.Quantity, volumeName string) kcore.Volume { return kcore.Volume{ - Name: _shmDirVolumeName, + Name: volumeName, VolumeSource: kcore.VolumeSource{ EmptyDir: &kcore.EmptyDirVolumeSource{ Medium: kcore.StorageMediumMemory, @@ -241,8 +241,8 @@ func ClusterConfigMount() kcore.VolumeMount { } } -func ShmMount() kcore.VolumeMount { - return k8s.EmptyDirVolumeMount(_shmDirVolumeName, _shmDirMountPath) +func ShmMount(volumeName string) kcore.VolumeMount { + return k8s.EmptyDirVolumeMount(volumeName, _shmDirMountPath) } func KubexitMount() kcore.VolumeMount { diff --git a/pkg/workloads/k8s.go b/pkg/workloads/k8s.go index 4cae268412..ed2b62752c 100644 --- a/pkg/workloads/k8s.go +++ b/pkg/workloads/k8s.go @@ -51,8 +51,7 @@ const ( _kubexitGraveyardName = "graveyard" _kubexitGraveyardMountPath = "/graveyard" - _shmDirVolumeName = "dshm" - _shmDirMountPath = "/dev/shm" + _shmDirMountPath = "/dev/shm" _clientConfigDirVolume = "client-config" _clientConfigConfigMap = "client-config" @@ -246,11 +245,6 @@ func userPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { ClientConfigMount(), } - if api.Pod.ShmSize != nil { - volumes = append(volumes, ShmVolume(api.Pod.ShmSize.Quantity)) - containerMounts = append(containerMounts, ShmMount()) - } - var containers []kcore.Container for _, container := range api.Pod.Containers { containerResourceList := kcore.ResourceList{} @@ -292,6 +286,11 @@ func userPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { } } + if container.Compute.Shm != nil { + volumes = append(volumes, ShmVolume(container.Compute.Shm.Quantity, "dshm-"+container.Name)) + containerMounts = append(containerMounts, ShmMount("dshm-"+container.Name)) + } + containerEnvVars := baseEnvVars containerEnvVars = append(containerEnvVars, kcore.EnvVar{ From ef413434fceb508d0ba2a793f85c7d72203c7e26 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 28 May 2021 15:35:22 -0700 Subject: [PATCH 44/82] Delete projectID --- pkg/operator/resources/asyncapi/api.go | 4 ++-- pkg/operator/resources/job/batchapi/api.go | 4 ++-- pkg/operator/resources/job/taskapi/api.go | 4 ++-- pkg/operator/resources/realtimeapi/api.go | 6 +++--- pkg/operator/resources/resources.go | 15 ++++++--------- pkg/types/spec/api.go | 6 +----- 6 files changed, 16 insertions(+), 23 deletions(-) diff --git a/pkg/operator/resources/asyncapi/api.go b/pkg/operator/resources/asyncapi/api.go index b71cd2387e..a93f005c54 100644 --- a/pkg/operator/resources/asyncapi/api.go +++ b/pkg/operator/resources/asyncapi/api.go @@ -66,7 +66,7 @@ func deploymentID() string { return k8s.RandomName()[:10] } -func UpdateAPI(apiConfig userconfig.API, projectID string, force bool) (*spec.API, string, error) { +func UpdateAPI(apiConfig userconfig.API, force bool) (*spec.API, string, error) { prevK8sResources, err := getK8sResources(apiConfig) if err != nil { return nil, "", err @@ -77,7 +77,7 @@ func UpdateAPI(apiConfig userconfig.API, projectID string, force bool) (*spec.AP deployID = prevK8sResources.apiDeployment.Labels["deploymentID"] } - api := spec.GetAPISpec(&apiConfig, projectID, deployID, config.ClusterConfig.ClusterUID) + api := spec.GetAPISpec(&apiConfig, deployID, config.ClusterConfig.ClusterUID) // resource creation if prevK8sResources.apiDeployment == nil { diff --git a/pkg/operator/resources/job/batchapi/api.go b/pkg/operator/resources/job/batchapi/api.go index df3fc6cc1e..9a88f84ec3 100644 --- a/pkg/operator/resources/job/batchapi/api.go +++ b/pkg/operator/resources/job/batchapi/api.go @@ -41,13 +41,13 @@ import ( const _batchDashboardUID = "batchapi" -func UpdateAPI(apiConfig *userconfig.API, projectID string) (*spec.API, string, error) { +func UpdateAPI(apiConfig *userconfig.API) (*spec.API, string, error) { prevVirtualService, err := config.K8s.GetVirtualService(workloads.K8sName(apiConfig.Name)) if err != nil { return nil, "", err } - api := spec.GetAPISpec(apiConfig, projectID, "", config.ClusterConfig.ClusterUID) // Deployment ID not needed for BatchAPI spec + api := spec.GetAPISpec(apiConfig, "", config.ClusterConfig.ClusterUID) // Deployment ID not needed for BatchAPI spec if prevVirtualService == nil { if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil { diff --git a/pkg/operator/resources/job/taskapi/api.go b/pkg/operator/resources/job/taskapi/api.go index 0c44a4daf3..f0f93cbf8a 100644 --- a/pkg/operator/resources/job/taskapi/api.go +++ b/pkg/operator/resources/job/taskapi/api.go @@ -38,13 +38,13 @@ import ( ) // UpdateAPI deploys or update a task api without triggering any task -func UpdateAPI(apiConfig *userconfig.API, projectID string) (*spec.API, string, error) { +func UpdateAPI(apiConfig *userconfig.API) (*spec.API, string, error) { prevVirtualService, err := config.K8s.GetVirtualService(workloads.K8sName(apiConfig.Name)) if err != nil { return nil, "", err } - api := spec.GetAPISpec(apiConfig, projectID, "", config.ClusterConfig.ClusterUID) // Deployment ID not needed for TaskAPI spec + api := spec.GetAPISpec(apiConfig, "", config.ClusterConfig.ClusterUID) // Deployment ID not needed for TaskAPI spec if prevVirtualService == nil { if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil { diff --git a/pkg/operator/resources/realtimeapi/api.go b/pkg/operator/resources/realtimeapi/api.go index 0cc58c8dd1..c1a0efe371 100644 --- a/pkg/operator/resources/realtimeapi/api.go +++ b/pkg/operator/resources/realtimeapi/api.go @@ -47,7 +47,7 @@ func deploymentID() string { return k8s.RandomName()[:10] } -func UpdateAPI(apiConfig *userconfig.API, projectID string, force bool) (*spec.API, string, error) { +func UpdateAPI(apiConfig *userconfig.API, force bool) (*spec.API, string, error) { prevDeployment, prevService, prevVirtualService, err := getK8sResources(apiConfig) if err != nil { return nil, "", err @@ -58,7 +58,7 @@ func UpdateAPI(apiConfig *userconfig.API, projectID string, force bool) (*spec.A deploymentID = prevDeployment.Labels["deploymentID"] } - api := spec.GetAPISpec(apiConfig, projectID, deploymentID, config.ClusterConfig.ClusterUID) + api := spec.GetAPISpec(apiConfig, deploymentID, config.ClusterConfig.ClusterUID) if prevDeployment == nil { if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil { @@ -146,7 +146,7 @@ func RefreshAPI(apiName string, force bool) (string, error) { return "", err } - api = spec.GetAPISpec(api.API, api.ProjectID, deploymentID(), config.ClusterConfig.ClusterUID) + api = spec.GetAPISpec(api.API, deploymentID(), config.ClusterConfig.ClusterUID) if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil { return "", errors.Wrap(err, "upload api spec") diff --git a/pkg/operator/resources/resources.go b/pkg/operator/resources/resources.go index 918174a1d6..aa6da29c9e 100644 --- a/pkg/operator/resources/resources.go +++ b/pkg/operator/resources/resources.go @@ -25,7 +25,6 @@ import ( batch "github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" - "github.com/cortexlabs/cortex/pkg/lib/hash" "github.com/cortexlabs/cortex/pkg/lib/logging" "github.com/cortexlabs/cortex/pkg/lib/parallel" "github.com/cortexlabs/cortex/pkg/lib/pointer" @@ -85,8 +84,6 @@ func GetDeployedResourceByNameOrNil(resourceName string) (*operator.DeployedReso } func Deploy(configFileName string, configBytes []byte, force bool) ([]schema.DeployResult, error) { - projectID := hash.Bytes(configBytes) - apiConfigs, err := spec.ExtractAPIConfigs(configBytes, configFileName) if err != nil { return nil, err @@ -105,7 +102,7 @@ func Deploy(configFileName string, configBytes []byte, force bool) ([]schema.Dep for i := range apiConfigs { apiConfig := apiConfigs[i] - api, msg, err := UpdateAPI(&apiConfig, projectID, force) + api, msg, err := UpdateAPI(&apiConfig, force) result := schema.DeployResult{ Message: msg, @@ -122,7 +119,7 @@ func Deploy(configFileName string, configBytes []byte, force bool) ([]schema.Dep return results, nil } -func UpdateAPI(apiConfig *userconfig.API, projectID string, force bool) (*schema.APIResponse, string, error) { +func UpdateAPI(apiConfig *userconfig.API, force bool) (*schema.APIResponse, string, error) { deployedResource, err := GetDeployedResourceByNameOrNil(apiConfig.Name) if err != nil { return nil, "", err @@ -138,13 +135,13 @@ func UpdateAPI(apiConfig *userconfig.API, projectID string, force bool) (*schema var msg string switch apiConfig.Kind { case userconfig.RealtimeAPIKind: - api, msg, err = realtimeapi.UpdateAPI(apiConfig, projectID, force) + api, msg, err = realtimeapi.UpdateAPI(apiConfig, force) case userconfig.BatchAPIKind: - api, msg, err = batchapi.UpdateAPI(apiConfig, projectID) + api, msg, err = batchapi.UpdateAPI(apiConfig) case userconfig.TaskAPIKind: - api, msg, err = taskapi.UpdateAPI(apiConfig, projectID) + api, msg, err = taskapi.UpdateAPI(apiConfig) case userconfig.AsyncAPIKind: - api, msg, err = asyncapi.UpdateAPI(*apiConfig, projectID, force) + api, msg, err = asyncapi.UpdateAPI(*apiConfig, force) case userconfig.TrafficSplitterKind: api, msg, err = trafficsplitter.UpdateAPI(apiConfig) default: diff --git a/pkg/types/spec/api.go b/pkg/types/spec/api.go index a3fa2eb3c9..952c52827a 100644 --- a/pkg/types/spec/api.go +++ b/pkg/types/spec/api.go @@ -38,7 +38,6 @@ type API struct { SpecID string `json:"spec_id"` HandlerID string `json:"handler_id"` DeploymentID string `json:"deployment_id"` - ProjectID string `json:"project_id"` Key string `json:"key"` HandlerKey string `json:"handler_key"` @@ -55,19 +54,17 @@ APIID (uniquely identifies an api configuration for a given deployment) * Containers * Compute * Pod - * ProjectID * Deployment Strategy * Autoscaling * Networking * APIs * DeploymentID (used for refreshing a deployment) */ -func GetAPISpec(apiConfig *userconfig.API, projectID string, deploymentID string, clusterUID string) *API { +func GetAPISpec(apiConfig *userconfig.API, deploymentID string, clusterUID string) *API { var buf bytes.Buffer buf.WriteString(s.Obj(apiConfig.Resource)) buf.WriteString(s.Obj(apiConfig.Pod)) - buf.WriteString(projectID) handlerID := hash.Bytes(buf.Bytes()) buf.Reset() @@ -90,7 +87,6 @@ func GetAPISpec(apiConfig *userconfig.API, projectID string, deploymentID string DeploymentID: deploymentID, LastUpdated: time.Now().Unix(), MetadataRoot: MetadataRoot(apiConfig.Name, clusterUID), - ProjectID: projectID, } } From 8501030d8d79162b11a560b2bcb24ed11ab416f7 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 28 May 2021 15:36:40 -0700 Subject: [PATCH 45/82] Fix projectID deletion --- pkg/operator/resources/trafficsplitter/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/operator/resources/trafficsplitter/api.go b/pkg/operator/resources/trafficsplitter/api.go index e2b908d3fd..655311b3ae 100644 --- a/pkg/operator/resources/trafficsplitter/api.go +++ b/pkg/operator/resources/trafficsplitter/api.go @@ -41,7 +41,7 @@ func UpdateAPI(apiConfig *userconfig.API) (*spec.API, string, error) { return nil, "", err } - api := spec.GetAPISpec(apiConfig, "", "", config.ClusterConfig.ClusterUID) + api := spec.GetAPISpec(apiConfig, "", config.ClusterConfig.ClusterUID) if prevVirtualService == nil { if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil { return nil, "", errors.Wrap(err, "failed to upload api spec") From 8d82a18aa2ce1d983eb3dc3d522cbe786d77b6aa Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 28 May 2021 15:54:49 -0700 Subject: [PATCH 46/82] Rename handlerID to podID --- .../batch/batchjob_controller_helpers.go | 4 +-- .../batch/batchjob_controller_test.go | 2 +- pkg/operator/endpoints/logs.go | 4 +-- pkg/operator/resources/asyncapi/api.go | 30 +++---------------- pkg/operator/resources/asyncapi/k8s_specs.go | 12 ++++---- pkg/operator/resources/asyncapi/status.go | 2 +- .../resources/job/batchapi/k8s_specs.go | 2 +- pkg/operator/resources/job/taskapi/job.go | 2 +- .../resources/job/taskapi/k8s_specs.go | 6 ++-- pkg/operator/resources/realtimeapi/api.go | 17 +---------- .../resources/realtimeapi/k8s_specs.go | 6 ++-- pkg/types/spec/api.go | 25 ++++------------ pkg/types/spec/job.go | 2 +- 13 files changed, 32 insertions(+), 82 deletions(-) diff --git a/pkg/crds/controllers/batch/batchjob_controller_helpers.go b/pkg/crds/controllers/batch/batchjob_controller_helpers.go index 70815b8a15..e9a7d4ae0c 100644 --- a/pkg/crds/controllers/batch/batchjob_controller_helpers.go +++ b/pkg/crds/controllers/batch/batchjob_controller_helpers.go @@ -338,7 +338,7 @@ func (r *BatchJobReconciler) desiredWorkerJob(batchJob batch.BatchJob, apiSpec s "apiName": batchJob.Spec.APIName, "apiID": batchJob.Spec.APIID, "specID": apiSpec.SpecID, - "handlerID": apiSpec.HandlerID, + "podID": apiSpec.PodID, "jobID": batchJob.Name, "cortex.dev/api": "true", "cortex.dev/batch": "worker", @@ -349,7 +349,7 @@ func (r *BatchJobReconciler) desiredWorkerJob(batchJob batch.BatchJob, apiSpec s "apiName": batchJob.Spec.APIName, "apiID": batchJob.Spec.APIID, "specID": apiSpec.SpecID, - "handlerID": apiSpec.HandlerID, + "podID": apiSpec.PodID, "jobID": batchJob.Name, "cortex.dev/api": "true", "cortex.dev/batch": "worker", diff --git a/pkg/crds/controllers/batch/batchjob_controller_test.go b/pkg/crds/controllers/batch/batchjob_controller_test.go index 005691cb3e..c439023067 100644 --- a/pkg/crds/controllers/batch/batchjob_controller_test.go +++ b/pkg/crds/controllers/batch/batchjob_controller_test.go @@ -57,7 +57,7 @@ func uploadTestAPISpec(apiName string, apiID string) error { }, ID: apiID, SpecID: random.String(5), - HandlerID: random.String(5), + PodID: random.String(5), DeploymentID: random.String(5), } apiSpecKey := spec.Key(apiName, apiID, clusterConfig.ClusterUID) diff --git a/pkg/operator/endpoints/logs.go b/pkg/operator/endpoints/logs.go index 62273c6a10..765a0318eb 100644 --- a/pkg/operator/endpoints/logs.go +++ b/pkg/operator/endpoints/logs.go @@ -50,7 +50,7 @@ func ReadLogs(w http.ResponseWriter, r *http.Request) { } deploymentID := deployedResource.VirtualService.Labels["deploymentID"] - handlerID := deployedResource.VirtualService.Labels["handlerID"] + podID := deployedResource.VirtualService.Labels["podID"] upgrader := websocket.Upgrader{} socket, err := upgrader.Upgrade(w, r, nil) @@ -60,7 +60,7 @@ func ReadLogs(w http.ResponseWriter, r *http.Request) { } defer socket.Close() - labels := map[string]string{"apiName": apiName, "deploymentID": deploymentID, "handlerID": handlerID} + labels := map[string]string{"apiName": apiName, "deploymentID": deploymentID, "podID": podID} if deployedResource.Kind == userconfig.AsyncAPIKind { labels["cortex.dev/async"] = "api" diff --git a/pkg/operator/resources/asyncapi/api.go b/pkg/operator/resources/asyncapi/api.go index a93f005c54..bf83c81bb8 100644 --- a/pkg/operator/resources/asyncapi/api.go +++ b/pkg/operator/resources/asyncapi/api.go @@ -81,8 +81,8 @@ func UpdateAPI(apiConfig userconfig.API, force bool) (*spec.API, string, error) // resource creation if prevK8sResources.apiDeployment == nil { - if err = uploadAPItoS3(*api); err != nil { - return nil, "", err + if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil { + return nil, "", errors.Wrap(err, "upload api spec") } tags := map[string]string{ @@ -121,8 +121,8 @@ func UpdateAPI(apiConfig userconfig.API, force bool) (*spec.API, string, error) return nil, "", ErrorAPIUpdating(api.Name) } - if err = uploadAPItoS3(*api); err != nil { - return nil, "", err + if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil { + return nil, "", errors.Wrap(err, "upload api spec") } queueURL, err := getQueueURL(api.Name, prevK8sResources.gatewayVirtualService.Labels["deploymentID"]) @@ -507,25 +507,3 @@ func deleteK8sResources(apiName string) error { return err } - -func uploadAPItoS3(api spec.API) error { - return parallel.RunFirstErr( - func() error { - var err error - err = config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key) - if err != nil { - err = errors.Wrap(err, "upload api spec") - } - return err - }, - func() error { - var err error - // Use api spec indexed by HandlerID for replicas to prevent rolling updates when SpecID changes without HandlerID changing - err = config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.HandlerKey) - if err != nil { - err = errors.Wrap(err, "upload handler spec") - } - return err - }, - ) -} diff --git a/pkg/operator/resources/asyncapi/k8s_specs.go b/pkg/operator/resources/asyncapi/k8s_specs.go index 79fad9e7e6..79d243728f 100644 --- a/pkg/operator/resources/asyncapi/k8s_specs.go +++ b/pkg/operator/resources/asyncapi/k8s_specs.go @@ -69,7 +69,7 @@ func gatewayDeploymentSpec(api spec.API, prevDeployment *kapps.Deployment, queue "apiID": api.ID, "specID": api.SpecID, "deploymentID": api.DeploymentID, - "handlerID": api.HandlerID, + "podID": api.PodID, "cortex.dev/api": "true", "cortex.dev/async": "gateway", }, @@ -78,7 +78,7 @@ func gatewayDeploymentSpec(api spec.API, prevDeployment *kapps.Deployment, queue "apiName": api.Name, "apiKind": api.Kind.String(), "deploymentID": api.DeploymentID, - "handlerID": api.HandlerID, + "podID": api.PodID, "cortex.dev/api": "true", "cortex.dev/async": "gateway", }, @@ -113,7 +113,7 @@ func gatewayHPASpec(api spec.API) (kautoscaling.HorizontalPodAutoscaler, error) "apiID": api.ID, "specID": api.SpecID, "deploymentID": api.DeploymentID, - "handlerID": api.HandlerID, + "podID": api.PodID, "cortex.dev/api": "true", "cortex.dev/async": "hpa", }, @@ -164,7 +164,7 @@ func gatewayVirtualServiceSpec(api spec.API) v1beta1.VirtualService { "apiID": api.ID, "specID": api.SpecID, "deploymentID": api.DeploymentID, - "handlerID": api.HandlerID, + "podID": api.PodID, "cortex.dev/api": "true", "cortex.dev/async": "gateway", }, @@ -218,7 +218,7 @@ func deploymentSpec(api spec.API, prevDeployment *kapps.Deployment, queueURL str "apiID": api.ID, "specID": api.SpecID, "deploymentID": api.DeploymentID, - "handlerID": api.HandlerID, + "podID": api.PodID, "cortex.dev/api": "true", "cortex.dev/async": "api", }, @@ -233,7 +233,7 @@ func deploymentSpec(api spec.API, prevDeployment *kapps.Deployment, queueURL str "apiName": api.Name, "apiKind": api.Kind.String(), "deploymentID": api.DeploymentID, - "handlerID": api.HandlerID, + "podID": api.PodID, "cortex.dev/api": "true", "cortex.dev/async": "api", }, diff --git a/pkg/operator/resources/asyncapi/status.go b/pkg/operator/resources/asyncapi/status.go index 3f90f6d85e..34e3f5c84d 100644 --- a/pkg/operator/resources/asyncapi/status.go +++ b/pkg/operator/resources/asyncapi/status.go @@ -297,6 +297,6 @@ func addPodToReplicaCounts(pod *kcore.Pod, deployment *kapps.Deployment, counts } func isPodSpecLatest(deployment *kapps.Deployment, pod *kcore.Pod) bool { - return deployment.Spec.Template.Labels["handlerID"] == pod.Labels["handlerID"] && + return deployment.Spec.Template.Labels["podID"] == pod.Labels["podID"] && deployment.Spec.Template.Labels["deploymentID"] == pod.Labels["deploymentID"] } diff --git a/pkg/operator/resources/job/batchapi/k8s_specs.go b/pkg/operator/resources/job/batchapi/k8s_specs.go index 5ae8b5aff8..39c1588a42 100644 --- a/pkg/operator/resources/job/batchapi/k8s_specs.go +++ b/pkg/operator/resources/job/batchapi/k8s_specs.go @@ -50,7 +50,7 @@ func virtualServiceSpec(api *spec.API) *istioclientnetworking.VirtualService { "apiName": api.Name, "apiID": api.ID, "specID": api.SpecID, - "handlerID": api.HandlerID, + "podID": api.PodID, "apiKind": api.Kind.String(), "cortex.dev/api": "true", }, diff --git a/pkg/operator/resources/job/taskapi/job.go b/pkg/operator/resources/job/taskapi/job.go index f95ce04b75..2362ced32c 100644 --- a/pkg/operator/resources/job/taskapi/job.go +++ b/pkg/operator/resources/job/taskapi/job.go @@ -61,7 +61,7 @@ func SubmitJob(apiName string, submission *schema.TaskJobSubmission) (*spec.Task RuntimeTaskJobConfig: submission.RuntimeTaskJobConfig, APIID: apiSpec.ID, SpecID: apiSpec.SpecID, - HandlerID: apiSpec.HandlerID, + PodID: apiSpec.PodID, StartTime: time.Now(), } diff --git a/pkg/operator/resources/job/taskapi/k8s_specs.go b/pkg/operator/resources/job/taskapi/k8s_specs.go index 554506472e..4a38ba7d29 100644 --- a/pkg/operator/resources/job/taskapi/k8s_specs.go +++ b/pkg/operator/resources/job/taskapi/k8s_specs.go @@ -52,7 +52,7 @@ func virtualServiceSpec(api *spec.API) *istioclientnetworking.VirtualService { "apiName": api.Name, "apiID": api.ID, "specID": api.SpecID, - "handlerID": api.HandlerID, + "podID": api.PodID, "apiKind": api.Kind.String(), "cortex.dev/api": "true", }, @@ -69,7 +69,7 @@ func k8sJobSpec(api *spec.API, job *spec.TaskJob) *kbatch.Job { "apiName": api.Name, "apiID": api.ID, "specID": api.SpecID, - "handlerID": api.HandlerID, + "podID": api.PodID, "jobID": job.ID, "apiKind": api.Kind.String(), "cortex.dev/api": "true", @@ -77,7 +77,7 @@ func k8sJobSpec(api *spec.API, job *spec.TaskJob) *kbatch.Job { PodSpec: k8s.PodSpec{ Labels: map[string]string{ "apiName": api.Name, - "handlerID": api.HandlerID, + "podID": api.PodID, "jobID": job.ID, "apiKind": api.Kind.String(), "cortex.dev/api": "true", diff --git a/pkg/operator/resources/realtimeapi/api.go b/pkg/operator/resources/realtimeapi/api.go index c1a0efe371..f07b717b5c 100644 --- a/pkg/operator/resources/realtimeapi/api.go +++ b/pkg/operator/resources/realtimeapi/api.go @@ -65,11 +65,6 @@ func UpdateAPI(apiConfig *userconfig.API, force bool) (*spec.API, string, error) return nil, "", errors.Wrap(err, "upload api spec") } - // Use api spec indexed by HandlerID for replicas to prevent rolling updates when SpecID changes without HandlerID changing - if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.HandlerKey); err != nil { - return nil, "", errors.Wrap(err, "upload handler spec") - } - if err := applyK8sResources(api, prevDeployment, prevService, prevVirtualService); err != nil { routines.RunWithPanicHandler(func() { deleteK8sResources(api.Name) @@ -93,11 +88,6 @@ func UpdateAPI(apiConfig *userconfig.API, force bool) (*spec.API, string, error) return nil, "", errors.Wrap(err, "upload api spec") } - // Use api spec indexed by HandlerID for replicas to prevent rolling updates when SpecID changes without HandlerID changing - if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.HandlerKey); err != nil { - return nil, "", errors.Wrap(err, "upload handler spec") - } - if err := applyK8sResources(api, prevDeployment, prevService, prevVirtualService); err != nil { return nil, "", err } @@ -152,11 +142,6 @@ func RefreshAPI(apiName string, force bool) (string, error) { return "", errors.Wrap(err, "upload api spec") } - // Reupload api spec to the same HandlerID but with the new DeploymentID - if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.HandlerKey); err != nil { - return "", errors.Wrap(err, "upload handler spec") - } - if err := applyK8sResources(api, prevDeployment, prevService, prevVirtualService); err != nil { return "", err } @@ -428,7 +413,7 @@ func isAPIUpdating(deployment *kapps.Deployment) (bool, error) { } func isPodSpecLatest(deployment *kapps.Deployment, pod *kcore.Pod) bool { - return deployment.Spec.Template.Labels["handlerID"] == pod.Labels["handlerID"] && + return deployment.Spec.Template.Labels["podID"] == pod.Labels["podID"] && deployment.Spec.Template.Labels["deploymentID"] == pod.Labels["deploymentID"] } diff --git a/pkg/operator/resources/realtimeapi/k8s_specs.go b/pkg/operator/resources/realtimeapi/k8s_specs.go index 903268f9f1..2790e9751f 100644 --- a/pkg/operator/resources/realtimeapi/k8s_specs.go +++ b/pkg/operator/resources/realtimeapi/k8s_specs.go @@ -47,7 +47,7 @@ func deploymentSpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Depl "apiID": api.ID, "specID": api.SpecID, "deploymentID": api.DeploymentID, - "handlerID": api.HandlerID, + "podID": api.PodID, "cortex.dev/api": "true", }, Annotations: api.ToK8sAnnotations(), @@ -60,7 +60,7 @@ func deploymentSpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Depl "apiName": api.Name, "apiKind": api.Kind.String(), "deploymentID": api.DeploymentID, - "handlerID": api.HandlerID, + "podID": api.PodID, "cortex.dev/api": "true", }, Annotations: map[string]string{ @@ -117,7 +117,7 @@ func virtualServiceSpec(api *spec.API) *istioclientnetworking.VirtualService { "apiID": api.ID, "specID": api.SpecID, "deploymentID": api.DeploymentID, - "handlerID": api.HandlerID, + "podID": api.PodID, "cortex.dev/api": "true", }, }) diff --git a/pkg/types/spec/api.go b/pkg/types/spec/api.go index 952c52827a..3ae80385e3 100644 --- a/pkg/types/spec/api.go +++ b/pkg/types/spec/api.go @@ -36,11 +36,10 @@ type API struct { *userconfig.API ID string `json:"id"` SpecID string `json:"spec_id"` - HandlerID string `json:"handler_id"` + PodID string `json:"pod_id"` DeploymentID string `json:"deployment_id"` - Key string `json:"key"` - HandlerKey string `json:"handler_key"` + Key string `json:"key"` LastUpdated int64 `json:"last_updated"` MetadataRoot string `json:"metadata_root"` @@ -49,7 +48,7 @@ type API struct { /* APIID (uniquely identifies an api configuration for a given deployment) * SpecID (uniquely identifies api configuration specified by user) - * HandlerID (used to determine when rolling updates need to happen) + * PodID (an ID representing the pod spec) * Resource * Containers * Compute @@ -65,10 +64,10 @@ func GetAPISpec(apiConfig *userconfig.API, deploymentID string, clusterUID strin buf.WriteString(s.Obj(apiConfig.Resource)) buf.WriteString(s.Obj(apiConfig.Pod)) - handlerID := hash.Bytes(buf.Bytes()) + podID := hash.Bytes(buf.Bytes()) buf.Reset() - buf.WriteString(handlerID) + buf.WriteString(podID) buf.WriteString(s.Obj(apiConfig.APIs)) buf.WriteString(s.Obj(apiConfig.Networking)) buf.WriteString(s.Obj(apiConfig.Autoscaling)) @@ -81,26 +80,14 @@ func GetAPISpec(apiConfig *userconfig.API, deploymentID string, clusterUID strin API: apiConfig, ID: apiID, SpecID: specID, - HandlerID: handlerID, + PodID: podID, Key: Key(apiConfig.Name, apiID, clusterUID), - HandlerKey: HandlerKey(apiConfig.Name, handlerID, clusterUID), DeploymentID: deploymentID, LastUpdated: time.Now().Unix(), MetadataRoot: MetadataRoot(apiConfig.Name, clusterUID), } } -func HandlerKey(apiName string, handlerID string, clusterUID string) string { - return filepath.Join( - clusterUID, - "apis", - apiName, - "handler", - handlerID, - consts.CortexVersion+"-spec.json", - ) -} - func Key(apiName string, apiID string, clusterUID string) string { return filepath.Join( clusterUID, diff --git a/pkg/types/spec/job.go b/pkg/types/spec/job.go index f1395d2881..784fb4f199 100644 --- a/pkg/types/spec/job.go +++ b/pkg/types/spec/job.go @@ -87,7 +87,7 @@ type TaskJob struct { RuntimeTaskJobConfig APIID string `json:"api_id"` SpecID string `json:"spec_id"` - HandlerID string `json:"handler_id"` + PodID string `json:"pod_id"` StartTime time.Time `json:"start_time"` } From da7cc02a40ba039dea3ba9fae4f0597a8b5ed054 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 28 May 2021 15:56:58 -0700 Subject: [PATCH 47/82] Delete unnecessary comment --- pkg/types/status/status.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/types/status/status.go b/pkg/types/status/status.go index faffbbe308..6dad4e1992 100644 --- a/pkg/types/status/status.go +++ b/pkg/types/status/status.go @@ -24,7 +24,7 @@ type Status struct { } type ReplicaCounts struct { - Updated SubReplicaCounts `json:"updated"` // fully up-to-date (compute and model) + Updated SubReplicaCounts `json:"updated"` Stale SubReplicaCounts `json:"stale"` Requested int32 `json:"requested"` } From b626d3912881914f88ae70d5d812ff81dad77c26 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 28 May 2021 15:58:54 -0700 Subject: [PATCH 48/82] Rename Container Implementation --- docs/summary.md | 8 ++++---- docs/workloads/async/container.md | 2 +- docs/workloads/batch/container.md | 2 +- docs/workloads/realtime/container.md | 2 +- docs/workloads/task/container.md | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/summary.md b/docs/summary.md index a407dda760..7ed2aa333c 100644 --- a/docs/summary.md +++ b/docs/summary.md @@ -32,7 +32,7 @@ * [Realtime APIs](workloads/realtime/realtime-apis.md) * [Example](workloads/realtime/example.md) * [Configuration](workloads/realtime/configuration.md) - * [Container Interface](workloads/realtime/container.md) + * [Container Implementation](workloads/realtime/container.md) * [Autoscaling](workloads/realtime/autoscaling.md) * [Traffic Splitter](workloads/realtime/traffic-splitter.md) * [Metrics](workloads/realtime/metrics.md) @@ -41,18 +41,18 @@ * [Async APIs](workloads/async/async-apis.md) * [Example](workloads/async/example.md) * [Configuration](workloads/async/configuration.md) - * [Container Interface](workloads/async/container.md) + * [Container Implementation](workloads/async/container.md) * [Statuses](workloads/async/statuses.md) * [Batch APIs](workloads/batch/batch-apis.md) * [Example](workloads/batch/example.md) * [Configuration](workloads/batch/configuration.md) - * [Container Interface](workloads/batch/container.md) + * [Container Implementation](workloads/batch/container.md) * [Jobs](workloads/batch/jobs.md) * [Statuses](workloads/batch/statuses.md) * [Task APIs](workloads/task/task-apis.md) * [Example](workloads/task/example.md) * [Configuration](workloads/task/configuration.md) - * [Container Interface](workloads/task/container.md) + * [Container Implementation](workloads/task/container.md) * [Jobs](workloads/task/jobs.md) * [Statuses](workloads/task/statuses.md) diff --git a/docs/workloads/async/container.md b/docs/workloads/async/container.md index 58f9f21202..f1369970da 100644 --- a/docs/workloads/async/container.md +++ b/docs/workloads/async/container.md @@ -1,4 +1,4 @@ -# Container Interface +# Container Implementation ## Handling requests diff --git a/docs/workloads/batch/container.md b/docs/workloads/batch/container.md index 368e54ec9f..f194828c1b 100644 --- a/docs/workloads/batch/container.md +++ b/docs/workloads/batch/container.md @@ -1,4 +1,4 @@ -# Container Interface +# Container Implementation ## Handling requests diff --git a/docs/workloads/realtime/container.md b/docs/workloads/realtime/container.md index 6a6c605d88..43f3517da4 100644 --- a/docs/workloads/realtime/container.md +++ b/docs/workloads/realtime/container.md @@ -1,4 +1,4 @@ -# Container Interface +# Container Implementation ## Handling requests diff --git a/docs/workloads/task/container.md b/docs/workloads/task/container.md index 05bcc3753a..1579dc6204 100644 --- a/docs/workloads/task/container.md +++ b/docs/workloads/task/container.md @@ -1,4 +1,4 @@ -# Container Interface +# Container Implementation ## Multiple containers From 486584ae796f3afe9995dd51eb8c4a47d5a5da09 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 28 May 2021 16:05:30 -0700 Subject: [PATCH 49/82] Update docs --- CONTRIBUTING.md | 2 +- docs/workloads/async/container.md | 2 +- docs/workloads/batch/container.md | 6 +++++- docs/workloads/task/container.md | 6 +++++- pkg/lib/aws/s3.go | 2 +- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f7f73e4af3..001805cd45 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,7 @@ ## Remote development -We recommend that you run your development environment on an EC2 instance due to frequent docker registry pushing. We've had a good experience using [Mutagen](https://mutagen.io/documentation/introduction) to synchronize local / remote file systems. +We recommend that you run your development environment on an EC2 instance due to frequent docker registry pushing. We've had a good experience using [Mutagen](https://mutagen.io/documentation/introduction) to synchronize local / remote filesystems. ## Prerequisites diff --git a/docs/workloads/async/container.md b/docs/workloads/async/container.md index f1369970da..290145c6e9 100644 --- a/docs/workloads/async/container.md +++ b/docs/workloads/async/container.md @@ -25,7 +25,7 @@ readiness_probe: Your API pod can contain multiple containers, only one of which can be listening for requests on the target port (it can be any of the containers). -The `/mnt` directory is mounted to each container's file system, and is shared across all containers. +The `/mnt` directory is mounted to each container's filesystem, and is shared across all containers. ## Using the Cortex CLI or client diff --git a/docs/workloads/batch/container.md b/docs/workloads/batch/container.md index f194828c1b..2be5439f94 100644 --- a/docs/workloads/batch/container.md +++ b/docs/workloads/batch/container.md @@ -10,6 +10,10 @@ Your web server must respond with status code 200 for the batch to be marked as Once all batches have been processed, one of your workers will receive an HTTP POST request to `/on-job-complete`. It is not necessary for your web server to handle requests to `/on-job-complete` (404 errors will be ignored). +## Job specification + +If you need access to any parameters in the job submission (e.g. `config`), the entire job specification is available at `/cortex/spec/job.json` in your API containers' filesystems. + ## Readiness checks It is often important to implement a readiness check for your API. By default, as soon as your web server has bound to the port, it will start receiving batches. In some cases, the web server may start listening on the port before its workers are ready to handle traffic (e.g. `tiangolo/uvicorn-gunicorn-fastapi` behaves this way). Readiness checks ensure that traffic is not sent into your web server before it's ready to handle them. @@ -27,7 +31,7 @@ readiness_probe: Your API pod can contain multiple containers, only one of which can be listening for requests on the target port (it can be any of the containers). -The `/mnt` directory is mounted to each container's file system, and is shared across all containers. +The `/mnt` directory is mounted to each container's filesystem, and is shared across all containers. ## Using the Cortex CLI or client diff --git a/docs/workloads/task/container.md b/docs/workloads/task/container.md index 1579dc6204..d68e206a9e 100644 --- a/docs/workloads/task/container.md +++ b/docs/workloads/task/container.md @@ -1,8 +1,12 @@ # Container Implementation +## Job specification + +If you need access to any parameters in the job submission (e.g. `config`), the entire job specification is available at `/cortex/spec/job.json` in your API containers' filesystems. + ## Multiple containers -Your Task's pod can contain multiple containers. The `/mnt` directory is mounted to each container's file system, and is shared across all containers. +Your Task's pod can contain multiple containers. The `/mnt` directory is mounted to each container's filesystem, and is shared across all containers. ## Using the Cortex CLI or client diff --git a/pkg/lib/aws/s3.go b/pkg/lib/aws/s3.go index 6a15f044af..b0d1357113 100644 --- a/pkg/lib/aws/s3.go +++ b/pkg/lib/aws/s3.go @@ -684,7 +684,7 @@ func (c *Client) ListS3PathDir(s3DirPath string, includeDirObjects bool, maxResu return c.ListS3PathPrefix(s3Path, includeDirObjects, maxResults, startAfter) } -// This behaves like you'd expect `ls` to behave on a local file system +// This behaves like you'd expect `ls` to behave on a local filesystem // "directory" names will be returned even if S3 directory objects don't exist func (c *Client) ListS3DirOneLevel(bucket string, s3Dir string, maxResults *int64, startAfter *string) ([]string, error) { s3Dir = s.EnsureSuffix(s3Dir, "/") From 30eea014f5ef951b2d7974d829e33c9c13408b37 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Sat, 29 May 2021 02:30:35 +0300 Subject: [PATCH 50/82] CaaS - Async/Batch Deployments (#2201) --- cmd/async-gateway/main.go | 2 +- pkg/consts/consts.go | 4 + .../batch/batchjob_controller_helpers.go | 9 +- pkg/operator/resources/asyncapi/k8s_specs.go | 6 +- .../resources/job/taskapi/k8s_specs.go | 2 +- .../resources/realtimeapi/k8s_specs.go | 6 +- pkg/types/clusterconfig/cluster_config.go | 11 ++ pkg/workloads/k8s.go | 112 +++++++++++++++--- .../image-classifier-alexnet/cortex_cpu.yaml | 2 +- .../image-classifier-alexnet/cortex_gpu.yaml | 2 +- .../batch/image-classifier-alexnet/submit.py | 1 + test/apis/batch/sum/cortex_cpu.yaml | 2 +- test/apis/batch/sum/submit.py | 1 + 13 files changed, 120 insertions(+), 40 deletions(-) diff --git a/cmd/async-gateway/main.go b/cmd/async-gateway/main.go index 5f940e5c0a..2d7bd21258 100644 --- a/cmd/async-gateway/main.go +++ b/cmd/async-gateway/main.go @@ -66,9 +66,9 @@ func main() { }() var ( + clusterConfigPath = flag.String("cluster-config", "", "cluster config path") port = flag.String("port", _defaultPort, "port on which the gateway server runs on") queueURL = flag.String("queue", "", "SQS queue URL") - clusterConfigPath = flag.String("cluster-config", "", "cluster config path") ) flag.Parse() diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 1ba32c84c5..139205e10b 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -36,6 +36,9 @@ var ( AdminPortStr = "15000" AdminPortInt32 = int32(15000) + StatsDPortStr = "9125" + StatsDPortInt32 = int32(9125) + AuthHeader = "X-Cortex-Authorization" DefaultInClusterConfigPath = "/configs/cluster/cluster.yaml" @@ -43,6 +46,7 @@ var ( AsyncWorkloadsExpirationDays = int64(7) ReservedContainerNames = []string{ + "dequeuer", "proxy", } ) diff --git a/pkg/crds/controllers/batch/batchjob_controller_helpers.go b/pkg/crds/controllers/batch/batchjob_controller_helpers.go index e9a7d4ae0c..7b35461e76 100644 --- a/pkg/crds/controllers/batch/batchjob_controller_helpers.go +++ b/pkg/crds/controllers/batch/batchjob_controller_helpers.go @@ -319,14 +319,7 @@ func (r *BatchJobReconciler) desiredEnqueuerJob(batchJob batch.BatchJob, queueUR } func (r *BatchJobReconciler) desiredWorkerJob(batchJob batch.BatchJob, apiSpec spec.API, jobSpec spec.BatchJob) (*kbatch.Job, error) { - var containers []kcore.Container - var volumes []kcore.Volume - - containers, volumes = workloads.BatchUserPodContainers(apiSpec, &jobSpec.JobKey) - - // TODO add the proxy as well - // use workloads.APIConfigMount(batchJob.Spec.APIName + "-" + batchJob.Name) to mount the probes - // the probes will be made available at /cortex/spec/probes.json + containers, volumes := workloads.BatchContainers(apiSpec, &jobSpec) job := k8s.Job( &k8s.JobSpec{ diff --git a/pkg/operator/resources/asyncapi/k8s_specs.go b/pkg/operator/resources/asyncapi/k8s_specs.go index 79d243728f..cea6c256a8 100644 --- a/pkg/operator/resources/asyncapi/k8s_specs.go +++ b/pkg/operator/resources/asyncapi/k8s_specs.go @@ -201,11 +201,7 @@ func deploymentSpec(api spec.API, prevDeployment *kapps.Deployment, queueURL str volumes []kcore.Volume ) - containers, volumes = workloads.AsyncUserPodContainers(api) - - // TODO add the proxy as well - // use workloads.APIConfigMount(workloads.K8sName(api.Name)) to mount the probes - // the probes will be made available at /cortex/spec/probes.json + containers, volumes = workloads.AsyncContainers(api, queueURL) return *k8s.Deployment(&k8s.DeploymentSpec{ Name: workloads.K8sName(api.Name), diff --git a/pkg/operator/resources/job/taskapi/k8s_specs.go b/pkg/operator/resources/job/taskapi/k8s_specs.go index 4a38ba7d29..71f30cb019 100644 --- a/pkg/operator/resources/job/taskapi/k8s_specs.go +++ b/pkg/operator/resources/job/taskapi/k8s_specs.go @@ -60,7 +60,7 @@ func virtualServiceSpec(api *spec.API) *istioclientnetworking.VirtualService { } func k8sJobSpec(api *spec.API, job *spec.TaskJob) *kbatch.Job { - containers, volumes := workloads.TaskUserPodContainers(*api, &job.JobKey) + containers, volumes := workloads.TaskContainers(*api, &job.JobKey) return k8s.Job(&k8s.JobSpec{ Name: job.JobKey.K8sName(), diff --git a/pkg/operator/resources/realtimeapi/k8s_specs.go b/pkg/operator/resources/realtimeapi/k8s_specs.go index 2790e9751f..5467bdaded 100644 --- a/pkg/operator/resources/realtimeapi/k8s_specs.go +++ b/pkg/operator/resources/realtimeapi/k8s_specs.go @@ -30,11 +30,7 @@ import ( var _terminationGracePeriodSeconds int64 = 60 // seconds func deploymentSpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Deployment { - containers, volumes := workloads.RealtimeUserPodContainers(*api) - proxyContainer, proxyVolume := workloads.RealtimeProxyContainer(*api) - - containers = append(containers, proxyContainer) - volumes = append(volumes, proxyVolume) + containers, volumes := workloads.RealtimeContainers(*api) return k8s.Deployment(&k8s.DeploymentSpec{ Name: workloads.K8sName(api.Name), diff --git a/pkg/types/clusterconfig/cluster_config.go b/pkg/types/clusterconfig/cluster_config.go index a27c1293af..37b3a7600b 100644 --- a/pkg/types/clusterconfig/cluster_config.go +++ b/pkg/types/clusterconfig/cluster_config.go @@ -92,6 +92,7 @@ type CoreConfig struct { ImageProxy string `json:"image_proxy" yaml:"image_proxy"` ImageAsyncGateway string `json:"image_async_gateway" yaml:"image_async_gateway"` ImageEnqueuer string `json:"image_enqueuer" yaml:"image_enqueuer"` + ImageDequeuer string `json:"image_dequeuer" yaml:"image_dequeuer"` ImageClusterAutoscaler string `json:"image_cluster_autoscaler" yaml:"image_cluster_autoscaler"` ImageMetricsServer string `json:"image_metrics_server" yaml:"image_metrics_server"` ImageInferentia string `json:"image_inferentia" yaml:"image_inferentia"` @@ -361,6 +362,13 @@ var CoreConfigStructFieldValidations = []*cr.StructFieldValidation{ Validator: validateImageVersion, }, }, + { + StructField: "ImageDequeuer", + StringValidation: &cr.StringValidation{ + Default: consts.DefaultRegistry() + "/dequeuer:" + consts.CortexVersion, + Validator: validateImageVersion, + }, + }, { StructField: "ImageClusterAutoscaler", StringValidation: &cr.StringValidation{ @@ -1316,6 +1324,9 @@ func (cc *CoreConfig) TelemetryEvent() map[string]interface{} { if !strings.HasPrefix(cc.ImageEnqueuer, "cortexlabs/") { event["image_enqueuer._is_custom"] = true } + if !strings.HasPrefix(cc.ImageDequeuer, "cortexlabs/") { + event["image_dequeuer._is_custom"] = true + } if !strings.HasPrefix(cc.ImageClusterAutoscaler, "cortexlabs/") { event["image_cluster_autoscaler._is_custom"] = true } diff --git a/pkg/workloads/k8s.go b/pkg/workloads/k8s.go index ed2b62752c..2739f38eed 100644 --- a/pkg/workloads/k8s.go +++ b/pkg/workloads/k8s.go @@ -44,9 +44,11 @@ const ( _emptyDirVolumeName = "mnt" _emptyDirMountPath = "/mnt" + _gatewayContainerName = "gateway" + _proxyContainerName = "proxy" - _gatewayContainerName = "gateway" + _dequeuerContainerName = "dequeuer" _kubexitGraveyardName = "graveyard" _kubexitGraveyardMountPath = "/graveyard" @@ -78,9 +80,9 @@ func AsyncGatewayContainer(api spec.API, queueURL string, volumeMounts []kcore.V Image: config.ClusterConfig.ImageAsyncGateway, ImagePullPolicy: kcore.PullAlways, Args: []string{ + "--cluster-config", consts.DefaultInClusterConfigPath, "--port", s.Int32(consts.ProxyListeningPortInt32), "--queue", queueURL, - "--cluster-config", consts.DefaultInClusterConfigPath, api.Name, }, Ports: []kcore.ContainerPort{ @@ -113,12 +115,78 @@ func AsyncGatewayContainer(api spec.API, queueURL string, volumeMounts []kcore.V } } -func RealtimeProxyContainer(api spec.API) (kcore.Container, kcore.Volume) { +func asyncDequeuerProxyContainer(api spec.API, queueURL string) (kcore.Container, kcore.Volume) { + return kcore.Container{ + Name: _dequeuerContainerName, + Image: config.ClusterConfig.ImageDequeuer, + ImagePullPolicy: kcore.PullAlways, + Command: []string{ + "/dequeuer", + }, + Args: []string{ + "--cluster-config", consts.DefaultInClusterConfigPath, + "--cluster-uid", config.ClusterConfig.ClusterUID, + "--queue", queueURL, + "--api-kind", api.Kind.String(), + "--api-name", api.Name, + "--user-port", s.Int32(*api.Pod.Port), + "--statsd-port", consts.StatsDPortStr, + }, + Env: append(baseEnvVars, kcore.EnvVar{ + Name: "HOST_IP", + ValueFrom: &kcore.EnvVarSource{ + FieldRef: &kcore.ObjectFieldSelector{ + FieldPath: "status.hostIP", + }, + }, + }), + VolumeMounts: []kcore.VolumeMount{ + ClusterConfigMount(), + }, + }, ClusterConfigVolume() +} + +func batchDequeuerProxyContainer(api spec.API, jobID, queueURL string) (kcore.Container, kcore.Volume) { + return kcore.Container{ + Name: _dequeuerContainerName, + Image: config.ClusterConfig.ImageDequeuer, + ImagePullPolicy: kcore.PullAlways, + Command: []string{ + "/dequeuer", + }, + Args: []string{ + "--cluster-config", consts.DefaultInClusterConfigPath, + "--cluster-uid", config.ClusterConfig.ClusterUID, + "--queue", queueURL, + "--api-kind", api.Kind.String(), + "--api-name", api.Name, + "--job-id", jobID, + "--user-port", s.Int32(*api.Pod.Port), + "--statsd-port", consts.StatsDPortStr, + }, + Env: append(baseEnvVars, kcore.EnvVar{ + Name: "HOST_IP", + ValueFrom: &kcore.EnvVarSource{ + FieldRef: &kcore.ObjectFieldSelector{ + FieldPath: "status.hostIP", + }, + }, + }), + VolumeMounts: []kcore.VolumeMount{ + ClusterConfigMount(), + CortexMount(), + }, + }, ClusterConfigVolume() +} + +func realtimeProxyContainer(api spec.API) (kcore.Container, kcore.Volume) { return kcore.Container{ Name: _proxyContainerName, Image: config.ClusterConfig.ImageProxy, ImagePullPolicy: kcore.PullAlways, Args: []string{ + "--cluster-config", + consts.DefaultInClusterConfigPath, "--port", consts.ProxyListeningPortStr, "--admin-port", @@ -129,8 +197,6 @@ func RealtimeProxyContainer(api spec.API) (kcore.Container, kcore.Volume) { s.Int32(int32(api.Pod.MaxConcurrency)), "--max-queue-length", s.Int32(int32(api.Pod.MaxQueueLength)), - "--cluster-config", - consts.DefaultInClusterConfigPath, }, Ports: []kcore.ContainerPort{ {Name: "admin", ContainerPort: consts.AdminPortInt32}, @@ -157,25 +223,30 @@ func RealtimeProxyContainer(api spec.API) (kcore.Container, kcore.Volume) { }, ClusterConfigVolume() } -func RealtimeUserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { - return userPodContainers(api) +func RealtimeContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { + containers, volumes := userPodContainers(api) + proxyContainer, proxyVolume := realtimeProxyContainer(api) + + containers = append(containers, proxyContainer) + volumes = append(volumes, proxyVolume) + + return containers, volumes } -func AsyncUserPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { - containers, volumes := userPodContainers(api) +func AsyncContainers(api spec.API, queueURL string) ([]kcore.Container, []kcore.Volume) { k8sName := K8sName(api.Name) - for i := range containers { - containers[i].VolumeMounts = append(containers[i].VolumeMounts, - APIConfigMount(k8sName), - ) - } - volumes = append(volumes, APIConfigVolume(k8sName)) + containers, volumes := userPodContainers(api) + dequeuerContainer, dequeuerVolume := asyncDequeuerProxyContainer(api, queueURL) + dequeuerContainer.VolumeMounts = append(dequeuerContainer.VolumeMounts, APIConfigMount(k8sName)) + + containers = append(containers, dequeuerContainer) + volumes = append(volumes, dequeuerVolume, APIConfigVolume(k8sName)) return containers, volumes } -func TaskUserPodContainers(api spec.API, job *spec.JobKey) ([]kcore.Container, []kcore.Volume) { +func TaskContainers(api spec.API, job *spec.JobKey) ([]kcore.Container, []kcore.Volume) { containers, volumes := userPodContainers(api) k8sName := job.K8sName() @@ -204,8 +275,13 @@ func TaskUserPodContainers(api spec.API, job *spec.JobKey) ([]kcore.Container, [ return containers, volumes } -func BatchUserPodContainers(api spec.API, job *spec.JobKey) ([]kcore.Container, []kcore.Volume) { +func BatchContainers(api spec.API, job *spec.BatchJob) ([]kcore.Container, []kcore.Volume) { containers, volumes := userPodContainers(api) + dequeuerContainer, dequeuerVolume := batchDequeuerProxyContainer(api, job.ID, job.SQSUrl) + + containers = append(containers, dequeuerContainer) + volumes = append(volumes, dequeuerVolume) + k8sName := job.K8sName() volumes = append(volumes, @@ -214,6 +290,8 @@ func BatchUserPodContainers(api spec.API, job *spec.JobKey) ([]kcore.Container, ) containerNames := userconfig.GetContainerNames(api.Pod.Containers) + containerNames.Add(dequeuerContainer.Name) + for i, c := range containers { containers[i].VolumeMounts = append(containers[i].VolumeMounts, KubexitMount(), diff --git a/test/apis/batch/image-classifier-alexnet/cortex_cpu.yaml b/test/apis/batch/image-classifier-alexnet/cortex_cpu.yaml index 1799203294..5891c01510 100644 --- a/test/apis/batch/image-classifier-alexnet/cortex_cpu.yaml +++ b/test/apis/batch/image-classifier-alexnet/cortex_cpu.yaml @@ -13,7 +13,7 @@ - "--threads" - "1" - "--bind" - - ":$CORTEX_PORT" + - ":$(CORTEX_PORT)" - "main:app" compute: cpu: 1 diff --git a/test/apis/batch/image-classifier-alexnet/cortex_gpu.yaml b/test/apis/batch/image-classifier-alexnet/cortex_gpu.yaml index 11469453b2..b5748f0c3c 100644 --- a/test/apis/batch/image-classifier-alexnet/cortex_gpu.yaml +++ b/test/apis/batch/image-classifier-alexnet/cortex_gpu.yaml @@ -13,7 +13,7 @@ - "--threads" - "1" - "--bind" - - ":$CORTEX_PORT" + - ":$(CORTEX_PORT)" - "main:app" compute: cpu: 200m diff --git a/test/apis/batch/image-classifier-alexnet/submit.py b/test/apis/batch/image-classifier-alexnet/submit.py index 8e9be6e80b..01ba0d55b5 100644 --- a/test/apis/batch/image-classifier-alexnet/submit.py +++ b/test/apis/batch/image-classifier-alexnet/submit.py @@ -30,6 +30,7 @@ def main(): # submit job job_spec = { + "workers": 1, "item_list": {"items": sample_items, "batch_size": 1}, "config": {"dest_s3_dir": dest_s3_dir}, } diff --git a/test/apis/batch/sum/cortex_cpu.yaml b/test/apis/batch/sum/cortex_cpu.yaml index 22e59ef857..5c3dafde3c 100644 --- a/test/apis/batch/sum/cortex_cpu.yaml +++ b/test/apis/batch/sum/cortex_cpu.yaml @@ -15,7 +15,7 @@ - "--threads" - "1" - "--bind" - - ":$CORTEX_PORT" + - ":$(CORTEX_PORT)" - "main:app" compute: cpu: 200m diff --git a/test/apis/batch/sum/submit.py b/test/apis/batch/sum/submit.py index b6baffdab9..c1ae977b8a 100644 --- a/test/apis/batch/sum/submit.py +++ b/test/apis/batch/sum/submit.py @@ -30,6 +30,7 @@ def main(): # submit job job_spec = { + "workers": 1, "item_list": {"items": sample_items, "batch_size": 1}, "config": {"dest_s3_dir": dest_s3_dir}, } From 30efa6d1234802d9eeff42f75de5eb6e5814f586 Mon Sep 17 00:00:00 2001 From: Vishal Bollu Date: Mon, 31 May 2021 09:51:53 -0400 Subject: [PATCH 51/82] Add configmap permissions to controller (#2205) --- pkg/crds/config/rbac/role.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/crds/config/rbac/role.yaml b/pkg/crds/config/rbac/role.yaml index d99ed03381..4b64fb36ab 100644 --- a/pkg/crds/config/rbac/role.yaml +++ b/pkg/crds/config/rbac/role.yaml @@ -6,6 +6,15 @@ metadata: creationTimestamp: null name: manager-role rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - list + - watch - apiGroups: - "" resources: From e4edc3355bc77d40d5e961b98ca718b8bab08ef5 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Mon, 31 May 2021 10:36:21 -0700 Subject: [PATCH 52/82] Update docs --- docs/workloads/async/configuration.md | 6 +++--- docs/workloads/batch/configuration.md | 6 +++--- docs/workloads/realtime/configuration.md | 8 ++++---- docs/workloads/task/configuration.md | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/workloads/async/configuration.md b/docs/workloads/async/configuration.md index 3b975b1413..d8e142fcef 100644 --- a/docs/workloads/async/configuration.md +++ b/docs/workloads/async/configuration.md @@ -8,8 +8,8 @@ containers: # configurations for the containers to run (at least one constainer must be provided) - name: # name of the container (required) image: # docker image to use for the container (required) - command: # entrypoint (default: the docker image's ENTRYPOINT) - args: # arguments to the entrypoint (default: the docker image's CMD) + command: # entrypoint (not executed within a shell); env vars can be used with e.g. $(CORTEX_PORT) (default: the docker image's ENTRYPOINT) + args: # arguments to the entrypoint; env vars can be used with e.g. $(CORTEX_PORT) (default: the docker image's CMD) env: # dictionary of environment variables to set in the container (optional) compute: # compute resource requests (default: see below) cpu: # CPU request for the container; one unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the "m" suffix (default: 200m) @@ -35,7 +35,7 @@ tcp_socket: # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified) port: # the port to access on the container (required) exec: # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified) - command: [] # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) + command: # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) initial_delay_seconds: # number of seconds after the container has started before the probe is initiated (default: 0) timeout_seconds: # number of seconds until the probe times out (default: 1) period_seconds: # how often (in seconds) to perform the probe (default: 10) diff --git a/docs/workloads/batch/configuration.md b/docs/workloads/batch/configuration.md index 31ef274089..cd6b1472ea 100644 --- a/docs/workloads/batch/configuration.md +++ b/docs/workloads/batch/configuration.md @@ -8,8 +8,8 @@ containers: # configurations for the containers to run (at least one constainer must be provided) - name: # name of the container (required) image: # docker image to use for the container (required) - command: # entrypoint (required) - args: # arguments to the entrypoint (default: no args) + command: # entrypoint (not executed within a shell); env vars can be used with e.g. $(CORTEX_PORT) (required) + args: # arguments to the entrypoint; env vars can be used with e.g. $(CORTEX_PORT) (default: no args) env: # dictionary of environment variables to set in the container (optional) compute: # compute resource requests (default: see below) cpu: # CPU request for the container; one unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the "m" suffix (default: 200m) @@ -35,7 +35,7 @@ tcp_socket: # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified) port: # the port to access on the container (required) exec: # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified) - command: [] # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) + command: # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) initial_delay_seconds: # number of seconds after the container has started before the probe is initiated (default: 0) timeout_seconds: # number of seconds until the probe times out (default: 1) period_seconds: # how often (in seconds) to perform the probe (default: 10) diff --git a/docs/workloads/realtime/configuration.md b/docs/workloads/realtime/configuration.md index 9d2bf85bf6..840a302983 100644 --- a/docs/workloads/realtime/configuration.md +++ b/docs/workloads/realtime/configuration.md @@ -10,8 +10,8 @@ containers: # configurations for the containers to run (at least one constainer must be provided) - name: # name of the container (required) image: # docker image to use for the container (required) - command: # entrypoint (default: the docker image's ENTRYPOINT) - args: # arguments to the entrypoint (default: the docker image's CMD) + command: # entrypoint (not executed within a shell); env vars can be used with e.g. $(CORTEX_PORT) (default: the docker image's ENTRYPOINT) + args: # arguments to the entrypoint; env vars can be used with e.g. $(CORTEX_PORT) (default: the docker image's CMD) env: # dictionary of environment variables to set in the container (optional) compute: # compute resource requests (default: see below) cpu: # CPU request for the container; one unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the "m" suffix (default: 200m) @@ -26,7 +26,7 @@ tcp_socket: # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified) port: # the port to access on the container (required) exec: # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified) - command: [] # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) + command: # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) initial_delay_seconds: # number of seconds after the container has started before the probe is initiated (default: 0) timeout_seconds: # number of seconds until the probe times out (default: 1) period_seconds: # how often (in seconds) to perform the probe (default: 10) @@ -39,7 +39,7 @@ tcp_socket: # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified) port: # the port to access on the container (required) exec: # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified) - command: [] # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) + command: # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) initial_delay_seconds: # number of seconds after the container has started before the probe is initiated (default: 0) timeout_seconds: # number of seconds until the probe times out (default: 1) period_seconds: # how often (in seconds) to perform the probe (default: 10) diff --git a/docs/workloads/task/configuration.md b/docs/workloads/task/configuration.md index 6eabb1d7f7..c6d63d5e41 100644 --- a/docs/workloads/task/configuration.md +++ b/docs/workloads/task/configuration.md @@ -7,8 +7,8 @@ containers: # configurations for the containers to run (at least one constainer must be provided) - name: # name of the container (required) image: # docker image to use for the container (required) - command: # entrypoint (required) - args: # arguments to the entrypoint (default: no args) + command: # entrypoint (not executed within a shell); env vars can be used with e.g. $(CORTEX_PORT) (required) + args: # arguments to the entrypoint; env vars can be used with e.g. $(CORTEX_PORT) (default: no args) env: # dictionary of environment variables to set in the container (optional) compute: # compute resource requests (default: see below) cpu: # CPU request for the container; one unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the "m" suffix (default: 200m) @@ -23,7 +23,7 @@ tcp_socket: # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified) port: # the port to access on the container (required) exec: # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified) - command: [] # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) + command: # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) initial_delay_seconds: # number of seconds after the container has started before the probe is initiated (default: 0) timeout_seconds: # number of seconds until the probe times out (default: 1) period_seconds: # how often (in seconds) to perform the probe (default: 10) From 7dadcd05d6a46b96cbb2706b31ba511aa1ab8c7c Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Tue, 1 Jun 2021 00:50:52 +0300 Subject: [PATCH 53/82] CaaS - healthcheck probes for Async/Batch/Realtime (#2206) --- cmd/dequeuer/main.go | 55 ++++++++++- cmd/proxy/main.go | 24 ++++- images/dequeuer/Dockerfile | 1 + pkg/dequeuer/dequeuer.go | 20 ++-- pkg/dequeuer/dequeuer_test.go | 12 ++- .../handler.go => dequeuer/http_handler.go} | 7 +- pkg/dequeuer/probes.go | 56 +++++++++++ pkg/dequeuer/probes_test.go | 98 +++++++++++++++++++ pkg/{proxy/consts.go => probe/handler.go} | 17 ++-- pkg/{proxy => }/probe/handler_test.go | 57 +++++++---- pkg/{proxy => }/probe/probe.go | 86 ++++++++++++++-- pkg/{proxy => }/probe/probe_test.go | 71 ++++++++++++-- pkg/proxy/handler.go | 9 +- pkg/workloads/k8s.go | 17 ++++ .../apis/async/text-generator/cortex_cpu.yaml | 1 - .../apis/async/text-generator/cortex_gpu.yaml | 1 - .../image-classifier-alexnet/cortex_cpu.yaml | 4 + .../image-classifier-alexnet/cortex_gpu.yaml | 4 + .../image-classifier-alexnet-cpu.dockerfile | 1 - .../image-classifier-alexnet-gpu.dockerfile | 1 - .../batch/image-classifier-alexnet/main.py | 15 ++- test/apis/batch/sum/cortex_cpu.yaml | 4 + test/apis/batch/sum/main.py | 13 +-- test/apis/batch/sum/sum-cpu.dockerfile | 1 - 24 files changed, 487 insertions(+), 88 deletions(-) rename pkg/{proxy/probe/handler.go => dequeuer/http_handler.go} (88%) create mode 100644 pkg/dequeuer/probes.go create mode 100644 pkg/dequeuer/probes_test.go rename pkg/{proxy/consts.go => probe/handler.go} (63%) rename pkg/{proxy => }/probe/handler_test.go (75%) rename pkg/{proxy => }/probe/probe.go (60%) rename pkg/{proxy => }/probe/probe_test.go (68%) diff --git a/cmd/dequeuer/main.go b/cmd/dequeuer/main.go index 124f0f465d..85d17c5505 100644 --- a/cmd/dequeuer/main.go +++ b/cmd/dequeuer/main.go @@ -19,6 +19,7 @@ package main import ( "flag" "fmt" + "net/http" "os" "os/signal" "strconv" @@ -28,8 +29,10 @@ import ( "github.com/cortexlabs/cortex/pkg/dequeuer" awslib "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" + "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/logging" "github.com/cortexlabs/cortex/pkg/lib/telemetry" + "github.com/cortexlabs/cortex/pkg/probe" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" "github.com/cortexlabs/cortex/pkg/types/userconfig" "go.uber.org/zap" @@ -39,21 +42,25 @@ func main() { var ( clusterConfigPath string clusterUID string + probesPath string queueURL string userContainerPort int apiName string jobID string statsdPort int apiKind string + adminPort int ) flag.StringVar(&clusterConfigPath, "cluster-config", "", "cluster config path") flag.StringVar(&clusterUID, "cluster-uid", "", "cluster unique identifier") + flag.StringVar(&probesPath, "probes-path", "", "path to the probes spec") flag.StringVar(&queueURL, "queue", "", "target queue URL from which the api messages will be dequeued") flag.StringVar(&apiKind, "api-kind", "", fmt.Sprintf("api kind (%s|%s)", userconfig.BatchAPIKind.String(), userconfig.AsyncAPIKind.String())) flag.StringVar(&apiName, "api-name", "", "api name") flag.StringVar(&jobID, "job-id", "", "job ID") flag.IntVar(&userContainerPort, "user-port", 8080, "target port to which the dequeued messages will be sent to") flag.IntVar(&statsdPort, "statsd-port", 9125, "port for to send udp statsd metrics") + flag.IntVar(&adminPort, "admin-port", 0, "port where the admin server (for the probes) will be exposed") flag.Parse() @@ -72,6 +79,8 @@ func main() { switch { case clusterConfigPath == "": log.Fatal("--cluster-config is a required option") + case probesPath == "": + log.Fatal("--probes-path is a required option") case queueURL == "": log.Fatal("--queue is a required option") case apiName == "": @@ -113,6 +122,18 @@ func main() { } defer telemetry.Close() + var probes []*probe.Probe + if files.IsFile(probesPath) { + probes, err = dequeuer.ProbesFromFile(probesPath, log) + if err != nil { + exit(log, err, fmt.Sprintf("unable to read probes from %s", probesPath)) + } + } + + if !dequeuer.HasTCPProbeTargetingUserPod(probes, userContainerPort) { + probes = append(probes, probe.NewDefaultProbe(fmt.Sprintf("http://localhost:%d", userContainerPort), log)) + } + metricsClient, err := statsd.New(fmt.Sprintf("%s:%d", hostIP, statsdPort)) if err != nil { exit(log, err, "unable to initialize metrics client") @@ -121,6 +142,8 @@ func main() { var dequeuerConfig dequeuer.SQSDequeuerConfig var messageHandler dequeuer.MessageHandler + errCh := make(chan error) + switch apiKind { case userconfig.BatchAPIKind.String(): if jobID == "" { @@ -146,6 +169,9 @@ func main() { if clusterUID == "" { log.Fatal("--cluster-uid is a required option") } + if adminPort == 0 { + log.Fatal("--admin-port is a required option") + } config := dequeuer.AsyncMessageHandlerConfig{ ClusterUID: clusterUID, @@ -160,6 +186,21 @@ func main() { QueueURL: queueURL, StopIfNoMessages: false, } + + adminHandler := http.NewServeMux() + adminHandler.Handle("/healthz", dequeuer.HealthcheckHandler(func() bool { + return probe.AreProbesHealthy(probes) + })) + + go func() { + server := &http.Server{ + Addr: ":" + strconv.Itoa(adminPort), + Handler: adminHandler, + } + log.Infof("Starting %s server on %s", "admin", server.Addr) + errCh <- server.ListenAndServe() + }() + default: exit(log, err, fmt.Sprintf("kind %s is not supported", apiKind)) } @@ -172,15 +213,23 @@ func main() { exit(log, err, "failed to create sqs dequeuer") } - errCh := make(chan error) go func() { log.Info("Starting dequeuer...") - errCh <- sqsDequeuer.Start(messageHandler) + errCh <- sqsDequeuer.Start(messageHandler, func() bool { + return probe.AreProbesHealthy(probes) + }) }() + for _, probe := range probes { + stopper := probe.StartProbing() + defer func() { + stopper <- struct{}{} + }() + } + select { case err = <-errCh: - exit(log, err, "error during message dequeueing") + exit(log, err, "error during message dequeueing or error from admin server") case <-sigint: log.Info("Received TERM signal, handling a graceful shutdown...") sqsDequeuer.Shutdown() diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index e241ba3d2e..06330016db 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -19,6 +19,7 @@ package main import ( "context" "flag" + "net" "net/http" "os" "os/signal" @@ -30,7 +31,6 @@ import ( "github.com/cortexlabs/cortex/pkg/lib/logging" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/proxy" - "github.com/cortexlabs/cortex/pkg/proxy/probe" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" "github.com/cortexlabs/cortex/pkg/types/userconfig" "go.uber.org/zap" @@ -116,7 +116,6 @@ func main() { ) promStats := proxy.NewPrometheusStatsReporter() - readinessProbe := probe.NewDefaultProbe(target, log) go func() { reportTicker := time.NewTicker(_reportInterval) @@ -142,7 +141,7 @@ func main() { adminHandler := http.NewServeMux() adminHandler.Handle("/metrics", promStats) - adminHandler.Handle("/healthz", probe.Handler(readinessProbe)) + adminHandler.Handle("/healthz", readinessTCPHandler(userContainerPort, log)) servers := map[string]*http.Server{ "proxy": { @@ -202,3 +201,22 @@ func exit(log *zap.SugaredLogger, err error, wrapStrs ...string) { telemetry.Close() os.Exit(1) } + +func readinessTCPHandler(port int, logger *zap.SugaredLogger) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + timeout := time.Duration(1) * time.Second + address := net.JoinHostPort("localhost", strconv.FormatInt(int64(port), 10)) + + conn, err := net.DialTimeout("tcp", address, timeout) + _ = conn.Close() + if err != nil { + logger.Warn(err) + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte("unhealthy")) + return + } + + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("healthy")) + } +} diff --git a/images/dequeuer/Dockerfile b/images/dequeuer/Dockerfile index 7e4a59cee5..5c3e7c31f7 100644 --- a/images/dequeuer/Dockerfile +++ b/images/dequeuer/Dockerfile @@ -10,6 +10,7 @@ COPY pkg/config pkg/config COPY pkg/consts pkg/consts COPY pkg/lib pkg/lib COPY pkg/dequeuer pkg/dequeuer +COPY pkg/probe pkg/probe COPY pkg/types pkg/types COPY pkg/crds pkg/crds COPY pkg/workloads pkg/workloads diff --git a/pkg/dequeuer/dequeuer.go b/pkg/dequeuer/dequeuer.go index e448b1cb1c..9584397fc7 100644 --- a/pkg/dequeuer/dequeuer.go +++ b/pkg/dequeuer/dequeuer.go @@ -28,11 +28,12 @@ import ( ) var ( - _messageAttributes = []string{"All"} - _waitTime = 10 * time.Second - _visibilityTimeout = 30 * time.Second - _notFoundSleepTime = 10 * time.Second - _renewalPeriod = 10 * time.Second + _messageAttributes = []string{"All"} + _waitTime = 10 * time.Second + _visibilityTimeout = 30 * time.Second + _notFoundSleepTime = 10 * time.Second + _renewalPeriod = 10 * time.Second + _probeRefreshPeriod = 1 * time.Second ) type SQSDequeuerConfig struct { @@ -49,6 +50,7 @@ type SQSDequeuer struct { visibilityTimeout *int64 notFoundSleepTime time.Duration renewalPeriod time.Duration + probeRefreshPeriod time.Duration log *zap.SugaredLogger done chan struct{} } @@ -67,6 +69,7 @@ func NewSQSDequeuer(config SQSDequeuerConfig, awsClient *awslib.Client, logger * visibilityTimeout: aws.Int64(int64(_visibilityTimeout.Seconds())), notFoundSleepTime: _notFoundSleepTime, renewalPeriod: _renewalPeriod, + probeRefreshPeriod: _probeRefreshPeriod, log: logger, done: make(chan struct{}), }, nil @@ -92,7 +95,7 @@ func (d *SQSDequeuer) ReceiveMessage() (*sqs.Message, error) { return output.Messages[0], nil } -func (d *SQSDequeuer) Start(messageHandler MessageHandler) error { +func (d *SQSDequeuer) Start(messageHandler MessageHandler, readinessProbeFunc func() bool) error { noMessagesInPreviousIteration := false loop: @@ -101,6 +104,11 @@ loop: case <-d.done: break loop default: + if !readinessProbeFunc() { + time.Sleep(d.probeRefreshPeriod) + continue + } + message, err := d.ReceiveMessage() if err != nil { return err diff --git a/pkg/dequeuer/dequeuer_test.go b/pkg/dequeuer/dequeuer_test.go index 6e13fec3c4..59887d6fb6 100644 --- a/pkg/dequeuer/dequeuer_test.go +++ b/pkg/dequeuer/dequeuer_test.go @@ -264,7 +264,9 @@ func TestSQSDequeuerTerminationOnEmptyQueue(t *testing.T) { errCh := make(chan error, 1) go func() { - errCh <- dq.Start(msgHandler) + errCh <- dq.Start(msgHandler, func() bool { + return true + }) }() time.AfterFunc(10*time.Second, func() { errCh <- errors.New("timeout: dequeuer did not finish") }) @@ -299,7 +301,9 @@ func TestSQSDequeuer_Shutdown(t *testing.T) { errCh := make(chan error, 1) go func() { - errCh <- dq.Start(msgHandler) + errCh <- dq.Start(msgHandler, func() bool { + return true + }) }() time.AfterFunc(5*time.Second, func() { errCh <- errors.New("timeout: dequeuer did not exit") }) @@ -346,7 +350,9 @@ func TestSQSDequeuer_Start_HandlerError(t *testing.T) { }) require.NoError(t, err) - err = dq.Start(msgHandler) + err = dq.Start(msgHandler, func() bool { + return true + }) require.NoError(t, err) require.Never(t, func() bool { diff --git a/pkg/proxy/probe/handler.go b/pkg/dequeuer/http_handler.go similarity index 88% rename from pkg/proxy/probe/handler.go rename to pkg/dequeuer/http_handler.go index 55c89b5c58..a5c8aa0059 100644 --- a/pkg/proxy/probe/handler.go +++ b/pkg/dequeuer/http_handler.go @@ -14,14 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package probe +package dequeuer import "net/http" -func Handler(pb *Probe) http.HandlerFunc { +func HealthcheckHandler(isHealthy func() bool) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - healthy := pb.ProbeContainer() - if !healthy { + if !isHealthy() { w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte("unhealthy")) return diff --git a/pkg/dequeuer/probes.go b/pkg/dequeuer/probes.go new file mode 100644 index 0000000000..909c54cc91 --- /dev/null +++ b/pkg/dequeuer/probes.go @@ -0,0 +1,56 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dequeuer + +import ( + "github.com/cortexlabs/cortex/pkg/lib/files" + libjson "github.com/cortexlabs/cortex/pkg/lib/json" + "github.com/cortexlabs/cortex/pkg/probe" + "go.uber.org/zap" + kcore "k8s.io/api/core/v1" +) + +func ProbesFromFile(probesPath string, logger *zap.SugaredLogger) ([]*probe.Probe, error) { + fileBytes, err := files.ReadFileBytes(probesPath) + if err != nil { + return nil, err + } + + probesMap := map[string]kcore.Probe{} + if err := libjson.Unmarshal(fileBytes, &probesMap); err != nil { + return nil, err + } + + probesSlice := []*probe.Probe{} + for _, p := range probesMap { + auxProbe := p + probesSlice = append(probesSlice, probe.NewProbe(&auxProbe, logger)) + } + return probesSlice, nil +} + +func HasTCPProbeTargetingUserPod(probes []*probe.Probe, userPort int) bool { + for _, probe := range probes { + if probe == nil { + continue + } + if probe.Handler.TCPSocket != nil && probe.Handler.TCPSocket.Port.IntValue() == userPort { + return true + } + } + return false +} diff --git a/pkg/dequeuer/probes_test.go b/pkg/dequeuer/probes_test.go new file mode 100644 index 0000000000..892b64c928 --- /dev/null +++ b/pkg/dequeuer/probes_test.go @@ -0,0 +1,98 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dequeuer + +import ( + "testing" + + "github.com/cortexlabs/cortex/pkg/probe" + "github.com/stretchr/testify/require" + kcore "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func TestDefaultTCPProbeNotPresent(t *testing.T) { + t.Parallel() + log := newLogger(t) + + userPodPort := 8080 + + probes := []*probe.Probe{ + probe.NewProbe(&kcore.Probe{ + Handler: kcore.Handler{ + Exec: &kcore.ExecAction{ + Command: []string{"/bin/bash", "python", "test.py"}, + }, + }, + }, log), + probe.NewProbe(&kcore.Probe{ + Handler: kcore.Handler{ + HTTPGet: &kcore.HTTPGetAction{ + Path: "some-path", + Port: intstr.FromInt(12345), + Host: "localhost", + }, + }, + }, log), + probe.NewProbe(&kcore.Probe{ + Handler: kcore.Handler{ + TCPSocket: &kcore.TCPSocketAction{ + Port: intstr.FromInt(8447), + Host: "localhost", + }, + }, + }, log), + } + + require.False(t, HasTCPProbeTargetingUserPod(probes, userPodPort)) +} + +func TestDefaultTCPProbePresent(t *testing.T) { + t.Parallel() + log := newLogger(t) + + userPodPort := intstr.FromInt(8080) + + probes := []*probe.Probe{ + probe.NewProbe(&kcore.Probe{ + Handler: kcore.Handler{ + Exec: &kcore.ExecAction{ + Command: []string{"/bin/bash", "python", "test.py"}, + }, + }, + }, log), + probe.NewProbe(&kcore.Probe{ + Handler: kcore.Handler{ + HTTPGet: &kcore.HTTPGetAction{ + Path: "some-path", + Port: intstr.FromInt(12345), + Host: "localhost", + }, + }, + }, log), + probe.NewProbe(&kcore.Probe{ + Handler: kcore.Handler{ + TCPSocket: &kcore.TCPSocketAction{ + Port: userPodPort, + Host: "localhost", + }, + }, + }, log), + } + + require.True(t, HasTCPProbeTargetingUserPod(probes, userPodPort.IntValue())) +} diff --git a/pkg/proxy/consts.go b/pkg/probe/handler.go similarity index 63% rename from pkg/proxy/consts.go rename to pkg/probe/handler.go index 67bb86f7fc..80c19ac474 100644 --- a/pkg/proxy/consts.go +++ b/pkg/probe/handler.go @@ -14,14 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package proxy +package probe -const ( - // UserAgentKey is the user agent header key - UserAgentKey = "User-Agent" - - // KubeProbeUserAgentPrefix is the user agent header prefix used in k8s probes - // Since K8s 1.8, prober requests have - // User-Agent = "kube-probe/{major-version}.{minor-version}". - KubeProbeUserAgentPrefix = "kube-probe/" +import ( + "net/http" + "strings" ) + +func IsRequestKubeletProbe(r *http.Request) bool { + return strings.HasPrefix(r.Header.Get(_userAgentKey), _kubeProbeUserAgentPrefix) +} diff --git a/pkg/proxy/probe/handler_test.go b/pkg/probe/handler_test.go similarity index 75% rename from pkg/proxy/probe/handler_test.go rename to pkg/probe/handler_test.go index da2db383f9..d288f8f136 100644 --- a/pkg/proxy/probe/handler_test.go +++ b/pkg/probe/handler_test.go @@ -21,28 +21,25 @@ import ( "net/http/httptest" "net/url" "testing" + "time" - "github.com/cortexlabs/cortex/pkg/proxy" - "github.com/cortexlabs/cortex/pkg/proxy/probe" + "github.com/cortexlabs/cortex/pkg/probe" "github.com/stretchr/testify/require" kcore "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" ) -func TestHandlerFailure(t *testing.T) { - t.Parallel() - log := newLogger(t) - - pb := probe.NewDefaultProbe("http://127.0.0.1:12345", log) - handler := probe.Handler(pb) - - r := httptest.NewRequest(http.MethodGet, "http://fake.cortex.dev/healthz", nil) - w := httptest.NewRecorder() - - handler(w, r) +func generateHandler(pb *probe.Probe) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if !pb.IsHealthy() { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte("unhealthy")) + return + } - require.Equal(t, http.StatusInternalServerError, w.Code) - require.Equal(t, "unhealthy", w.Body.String()) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("healthy")) + } } func TestHandlerSuccessTCP(t *testing.T) { @@ -55,11 +52,23 @@ func TestHandlerSuccessTCP(t *testing.T) { server := httptest.NewServer(userHandler) pb := probe.NewDefaultProbe(server.URL, log) - handler := probe.Handler(pb) + handler := generateHandler(pb) r := httptest.NewRequest(http.MethodGet, "http://fake.cortex.dev/healthz", nil) w := httptest.NewRecorder() + stopper := pb.StartProbing() + defer func() { + stopper <- struct{}{} + }() + + for { + if pb.HasRunOnce() { + break + } + time.Sleep(time.Second) + } + handler(w, r) require.Equal(t, http.StatusOK, w.Code) @@ -78,7 +87,7 @@ func TestHandlerSuccessHTTP(t *testing.T) { } var userHandler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - require.Contains(t, r.Header.Get(proxy.UserAgentKey), proxy.KubeProbeUserAgentPrefix) + require.True(t, probe.IsRequestKubeletProbe(r)) for _, header := range headers { require.Equal(t, header.Value, r.Header.Get(header.Name)) } @@ -105,11 +114,23 @@ func TestHandlerSuccessHTTP(t *testing.T) { FailureThreshold: 3, }, log, ) - handler := probe.Handler(pb) + handler := generateHandler(pb) r := httptest.NewRequest(http.MethodGet, "http://fake.cortex.dev/healthz", nil) w := httptest.NewRecorder() + stopper := pb.StartProbing() + defer func() { + stopper <- struct{}{} + }() + + for { + if pb.HasRunOnce() { + break + } + time.Sleep(time.Second) + } + handler(w, r) require.Equal(t, http.StatusOK, w.Code) diff --git a/pkg/proxy/probe/probe.go b/pkg/probe/probe.go similarity index 60% rename from pkg/proxy/probe/probe.go rename to pkg/probe/probe.go index 3eeee7a9f9..94a554a71e 100644 --- a/pkg/proxy/probe/probe.go +++ b/pkg/probe/probe.go @@ -26,19 +26,34 @@ import ( "time" s "github.com/cortexlabs/cortex/pkg/lib/strings" - "github.com/cortexlabs/cortex/pkg/proxy" "go.uber.org/zap" kcore "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" ) +var ( + // userAgentKey is the user agent header key + _userAgentKey = "User-Agent" + + // kubeProbeUserAgentPrefix is the user agent header prefix used in k8s probes + // Since K8s 1.8, prober requests have + // User-Agent = "kube-probe/{major-version}.{minor-version}". + _kubeProbeUserAgentPrefix = "kube-probe/" +) + const ( - _defaultTimeoutSeconds = 1 + _defaultInitialDelaySeconds = int32(1) + _defaultTimeoutSeconds = int32(1) + _defaultPeriodSeconds = int32(1) + _defaultSuccessThreshold = int32(1) + _defaultFailureThreshold = int32(1) ) type Probe struct { *kcore.Probe - logger *zap.SugaredLogger + logger *zap.SugaredLogger + healthy bool + hasRunOnce bool } func NewProbe(probe *kcore.Probe, logger *zap.SugaredLogger) *Probe { @@ -62,13 +77,72 @@ func NewDefaultProbe(target string, logger *zap.SugaredLogger) *Probe { Host: targetURL.Hostname(), }, }, - TimeoutSeconds: _defaultTimeoutSeconds, + InitialDelaySeconds: _defaultInitialDelaySeconds, + TimeoutSeconds: _defaultTimeoutSeconds, + PeriodSeconds: _defaultPeriodSeconds, + SuccessThreshold: _defaultSuccessThreshold, + FailureThreshold: _defaultFailureThreshold, }, logger: logger, } } -func (p *Probe) ProbeContainer() bool { +func (p *Probe) StartProbing() chan struct{} { + stop := make(chan struct{}) + + ticker := time.NewTicker(time.Duration(p.PeriodSeconds) * time.Second) + time.AfterFunc(time.Duration(p.InitialDelaySeconds)*time.Second, func() { + successCount := int32(0) + failureCount := int32(0) + + for { + select { + case <-stop: + return + case <-ticker.C: + healthy := p.probeContainer() + if healthy { + successCount++ + failureCount = 0 + } else { + failureCount++ + successCount = 0 + } + + if successCount >= p.SuccessThreshold { + p.healthy = true + } else if failureCount >= p.FailureThreshold { + p.healthy = false + } + p.hasRunOnce = true + } + } + }) + + return stop +} + +func (p *Probe) IsHealthy() bool { + return p.healthy +} + +func (p *Probe) HasRunOnce() bool { + return p.hasRunOnce +} + +func AreProbesHealthy(probes []*Probe) bool { + for _, probe := range probes { + if probe == nil { + continue + } + if !probe.IsHealthy() { + return false + } + } + return true +} + +func (p *Probe) probeContainer() bool { var err error switch { @@ -104,7 +178,7 @@ func (p *Probe) httpProbe() error { return err } - req.Header.Add(proxy.UserAgentKey, proxy.KubeProbeUserAgentPrefix) + req.Header.Add(_userAgentKey, _kubeProbeUserAgentPrefix) for _, header := range p.HTTPGet.HTTPHeaders { req.Header.Add(header.Name, header.Value) diff --git a/pkg/proxy/probe/probe_test.go b/pkg/probe/probe_test.go similarity index 68% rename from pkg/proxy/probe/probe_test.go rename to pkg/probe/probe_test.go index b0518396f3..c33b7bd0a8 100644 --- a/pkg/proxy/probe/probe_test.go +++ b/pkg/probe/probe_test.go @@ -21,8 +21,9 @@ import ( "net/http/httptest" "net/url" "testing" + "time" - "github.com/cortexlabs/cortex/pkg/proxy/probe" + "github.com/cortexlabs/cortex/pkg/probe" "github.com/stretchr/testify/require" "go.uber.org/zap" kcore "k8s.io/api/core/v1" @@ -52,7 +53,19 @@ func TestDefaultProbeSuccess(t *testing.T) { server := httptest.NewServer(handler) pb := probe.NewDefaultProbe(server.URL, log) - require.True(t, pb.ProbeContainer()) + stopper := pb.StartProbing() + defer func() { + stopper <- struct{}{} + }() + + for { + if pb.HasRunOnce() { + break + } + time.Sleep(time.Second) + } + + require.True(t, pb.IsHealthy()) } func TestDefaultProbeFailure(t *testing.T) { @@ -62,7 +75,19 @@ func TestDefaultProbeFailure(t *testing.T) { target := "http://127.0.0.1:12345" pb := probe.NewDefaultProbe(target, log) - require.False(t, pb.ProbeContainer()) + stopper := pb.StartProbing() + defer func() { + stopper <- struct{}{} + }() + + for { + if pb.HasRunOnce() { + break + } + time.Sleep(time.Second) + } + + require.False(t, pb.IsHealthy()) } func TestProbeHTTPFailure(t *testing.T) { @@ -78,11 +103,27 @@ func TestProbeHTTPFailure(t *testing.T) { Host: "127.0.0.1", }, }, - TimeoutSeconds: 3, + InitialDelaySeconds: 1, + TimeoutSeconds: 3, + PeriodSeconds: 1, + SuccessThreshold: 1, + FailureThreshold: 1, }, log, ) - require.False(t, pb.ProbeContainer()) + stopper := pb.StartProbing() + defer func() { + stopper <- struct{}{} + }() + + for { + if pb.HasRunOnce() { + break + } + time.Sleep(time.Second) + } + + require.False(t, pb.IsHealthy()) } func TestProbeHTTPSuccess(t *testing.T) { @@ -105,9 +146,25 @@ func TestProbeHTTPSuccess(t *testing.T) { Host: targetURL.Hostname(), }, }, - TimeoutSeconds: 3, + InitialDelaySeconds: 1, + TimeoutSeconds: 3, + PeriodSeconds: 1, + SuccessThreshold: 1, + FailureThreshold: 1, }, log, ) - require.True(t, pb.ProbeContainer()) + stopper := pb.StartProbing() + defer func() { + stopper <- struct{}{} + }() + + for { + if pb.HasRunOnce() { + break + } + time.Sleep(time.Second) + } + + require.True(t, pb.IsHealthy()) } diff --git a/pkg/proxy/handler.go b/pkg/proxy/handler.go index 39ba5f0b6f..824d3cf7e8 100644 --- a/pkg/proxy/handler.go +++ b/pkg/proxy/handler.go @@ -20,12 +20,13 @@ import ( "context" "errors" "net/http" - "strings" + + "github.com/cortexlabs/cortex/pkg/probe" ) func Handler(breaker *Breaker, next http.Handler) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - if isKubeletProbe(r) || breaker == nil { + if probe.IsRequestKubeletProbe(r) || breaker == nil { next.ServeHTTP(w, r) return } @@ -41,7 +42,3 @@ func Handler(breaker *Breaker, next http.Handler) http.HandlerFunc { } } } - -func isKubeletProbe(r *http.Request) bool { - return strings.HasPrefix(r.Header.Get(UserAgentKey), KubeProbeUserAgentPrefix) -} diff --git a/pkg/workloads/k8s.go b/pkg/workloads/k8s.go index 2739f38eed..74e528adc4 100644 --- a/pkg/workloads/k8s.go +++ b/pkg/workloads/k8s.go @@ -17,6 +17,7 @@ limitations under the License. package workloads import ( + "path" "strings" "github.com/cortexlabs/cortex/pkg/config" @@ -126,11 +127,13 @@ func asyncDequeuerProxyContainer(api spec.API, queueURL string) (kcore.Container Args: []string{ "--cluster-config", consts.DefaultInClusterConfigPath, "--cluster-uid", config.ClusterConfig.ClusterUID, + "--probes-path", path.Join(_cortexDirMountPath, "spec", "probes.json"), "--queue", queueURL, "--api-kind", api.Kind.String(), "--api-name", api.Name, "--user-port", s.Int32(*api.Pod.Port), "--statsd-port", consts.StatsDPortStr, + "--admin-port", consts.AdminPortStr, }, Env: append(baseEnvVars, kcore.EnvVar{ Name: "HOST_IP", @@ -140,6 +143,19 @@ func asyncDequeuerProxyContainer(api spec.API, queueURL string) (kcore.Container }, }, }), + ReadinessProbe: &kcore.Probe{ + Handler: kcore.Handler{ + HTTPGet: &kcore.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(int(consts.AdminPortInt32)), + }, + }, + InitialDelaySeconds: 1, + TimeoutSeconds: 1, + PeriodSeconds: 5, + SuccessThreshold: 1, + FailureThreshold: 1, + }, VolumeMounts: []kcore.VolumeMount{ ClusterConfigMount(), }, @@ -157,6 +173,7 @@ func batchDequeuerProxyContainer(api spec.API, jobID, queueURL string) (kcore.Co Args: []string{ "--cluster-config", consts.DefaultInClusterConfigPath, "--cluster-uid", config.ClusterConfig.ClusterUID, + "--probes-path", path.Join(_cortexDirMountPath, "spec", "probes.json"), "--queue", queueURL, "--api-kind", api.Kind.String(), "--api-name", api.Name, diff --git a/test/apis/async/text-generator/cortex_cpu.yaml b/test/apis/async/text-generator/cortex_cpu.yaml index 0863c54775..4e59b10efe 100644 --- a/test/apis/async/text-generator/cortex_cpu.yaml +++ b/test/apis/async/text-generator/cortex_cpu.yaml @@ -2,7 +2,6 @@ kind: AsyncAPI pod: port: 9000 - max_concurrency: 1 containers: - name: api image: quay.io/cortexlabs-test/async-text-generator-cpu:latest diff --git a/test/apis/async/text-generator/cortex_gpu.yaml b/test/apis/async/text-generator/cortex_gpu.yaml index 3d0ed0c94c..6706058859 100644 --- a/test/apis/async/text-generator/cortex_gpu.yaml +++ b/test/apis/async/text-generator/cortex_gpu.yaml @@ -2,7 +2,6 @@ kind: AsyncAPI pod: port: 9000 - max_concurrency: 1 containers: - name: api image: quay.io/cortexlabs-test/async-text-generator-gpu:latest diff --git a/test/apis/batch/image-classifier-alexnet/cortex_cpu.yaml b/test/apis/batch/image-classifier-alexnet/cortex_cpu.yaml index 5891c01510..3702ba8a36 100644 --- a/test/apis/batch/image-classifier-alexnet/cortex_cpu.yaml +++ b/test/apis/batch/image-classifier-alexnet/cortex_cpu.yaml @@ -15,6 +15,10 @@ - "--bind" - ":$(CORTEX_PORT)" - "main:app" + readiness_probe: + http_get: + path: "/healthz" + port: 8080 compute: cpu: 1 mem: 2G diff --git a/test/apis/batch/image-classifier-alexnet/cortex_gpu.yaml b/test/apis/batch/image-classifier-alexnet/cortex_gpu.yaml index b5748f0c3c..ec1b10693b 100644 --- a/test/apis/batch/image-classifier-alexnet/cortex_gpu.yaml +++ b/test/apis/batch/image-classifier-alexnet/cortex_gpu.yaml @@ -15,6 +15,10 @@ - "--bind" - ":$(CORTEX_PORT)" - "main:app" + readiness_probe: + http_get: + path: "/healthz" + port: 8080 compute: cpu: 200m gpu: 1 diff --git a/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-cpu.dockerfile b/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-cpu.dockerfile index cee0dd0d34..d63d8f72ad 100644 --- a/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-cpu.dockerfile +++ b/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-cpu.dockerfile @@ -8,7 +8,6 @@ RUN pip install --no-cache-dir \ "uvicorn[standard]" \ gunicorn \ fastapi \ - pydantic \ requests \ torchvision \ torch \ diff --git a/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-gpu.dockerfile b/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-gpu.dockerfile index 442fdc22e3..d339842e9d 100644 --- a/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-gpu.dockerfile +++ b/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-gpu.dockerfile @@ -18,7 +18,6 @@ RUN pip3 install --no-cache-dir \ "uvicorn[standard]" \ gunicorn \ fastapi \ - pydantic \ requests \ torchvision \ torch \ diff --git a/test/apis/batch/image-classifier-alexnet/main.py b/test/apis/batch/image-classifier-alexnet/main.py index 45fa3ad2f1..d899d88a5f 100644 --- a/test/apis/batch/image-classifier-alexnet/main.py +++ b/test/apis/batch/image-classifier-alexnet/main.py @@ -1,8 +1,7 @@ import os, json, re -from typing import Any, List +from typing import List -from fastapi import FastAPI, Response, status -from pydantic import BaseModel +from fastapi import FastAPI, Response, Request, status import requests import torch @@ -13,10 +12,6 @@ import boto3 -class Request(BaseModel): - payload: List[Any] - - state = { "ready": False, "model": None, @@ -71,8 +66,8 @@ def healthz(response: Response): @app.post("/") -def handle_batch(request: Request): - payload = request.payload +async def handle_batch(request: Request): + payload: List[str] = await request.json() job_id = state["job_id"] tensor_list = [] @@ -113,6 +108,8 @@ def on_job_complete(): # aggregate all classifications paginator = s3.get_paginator("list_objects_v2") for page in paginator.paginate(Bucket=state["bucket"], Prefix=state["key"]): + if "Contents" not in page: + continue for obj in page["Contents"]: body = s3.get_object(Bucket=state["bucket"], Key=obj["Key"])["Body"] all_results += json.loads(body.read().decode("utf8")) diff --git a/test/apis/batch/sum/cortex_cpu.yaml b/test/apis/batch/sum/cortex_cpu.yaml index 5c3dafde3c..267f50e74e 100644 --- a/test/apis/batch/sum/cortex_cpu.yaml +++ b/test/apis/batch/sum/cortex_cpu.yaml @@ -17,6 +17,10 @@ - "--bind" - ":$(CORTEX_PORT)" - "main:app" + readiness_probe: + http_get: + path: "/healthz" + port: 8080 compute: cpu: 200m mem: 256Mi diff --git a/test/apis/batch/sum/main.py b/test/apis/batch/sum/main.py index 7c82267d88..d3a732e28f 100644 --- a/test/apis/batch/sum/main.py +++ b/test/apis/batch/sum/main.py @@ -4,13 +4,7 @@ import re from typing import List -from pydantic import BaseModel -from fastapi import FastAPI, Response, status - - -class Request(BaseModel): - payload: List[List[int]] - +from fastapi import FastAPI, Response, Request, status state = { "ready": False, @@ -51,9 +45,10 @@ def healthz(response: Response): @app.post("/") -def handle_batch(request: Request): +async def handle_batch(request: Request): global state - for numbers_list in request.payload: + payload: List[List[int]] = await request.json() + for numbers_list in payload: state["numbers_list"].append(sum(numbers_list)) diff --git a/test/apis/batch/sum/sum-cpu.dockerfile b/test/apis/batch/sum/sum-cpu.dockerfile index 264ae82b32..f7b6b29763 100644 --- a/test/apis/batch/sum/sum-cpu.dockerfile +++ b/test/apis/batch/sum/sum-cpu.dockerfile @@ -8,7 +8,6 @@ RUN pip install --no-cache-dir \ "uvicorn[standard]" \ gunicorn \ fastapi \ - pydantic \ boto3==1.17.72 # Copy local code to the container image. From 2b4badde6dba6e163b303e73b060460924d106ca Mon Sep 17 00:00:00 2001 From: Vishal Bollu Date: Tue, 1 Jun 2021 00:57:37 -0400 Subject: [PATCH 54/82] Add cortex prefix to labels in fluent bit (#2197) --- docs/clusters/observability/logging.md | 28 ++++++++++++------- manager/manifests/fluent-bit.yaml.j2 | 12 ++++++-- .../controllers/batch/batchjob_controller.go | 27 ++++++++++++------ pkg/lib/logging/logging.go | 2 +- 4 files changed, 46 insertions(+), 23 deletions(-) diff --git a/docs/clusters/observability/logging.md b/docs/clusters/observability/logging.md index 1e484649c4..d8cfe5ffc9 100644 --- a/docs/clusters/observability/logging.md +++ b/docs/clusters/observability/logging.md @@ -27,8 +27,8 @@ Below are some sample CloudWatch Log Insight queries: ```text fields @timestamp, message -| filter labels.apiName="" -| filter labels.apiKind="RealtimeAPI" +| filter cortex.labels.apiName="" +| filter cortex.labels.apiKind="RealtimeAPI" | sort @timestamp asc | limit 1000 ``` @@ -37,8 +37,8 @@ fields @timestamp, message ```text fields @timestamp, message -| filter labels.apiName="" -| filter labels.apiKind="AsyncAPI" +| filter cortex.labels.apiName="" +| filter cortex.labels.apiKind="AsyncAPI" | sort @timestamp asc | limit 1000 ``` @@ -47,9 +47,9 @@ fields @timestamp, message ```text fields @timestamp, message -| filter labels.apiName="" -| filter labels.jobID="" -| filter labels.apiKind="BatchAPI" +| filter cortex.labels.apiName="" +| filter cortex.labels.jobID="" +| filter cortex.labels.apiKind="BatchAPI" | sort @timestamp asc | limit 1000 ``` @@ -58,9 +58,17 @@ fields @timestamp, message ```text fields @timestamp, message -| filter labels.apiName="" -| filter labels.jobID="" -| filter labels.apiKind="TaskAPI" +| filter cortex.labels.apiName="" +| filter cortex.labels.jobID="" +| filter cortex.labels.apiKind="TaskAPI" | sort @timestamp asc | limit 1000 ``` + +## Structured logging + +If you log JSON strings from your APIs, they will be automatically parsed before pushing to CloudWatch. + +It is recommended to configure your JSON logger to use `message` or `msg` as the key for the log line if you would like the sample queries above to display the messages in your logs. + +Avoid using top-level keys that start with "cortex" to prevent collisions with Cortex's internal logging. diff --git a/manager/manifests/fluent-bit.yaml.j2 b/manager/manifests/fluent-bit.yaml.j2 index 7f586248a4..41a0ae87e6 100644 --- a/manager/manifests/fluent-bit.yaml.j2 +++ b/manager/manifests/fluent-bit.yaml.j2 @@ -100,6 +100,12 @@ data: Condition Key_Exists message Hard_rename message log + [FILTER] + Name modify + Match k8s_container.* + Condition Key_Exists msg + Hard_rename msg log + [FILTER] Name nest Match k8s_container.* @@ -110,8 +116,8 @@ data: [FILTER] Name modify Match k8s_container.* - Condition Key_Does_Not_Exist labels - Rename k8s.labels labels + Condition Key_Does_Not_Exist cortex.labels + Rename k8s.labels cortex.labels [FILTER] Name modify @@ -136,7 +142,7 @@ data: Name modify Match k8s_container.*.event-exporter-* Condition Key_exists involvedObject.labels - Hard_copy involvedObject.labels labels + Hard_copy involvedObject.labels cortex.labels [FILTER] Name nest diff --git a/pkg/crds/controllers/batch/batchjob_controller.go b/pkg/crds/controllers/batch/batchjob_controller.go index edd656c523..dd15210b31 100644 --- a/pkg/crds/controllers/batch/batchjob_controller.go +++ b/pkg/crds/controllers/batch/batchjob_controller.go @@ -62,9 +62,11 @@ type BatchJobReconciler struct { // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. func (r *BatchJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - log := r.Log.WithValues( - "batchjob", req.NamespacedName, - "apiKind", userconfig.BatchAPIKind.String(), + log := r.Log.WithValues("cortex.labels", + map[string]string{ + "batchjob": req.NamespacedName.String(), + "apiKind": userconfig.BatchAPIKind.String(), + }, ) // Step 1: get resource from request @@ -77,8 +79,15 @@ func (r *BatchJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c return ctrl.Result{}, client.IgnoreNotFound(err) } - log = log.WithValues("apiName", batchJob.Spec.APIName, "apiID", batchJob.Spec.APIID, "jobID", batchJob.Name) - + log = r.Log.WithValues("cortex.labels", + map[string]string{ + "batchjob": req.NamespacedName.String(), + "apiKind": userconfig.BatchAPIKind.String(), + "apiName": batchJob.Spec.APIName, + "apiID": batchJob.Spec.APIID, + "jobID": batchJob.Name, + }, + ) // Step 2: create finalizer or handle deletion if batchJob.ObjectMeta.DeletionTimestamp.IsZero() { // The object is not being deleted, so we add our finalizer if it does not exist yet, @@ -216,7 +225,7 @@ func (r *BatchJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c // Step 5: Create resources var queueURL string if !queueExists { - log.V(1).Info("creating queue") + log.Info("creating queue") queueURL, err = r.createQueue(batchJob) if err != nil { log.Error(err, "failed to create queue") @@ -228,7 +237,7 @@ func (r *BatchJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c switch enqueuingStatus { case batch.EnqueuingNotStarted: - log.V(1).Info("enqueuing payload") + log.Info("enqueuing payload") if err = r.enqueuePayload(ctx, batchJob, queueURL); err != nil { log.Error(err, "failed to start enqueuing the payload") return ctrl.Result{}, err @@ -249,7 +258,7 @@ func (r *BatchJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c } if !workerJobExists { - log.V(1).Info("creating worker job") + log.Info("creating worker job") if err = r.createWorkerJob(ctx, batchJob, queueURL); err != nil { log.Error(err, "failed to create worker job") return ctrl.Result{}, err @@ -264,7 +273,7 @@ func (r *BatchJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c if err = r.Delete(ctx, &batchJob); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } - log.V(1).Info("TTL exceeded, deleting resource") + log.Info("TTL exceeded, deleting resource") return ctrl.Result{}, nil } log.V(1).Info("scheduling reconciliation requeue", "time", batchJob.Spec.TTL.Duration) diff --git a/pkg/lib/logging/logging.go b/pkg/lib/logging/logging.go index c4ec6bddb4..e722e7c3cc 100644 --- a/pkg/lib/logging/logging.go +++ b/pkg/lib/logging/logging.go @@ -77,7 +77,7 @@ func DefaultZapConfig(level userconfig.LogLevel, fields ...map[string]interface{ initialFields := map[string]interface{}{} if len(labels) > 0 { - initialFields["labels"] = labels + initialFields["cortex.labels"] = labels } return zap.Config{ From 8b271817186ab3a0342d426b20183653df221022 Mon Sep 17 00:00:00 2001 From: Vishal Bollu Date: Tue, 1 Jun 2021 10:56:39 -0400 Subject: [PATCH 55/82] Fix statsd metrics exporting to prometheus (#2207) --- manager/manifests/prometheus-monitoring.yaml.j2 | 2 +- pkg/dequeuer/batch_handler.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/manager/manifests/prometheus-monitoring.yaml.j2 b/manager/manifests/prometheus-monitoring.yaml.j2 index d7868c1755..cbc8af579d 100644 --- a/manager/manifests/prometheus-monitoring.yaml.j2 +++ b/manager/manifests/prometheus-monitoring.yaml.j2 @@ -221,7 +221,7 @@ metadata: spec: jobLabel: "statsd-exporter" podMetricsEndpoints: - - port: admin + - port: metrics scheme: http path: /metrics interval: 20s diff --git a/pkg/dequeuer/batch_handler.go b/pkg/dequeuer/batch_handler.go index 2fc94f6f7c..150e88a322 100644 --- a/pkg/dequeuer/batch_handler.go +++ b/pkg/dequeuer/batch_handler.go @@ -58,7 +58,7 @@ type BatchMessageHandlerConfig struct { func NewBatchMessageHandler(config BatchMessageHandlerConfig, awsClient *awslib.Client, statsdClient statsd.ClientInterface, log *zap.SugaredLogger) *BatchMessageHandler { tags := []string{ "api_name:" + config.APIName, - "job_id" + config.JobID, + "job_id:" + config.JobID, } return &BatchMessageHandler{ From 750e7a0aa0346281464ef3a51bff3aff277b9fdb Mon Sep 17 00:00:00 2001 From: Vishal Bollu Date: Tue, 1 Jun 2021 17:47:11 -0400 Subject: [PATCH 56/82] Display cloudwatch url as output for cortex logs (#2208) --- .github/ISSUE_TEMPLATE/bug-report.md | 2 +- cli/cluster/logs.go | 32 ++++- cli/cmd/deploy.go | 2 +- cli/cmd/logs.go | 56 ++++---- cmd/operator/main.go | 3 +- docs/clients/cli.md | 3 +- docs/clients/python.md | 31 +---- docs/clusters/management/environments.md | 4 - docs/clusters/observability/logging.md | 31 ++--- docs/workloads/async/example.md | 5 +- docs/workloads/batch/example.md | 2 +- docs/workloads/realtime/example.md | 2 +- docs/workloads/realtime/troubleshooting.md | 8 +- pkg/operator/endpoints/errors.go | 2 +- pkg/operator/endpoints/logs.go | 57 ++++++++ pkg/operator/endpoints/logs_job.go | 60 +++++++++ pkg/operator/operator/workload_logging.go | 150 +++++++++++++++++++++ pkg/operator/schema/schema.go | 4 + python/client/cortex/binary/__init__.py | 5 - python/client/cortex/client.py | 69 ++-------- test/e2e/e2e/tests.py | 18 +-- test/e2e/e2e/utils.py | 8 ++ 22 files changed, 389 insertions(+), 165 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index aff8014321..2fd7bca1b2 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -39,7 +39,7 @@ assignees: '' ### Stack traces -(error output from `cortex logs `) +(error output from CloudWatch Insights or from a random pod `cortex logs `) ```text diff --git a/cli/cluster/logs.go b/cli/cluster/logs.go index 99b9f61552..8218c16961 100644 --- a/cli/cluster/logs.go +++ b/cli/cluster/logs.go @@ -35,12 +35,40 @@ import ( "github.com/gorilla/websocket" ) +func GetLogs(operatorConfig OperatorConfig, apiName string) (schema.LogResponse, error) { + httpRes, err := HTTPGet(operatorConfig, "/logs/"+apiName) + if err != nil { + return schema.LogResponse{}, err + } + + var logResponse schema.LogResponse + if err = json.Unmarshal(httpRes, &logResponse); err != nil { + return schema.LogResponse{}, errors.Wrap(err, "/logs/"+apiName, string(httpRes)) + } + + return logResponse, nil +} + +func GetJobLogs(operatorConfig OperatorConfig, apiName string, jobID string) (schema.LogResponse, error) { + httpRes, err := HTTPGet(operatorConfig, "/logs/"+apiName, map[string]string{"jobID": jobID}) + if err != nil { + return schema.LogResponse{}, err + } + + var logResponse schema.LogResponse + if err = json.Unmarshal(httpRes, &logResponse); err != nil { + return schema.LogResponse{}, errors.Wrap(err, "/logs/"+apiName, string(httpRes)) + } + + return logResponse, nil +} + func StreamLogs(operatorConfig OperatorConfig, apiName string) error { - return streamLogs(operatorConfig, "/logs/"+apiName) + return streamLogs(operatorConfig, "/streamlogs/"+apiName) } func StreamJobLogs(operatorConfig OperatorConfig, apiName string, jobID string) error { - return streamLogs(operatorConfig, "/logs/"+apiName, map[string]string{"jobID": jobID}) + return streamLogs(operatorConfig, "/streamlogs/"+apiName, map[string]string{"jobID": jobID}) } func streamLogs(operatorConfig OperatorConfig, path string, qParams ...map[string]string) error { diff --git a/cli/cmd/deploy.go b/cli/cmd/deploy.go index 9f17140da1..f70ea3b77b 100644 --- a/cli/cmd/deploy.go +++ b/cli/cmd/deploy.go @@ -240,7 +240,7 @@ func getAPICommandsMessage(results []schema.DeployResult, envName string) (strin continue } if result.API != nil && result.API.Spec.Kind == userconfig.RealtimeAPIKind { - items.Add(fmt.Sprintf("cortex logs %s%s", apiName, envArg), "(stream api logs)") + items.Add(fmt.Sprintf("cortex logs %s%s", apiName, envArg), "(access logs)") break } } diff --git a/cli/cmd/logs.go b/cli/cmd/logs.go index 9da9cf95b1..f7fd6e1286 100644 --- a/cli/cmd/logs.go +++ b/cli/cmd/logs.go @@ -17,28 +17,36 @@ limitations under the License. package cmd import ( + "fmt" + "github.com/cortexlabs/cortex/cli/cluster" "github.com/cortexlabs/cortex/pkg/lib/exit" - "github.com/cortexlabs/cortex/pkg/lib/prompt" "github.com/cortexlabs/cortex/pkg/lib/telemetry" - "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/spf13/cobra" ) var ( _flagLogsEnv string _flagLogsDisallowPrompt bool + _flagRandomPod bool + _logsOutput = `Navigate to the link below and click "Run Query": + +%s + +NOTE: there may be 1-2 minutes of delay for the logs to show up in the results of CloudWatch Insight queries +` ) func logsInit() { _logsCmd.Flags().SortFlags = false _logsCmd.Flags().StringVarP(&_flagLogsEnv, "env", "e", "", "environment to use") _logsCmd.Flags().BoolVarP(&_flagLogsDisallowPrompt, "yes", "y", false, "skip prompts") + _logsCmd.Flags().BoolVarP(&_flagRandomPod, "random-pod", "", false, "stream logs from a random pod") } var _logsCmd = &cobra.Command{ Use: "logs API_NAME [JOB_ID]", - Short: "stream logs from a single replica of an api or a single worker for a job", + Short: "get the logs for an API or a job", Args: cobra.RangeArgs(1, 2), Run: func(cmd *cobra.Command, args []string) { envName, err := getEnvFromFlag(_flagLogsEnv) @@ -52,7 +60,7 @@ var _logsCmd = &cobra.Command{ telemetry.Event("cli.logs") exit.Error(err) } - telemetry.Event("cli.logs", map[string]interface{}{"env_name": env.Name}) + telemetry.Event("cli.logs", map[string]interface{}{"env_name": env.Name, "random_pod": _flagRandomPod}) err = printEnvIfNotSpecified(env.Name, cmd) if err != nil { @@ -62,38 +70,36 @@ var _logsCmd = &cobra.Command{ operatorConfig := MustGetOperatorConfig(env.Name) apiName := args[0] - apiResponse, err := cluster.GetAPI(operatorConfig, apiName) - if err != nil { - exit.Error(err) - } - if len(args) == 1 { - if apiResponse[0].Spec.Kind == userconfig.RealtimeAPIKind && apiResponse[0].Status.Requested > 1 && !_flagLogsDisallowPrompt { - prompt.YesOrExit("logs from a single random replica will be streamed\n\nfor aggregated logs please visit your cloudwatch logging dashboard; see https://docs.cortex.dev for details", "", "") + if _flagRandomPod { + err := cluster.StreamLogs(operatorConfig, apiName) + if err != nil { + exit.Error(err) + } + return } - err = cluster.StreamLogs(operatorConfig, apiName) + logResponse, err := cluster.GetLogs(operatorConfig, apiName) if err != nil { exit.Error(err) } + fmt.Printf(_logsOutput, logResponse.LogURL) + return } - if len(args) == 2 { - if apiResponse[0].Spec.Kind == userconfig.BatchAPIKind { - jobResponse, err := cluster.GetBatchJob(operatorConfig, apiName, args[1]) - if err != nil { - exit.Error(err) - } - - if jobResponse.JobStatus.Workers > 1 && !_flagLogsDisallowPrompt { - prompt.YesOrExit("logs from a single random worker will be streamed\n\nfor aggregated logs please visit your cloudwatch logging dashboard; see https://docs.cortex.dev for details", "", "") - } - } - - err = cluster.StreamJobLogs(operatorConfig, apiName, args[1]) + jobID := args[1] + if _flagRandomPod { + err := cluster.StreamJobLogs(operatorConfig, apiName, jobID) if err != nil { exit.Error(err) } + return + } + + logResponse, err := cluster.GetJobLogs(operatorConfig, apiName, jobID) + if err != nil { + exit.Error(err) } + fmt.Printf(_logsOutput, logResponse.LogURL) }, } diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 0b40de677c..fe8e30f0ec 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -122,7 +122,8 @@ func main() { routerWithAuth.HandleFunc("/get", endpoints.GetAPIs).Methods("GET") routerWithAuth.HandleFunc("/get/{apiName}", endpoints.GetAPI).Methods("GET") routerWithAuth.HandleFunc("/get/{apiName}/{apiID}", endpoints.GetAPIByID).Methods("GET") - routerWithAuth.HandleFunc("/logs/{apiName}", endpoints.ReadLogs) + routerWithAuth.HandleFunc("/streamlogs/{apiName}", endpoints.ReadLogs) + routerWithAuth.HandleFunc("/logs/{apiName}", endpoints.GetLogURL).Methods("GET") operatorLogger.Info("Running on port " + _operatorPortStr) diff --git a/docs/clients/cli.md b/docs/clients/cli.md index faae5c0be9..5b4c843f4c 100644 --- a/docs/clients/cli.md +++ b/docs/clients/cli.md @@ -35,7 +35,7 @@ Flags: ## logs ```text -stream logs from a single replica of an api or a single worker for a job +get the logs for an API or a job Usage: cortex logs API_NAME [JOB_ID] [flags] @@ -43,6 +43,7 @@ Usage: Flags: -e, --env string environment to use -y, --yes skip prompts + --random-pod stream logs from a random pod -h, --help help for logs ``` diff --git a/docs/clients/python.md b/docs/clients/python.md index 2e7cb704ad..42840696f1 100644 --- a/docs/clients/python.md +++ b/docs/clients/python.md @@ -14,8 +14,6 @@ * [refresh](#refresh) * [delete](#delete) * [stop\_job](#stop_job) - * [stream\_api\_logs](#stream_api_logs) - * [stream\_job\_logs](#stream_job_logs) # cortex @@ -90,7 +88,7 @@ Deploy or update an API. - `api_spec` - A dictionary defining a single Cortex API. See https://docs.cortex.dev/v/master/ for schema. - `force` - Override any in-progress api updates. -- `wait` - Streams logs until the API is ready. +- `wait` - Block until the API is ready. **Returns**: @@ -111,7 +109,7 @@ Deploy or update APIs specified in a configuration file. - `config_file` - Local path to a yaml file defining Cortex API(s). See https://docs.cortex.dev/v/master/ for schema. - `force` - Override any in-progress api updates. -- `wait` - Streams logs until the APIs are ready. +- `wait` - Block until the API is ready. **Returns**: @@ -203,28 +201,3 @@ Stop a running job. - `api_name` - Name of the Batch/Task API. - `job_id` - ID of the Job to stop. - -## stream\_api\_logs - -```python - | stream_api_logs(api_name: str) -``` - -Stream the logs of an API. - -**Arguments**: - -- `api_name` - Name of the API. - -## stream\_job\_logs - -```python - | stream_job_logs(api_name: str, job_id: str) -``` - -Stream the logs of a Job. - -**Arguments**: - -- `api_name` - Name of the Batch API. -- `job_id` - Job ID. diff --git a/docs/clusters/management/environments.md b/docs/clusters/management/environments.md index 26c5592c7e..03490c9b13 100644 --- a/docs/clusters/management/environments.md +++ b/docs/clusters/management/environments.md @@ -11,11 +11,9 @@ cortex cluster up cluster1.yaml --configure-env cluster1 # configures the clust cortex cluster up cluster2.yaml --configure-env cluster2 # configures the cluster2 env cortex deploy --env cluster1 -cortex logs my-api --env cluster1 cortex delete my-api --env cluster1 cortex deploy --env cluster2 -cortex logs my-api --env cluster2 cortex delete my-api --env cluster2 ``` @@ -26,11 +24,9 @@ cortex cluster info cluster1.yaml --configure-env cluster1 # configures the clu cortex cluster info cluster2.yaml --configure-env cluster2 # configures the cluster2 env cortex deploy --env cluster1 -cortex logs my-api --env cluster1 cortex delete my-api --env cluster1 cortex deploy --env cluster2 -cortex logs my-api --env cluster2 cortex delete my-api --env cluster2 ``` diff --git a/docs/clusters/observability/logging.md b/docs/clusters/observability/logging.md index d8cfe5ffc9..f94e99da22 100644 --- a/docs/clusters/observability/logging.md +++ b/docs/clusters/observability/logging.md @@ -1,27 +1,12 @@ # Logging -By default, logs are collected with Fluent Bit and are exported to CloudWatch. It is also possible to view the logs of a single replica using the `cortex logs` command. - -## `cortex logs` - -The CLI includes a command to get the logs for a single API replica for debugging purposes: - -```bash -# RealtimeAPI -cortex logs - -# BatchAPI or TaskAPI -cortex logs # the job needs to be in a running state -``` - -**Important:** this method won't show the logs for all the API replicas and therefore is not a complete logging -solution. +Logs are collected with Fluent Bit and are exported to CloudWatch. ## Logs on AWS Logs will automatically be pushed to CloudWatch and a log group with the same name as your cluster will be created to store your logs. API logs are tagged with labels to help with log aggregation and filtering. -Below are some sample CloudWatch Log Insight queries: +You can use the `cortex logs` command to get a CloudWatch Insights URL of query to fetch logs for your API. Please note that there may be a few minutes of delay from when a message is logged to when it is available in CloudWatch Insights. **RealtimeAPI:** @@ -65,6 +50,18 @@ fields @timestamp, message | limit 1000 ``` +## Streaming logs for an API or a running job + +You can stream logs directly from a random pod of an API or a running job to iterate and debug quickly. These logs will not be as comprehensive as the logs that are available in CloudWatch. + +```bash +# RealtimeAPI +cortex logs --random-pod + +# BatchAPI or TaskAPI +cortex logs --random-pod # the job must be in a running state +``` + ## Structured logging If you log JSON strings from your APIs, they will be automatically parsed before pushing to CloudWatch. diff --git a/docs/workloads/async/example.md b/docs/workloads/async/example.md index 9734cfc071..3869b85fae 100644 --- a/docs/workloads/async/example.md +++ b/docs/workloads/async/example.md @@ -150,10 +150,9 @@ is `completed`. The result will remain queryable for 7 days after the request wa It is also possible to setup a webhook in your handler to get the response sent to a pre-defined web server once the workload completes or fails. -## Stream logs +## Debugging logs -If necessary, you can stream the logs from a random running pod from your API with the `cortex logs` command. This is -intended for debugging purposes only. For production logs, you can view the logs in cloudwatch logs. +If necessary, you can view logs for your API in CloudWatch using the `cortex logs` command. ```bash cortex logs iris-classifier diff --git a/docs/workloads/batch/example.md b/docs/workloads/batch/example.md index 32f5903c9d..358e99d9cd 100644 --- a/docs/workloads/batch/example.md +++ b/docs/workloads/batch/example.md @@ -125,7 +125,7 @@ print(response.text) cortex get image-classifier 69b183ed6bdf3e9b ``` -## Stream logs +## Debugging logs ```bash cortex logs image-classifier 69b183ed6bdf3e9b diff --git a/docs/workloads/realtime/example.md b/docs/workloads/realtime/example.md index f845723dbf..46d38b12ab 100644 --- a/docs/workloads/realtime/example.md +++ b/docs/workloads/realtime/example.md @@ -55,7 +55,7 @@ cortex deploy text_generator.yaml cortex get text-generator --watch ``` -### Stream logs +### Debugging logs ```bash cortex logs text-generator diff --git a/docs/workloads/realtime/troubleshooting.md b/docs/workloads/realtime/troubleshooting.md index 7e01e6f524..b82ed7660a 100644 --- a/docs/workloads/realtime/troubleshooting.md +++ b/docs/workloads/realtime/troubleshooting.md @@ -4,8 +4,8 @@ When making requests to your API, it's possible to get a `no healthy upstream` error message (with HTTP status code `503`). This means that there are currently no live replicas running for your API. This could happen for a few reasons: -1. It's possible that your API is simply not ready yet. You can check the status of your API with `cortex get API_NAME`, and stream the logs for a single replica (at random) with `cortex logs API_NAME`. -1. Your API may have errored during initialization or while responding to a previous request. `cortex get API_NAME` will show the status of your API, and you can view the logs for all replicas via Cloudwatch Logs Insights. +1. It's possible that your API is simply not ready yet. You can check the status of your API with `cortex get API_NAME`, and inspect the logs in CloudWatch with the help of `cortex logs API_NAME`. +1. Your API may have errored during initialization or while responding to a previous request. `cortex get API_NAME` will show the status of your API, and you can view the logs for all replicas by visiting the CloudWatch Insights URL from `cortex logs API_NAME`. If you are using API Gateway in front of your API endpoints, it is also possible to receive a `{"message":"Service Unavailable"}` error message (with HTTP status code `503`) after 29 seconds if your request exceeds API Gateway's 29 second timeout. If this is the case, you can either modify your code to take less time, run on faster hardware (e.g. GPUs), or don't use API Gateway (there is no timeout when using the API's endpoint directly). @@ -13,9 +13,9 @@ If you are using API Gateway in front of your API endpoints, it is also possible If your API is stuck in the "updating" or "compute unavailable" state (which is displayed when running `cortex get`), there are a few possible causes. Here are some things to check: -### Check `cortex logs API_NAME` +### Inspect API logs in CloudWatch -If no logs appear (e.g. it just says "fetching logs..."), continue down this list. +Use `cortex logs API_NAME` for a URL to view logs for your API in CloudWatch. In addition to output from your containers, you will find logs from other parts of the Cortex infrastructure that may help your troubleshooting. ### Check `max_instances` for your cluster diff --git a/pkg/operator/endpoints/errors.go b/pkg/operator/endpoints/errors.go index 09cedbe437..a005323489 100644 --- a/pkg/operator/endpoints/errors.go +++ b/pkg/operator/endpoints/errors.go @@ -120,6 +120,6 @@ func ErrorAnyPathParamRequired(param string, params ...string) error { func ErrorLogsJobIDRequired(resource operator.DeployedResource) error { return errors.WithStack(&errors.Error{ Kind: ErrLogsJobIDRequired, - Message: fmt.Sprintf("job id is required to stream logs for %s; you can get a list of latest job ids with `cortex get %s` and use `cortex logs %s JOB_ID` to stream logs for a job", resource.UserString(), resource.Name, resource.Name), + Message: fmt.Sprintf("job id is required for %s; you can get a list of latest job ids with `cortex get %s` and use `cortex logs %s JOB_ID` to get the logs", resource.UserString(), resource.Name, resource.Name), }) } diff --git a/pkg/operator/endpoints/logs.go b/pkg/operator/endpoints/logs.go index 765a0318eb..2d335e27da 100644 --- a/pkg/operator/endpoints/logs.go +++ b/pkg/operator/endpoints/logs.go @@ -21,6 +21,9 @@ import ( "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/operator/resources" + "github.com/cortexlabs/cortex/pkg/operator/resources/asyncapi" + "github.com/cortexlabs/cortex/pkg/operator/resources/realtimeapi" + "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/gorilla/mux" "github.com/gorilla/websocket" @@ -67,3 +70,57 @@ func ReadLogs(w http.ResponseWriter, r *http.Request) { } operator.StreamLogsFromRandomPod(labels, socket) } + +func GetLogURL(w http.ResponseWriter, r *http.Request) { + apiName := mux.Vars(r)["apiName"] + jobID := getOptionalQParam("jobID", r) + + if jobID != "" { + GetJobLogURL(w, r) + return + } + + deployedResource, err := resources.GetDeployedResourceByName(apiName) + if err != nil { + respondError(w, r, err) + return + } + + if deployedResource.Kind == userconfig.BatchAPIKind || deployedResource.Kind == userconfig.TaskAPIKind { + respondError(w, r, ErrorLogsJobIDRequired(*deployedResource)) + return + } + + switch deployedResource.Kind { + case userconfig.AsyncAPIKind: + apiResponse, err := asyncapi.GetAPIByName(deployedResource) + if err != nil { + respondError(w, r, err) + return + } + logURL, err := operator.APILogURL(apiResponse[0].Spec) + if err != nil { + respondError(w, r, err) + return + } + respondJSON(w, r, schema.LogResponse{ + LogURL: logURL, + }) + case userconfig.RealtimeAPIKind: + apiResponse, err := realtimeapi.GetAPIByName(deployedResource) + if err != nil { + respondError(w, r, err) + return + } + logURL, err := operator.APILogURL(apiResponse[0].Spec) + if err != nil { + respondError(w, r, err) + return + } + respondJSON(w, r, schema.LogResponse{ + LogURL: logURL, + }) + default: + respondError(w, r, resources.ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.RealtimeAPIKind, userconfig.AsyncAPIKind)) + } +} diff --git a/pkg/operator/endpoints/logs_job.go b/pkg/operator/endpoints/logs_job.go index 6232466569..704bef376c 100644 --- a/pkg/operator/endpoints/logs_job.go +++ b/pkg/operator/endpoints/logs_job.go @@ -21,6 +21,10 @@ import ( "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/operator/resources" + "github.com/cortexlabs/cortex/pkg/operator/resources/job/batchapi" + "github.com/cortexlabs/cortex/pkg/operator/resources/job/taskapi" + "github.com/cortexlabs/cortex/pkg/operator/schema" + "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/gorilla/mux" "github.com/gorilla/websocket" @@ -60,3 +64,59 @@ func ReadJobLogs(w http.ResponseWriter, r *http.Request) { operator.StreamLogsFromRandomPod(labels, socket) } + +func GetJobLogURL(w http.ResponseWriter, r *http.Request) { + apiName := mux.Vars(r)["apiName"] + jobID, err := getRequiredQueryParam("jobID", r) + if err != nil { + respondError(w, r, err) + return + } + + deployedResource, err := resources.GetDeployedResourceByName(apiName) + if err != nil { + respondError(w, r, err) + return + } + + switch deployedResource.Kind { + case userconfig.BatchAPIKind: + jobStatus, err := batchapi.GetJobStatus(spec.JobKey{ + ID: jobID, + APIName: apiName, + Kind: userconfig.BatchAPIKind, + }) + if err != nil { + respondError(w, r, err) + return + } + logURL, err := operator.BatchJobLogURL(apiName, *jobStatus) + if err != nil { + respondError(w, r, err) + return + } + respondJSON(w, r, schema.LogResponse{ + LogURL: logURL, + }) + case userconfig.TaskAPIKind: + jobStatus, err := taskapi.GetJobStatus(spec.JobKey{ + ID: jobID, + APIName: apiName, + Kind: userconfig.TaskAPIKind, + }) + if err != nil { + respondError(w, r, err) + return + } + logURL, err := operator.TaskJobLogURL(apiName, *jobStatus) + if err != nil { + respondError(w, r, err) + return + } + respondJSON(w, r, schema.LogResponse{ + LogURL: logURL, + }) + default: + respondError(w, r, resources.ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.BatchAPIKind, userconfig.TaskAPIKind)) + } +} diff --git a/pkg/operator/operator/workload_logging.go b/pkg/operator/operator/workload_logging.go index a180b906dc..a2b4ee198a 100644 --- a/pkg/operator/operator/workload_logging.go +++ b/pkg/operator/operator/workload_logging.go @@ -18,17 +18,23 @@ package operator import ( "bufio" + "bytes" "encoding/json" "fmt" "io" "os/exec" + "strings" + "text/template" "time" "github.com/cortexlabs/cortex/pkg/config" + awslib "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/operator/lib/routines" + "github.com/cortexlabs/cortex/pkg/types/spec" + "github.com/cortexlabs/cortex/pkg/types/status" "github.com/gorilla/websocket" ) @@ -42,6 +48,150 @@ const ( _pollPeriod = 250 * time.Millisecond ) +func timeString(t time.Time) string { + return fmt.Sprintf("%sT%02d*3a%02d*3a%02d", t.Format("2006-01-02"), t.Hour(), t.Minute(), t.Second()) +} + +var _apiLogURLTemplate *template.Template = template.Must(template.New("api_log_url_template").Parse(strings.TrimSpace(` + https://console.{{.Partition}}.com/cloudwatch/home?region={{.Region}}#logsV2:logs-insights$3FqueryDetail$3D$257E$2528end$257E0$257Estart$257E-3600$257EtimeType$257E$2527RELATIVE$257Eunit$257E$2527seconds$257EeditorString$257E$2527fields*20*40timestamp*2c*20message*0a*7c*20filter*20cortex.labels.apiName*3d*22{{.APIName}}*22*0a*7c*20sort*20*40timestamp*20asc*0a$257Esource$257E$2528$257E$2527{{.LogGroup}}$2529$2529 +`))) + +var _completedJobLogURLTemplate *template.Template = template.Must(template.New("completed_job_log_url_template").Parse(strings.TrimSpace(` +https://console.{{.Partition}}.com/cloudwatch/home?region={{.Region}}#logsV2:logs-insights$3FqueryDetail$3D$257E$2528end$257E$2527{{.EndTime}}$257Estart$257E$2527{{.StartTime}}$257EtimeType$257E$2527ABSOLUTE$257Etz$257E$2527Local$257EeditorString$257E$2527fields*20*40timestamp*2c*20message*0a*7c*20filter*20cortex.labels.apiName*3d*22sum*22*20and*20cortex.labels.jobID*3d*22697ba471c134ea2e*22*0a*7c*20sort*20*40timestamp*20asc*0a$257Esource$257E$2528$257E$2527{{.LogGroup}}$2529$2529 +`))) + +var _inProgressJobLogsURLTemplate *template.Template = template.Must(template.New("in_progress_job_log_url_template").Parse(strings.TrimSpace(` +https://console.{{.Partition}}.com/cloudwatch/home?region={{.Region}}#logsV2:logs-insights$3FqueryDetail$3D$257E$2528end$257E0$257Estart$257E-3600$257EtimeType$257E$2527RELATIVE$257Eunit$257E$2527seconds$257EeditorString$257E$2527fields*20*40timestamp*2c*20message*0a*7c*20filter*20cortex.labels.apiName*3d*22{{.APIName}}*22*20and*20cortex.labels.jobID*3d*22{{.JobID}}*22*0a*7c*20sort*20*40timestamp*20asc*0a$257Esource$257E$2528$257E$2527{{.LogGroup}}$2529$2529 +`))) + +type apiLogURLTemplateArgs struct { + Partition string + Region string + LogGroup string + APIName string +} + +type completedJobLogURLTemplateArgs struct { + Partition string + Region string + StartTime string + EndTime string + LogGroup string + APIName string + JobID string +} + +type inProgressJobLogURLTemplateArgs struct { + Partition string + Region string + LogGroup string + APIName string + JobID string +} + +func completedBatchJobLogsURL(args completedJobLogURLTemplateArgs) (string, error) { + buf := &bytes.Buffer{} + err := _completedJobLogURLTemplate.Execute(buf, args) + if err != nil { + return "", err + } + + return strings.TrimSpace(buf.String()), nil +} + +func inProgressBatchJobLogsURL(args inProgressJobLogURLTemplateArgs) (string, error) { + buf := &bytes.Buffer{} + err := _inProgressJobLogsURLTemplate.Execute(buf, args) + if err != nil { + return "", err + } + + return strings.TrimSpace(buf.String()), nil +} + +func APILogURL(api spec.API) (string, error) { + partition := "aws.amazon" + region := config.ClusterConfig.Region + if awslib.PartitionFromRegion(region) == "aws-us-gov" { + partition = "amazonaws-us-gov" + } + logGroup := config.ClusterConfig.ClusterName + + args := apiLogURLTemplateArgs{ + Partition: partition, + Region: region, + LogGroup: logGroup, + APIName: api.Name, + } + + buf := &bytes.Buffer{} + err := _apiLogURLTemplate.Execute(buf, args) + if err != nil { + return "", err + } + + return strings.TrimSpace(buf.String()), nil +} + +func BatchJobLogURL(apiName string, jobStatus status.BatchJobStatus) (string, error) { + partition := "aws.amazon" + region := config.ClusterConfig.Region + if awslib.PartitionFromRegion(region) == "aws-us-gov" { + partition = "amazonaws-us-gov" + } + logGroup := config.ClusterConfig.ClusterName + + if jobStatus.EndTime != nil { + endTime := *jobStatus.EndTime + endTime = endTime.Add(60 * time.Second) + return completedBatchJobLogsURL(completedJobLogURLTemplateArgs{ + Partition: partition, + Region: region, + StartTime: timeString(jobStatus.StartTime), + EndTime: timeString(endTime), + LogGroup: logGroup, + APIName: apiName, + JobID: jobStatus.ID, + }) + } + return inProgressBatchJobLogsURL(inProgressJobLogURLTemplateArgs{ + Partition: partition, + Region: region, + LogGroup: logGroup, + APIName: apiName, + JobID: jobStatus.ID, + }) +} + +func TaskJobLogURL(apiName string, jobStatus status.TaskJobStatus) (string, error) { + partition := "aws.amazon" + region := config.ClusterConfig.Region + if awslib.PartitionFromRegion(region) == "aws-us-gov" { + partition = "amazonaws-us-gov" + } + logGroup := config.ClusterConfig.ClusterName + if jobStatus.EndTime != nil { + endTime := *jobStatus.EndTime + endTime = endTime.Add(60 * time.Second) + return completedBatchJobLogsURL(completedJobLogURLTemplateArgs{ + Partition: partition, + Region: region, + StartTime: timeString(jobStatus.StartTime), + EndTime: timeString(endTime), + LogGroup: logGroup, + APIName: apiName, + JobID: jobStatus.ID, + }) + } + return inProgressBatchJobLogsURL(inProgressJobLogURLTemplateArgs{ + Partition: partition, + Region: region, + LogGroup: logGroup, + APIName: apiName, + JobID: jobStatus.ID, + }) +} + func waitForPodToBeNotPending(podName string, cancelListener chan struct{}, socket *websocket.Conn) bool { wrotePending := false timer := time.NewTimer(0) diff --git a/pkg/operator/schema/schema.go b/pkg/operator/schema/schema.go index d2b1433340..8a896d4d10 100644 --- a/pkg/operator/schema/schema.go +++ b/pkg/operator/schema/schema.go @@ -60,6 +60,10 @@ type APIResponse struct { APIVersions []APIVersion `json:"api_versions,omitempty"` } +type LogResponse struct { + LogURL string `json:"log_url"` +} + type BatchJobResponse struct { APISpec spec.API `json:"api_spec"` JobStatus status.BatchJobStatus `json:"job_status"` diff --git a/python/client/cortex/binary/__init__.py b/python/client/cortex/binary/__init__.py index 30a6f5fafa..07674080ce 100644 --- a/python/client/cortex/binary/__init__.py +++ b/python/client/cortex/binary/__init__.py @@ -61,8 +61,6 @@ def run_cli( output = "" result = "" - processing_result = False - processed_result = False for c in iter(lambda: process.stdout.read(1), ""): output += c @@ -71,9 +69,6 @@ def run_cli( sys.stdout.write(c) sys.stdout.flush() - if processed_result == True: - processing_result = False - process.wait() if process.returncode == 0: diff --git a/python/client/cortex/client.py b/python/client/cortex/client.py index 17c3f4366b..bf09da80b5 100644 --- a/python/client/cortex/client.py +++ b/python/client/cortex/client.py @@ -55,7 +55,7 @@ def deploy( Args: api_spec: A dictionary defining a single Cortex API. See https://docs.cortex.dev/v/master/ for schema. force: Override any in-progress api updates. - wait: Streams logs until the API is ready. + wait: Block until the API is ready. Returns: Deployment status, API specification, and endpoint for each API. @@ -86,7 +86,7 @@ def deploy_from_file( Args: config_file: Local path to a yaml file defining Cortex API(s). See https://docs.cortex.dev/v/master/ for schema. force: Override any in-progress api updates. - wait: Streams logs until the APIs are ready. + wait: Block until the API is ready. Returns: Deployment status, API specification, and endpoint for each API. @@ -114,41 +114,18 @@ def deploy_from_file( if not wait: return deploy_result - # logging immediately will show previous versions of the replica terminating; - # wait a few seconds for the new replicas to start initializing - time.sleep(5) - - def stream_to_stdout(process): - for c in iter(lambda: process.stdout.read(1), ""): - sys.stdout.write(c) - sys.stdout.flush() - api_name = deploy_result["api"]["spec"]["name"] - if deploy_result["api"]["spec"]["kind"] != "RealtimeAPI": + if ( + deploy_result["api"]["spec"]["kind"] != "RealtimeAPI" + and deploy_result["api"]["spec"]["kind"] != "AsyncAPI" + ): return deploy_result - env = os.environ.copy() - env["CORTEX_CLI_INVOKER"] = "python" - process = subprocess.Popen( - [get_cli_path(), "logs", "--env", self.env_name, api_name, "-y"], - stderr=subprocess.STDOUT, - stdout=subprocess.PIPE, - encoding="utf8", - errors="replace", # replace non-utf8 characters with `?` instead of failing - env=env, - ) - - streamer = threading.Thread(target=stream_to_stdout, args=[process]) - streamer.start() - - while process.poll() is None: + while True: + time.sleep(5) api = self.get_api(api_name) if api["status"]["status_code"] != "status_updating": - time.sleep(5) # accommodate latency in log streaming from the cluster - process.terminate() break - time.sleep(5) - streamer.join(timeout=10) return api @@ -260,33 +237,3 @@ def stop_job(self, api_name: str, job_id: str, keep_cache: bool = False): ] run_cli(args) - - @sentry_wrapper - def stream_api_logs( - self, - api_name: str, - ): - """ - Stream the logs of an API. - - Args: - api_name: Name of the API. - """ - args = ["logs", api_name, "--env", self.env_name, "-y"] - run_cli(args) - - @sentry_wrapper - def stream_job_logs( - self, - api_name: str, - job_id: str, - ): - """ - Stream the logs of a Job. - - Args: - api_name: Name of the Batch API. - job_id: Job ID. - """ - args = ["logs", api_name, job_id, "--env", self.env_name, "-y"] - run_cli(args) diff --git a/test/e2e/e2e/tests.py b/test/e2e/e2e/tests.py index 0f560de23c..3bc72af04d 100644 --- a/test/e2e/e2e/tests.py +++ b/test/e2e/e2e/tests.py @@ -51,6 +51,8 @@ make_requests_concurrently, check_futures_healthy, retrieve_results_concurrently, + stream_api_logs, + stream_job_logs, ) TEST_APIS_DIR = Path(__file__).parent.parent.parent / "apis" @@ -122,7 +124,7 @@ def test_realtime_api( try: api_info = client.get_api(api_name) printer(json.dumps(api_info, indent=2)) - td.Thread(target=lambda: client.stream_api_logs(api_name), daemon=True).start() + td.Thread(target=lambda: stream_api_logs(client, api_name), daemon=True).start() time.sleep(5) finally: raise @@ -204,8 +206,10 @@ def test_batch_api( job_status = client.get_job(api_name, job_spec["job_id"]) printer(json.dumps(job_status, indent=2)) + td.Thread( - target=lambda: client.stream_job_logs(api_name, job_spec["job_id"]), daemon=True + target=lambda: stream_job_logs(client, api_name, job_spec["job_id"]), + daemon=True, ).start() time.sleep(5) finally: @@ -311,11 +315,9 @@ def test_async_api( try: api_info = client.get_api(api_name) printer(json.dumps(api_info, indent=2)) - - job_status = client.get_job(api_name, result_response_json["id"]) - printer(json.dumps(job_status, indent=2)) + printer(json.dumps(result_response_json, indent=2)) td.Thread( - target=lambda: client.stream_job_logs(api_name, result_response_json["id"]), + target=lambda: stream_api_logs(client, api_name), daemon=True, ).start() time.sleep(5) @@ -386,7 +388,7 @@ def test_task_api( job_status = client.get_job(api_name, job_spec["job_id"]) printer(json.dumps(job_status, indent=2)) td.Thread( - target=lambda: client.stream_job_logs(api_name, job_spec["job_id"]), daemon=True + target=lambda: stream_job_logs(client, api_name, job_spec["job_id"]), daemon=True ).start() time.sleep(5) except: @@ -986,7 +988,7 @@ def test_long_running_realtime( try: api_info = client.get_api(api_name) printer(json.dumps(api_info, indent=2)) - td.Thread(target=lambda: client.stream_api_logs(api_name), daemon=True).start() + td.Thread(target=lambda: stream_api_logs(client, api_name), daemon=True).start() time.sleep(5) finally: raise diff --git a/test/e2e/e2e/utils.py b/test/e2e/e2e/utils.py index 013f1c0f57..f5223f3d47 100644 --- a/test/e2e/e2e/utils.py +++ b/test/e2e/e2e/utils.py @@ -416,3 +416,11 @@ def client_from_config(config_path: str) -> cx.Client: cluster_name = config["cluster_name"] return cx.client(f"{cluster_name}") + + +def stream_api_logs(client: cx.Client, api_name: str): + cx.run_cli(["logs", api_name, "--random-pod", "-e", client.env_name]) + + +def stream_job_logs(client: cx.Client, api_name: str, job_id: str): + cx.run_cli(["logs", api_name, job_id, "--random-pod", "-e", client.env_name]) From b9bdf43fb69463550f9fe60a288c277278d3601e Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Wed, 2 Jun 2021 01:18:41 +0300 Subject: [PATCH 57/82] Update respond.go (#2209) --- pkg/operator/endpoints/respond.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/operator/endpoints/respond.go b/pkg/operator/endpoints/respond.go index 039b49fc2c..81d5be78b6 100644 --- a/pkg/operator/endpoints/respond.go +++ b/pkg/operator/endpoints/respond.go @@ -31,6 +31,7 @@ var operatorLogger = logging.GetLogger() func respondJSON(w http.ResponseWriter, r *http.Request, response interface{}) { if err := json.NewEncoder(w).Encode(response); err != nil { respondError(w, r, errors.Wrap(err, "failed to encode response")) + return } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) From 527bf25ec9d4f35d877cc9cceb7841c1949c85b4 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Wed, 2 Jun 2021 01:32:16 +0300 Subject: [PATCH 58/82] CaaS - fix respond function (#2213) --- pkg/operator/endpoints/respond.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/operator/endpoints/respond.go b/pkg/operator/endpoints/respond.go index 81d5be78b6..12b9de78de 100644 --- a/pkg/operator/endpoints/respond.go +++ b/pkg/operator/endpoints/respond.go @@ -21,6 +21,7 @@ import ( "net/http" "github.com/cortexlabs/cortex/pkg/lib/errors" + libjson "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/logging" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/operator/schema" @@ -29,12 +30,15 @@ import ( var operatorLogger = logging.GetLogger() func respondJSON(w http.ResponseWriter, r *http.Request, response interface{}) { - if err := json.NewEncoder(w).Encode(response); err != nil { + jsonBytes, err := libjson.Marshal(response) + if err != nil { respondError(w, r, errors.Wrap(err, "failed to encode response")) return } + w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) + w.Write(jsonBytes) } func respondError(w http.ResponseWriter, r *http.Request, err error, strs ...string) { From fae614a9a938505b46998557bcf0e07f97f5a9ee Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Tue, 1 Jun 2021 21:58:10 -0700 Subject: [PATCH 59/82] Update test image build scripts --- Makefile | 5 - test/apis/async/hello-world/.dockerignore | 9 ++ test/apis/async/hello-world/build-cpu.sh | 4 + test/apis/async/hello-world/cortex_cpu.yaml | 14 +++ .../hello-world/hello-world-cpu.dockerfile | 17 ++++ test/apis/async/hello-world/main.py | 16 ++++ test/apis/async/text-generator/.dockerignore | 7 +- test/apis/async/text-generator/build-cpu.sh | 4 + test/apis/async/text-generator/build-gpu.sh | 4 + .../image-classifier-alexnet/.dockerignore | 7 +- .../image-classifier-alexnet/build-cpu.sh | 4 + .../image-classifier-alexnet/build-gpu.sh | 4 + test/apis/batch/sum/.dockerignore | 6 +- test/apis/batch/sum/build-cpu.sh | 4 + test/apis/realtime/hello-world/.dockerignore | 5 +- test/apis/realtime/hello-world/sample.json | 1 - .../image-classifier-resnet50/.dockerignore | 6 +- .../image-classifier-resnet50/build-cpu.sh | 4 + .../image-classifier-resnet50/build-gpu.sh | 4 + .../realtime/prime-generator/.dockerignore | 6 +- .../realtime/prime-generator/build-cpu.sh | 4 + test/apis/realtime/sleep/.dockerignore | 5 +- test/apis/realtime/sleep/build-cpu.sh | 4 + test/apis/realtime/sleep/sample.json | 1 - .../realtime/text-generator/.dockerignore | 6 +- .../apis/realtime/text-generator/build-cpu.sh | 4 + .../apis/realtime/text-generator/build-gpu.sh | 4 + .../iris-classifier-trainer/.dockerignore | 6 +- .../task/iris-classifier-trainer/build-cpu.sh | 4 + test/utils/build-and-push-images.sh | 59 ------------ test/utils/build.sh | 94 +++++++++++++++++++ 31 files changed, 237 insertions(+), 85 deletions(-) create mode 100644 test/apis/async/hello-world/.dockerignore create mode 100755 test/apis/async/hello-world/build-cpu.sh create mode 100644 test/apis/async/hello-world/cortex_cpu.yaml create mode 100644 test/apis/async/hello-world/hello-world-cpu.dockerfile create mode 100644 test/apis/async/hello-world/main.py create mode 100755 test/apis/async/text-generator/build-cpu.sh create mode 100755 test/apis/async/text-generator/build-gpu.sh create mode 100755 test/apis/batch/image-classifier-alexnet/build-cpu.sh create mode 100755 test/apis/batch/image-classifier-alexnet/build-gpu.sh create mode 100755 test/apis/batch/sum/build-cpu.sh delete mode 100644 test/apis/realtime/hello-world/sample.json create mode 100755 test/apis/realtime/image-classifier-resnet50/build-cpu.sh create mode 100755 test/apis/realtime/image-classifier-resnet50/build-gpu.sh create mode 100755 test/apis/realtime/prime-generator/build-cpu.sh create mode 100755 test/apis/realtime/sleep/build-cpu.sh delete mode 100644 test/apis/realtime/sleep/sample.json create mode 100755 test/apis/realtime/text-generator/build-cpu.sh create mode 100755 test/apis/realtime/text-generator/build-gpu.sh create mode 100755 test/apis/task/iris-classifier-trainer/build-cpu.sh delete mode 100755 test/utils/build-and-push-images.sh create mode 100755 test/utils/build.sh diff --git a/Makefile b/Makefile index 798cc3e615..ddecc7367f 100644 --- a/Makefile +++ b/Makefile @@ -165,11 +165,6 @@ format: test: @./build/test.sh go -# build test api images -# make sure you login with your quay credentials -build-and-push-test-images: - @./test/utils/build-and-push-images.sh quay.io - # run e2e tests on an existing cluster # read test/e2e/README.md for instructions first test-e2e: diff --git a/test/apis/async/hello-world/.dockerignore b/test/apis/async/hello-world/.dockerignore new file mode 100644 index 0000000000..4a8a96662f --- /dev/null +++ b/test/apis/async/hello-world/.dockerignore @@ -0,0 +1,9 @@ +*.dockerfile +build*.sh +cortex*.yaml +*.pyc +*.pyo +*.pyd +__pycache__ +.pytest_cache +*.md diff --git a/test/apis/async/hello-world/build-cpu.sh b/test/apis/async/hello-world/build-cpu.sh new file mode 100755 index 0000000000..12960564d9 --- /dev/null +++ b/test/apis/async/hello-world/build-cpu.sh @@ -0,0 +1,4 @@ +# usage: build-cpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" diff --git a/test/apis/async/hello-world/cortex_cpu.yaml b/test/apis/async/hello-world/cortex_cpu.yaml new file mode 100644 index 0000000000..cda010a135 --- /dev/null +++ b/test/apis/async/hello-world/cortex_cpu.yaml @@ -0,0 +1,14 @@ +- name: hello-world + kind: AsyncAPI + pod: + port: 9000 + containers: + - name: api + image: quay.io/cortexlabs-test/async-hello-world-cpu:latest + readiness_probe: + http_get: + path: "/healthz" + port: 9000 + compute: + cpu: 200m + mem: 128M diff --git a/test/apis/async/hello-world/hello-world-cpu.dockerfile b/test/apis/async/hello-world/hello-world-cpu.dockerfile new file mode 100644 index 0000000000..a496f7c881 --- /dev/null +++ b/test/apis/async/hello-world/hello-world-cpu.dockerfile @@ -0,0 +1,17 @@ +FROM python:3.8-slim + +# Allow statements and log messages to immediately appear in the logs +ENV PYTHONUNBUFFERED True + +# Install production dependencies +RUN pip install --no-cache-dir "uvicorn[standard]" gunicorn fastapi + +# Copy local code to the container image. +COPY . /app +WORKDIR /app/ + +ENV PYTHONPATH=/app +ENV CORTEX_PORT=9000 + +# Run the web service on container startup. +CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/async/hello-world/main.py b/test/apis/async/hello-world/main.py new file mode 100644 index 0000000000..d1a6cd91e9 --- /dev/null +++ b/test/apis/async/hello-world/main.py @@ -0,0 +1,16 @@ +import os +from fastapi import FastAPI + +app = FastAPI() + +response_str = os.getenv("RESPONSE", "hello world") + + +@app.get("/healthz") +def healthz(): + return "ok" + + +@app.post("/") +def handler(): + return {"message": response_str} diff --git a/test/apis/async/text-generator/.dockerignore b/test/apis/async/text-generator/.dockerignore index 7fa2250d13..414f79e41f 100644 --- a/test/apis/async/text-generator/.dockerignore +++ b/test/apis/async/text-generator/.dockerignore @@ -1,9 +1,10 @@ -*.dockerfile -README.md -sample.json expectations.yaml +*.dockerfile +build*.sh +cortex*.yaml *.pyc *.pyo *.pyd __pycache__ .pytest_cache +*.md diff --git a/test/apis/async/text-generator/build-cpu.sh b/test/apis/async/text-generator/build-cpu.sh new file mode 100755 index 0000000000..12960564d9 --- /dev/null +++ b/test/apis/async/text-generator/build-cpu.sh @@ -0,0 +1,4 @@ +# usage: build-cpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" diff --git a/test/apis/async/text-generator/build-gpu.sh b/test/apis/async/text-generator/build-gpu.sh new file mode 100755 index 0000000000..5acedee826 --- /dev/null +++ b/test/apis/async/text-generator/build-gpu.sh @@ -0,0 +1,4 @@ +# usage: build-gpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" diff --git a/test/apis/batch/image-classifier-alexnet/.dockerignore b/test/apis/batch/image-classifier-alexnet/.dockerignore index 12657957ef..014d70d990 100644 --- a/test/apis/batch/image-classifier-alexnet/.dockerignore +++ b/test/apis/batch/image-classifier-alexnet/.dockerignore @@ -1,8 +1,11 @@ -*.dockerfile -README.md sample.json +submit.py +*.dockerfile +build*.sh +cortex*.yaml *.pyc *.pyo *.pyd __pycache__ .pytest_cache +*.md diff --git a/test/apis/batch/image-classifier-alexnet/build-cpu.sh b/test/apis/batch/image-classifier-alexnet/build-cpu.sh new file mode 100755 index 0000000000..12960564d9 --- /dev/null +++ b/test/apis/batch/image-classifier-alexnet/build-cpu.sh @@ -0,0 +1,4 @@ +# usage: build-cpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" diff --git a/test/apis/batch/image-classifier-alexnet/build-gpu.sh b/test/apis/batch/image-classifier-alexnet/build-gpu.sh new file mode 100755 index 0000000000..5acedee826 --- /dev/null +++ b/test/apis/batch/image-classifier-alexnet/build-gpu.sh @@ -0,0 +1,4 @@ +# usage: build-gpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" diff --git a/test/apis/batch/sum/.dockerignore b/test/apis/batch/sum/.dockerignore index 30fea70be8..8030157a8b 100644 --- a/test/apis/batch/sum/.dockerignore +++ b/test/apis/batch/sum/.dockerignore @@ -1,10 +1,12 @@ -*.dockerfile -README.md sample.json submit.py sample_generator.py +*.dockerfile +build*.sh +cortex*.yaml *.pyc *.pyo *.pyd __pycache__ .pytest_cache +*.md diff --git a/test/apis/batch/sum/build-cpu.sh b/test/apis/batch/sum/build-cpu.sh new file mode 100755 index 0000000000..12960564d9 --- /dev/null +++ b/test/apis/batch/sum/build-cpu.sh @@ -0,0 +1,4 @@ +# usage: build-cpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" diff --git a/test/apis/realtime/hello-world/.dockerignore b/test/apis/realtime/hello-world/.dockerignore index 12657957ef..4a8a96662f 100644 --- a/test/apis/realtime/hello-world/.dockerignore +++ b/test/apis/realtime/hello-world/.dockerignore @@ -1,8 +1,9 @@ *.dockerfile -README.md -sample.json +build*.sh +cortex*.yaml *.pyc *.pyo *.pyd __pycache__ .pytest_cache +*.md diff --git a/test/apis/realtime/hello-world/sample.json b/test/apis/realtime/hello-world/sample.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/test/apis/realtime/hello-world/sample.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/test/apis/realtime/image-classifier-resnet50/.dockerignore b/test/apis/realtime/image-classifier-resnet50/.dockerignore index 6482ea768a..a0dd6cdad9 100644 --- a/test/apis/realtime/image-classifier-resnet50/.dockerignore +++ b/test/apis/realtime/image-classifier-resnet50/.dockerignore @@ -1,8 +1,10 @@ +sample.json *.dockerfile -README.md -client.py +build*.sh +cortex*.yaml *.pyc *.pyo *.pyd __pycache__ .pytest_cache +*.md diff --git a/test/apis/realtime/image-classifier-resnet50/build-cpu.sh b/test/apis/realtime/image-classifier-resnet50/build-cpu.sh new file mode 100755 index 0000000000..12960564d9 --- /dev/null +++ b/test/apis/realtime/image-classifier-resnet50/build-cpu.sh @@ -0,0 +1,4 @@ +# usage: build-cpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" diff --git a/test/apis/realtime/image-classifier-resnet50/build-gpu.sh b/test/apis/realtime/image-classifier-resnet50/build-gpu.sh new file mode 100755 index 0000000000..5acedee826 --- /dev/null +++ b/test/apis/realtime/image-classifier-resnet50/build-gpu.sh @@ -0,0 +1,4 @@ +# usage: build-gpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" diff --git a/test/apis/realtime/prime-generator/.dockerignore b/test/apis/realtime/prime-generator/.dockerignore index 12657957ef..a0dd6cdad9 100644 --- a/test/apis/realtime/prime-generator/.dockerignore +++ b/test/apis/realtime/prime-generator/.dockerignore @@ -1,8 +1,10 @@ -*.dockerfile -README.md sample.json +*.dockerfile +build*.sh +cortex*.yaml *.pyc *.pyo *.pyd __pycache__ .pytest_cache +*.md diff --git a/test/apis/realtime/prime-generator/build-cpu.sh b/test/apis/realtime/prime-generator/build-cpu.sh new file mode 100755 index 0000000000..12960564d9 --- /dev/null +++ b/test/apis/realtime/prime-generator/build-cpu.sh @@ -0,0 +1,4 @@ +# usage: build-cpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" diff --git a/test/apis/realtime/sleep/.dockerignore b/test/apis/realtime/sleep/.dockerignore index 12657957ef..4a8a96662f 100644 --- a/test/apis/realtime/sleep/.dockerignore +++ b/test/apis/realtime/sleep/.dockerignore @@ -1,8 +1,9 @@ *.dockerfile -README.md -sample.json +build*.sh +cortex*.yaml *.pyc *.pyo *.pyd __pycache__ .pytest_cache +*.md diff --git a/test/apis/realtime/sleep/build-cpu.sh b/test/apis/realtime/sleep/build-cpu.sh new file mode 100755 index 0000000000..12960564d9 --- /dev/null +++ b/test/apis/realtime/sleep/build-cpu.sh @@ -0,0 +1,4 @@ +# usage: build-cpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" diff --git a/test/apis/realtime/sleep/sample.json b/test/apis/realtime/sleep/sample.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/test/apis/realtime/sleep/sample.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/test/apis/realtime/text-generator/.dockerignore b/test/apis/realtime/text-generator/.dockerignore index 12657957ef..a0dd6cdad9 100644 --- a/test/apis/realtime/text-generator/.dockerignore +++ b/test/apis/realtime/text-generator/.dockerignore @@ -1,8 +1,10 @@ -*.dockerfile -README.md sample.json +*.dockerfile +build*.sh +cortex*.yaml *.pyc *.pyo *.pyd __pycache__ .pytest_cache +*.md diff --git a/test/apis/realtime/text-generator/build-cpu.sh b/test/apis/realtime/text-generator/build-cpu.sh new file mode 100755 index 0000000000..12960564d9 --- /dev/null +++ b/test/apis/realtime/text-generator/build-cpu.sh @@ -0,0 +1,4 @@ +# usage: build-cpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" diff --git a/test/apis/realtime/text-generator/build-gpu.sh b/test/apis/realtime/text-generator/build-gpu.sh new file mode 100755 index 0000000000..5acedee826 --- /dev/null +++ b/test/apis/realtime/text-generator/build-gpu.sh @@ -0,0 +1,4 @@ +# usage: build-gpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" diff --git a/test/apis/task/iris-classifier-trainer/.dockerignore b/test/apis/task/iris-classifier-trainer/.dockerignore index 5bb0c5ed3b..30f9c4551c 100644 --- a/test/apis/task/iris-classifier-trainer/.dockerignore +++ b/test/apis/task/iris-classifier-trainer/.dockerignore @@ -1,8 +1,10 @@ -*.dockerfile -README.md submit.py +*.dockerfile +build*.sh +cortex*.yaml *.pyc *.pyo *.pyd __pycache__ .pytest_cache +*.md diff --git a/test/apis/task/iris-classifier-trainer/build-cpu.sh b/test/apis/task/iris-classifier-trainer/build-cpu.sh new file mode 100755 index 0000000000..12960564d9 --- /dev/null +++ b/test/apis/task/iris-classifier-trainer/build-cpu.sh @@ -0,0 +1,4 @@ +# usage: build-cpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" diff --git a/test/utils/build-and-push-images.sh b/test/utils/build-and-push-images.sh deleted file mode 100755 index dc3cd57e54..0000000000 --- a/test/utils/build-and-push-images.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -# Copyright 2021 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. >/dev/null && pwd)" -source $ROOT/dev/util.sh - -# registry address -host=$1 - -# images to build -api_images=( - "async-text-generator-cpu" - "async-text-generator-gpu" - "batch-image-classifier-alexnet-cpu" - "batch-image-classifier-alexnet-gpu" - "batch-sum-cpu" - "realtime-image-classifier-resnet50-cpu" - "realtime-image-classifier-resnet50-gpu" - "realtime-prime-generator-cpu" - "realtime-sleep-cpu" - "realtime-text-generator-cpu" - "realtime-text-generator-gpu" - "realtime-hello-world-cpu" - "task-iris-classifier-trainer-cpu" -) - -# build the images -for image in "${api_images[@]}"; do - kind=$(python -c "first_element='$image'.split('-', 1)[0]; print(first_element)") - api_name=$(python -c "right_tail='$image'.split('-', 1)[1]; mid_section=right_tail.rsplit('-', 1)[0]; print(mid_section)") - compute_type=$(python -c "last_element='$image'.rsplit('-', 1)[1]; print(last_element)") - dir="${ROOT}/test/apis/${kind}/${api_name}" - - blue_echo "Building $host/cortexlabs-test/$image:latest..." - docker build $dir -f $dir/$api_name-$compute_type.dockerfile -t cortexlabs-test/$image -t $host/cortexlabs-test/$image - green_echo "Built $host/cortexlabs-test/$image:latest\n" -done - -# push the images -for image in "${api_images[@]}"; do - blue_echo "Pushing $host/cortexlabs-test/$image:latest..." - docker push $host/cortexlabs-test/${image} - green_echo "Pushed $host/cortexlabs-test/$image:latest\n" -done diff --git a/test/utils/build.sh b/test/utils/build.sh new file mode 100755 index 0000000000..0020fea488 --- /dev/null +++ b/test/utils/build.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +# Copyright 2021 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# usage: ./build.sh build PATH [REGISTRY] [--skip-push] +# PATH is e.g. /home/ubuntu/src/github.com/cortexlabs/cortex/test/apis/realtime/sleep/build-cpu.sh +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +set -eo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. >/dev/null && pwd)" +source $ROOT/dev/util.sh + +function registry_login() { + login_url=$1 + region=$2 + + blue_echo "\nLogging in to ECR..." + aws ecr get-login-password --region $region | docker login --username AWS --password-stdin $login_url + green_echo "\nSuccess" +} + +function create_ecr_repo() { + repo_name=$1 + region=$2 + + blue_echo "\nCreating ECR repo $repo_name..." + aws ecr create-repository --repository-name=$repo_name --region=$region + green_echo "\nSuccess" +} + +path="$1" +registry="$CORTEX_DEV_DEFAULT_IMAGE_REGISTRY" +should_skip_push="false" +for arg in "${@:2}"; do + if [ "$arg" = "--skip-push" ]; then + should_skip_push="true" + else + registry="$arg" + fi +done + +if [ -z "$registry" ]; then + error_echo "registry must be provided as a positional arg, or $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY must be set" +fi + +name="$(basename $(dirname "$path"))" # e.g. sleep +kind="$(basename $(dirname $(dirname "$path")))" # e.g. realtime +architecture="$(echo "$(basename "$path" .sh)" | sed 's/.*-//')" # e.g. cpu +image_url="$registry/$kind-$name-$architecture" # e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-cpu +if [[ "$image_url" == *".ecr."* ]]; then + login_url="$(echo "$image_url" | sed 's/\/.*//')" # e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com + repo_name="$(echo $image_url | sed 's/[^\/]*\///')" # e.g. cortexlabs/realtime-sleep-cpu + region="$(echo "$image_url" | sed 's/.*\.ecr\.//' | sed 's/\..*//')" # e.g. us-west-2 +fi + +blue_echo "Building $image_url:latest...\n" +docker build "$(dirname "$path")" -f "$(dirname "$path")/$name-$architecture.dockerfile" -t "$image_url" +green_echo "\nBuilt $image_url:latest" + +if [ "$should_skip_push" = "true" ]; then + exit 0 +fi + +while true; do + blue_echo "\nPushing $image_url:latest..." + exec 5>&1 + set +e + out=$(docker push $image_url 2>&1 | tee >(cat - >&5)) + set -e + if [[ "$image_url" == *".ecr."* ]]; then + if [[ "$out" == *"authorization token has expired"* ]] || [[ "$out" == *"no basic auth credentials"* ]]; then + registry_login $login_url $region + continue + elif [[ "$out" == *"does not exist"* ]]; then + create_ecr_repo $repo_name $region + continue + fi + fi + green_echo "\nPushed $image_url:latest" + break +done From c0b85bf1851609e2bbe4b2eb467113fc46d34d28 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 2 Jun 2021 09:37:47 -0700 Subject: [PATCH 60/82] Update test/utils/build.sh --- test/utils/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/utils/build.sh b/test/utils/build.sh index 0020fea488..baa643514a 100755 --- a/test/utils/build.sh +++ b/test/utils/build.sh @@ -24,7 +24,7 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. >/dev/null && pwd)" source $ROOT/dev/util.sh function registry_login() { - login_url=$1 + login_url=$1 # e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-cpu region=$2 blue_echo "\nLogging in to ECR..." @@ -33,7 +33,7 @@ function registry_login() { } function create_ecr_repo() { - repo_name=$1 + repo_name=$1 # e.g. cortexlabs/realtime-sleep-cpu region=$2 blue_echo "\nCreating ECR repo $repo_name..." @@ -78,7 +78,7 @@ while true; do blue_echo "\nPushing $image_url:latest..." exec 5>&1 set +e - out=$(docker push $image_url 2>&1 | tee >(cat - >&5)) + out=$(docker push $image_url 2>&1 | tee /dev/fd/5; exit ${PIPESTATUS[0]}) set -e if [[ "$image_url" == *".ecr."* ]]; then if [[ "$out" == *"authorization token has expired"* ]] || [[ "$out" == *"no basic auth credentials"* ]]; then From 596a5564ba820a828f808cdc4c50ccbe345008fa Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 2 Jun 2021 10:42:36 -0700 Subject: [PATCH 61/82] Add docs for chaining APIs --- docs/workloads/async/container.md | 21 +++++++++++++++++++++ docs/workloads/batch/container.md | 23 +++++++++++++++++++++++ docs/workloads/realtime/container.md | 11 ++++++++--- docs/workloads/task/container.md | 17 +++++++++++++++++ 4 files changed, 69 insertions(+), 3 deletions(-) diff --git a/docs/workloads/async/container.md b/docs/workloads/async/container.md index 290145c6e9..4ed9fb5f72 100644 --- a/docs/workloads/async/container.md +++ b/docs/workloads/async/container.md @@ -32,3 +32,24 @@ The `/mnt` directory is mounted to each container's filesystem, and is shared ac It is possible to use the Cortex CLI or client to interact with your cluster's APIs from within your API containers. All containers will have a CLI configuration file present at `/cortex/client/cli.yaml`, which is configured to connect to the cluster. In addition, the `CORTEX_CLI_CONFIG_DIR` environment variable is set to `/cortex/client` by default. Therefore, no additional configuration is required to use the CLI or Python client (which can be instantiated via `cortex.client()`). Note: your Cortex CLI or client must match the version of your cluster (available in the `CORTEX_VERSION` environment variable). + +## Chaining APIs + +It is possible to submit requests to Async APIs from any Cortex API within a Cortex cluster. Requests can be made to `http://ingressgateway-apis.istio-system.svc.cluster.local/`, where `` is the name of the Async API you are making a request to. + +For example, if there is an Async API named `my-api` running in the cluster, you can make a request to it from a different API in Python by using: + +```python +import requests + +# make a request to an Async API +response = requests.post( + "http://ingressgateway-apis.istio-system.svc.cluster.local/my-api", + json={"text": "hello world"}, +) + +# retreive a result from an Async API +response = requests.get("http://ingressgateway-apis.istio-system.svc.cluster.local/my-api/") +``` + +To make requests from your Async API to a Realtime, Batch, or Task API running within the cluster, see the "Chaining APIs" docs associated with the target workload type. diff --git a/docs/workloads/batch/container.md b/docs/workloads/batch/container.md index 2be5439f94..39ff603708 100644 --- a/docs/workloads/batch/container.md +++ b/docs/workloads/batch/container.md @@ -38,3 +38,26 @@ The `/mnt` directory is mounted to each container's filesystem, and is shared ac It is possible to use the Cortex CLI or client to interact with your cluster's APIs from within your API containers. All containers will have a CLI configuration file present at `/cortex/client/cli.yaml`, which is configured to connect to the cluster. In addition, the `CORTEX_CLI_CONFIG_DIR` environment variable is set to `/cortex/client` by default. Therefore, no additional configuration is required to use the CLI or Python client (which can be instantiated via `cortex.client()`). Note: your Cortex CLI or client must match the version of your cluster (available in the `CORTEX_VERSION` environment variable). + +## Chaining APIs + +It is possible to submit Batch jobs from any Cortex API within a Cortex cluster. Jobs can be submitted to `http://ingressgateway-operator.istio-system.svc.cluster.local/batch/`, where `` is the name of the Batch API you are making a request to. + +For example, if there is a Batch API named `my-api` running in the cluster, you can make a request to it from a different API in Python by using: + +```python +import requests + +job_spec = { + "workers": 1, + "item_list": {"items": [...], "batch_size": 10}, + "config": {"my_key": "my_value"}, +} + +response = requests.post( + "http://ingressgateway-operator.istio-system.svc.cluster.local/batch/my-api", + json=job_spec, +) +``` + +To make requests from your Batch API to a Realtime, Task, or Async API running within the cluster, see the "Chaining APIs" docs associated with the target workload type. diff --git a/docs/workloads/realtime/container.md b/docs/workloads/realtime/container.md index 43f3517da4..c0e24650ee 100644 --- a/docs/workloads/realtime/container.md +++ b/docs/workloads/realtime/container.md @@ -33,14 +33,19 @@ Note: your Cortex CLI or client must match the version of your cluster (availabl ## Chaining APIs -It is possible to make requests from any Cortex API type to a Realtime API within a Cortex cluster. All running APIs are accessible at `http://api-:8888/`, where `` is the name of the API you are making a request to. +It is possible to make requests to Realtime APIs from any Cortex API within a Cortex cluster. Requests can be made to `http://ingressgateway-apis.istio-system.svc.cluster.local/`, where `` is the name of the Realtime API you are making a request to. -For example, if there is a Realtime api named `my-api` running in the cluster, you could make a request to it from a different API by using: +For example, if there is a Realtime API named `my-api` running in the cluster, you can make a request to it from a different API in Python by using: ```python import requests -response = requests.post("http://api-my-api:8888/", json={"text": "hello world"}) +response = requests.post( + "http://ingressgateway-apis.istio-system.svc.cluster.local/my-api", + json={"text": "hello world"}, +) ``` Note that if the API making the request is a Realtime API or Async API, its autoscaling configuration (i.e. `target_in_flight`) should be modified with the understanding that requests will be considered "in-flight" in the first API as the request is being fulfilled by the second API. + +To make requests from your Realtime API to a Batch, Async, or Task API running within the cluster, see the "Chaining APIs" docs associated with the target workload type. diff --git a/docs/workloads/task/container.md b/docs/workloads/task/container.md index d68e206a9e..2f74096d1f 100644 --- a/docs/workloads/task/container.md +++ b/docs/workloads/task/container.md @@ -13,3 +13,20 @@ Your Task's pod can contain multiple containers. The `/mnt` directory is mounted It is possible to use the Cortex CLI or client to interact with your cluster's APIs from within your API containers. All containers will have a CLI configuration file present at `/cortex/client/cli.yaml`, which is configured to connect to the cluster. In addition, the `CORTEX_CLI_CONFIG_DIR` environment variable is set to `/cortex/client` by default. Therefore, no additional configuration is required to use the CLI or Python client (which can be instantiated via `cortex.client()`). Note: your Cortex CLI or client must match the version of your cluster (available in the `CORTEX_VERSION` environment variable). + +## Chaining APIs + +It is possible to submit Task jobs from any Cortex API within a Cortex cluster. Jobs can be submitted to `http://ingressgateway-operator.istio-system.svc.cluster.local/tasks/`, where `` is the name of the Task API you are making a request to. + +For example, if there is a Task API named `my-api` running in the cluster, you can make a request to it from a different API in Python by using: + +```python +import requests + +response = requests.post( + "http://ingressgateway-operator.istio-system.svc.cluster.local/tasks/my-api", + json={"config": {"my_key": "my_value"}}, +) +``` + +To make requests from your Task API to a Realtime, Batch, or Async API running within the cluster, see the "Chaining APIs" docs associated with the target workload type. From 6a966bf904771ac84ef5aa009a0eb1cdfa3e57cb Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 2 Jun 2021 11:00:17 -0700 Subject: [PATCH 62/82] Update docs --- docs/clusters/observability/logging.md | 2 +- docs/workloads/async/container.md | 4 ++++ docs/workloads/batch/container.md | 4 ++++ docs/workloads/realtime/container.md | 4 ++++ docs/workloads/task/container.md | 4 ++++ 5 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/clusters/observability/logging.md b/docs/clusters/observability/logging.md index f94e99da22..ea223f3a59 100644 --- a/docs/clusters/observability/logging.md +++ b/docs/clusters/observability/logging.md @@ -4,7 +4,7 @@ Logs are collected with Fluent Bit and are exported to CloudWatch. ## Logs on AWS -Logs will automatically be pushed to CloudWatch and a log group with the same name as your cluster will be created to store your logs. API logs are tagged with labels to help with log aggregation and filtering. +Logs will automatically be pushed to CloudWatch and a log group with the same name as your cluster will be created to store your logs. API logs are tagged with labels to help with log aggregation and filtering. Log lines greater than 5 MB in size will be ignored. You can use the `cortex logs` command to get a CloudWatch Insights URL of query to fetch logs for your API. Please note that there may be a few minutes of delay from when a message is logged to when it is available in CloudWatch Insights. diff --git a/docs/workloads/async/container.md b/docs/workloads/async/container.md index 4ed9fb5f72..51474468fe 100644 --- a/docs/workloads/async/container.md +++ b/docs/workloads/async/container.md @@ -27,6 +27,10 @@ Your API pod can contain multiple containers, only one of which can be listening The `/mnt` directory is mounted to each container's filesystem, and is shared across all containers. +## Observability + +See docs for [logging](../../clusters/observability/logging.md), [metrics](../../clusters/observability/metrics.md), and [alerting](../../clusters/observability/metrics.md). + ## Using the Cortex CLI or client It is possible to use the Cortex CLI or client to interact with your cluster's APIs from within your API containers. All containers will have a CLI configuration file present at `/cortex/client/cli.yaml`, which is configured to connect to the cluster. In addition, the `CORTEX_CLI_CONFIG_DIR` environment variable is set to `/cortex/client` by default. Therefore, no additional configuration is required to use the CLI or Python client (which can be instantiated via `cortex.client()`). diff --git a/docs/workloads/batch/container.md b/docs/workloads/batch/container.md index 39ff603708..1634dfcb22 100644 --- a/docs/workloads/batch/container.md +++ b/docs/workloads/batch/container.md @@ -33,6 +33,10 @@ Your API pod can contain multiple containers, only one of which can be listening The `/mnt` directory is mounted to each container's filesystem, and is shared across all containers. +## Observability + +See docs for [logging](../../clusters/observability/logging.md), [metrics](../../clusters/observability/metrics.md), and [alerting](../../clusters/observability/metrics.md). + ## Using the Cortex CLI or client It is possible to use the Cortex CLI or client to interact with your cluster's APIs from within your API containers. All containers will have a CLI configuration file present at `/cortex/client/cli.yaml`, which is configured to connect to the cluster. In addition, the `CORTEX_CLI_CONFIG_DIR` environment variable is set to `/cortex/client` by default. Therefore, no additional configuration is required to use the CLI or Python client (which can be instantiated via `cortex.client()`). diff --git a/docs/workloads/realtime/container.md b/docs/workloads/realtime/container.md index c0e24650ee..4fe57c985a 100644 --- a/docs/workloads/realtime/container.md +++ b/docs/workloads/realtime/container.md @@ -25,6 +25,10 @@ Your API pod can contain multiple containers, only one of which can be listening The `/mnt` directory is mounted to each container's file system, and is shared across all containers. +## Observability + +See docs for [logging](../../clusters/observability/logging.md), [metrics](../../clusters/observability/metrics.md), and [alerting](../../clusters/observability/metrics.md). + ## Using the Cortex CLI or client It is possible to use the Cortex CLI or client to interact with your cluster's APIs from within your API containers. All containers will have a CLI configuration file present at `/cortex/client/cli.yaml`, which is configured to connect to the cluster. In addition, the `CORTEX_CLI_CONFIG_DIR` environment variable is set to `/cortex/client` by default. Therefore, no additional configuration is required to use the CLI or Python client (which can be instantiated via `cortex.client()`). diff --git a/docs/workloads/task/container.md b/docs/workloads/task/container.md index 2f74096d1f..bf742d57ff 100644 --- a/docs/workloads/task/container.md +++ b/docs/workloads/task/container.md @@ -8,6 +8,10 @@ If you need access to any parameters in the job submission (e.g. `config`), the Your Task's pod can contain multiple containers. The `/mnt` directory is mounted to each container's filesystem, and is shared across all containers. +## Observability + +See docs for [logging](../../clusters/observability/logging.md), [metrics](../../clusters/observability/metrics.md), and [alerting](../../clusters/observability/metrics.md). + ## Using the Cortex CLI or client It is possible to use the Cortex CLI or client to interact with your cluster's APIs from within your API containers. All containers will have a CLI configuration file present at `/cortex/client/cli.yaml`, which is configured to connect to the cluster. In addition, the `CORTEX_CLI_CONFIG_DIR` environment variable is set to `/cortex/client` by default. Therefore, no additional configuration is required to use the CLI or Python client (which can be instantiated via `cortex.client()`). From 8a05a140eeb1be8d564cea0aea85c44438897413 Mon Sep 17 00:00:00 2001 From: vishal Date: Wed, 2 Jun 2021 14:35:51 -0400 Subject: [PATCH 63/82] Remove hard coded API name and Job ID values in _completedJobLogURLTemplate --- pkg/operator/operator/workload_logging.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/operator/operator/workload_logging.go b/pkg/operator/operator/workload_logging.go index a2b4ee198a..47d97c405e 100644 --- a/pkg/operator/operator/workload_logging.go +++ b/pkg/operator/operator/workload_logging.go @@ -53,11 +53,11 @@ func timeString(t time.Time) string { } var _apiLogURLTemplate *template.Template = template.Must(template.New("api_log_url_template").Parse(strings.TrimSpace(` - https://console.{{.Partition}}.com/cloudwatch/home?region={{.Region}}#logsV2:logs-insights$3FqueryDetail$3D$257E$2528end$257E0$257Estart$257E-3600$257EtimeType$257E$2527RELATIVE$257Eunit$257E$2527seconds$257EeditorString$257E$2527fields*20*40timestamp*2c*20message*0a*7c*20filter*20cortex.labels.apiName*3d*22{{.APIName}}*22*0a*7c*20sort*20*40timestamp*20asc*0a$257Esource$257E$2528$257E$2527{{.LogGroup}}$2529$2529 +https://console.{{.Partition}}.com/cloudwatch/home?region={{.Region}}#logsV2:logs-insights$3FqueryDetail$3D$257E$2528end$257E0$257Estart$257E-3600$257EtimeType$257E$2527RELATIVE$257Eunit$257E$2527seconds$257EeditorString$257E$2527fields*20*40timestamp*2c*20message*0a*7c*20filter*20cortex.labels.apiName*3d*22{{.APIName}}*22*0a*7c*20sort*20*40timestamp*20asc*0a$257Esource$257E$2528$257E$2527{{.LogGroup}}$2529$2529 `))) var _completedJobLogURLTemplate *template.Template = template.Must(template.New("completed_job_log_url_template").Parse(strings.TrimSpace(` -https://console.{{.Partition}}.com/cloudwatch/home?region={{.Region}}#logsV2:logs-insights$3FqueryDetail$3D$257E$2528end$257E$2527{{.EndTime}}$257Estart$257E$2527{{.StartTime}}$257EtimeType$257E$2527ABSOLUTE$257Etz$257E$2527Local$257EeditorString$257E$2527fields*20*40timestamp*2c*20message*0a*7c*20filter*20cortex.labels.apiName*3d*22sum*22*20and*20cortex.labels.jobID*3d*22697ba471c134ea2e*22*0a*7c*20sort*20*40timestamp*20asc*0a$257Esource$257E$2528$257E$2527{{.LogGroup}}$2529$2529 +https://console.{{.Partition}}.com/cloudwatch/home?region={{.Region}}#logsV2:logs-insights$3FqueryDetail$3D$257E$2528end$257E$2527{{.EndTime}}$257Estart$257E$2527{{.StartTime}}$257EtimeType$257E$2527ABSOLUTE$257Etz$257E$2527Local$257EeditorString$257E$2527fields*20*40timestamp*2c*20message*0a*7c*20filter*20cortex.labels.apiName*3d*22{{.APIName}}*22*20and*20cortex.labels.jobID*3d*22{{.JobID}}*22*0a*7c*20sort*20*40timestamp*20asc*0a$257Esource$257E$2528$257E$2527{{.LogGroup}}$2529$2529 `))) var _inProgressJobLogsURLTemplate *template.Template = template.Must(template.New("in_progress_job_log_url_template").Parse(strings.TrimSpace(` From 234338398353948acd9bd5a1ee6371c99f3a3129 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 2 Jun 2021 11:49:41 -0700 Subject: [PATCH 64/82] Fix async API queue deletion during cluster down (#2216) --- cli/cmd/cluster.go | 16 ++++------------ pkg/lib/aws/sqs.go | 11 +++++++---- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/cli/cmd/cluster.go b/cli/cmd/cluster.go index 7bb7f38387..b26744e872 100644 --- a/cli/cmd/cluster.go +++ b/cli/cmd/cluster.go @@ -484,25 +484,17 @@ var _clusterDownCmd = &cobra.Command{ loadBalancer, _ := getLoadBalancer(accessConfig.ClusterName, OperatorLoadBalancer, awsClient) fmt.Print("○ deleting sqs queues ... ") - if queueExists, err := awsClient.DoesQueueExist(clusterconfig.SQSNamePrefix(accessConfig.ClusterName)); err != nil { + numDeleted, err := awsClient.DeleteQueuesWithPrefix(clusterconfig.SQSNamePrefix(accessConfig.ClusterName)) + if err != nil { errorsList = append(errorsList, err) fmt.Print("failed ✗") fmt.Printf("\n\nfailed to delete all sqs queues; please delete queues starting with the name %s via the cloudwatch console: https://%s.console.aws.amazon.com/sqs/v2/home\n", clusterconfig.SQSNamePrefix(accessConfig.ClusterName), accessConfig.Region) errors.PrintError(err) fmt.Println() - } else if !queueExists { + } else if numDeleted == 0 { fmt.Println("no sqs queues exist ✓") } else { - err = awsClient.DeleteQueuesWithPrefix(clusterconfig.SQSNamePrefix(accessConfig.ClusterName)) - if err != nil { - fmt.Print("failed ✗") - errorsList = append(errorsList, err) - fmt.Printf("\n\nfailed to delete all sqs queues; please delete queues starting with the name %s via the cloudwatch console: https://%s.console.aws.amazon.com/sqs/v2/home\n", clusterconfig.SQSNamePrefix(accessConfig.ClusterName), accessConfig.Region) - errors.PrintError(err) - fmt.Println() - } else { - fmt.Println("✓") - } + fmt.Println("✓") } clusterDoesntExist := !clusterExists diff --git a/pkg/lib/aws/sqs.go b/pkg/lib/aws/sqs.go index 2c7a49e0d5..1c68849d4e 100644 --- a/pkg/lib/aws/sqs.go +++ b/pkg/lib/aws/sqs.go @@ -64,7 +64,8 @@ func (c *Client) DoesQueueExist(queueName string) (bool, error) { return true, nil } -func (c *Client) DeleteQueuesWithPrefix(queueNamePrefix string) error { +func (c *Client) DeleteQueuesWithPrefix(queueNamePrefix string) (int, error) { + var numDeleted int var deleteError error err := c.SQS().ListQueuesPages(&sqs.ListQueuesInput{ @@ -75,6 +76,8 @@ func (c *Client) DeleteQueuesWithPrefix(queueNamePrefix string) error { QueueUrl: queueURL, }) + numDeleted++ + if deleteError != nil { deleteError = err } @@ -83,12 +86,12 @@ func (c *Client) DeleteQueuesWithPrefix(queueNamePrefix string) error { }) if err != nil { - return errors.WithStack(err) + return 0, errors.WithStack(err) } if deleteError != nil { - return errors.WithStack(deleteError) + return 0, errors.WithStack(deleteError) } - return nil + return numDeleted, nil } From d4d7095ffe8f1071acf6c374f31a9f76f005fb2e Mon Sep 17 00:00:00 2001 From: Vishal Bollu Date: Wed, 2 Jun 2021 15:00:29 -0400 Subject: [PATCH 65/82] Prevent nil pointer and improve logging on failed health checks (#2211) --- cmd/proxy/main.go | 4 ++-- pkg/probe/probe.go | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 06330016db..f162be3d67 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -208,13 +208,13 @@ func readinessTCPHandler(port int, logger *zap.SugaredLogger) http.HandlerFunc { address := net.JoinHostPort("localhost", strconv.FormatInt(int64(port), 10)) conn, err := net.DialTimeout("tcp", address, timeout) - _ = conn.Close() if err != nil { - logger.Warn(err) + logger.Warn(errors.Wrap(err, "TCP probe to user-provided container port failed")) w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte("unhealthy")) return } + _ = conn.Close() w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("healthy")) diff --git a/pkg/probe/probe.go b/pkg/probe/probe.go index 94a554a71e..e79e141975 100644 --- a/pkg/probe/probe.go +++ b/pkg/probe/probe.go @@ -25,6 +25,7 @@ import ( "net/url" "time" + "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" "go.uber.org/zap" kcore "k8s.io/api/core/v1" @@ -160,7 +161,7 @@ func (p *Probe) probeContainer() bool { } if err != nil { - p.logger.Warn(err) + p.logger.Warn(errors.Wrap(err, "probe to user provided containers failed")) return false } return true From 8a9a9a60f4b1f99eb2d7d7ae70f473861562d273 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Wed, 2 Jun 2021 22:05:55 +0300 Subject: [PATCH 66/82] CaaS - E2E fixes (#2212) --- .circleci/config.yml | 4 ++-- test/e2e/e2e/tests.py | 20 +++++++++++++++++++- test/e2e/e2e/utils.py | 26 ++++++++++++++++++-------- test/e2e/tests/conftest.py | 16 ++++++++-------- 4 files changed, 47 insertions(+), 19 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f32ce2ef1a..60ad133371 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -159,8 +159,8 @@ jobs: node_groups: - name: spot instance_type: t3.medium - min_instances: 10 - max_instances: 10 + min_instances: 16 + max_instances: 16 spot: true - name: cpu instance_type: c5.xlarge diff --git a/test/e2e/e2e/tests.py b/test/e2e/e2e/tests.py index 3bc72af04d..a78fb5e058 100644 --- a/test/e2e/e2e/tests.py +++ b/test/e2e/e2e/tests.py @@ -542,6 +542,7 @@ def test_load_realtime( # controls the flow of requests request_stopper = td.Event() latencies: List[float] = [] + failed = False try: printer(f"getting {desired_replicas} replicas ready") assert apis_ready( @@ -623,6 +624,7 @@ def test_load_realtime( except: # best effort + failed = True try: api_info = client.get_api(api_name) printer(json.dumps(api_info, indent=2)) @@ -632,6 +634,8 @@ def test_load_realtime( finally: request_stopper.set() delete_apis(client, [api_name]) + if failed: + time.sleep(30) def test_load_async( @@ -665,6 +669,7 @@ def test_load_async( request_stopper = td.Event() map_stopper = td.Event() responses: List[Dict[str, Any]] = [] + failed = False try: printer(f"getting {desired_replicas} replicas ready") assert apis_ready( @@ -738,6 +743,7 @@ def test_load_async( except: # best effort + failed = True try: api_info = client.get_api(api_name) printer(json.dumps(api_info, indent=2)) @@ -749,6 +755,8 @@ def test_load_async( printer(f"{len(results)}/{total_requests} have been successfully retrieved") map_stopper.set() delete_apis(client, [api_name]) + if failed: + time.sleep(30) def test_load_batch( @@ -786,7 +794,7 @@ def test_load_batch( api_name = api_specs[0]["name"] client.deploy(api_spec=api_specs[0]) api_endpoint = client.get_api(api_name)["endpoint"] - + failed = False try: assert endpoint_ready( client=client, api_name=api_name, timeout=deploy_timeout @@ -840,6 +848,7 @@ def test_load_batch( except: # best effort + failed = True try: api_info = client.get_api(api_name) @@ -853,6 +862,8 @@ def test_load_batch( finally: delete_apis(client, [api_name]) + if failed: + time.sleep(30) def test_load_task( @@ -881,6 +892,7 @@ def test_load_task( request_stopper = td.Event() map_stopper = td.Event() + failed = False try: assert endpoint_ready( client=client, api_name=api_name, timeout=deploy_timeout @@ -902,6 +914,9 @@ def test_load_task( check_futures_healthy(threads_futures) wait_on_futures(threads_futures) + # give it a bit of a delay to avoid overloading + time.sleep(1) + printer("waiting on the jobs") job_ids = [job_spec.json()["job_id"] for job_spec in job_specs] retrieve_results_concurrently( @@ -916,6 +931,7 @@ def test_load_task( except: # best effort + failed = True try: api_info = client.get_api(api_name) @@ -930,6 +946,8 @@ def test_load_task( finally: map_stopper.set() delete_apis(client, [api_name]) + if failed: + time.sleep(30) def test_long_running_realtime( diff --git a/test/e2e/e2e/utils.py b/test/e2e/e2e/utils.py index f5223f3d47..6a4cfc1252 100644 --- a/test/e2e/e2e/utils.py +++ b/test/e2e/e2e/utils.py @@ -374,19 +374,29 @@ def _retriever(request_id: str): continue result_response_json = result_response.json() - if ( - async_kind - and "status" in result_response_json - and result_response_json["status"] == "completed" - ): - break + if async_kind and "status" in result_response_json: + if result_response_json["status"] == "completed": + break + if result_response_json["status"] not in ["in_progress", "in_queue"]: + raise RuntimeError( + f"status for request ID {request_id} got set to {result_response_json['status']}" + ) + if ( task_kind and "job_status" in result_response_json and "status" in result_response_json["job_status"] - and result_response_json["job_status"]["status"] == "status_succeeded" ): - break + if result_response_json["job_status"]["status"] == "succeeded": + break + if result_response_json["job_status"]["status"] not in [ + "pending", + "enqueuing", + "running", + ]: + raise RuntimeError( + f"status for job ID {request_id} got set to {result_response_json['job_status']['status']}" + ) if event_stopper.is_set(): return diff --git a/test/e2e/tests/conftest.py b/test/e2e/tests/conftest.py index 147a906abb..9a23ac1d0f 100644 --- a/test/e2e/tests/conftest.py +++ b/test/e2e/tests/conftest.py @@ -86,13 +86,13 @@ def pytest_configure(config): "realtime_deploy_timeout": int( os.environ.get("CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT", 200) ), - "batch_deploy_timeout": int(os.environ.get("CORTEX_TEST_BATCH_DEPLOY_TIMEOUT", 30)), + "batch_deploy_timeout": int(os.environ.get("CORTEX_TEST_BATCH_DEPLOY_TIMEOUT", 150)), "batch_job_timeout": int(os.environ.get("CORTEX_TEST_BATCH_JOB_TIMEOUT", 200)), - "async_deploy_timeout": int(os.environ.get("CORTEX_TEST_ASYNC_DEPLOY_TIMEOUT", 120)), + "async_deploy_timeout": int(os.environ.get("CORTEX_TEST_ASYNC_DEPLOY_TIMEOUT", 150)), "async_workload_timeout": int( os.environ.get("CORTEX_TEST_ASYNC_WORKLOAD_TIMEOUT", 200) ), - "task_deploy_timeout": int(os.environ.get("CORTEX_TEST_TASK_DEPLOY_TIMEOUT", 30)), + "task_deploy_timeout": int(os.environ.get("CORTEX_TEST_TASK_DEPLOY_TIMEOUT", 75)), "task_job_timeout": int(os.environ.get("CORTEX_TEST_TASK_JOB_TIMEOUT", 200)), "skip_gpus": config.getoption("--skip-gpus"), "skip_infs": config.getoption("--skip-infs"), @@ -104,7 +104,7 @@ def pytest_configure(config): }, "load_test_config": { "realtime": { - "total_requests": 10 ** 6, + "total_requests": 10 ** 5, "desired_replicas": 50, "concurrency": 50, "min_rtt": 0.004, # measured in seconds @@ -115,7 +115,7 @@ def pytest_configure(config): }, "async": { "total_requests": 10 ** 3, - "desired_replicas": 50, + "desired_replicas": 20, "concurrency": 10, "submit_timeout": 120, # measured in seconds "workload_timeout": 120, # measured in seconds @@ -125,13 +125,13 @@ def pytest_configure(config): "workers_per_job": 10, "items_per_job": 10 ** 5, "batch_size": 10 * 2, - "workload_timeout": 210, # measured in seconds + "workload_timeout": 200, # measured in seconds }, "task": { "jobs": 10 ** 2, "concurrency": 4, - "submit_timeout": 240, # measured in seconds - "workload_timeout": 180, # measured in seconds + "submit_timeout": 200, # measured in seconds + "workload_timeout": 400, # measured in seconds }, }, "long_running_test_config": { From cd19bbf2b18cb35a9519717a94e3bae980959d49 Mon Sep 17 00:00:00 2001 From: Vishal Bollu Date: Wed, 2 Jun 2021 15:11:45 -0400 Subject: [PATCH 67/82] Schedule dequeuer first and then user provided containers for BatchAPI (#2218) --- pkg/workloads/k8s.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/workloads/k8s.go b/pkg/workloads/k8s.go index 74e528adc4..c110163020 100644 --- a/pkg/workloads/k8s.go +++ b/pkg/workloads/k8s.go @@ -293,11 +293,12 @@ func TaskContainers(api spec.API, job *spec.JobKey) ([]kcore.Container, []kcore. } func BatchContainers(api spec.API, job *spec.BatchJob) ([]kcore.Container, []kcore.Volume) { - containers, volumes := userPodContainers(api) + userContainers, userVolumes := userPodContainers(api) dequeuerContainer, dequeuerVolume := batchDequeuerProxyContainer(api, job.ID, job.SQSUrl) - containers = append(containers, dequeuerContainer) - volumes = append(volumes, dequeuerVolume) + // make sure the dequeuer starts first to allow it to start watching the graveyard before user containers begin + containers := append([]kcore.Container{dequeuerContainer}, userContainers...) + volumes := append([]kcore.Volume{dequeuerVolume}, userVolumes...) k8sName := job.K8sName() From 7705d643c27dd878fc53144dd7d79ecac02b90d1 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 2 Jun 2021 12:12:47 -0700 Subject: [PATCH 68/82] Add test/apis/realtime/hello-world/build-cpu.sh --- test/apis/realtime/hello-world/build-cpu.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100755 test/apis/realtime/hello-world/build-cpu.sh diff --git a/test/apis/realtime/hello-world/build-cpu.sh b/test/apis/realtime/hello-world/build-cpu.sh new file mode 100755 index 0000000000..12960564d9 --- /dev/null +++ b/test/apis/realtime/hello-world/build-cpu.sh @@ -0,0 +1,4 @@ +# usage: build-cpu.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" From 531337f56e38cc115bba6a2c0c6e64f5a152454d Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 2 Jun 2021 12:42:45 -0700 Subject: [PATCH 69/82] cache dequeuer in registry.sh --- dev/registry.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev/registry.sh b/dev/registry.sh index 02b17b27b7..2b421a4afd 100755 --- a/dev/registry.sh +++ b/dev/registry.sh @@ -246,6 +246,9 @@ elif [ "$cmd" = "update" ]; then if [[ " ${images_to_build[@]} " =~ " enqueuer " ]]; then cache_builder enqueuer fi + if [[ " ${images_to_build[@]} " =~ " dequeuer " ]]; then + cache_builder dequeuer + fi if [[ " ${images_to_build[@]} " =~ " controller-manager " ]]; then cache_builder controller-manager fi From d21509e1bd3a0a9bdda170a0128be42aaa5f4479 Mon Sep 17 00:00:00 2001 From: Vishal Bollu Date: Wed, 2 Jun 2021 22:56:22 +0000 Subject: [PATCH 70/82] Close the request body in batch handler --- pkg/dequeuer/batch_handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/dequeuer/batch_handler.go b/pkg/dequeuer/batch_handler.go index 150e88a322..f497b39a80 100644 --- a/pkg/dequeuer/batch_handler.go +++ b/pkg/dequeuer/batch_handler.go @@ -128,6 +128,7 @@ func (h *BatchMessageHandler) submitRequest(messageBody string, isOnJobComplete if err != nil { return ErrorUserContainerNotReachable(err) } + defer response.Body.Close() if response.StatusCode == http.StatusNotFound && isOnJobComplete { return nil From 76547a2ae95a289eabb24a637a168fe649719bb9 Mon Sep 17 00:00:00 2001 From: Vishal Bollu Date: Wed, 2 Jun 2021 22:57:00 +0000 Subject: [PATCH 71/82] Persist metrics when status is in completed with failures state --- pkg/crds/controllers/batch/batchjob_controller_helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/crds/controllers/batch/batchjob_controller_helpers.go b/pkg/crds/controllers/batch/batchjob_controller_helpers.go index 7b35461e76..6fcfadfc89 100644 --- a/pkg/crds/controllers/batch/batchjob_controller_helpers.go +++ b/pkg/crds/controllers/batch/batchjob_controller_helpers.go @@ -636,7 +636,7 @@ func (r *BatchJobReconciler) updateCompletedTimestamp(ctx context.Context, batch func (r *BatchJobReconciler) persistJobToS3(batchJob batch.BatchJob) error { return parallel.RunFirstErr( func() error { - if batchJob.Status.Status != status.JobSucceeded { + if batchJob.Status.Status != status.JobSucceeded && batchJob.Status.Status != status.JobCompletedWithFailures { return nil } return r.Config.SaveJobMetrics(r, batchJob) From 0d871ce3156e19baea86e042ec848ec51dcf3fd3 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 2 Jun 2021 21:55:07 -0700 Subject: [PATCH 72/82] Update test APIs --- test/apis/async/hello-world/.dockerignore | 9 ----- test/apis/async/hello-world/{ => app}/main.py | 4 ++- .../async/hello-world/app/requirements.txt | 2 ++ test/apis/async/hello-world/cortex_cpu.yaml | 4 +-- test/apis/async/hello-world/cpu.Dockerfile | 13 +++++++ .../hello-world/hello-world-cpu.dockerfile | 17 --------- test/apis/async/text-generator/.dockerignore | 10 ------ .../async/text-generator/{ => app}/main.py | 31 ++++++++-------- .../text-generator/app/requirements-cpu.txt | 6 ++++ .../text-generator/app/requirements-gpu.txt | 5 +++ .../apis/async/text-generator/cortex_cpu.yaml | 4 +-- .../apis/async/text-generator/cortex_gpu.yaml | 4 +-- test/apis/async/text-generator/cpu.Dockerfile | 13 +++++++ .../async/text-generator/expectations.yaml | 4 +-- test/apis/async/text-generator/gpu.Dockerfile | 24 +++++++++++++ .../text-generator-cpu.dockerfile | 23 ------------ .../text-generator-gpu.dockerfile | 27 -------------- .../image-classifier-alexnet/.dockerignore | 11 ------ .../{ => app}/main.py | 26 +++++++------- .../app/requirements-cpu.txt | 7 ++++ .../app/requirements-gpu.txt | 6 ++++ .../image-classifier-alexnet/cortex_cpu.yaml | 12 +------ .../image-classifier-alexnet/cortex_gpu.yaml | 12 +------ .../image-classifier-alexnet/cpu.Dockerfile | 13 +++++++ .../image-classifier-alexnet/gpu.Dockerfile | 24 +++++++++++++ .../image-classifier-alexnet-cpu.dockerfile | 24 ------------- .../image-classifier-alexnet-gpu.dockerfile | 35 ------------------- test/apis/batch/sum/.dockerignore | 12 ------- test/apis/batch/sum/{ => app}/main.py | 14 ++++---- test/apis/batch/sum/app/requirements.txt | 3 ++ test/apis/batch/sum/cortex_cpu.yaml | 12 +------ test/apis/batch/sum/cpu.Dockerfile | 13 +++++++ test/apis/batch/sum/sum-cpu.dockerfile | 21 ----------- test/apis/realtime/hello-world/.dockerignore | 9 ----- .../realtime/hello-world/{ => app}/main.py | 6 ++-- .../realtime/hello-world/app/requirements.txt | 2 ++ .../apis/realtime/hello-world/cortex_cpu.yaml | 4 +-- test/apis/realtime/hello-world/cpu.Dockerfile | 13 +++++++ .../hello-world/hello-world-cpu.dockerfile | 17 --------- .../image-classifier-resnet50/.dockerignore | 10 ------ ...resnet50-cpu.dockerfile => cpu.Dockerfile} | 4 +-- ...resnet50-gpu.dockerfile => gpu.Dockerfile} | 0 .../realtime/prime-generator/.dockerignore | 10 ------ .../prime-generator/{ => app}/main.py | 14 ++++---- .../prime-generator/app/requirements.txt | 3 ++ .../realtime/prime-generator/cortex_cpu.yaml | 4 +-- .../realtime/prime-generator/cpu.Dockerfile | 13 +++++++ .../prime-generator-cpu.dockerfile | 17 --------- test/apis/realtime/sleep/.dockerignore | 9 ----- test/apis/realtime/sleep/{ => app}/main.py | 4 ++- test/apis/realtime/sleep/app/requirements.txt | 2 ++ test/apis/realtime/sleep/cortex_cpu.yaml | 4 +-- test/apis/realtime/sleep/cpu.Dockerfile | 13 +++++++ test/apis/realtime/sleep/sleep-cpu.dockerfile | 17 --------- .../realtime/text-generator/.dockerignore | 10 ------ .../realtime/text-generator/{ => app}/main.py | 31 ++++++++-------- .../text-generator/app/requirements-cpu.txt | 6 ++++ .../text-generator/app/requirements-gpu.txt | 5 +++ .../realtime/text-generator/cortex_cpu.yaml | 4 +-- .../realtime/text-generator/cortex_gpu.yaml | 4 +-- .../realtime/text-generator/cpu.Dockerfile | 13 +++++++ .../realtime/text-generator/gpu.Dockerfile | 24 +++++++++++++ .../text-generator-cpu.dockerfile | 23 ------------ .../text-generator-gpu.dockerfile | 27 -------------- .../iris-classifier-trainer/.dockerignore | 10 ------ .../iris-classifier-trainer/{ => app}/main.py | 6 +++- .../app/requirements.txt | 3 ++ .../iris-classifier-trainer/cpu.Dockerfile | 11 ++++++ .../iris-classifier-trainer-cpu.dockerfile | 17 --------- .../hello-world/cortex_cpu.yaml | 12 +++---- test/utils/build.sh | 10 +++--- 71 files changed, 343 insertions(+), 493 deletions(-) delete mode 100644 test/apis/async/hello-world/.dockerignore rename test/apis/async/hello-world/{ => app}/main.py (71%) create mode 100644 test/apis/async/hello-world/app/requirements.txt create mode 100644 test/apis/async/hello-world/cpu.Dockerfile delete mode 100644 test/apis/async/hello-world/hello-world-cpu.dockerfile delete mode 100644 test/apis/async/text-generator/.dockerignore rename test/apis/async/text-generator/{ => app}/main.py (66%) create mode 100644 test/apis/async/text-generator/app/requirements-cpu.txt create mode 100644 test/apis/async/text-generator/app/requirements-gpu.txt create mode 100644 test/apis/async/text-generator/cpu.Dockerfile create mode 100644 test/apis/async/text-generator/gpu.Dockerfile delete mode 100644 test/apis/async/text-generator/text-generator-cpu.dockerfile delete mode 100644 test/apis/async/text-generator/text-generator-gpu.dockerfile delete mode 100644 test/apis/batch/image-classifier-alexnet/.dockerignore rename test/apis/batch/image-classifier-alexnet/{ => app}/main.py (92%) create mode 100644 test/apis/batch/image-classifier-alexnet/app/requirements-cpu.txt create mode 100644 test/apis/batch/image-classifier-alexnet/app/requirements-gpu.txt create mode 100644 test/apis/batch/image-classifier-alexnet/cpu.Dockerfile create mode 100644 test/apis/batch/image-classifier-alexnet/gpu.Dockerfile delete mode 100644 test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-cpu.dockerfile delete mode 100644 test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-gpu.dockerfile delete mode 100644 test/apis/batch/sum/.dockerignore rename test/apis/batch/sum/{ => app}/main.py (82%) create mode 100644 test/apis/batch/sum/app/requirements.txt create mode 100644 test/apis/batch/sum/cpu.Dockerfile delete mode 100644 test/apis/batch/sum/sum-cpu.dockerfile delete mode 100644 test/apis/realtime/hello-world/.dockerignore rename test/apis/realtime/hello-world/{ => app}/main.py (59%) create mode 100644 test/apis/realtime/hello-world/app/requirements.txt create mode 100644 test/apis/realtime/hello-world/cpu.Dockerfile delete mode 100644 test/apis/realtime/hello-world/hello-world-cpu.dockerfile delete mode 100644 test/apis/realtime/image-classifier-resnet50/.dockerignore rename test/apis/realtime/image-classifier-resnet50/{image-classifier-resnet50-cpu.dockerfile => cpu.Dockerfile} (89%) rename test/apis/realtime/image-classifier-resnet50/{image-classifier-resnet50-gpu.dockerfile => gpu.Dockerfile} (100%) delete mode 100644 test/apis/realtime/prime-generator/.dockerignore rename test/apis/realtime/prime-generator/{ => app}/main.py (99%) create mode 100644 test/apis/realtime/prime-generator/app/requirements.txt create mode 100644 test/apis/realtime/prime-generator/cpu.Dockerfile delete mode 100644 test/apis/realtime/prime-generator/prime-generator-cpu.dockerfile delete mode 100644 test/apis/realtime/sleep/.dockerignore rename test/apis/realtime/sleep/{ => app}/main.py (58%) create mode 100644 test/apis/realtime/sleep/app/requirements.txt create mode 100644 test/apis/realtime/sleep/cpu.Dockerfile delete mode 100644 test/apis/realtime/sleep/sleep-cpu.dockerfile delete mode 100644 test/apis/realtime/text-generator/.dockerignore rename test/apis/realtime/text-generator/{ => app}/main.py (66%) create mode 100644 test/apis/realtime/text-generator/app/requirements-cpu.txt create mode 100644 test/apis/realtime/text-generator/app/requirements-gpu.txt create mode 100644 test/apis/realtime/text-generator/cpu.Dockerfile create mode 100644 test/apis/realtime/text-generator/gpu.Dockerfile delete mode 100644 test/apis/realtime/text-generator/text-generator-cpu.dockerfile delete mode 100644 test/apis/realtime/text-generator/text-generator-gpu.dockerfile delete mode 100644 test/apis/task/iris-classifier-trainer/.dockerignore rename test/apis/task/iris-classifier-trainer/{ => app}/main.py (95%) create mode 100644 test/apis/task/iris-classifier-trainer/app/requirements.txt create mode 100644 test/apis/task/iris-classifier-trainer/cpu.Dockerfile delete mode 100644 test/apis/task/iris-classifier-trainer/iris-classifier-trainer-cpu.dockerfile diff --git a/test/apis/async/hello-world/.dockerignore b/test/apis/async/hello-world/.dockerignore deleted file mode 100644 index 4a8a96662f..0000000000 --- a/test/apis/async/hello-world/.dockerignore +++ /dev/null @@ -1,9 +0,0 @@ -*.dockerfile -build*.sh -cortex*.yaml -*.pyc -*.pyo -*.pyd -__pycache__ -.pytest_cache -*.md diff --git a/test/apis/async/hello-world/main.py b/test/apis/async/hello-world/app/main.py similarity index 71% rename from test/apis/async/hello-world/main.py rename to test/apis/async/hello-world/app/main.py index d1a6cd91e9..ba4eac1bfb 100644 --- a/test/apis/async/hello-world/main.py +++ b/test/apis/async/hello-world/app/main.py @@ -1,5 +1,7 @@ import os + from fastapi import FastAPI +from fastapi.responses import PlainTextResponse app = FastAPI() @@ -8,7 +10,7 @@ @app.get("/healthz") def healthz(): - return "ok" + return PlainTextResponse("ok") @app.post("/") diff --git a/test/apis/async/hello-world/app/requirements.txt b/test/apis/async/hello-world/app/requirements.txt new file mode 100644 index 0000000000..190ccb7716 --- /dev/null +++ b/test/apis/async/hello-world/app/requirements.txt @@ -0,0 +1,2 @@ +uvicorn[standard] +fastapi diff --git a/test/apis/async/hello-world/cortex_cpu.yaml b/test/apis/async/hello-world/cortex_cpu.yaml index cda010a135..4a1143af7b 100644 --- a/test/apis/async/hello-world/cortex_cpu.yaml +++ b/test/apis/async/hello-world/cortex_cpu.yaml @@ -1,14 +1,14 @@ - name: hello-world kind: AsyncAPI pod: - port: 9000 + port: 8080 containers: - name: api image: quay.io/cortexlabs-test/async-hello-world-cpu:latest readiness_probe: http_get: path: "/healthz" - port: 9000 + port: 8080 compute: cpu: 200m mem: 128M diff --git a/test/apis/async/hello-world/cpu.Dockerfile b/test/apis/async/hello-world/cpu.Dockerfile new file mode 100644 index 0000000000..ea648acdf0 --- /dev/null +++ b/test/apis/async/hello-world/cpu.Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.8-slim + +ENV PYTHONUNBUFFERED TRUE + +COPY app/requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY app /app +WORKDIR /app/ +ENV PYTHONPATH=/app + +ENV CORTEX_PORT=8080 +CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app diff --git a/test/apis/async/hello-world/hello-world-cpu.dockerfile b/test/apis/async/hello-world/hello-world-cpu.dockerfile deleted file mode 100644 index a496f7c881..0000000000 --- a/test/apis/async/hello-world/hello-world-cpu.dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.8-slim - -# Allow statements and log messages to immediately appear in the logs -ENV PYTHONUNBUFFERED True - -# Install production dependencies -RUN pip install --no-cache-dir "uvicorn[standard]" gunicorn fastapi - -# Copy local code to the container image. -COPY . /app -WORKDIR /app/ - -ENV PYTHONPATH=/app -ENV CORTEX_PORT=9000 - -# Run the web service on container startup. -CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/async/text-generator/.dockerignore b/test/apis/async/text-generator/.dockerignore deleted file mode 100644 index 414f79e41f..0000000000 --- a/test/apis/async/text-generator/.dockerignore +++ /dev/null @@ -1,10 +0,0 @@ -expectations.yaml -*.dockerfile -build*.sh -cortex*.yaml -*.pyc -*.pyo -*.pyd -__pycache__ -.pytest_cache -*.md diff --git a/test/apis/async/text-generator/main.py b/test/apis/async/text-generator/app/main.py similarity index 66% rename from test/apis/async/text-generator/main.py rename to test/apis/async/text-generator/app/main.py index 020c7fc9e2..39939e4fc6 100644 --- a/test/apis/async/text-generator/main.py +++ b/test/apis/async/text-generator/app/main.py @@ -1,21 +1,17 @@ import os -from fastapi import FastAPI, Response, status +from fastapi import FastAPI, status +from fastapi.responses import PlainTextResponse from pydantic import BaseModel from transformers import GPT2Tokenizer, GPT2LMHeadModel - -class Request(BaseModel): - text: str - - +app = FastAPI() +device = os.getenv("TARGET_DEVICE", "cpu") state = { - "model_ready": False, + "ready": False, "tokenizer": None, "model": None, } -device = os.getenv("TARGET_DEVICE", "cpu") -app = FastAPI() @app.on_event("startup") @@ -23,13 +19,18 @@ def startup(): global state state["tokenizer"] = GPT2Tokenizer.from_pretrained("gpt2") state["model"] = GPT2LMHeadModel.from_pretrained("gpt2").to(device) - state["model_ready"] = True + state["ready"] = True @app.get("/healthz") -def healthz(response: Response): - if not state["model_ready"]: - response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE +def healthz(): + if state["ready"]: + return PlainTextResponse("ok") + return PlainTextResponse("service unavailable", status_code=status.HTTP_503_SERVICE_UNAVAILABLE) + + +class Request(BaseModel): + text: str @app.post("/") @@ -37,6 +38,4 @@ def text_generator(request: Request): input_length = len(request.text.split()) tokens = state["tokenizer"].encode(request.text, return_tensors="pt").to(device) prediction = state["model"].generate(tokens, max_length=input_length + 20, do_sample=True) - return { - "prediction": state["tokenizer"].decode(prediction[0]), - } + return {"text": state["tokenizer"].decode(prediction[0])} diff --git a/test/apis/async/text-generator/app/requirements-cpu.txt b/test/apis/async/text-generator/app/requirements-cpu.txt new file mode 100644 index 0000000000..91ef8d2188 --- /dev/null +++ b/test/apis/async/text-generator/app/requirements-cpu.txt @@ -0,0 +1,6 @@ +uvicorn[standard] +fastapi +pydantic +transformers==3.0.* +-f https://download.pytorch.org/whl/torch_stable.html +torch==1.7.1+cpu diff --git a/test/apis/async/text-generator/app/requirements-gpu.txt b/test/apis/async/text-generator/app/requirements-gpu.txt new file mode 100644 index 0000000000..9433f26464 --- /dev/null +++ b/test/apis/async/text-generator/app/requirements-gpu.txt @@ -0,0 +1,5 @@ +uvicorn[standard] +fastapi +pydantic +transformers==3.0.* +torch==1.7.* diff --git a/test/apis/async/text-generator/cortex_cpu.yaml b/test/apis/async/text-generator/cortex_cpu.yaml index 4e59b10efe..d78afacdcb 100644 --- a/test/apis/async/text-generator/cortex_cpu.yaml +++ b/test/apis/async/text-generator/cortex_cpu.yaml @@ -1,14 +1,14 @@ - name: text-generator kind: AsyncAPI pod: - port: 9000 + port: 8080 containers: - name: api image: quay.io/cortexlabs-test/async-text-generator-cpu:latest readiness_probe: http_get: path: "/healthz" - port: 9000 + port: 8080 compute: cpu: 1 mem: 2.5G diff --git a/test/apis/async/text-generator/cortex_gpu.yaml b/test/apis/async/text-generator/cortex_gpu.yaml index 6706058859..a18a8bc612 100644 --- a/test/apis/async/text-generator/cortex_gpu.yaml +++ b/test/apis/async/text-generator/cortex_gpu.yaml @@ -1,7 +1,7 @@ - name: text-generator kind: AsyncAPI pod: - port: 9000 + port: 8080 containers: - name: api image: quay.io/cortexlabs-test/async-text-generator-gpu:latest @@ -10,7 +10,7 @@ readiness_probe: http_get: path: "/healthz" - port: 9000 + port: 8080 compute: cpu: 1 gpu: 1 diff --git a/test/apis/async/text-generator/cpu.Dockerfile b/test/apis/async/text-generator/cpu.Dockerfile new file mode 100644 index 0000000000..54a36928ff --- /dev/null +++ b/test/apis/async/text-generator/cpu.Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.8-slim + +ENV PYTHONUNBUFFERED TRUE + +COPY app/requirements-cpu.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY app /app +WORKDIR /app/ +ENV PYTHONPATH=/app + +ENV CORTEX_PORT=8080 +CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app diff --git a/test/apis/async/text-generator/expectations.yaml b/test/apis/async/text-generator/expectations.yaml index 5797bcebae..2f6bb2a07c 100644 --- a/test/apis/async/text-generator/expectations.yaml +++ b/test/apis/async/text-generator/expectations.yaml @@ -3,7 +3,7 @@ response: json_schema: type: "object" properties: - prediction: + text: type: "string" required: - - "prediction" + - "text" diff --git a/test/apis/async/text-generator/gpu.Dockerfile b/test/apis/async/text-generator/gpu.Dockerfile new file mode 100644 index 0000000000..81f9169949 --- /dev/null +++ b/test/apis/async/text-generator/gpu.Dockerfile @@ -0,0 +1,24 @@ +FROM nvidia/cuda:10.2-cudnn8-runtime-ubuntu18.04 + +RUN apt-get update \ + && apt-get install -y \ + python3 \ + python3-pip \ + pkg-config \ + build-essential \ + git \ + cmake \ + && apt-get clean -qq && rm -rf /var/lib/apt/lists/* + +ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 +ENV PYTHONUNBUFFERED TRUE + +COPY app/requirements-gpu.txt /app/requirements.txt +RUN pip3 install --no-cache-dir -r /app/requirements.txt + +COPY app /app +WORKDIR /app/ +ENV PYTHONPATH=/app + +ENV CORTEX_PORT=8080 +CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app diff --git a/test/apis/async/text-generator/text-generator-cpu.dockerfile b/test/apis/async/text-generator/text-generator-cpu.dockerfile deleted file mode 100644 index 6b5367823f..0000000000 --- a/test/apis/async/text-generator/text-generator-cpu.dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM python:3.8-slim - -# Allow statements and log messages to immediately appear in the logs -ENV PYTHONUNBUFFERED True - -# Install production dependencies -RUN pip install --no-cache-dir \ - "uvicorn[standard]" \ - gunicorn \ - fastapi \ - pydantic \ - transformers==3.0.* \ - torch==1.7.1+cpu -f https://download.pytorch.org/whl/torch_stable.html - -# Copy local code to the container image. -COPY . /app -WORKDIR /app/ - -ENV PYTHONPATH=/app -ENV CORTEX_PORT=9000 - -# Run the web service on container startup -CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/async/text-generator/text-generator-gpu.dockerfile b/test/apis/async/text-generator/text-generator-gpu.dockerfile deleted file mode 100644 index 2de181d169..0000000000 --- a/test/apis/async/text-generator/text-generator-gpu.dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -FROM nvidia/cuda:10.2-cudnn8-runtime-ubuntu18.04 - -RUN apt-get update \ - && apt-get install \ - python3 \ - python3-pip \ - pkg-config \ - git \ - build-essential \ - cmake -y \ - && apt-get clean -qq && rm -rf /var/lib/apt/lists/* - -# Allow statements and log messages to immediately appear in the logs -ENV PYTHONUNBUFFERED True - -# Install production dependencies -RUN pip3 install --no-cache-dir "uvicorn[standard]" gunicorn fastapi pydantic transformers==3.0.* torch==1.7.* - -# Copy local code to the container image. -COPY . /app -WORKDIR /app/ - -ENV PYTHONPATH=/app -ENV CORTEX_PORT=9000 - -# Run the web service on container startup. -CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/batch/image-classifier-alexnet/.dockerignore b/test/apis/batch/image-classifier-alexnet/.dockerignore deleted file mode 100644 index 014d70d990..0000000000 --- a/test/apis/batch/image-classifier-alexnet/.dockerignore +++ /dev/null @@ -1,11 +0,0 @@ -sample.json -submit.py -*.dockerfile -build*.sh -cortex*.yaml -*.pyc -*.pyo -*.pyd -__pycache__ -.pytest_cache -*.md diff --git a/test/apis/batch/image-classifier-alexnet/main.py b/test/apis/batch/image-classifier-alexnet/app/main.py similarity index 92% rename from test/apis/batch/image-classifier-alexnet/main.py rename to test/apis/batch/image-classifier-alexnet/app/main.py index d899d88a5f..0ca1aa416f 100644 --- a/test/apis/batch/image-classifier-alexnet/main.py +++ b/test/apis/batch/image-classifier-alexnet/app/main.py @@ -1,17 +1,19 @@ import os, json, re -from typing import List - -from fastapi import FastAPI, Response, Request, status - import requests import torch import torchvision -from torchvision import transforms -from PIL import Image -from io import BytesIO import boto3 +from typing import List +from PIL import Image +from io import BytesIO +from torchvision import transforms +from fastapi import FastAPI, Request, status +from fastapi.responses import PlainTextResponse +app = FastAPI() +device = os.getenv("TARGET_DEVICE", "cpu") +s3 = boto3.client("s3") state = { "ready": False, "model": None, @@ -21,9 +23,6 @@ "bucket": None, "key": None, } -device = os.getenv("TARGET_DEVICE", "cpu") -s3 = boto3.client("s3") -app = FastAPI() @app.on_event("startup") @@ -60,9 +59,10 @@ def startup(): @app.get("/healthz") -def healthz(response: Response): - if not state["ready"]: - response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE +def healthz(): + if state["ready"]: + return PlainTextResponse("ok") + return PlainTextResponse("service unavailable", status_code=status.HTTP_503_SERVICE_UNAVAILABLE) @app.post("/") diff --git a/test/apis/batch/image-classifier-alexnet/app/requirements-cpu.txt b/test/apis/batch/image-classifier-alexnet/app/requirements-cpu.txt new file mode 100644 index 0000000000..6d199faff8 --- /dev/null +++ b/test/apis/batch/image-classifier-alexnet/app/requirements-cpu.txt @@ -0,0 +1,7 @@ +uvicorn[standard] +fastapi +requests +boto3==1.17.* +-f https://download.pytorch.org/whl/torch_stable.html +torchvision==0.8.2+cpu +torch==1.7.1+cpu diff --git a/test/apis/batch/image-classifier-alexnet/app/requirements-gpu.txt b/test/apis/batch/image-classifier-alexnet/app/requirements-gpu.txt new file mode 100644 index 0000000000..2c8780d2a2 --- /dev/null +++ b/test/apis/batch/image-classifier-alexnet/app/requirements-gpu.txt @@ -0,0 +1,6 @@ +uvicorn[standard] +fastapi +requests +boto3==1.17.* +torchvision==0.8.* +torch==1.7.* diff --git a/test/apis/batch/image-classifier-alexnet/cortex_cpu.yaml b/test/apis/batch/image-classifier-alexnet/cortex_cpu.yaml index 3702ba8a36..a96c29b861 100644 --- a/test/apis/batch/image-classifier-alexnet/cortex_cpu.yaml +++ b/test/apis/batch/image-classifier-alexnet/cortex_cpu.yaml @@ -4,17 +4,7 @@ containers: - name: api image: quay.io/cortexlabs-test/batch-image-classifier-alexnet-cpu:latest - command: - - "gunicorn" - - "-k" - - "uvicorn.workers.UvicornWorker" - - "--workers" - - "1" - - "--threads" - - "1" - - "--bind" - - ":$(CORTEX_PORT)" - - "main:app" + command: ["uvicorn", "--workers", "1", "--host", "0.0.0.0", "--port", "$(CORTEX_PORT)", "main:app"] readiness_probe: http_get: path: "/healthz" diff --git a/test/apis/batch/image-classifier-alexnet/cortex_gpu.yaml b/test/apis/batch/image-classifier-alexnet/cortex_gpu.yaml index ec1b10693b..b36fb917a2 100644 --- a/test/apis/batch/image-classifier-alexnet/cortex_gpu.yaml +++ b/test/apis/batch/image-classifier-alexnet/cortex_gpu.yaml @@ -4,17 +4,7 @@ containers: - name: api image: quay.io/cortexlabs-test/batch-image-classifier-alexnet-gpu:latest - command: - - "gunicorn" - - "-k" - - "uvicorn.workers.UvicornWorker" - - "--workers" - - "1" - - "--threads" - - "1" - - "--bind" - - ":$(CORTEX_PORT)" - - "main:app" + command: ["uvicorn", "--workers", "1", "--host", "0.0.0.0", "--port", "$(CORTEX_PORT)", "main:app"] readiness_probe: http_get: path: "/healthz" diff --git a/test/apis/batch/image-classifier-alexnet/cpu.Dockerfile b/test/apis/batch/image-classifier-alexnet/cpu.Dockerfile new file mode 100644 index 0000000000..54a36928ff --- /dev/null +++ b/test/apis/batch/image-classifier-alexnet/cpu.Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.8-slim + +ENV PYTHONUNBUFFERED TRUE + +COPY app/requirements-cpu.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY app /app +WORKDIR /app/ +ENV PYTHONPATH=/app + +ENV CORTEX_PORT=8080 +CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app diff --git a/test/apis/batch/image-classifier-alexnet/gpu.Dockerfile b/test/apis/batch/image-classifier-alexnet/gpu.Dockerfile new file mode 100644 index 0000000000..81f9169949 --- /dev/null +++ b/test/apis/batch/image-classifier-alexnet/gpu.Dockerfile @@ -0,0 +1,24 @@ +FROM nvidia/cuda:10.2-cudnn8-runtime-ubuntu18.04 + +RUN apt-get update \ + && apt-get install -y \ + python3 \ + python3-pip \ + pkg-config \ + build-essential \ + git \ + cmake \ + && apt-get clean -qq && rm -rf /var/lib/apt/lists/* + +ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 +ENV PYTHONUNBUFFERED TRUE + +COPY app/requirements-gpu.txt /app/requirements.txt +RUN pip3 install --no-cache-dir -r /app/requirements.txt + +COPY app /app +WORKDIR /app/ +ENV PYTHONPATH=/app + +ENV CORTEX_PORT=8080 +CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app diff --git a/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-cpu.dockerfile b/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-cpu.dockerfile deleted file mode 100644 index d63d8f72ad..0000000000 --- a/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-cpu.dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM python:3.8-slim - -# Allow statements and log messages to immediately appear in the logs -ENV PYTHONUNBUFFERED True - -# Install production dependencies -RUN pip install --no-cache-dir \ - "uvicorn[standard]" \ - gunicorn \ - fastapi \ - requests \ - torchvision \ - torch \ - boto3==1.17.72 - -# Copy local code to the container image. -COPY . /app -WORKDIR /app/ - -ENV PYTHONPATH=/app -ENV CORTEX_PORT=9000 - -# Run the web service on container startup. -CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-gpu.dockerfile b/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-gpu.dockerfile deleted file mode 100644 index d339842e9d..0000000000 --- a/test/apis/batch/image-classifier-alexnet/image-classifier-alexnet-gpu.dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -FROM nvidia/cuda:10.2-cudnn8-runtime-ubuntu18.04 - -RUN apt-get update \ - && apt-get install \ - python3 \ - python3-pip \ - pkg-config \ - git \ - build-essential \ - cmake -y \ - && apt-get clean -qq && rm -rf /var/lib/apt/lists/* - -# allow statements and log messages to immediately appear in the logs -ENV PYTHONUNBUFFERED True - -# install production dependencies -RUN pip3 install --no-cache-dir \ - "uvicorn[standard]" \ - gunicorn \ - fastapi \ - requests \ - torchvision \ - torch \ - boto3==1.17.72 - - -# copy local code to the container image. -COPY . /app -WORKDIR /app/ - -ENV PYTHONPATH=/app -ENV CORTEX_PORT=9000 - -# run the web service on container startup. -CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/batch/sum/.dockerignore b/test/apis/batch/sum/.dockerignore deleted file mode 100644 index 8030157a8b..0000000000 --- a/test/apis/batch/sum/.dockerignore +++ /dev/null @@ -1,12 +0,0 @@ -sample.json -submit.py -sample_generator.py -*.dockerfile -build*.sh -cortex*.yaml -*.pyc -*.pyo -*.pyd -__pycache__ -.pytest_cache -*.md diff --git a/test/apis/batch/sum/main.py b/test/apis/batch/sum/app/main.py similarity index 82% rename from test/apis/batch/sum/main.py rename to test/apis/batch/sum/app/main.py index d3a732e28f..26ac026c42 100644 --- a/test/apis/batch/sum/main.py +++ b/test/apis/batch/sum/app/main.py @@ -4,16 +4,17 @@ import re from typing import List -from fastapi import FastAPI, Response, Request, status +from fastapi import FastAPI, Request, status +from fastapi.responses import PlainTextResponse +app = FastAPI() +s3 = boto3.client("s3") state = { "ready": False, "bucket": None, "key": None, "numbers_list": [], } -app = FastAPI() -s3 = boto3.client("s3") @app.on_event("startup") @@ -39,9 +40,10 @@ def startup(): @app.get("/healthz") -def healthz(response: Response): - if not state["ready"]: - response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE +def healthz(): + if state["ready"]: + return PlainTextResponse("ok") + return PlainTextResponse("service unavailable", status_code=status.HTTP_503_SERVICE_UNAVAILABLE) @app.post("/") diff --git a/test/apis/batch/sum/app/requirements.txt b/test/apis/batch/sum/app/requirements.txt new file mode 100644 index 0000000000..d34d746b43 --- /dev/null +++ b/test/apis/batch/sum/app/requirements.txt @@ -0,0 +1,3 @@ +uvicorn[standard] +fastapi +boto3==1.17.* diff --git a/test/apis/batch/sum/cortex_cpu.yaml b/test/apis/batch/sum/cortex_cpu.yaml index 267f50e74e..7cb20d7bf3 100644 --- a/test/apis/batch/sum/cortex_cpu.yaml +++ b/test/apis/batch/sum/cortex_cpu.yaml @@ -6,17 +6,7 @@ containers: - name: api image: quay.io/cortexlabs-test/batch-sum-cpu:latest - command: - - "gunicorn" - - "-k" - - "uvicorn.workers.UvicornWorker" - - "--workers" - - "1" - - "--threads" - - "1" - - "--bind" - - ":$(CORTEX_PORT)" - - "main:app" + command: ["uvicorn", "--workers", "1", "--host", "0.0.0.0", "--port", "$(CORTEX_PORT)", "main:app"] readiness_probe: http_get: path: "/healthz" diff --git a/test/apis/batch/sum/cpu.Dockerfile b/test/apis/batch/sum/cpu.Dockerfile new file mode 100644 index 0000000000..ea648acdf0 --- /dev/null +++ b/test/apis/batch/sum/cpu.Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.8-slim + +ENV PYTHONUNBUFFERED TRUE + +COPY app/requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY app /app +WORKDIR /app/ +ENV PYTHONPATH=/app + +ENV CORTEX_PORT=8080 +CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app diff --git a/test/apis/batch/sum/sum-cpu.dockerfile b/test/apis/batch/sum/sum-cpu.dockerfile deleted file mode 100644 index f7b6b29763..0000000000 --- a/test/apis/batch/sum/sum-cpu.dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM python:3.8-slim - -# Allow statements and log messages to immediately appear in the logs -ENV PYTHONUNBUFFERED True - -# Install production dependencies -RUN pip install --no-cache-dir \ - "uvicorn[standard]" \ - gunicorn \ - fastapi \ - boto3==1.17.72 - -# Copy local code to the container image. -COPY ./main.py /app/ -WORKDIR /app/ - -ENV PYTHONPATH=/app -ENV CORTEX_PORT=9000 - -# Run the web service on container startup. -CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/realtime/hello-world/.dockerignore b/test/apis/realtime/hello-world/.dockerignore deleted file mode 100644 index 4a8a96662f..0000000000 --- a/test/apis/realtime/hello-world/.dockerignore +++ /dev/null @@ -1,9 +0,0 @@ -*.dockerfile -build*.sh -cortex*.yaml -*.pyc -*.pyo -*.pyd -__pycache__ -.pytest_cache -*.md diff --git a/test/apis/realtime/hello-world/main.py b/test/apis/realtime/hello-world/app/main.py similarity index 59% rename from test/apis/realtime/hello-world/main.py rename to test/apis/realtime/hello-world/app/main.py index a164dbf79a..b0dfb5c7bf 100644 --- a/test/apis/realtime/hello-world/main.py +++ b/test/apis/realtime/hello-world/app/main.py @@ -1,5 +1,7 @@ import os + from fastapi import FastAPI +from fastapi.responses import PlainTextResponse app = FastAPI() @@ -8,9 +10,9 @@ @app.get("/healthz") def healthz(): - return "ok" + return PlainTextResponse("ok") @app.post("/") def post_handler(): - return response_str + return PlainTextResponse(response_str) diff --git a/test/apis/realtime/hello-world/app/requirements.txt b/test/apis/realtime/hello-world/app/requirements.txt new file mode 100644 index 0000000000..190ccb7716 --- /dev/null +++ b/test/apis/realtime/hello-world/app/requirements.txt @@ -0,0 +1,2 @@ +uvicorn[standard] +fastapi diff --git a/test/apis/realtime/hello-world/cortex_cpu.yaml b/test/apis/realtime/hello-world/cortex_cpu.yaml index b913aeef3d..3edc98541d 100644 --- a/test/apis/realtime/hello-world/cortex_cpu.yaml +++ b/test/apis/realtime/hello-world/cortex_cpu.yaml @@ -1,7 +1,7 @@ - name: hello-world kind: RealtimeAPI pod: - port: 9000 + port: 8080 max_concurrency: 1 containers: - name: api @@ -9,7 +9,7 @@ readiness_probe: http_get: path: "/healthz" - port: 9000 + port: 8080 compute: cpu: 200m mem: 128M diff --git a/test/apis/realtime/hello-world/cpu.Dockerfile b/test/apis/realtime/hello-world/cpu.Dockerfile new file mode 100644 index 0000000000..ea648acdf0 --- /dev/null +++ b/test/apis/realtime/hello-world/cpu.Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.8-slim + +ENV PYTHONUNBUFFERED TRUE + +COPY app/requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY app /app +WORKDIR /app/ +ENV PYTHONPATH=/app + +ENV CORTEX_PORT=8080 +CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app diff --git a/test/apis/realtime/hello-world/hello-world-cpu.dockerfile b/test/apis/realtime/hello-world/hello-world-cpu.dockerfile deleted file mode 100644 index a496f7c881..0000000000 --- a/test/apis/realtime/hello-world/hello-world-cpu.dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.8-slim - -# Allow statements and log messages to immediately appear in the logs -ENV PYTHONUNBUFFERED True - -# Install production dependencies -RUN pip install --no-cache-dir "uvicorn[standard]" gunicorn fastapi - -# Copy local code to the container image. -COPY . /app -WORKDIR /app/ - -ENV PYTHONPATH=/app -ENV CORTEX_PORT=9000 - -# Run the web service on container startup. -CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/realtime/image-classifier-resnet50/.dockerignore b/test/apis/realtime/image-classifier-resnet50/.dockerignore deleted file mode 100644 index a0dd6cdad9..0000000000 --- a/test/apis/realtime/image-classifier-resnet50/.dockerignore +++ /dev/null @@ -1,10 +0,0 @@ -sample.json -*.dockerfile -build*.sh -cortex*.yaml -*.pyc -*.pyo -*.pyd -__pycache__ -.pytest_cache -*.md diff --git a/test/apis/realtime/image-classifier-resnet50/image-classifier-resnet50-cpu.dockerfile b/test/apis/realtime/image-classifier-resnet50/cpu.Dockerfile similarity index 89% rename from test/apis/realtime/image-classifier-resnet50/image-classifier-resnet50-cpu.dockerfile rename to test/apis/realtime/image-classifier-resnet50/cpu.Dockerfile index e64f7cdfa5..527977184c 100644 --- a/test/apis/realtime/image-classifier-resnet50/image-classifier-resnet50-cpu.dockerfile +++ b/test/apis/realtime/image-classifier-resnet50/cpu.Dockerfile @@ -1,7 +1,7 @@ FROM tensorflow/serving:2.3.0 -RUN apt-get update -qq && apt-get install -y -q \ - wget \ +RUN apt-get update -qq && apt-get install -y --no-install-recommends -q \ + wget \ && apt-get clean -qq && rm -rf /var/lib/apt/lists/* RUN TFS_PROBE_VERSION=1.0.1 \ diff --git a/test/apis/realtime/image-classifier-resnet50/image-classifier-resnet50-gpu.dockerfile b/test/apis/realtime/image-classifier-resnet50/gpu.Dockerfile similarity index 100% rename from test/apis/realtime/image-classifier-resnet50/image-classifier-resnet50-gpu.dockerfile rename to test/apis/realtime/image-classifier-resnet50/gpu.Dockerfile diff --git a/test/apis/realtime/prime-generator/.dockerignore b/test/apis/realtime/prime-generator/.dockerignore deleted file mode 100644 index a0dd6cdad9..0000000000 --- a/test/apis/realtime/prime-generator/.dockerignore +++ /dev/null @@ -1,10 +0,0 @@ -sample.json -*.dockerfile -build*.sh -cortex*.yaml -*.pyc -*.pyo -*.pyd -__pycache__ -.pytest_cache -*.md diff --git a/test/apis/realtime/prime-generator/main.py b/test/apis/realtime/prime-generator/app/main.py similarity index 99% rename from test/apis/realtime/prime-generator/main.py rename to test/apis/realtime/prime-generator/app/main.py index 59099d8202..54c6f33eba 100644 --- a/test/apis/realtime/prime-generator/main.py +++ b/test/apis/realtime/prime-generator/app/main.py @@ -1,8 +1,9 @@ from typing import DefaultDict - from fastapi import FastAPI from pydantic import BaseModel +app = FastAPI() + def generate_primes(limit=None): """Sieve of Eratosthenes""" @@ -19,18 +20,15 @@ def generate_primes(limit=None): num += 1 -class Request(BaseModel): - primes_to_generate: float - - -app = FastAPI() - - @app.get("/healthz") def healthz(): return "ok" +class Request(BaseModel): + primes_to_generate: float + + @app.post("/") def prime_numbers(request: Request): return {"prime_numbers": list(generate_primes(request.primes_to_generate))} diff --git a/test/apis/realtime/prime-generator/app/requirements.txt b/test/apis/realtime/prime-generator/app/requirements.txt new file mode 100644 index 0000000000..81c6e29af4 --- /dev/null +++ b/test/apis/realtime/prime-generator/app/requirements.txt @@ -0,0 +1,3 @@ +uvicorn[standard] +fastapi +pydantic diff --git a/test/apis/realtime/prime-generator/cortex_cpu.yaml b/test/apis/realtime/prime-generator/cortex_cpu.yaml index b5ffaf96c8..146c78267a 100644 --- a/test/apis/realtime/prime-generator/cortex_cpu.yaml +++ b/test/apis/realtime/prime-generator/cortex_cpu.yaml @@ -1,7 +1,7 @@ - name: prime-generator kind: RealtimeAPI pod: - port: 9000 + port: 8080 max_concurrency: 1 containers: - name: api @@ -9,7 +9,7 @@ readiness_probe: http_get: path: "/healthz" - port: 9000 + port: 8080 compute: cpu: 200m mem: 128M diff --git a/test/apis/realtime/prime-generator/cpu.Dockerfile b/test/apis/realtime/prime-generator/cpu.Dockerfile new file mode 100644 index 0000000000..ea648acdf0 --- /dev/null +++ b/test/apis/realtime/prime-generator/cpu.Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.8-slim + +ENV PYTHONUNBUFFERED TRUE + +COPY app/requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY app /app +WORKDIR /app/ +ENV PYTHONPATH=/app + +ENV CORTEX_PORT=8080 +CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app diff --git a/test/apis/realtime/prime-generator/prime-generator-cpu.dockerfile b/test/apis/realtime/prime-generator/prime-generator-cpu.dockerfile deleted file mode 100644 index a8f376e7d4..0000000000 --- a/test/apis/realtime/prime-generator/prime-generator-cpu.dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.8-slim - -# Allow statements and log messages to immediately appear in the logs -ENV PYTHONUNBUFFERED True - -# Install production dependencies -RUN pip install --no-cache-dir "uvicorn[standard]" gunicorn fastapi pydantic - -# Copy local code to the container image. -COPY . /app -WORKDIR /app/ - -ENV PYTHONPATH=/app -ENV CORTEX_PORT=9000 - -# Run the web service on container startup. -CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/realtime/sleep/.dockerignore b/test/apis/realtime/sleep/.dockerignore deleted file mode 100644 index 4a8a96662f..0000000000 --- a/test/apis/realtime/sleep/.dockerignore +++ /dev/null @@ -1,9 +0,0 @@ -*.dockerfile -build*.sh -cortex*.yaml -*.pyc -*.pyo -*.pyd -__pycache__ -.pytest_cache -*.md diff --git a/test/apis/realtime/sleep/main.py b/test/apis/realtime/sleep/app/main.py similarity index 58% rename from test/apis/realtime/sleep/main.py rename to test/apis/realtime/sleep/app/main.py index 6dd675055c..4c44187682 100644 --- a/test/apis/realtime/sleep/main.py +++ b/test/apis/realtime/sleep/app/main.py @@ -1,15 +1,17 @@ import time from fastapi import FastAPI +from fastapi.responses import PlainTextResponse app = FastAPI() @app.get("/healthz") def healthz(): - return "ok" + return PlainTextResponse("ok") @app.post("/") def sleep(sleep: float = 0): time.sleep(sleep) + return PlainTextResponse("ok") diff --git a/test/apis/realtime/sleep/app/requirements.txt b/test/apis/realtime/sleep/app/requirements.txt new file mode 100644 index 0000000000..190ccb7716 --- /dev/null +++ b/test/apis/realtime/sleep/app/requirements.txt @@ -0,0 +1,2 @@ +uvicorn[standard] +fastapi diff --git a/test/apis/realtime/sleep/cortex_cpu.yaml b/test/apis/realtime/sleep/cortex_cpu.yaml index cae1a97b39..8b27f7e92d 100644 --- a/test/apis/realtime/sleep/cortex_cpu.yaml +++ b/test/apis/realtime/sleep/cortex_cpu.yaml @@ -1,7 +1,7 @@ - name: sleep kind: RealtimeAPI pod: - port: 9000 + port: 8080 max_concurrency: 1 max_queue_length: 128 containers: @@ -10,7 +10,7 @@ readiness_probe: http_get: path: "/healthz" - port: 9000 + port: 8080 compute: cpu: 200m mem: 128M diff --git a/test/apis/realtime/sleep/cpu.Dockerfile b/test/apis/realtime/sleep/cpu.Dockerfile new file mode 100644 index 0000000000..ea648acdf0 --- /dev/null +++ b/test/apis/realtime/sleep/cpu.Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.8-slim + +ENV PYTHONUNBUFFERED TRUE + +COPY app/requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY app /app +WORKDIR /app/ +ENV PYTHONPATH=/app + +ENV CORTEX_PORT=8080 +CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app diff --git a/test/apis/realtime/sleep/sleep-cpu.dockerfile b/test/apis/realtime/sleep/sleep-cpu.dockerfile deleted file mode 100644 index a496f7c881..0000000000 --- a/test/apis/realtime/sleep/sleep-cpu.dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.8-slim - -# Allow statements and log messages to immediately appear in the logs -ENV PYTHONUNBUFFERED True - -# Install production dependencies -RUN pip install --no-cache-dir "uvicorn[standard]" gunicorn fastapi - -# Copy local code to the container image. -COPY . /app -WORKDIR /app/ - -ENV PYTHONPATH=/app -ENV CORTEX_PORT=9000 - -# Run the web service on container startup. -CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/realtime/text-generator/.dockerignore b/test/apis/realtime/text-generator/.dockerignore deleted file mode 100644 index a0dd6cdad9..0000000000 --- a/test/apis/realtime/text-generator/.dockerignore +++ /dev/null @@ -1,10 +0,0 @@ -sample.json -*.dockerfile -build*.sh -cortex*.yaml -*.pyc -*.pyo -*.pyd -__pycache__ -.pytest_cache -*.md diff --git a/test/apis/realtime/text-generator/main.py b/test/apis/realtime/text-generator/app/main.py similarity index 66% rename from test/apis/realtime/text-generator/main.py rename to test/apis/realtime/text-generator/app/main.py index 020c7fc9e2..39939e4fc6 100644 --- a/test/apis/realtime/text-generator/main.py +++ b/test/apis/realtime/text-generator/app/main.py @@ -1,21 +1,17 @@ import os -from fastapi import FastAPI, Response, status +from fastapi import FastAPI, status +from fastapi.responses import PlainTextResponse from pydantic import BaseModel from transformers import GPT2Tokenizer, GPT2LMHeadModel - -class Request(BaseModel): - text: str - - +app = FastAPI() +device = os.getenv("TARGET_DEVICE", "cpu") state = { - "model_ready": False, + "ready": False, "tokenizer": None, "model": None, } -device = os.getenv("TARGET_DEVICE", "cpu") -app = FastAPI() @app.on_event("startup") @@ -23,13 +19,18 @@ def startup(): global state state["tokenizer"] = GPT2Tokenizer.from_pretrained("gpt2") state["model"] = GPT2LMHeadModel.from_pretrained("gpt2").to(device) - state["model_ready"] = True + state["ready"] = True @app.get("/healthz") -def healthz(response: Response): - if not state["model_ready"]: - response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE +def healthz(): + if state["ready"]: + return PlainTextResponse("ok") + return PlainTextResponse("service unavailable", status_code=status.HTTP_503_SERVICE_UNAVAILABLE) + + +class Request(BaseModel): + text: str @app.post("/") @@ -37,6 +38,4 @@ def text_generator(request: Request): input_length = len(request.text.split()) tokens = state["tokenizer"].encode(request.text, return_tensors="pt").to(device) prediction = state["model"].generate(tokens, max_length=input_length + 20, do_sample=True) - return { - "prediction": state["tokenizer"].decode(prediction[0]), - } + return {"text": state["tokenizer"].decode(prediction[0])} diff --git a/test/apis/realtime/text-generator/app/requirements-cpu.txt b/test/apis/realtime/text-generator/app/requirements-cpu.txt new file mode 100644 index 0000000000..91ef8d2188 --- /dev/null +++ b/test/apis/realtime/text-generator/app/requirements-cpu.txt @@ -0,0 +1,6 @@ +uvicorn[standard] +fastapi +pydantic +transformers==3.0.* +-f https://download.pytorch.org/whl/torch_stable.html +torch==1.7.1+cpu diff --git a/test/apis/realtime/text-generator/app/requirements-gpu.txt b/test/apis/realtime/text-generator/app/requirements-gpu.txt new file mode 100644 index 0000000000..9433f26464 --- /dev/null +++ b/test/apis/realtime/text-generator/app/requirements-gpu.txt @@ -0,0 +1,5 @@ +uvicorn[standard] +fastapi +pydantic +transformers==3.0.* +torch==1.7.* diff --git a/test/apis/realtime/text-generator/cortex_cpu.yaml b/test/apis/realtime/text-generator/cortex_cpu.yaml index ffea3ec1fb..c31bb959b0 100644 --- a/test/apis/realtime/text-generator/cortex_cpu.yaml +++ b/test/apis/realtime/text-generator/cortex_cpu.yaml @@ -1,7 +1,7 @@ - name: text-generator kind: RealtimeAPI pod: - port: 9000 + port: 8080 max_concurrency: 1 containers: - name: api @@ -9,7 +9,7 @@ readiness_probe: http_get: path: "/healthz" - port: 9000 + port: 8080 compute: cpu: 1 mem: 2.5G diff --git a/test/apis/realtime/text-generator/cortex_gpu.yaml b/test/apis/realtime/text-generator/cortex_gpu.yaml index 11a70b57a9..a0659ff234 100644 --- a/test/apis/realtime/text-generator/cortex_gpu.yaml +++ b/test/apis/realtime/text-generator/cortex_gpu.yaml @@ -1,7 +1,7 @@ - name: text-generator kind: RealtimeAPI pod: - port: 9000 + port: 8080 max_concurrency: 1 containers: - name: api @@ -11,7 +11,7 @@ readiness_probe: http_get: path: "/healthz" - port: 9000 + port: 8080 compute: cpu: 1 gpu: 1 diff --git a/test/apis/realtime/text-generator/cpu.Dockerfile b/test/apis/realtime/text-generator/cpu.Dockerfile new file mode 100644 index 0000000000..54a36928ff --- /dev/null +++ b/test/apis/realtime/text-generator/cpu.Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.8-slim + +ENV PYTHONUNBUFFERED TRUE + +COPY app/requirements-cpu.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY app /app +WORKDIR /app/ +ENV PYTHONPATH=/app + +ENV CORTEX_PORT=8080 +CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app diff --git a/test/apis/realtime/text-generator/gpu.Dockerfile b/test/apis/realtime/text-generator/gpu.Dockerfile new file mode 100644 index 0000000000..81f9169949 --- /dev/null +++ b/test/apis/realtime/text-generator/gpu.Dockerfile @@ -0,0 +1,24 @@ +FROM nvidia/cuda:10.2-cudnn8-runtime-ubuntu18.04 + +RUN apt-get update \ + && apt-get install -y \ + python3 \ + python3-pip \ + pkg-config \ + build-essential \ + git \ + cmake \ + && apt-get clean -qq && rm -rf /var/lib/apt/lists/* + +ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 +ENV PYTHONUNBUFFERED TRUE + +COPY app/requirements-gpu.txt /app/requirements.txt +RUN pip3 install --no-cache-dir -r /app/requirements.txt + +COPY app /app +WORKDIR /app/ +ENV PYTHONPATH=/app + +ENV CORTEX_PORT=8080 +CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app diff --git a/test/apis/realtime/text-generator/text-generator-cpu.dockerfile b/test/apis/realtime/text-generator/text-generator-cpu.dockerfile deleted file mode 100644 index b90ff40a7f..0000000000 --- a/test/apis/realtime/text-generator/text-generator-cpu.dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM python:3.8-slim - -# Allow statements and log messages to immediately appear in the logs -ENV PYTHONUNBUFFERED True - -# Install production dependencies -RUN pip install --no-cache-dir \ - "uvicorn[standard]" \ - gunicorn \ - fastapi \ - pydantic \ - transformers==3.0.* \ - torch==1.7.1+cpu -f https://download.pytorch.org/whl/torch_stable.html - -# Copy local code to the container image. -COPY . /app -WORKDIR /app/ - -ENV PYTHONPATH=/app -ENV CORTEX_PORT=9000 - -# Run the web service on container startup. -CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/realtime/text-generator/text-generator-gpu.dockerfile b/test/apis/realtime/text-generator/text-generator-gpu.dockerfile deleted file mode 100644 index 2de181d169..0000000000 --- a/test/apis/realtime/text-generator/text-generator-gpu.dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -FROM nvidia/cuda:10.2-cudnn8-runtime-ubuntu18.04 - -RUN apt-get update \ - && apt-get install \ - python3 \ - python3-pip \ - pkg-config \ - git \ - build-essential \ - cmake -y \ - && apt-get clean -qq && rm -rf /var/lib/apt/lists/* - -# Allow statements and log messages to immediately appear in the logs -ENV PYTHONUNBUFFERED True - -# Install production dependencies -RUN pip3 install --no-cache-dir "uvicorn[standard]" gunicorn fastapi pydantic transformers==3.0.* torch==1.7.* - -# Copy local code to the container image. -COPY . /app -WORKDIR /app/ - -ENV PYTHONPATH=/app -ENV CORTEX_PORT=9000 - -# Run the web service on container startup. -CMD gunicorn -k uvicorn.workers.UvicornWorker --workers 1 --threads 1 --bind :$CORTEX_PORT main:app diff --git a/test/apis/task/iris-classifier-trainer/.dockerignore b/test/apis/task/iris-classifier-trainer/.dockerignore deleted file mode 100644 index 30f9c4551c..0000000000 --- a/test/apis/task/iris-classifier-trainer/.dockerignore +++ /dev/null @@ -1,10 +0,0 @@ -submit.py -*.dockerfile -build*.sh -cortex*.yaml -*.pyc -*.pyo -*.pyd -__pycache__ -.pytest_cache -*.md diff --git a/test/apis/task/iris-classifier-trainer/main.py b/test/apis/task/iris-classifier-trainer/app/main.py similarity index 95% rename from test/apis/task/iris-classifier-trainer/main.py rename to test/apis/task/iris-classifier-trainer/app/main.py index 99ce8ec1d8..6c95513d70 100644 --- a/test/apis/task/iris-classifier-trainer/main.py +++ b/test/apis/task/iris-classifier-trainer/app/main.py @@ -1,4 +1,8 @@ -import json, pickle, re, os, boto3 +import json +import pickle +import re +import os +import boto3 from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split diff --git a/test/apis/task/iris-classifier-trainer/app/requirements.txt b/test/apis/task/iris-classifier-trainer/app/requirements.txt new file mode 100644 index 0000000000..ead4e24356 --- /dev/null +++ b/test/apis/task/iris-classifier-trainer/app/requirements.txt @@ -0,0 +1,3 @@ +boto3==1.17.* +numpy==1.18.* +scikit-learn==0.21.* diff --git a/test/apis/task/iris-classifier-trainer/cpu.Dockerfile b/test/apis/task/iris-classifier-trainer/cpu.Dockerfile new file mode 100644 index 0000000000..a1987f954e --- /dev/null +++ b/test/apis/task/iris-classifier-trainer/cpu.Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.7-slim + +ENV PYTHONUNBUFFERED TRUE + +COPY app/requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY app /app +WORKDIR /app/ + +CMD exec python app/main.py diff --git a/test/apis/task/iris-classifier-trainer/iris-classifier-trainer-cpu.dockerfile b/test/apis/task/iris-classifier-trainer/iris-classifier-trainer-cpu.dockerfile deleted file mode 100644 index 06cdb86fcc..0000000000 --- a/test/apis/task/iris-classifier-trainer/iris-classifier-trainer-cpu.dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -# Use the official lightweight Python image. -# https://hub.docker.com/_/python -FROM python:3.7-slim - -# Allow statements and log messages to immediately appear in the logs -ENV PYTHONUNBUFFERED True - -# Install production dependencies. -RUN pip install numpy==1.18.5 boto3==1.17.72 scikit-learn==0.21.3 - -# Copy local code to the container image. -ENV APP_HOME /app -WORKDIR $APP_HOME -COPY . ./ - -# Run task -CMD exec python main.py diff --git a/test/apis/trafficsplitter/hello-world/cortex_cpu.yaml b/test/apis/trafficsplitter/hello-world/cortex_cpu.yaml index 73fec8f126..20292f1fc6 100644 --- a/test/apis/trafficsplitter/hello-world/cortex_cpu.yaml +++ b/test/apis/trafficsplitter/hello-world/cortex_cpu.yaml @@ -1,7 +1,7 @@ - name: hello-world-a kind: RealtimeAPI pod: - port: 9000 + port: 8080 max_concurrency: 1 containers: - name: api @@ -11,7 +11,7 @@ readiness_probe: http_get: path: "/healthz" - port: 9000 + port: 8080 compute: cpu: 200m mem: 128M @@ -19,7 +19,7 @@ - name: hello-world-b kind: RealtimeAPI pod: - port: 9000 + port: 8080 max_concurrency: 1 containers: - name: api @@ -29,7 +29,7 @@ readiness_probe: http_get: path: "/healthz" - port: 9000 + port: 8080 compute: cpu: 200m mem: 128M @@ -37,7 +37,7 @@ - name: hello-world-shadow kind: RealtimeAPI pod: - port: 9000 + port: 8080 max_concurrency: 1 containers: - name: api @@ -47,7 +47,7 @@ readiness_probe: http_get: path: "/healthz" - port: 9000 + port: 8080 compute: cpu: 200m mem: 128M diff --git a/test/utils/build.sh b/test/utils/build.sh index baa643514a..0ac129bea9 100755 --- a/test/utils/build.sh +++ b/test/utils/build.sh @@ -27,7 +27,7 @@ function registry_login() { login_url=$1 # e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-cpu region=$2 - blue_echo "\nLogging in to ECR..." + blue_echo "\nLogging in to ECR" aws ecr get-login-password --region $region | docker login --username AWS --password-stdin $login_url green_echo "\nSuccess" } @@ -36,7 +36,7 @@ function create_ecr_repo() { repo_name=$1 # e.g. cortexlabs/realtime-sleep-cpu region=$2 - blue_echo "\nCreating ECR repo $repo_name..." + blue_echo "\nCreating ECR repo $repo_name" aws ecr create-repository --repository-name=$repo_name --region=$region green_echo "\nSuccess" } @@ -66,8 +66,8 @@ if [[ "$image_url" == *".ecr."* ]]; then region="$(echo "$image_url" | sed 's/.*\.ecr\.//' | sed 's/\..*//')" # e.g. us-west-2 fi -blue_echo "Building $image_url:latest...\n" -docker build "$(dirname "$path")" -f "$(dirname "$path")/$name-$architecture.dockerfile" -t "$image_url" +blue_echo "Building $image_url:latest\n" +docker build "$(dirname "$path")" -f "$(dirname "$path")/$architecture.Dockerfile" -t "$image_url" green_echo "\nBuilt $image_url:latest" if [ "$should_skip_push" = "true" ]; then @@ -75,7 +75,7 @@ if [ "$should_skip_push" = "true" ]; then fi while true; do - blue_echo "\nPushing $image_url:latest..." + blue_echo "\nPushing $image_url:latest" exec 5>&1 set +e out=$(docker push $image_url 2>&1 | tee /dev/fd/5; exit ${PIPESTATUS[0]}) From 27d3cd8eb77e6536350c825d265dc8932ba5a5cd Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 2 Jun 2021 22:19:24 -0700 Subject: [PATCH 73/82] Add build-and-push-test-images command --- Makefile | 5 +++++ test/utils/build-all.sh | 27 +++++++++++++++++++++++++++ test/utils/build.sh | 21 +++++++++++++-------- 3 files changed, 45 insertions(+), 8 deletions(-) create mode 100755 test/utils/build-all.sh diff --git a/Makefile b/Makefile index ddecc7367f..8bfcd2d685 100644 --- a/Makefile +++ b/Makefile @@ -162,6 +162,11 @@ format: # Tests # ######### +# build test api images +# make sure you login with your quay credentials +build-and-push-test-images: + @./test/utils/build-all.sh quay.io/cortexlabs-test + test: @./build/test.sh go diff --git a/test/utils/build-all.sh b/test/utils/build-all.sh new file mode 100755 index 0000000000..52140082f6 --- /dev/null +++ b/test/utils/build-all.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright 2021 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# usage: ./build-all.sh [REGISTRY] [--skip-push] +# REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test + +set -eo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. >/dev/null && pwd)" +source $ROOT/dev/util.sh + +for f in $(find $ROOT/test/apis -type f -name 'build-*.sh'); do + $ROOT/test/utils/build.sh $f "$@" +done diff --git a/test/utils/build.sh b/test/utils/build.sh index 0ac129bea9..2a1cdd4fb7 100755 --- a/test/utils/build.sh +++ b/test/utils/build.sh @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# usage: ./build.sh build PATH [REGISTRY] [--skip-push] +# usage: ./build.sh PATH [REGISTRY] [--skip-push] # PATH is e.g. /home/ubuntu/src/github.com/cortexlabs/cortex/test/apis/realtime/sleep/build-cpu.sh # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test @@ -79,14 +79,19 @@ while true; do exec 5>&1 set +e out=$(docker push $image_url 2>&1 | tee /dev/fd/5; exit ${PIPESTATUS[0]}) + exit_code=$? set -e - if [[ "$image_url" == *".ecr."* ]]; then - if [[ "$out" == *"authorization token has expired"* ]] || [[ "$out" == *"no basic auth credentials"* ]]; then - registry_login $login_url $region - continue - elif [[ "$out" == *"does not exist"* ]]; then - create_ecr_repo $repo_name $region - continue + if [ $exit_code -ne 0 ]; then + if [[ "$image_url" != *".ecr."* ]]; then + exit $exit_code + else + if [[ "$out" == *"authorization token has expired"* ]] || [[ "$out" == *"no basic auth credentials"* ]]; then + registry_login $login_url $region + continue + elif [[ "$out" == *"does not exist"* ]]; then + create_ecr_repo $repo_name $region + continue + fi fi fi green_echo "\nPushed $image_url:latest" From 8a246af3177c02a9f6b3f5c99cf076240f3f50e9 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 2 Jun 2021 22:20:32 -0700 Subject: [PATCH 74/82] Rename build-test-api-images --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8bfcd2d685..1461188697 100644 --- a/Makefile +++ b/Makefile @@ -164,7 +164,7 @@ format: # build test api images # make sure you login with your quay credentials -build-and-push-test-images: +build-test-api-images: @./test/utils/build-all.sh quay.io/cortexlabs-test test: From 62158832c4b799ce81c4084ef048e8a408392980 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 2 Jun 2021 22:26:27 -0700 Subject: [PATCH 75/82] Rename containers.md --- docs/summary.md | 8 ++++---- docs/workloads/async/{container.md => containers.md} | 2 +- docs/workloads/batch/{container.md => containers.md} | 2 +- docs/workloads/realtime/{container.md => containers.md} | 2 +- docs/workloads/task/{container.md => containers.md} | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) rename docs/workloads/async/{container.md => containers.md} (99%) rename docs/workloads/batch/{container.md => containers.md} (99%) rename docs/workloads/realtime/{container.md => containers.md} (99%) rename docs/workloads/task/{container.md => containers.md} (98%) diff --git a/docs/summary.md b/docs/summary.md index 7ed2aa333c..713a053406 100644 --- a/docs/summary.md +++ b/docs/summary.md @@ -32,7 +32,7 @@ * [Realtime APIs](workloads/realtime/realtime-apis.md) * [Example](workloads/realtime/example.md) * [Configuration](workloads/realtime/configuration.md) - * [Container Implementation](workloads/realtime/container.md) + * [Containers](workloads/realtime/containers.md) * [Autoscaling](workloads/realtime/autoscaling.md) * [Traffic Splitter](workloads/realtime/traffic-splitter.md) * [Metrics](workloads/realtime/metrics.md) @@ -41,18 +41,18 @@ * [Async APIs](workloads/async/async-apis.md) * [Example](workloads/async/example.md) * [Configuration](workloads/async/configuration.md) - * [Container Implementation](workloads/async/container.md) + * [Containers](workloads/async/containers.md) * [Statuses](workloads/async/statuses.md) * [Batch APIs](workloads/batch/batch-apis.md) * [Example](workloads/batch/example.md) * [Configuration](workloads/batch/configuration.md) - * [Container Implementation](workloads/batch/container.md) + * [Containers](workloads/batch/containers.md) * [Jobs](workloads/batch/jobs.md) * [Statuses](workloads/batch/statuses.md) * [Task APIs](workloads/task/task-apis.md) * [Example](workloads/task/example.md) * [Configuration](workloads/task/configuration.md) - * [Container Implementation](workloads/task/container.md) + * [Containers](workloads/task/containers.md) * [Jobs](workloads/task/jobs.md) * [Statuses](workloads/task/statuses.md) diff --git a/docs/workloads/async/container.md b/docs/workloads/async/containers.md similarity index 99% rename from docs/workloads/async/container.md rename to docs/workloads/async/containers.md index 51474468fe..d61d1ced57 100644 --- a/docs/workloads/async/container.md +++ b/docs/workloads/async/containers.md @@ -1,4 +1,4 @@ -# Container Implementation +# Containers ## Handling requests diff --git a/docs/workloads/batch/container.md b/docs/workloads/batch/containers.md similarity index 99% rename from docs/workloads/batch/container.md rename to docs/workloads/batch/containers.md index 1634dfcb22..eb3a971e79 100644 --- a/docs/workloads/batch/container.md +++ b/docs/workloads/batch/containers.md @@ -1,4 +1,4 @@ -# Container Implementation +# Containers ## Handling requests diff --git a/docs/workloads/realtime/container.md b/docs/workloads/realtime/containers.md similarity index 99% rename from docs/workloads/realtime/container.md rename to docs/workloads/realtime/containers.md index 4fe57c985a..b3b502a135 100644 --- a/docs/workloads/realtime/container.md +++ b/docs/workloads/realtime/containers.md @@ -1,4 +1,4 @@ -# Container Implementation +# Containers ## Handling requests diff --git a/docs/workloads/task/container.md b/docs/workloads/task/containers.md similarity index 98% rename from docs/workloads/task/container.md rename to docs/workloads/task/containers.md index bf742d57ff..3b53eb35f2 100644 --- a/docs/workloads/task/container.md +++ b/docs/workloads/task/containers.md @@ -1,4 +1,4 @@ -# Container Implementation +# Containers ## Job specification From 2882436da75a42d114b4723b4d44fe7c7bf8f321 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Thu, 3 Jun 2021 15:40:08 +0300 Subject: [PATCH 76/82] Nits --- cmd/enqueuer/main.go | 8 ++++---- pkg/types/userconfig/log_level.go | 16 ---------------- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/cmd/enqueuer/main.go b/cmd/enqueuer/main.go index c73724be1c..2a1a79c7a6 100644 --- a/cmd/enqueuer/main.go +++ b/cmd/enqueuer/main.go @@ -28,16 +28,16 @@ import ( ) func createLogger() (*zap.Logger, error) { - logLevelEnv := strings.ToUpper(os.Getenv("CORTEX_LOG_LEVEL")) + logLevelEnv := strings.ToLower(os.Getenv("CORTEX_LOG_LEVEL")) disableJSONLogging := os.Getenv("CORTEX_DISABLE_JSON_LOGGING") var logLevelZap zapcore.Level switch logLevelEnv { - case "DEBUG": + case "debug": logLevelZap = zapcore.DebugLevel - case "WARNING": + case "warning": logLevelZap = zapcore.WarnLevel - case "ERROR": + case "error": logLevelZap = zapcore.ErrorLevel default: logLevelZap = zapcore.InfoLevel diff --git a/pkg/types/userconfig/log_level.go b/pkg/types/userconfig/log_level.go index 6279a842fe..b96af6470f 100644 --- a/pkg/types/userconfig/log_level.go +++ b/pkg/types/userconfig/log_level.go @@ -83,22 +83,6 @@ func (t LogLevel) MarshalBinary() ([]byte, error) { return []byte(t.String()), nil } -// Value to set for TF_CPP_MIN_LOG_LEVEL environment variable. Default is 0. -func TFNumericLogLevelFromLogLevel(logLevel LogLevel) int { - var tfLogLevelNumber int - switch logLevel.String() { - case "debug": - tfLogLevelNumber = 0 - case "info": - tfLogLevelNumber = 0 - case "warning": - tfLogLevelNumber = 1 - case "error": - tfLogLevelNumber = 2 - } - return tfLogLevelNumber -} - func ToZapLogLevel(logLevel LogLevel) zapcore.Level { switch logLevel { case InfoLogLevel: From 0e8839ef1d10c94e4c94e3d4e4eab184ef2699c3 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Thu, 3 Jun 2021 10:13:54 -0700 Subject: [PATCH 77/82] Update env list formatting --- cli/types/cliconfig/environment.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/cli/types/cliconfig/environment.go b/cli/types/cliconfig/environment.go index 30dcaef2e3..16bec370b8 100644 --- a/cli/types/cliconfig/environment.go +++ b/cli/types/cliconfig/environment.go @@ -17,12 +17,12 @@ limitations under the License. package cliconfig import ( + "fmt" "strings" cr "github.com/cortexlabs/cortex/pkg/lib/configreader" + "github.com/cortexlabs/cortex/pkg/lib/console" "github.com/cortexlabs/cortex/pkg/lib/errors" - "github.com/cortexlabs/cortex/pkg/lib/pointer" - "github.com/cortexlabs/cortex/pkg/lib/table" "github.com/cortexlabs/cortex/pkg/lib/urls" ) @@ -32,19 +32,17 @@ type Environment struct { } func (env Environment) String(isDefault bool) string { - var items table.KeyValuePairs + var envStr string if isDefault { - items.Add("name", env.Name+" (default)") + envStr += console.Bold(env.Name + " (default)") } else { - items.Add("name", env.Name) + envStr += console.Bold(env.Name) } - items.Add("cortex operator endpoint", env.OperatorEndpoint) + envStr += fmt.Sprintf("\ncortex operator endpoint: %s\n", env.OperatorEndpoint) - return items.String(&table.KeyValuePairOpts{ - BoldFirstLine: pointer.Bool(true), - }) + return envStr } func CortexEndpointValidator(val string) (string, error) { From 38ffa7d8924d11538bd4706df72481ccd4180d2e Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Thu, 3 Jun 2021 20:39:08 +0300 Subject: [PATCH 78/82] CaaS - batch probe fixes (#2220) --- cmd/dequeuer/main.go | 37 ++++++++++++++++++------------------- pkg/probe/probe.go | 4 +++- pkg/workloads/k8s.go | 18 ++++++++++++++++-- 3 files changed, 37 insertions(+), 22 deletions(-) diff --git a/cmd/dequeuer/main.go b/cmd/dequeuer/main.go index 85d17c5505..bc29a674ae 100644 --- a/cmd/dequeuer/main.go +++ b/cmd/dequeuer/main.go @@ -87,6 +87,8 @@ func main() { log.Fatal("--api-name is a required option") case apiKind == "": log.Fatal("--api-kind is a required option") + case adminPort == 0: + log.Fatal("--admin-port is a required option") } targetURL := "http://127.0.0.1:" + strconv.Itoa(userContainerPort) @@ -142,8 +144,6 @@ func main() { var dequeuerConfig dequeuer.SQSDequeuerConfig var messageHandler dequeuer.MessageHandler - errCh := make(chan error) - switch apiKind { case userconfig.BatchAPIKind.String(): if jobID == "" { @@ -169,9 +169,6 @@ func main() { if clusterUID == "" { log.Fatal("--cluster-uid is a required option") } - if adminPort == 0 { - log.Fatal("--admin-port is a required option") - } config := dequeuer.AsyncMessageHandlerConfig{ ClusterUID: clusterUID, @@ -187,24 +184,26 @@ func main() { StopIfNoMessages: false, } - adminHandler := http.NewServeMux() - adminHandler.Handle("/healthz", dequeuer.HealthcheckHandler(func() bool { - return probe.AreProbesHealthy(probes) - })) - - go func() { - server := &http.Server{ - Addr: ":" + strconv.Itoa(adminPort), - Handler: adminHandler, - } - log.Infof("Starting %s server on %s", "admin", server.Addr) - errCh <- server.ListenAndServe() - }() - default: exit(log, err, fmt.Sprintf("kind %s is not supported", apiKind)) } + errCh := make(chan error) + + adminHandler := http.NewServeMux() + adminHandler.Handle("/healthz", dequeuer.HealthcheckHandler(func() bool { + return probe.AreProbesHealthy(probes) + })) + + go func() { + server := &http.Server{ + Addr: ":" + strconv.Itoa(adminPort), + Handler: adminHandler, + } + log.Infof("Starting %s server on %s", "admin", server.Addr) + errCh <- server.ListenAndServe() + }() + sigint := make(chan os.Signal, 1) signal.Notify(sigint, os.Interrupt) diff --git a/pkg/probe/probe.go b/pkg/probe/probe.go index e79e141975..11834cdd9e 100644 --- a/pkg/probe/probe.go +++ b/pkg/probe/probe.go @@ -173,7 +173,9 @@ func (p *Probe) httpProbe() error { "http://", ) - httpClient := &http.Client{} + httpClient := &http.Client{ + Timeout: time.Duration(p.TimeoutSeconds) * time.Second, + } req, err := http.NewRequest(http.MethodGet, targetURL, nil) if err != nil { return err diff --git a/pkg/workloads/k8s.go b/pkg/workloads/k8s.go index c110163020..62a2329410 100644 --- a/pkg/workloads/k8s.go +++ b/pkg/workloads/k8s.go @@ -152,7 +152,7 @@ func asyncDequeuerProxyContainer(api spec.API, queueURL string) (kcore.Container }, InitialDelaySeconds: 1, TimeoutSeconds: 1, - PeriodSeconds: 5, + PeriodSeconds: 10, SuccessThreshold: 1, FailureThreshold: 1, }, @@ -180,6 +180,7 @@ func batchDequeuerProxyContainer(api spec.API, jobID, queueURL string) (kcore.Co "--job-id", jobID, "--user-port", s.Int32(*api.Pod.Port), "--statsd-port", consts.StatsDPortStr, + "--admin-port", consts.AdminPortStr, }, Env: append(baseEnvVars, kcore.EnvVar{ Name: "HOST_IP", @@ -189,6 +190,19 @@ func batchDequeuerProxyContainer(api spec.API, jobID, queueURL string) (kcore.Co }, }, }), + ReadinessProbe: &kcore.Probe{ + Handler: kcore.Handler{ + HTTPGet: &kcore.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(int(consts.AdminPortInt32)), + }, + }, + InitialDelaySeconds: 1, + TimeoutSeconds: 1, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 1, + }, VolumeMounts: []kcore.VolumeMount{ ClusterConfigMount(), CortexMount(), @@ -233,7 +247,7 @@ func realtimeProxyContainer(api spec.API) (kcore.Container, kcore.Volume) { }, InitialDelaySeconds: 1, TimeoutSeconds: 1, - PeriodSeconds: 5, + PeriodSeconds: 10, SuccessThreshold: 1, FailureThreshold: 1, }, From 3d10a8693e9a4678c284bb7768858b04a3b5ab6d Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Thu, 3 Jun 2021 23:01:02 +0300 Subject: [PATCH 79/82] CaaS - nits and fixes (#2221) --- pkg/consts/consts.go | 4 +++ pkg/operator/resources/job/taskapi/job.go | 33 ++++++++++++----------- pkg/types/spec/validations.go | 15 +++-------- 3 files changed, 24 insertions(+), 28 deletions(-) diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 139205e10b..7bb71a82b8 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -45,6 +45,10 @@ var ( MaxBucketLifecycleRules = 100 AsyncWorkloadsExpirationDays = int64(7) + ReservedContainerPorts = []int32{ + ProxyListeningPortInt32, + AdminPortInt32, + } ReservedContainerNames = []string{ "dequeuer", "proxy", diff --git a/pkg/operator/resources/job/taskapi/job.go b/pkg/operator/resources/job/taskapi/job.go index 2362ced32c..8410e21786 100644 --- a/pkg/operator/resources/job/taskapi/job.go +++ b/pkg/operator/resources/job/taskapi/job.go @@ -69,10 +69,6 @@ func SubmitJob(apiName string, submission *schema.TaskJobSubmission) (*spec.Task return nil, err } - if err := createJobConfigMap(*apiSpec, jobSpec); err != nil { - return nil, err - } - deployJob(apiSpec, &jobSpec) return &jobSpec, nil @@ -88,21 +84,13 @@ func uploadJobSpec(jobSpec *spec.TaskJob) error { return nil } -func createJobConfigMap(apiSpec spec.API, jobSpec spec.TaskJob) error { - configMapConfig := workloads.ConfigMapConfig{ - TaskJob: &jobSpec, - } - - configMapData, err := configMapConfig.GenerateConfigMapData() +func deployJob(apiSpec *spec.API, jobSpec *spec.TaskJob) { + err := createJobConfigMap(*apiSpec, *jobSpec) if err != nil { - return err + handleJobSubmissionError(jobSpec.JobKey, err) } - return createK8sConfigMap(k8sConfigMap(apiSpec, jobSpec, configMapData)) -} - -func deployJob(apiSpec *spec.API, jobSpec *spec.TaskJob) { - err := createK8sJob(apiSpec, jobSpec) + err = createK8sJob(apiSpec, jobSpec) if err != nil { handleJobSubmissionError(jobSpec.JobKey, err) } @@ -113,6 +101,19 @@ func deployJob(apiSpec *spec.API, jobSpec *spec.TaskJob) { } } +func createJobConfigMap(apiSpec spec.API, jobSpec spec.TaskJob) error { + configMapConfig := workloads.ConfigMapConfig{ + TaskJob: &jobSpec, + } + + configMapData, err := configMapConfig.GenerateConfigMapData() + if err != nil { + return err + } + + return createK8sConfigMap(k8sConfigMap(apiSpec, jobSpec, configMapData)) +} + func handleJobSubmissionError(jobKey spec.JobKey, jobErr error) { jobLogger, err := operator.GetJobLogger(jobKey) if err != nil { diff --git a/pkg/types/spec/validations.go b/pkg/types/spec/validations.go index 1bea4949ed..db997c1272 100644 --- a/pkg/types/spec/validations.go +++ b/pkg/types/spec/validations.go @@ -165,10 +165,7 @@ func podValidation(kind userconfig.Kind) *cr.StructFieldValidation { AllowExplicitNull: true, GreaterThan: pointer.Int32(0), LessThanOrEqualTo: pointer.Int32(65535), - DisallowedValues: []int32{ - consts.ProxyListeningPortInt32, - consts.AdminPortInt32, - }, + DisallowedValues: consts.ReservedContainerPorts, }, }, containersValidation(kind), @@ -366,10 +363,7 @@ func httpGetProbeValidation() *cr.StructFieldValidation { Required: true, GreaterThan: pointer.Int32(0), LessThanOrEqualTo: pointer.Int32(65535), - DisallowedValues: []int32{ - consts.ProxyListeningPortInt32, - consts.AdminPortInt32, - }, + DisallowedValues: consts.ReservedContainerPorts, }, }, }, @@ -391,10 +385,7 @@ func tcpSocketProbeValidation() *cr.StructFieldValidation { Required: true, GreaterThan: pointer.Int32(0), LessThanOrEqualTo: pointer.Int32(65535), - DisallowedValues: []int32{ - consts.ProxyListeningPortInt32, - consts.AdminPortInt32, - }, + DisallowedValues: consts.ReservedContainerPorts, }, }, }, From 7f2aaedc6b9e9fcb9012a251a036d32abd6e1d14 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Thu, 3 Jun 2021 12:40:07 -0700 Subject: [PATCH 80/82] Update examples --- test/apis/async/hello-world/app/main.py | 5 +- test/apis/async/text-generator/app/main.py | 29 +++++----- .../text-generator/app/requirements-cpu.txt | 1 - .../text-generator/app/requirements-gpu.txt | 1 - .../image-classifier-alexnet/app/main.py | 53 +++++++------------ .../app/requirements-cpu.txt | 2 +- .../app/requirements-gpu.txt | 2 +- test/apis/batch/sum/app/main.py | 33 +++++------- test/apis/realtime/hello-world/app/main.py | 5 +- .../apis/realtime/prime-generator/app/main.py | 6 +-- .../prime-generator/app/requirements.txt | 1 - test/apis/realtime/text-generator/app/main.py | 29 +++++----- .../text-generator/app/requirements-cpu.txt | 1 - .../text-generator/app/requirements-gpu.txt | 1 - .../app/requirements.txt | 2 +- 15 files changed, 67 insertions(+), 104 deletions(-) diff --git a/test/apis/async/hello-world/app/main.py b/test/apis/async/hello-world/app/main.py index ba4eac1bfb..61fef2b68b 100644 --- a/test/apis/async/hello-world/app/main.py +++ b/test/apis/async/hello-world/app/main.py @@ -4,8 +4,7 @@ from fastapi.responses import PlainTextResponse app = FastAPI() - -response_str = os.getenv("RESPONSE", "hello world") +app.response_str = os.getenv("RESPONSE", "hello world") @app.get("/healthz") @@ -15,4 +14,4 @@ def healthz(): @app.post("/") def handler(): - return {"message": response_str} + return {"message": app.response_str} diff --git a/test/apis/async/text-generator/app/main.py b/test/apis/async/text-generator/app/main.py index 39939e4fc6..7da2b4ea88 100644 --- a/test/apis/async/text-generator/app/main.py +++ b/test/apis/async/text-generator/app/main.py @@ -6,36 +6,31 @@ from transformers import GPT2Tokenizer, GPT2LMHeadModel app = FastAPI() -device = os.getenv("TARGET_DEVICE", "cpu") -state = { - "ready": False, - "tokenizer": None, - "model": None, -} +app.device = os.getenv("TARGET_DEVICE", "cpu") +app.ready = False @app.on_event("startup") def startup(): - global state - state["tokenizer"] = GPT2Tokenizer.from_pretrained("gpt2") - state["model"] = GPT2LMHeadModel.from_pretrained("gpt2").to(device) - state["ready"] = True + app.tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + app.model = GPT2LMHeadModel.from_pretrained("gpt2").to(app.device) + app.ready = True @app.get("/healthz") def healthz(): - if state["ready"]: + if app.ready: return PlainTextResponse("ok") return PlainTextResponse("service unavailable", status_code=status.HTTP_503_SERVICE_UNAVAILABLE) -class Request(BaseModel): +class Body(BaseModel): text: str @app.post("/") -def text_generator(request: Request): - input_length = len(request.text.split()) - tokens = state["tokenizer"].encode(request.text, return_tensors="pt").to(device) - prediction = state["model"].generate(tokens, max_length=input_length + 20, do_sample=True) - return {"text": state["tokenizer"].decode(prediction[0])} +def text_generator(body: Body): + input_length = len(body.text.split()) + tokens = app.tokenizer.encode(body.text, return_tensors="pt").to(app.device) + prediction = app.model.generate(tokens, max_length=input_length + 20, do_sample=True) + return {"text": app.tokenizer.decode(prediction[0])} diff --git a/test/apis/async/text-generator/app/requirements-cpu.txt b/test/apis/async/text-generator/app/requirements-cpu.txt index 91ef8d2188..91ec4ae257 100644 --- a/test/apis/async/text-generator/app/requirements-cpu.txt +++ b/test/apis/async/text-generator/app/requirements-cpu.txt @@ -1,6 +1,5 @@ uvicorn[standard] fastapi -pydantic transformers==3.0.* -f https://download.pytorch.org/whl/torch_stable.html torch==1.7.1+cpu diff --git a/test/apis/async/text-generator/app/requirements-gpu.txt b/test/apis/async/text-generator/app/requirements-gpu.txt index 9433f26464..f32b4a9708 100644 --- a/test/apis/async/text-generator/app/requirements-gpu.txt +++ b/test/apis/async/text-generator/app/requirements-gpu.txt @@ -1,5 +1,4 @@ uvicorn[standard] fastapi -pydantic transformers==3.0.* torch==1.7.* diff --git a/test/apis/batch/image-classifier-alexnet/app/main.py b/test/apis/batch/image-classifier-alexnet/app/main.py index 0ca1aa416f..dbdc8fbaaf 100644 --- a/test/apis/batch/image-classifier-alexnet/app/main.py +++ b/test/apis/batch/image-classifier-alexnet/app/main.py @@ -12,23 +12,13 @@ from fastapi.responses import PlainTextResponse app = FastAPI() -device = os.getenv("TARGET_DEVICE", "cpu") +app.device = os.getenv("TARGET_DEVICE", "cpu") +app.ready = False s3 = boto3.client("s3") -state = { - "ready": False, - "model": None, - "preprocess": None, - "job_id": None, - "labels": None, - "bucket": None, - "key": None, -} @app.on_event("startup") def startup(): - global state - # read job spec with open("/cortex/spec/job.json", "r") as f: job_spec = json.load(f) @@ -36,43 +26,40 @@ def startup(): # get metadata config = job_spec["config"] - job_id = job_spec["job_id"] - state["job_id"] = job_spec["job_id"] + app.job_id = job_spec["job_id"] if len(config.get("dest_s3_dir", "")) == 0: raise Exception("'dest_s3_dir' field was not provided in job submission") # s3 info - state["bucket"], state["key"] = re.match("s3://(.+?)/(.+)", config["dest_s3_dir"]).groups() - state["key"] = os.path.join(state["key"], job_id) + app.bucket, app.key = re.match("s3://(.+?)/(.+)", config["dest_s3_dir"]).groups() + app.key = os.path.join(app.key, app.job_id) # loading model normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) - state["preprocess"] = transforms.Compose( + app.preprocess = transforms.Compose( [transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize] ) - state["labels"] = requests.get( + app.labels = requests.get( "https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt" ).text.split("\n")[1:] - state["model"] = torchvision.models.alexnet(pretrained=True).eval().to(device) + app.model = torchvision.models.alexnet(pretrained=True).eval().to(app.device) - state["ready"] = True + app.ready = True @app.get("/healthz") def healthz(): - if state["ready"]: + if app.app.ready: return PlainTextResponse("ok") return PlainTextResponse("service unavailable", status_code=status.HTTP_503_SERVICE_UNAVAILABLE) @app.post("/") -async def handle_batch(request: Request): - payload: List[str] = await request.json() - job_id = state["job_id"] +def handle_batch(image_urls: List[str]): tensor_list = [] # download and preprocess each image - for image_url in payload: + for image_url in image_urls: if image_url.startswith("s3://"): bucket, image_key = re.match("s3://(.+?)/(.+)", image_url).groups() image_bytes = s3.get_object(Bucket=bucket, Key=image_key)["Body"].read() @@ -82,23 +69,23 @@ async def handle_batch(request: Request): raise RuntimeError(f"{image_url}: invalid image url") img_pil = Image.open(BytesIO(image_bytes)) - tensor_list.append(state["preprocess"](img_pil)) + tensor_list.append(app.preprocess(img_pil)) # classify the batch of images img_tensor = torch.stack(tensor_list) with torch.no_grad(): - prediction = state["model"](img_tensor) + prediction = app.model(img_tensor) _, indices = prediction.max(1) # extract predicted classes results = [ - {"url": payload[i], "class": state["labels"][class_idx]} + {"url": image_urls[i], "class": app.labels[class_idx]} for i, class_idx in enumerate(indices) ] json_output = json.dumps(results) # save results - s3.put_object(Bucket=state["bucket"], Key=f"{state['key']}/{job_id}.json", Body=json_output) + s3.put_object(Bucket=app.bucket, Key=f"{app.key}/{app.job_id}.json", Body=json_output) @app.post("/on-job-complete") @@ -107,16 +94,16 @@ def on_job_complete(): # aggregate all classifications paginator = s3.get_paginator("list_objects_v2") - for page in paginator.paginate(Bucket=state["bucket"], Prefix=state["key"]): + for page in paginator.paginate(Bucket=app.bucket, Prefix=app.key): if "Contents" not in page: continue for obj in page["Contents"]: - body = s3.get_object(Bucket=state["bucket"], Key=obj["Key"])["Body"] + body = s3.get_object(Bucket=app.bucket, Key=obj["Key"])["Body"] all_results += json.loads(body.read().decode("utf8")) # save single file containing aggregated classifications s3.put_object( - Bucket=state["bucket"], - Key=os.path.join(state["key"], "aggregated_results.json"), + Bucket=app.bucket, + Key=os.path.join(app.key, "aggregated_results.json"), Body=json.dumps(all_results), ) diff --git a/test/apis/batch/image-classifier-alexnet/app/requirements-cpu.txt b/test/apis/batch/image-classifier-alexnet/app/requirements-cpu.txt index 6d199faff8..9b04d0f0cb 100644 --- a/test/apis/batch/image-classifier-alexnet/app/requirements-cpu.txt +++ b/test/apis/batch/image-classifier-alexnet/app/requirements-cpu.txt @@ -3,5 +3,5 @@ fastapi requests boto3==1.17.* -f https://download.pytorch.org/whl/torch_stable.html -torchvision==0.8.2+cpu torch==1.7.1+cpu +torchvision==0.8.2+cpu diff --git a/test/apis/batch/image-classifier-alexnet/app/requirements-gpu.txt b/test/apis/batch/image-classifier-alexnet/app/requirements-gpu.txt index 2c8780d2a2..fb88b92e91 100644 --- a/test/apis/batch/image-classifier-alexnet/app/requirements-gpu.txt +++ b/test/apis/batch/image-classifier-alexnet/app/requirements-gpu.txt @@ -2,5 +2,5 @@ uvicorn[standard] fastapi requests boto3==1.17.* -torchvision==0.8.* torch==1.7.* +torchvision==0.8.* diff --git a/test/apis/batch/sum/app/main.py b/test/apis/batch/sum/app/main.py index 26ac026c42..a1d749505a 100644 --- a/test/apis/batch/sum/app/main.py +++ b/test/apis/batch/sum/app/main.py @@ -4,23 +4,17 @@ import re from typing import List -from fastapi import FastAPI, Request, status +from fastapi import FastAPI, status from fastapi.responses import PlainTextResponse app = FastAPI() +app.ready = False +app.numbers_list = [] s3 = boto3.client("s3") -state = { - "ready": False, - "bucket": None, - "key": None, - "numbers_list": [], -} @app.on_event("startup") def startup(): - global state - # read job spec with open("/cortex/spec/job.json", "r") as f: job_spec = json.load(f) @@ -33,28 +27,27 @@ def startup(): raise Exception("'dest_s3_dir' field was not provided in job submission") # s3 info - state["bucket"], state["key"] = re.match("s3://(.+?)/(.+)", config["dest_s3_dir"]).groups() - state["key"] = os.path.join(state["key"], job_id) + app.bucket, app.key = re.match("s3://(.+?)/(.+)", config["dest_s3_dir"]).groups() + app.key = os.path.join(app.key, job_id) - state["ready"] = True + app.ready = True @app.get("/healthz") def healthz(): - if state["ready"]: + if app.ready: return PlainTextResponse("ok") return PlainTextResponse("service unavailable", status_code=status.HTTP_503_SERVICE_UNAVAILABLE) @app.post("/") -async def handle_batch(request: Request): - global state - payload: List[List[int]] = await request.json() - for numbers_list in payload: - state["numbers_list"].append(sum(numbers_list)) +def handle_batch(batches: List[List[int]]): + for numbers_list in batches: + app.numbers_list.append(sum(numbers_list)) @app.post("/on-job-complete") def on_job_complete(): - json_output = json.dumps(state["numbers_list"]) - s3.put_object(Bucket=state["bucket"], Key=f"{state['key']}.json", Body=json_output) + # this is only intended to work if 1 worker is used (since on-job-complete runs once across all workers) + json_output = json.dumps(app.numbers_list) + s3.put_object(Bucket=app.bucket, Key=f"{app.key}.json", Body=json_output) diff --git a/test/apis/realtime/hello-world/app/main.py b/test/apis/realtime/hello-world/app/main.py index b0dfb5c7bf..6c49d69293 100644 --- a/test/apis/realtime/hello-world/app/main.py +++ b/test/apis/realtime/hello-world/app/main.py @@ -4,8 +4,7 @@ from fastapi.responses import PlainTextResponse app = FastAPI() - -response_str = os.getenv("RESPONSE", "hello world") +app.response_str = os.getenv("RESPONSE", "hello world") @app.get("/healthz") @@ -15,4 +14,4 @@ def healthz(): @app.post("/") def post_handler(): - return PlainTextResponse(response_str) + return PlainTextResponse(app.response_str) diff --git a/test/apis/realtime/prime-generator/app/main.py b/test/apis/realtime/prime-generator/app/main.py index 54c6f33eba..78c8227c8d 100644 --- a/test/apis/realtime/prime-generator/app/main.py +++ b/test/apis/realtime/prime-generator/app/main.py @@ -25,10 +25,10 @@ def healthz(): return "ok" -class Request(BaseModel): +class Body(BaseModel): primes_to_generate: float @app.post("/") -def prime_numbers(request: Request): - return {"prime_numbers": list(generate_primes(request.primes_to_generate))} +def prime_numbers(body: Body): + return {"prime_numbers": list(generate_primes(body.primes_to_generate))} diff --git a/test/apis/realtime/prime-generator/app/requirements.txt b/test/apis/realtime/prime-generator/app/requirements.txt index 81c6e29af4..190ccb7716 100644 --- a/test/apis/realtime/prime-generator/app/requirements.txt +++ b/test/apis/realtime/prime-generator/app/requirements.txt @@ -1,3 +1,2 @@ uvicorn[standard] fastapi -pydantic diff --git a/test/apis/realtime/text-generator/app/main.py b/test/apis/realtime/text-generator/app/main.py index 39939e4fc6..7da2b4ea88 100644 --- a/test/apis/realtime/text-generator/app/main.py +++ b/test/apis/realtime/text-generator/app/main.py @@ -6,36 +6,31 @@ from transformers import GPT2Tokenizer, GPT2LMHeadModel app = FastAPI() -device = os.getenv("TARGET_DEVICE", "cpu") -state = { - "ready": False, - "tokenizer": None, - "model": None, -} +app.device = os.getenv("TARGET_DEVICE", "cpu") +app.ready = False @app.on_event("startup") def startup(): - global state - state["tokenizer"] = GPT2Tokenizer.from_pretrained("gpt2") - state["model"] = GPT2LMHeadModel.from_pretrained("gpt2").to(device) - state["ready"] = True + app.tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + app.model = GPT2LMHeadModel.from_pretrained("gpt2").to(app.device) + app.ready = True @app.get("/healthz") def healthz(): - if state["ready"]: + if app.ready: return PlainTextResponse("ok") return PlainTextResponse("service unavailable", status_code=status.HTTP_503_SERVICE_UNAVAILABLE) -class Request(BaseModel): +class Body(BaseModel): text: str @app.post("/") -def text_generator(request: Request): - input_length = len(request.text.split()) - tokens = state["tokenizer"].encode(request.text, return_tensors="pt").to(device) - prediction = state["model"].generate(tokens, max_length=input_length + 20, do_sample=True) - return {"text": state["tokenizer"].decode(prediction[0])} +def text_generator(body: Body): + input_length = len(body.text.split()) + tokens = app.tokenizer.encode(body.text, return_tensors="pt").to(app.device) + prediction = app.model.generate(tokens, max_length=input_length + 20, do_sample=True) + return {"text": app.tokenizer.decode(prediction[0])} diff --git a/test/apis/realtime/text-generator/app/requirements-cpu.txt b/test/apis/realtime/text-generator/app/requirements-cpu.txt index 91ef8d2188..91ec4ae257 100644 --- a/test/apis/realtime/text-generator/app/requirements-cpu.txt +++ b/test/apis/realtime/text-generator/app/requirements-cpu.txt @@ -1,6 +1,5 @@ uvicorn[standard] fastapi -pydantic transformers==3.0.* -f https://download.pytorch.org/whl/torch_stable.html torch==1.7.1+cpu diff --git a/test/apis/realtime/text-generator/app/requirements-gpu.txt b/test/apis/realtime/text-generator/app/requirements-gpu.txt index 9433f26464..f32b4a9708 100644 --- a/test/apis/realtime/text-generator/app/requirements-gpu.txt +++ b/test/apis/realtime/text-generator/app/requirements-gpu.txt @@ -1,5 +1,4 @@ uvicorn[standard] fastapi -pydantic transformers==3.0.* torch==1.7.* diff --git a/test/apis/task/iris-classifier-trainer/app/requirements.txt b/test/apis/task/iris-classifier-trainer/app/requirements.txt index ead4e24356..28d8b71cf2 100644 --- a/test/apis/task/iris-classifier-trainer/app/requirements.txt +++ b/test/apis/task/iris-classifier-trainer/app/requirements.txt @@ -1,3 +1,3 @@ -boto3==1.17.* numpy==1.18.* scikit-learn==0.21.* +boto3==1.17.* From b178912b358a493fc589b225765b013cb67a22d5 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Thu, 3 Jun 2021 12:48:37 -0700 Subject: [PATCH 81/82] Update main.py --- test/apis/batch/image-classifier-alexnet/app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/apis/batch/image-classifier-alexnet/app/main.py b/test/apis/batch/image-classifier-alexnet/app/main.py index dbdc8fbaaf..a8f1211df2 100644 --- a/test/apis/batch/image-classifier-alexnet/app/main.py +++ b/test/apis/batch/image-classifier-alexnet/app/main.py @@ -49,7 +49,7 @@ def startup(): @app.get("/healthz") def healthz(): - if app.app.ready: + if app.ready: return PlainTextResponse("ok") return PlainTextResponse("service unavailable", status_code=status.HTTP_503_SERVICE_UNAVAILABLE) From ca7e454125d7d861b2b158585431207458e791ac Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Thu, 3 Jun 2021 17:15:43 -0700 Subject: [PATCH 82/82] Update dev workflow --- test/apis/async/hello-world/build-cpu.sh | 4 +- test/apis/async/text-generator/build-cpu.sh | 4 +- test/apis/async/text-generator/build-gpu.sh | 4 +- .../image-classifier-alexnet/build-cpu.sh | 4 +- .../image-classifier-alexnet/build-gpu.sh | 4 +- test/apis/batch/sum/build-cpu.sh | 4 +- test/apis/realtime/hello-world/build-cpu.sh | 4 +- .../image-classifier-resnet50/build-cpu.sh | 4 +- .../image-classifier-resnet50/build-gpu.sh | 4 +- .../realtime/prime-generator/build-cpu.sh | 4 +- test/apis/realtime/sleep/build-cpu.sh | 4 +- .../apis/realtime/text-generator/build-cpu.sh | 4 +- .../apis/realtime/text-generator/build-gpu.sh | 4 +- .../task/iris-classifier-trainer/build-cpu.sh | 4 +- test/utils/build-all.sh | 3 +- test/utils/build.sh | 52 +++++++++++++------ 16 files changed, 79 insertions(+), 32 deletions(-) diff --git a/test/apis/async/hello-world/build-cpu.sh b/test/apis/async/hello-world/build-cpu.sh index 12960564d9..259930bcaa 100755 --- a/test/apis/async/hello-world/build-cpu.sh +++ b/test/apis/async/hello-world/build-cpu.sh @@ -1,4 +1,6 @@ # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test -./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" +image_name="async-hello-world-cpu" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/async/text-generator/build-cpu.sh b/test/apis/async/text-generator/build-cpu.sh index 12960564d9..2f7c3dbb35 100755 --- a/test/apis/async/text-generator/build-cpu.sh +++ b/test/apis/async/text-generator/build-cpu.sh @@ -1,4 +1,6 @@ # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test -./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" +image_name="async-text-generator-cpu" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/async/text-generator/build-gpu.sh b/test/apis/async/text-generator/build-gpu.sh index 5acedee826..4048a3217f 100755 --- a/test/apis/async/text-generator/build-gpu.sh +++ b/test/apis/async/text-generator/build-gpu.sh @@ -1,4 +1,6 @@ # usage: build-gpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test -./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" +image_name="async-text-generator-gpu" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/batch/image-classifier-alexnet/build-cpu.sh b/test/apis/batch/image-classifier-alexnet/build-cpu.sh index 12960564d9..feb70a3d9c 100755 --- a/test/apis/batch/image-classifier-alexnet/build-cpu.sh +++ b/test/apis/batch/image-classifier-alexnet/build-cpu.sh @@ -1,4 +1,6 @@ # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test -./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" +image_name="batch-image-classifier-alexnet-cpu" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/batch/image-classifier-alexnet/build-gpu.sh b/test/apis/batch/image-classifier-alexnet/build-gpu.sh index 5acedee826..a4ef824dbb 100755 --- a/test/apis/batch/image-classifier-alexnet/build-gpu.sh +++ b/test/apis/batch/image-classifier-alexnet/build-gpu.sh @@ -1,4 +1,6 @@ # usage: build-gpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test -./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" +image_name="batch-image-classifier-alexnet-gpu" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/batch/sum/build-cpu.sh b/test/apis/batch/sum/build-cpu.sh index 12960564d9..6dbeb797d2 100755 --- a/test/apis/batch/sum/build-cpu.sh +++ b/test/apis/batch/sum/build-cpu.sh @@ -1,4 +1,6 @@ # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test -./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" +image_name="batch-sum-cpu" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/realtime/hello-world/build-cpu.sh b/test/apis/realtime/hello-world/build-cpu.sh index 12960564d9..d85bf31a36 100755 --- a/test/apis/realtime/hello-world/build-cpu.sh +++ b/test/apis/realtime/hello-world/build-cpu.sh @@ -1,4 +1,6 @@ # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test -./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" +image_name="realtime-hello-world-cpu" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/realtime/image-classifier-resnet50/build-cpu.sh b/test/apis/realtime/image-classifier-resnet50/build-cpu.sh index 12960564d9..d6ccd20732 100755 --- a/test/apis/realtime/image-classifier-resnet50/build-cpu.sh +++ b/test/apis/realtime/image-classifier-resnet50/build-cpu.sh @@ -1,4 +1,6 @@ # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test -./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" +image_name="realtime-image-classifier-resnet50-cpu" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/realtime/image-classifier-resnet50/build-gpu.sh b/test/apis/realtime/image-classifier-resnet50/build-gpu.sh index 5acedee826..3c45cf67d0 100755 --- a/test/apis/realtime/image-classifier-resnet50/build-gpu.sh +++ b/test/apis/realtime/image-classifier-resnet50/build-gpu.sh @@ -1,4 +1,6 @@ # usage: build-gpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test -./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" +image_name="realtime-image-classifier-resnet50-gpu" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/realtime/prime-generator/build-cpu.sh b/test/apis/realtime/prime-generator/build-cpu.sh index 12960564d9..08fa5bcafc 100755 --- a/test/apis/realtime/prime-generator/build-cpu.sh +++ b/test/apis/realtime/prime-generator/build-cpu.sh @@ -1,4 +1,6 @@ # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test -./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" +image_name="realtime-prime-generator-cpu" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/realtime/sleep/build-cpu.sh b/test/apis/realtime/sleep/build-cpu.sh index 12960564d9..f6e0431d08 100755 --- a/test/apis/realtime/sleep/build-cpu.sh +++ b/test/apis/realtime/sleep/build-cpu.sh @@ -1,4 +1,6 @@ # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test -./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" +image_name="realtime-sleep-cpu" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/realtime/text-generator/build-cpu.sh b/test/apis/realtime/text-generator/build-cpu.sh index 12960564d9..c6cd41e8a0 100755 --- a/test/apis/realtime/text-generator/build-cpu.sh +++ b/test/apis/realtime/text-generator/build-cpu.sh @@ -1,4 +1,6 @@ # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test -./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" +image_name="realtime-text-generator-cpu" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/realtime/text-generator/build-gpu.sh b/test/apis/realtime/text-generator/build-gpu.sh index 5acedee826..1dcd765695 100755 --- a/test/apis/realtime/text-generator/build-gpu.sh +++ b/test/apis/realtime/text-generator/build-gpu.sh @@ -1,4 +1,6 @@ # usage: build-gpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test -./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" +image_name="realtime-text-generator-gpu" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/apis/task/iris-classifier-trainer/build-cpu.sh b/test/apis/task/iris-classifier-trainer/build-cpu.sh index 12960564d9..3690ec24fe 100755 --- a/test/apis/task/iris-classifier-trainer/build-cpu.sh +++ b/test/apis/task/iris-classifier-trainer/build-cpu.sh @@ -1,4 +1,6 @@ # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test -./"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$@" +image_name="task-iris-classifier-trainer-cpu" + +"$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" diff --git a/test/utils/build-all.sh b/test/utils/build-all.sh index 52140082f6..ebaed61ab0 100755 --- a/test/utils/build-all.sh +++ b/test/utils/build-all.sh @@ -20,8 +20,7 @@ set -eo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. >/dev/null && pwd)" -source $ROOT/dev/util.sh for f in $(find $ROOT/test/apis -type f -name 'build-*.sh'); do - $ROOT/test/utils/build.sh $f "$@" + "$f" "$@" done diff --git a/test/utils/build.sh b/test/utils/build.sh index 2a1cdd4fb7..52eae868dc 100755 --- a/test/utils/build.sh +++ b/test/utils/build.sh @@ -14,7 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# usage: ./build.sh PATH [REGISTRY] [--skip-push] +# note: this only meant to be called from the build*.sh files in each test api directory +# usage: ./build.sh BUILDER_PATH IMAGE_NAME [REGISTRY] [--skip-push] # PATH is e.g. /home/ubuntu/src/github.com/cortexlabs/cortex/test/apis/realtime/sleep/build-cpu.sh # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test @@ -41,33 +42,48 @@ function create_ecr_repo() { green_echo "\nSuccess" } -path="$1" -registry="$CORTEX_DEV_DEFAULT_IMAGE_REGISTRY" should_skip_push="false" -for arg in "${@:2}"; do - if [ "$arg" = "--skip-push" ]; then +positional_args=() +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + -s|--skip-push) should_skip_push="true" - else - registry="$arg" - fi + shift + ;; + *) + positional_args+=("$1") + shift + ;; + esac done +set -- "${positional_args[@]}" + +builder_path="$1" # e.g. /home/ubuntu/src/github.com/cortexlabs/cortex/test/apis/realtime/hello-world/build-cpu.sh +image_name="$2" # e.g. realtime-hello-world-cpu +registry=${3:-$CORTEX_DEV_DEFAULT_IMAGE_REGISTRY} # e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs if [ -z "$registry" ]; then error_echo "registry must be provided as a positional arg, or $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY must be set" fi -name="$(basename $(dirname "$path"))" # e.g. sleep -kind="$(basename $(dirname $(dirname "$path")))" # e.g. realtime -architecture="$(echo "$(basename "$path" .sh)" | sed 's/.*-//')" # e.g. cpu -image_url="$registry/$kind-$name-$architecture" # e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-cpu -if [[ "$image_url" == *".ecr."* ]]; then - login_url="$(echo "$image_url" | sed 's/\/.*//')" # e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com +image_url="${registry}/${image_name}" # e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-cpu +dockerfile_name="$(echo "$builder_path" | sed 's/.*build-//' | sed 's/\..*//')" # e.g. cpu +api_dir="$(dirname $builder_path)" # e.g. /home/ubuntu/src/github.com/cortexlabs/cortex/test/apis/realtime/hello-world +dockerfile_path="${api_dir}/${dockerfile_name}.Dockerfile" # e.g. /home/ubuntu/src/github.com/cortexlabs/cortex/test/apis/realtime/hello-world/cpu.Dockerfile +if [[ "$registry" == *".ecr."* ]]; then + login_url="$(echo "$registry" | sed 's/\/.*//')" # e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com repo_name="$(echo $image_url | sed 's/[^\/]*\///')" # e.g. cortexlabs/realtime-sleep-cpu - region="$(echo "$image_url" | sed 's/.*\.ecr\.//' | sed 's/\..*//')" # e.g. us-west-2 + region="$(echo "$registry" | sed 's/.*\.ecr\.//' | sed 's/\..*//')" # e.g. us-west-2 +fi + +if [ ! -f "$dockerfile_path" ]; then + error_echo "$dockerfile_path does not exist" + exit 1 fi blue_echo "Building $image_url:latest\n" -docker build "$(dirname "$path")" -f "$(dirname "$path")/$architecture.Dockerfile" -t "$image_url" +docker build "$api_dir" -f "$dockerfile_path" -t "$image_url" green_echo "\nBuilt $image_url:latest" if [ "$should_skip_push" = "true" ]; then @@ -97,3 +113,7 @@ while true; do green_echo "\nPushed $image_url:latest" break done + +# update api config +find $api_dir -type f -name 'cortex_*.yaml' \ + -exec sed -i "s|quay.io/cortexlabs-test/${image_name}|${image_url}|g" {} \;