Reworking logs (#1778)

Author: vishalbollu

CONTRIBUTING.md

@@ -186,7 +186,7 @@ image_metrics_server: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/met
 image_inferentia: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/inferentia:latest
 image_neuron_rtd: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/neuron-rtd:latest
 image_nvidia: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/nvidia:latest
-image_fluentd: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/fluentd:latest
+image_fluent_bit: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/fluent_bit:latest
 image_statsd: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/statsd:latest
 image_istio_proxy: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/istio-proxy:latest
 image_istio_pilot: <account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs/istio-pilot:latest

build/images.sh

@@ -67,6 +67,7 @@ non_dev_images_cluster=(
   "operator"
   "istio-proxy"
   "istio-pilot"
+  "fluent-bit"
 )
 non_dev_images_aws=(
   # includes non_dev_images_cluster
@@ -76,7 +77,6 @@ non_dev_images_aws=(
   "inferentia"
   "neuron-rtd"
   "nvidia"
-  "fluentd"
   "statsd"
 )
 non_dev_images_gcp=(

cli/cluster/logs.go

@@ -25,11 +25,11 @@ import (
 	"os/signal"
 	"strings"
 
+	"github.com/cortexlabs/cortex/cli/lib/routines"
 	"github.com/cortexlabs/cortex/pkg/consts"
 	"github.com/cortexlabs/cortex/pkg/lib/errors"
 	"github.com/cortexlabs/cortex/pkg/lib/exit"
 	"github.com/cortexlabs/cortex/pkg/lib/json"
-	"github.com/cortexlabs/cortex/pkg/lib/routines"
 	"github.com/cortexlabs/cortex/pkg/operator/schema"
 	"github.com/gorilla/websocket"
 )
@@ -120,7 +120,7 @@ func handleConnection(connection *websocket.Conn, done chan struct{}) {
 			if err != nil {
 				exit.Error(ErrorOperatorSocketRead(err))
 			}
-			fmt.Println(string(message))
+			fmt.Print(string(message))
 		}
 	}, false)
 }

cli/cmd/cluster_gcp.go

@@ -435,6 +435,7 @@ func createGKECluster(clusterConfig *clusterconfig.GCPConfig, gcpClient *gcp.Cli
 	gkeClusterConfig := containerpb.Cluster{
 		Name:                  clusterConfig.ClusterName,
 		InitialClusterVersion: "1.17",
+		LoggingService:        "none",
 		NodePools: []*containerpb.NodePool{
 			{
 				Name: "ng-cortex-operator",

cli/cmd/lib_cluster_config_aws.go

@@ -383,10 +383,10 @@ func setConfigFieldsFromCached(userClusterConfig *clusterconfig.Config, cachedCl
 	}
 	userClusterConfig.ImageNvidia = cachedClusterConfig.ImageNvidia
 
-	if s.Obj(cachedClusterConfig.ImageFluentd) != s.Obj(userClusterConfig.ImageFluentd) {
-		return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImageFluentdKey, cachedClusterConfig.ImageFluentd)
+	if s.Obj(cachedClusterConfig.ImageFluentBit) != s.Obj(userClusterConfig.ImageFluentBit) {
+		return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImageFluentBitKey, cachedClusterConfig.ImageFluentBit)
 	}
-	userClusterConfig.ImageFluentd = cachedClusterConfig.ImageFluentd
+	userClusterConfig.ImageFluentBit = cachedClusterConfig.ImageFluentBit
 
 	if s.Obj(cachedClusterConfig.ImageStatsd) != s.Obj(userClusterConfig.ImageStatsd) {
 		return clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.ImageStatsdKey, cachedClusterConfig.ImageStatsd)
@@ -691,8 +691,8 @@ func clusterConfigConfirmationStr(clusterConfig clusterconfig.Config, awsCreds A
 	if clusterConfig.ImageNvidia != defaultConfig.ImageNvidia {
 		items.Add(clusterconfig.ImageNvidiaUserKey, clusterConfig.ImageNvidia)
 	}
-	if clusterConfig.ImageFluentd != defaultConfig.ImageFluentd {
-		items.Add(clusterconfig.ImageFluentdUserKey, clusterConfig.ImageFluentd)
+	if clusterConfig.ImageFluentBit != defaultConfig.ImageFluentBit {
+		items.Add(clusterconfig.ImageFluentBitUserKey, clusterConfig.ImageFluentBit)
 	}
 	if clusterConfig.ImageStatsd != defaultConfig.ImageStatsd {
 		items.Add(clusterconfig.ImageStatsdUserKey, clusterConfig.ImageStatsd)

cli/cmd/lib_manager.go

@@ -28,13 +28,13 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/cortexlabs/cortex/cli/lib/routines"
 	"github.com/cortexlabs/cortex/pkg/consts"
 	"github.com/cortexlabs/cortex/pkg/lib/archive"
 	"github.com/cortexlabs/cortex/pkg/lib/docker"
 	"github.com/cortexlabs/cortex/pkg/lib/errors"
 	"github.com/cortexlabs/cortex/pkg/lib/exit"
 	"github.com/cortexlabs/cortex/pkg/lib/files"
-	"github.com/cortexlabs/cortex/pkg/lib/routines"
 	"github.com/cortexlabs/cortex/pkg/types/clusterconfig"
 	"github.com/cortexlabs/yaml"
 	dockertypes "github.com/docker/docker/api/types"

cli/cmd/logs.go

@@ -17,28 +17,27 @@ limitations under the License.
 package cmd
 
 import (
-	"fmt"
-	"net/http"
-	"net/url"
-
 	"github.com/cortexlabs/cortex/cli/cluster"
-	"github.com/cortexlabs/cortex/pkg/lib/console"
 	"github.com/cortexlabs/cortex/pkg/lib/exit"
+	"github.com/cortexlabs/cortex/pkg/lib/prompt"
 	"github.com/cortexlabs/cortex/pkg/lib/telemetry"
-	"github.com/cortexlabs/cortex/pkg/types"
 	"github.com/spf13/cobra"
 )
 
-var _flagLogsEnv string
+var (
+	_flagLogsEnv            string
+	_flagLogsDisallowPrompt bool
+)
 
 func logsInit() {
 	_logsCmd.Flags().SortFlags = false
 	_logsCmd.Flags().StringVarP(&_flagLogsEnv, "env", "e", "", "environment to use")
+	_logsCmd.Flags().BoolVarP(&_flagLogsDisallowPrompt, "yes", "y", false, "skip prompts")
 }
 
 var _logsCmd = &cobra.Command{
 	Use:   "logs API_NAME [JOB_ID]",
-	Short: "stream logs from an api",
+	Short: "stream logs from a single replica of an api or a single worker for a job",
 	Args:  cobra.RangeArgs(1, 2),
 	Run: func(cmd *cobra.Command, args []string) {
 		envName, err := getEnvFromFlag(_flagLogsEnv)
@@ -59,43 +58,39 @@ var _logsCmd = &cobra.Command{
 			exit.Error(err)
 		}
 
+		operatorConfig := MustGetOperatorConfig(env.Name)
 		apiName := args[0]
-		if env.Provider == types.AWSProviderType {
-			if len(args) == 1 {
-				err := cluster.StreamLogs(MustGetOperatorConfig(env.Name), apiName)
-				if err != nil {
-					exit.Error(err)
-				}
+
+		if len(args) == 1 {
+			apiResponse, err := cluster.GetAPI(operatorConfig, apiName)
+			if err != nil {
+				exit.Error(err)
 			}
-			if len(args) == 2 {
-				err := cluster.StreamJobLogs(MustGetOperatorConfig(env.Name), apiName, args[1])
-				if err != nil {
-					exit.Error(err)
-				}
+
+			if apiResponse[0].Status.Requested > 1 && !_flagLogsDisallowPrompt {
+				prompt.YesOrExit("logs from a single random replica will be streamed\n\nfor aggregated logs please visit your cloud provider's logging dashboard; see https://docs.cortex.dev for details", "", "")
 			}
-		}
 
-		if env.Provider == types.GCPProviderType {
-			gcpLogsResponse, err := cluster.GetGCPLogsURL(MustGetOperatorConfig(env.Name), apiName)
+			err = cluster.StreamLogs(operatorConfig, apiName)
 			if err != nil {
 				exit.Error(err)
 			}
-
-			gcpReq, err := http.NewRequest("GET", "https://console.cloud.google.com/logs/query", nil)
+		}
+		if len(args) == 2 {
+			jobResponse, err := cluster.GetJob(operatorConfig, apiName, args[1])
 			if err != nil {
 				exit.Error(err)
 			}
-			query := ""
-			for q, v := range gcpLogsResponse.QueryParams {
-				query += fmt.Sprintf("%s=\"%s\"\n", q, v)
+
+			if jobResponse.JobStatus.Job.Workers > 1 && !_flagLogsDisallowPrompt {
+				prompt.YesOrExit("logs from a single random worker will be streamed\n\nfor aggregated logs please visit your cloud provider's logging dashboard; see https://docs.cortex.dev for details", "", "")
 			}
-			queryValues := make(url.Values)
-			queryValues.Add("query", query)
-			gcpReq.URL.RawQuery = queryValues.Encode()
 
-			gcpLogsURL := gcpReq.URL.String()
-			consoleOutput := console.Bold(fmt.Sprintf("visit the following link to view logs for api %s: ", apiName)) + gcpLogsURL
-			fmt.Println(consoleOutput)
+			err = cluster.StreamJobLogs(operatorConfig, apiName, args[1])
+			if err != nil {
+				exit.Error(err)
+			}
 		}
+
 	},
 }

pkg/lib/routines/routines.go renamed to cli/lib/routines/routines.go

@@ -235,12 +235,11 @@ Note: overriding horizontal-pod-autoscaler-sync-period on EKS is currently not s
 1. In the [GitHub Repo](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws), set the tree to the tag for the chosen release, and open `cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml` (e.g. <https://github.com/kubernetes/autoscaler/blob/cluster-autoscaler-1.16.5/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml>)
 1. Resolve merge conflicts with the template in `manager/manifests/cluster-autoscaler.yaml.j2`
 
-## Fluentd
+## FluentBit
 
-1. Find the latest release on [Dockerhub](https://hub.docker.com/r/fluent/fluentd-kubernetes-daemonset/)
-1. Update the base image version in `images/fluentd/Dockerfile`
-1. Update record-modifier in `images/fluentd/Dockerfile` to the latest version [here](https://github.com/repeatedly/fluent-plugin-record-modifier/blob/master/VERSION)
-1. Update `fluentd.yaml` as necessary (make sure to maintain all Cortex environment variables)
+1. Find the latest release on [Dockerhub](https://hub.docker.com/r/amazon/aws-for-fluent-bit/tags?page=1&ordering=last_updated)
+1. Update the base image version in `images/fluent-bit/Dockerfile`
+1. Update `fluent-bit.yaml` as necessary (make sure to maintain all Cortex environment variables)
 
 ## Statsd
 
@@ -258,7 +257,7 @@ Note: overriding horizontal-pod-autoscaler-sync-period on EKS is currently not s
 ## kubectl
 
 1. Find the latest release [here](https://storage.googleapis.com/kubernetes-release/release/stable.txt)
-1. Update the version in `images/manager/Dockerfile`
+1. Update the version in `images/manager/Dockerfile` and `images/operator/Dockerfile`
 1. Update your local version and alert developers
    * Linux:
      1. `curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl`

dev/versions.md

@@ -35,13 +35,14 @@ Flags:
 ## logs
 
 ```text
-stream logs from an api
+stream logs from a single replica of an api or a single worker for a job
 
 Usage:
   cortex logs API_NAME [JOB_ID] [flags]
 
 Flags:
   -e, --env string   environment to use
+  -y, --yes          skip prompts
   -h, --help         help for logs
 ```