correct alert json in demo script, refactor querier selection

Author: nikhilsinhaparseable

Cargo.toml

@@ -132,7 +132,7 @@ ulid = { version = "1.0", features = ["serde"] }
 xxhash-rust = { version = "0.8", features = ["xxh3"] }
 futures-core = "0.3.31"
 tempfile = "3.20.0"
-lazy_static = "1.5.0"
+lazy_static = "1.4.0"
 
 [build-dependencies]
 cargo_toml = "0.21"

resources/ingest_demo_data.sh

@@ -437,7 +437,7 @@ create_alerts() {
     fi
 
     # Alert 3: Trace ID null
-    alert3_json="{\"severity\":\"high\",\"title\":\"Trace ID null\",\"alertType\":\"threshold\",\"query\": \"select count(trace_id) as count_trace_id from demodata where trace_id is null\",\"thresholdConfig\":{\"operator\":\"is null\",\"value\":null},\"evalConfig\":{\"rollingWindow\":{\"evalStart\":\"5h\",\"evalEnd\":\"now\",\"evalFrequency\":1}},\"targets\":[\"$target_id\"]}"
+    alert3_json="{\"severity\":\"high\",\"title\":\"Trace ID null\",\"alertType\":\"threshold\",\"query\": \"select count(trace_id) as count_trace_id from demodata where trace_id is null\",\"thresholdConfig\":{\"operator\":\">\",\"value\":0},\"evalConfig\":{\"rollingWindow\":{\"evalStart\":\"5h\",\"evalEnd\":\"now\",\"evalFrequency\":1}},\"targets\":[\"$target_id\"]}"
     response3=$(curl_with_retry "$P_URL/api/v1/alerts" "POST" "$alert3_json" "application/json" 3)
     if [[ $? -eq 0 ]]; then
         echo "Alert 3 (Trace ID null) created successfully"

src/handlers/http/cluster/mod.rs

@@ -1308,6 +1308,9 @@ pub async fn mark_querier_available(domain_name: &str) {
 
 pub async fn send_query_request(query_request: &Query) -> Result<(JsonValue, String), QueryError> {
     let querier = get_available_querier().await?;
+    let domain_name = querier.domain_name.clone();
+
+    // Perform the query request
     let fields = query_request.fields;
     let streaming = query_request.streaming;
     let send_null = query_request.send_null;
@@ -1316,16 +1319,30 @@ pub async fn send_query_request(query_request: &Query) -> Result<(JsonValue, Str
         &querier.domain_name,
     );
 
-    let body = serde_json::to_string(&query_request)?;
-    let res = INTRA_CLUSTER_CLIENT
+    let body = match serde_json::to_string(&query_request) {
+        Ok(body) => body,
+        Err(err) => {
+            mark_querier_available(&domain_name).await;
+            return Err(QueryError::from(err));
+        }
+    };
+
+    let res = match INTRA_CLUSTER_CLIENT
         .post(uri)
         .header(header::AUTHORIZATION, &querier.token)
         .header(header::CONTENT_TYPE, "application/json")
         .body(body)
         .send()
-        .await?;
+        .await
+    {
+        Ok(res) => res,
+        Err(err) => {
+            mark_querier_available(&domain_name).await;
+            return Err(QueryError::from(err));
+        }
+    };
 
-    let domain_name = querier.domain_name.clone();
+    // Mark querier as available immediately after the HTTP request completes
     mark_querier_available(&domain_name).await;
 
     let headers = res.headers();