Skip to content

Commit 8180cbe

Browse files
nitishtnitisht
committed
Fix help messages for environment variables / flags
Fixes #586 This PR also adds * gRPC port number in startup banner. * Human readable input for size / capacity type environment variable
1 parent 83f3cc9 commit 8180cbe

File tree

4 files changed

+170
-39
lines changed

4 files changed

+170
-39
lines changed

‎server/src/banner.rs‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,13 @@ fn print_ascii_art() {
5151
}
5252

5353
fn status_info(config: &Config, scheme: &str, id: Uid) {
54-
let url = format!("\"{}://{}\"", scheme, config.parseable.address).underlined();
54+
let address = format!(
55+
"\"{}://{}\" ({}), \":{}\" (gRPC)",
56+
scheme,
57+
config.parseable.address,
58+
scheme.to_ascii_uppercase(),
59+
config.parseable.grpc_port
60+
);
5561
let mut credentials =
5662
String::from("\"As set in P_USERNAME and P_PASSWORD environment variables\"");
5763

@@ -67,12 +73,12 @@ fn status_info(config: &Config, scheme: &str, id: Uid) {
6773
eprintln!(
6874
"
6975
{}
70-
URL: {}
76+
Address: {}
7177
Credentials: {}
7278
Deployment UID: \"{}\"
73-
LLM Status: \"{}\"",
79+
LLM: \"{}\"",
7480
"Server:".to_string().bold(),
75-
url,
81+
address,
7682
credentials,
7783
id.to_string(),
7884
llm_status

‎server/src/option.rs‎

Lines changed: 41 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -205,8 +205,8 @@ pub struct Server {
205205
/// Rows in Parquet Rowgroup
206206
pub row_group_size: usize,
207207

208-
/// Query memory limit in bytes
209-
pub query_memory_pool_size: Option<usize>,
208+
/// Query memory limit in human readable format (e.g 1GiB, 2GiB, 100MB)
209+
pub query_memory_pool_size: Option<String>,
210210

211211
/// Parquet compression algorithm
212212
pub parquet_compression: Compression,
@@ -263,12 +263,9 @@ impl FromArgMatches for Server {
263263
self.livetail_channel_capacity = m
264264
.get_one::<usize>(Self::LIVETAIL_CAPACITY)
265265
.cloned()
266-
.expect("default for livetail port");
266+
.expect("default for livetail capacity");
267267
// converts Gib to bytes before assigning
268-
self.query_memory_pool_size = m
269-
.get_one::<u8>(Self::QUERY_MEM_POOL_SIZE)
270-
.cloned()
271-
.map(|gib| gib as usize * 1024usize.pow(3));
268+
self.query_memory_pool_size = m.get_one::<String>(Self::QUERY_MEM_POOL_SIZE).cloned();
272269
self.row_group_size = m
273270
.get_one::<usize>(Self::ROW_GROUP_SIZE)
274271
.cloned()
@@ -356,15 +353,15 @@ impl Server {
356353
.env("P_TLS_CERT_PATH")
357354
.value_name("PATH")
358355
.value_parser(validation::file_path)
359-
.help("The location of TLS Cert file"),
356+
.help("Local path on this device where certificate file is located. Required to enable TLS"),
360357
)
361358
.arg(
362359
Arg::new(Self::TLS_KEY)
363360
.long(Self::TLS_KEY)
364361
.env("P_TLS_KEY_PATH")
365362
.value_name("PATH")
366363
.value_parser(validation::file_path)
367-
.help("The location of TLS Private Key file"),
364+
.help("Local path on this device where private key file is located. Required to enable TLS"),
368365
)
369366
.arg(
370367
Arg::new(Self::ADDRESS)
@@ -373,7 +370,7 @@ impl Server {
373370
.value_name("ADDR:PORT")
374371
.default_value("0.0.0.0:8000")
375372
.value_parser(validation::socket_addr)
376-
.help("The address on which the http server will listen."),
373+
.help("Address and port for Parseable HTTP(s) server"),
377374
)
378375
.arg(
379376
Arg::new(Self::STAGING)
@@ -382,7 +379,7 @@ impl Server {
382379
.value_name("DIR")
383380
.default_value("./staging")
384381
.value_parser(validation::canonicalize_path)
385-
.help("The local staging path is used as a temporary landing point for incoming events and local cache")
382+
.help("Local path on this device to be used as landing point for incoming events")
386383
.next_line_help(true),
387384
)
388385
.arg(
@@ -392,7 +389,7 @@ impl Server {
392389
.value_name("SECONDS")
393390
.default_value("60")
394391
.value_parser(value_parser!(u64))
395-
.help("Interval in seconds after which un-committed data would be sent to the storage")
392+
.help("Interval in seconds after which staging data would be sent to the storage")
396393
.next_line_help(true),
397394
)
398395
.arg(
@@ -401,15 +398,15 @@ impl Server {
401398
.env("P_USERNAME")
402399
.value_name("STRING")
403400
.required(true)
404-
.help("Username for the basic authentication on the server"),
401+
.help("Admin username for this server"),
405402
)
406403
.arg(
407404
Arg::new(Self::PASSWORD)
408405
.long(Self::PASSWORD)
409406
.env("P_PASSWORD")
410407
.value_name("STRING")
411408
.required(true)
412-
.help("Password for the basic authentication on the server"),
409+
.help("Admin password for this server"),
413410
)
414411
.arg(
415412
Arg::new(Self::CHECK_UPDATE)
@@ -429,31 +426,31 @@ impl Server {
429426
.required(false)
430427
.default_value("true")
431428
.value_parser(value_parser!(bool))
432-
.help("Disable/Enable sending anonymous user data"),
429+
.help("Disable/Enable sending anonymous telemetry"),
433430
)
434431
.arg(
435432
Arg::new(Self::OPEN_AI_KEY)
436433
.long(Self::OPEN_AI_KEY)
437434
.env("P_OPENAI_API_KEY")
438435
.value_name("STRING")
439436
.required(false)
440-
.help("Set OpenAI key to enable llm feature"),
437+
.help("OpenAI key to enable llm feature"),
441438
)
442439
.arg(
443440
Arg::new(Self::OPENID_CLIENT_ID)
444441
.long(Self::OPENID_CLIENT_ID)
445442
.env("P_OIDC_CLIENT_ID")
446443
.value_name("STRING")
447444
.required(false)
448-
.help("Set client id for oidc provider"),
445+
.help("Client id for OIDC provider"),
449446
)
450447
.arg(
451448
Arg::new(Self::OPENID_CLIENT_SECRET)
452449
.long(Self::OPENID_CLIENT_SECRET)
453450
.env("P_OIDC_CLIENT_SECRET")
454451
.value_name("STRING")
455452
.required(false)
456-
.help("Set client secret for oidc provider"),
453+
.help("Client secret for OIDC provider"),
457454
)
458455
.arg(
459456
Arg::new(Self::OPENID_ISSUER)
@@ -462,7 +459,7 @@ impl Server {
462459
.value_name("URl")
463460
.required(false)
464461
.value_parser(validation::url)
465-
.help("Set OIDC provider's host address."),
462+
.help("OIDC provider's host address"),
466463
)
467464
.arg(
468465
Arg::new(Self::DOMAIN_URI)
@@ -471,7 +468,7 @@ impl Server {
471468
.value_name("URL")
472469
.required(false)
473470
.value_parser(validation::url)
474-
.help("Set host global domain address"),
471+
.help("Parseable server global domain address"),
475472
)
476473
.arg(
477474
Arg::new(Self::GRPC_PORT)
@@ -481,7 +478,7 @@ impl Server {
481478
.default_value("8001")
482479
.required(false)
483480
.value_parser(value_parser!(u16))
484-
.help("Set port for livetail arrow flight server"),
481+
.help("Port for gRPC server"),
485482
)
486483
.arg(
487484
Arg::new(Self::LIVETAIL_CAPACITY)
@@ -491,16 +488,16 @@ impl Server {
491488
.default_value("1000")
492489
.required(false)
493490
.value_parser(value_parser!(usize))
494-
.help("Set port for livetail arrow flight server"),
491+
.help("Number of rows in livetail channel"),
495492
)
496493
.arg(
497494
Arg::new(Self::QUERY_MEM_POOL_SIZE)
498495
.long(Self::QUERY_MEM_POOL_SIZE)
499496
.env("P_QUERY_MEMORY_LIMIT")
500-
.value_name("Gib")
497+
.value_name("STRING")
501498
.required(false)
502-
.value_parser(value_parser!(u8))
503-
.help("Set a fixed memory limit for query"),
499+
.value_parser(validation::size)
500+
.help("Memory allocated to query in human readable format (e.g 1GiB, 2GiB, 100MB)"),
504501
)
505502
.arg(
506503
Arg::new(Self::ROW_GROUP_SIZE)
@@ -510,7 +507,7 @@ impl Server {
510507
.required(false)
511508
.default_value("16384")
512509
.value_parser(value_parser!(usize))
513-
.help("Number of rows in a row groups"),
510+
.help("Number of rows in a row group"),
514511
)
515512
.arg(
516513
Arg::new(Self::PARQUET_COMPRESSION_ALGO)
@@ -606,4 +603,22 @@ pub mod validation {
606603
pub fn url(s: &str) -> Result<url::Url, String> {
607604
url::Url::parse(s).map_err(|_| "Invalid URL provided".to_string())
608605
}
606+
607+
pub fn size(s: &str) -> Result<(), String> {
608+
if let Ok(re) = regex::Regex::new(r"^(\d+)([TGMKP][iI]?[Bb]?)$") {
609+
if re.captures(s).is_some() {
610+
Ok(())
611+
} else {
612+
Err(
613+
"Invalid size provided. Please provide size in the format '1GiB', '3MiB', etc"
614+
.to_string(),
615+
)
616+
}
617+
} else {
618+
Err(
619+
"Invalid size provided. Please provide size in the format '1GiB', '3MiB', etc"
620+
.to_string(),
621+
)
622+
}
623+
}
609624
}

‎server/src/query.rs‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ use sysinfo::{System, SystemExt};
4242
use crate::event;
4343
use crate::option::CONFIG;
4444
use crate::storage::{ObjectStorageProvider, StorageDir};
45+
use crate::utils;
4546

4647
use self::error::ExecuteError;
4748

@@ -68,15 +69,16 @@ impl Query {
6869
.get_datafusion_runtime()
6970
.with_disk_manager(DiskManagerConfig::NewOs);
7071

71-
let (pool_size, fraction) = match CONFIG.parseable.query_memory_pool_size {
72-
Some(size) => (size, 1.),
73-
None => {
74-
let mut system = System::new();
75-
system.refresh_memory();
76-
let available_mem = system.available_memory();
77-
(available_mem as usize, 0.85)
78-
}
79-
};
72+
let (pool_size, fraction) =
73+
match utils::parse_storage_size(CONFIG.parseable.query_memory_pool_size.clone()) {
74+
Some(size) => (size as usize, 1.),
75+
None => {
76+
let mut system = System::new();
77+
system.refresh_memory();
78+
let available_mem = system.available_memory();
79+
(available_mem as usize, 0.85)
80+
}
81+
};
8082

8183
let runtime_config = runtime_config.with_memory_limit(pool_size, fraction);
8284
let runtime = Arc::new(RuntimeEnv::new(runtime_config).unwrap());

‎server/src/utils.rs‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,11 +235,65 @@ impl TimePeriod {
235235
}
236236
}
237237

238+
pub fn parse_storage_size(size: Option<String>) -> Option<u64> {
239+
let re = regex::Regex::new(r"^(\d+)([TGMKP][iI]?[Bb]?)$").unwrap();
240+
241+
if let Some(size) = size {
242+
if let Some(captures) = re.captures(size.as_str()) {
243+
if let Some(number) = captures.get(1) {
244+
if let Some(unit) = captures.get(2) {
245+
match unit.as_str() {
246+
"b" => Some(number.as_str().parse::<u64>().unwrap()),
247+
"kb" => Some((number.as_str().parse::<f64>().unwrap() * 1e3) as u64),
248+
"mb" => Some((number.as_str().parse::<f64>().unwrap() * 1e6) as u64),
249+
"gb" => Some((number.as_str().parse::<f64>().unwrap() * 1e9) as u64),
250+
"tb" => Some((number.as_str().parse::<f64>().unwrap() * 1e12) as u64),
251+
"pb" => Some((number.as_str().parse::<f64>().unwrap() * 1e15) as u64),
252+
"kib" => Some((number.as_str().parse::<f64>().unwrap() * 1024.0) as u64),
253+
"mib" => {
254+
Some((number.as_str().parse::<f64>().unwrap() * 1024.0 * 1024.0) as u64)
255+
}
256+
"gib" => Some(
257+
(number.as_str().parse::<f64>().unwrap() * 1024.0 * 1024.0 * 1024.0)
258+
as u64,
259+
),
260+
"tib" => Some(
261+
(number.as_str().parse::<f64>().unwrap()
262+
* 1024.0
263+
* 1024.0
264+
* 1024.0
265+
* 1024.0) as u64,
266+
),
267+
"pib" => Some(
268+
(number.as_str().parse::<f64>().unwrap()
269+
* 1024.0
270+
* 1024.0
271+
* 1024.0
272+
* 1024.0
273+
* 1024.0) as u64,
274+
),
275+
_ => None,
276+
}
277+
} else {
278+
None
279+
}
280+
} else {
281+
None
282+
}
283+
} else {
284+
None
285+
}
286+
} else {
287+
None
288+
}
289+
}
290+
238291
#[cfg(test)]
239292
mod tests {
240293
use chrono::DateTime;
241294
use rstest::*;
242295

296+
use super::parse_storage_size;
243297
use super::TimePeriod;
244298

245299
fn time_period_from_str(start: &str, end: &str) -> TimePeriod {
@@ -316,4 +370,58 @@ mod tests {
316370
let left = prefixes.iter().map(String::as_str).collect::<Vec<&str>>();
317371
assert_eq!(left.as_slice(), right);
318372
}
373+
374+
#[rstest]
375+
fn test_parse_storage_size() {
376+
// Valid SI (decimal) formats
377+
assert_eq!(parse_storage_size(Some(String::from("1 b"))), Some(1));
378+
assert_eq!(
379+
parse_storage_size(Some(String::from("1 KB"))),
380+
Some(1e3 as u64)
381+
);
382+
assert_eq!(
383+
parse_storage_size(Some(String::from("1 MB"))),
384+
Some(1e6 as u64)
385+
);
386+
assert_eq!(
387+
parse_storage_size(Some(String::from("1 GB"))),
388+
Some(1e9 as u64)
389+
);
390+
assert_eq!(
391+
parse_storage_size(Some(String::from("1 TB"))),
392+
Some(1e12 as u64)
393+
);
394+
assert_eq!(
395+
parse_storage_size(Some(String::from("1 PB"))),
396+
Some(1e15 as u64)
397+
);
398+
399+
// Valid IEC (binary) formats
400+
assert_eq!(parse_storage_size(Some(String::from("1 KiB"))), Some(1024));
401+
assert_eq!(
402+
parse_storage_size(Some(String::from("1 MiB"))),
403+
Some(1024 * 1024)
404+
);
405+
assert_eq!(
406+
parse_storage_size(Some(String::from("1 GiB"))),
407+
Some(1024 * 1024 * 1024)
408+
);
409+
assert_eq!(
410+
parse_storage_size(Some(String::from("1 TiB"))),
411+
Some(1024 * 1024 * 1024 * 1024)
412+
);
413+
assert_eq!(
414+
parse_storage_size(Some(String::from("1 PiB"))),
415+
Some(1024 * 1024 * 1024 * 1024 * 1024)
416+
);
417+
418+
// Invalid formats
419+
assert_eq!(parse_storage_size(Some(String::from("1MiB"))), None);
420+
assert_eq!(parse_storage_size(Some(String::from("invalid"))), None);
421+
assert_eq!(parse_storage_size(Some(String::from("1"))), None);
422+
assert_eq!(parse_storage_size(Some(String::from("one GB"))), None);
423+
424+
// Test for None input
425+
assert_eq!(parse_storage_size(None), None);
426+
}
319427
}

0 commit comments

Comments
 (0)