feat: First stab at urwid UI & async

- Adds async backends using asyncio for ClickHouse direct, Ralph, CSV,
and CHDB/S3 using a new task management system
- Adds an urwid based UI for displaying the load state as the various
loads happen
- Refactors several configuration variables

Author: bmtcril

.coverage.TCRIL-MBP-C29V19Y9V1-SPARE6.local.99610.XDPXxOgx

@@ -87,9 +87,9 @@ test::
     log_dir: logs
 
     # xAPI statements will be generated in batches, the total number of
-    # statements is ``num_batches * batch_size``. The batch size is the number
+    # statements is ``num_xapi_batches * batch_size``. The batch size is the number
     # of statements sent to the backend (Ralph POST, ClickHouse insert, etc.)
-    num_batches: 3
+    num_xapi_batches: 3
     batch_size: 100
 
     # Overall start and end date for the entire run. All xAPI statements

README.rst

@@ -1,6 +1,6 @@
 # CSV backend configuration
 # #########################
-backend: csv_file
+backend: csv
 # This can be anything smart_open can handle, a local directory or
 # an S3 bucket, etc., but importing to ClickHouse only supports S3 right now
 # https://pypi.org/project/smart-open/
@@ -37,8 +37,13 @@ csv_load_from_s3_after: false
 
 # Run options
 log_dir: logs
-num_batches: 3
-batch_size: 100
+num_xapi_batches: 300
+batch_size: 1000
+
+# This number is used for each QueueBackend that use workers, so the number of threads if
+# multiplicative. Generally this performs best less than 10, as more threads will cost more
+# in context switching than they save.
+num_workers: 4
 
 # Overall start and end date for the entire run
 start_date: 2014-01-01

default_config.yaml

@@ -12,7 +12,7 @@ s3_secret: ...
 
 # Run options
 log_dir: logs
-num_batches: 3
+num_xapi_batches: 3
 batch_size: 100
 
 # Overall start and end date for the entire run

example_configs/clickhouse_example.yaml

@@ -24,7 +24,7 @@ s3_secret: ...
 
 # Run options
 log_dir: logs
-num_batches: 100000
+num_xapi_batches: 100000
 batch_size: 10000
 
 # Overall start and end date for the entire run

example_configs/large_csv_load.yaml

@@ -5,7 +5,7 @@ csv_output_destination: logs
 
 # Run options
 log_dir: logs
-num_batches: 3
+num_xapi_batches: 3
 batch_size: 100
 
 # Overall start and end date for the entire run

example_configs/local_csv.yaml

@@ -12,7 +12,7 @@ lrs_password: test
 
 # Run options
 log_dir: logs
-num_batches: 3
+num_xapi_batches: 3
 batch_size: 100
 
 # Overall start and end date for the entire run

example_configs/ralph_clickhouse_example.yaml

@@ -9,7 +9,7 @@ csv_load_from_s3_after: false
 
 # Run options
 log_dir: logs
-num_batches: 3
+num_xapi_batches: 3
 batch_size: 100
 
 # Overall start and end date for the entire run

example_configs/s3_csv.yaml

@@ -24,7 +24,7 @@ s3_secret: ...
 
 # Run options
 log_dir: logs
-num_batches: 3
+num_xapi_batches: 3
 batch_size: 100
 
 # Overall start and end date for the entire run

example_configs/s3_csv_clickhouse.yaml

@@ -1,8 +1,11 @@
 # Core requirements for using this application
 -c constraints.txt
 
-click
-clickhouse-connect>=0.5,<0.7
-pyyaml
-requests
-smart_open[s3]
+chdb                   # Embedded ClickHouse client
+click                  # Command line interface library
+clickhouse-connect     # ClickHouse client
+pyyaml                 # YAML parser
+requests               # HTTP library
+smart_open[s3]         # Consistent file handing between local and object storage
+urwid                  # Console UI library
+uvloop                 # Improved performance with asyncio