Migrate refactor of catalog to rust

Author: faysou

.docker/DockerfileUbuntu

@@ -0,0 +1,72 @@
+# Dockerfile to reproduce locally an environment similar to what is run on the github runner
+#
+# From nautilus project's root folder:
+#
+# Build the image:
+# docker build -f .docker/DockerfileUbuntu -t nautilus-dev .
+#
+# Run interactively with local directory mounted:
+# docker run --rm -itv "$(pwd)":/workspace nautilus-dev bash
+#
+# Or run the default entrypoint:
+# docker run --rm -itv "$(pwd)":/workspace nautilus-dev
+#
+# Remove the image
+# docker image rm nautilus-dev
+
+FROM ubuntu:22.04
+
+# Set environment variables
+ENV DEBIAN_FRONTEND=noninteractive
+ENV BUILD_MODE=release
+ENV RUST_BACKTRACE=1
+ENV CARGO_INCREMENTAL=1
+ENV CC="clang"
+ENV CXX="clang++"
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    curl \
+    clang \
+    git \
+    pkg-config \
+    make \
+    capnproto \
+    libcapnp-dev \
+    gcc-aarch64-linux-gnu \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install mold linker
+RUN curl -L https://github.com/rui314/mold/releases/download/v2.35.1/mold-2.35.1-x86_64-linux.tar.gz | tar -xz -C /usr/local --strip-components=1
+
+# Install uv
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+ENV PATH="/root/.cargo/bin:/root/.local/bin:${PATH}"
+
+# Install Python 3.13
+RUN uv python install
+
+# Set working directory
+WORKDIR /workspace
+
+# Copy only necessary files for dependency setup
+# The actual source code will be mounted as a volume
+COPY ../scripts/rust-toolchain.sh scripts/
+COPY ../Cargo.toml Cargo.lock pyproject.toml rust-toolchain.toml ./
+
+# Set up Rust toolchain based on project requirements
+RUN bash scripts/rust-toolchain.sh > /tmp/toolchain.txt && \
+    TOOLCHAIN=$(cat /tmp/toolchain.txt) && \
+    rustup toolchain install $TOOLCHAIN && \
+    rustup default $TOOLCHAIN && \
+    rustup component add clippy rustfmt
+
+# Copy and set up entrypoint script for interactive development
+COPY .docker/entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

.docker/entrypoint.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# entrypoint script for DockerfileUbuntu
+
+echo "=== Nautilus Trader Development Environment ==="
+echo "Rust version: $(rustc --version)"
+echo "UV version: $(uv --version)"
+echo "Working directory: $(pwd)"
+echo
+
+echo "=== Setting PyO3 environment ==="
+export PYO3_PYTHON=/workspace/.venv/bin/python3
+echo "PYO3_PYTHON: $PYO3_PYTHON"
+echo
+
+echo "=== Development environment ready! ==="
+echo "You can now run for example:"
+echo "  make install-debug                                            # Install nautilus in debug mode"
+echo "  make cargo-test                                               # Test Rust code"
+echo "  make pytest                                                   # Run Python tests"
+echo "  uv run python -c \"import nautilus_trader.backtest.engine;\"    # Run a Python instruction"
+echo
+
+# If no command is provided, check if we have a TTY and start appropriate shell
+if [ $# -eq 0 ]; then
+    if [ -t 0 ]; then
+        echo "Starting interactive shell..."
+        exec bash
+    else
+        echo "No TTY detected. Use docker run -it for interactive mode."
+        echo "Container ready for commands. Example:"
+        echo "  docker run --rm -itv \"\$(pwd)\":/workspace nautilus-dev"
+    fi
+else
+    exec "$@"
+fi

Cargo.lock

@@ -220,7 +220,6 @@ url = { version = "2.5.4", default-features = false }
 urlencoding = "2.1.3"
 ustr = { version = "1.1.0", features = ["serde"] }
 uuid = { version = "1.17.0", features = ["v4", "serde"] }
-walkdir = "2.5.0"
 webpki-roots = "1.0.0"
 
 # dev-dependencies

Cargo.toml

@@ -40,6 +40,7 @@ rustdoc-args = ["--cfg", "docsrs"]
 [dependencies]
 nautilus-core = { workspace = true }
 nautilus-model = { workspace = true, features = ["python"] }
+nautilus-persistence = { workspace = true }
 nautilus-serialization = { workspace = true }
 
 anyhow = { workspace = true }

crates/adapters/tardis/Cargo.toml

@@ -32,13 +32,11 @@ use nautilus_model::{
     },
     identifiers::InstrumentId,
 };
-use nautilus_serialization::{
-    arrow::{
-        bars_to_arrow_record_batch_bytes, book_deltas_to_arrow_record_batch_bytes,
-        book_depth10_to_arrow_record_batch_bytes, quotes_to_arrow_record_batch_bytes,
-        trades_to_arrow_record_batch_bytes,
-    },
-    parquet::write_batch_to_parquet,
+use nautilus_persistence::parquet::write_batch_to_parquet;
+use nautilus_serialization::arrow::{
+    bars_to_arrow_record_batch_bytes, book_deltas_to_arrow_record_batch_bytes,
+    book_depth10_to_arrow_record_batch_bytes, quotes_to_arrow_record_batch_bytes,
+    trades_to_arrow_record_batch_bytes,
 };
 use thousands::Separable;
 use ustr::Ustr;
@@ -431,7 +429,16 @@ fn batch_and_write_bars(bars: Vec<Bar>, bar_type: &BarType, date: NaiveDate, pat
     };
 
     let filepath = path.join(parquet_filepath_bars(bar_type, date));
-    match write_batch_to_parquet(batch, &filepath, None, None, None) {
+    let filepath_str = filepath.to_string_lossy();
+
+    let rt = tokio::runtime::Runtime::new().unwrap();
+    match rt.block_on(write_batch_to_parquet(
+        batch,
+        &filepath_str,
+        None,
+        None,
+        None,
+    )) {
         Ok(()) => tracing::info!("File written: {filepath:?}"),
         Err(e) => tracing::error!("Error writing {filepath:?}: {e:?}"),
     }
@@ -464,7 +471,16 @@ fn write_batch(
     path: &Path,
 ) {
     let filepath = path.join(parquet_filepath(typename, instrument_id, date));
-    match write_batch_to_parquet(batch, &filepath, None, None, None) {
+    let filepath_str = filepath.to_string_lossy();
+
+    let rt = tokio::runtime::Runtime::new().unwrap();
+    match rt.block_on(write_batch_to_parquet(
+        batch,
+        &filepath_str,
+        None,
+        None,
+        None,
+    )) {
         Ok(()) => tracing::info!("File written: {filepath:?}"),
         Err(e) => tracing::error!("Error writing {filepath:?}: {e:?}"),
     }

crates/adapters/tardis/src/replay.rs

@@ -117,6 +117,55 @@ pub fn unix_nanos_to_iso8601(unix_nanos: UnixNanos) -> String {
     datetime.to_rfc3339_opts(SecondsFormat::Nanos, true)
 }
 
+/// Converts an ISO 8601 (RFC 3339) format string to UNIX nanoseconds timestamp.
+///
+/// This function accepts various ISO 8601 formats including:
+/// - Full RFC 3339 with nanosecond precision: "2024-02-10T14:58:43.456789Z"
+/// - RFC 3339 without fractional seconds: "2024-02-10T14:58:43Z"
+/// - Simple date format: "2024-02-10" (interpreted as midnight UTC)
+///
+/// # Parameters
+///
+/// - `date_string`: The ISO 8601 formatted date string to parse
+///
+/// # Returns
+///
+/// Returns `Ok(UnixNanos)` if the string is successfully parsed, or an error if the format
+/// is invalid or the timestamp is out of range.
+///
+/// # Errors
+///
+/// Returns an error if:
+/// - The string format is not a valid ISO 8601 format
+/// - The timestamp is out of range for `UnixNanos`
+/// - The date/time values are invalid
+///
+/// # Examples
+///
+/// ```rust
+/// use nautilus_core::datetime::iso8601_to_unix_nanos;
+/// use nautilus_core::UnixNanos;
+///
+/// // Full RFC 3339 format
+/// let nanos = iso8601_to_unix_nanos("2024-02-10T14:58:43.456789Z".to_string())?;
+/// assert_eq!(nanos, UnixNanos::from(1_707_577_123_456_789_000));
+///
+/// // Without fractional seconds
+/// let nanos = iso8601_to_unix_nanos("2024-02-10T14:58:43Z".to_string())?;
+/// assert_eq!(nanos, UnixNanos::from(1_707_577_123_000_000_000));
+///
+/// // Simple date format (midnight UTC)
+/// let nanos = iso8601_to_unix_nanos("2024-02-10".to_string())?;
+/// assert_eq!(nanos, UnixNanos::from(1_707_523_200_000_000_000));
+/// # Ok::<(), anyhow::Error>(())
+/// ```
+#[inline]
+pub fn iso8601_to_unix_nanos(date_string: String) -> anyhow::Result<UnixNanos> {
+    date_string
+        .parse::<UnixNanos>()
+        .map_err(|e| anyhow::anyhow!("Failed to parse ISO 8601 string '{}': {}", date_string, e))
+}
+
 /// Converts a UNIX nanoseconds timestamp to an ISO 8601 (RFC 3339) format string
 /// with millisecond precision.
 #[inline]
@@ -492,4 +541,36 @@ mod tests {
         let result = is_leap_year(year);
         assert_eq!(result, expected);
     }
+
+    #[rstest]
+    #[case("1970-01-01T00:00:00.000000000Z", 0)] // Unix epoch
+    #[case("1970-01-01T00:00:00.000000001Z", 1)] // 1 nanosecond
+    #[case("1970-01-01T00:00:00.001000000Z", 1_000_000)] // 1 millisecond
+    #[case("1970-01-01T00:00:01.000000000Z", 1_000_000_000)] // 1 second
+    #[case("2023-12-18T00:00:00.000000000Z", 1_702_857_600_000_000_000)] // Specific date
+    #[case("2024-02-10T14:58:43.456789Z", 1_707_577_123_456_789_000)] // RFC3339 with fractions
+    #[case("2024-02-10T14:58:43Z", 1_707_577_123_000_000_000)] // RFC3339 without fractions
+    #[case("2024-02-10", 1_707_523_200_000_000_000)] // Simple date format
+    fn test_iso8601_to_unix_nanos(#[case] input: &str, #[case] expected: u64) {
+        let result = iso8601_to_unix_nanos(input.to_string()).unwrap();
+        assert_eq!(result.as_u64(), expected);
+    }
+
+    #[rstest]
+    #[case("invalid-date")] // Invalid format
+    #[case("2024-02-30")] // Invalid date
+    #[case("2024-13-01")] // Invalid month
+    #[case("not a timestamp")] // Random string
+    fn test_iso8601_to_unix_nanos_invalid(#[case] input: &str) {
+        let result = iso8601_to_unix_nanos(input.to_string());
+        assert!(result.is_err());
+    }
+
+    #[rstest]
+    fn test_iso8601_roundtrip() {
+        let original_nanos = UnixNanos::from(1_707_577_123_456_789_000);
+        let iso8601_string = unix_nanos_to_iso8601(original_nanos);
+        let parsed_nanos = iso8601_to_unix_nanos(iso8601_string).unwrap();
+        assert_eq!(parsed_nanos, original_nanos);
+    }
 }

crates/core/src/datetime.rs

@@ -52,8 +52,10 @@ nautilus-model = { workspace = true, features = ["stubs"] }
 nautilus-serialization = { workspace = true, features = ["python"] }
 
 anyhow = { workspace = true }
+arrow = { workspace = true }
 binary-heap-plus = { workspace = true }
 compare = { workspace = true }
+chrono = { workspace = true }
 datafusion = { workspace = true }
 futures = { workspace = true }
 heck = { workspace = true }
@@ -65,7 +67,8 @@ pyo3 = { workspace = true, optional = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 tokio = { workspace = true }
-walkdir = { workspace = true }
+unbounded-interval-tree = { workspace = true }
+url = { workspace = true }
 
 [dev-dependencies]
 nautilus-testkit = { workspace = true }