feat(odbc source): Add ODBC source implementation

Cargo.toml

@@ -326,6 +326,10 @@ deadpool = { version = "0.12.2", default-features = false, features = ["managed"
 async-graphql = { version = "7.0.17", default-features = false, optional = true, features = ["chrono", "playground"] }
 async-graphql-warp = { version = "7.0.17", default-features = false, optional = true }
 
+# ODBC
+odbc-api = { version = "19.1.0", optional = true }
+cron = { version = "0.15.0", features = ["serde"], optional = true }
+
 # Opentelemetry
 
 hex = { version = "0.4.3", default-features = false, optional = true }
@@ -608,6 +612,7 @@ sources-logs = [
   "sources-logstash",
   "sources-mqtt",
   "sources-nats",
+  "sources-odbc",
   "sources-okta",
   "sources-opentelemetry",
   "sources-pulsar",
@@ -668,6 +673,7 @@ sources-mongodb_metrics = ["dep:mongodb"]
 sources-mqtt = ["dep:rumqttc"]
 sources-nats = ["dep:async-nats", "dep:nkeys"]
 sources-nginx_metrics = ["dep:nom"]
+sources-odbc = ["dep:odbc-api", "dep:cron"]
 sources-okta = ["sources-utils-http-client"]
 sources-opentelemetry = [
   "dep:hex",
@@ -917,6 +923,7 @@ all-integration-tests = [
   "mongodb_metrics-integration-tests",
   "nats-integration-tests",
   "nginx-integration-tests",
+  "odbc-integration-tests",
   "opentelemetry-integration-tests",
   "postgresql_metrics-integration-tests",
   "postgres_sink-integration-tests",
@@ -982,6 +989,7 @@ mongodb_metrics-integration-tests = ["sources-mongodb_metrics"]
 mqtt-integration-tests = ["sinks-mqtt", "sources-mqtt"]
 nats-integration-tests = ["sinks-nats", "sources-nats"]
 nginx-integration-tests = ["sources-nginx_metrics"]
+odbc-integration-tests = ["sources-odbc"]
 opentelemetry-integration-tests = ["sources-opentelemetry", "dep:prost"]
 postgresql_metrics-integration-tests = ["sources-postgresql_metrics"]
 postgres_sink-integration-tests = ["sinks-postgres"]

scripts/integration/Dockerfile

@@ -14,6 +14,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     zlib1g-dev \
     unzip \
     git \
+    unixodbc \
+    unixodbc-dev \
+    odbcinst \
+    odbc-mariadb \
+    odbc-postgresql \
   && rm -rf /var/lib/apt/lists/*
 
 COPY tests/data/ca/certs /certs

scripts/integration/odbc-mariadb/compose.yaml

@@ -0,0 +1,17 @@
+version: '3'
+
+services:
+  mariadb:
+    image: docker.io/mariadb:${CONFIG_VERSION}
+    ports:
+      - "3306:3306"
+    environment:
+      - MYSQL_USER=vector
+      - MYSQL_PASSWORD=vector
+      - MYSQL_ROOT_PASSWORD=vector
+      - MYSQL_DATABASE=vector_db
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/odbc-mariadb/test.yaml

@@ -0,0 +1,17 @@
+features:
+  - odbc-integration-tests
+
+test_filter: '::odbc::'
+
+env:
+  ODBC_DB_TYPE: "mariadb"
+  ODBC_CONN_STRING: "driver={MariaDB Unicode};server=mariadb;port=3306;database=vector_db;uid=vector;pwd=vector;"
+
+matrix:
+  version: [ '11-jammy' ]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+  - "src/sources/odbc/**"
+  - "scripts/integration/odbc/**"

scripts/integration/odbc-postgresql/compose.yaml

@@ -0,0 +1,16 @@
+version: '3'
+
+services:
+  postgresql:
+    image: docker.io/postgres:${CONFIG_VERSION}
+    ports:
+      - "5432:5432"
+    environment:
+      - POSTGRES_USER=vector
+      - POSTGRES_PASSWORD=vector
+      - POSTGRES_DB=vector_db
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/integration/odbc-postgresql/test.yaml

@@ -0,0 +1,17 @@
+features:
+  - odbc-integration-tests
+
+test_filter: '::odbc::'
+
+env:
+  ODBC_DB_TYPE: "postgresql"
+  ODBC_CONN_STRING: "driver={PostgreSQL Unicode};server=postgresql;port=5432;database=vector_db;uid=vector;pwd=vector;"
+
+matrix:
+  version: [ '16' ]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+  - "src/sources/odbc/**"
+  - "scripts/integration/odbc/**"

src/internal_events/mod.rs

@@ -152,6 +152,9 @@ mod file;
 #[cfg(windows)]
 mod windows;
 
+#[cfg(feature = "sources-odbc")]
+mod odbc_metrics;
+
 pub mod config;
 #[cfg(any(feature = "transforms-log_to_metric", feature = "sinks-loki"))]
 mod expansion;
@@ -248,6 +251,9 @@ pub(crate) use self::metric_to_log::*;
 pub(crate) use self::mqtt::*;
 #[cfg(feature = "sources-nginx_metrics")]
 pub(crate) use self::nginx_metrics::*;
+#[cfg(feature = "sources-odbc")]
+pub(crate) use self::odbc_metrics::*;
+#[allow(unused_imports)]
 #[cfg(any(
     feature = "sources-kubernetes_logs",
     feature = "transforms-log_to_metric",

src/internal_events/odbc_metrics.rs

@@ -0,0 +1,64 @@
+use metrics::counter;
+use vector_common::internal_event::{InternalEvent, error_type};
+
+#[derive(Debug)]
+pub struct OdbcEventsReceived {
+    pub count: usize,
+}
+
+impl InternalEvent for OdbcEventsReceived {
+    fn emit(self) {
+        trace!(
+            message = "Events received.",
+            count = %self.count,
+        );
+        counter!(
+            "component_received_events_total",
+            "protocol" => "odbc"
+        )
+        .increment(self.count as u64);
+        counter!(
+            "component_received_event_bytes_total",
+            "protocol" => "odbc"
+        )
+        .increment(0);
+    }
+}
+
+#[derive(Debug)]
+pub struct OdbcFailedError<'a> {
+    pub statement: &'a str,
+}
+
+impl InternalEvent for OdbcFailedError<'_> {
+    fn emit(self) {
+        error!(
+            message = "Unable to execute statement.",
+            statement = %self.statement,
+            error = error_type::COMMAND_FAILED
+        );
+        counter!(
+            "component_errors_total",
+            "statement" => self.statement.to_owned(),
+            "error_type" => error_type::COMMAND_FAILED
+        )
+        .increment(1);
+    }
+}
+
+#[derive(Debug)]
+pub struct OdbcQueryExecuted<'a> {
+    pub statement: &'a str,
+    pub elapsed: u128,
+}
+
+impl InternalEvent for OdbcQueryExecuted<'_> {
+    fn emit(self) {
+        trace!(
+            message = "Executed statement.",
+            statement = %self.statement,
+            elapsedMs = %self.elapsed
+        );
+        counter!("component_executed_events_total").increment(1);
+    }
+}

src/sources/mod.rs

@@ -64,6 +64,8 @@ pub mod mqtt;
 pub mod nats;
 #[cfg(feature = "sources-nginx_metrics")]
 pub mod nginx_metrics;
+#[cfg(feature = "sources-odbc")]
+pub mod odbc;
 #[cfg(feature = "sources-okta")]
 pub mod okta;
 #[cfg(feature = "sources-opentelemetry")]

src/sources/odbc/README.md

@@ -0,0 +1,46 @@
+# ODBC Development
+
+## Setup
+
+#### MacOS
+
+```shell
+brew install unixodbc
+
+brew install [email protected]  # Install MariaDB ODBC, and you need to configure odbcinst.ini.
+```
+
+Refs
+
+- MySQL Connector/ODBC: <https://dev.mysql.com/doc/connector-odbc/en/connector-odbc-installation-binary-macos.html>
+- MariaDB Connector/ODBC: <https://mariadb.com/kb/en/about-mariadb-connector-odbc/>
+  - Homebrew mariadb-connector-odbc: <https://formulae.brew.sh/formula/mariadb-connector-odbc>
+  - ODBC Configuration: <https://mariadb.com/kb/en/creating-a-data-source-with-mariadb-connectorodbc/>
+    ```shell
+    cat << EOF >> /opt/homebrew/etc/odbcinst.ini
+
+    [MariaDB ODBC 3.0 Driver]
+    Description = MariaDB Connector/ODBC v.3.0
+    Driver = /opt/homebrew/Cellar/mariadb-connector-odbc/3.2.6/lib/mariadb/libmaodbc.dylib
+    EOF
+    ```
+- MSSQL
+  Connector/ODBC: <https://learn.microsoft.com/ko-kr/sql/connect/odbc/linux-mac/install-microsoft-odbc-driver-sql-server-macos?view=sql-server-ver16>
+
+## ODBC Tips
+
+Show ODBC configuration
+
+```shell
+odbcinst -j
+
+### Output Example ###
+# unixODBC 2.3.12
+# DRIVERS............: /opt/homebrew/etc/odbcinst.ini
+# SYSTEM DATA SOURCES: /opt/homebrew/etc/odbc.ini
+# FILE DATA SOURCES..: /opt/homebrew/etc/ODBCDataSources
+# USER DATA SOURCES..: /Users/powerumc/.odbc.ini
+# SQLULEN Size.......: 8
+# SQLLEN Size........: 8
+# SQLSETPOSIROW Size.: 8
+```