Merge pull request #247 from datafold/issue245

Added validation for UUID columns (Also fixes issue #245)

Author: erezsh

data_diff/diff_tables.py

@@ -14,7 +14,7 @@
 
 from .utils import safezip, run_as_daemon
 from .thread_utils import ThreadedYielder
-from .databases.database_types import IKey, NumericType, PrecisionType, StringType
+from .databases.database_types import IKey, NumericType, PrecisionType, StringType, ColType_UUID
 from .table_segment import TableSegment
 from .tracking import create_end_event_json, create_start_event_json, send_event_json, is_tracking_enabled
 
@@ -209,6 +209,10 @@ def _validate_and_adjust_columns(self, table1, table2):
                 table1._schema[c1] = col1.replace(precision=lowest.precision)
                 table2._schema[c2] = col2.replace(precision=lowest.precision)
 
+            elif isinstance(col1, ColType_UUID):
+                if not isinstance(col2, ColType_UUID):
+                    raise TypeError(f"Incompatible types for column '{c1}':  {col1} <-> {col2}")
+
             elif isinstance(col1, StringType):
                 if not isinstance(col2, StringType):
                     raise TypeError(f"Incompatible types for column '{c1}':  {col1} <-> {col2}")
@@ -222,7 +226,9 @@ def _validate_and_adjust_columns(self, table1, table2):
                         "If encoding/formatting differs between databases, it may result in false positives."
                     )
 
-    def _bisect_and_diff_tables(self, ti: ThreadedYielder, table1: TableSegment, table2: TableSegment, level=0, max_rows=None):
+    def _bisect_and_diff_tables(
+        self, ti: ThreadedYielder, table1: TableSegment, table2: TableSegment, level=0, max_rows=None
+    ):
         assert table1.is_bounded and table2.is_bounded
 
         if max_rows is None:
@@ -259,7 +265,16 @@ def _bisect_and_diff_tables(self, ti: ThreadedYielder, table1: TableSegment, tab
         for i, (t1, t2) in enumerate(safezip(segmented1, segmented2)):
             ti.submit(self._diff_tables, ti, t1, t2, max_rows, level + 1, i + 1, len(segmented1), priority=level)
 
-    def _diff_tables(self, ti: ThreadedYielder, table1: TableSegment, table2: TableSegment, max_rows: int, level=0, segment_index=None, segment_count=None):
+    def _diff_tables(
+        self,
+        ti: ThreadedYielder,
+        table1: TableSegment,
+        table2: TableSegment,
+        max_rows: int,
+        level=0,
+        segment_index=None,
+        segment_count=None,
+    ):
         logger.info(
             ". " * level + f"Diffing segment {segment_index}/{segment_count}, "
             f"key-range: {table1.min_key}..{table2.max_key}, "

tests/test_postgresql.py

@@ -2,21 +2,21 @@
 
 from data_diff.databases.connect import connect
 from data_diff import TableSegment, TableDiffer
-from .common import TEST_POSTGRESQL_CONN_STRING, random_table_suffix
+from .common import TEST_POSTGRESQL_CONN_STRING, TEST_MYSQL_CONN_STRING, random_table_suffix
 
 
-class TestWithConnection(unittest.TestCase):
+class TestUUID(unittest.TestCase):
     def setUp(self) -> None:
         self.connection = connect(TEST_POSTGRESQL_CONN_STRING)
 
-        self.connection.query('CREATE EXTENSION IF NOT EXISTS "uuid-ossp";', None)
-
         table_suffix = random_table_suffix()
 
         self.table_src = f"src{table_suffix}"
         self.table_dst = f"dst{table_suffix}"
 
     def test_uuid(self):
+        self.connection.query('CREATE EXTENSION IF NOT EXISTS "uuid-ossp";', None)
+
         queries = [
             f"DROP TABLE IF EXISTS {self.table_src}",
             f"DROP TABLE IF EXISTS {self.table_dst}",
@@ -33,6 +33,7 @@ def test_uuid(self):
         ]
 
         queries.append(f"INSERT INTO {self.table_src}(comment) VALUES ('This one is different')")
+        queries.append("COMMIT")
 
         for query in queries:
             self.connection.query(query, None)
@@ -44,3 +45,24 @@ def test_uuid(self):
         diff = list(differ.diff_tables(a, b))
         uuid = diff[0][1][0]
         self.assertEqual(diff, [("-", (uuid, "This one is different"))])
+
+        # Compare with MySql
+        mysql_conn = connect(TEST_MYSQL_CONN_STRING)
+
+        rows = self.connection.query(f"SELECT * FROM {self.table_src}", list)
+
+        mysql_conn.query(f"CREATE TABLE {self.table_dst} (id VARCHAR(128), comment VARCHAR(128))", None)
+        mysql_conn.query(f"COMMIT", None)
+        for uuid, comment in rows:
+            mysql_conn.query(f"INSERT INTO {self.table_dst}(id, comment) VALUES ('{uuid}', '{comment}')", None)
+        mysql_conn.query(f"COMMIT", None)
+
+        c = TableSegment(mysql_conn, (self.table_dst,), "id", "comment")
+        diff = list(differ.diff_tables(a, c))
+        assert not diff, diff
+        diff = list(differ.diff_tables(c, a))
+        assert not diff, diff
+
+        self.connection.query(f"DROP TABLE {self.table_src}", None)
+        self.connection.query(f"DROP TABLE {self.table_dst}", None)
+        mysql_conn.query(f"DROP TABLE {self.table_dst}", None)