Use lightweight deletes

pom.xml

@@ -67,22 +67,23 @@
         <project.build.sourceEncoding>${encoding}</project.build.sourceEncoding>
         <maven.compiler.source>1.8</maven.compiler.source>
         <maven.compiler.target>1.8</maven.compiler.target>
-        <clickhouse-jdbc.version>0.3.2</clickhouse-jdbc.version>
-        <liquibase.version>4.15.0</liquibase.version>
-        <junit.version>5.4.0</junit.version>
-        <test-containers.version>1.17.3</test-containers.version>
-        <nexus-staging-maven-plugin.version>1.6.8</nexus-staging-maven-plugin.version>
+        <clickhouse-jdbc.version>0.4.5</clickhouse-jdbc.version>
+        <liquibase.version>4.21.1</liquibase.version>
+        <junit.version>5.9.2</junit.version>
+        <test-containers.version>1.18.0</test-containers.version>
+        <nexus-staging-maven-plugin.version>1.6.13</nexus-staging-maven-plugin.version>
         <maven-plugin-plugin.version>3.6.0</maven-plugin-plugin.version>
-        <maven-surefire-plugin.version>2.22.1</maven-surefire-plugin.version>
+        <maven-surefire-plugin.version>3.0.0</maven-surefire-plugin.version>
         <license-maven-plugin.version>2.0.0</license-maven-plugin.version>
-        <maven-shade-plugin.version>3.2.0</maven-shade-plugin.version>
-        <fmt-maven-plugin.version>2.9.1</fmt-maven-plugin.version>
-        <impsort-maven-plugin.version>1.4.1</impsort-maven-plugin.version>
+        <maven-shade-plugin.version>3.4.1</maven-shade-plugin.version>
+        <fmt-maven-plugin.version>2.19</fmt-maven-plugin.version>
+        <impsort-maven-plugin.version>1.8.0</impsort-maven-plugin.version>
         <maven-gpg-plugin.version>1.6</maven-gpg-plugin.version>
-        <maven-javadoc-plugin.version>3.2.0</maven-javadoc-plugin.version>
+        <maven-javadoc-plugin.version>3.5.0</maven-javadoc-plugin.version>
         <maven-source-plugin.version>3.2.1</maven-source-plugin.version>
-        <snakeyaml.version>1.28</snakeyaml.version>
-        <typesafe-config.version>1.4.1</typesafe-config.version>
+        <snakeyaml.version>2.0</snakeyaml.version>
+        <typesafe-config.version>1.4.2</typesafe-config.version>
+        <lz4.version>1.8.0</lz4.version>
     </properties>
 
     <dependencies>
@@ -125,6 +126,11 @@
             <artifactId>config</artifactId>
             <version>${typesafe-config.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.lz4</groupId>
+            <artifactId>lz4-java</artifactId>
+            <version>${lz4.version}</version>
+        </dependency>
     </dependencies>
 
     <build>
@@ -249,7 +255,7 @@
                 </executions>
             </plugin>
             <plugin>
-                <groupId>com.coveo</groupId>
+                <groupId>com.spotify.fmt</groupId>
                 <artifactId>fmt-maven-plugin</artifactId>
                 <version>${fmt-maven-plugin.version}</version>
                 <configuration>

src/main/java/liquibase/ext/clickhouse/database/ClickHouseDatabase.java

@@ -26,6 +26,8 @@
 import liquibase.database.AbstractJdbcDatabase;
 import liquibase.database.DatabaseConnection;
 import liquibase.exception.DatabaseException;
+import liquibase.statement.SqlStatement;
+import liquibase.statement.core.RawSqlStatement;
 
 public class ClickHouseDatabase extends AbstractJdbcDatabase {
 
@@ -89,11 +91,16 @@ public boolean supportsSequences() {
 
   @Override
   public boolean supportsSchemas() {
-    return false;
+    return true;
   }
 
   @Override
   public boolean supportsDDLInTransaction() {
     return false;
   }
+
+  @Override
+  protected SqlStatement getConnectionSchemaNameCallStatement() {
+    return new RawSqlStatement("SELECT currentDatabase()");
+  }
 }

src/main/java/liquibase/ext/clickhouse/lockservice/ClickHouseLockService.java

@@ -22,14 +22,17 @@
 import liquibase.ext.clickhouse.database.ClickHouseDatabase;
 
 import liquibase.Scope;
+import liquibase.changelog.ChangeLogHistoryServiceFactory;
 import liquibase.database.Database;
 import liquibase.exception.DatabaseException;
 import liquibase.exception.LiquibaseException;
+import liquibase.exception.LockException;
 import liquibase.exception.UnexpectedLiquibaseException;
 import liquibase.executor.Executor;
 import liquibase.executor.ExecutorService;
 import liquibase.lockservice.StandardLockService;
 import liquibase.logging.Logger;
+import liquibase.statement.core.LockDatabaseChangeLogStatement;
 import liquibase.statement.core.RawSqlStatement;
 
 public class ClickHouseLockService extends StandardLockService {
@@ -68,21 +71,81 @@ public boolean isDatabaseChangeLogLockTableInitialized(boolean tableJustCreated)
   }
 
   @Override
-  public boolean hasDatabaseChangeLogLockTable() {
-    boolean hasTable = false;
+  public boolean hasDatabaseChangeLogLockTable(boolean forceRecheck) {
+    if (forceRecheck || hasDatabaseChangeLogLockTable == null) {
+      try {
+        String query =
+            String.format(
+                "SELECT ID FROM %s.%s LIMIT 1",
+                database.getDefaultSchemaName(), database.getDatabaseChangeLogLockTableName());
+        getExecutor().execute(new RawSqlStatement(query));
+        hasDatabaseChangeLogLockTable = true;
+      } catch (DatabaseException e) {
+        getLogger()
+            .info(
+                String.format(
+                    "No %s table available", database.getDatabaseChangeLogLockTableName()));
+        hasDatabaseChangeLogLockTable = false;
+      }
+    }
+    return hasDatabaseChangeLogLockTable;
+  }
+
+  @Override
+  public boolean acquireLock() throws LockException {
+    if (hasChangeLogLock) {
+      return true;
+    }
+
+    quotingStrategy = database.getObjectQuotingStrategy();
+
+    Executor executor =
+        Scope.getCurrentScope().getSingleton(ExecutorService.class).getExecutor("jdbc", database);
+
     try {
+      database.rollback();
+      this.init();
+
       String query =
           String.format(
-              "SELECT ID FROM %s.%s LIMIT 1",
+              "SELECT MAX(LOCKED) FROM %s.%s",
               database.getDefaultSchemaName(), database.getDatabaseChangeLogLockTableName());
-      getExecutor().execute(new RawSqlStatement(query));
-      hasTable = true;
-    } catch (DatabaseException e) {
-      getLogger()
-          .info(
-              String.format("No %s table available", database.getDatabaseChangeLogLockTableName()));
+      boolean locked = executor.queryForInt(new RawSqlStatement(query)) > 0;
+
+      if (locked) {
+        return false;
+      } else {
+
+        executor.comment("Lock Database");
+        int rowsUpdated = executor.update(new LockDatabaseChangeLogStatement());
+
+        if (rowsUpdated > 1) {
+          throw new LockException("Did not update change log lock correctly");
+        }
+        if (rowsUpdated == 0) {
+          // another node was faster
+          return false;
+        }
+        database.commit();
+        Scope.getCurrentScope()
+            .getLog(getClass())
+            .info(coreBundle.getString("successfully.acquired.change.log.lock"));
+
+        hasChangeLogLock = true;
+
+        ChangeLogHistoryServiceFactory.getInstance().resetAll();
+        database.setCanCacheLiquibaseTableInfo(true);
+        return true;
+      }
+    } catch (Exception e) {
+      throw new LockException(e);
+    } finally {
+      try {
+        database.rollback();
+      } catch (DatabaseException ignored) {
+
+      }
     }
-    return hasTable;
   }
 
   private Executor getExecutor() {

src/main/java/liquibase/ext/clickhouse/params/ParamsLoader.java

@@ -2,7 +2,7 @@
  * #%L
  * Liquibase extension for ClickHouse
  * %%
- * Copyright (C) 2020 - 2022 Mediarithmics
+ * Copyright (C) 2020 - 2023 Mediarithmics
  * %%
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/java/liquibase/ext/clickhouse/sqlgenerator/CreateDatabaseChangeLogLockTableClickHouse.java

@@ -2,7 +2,7 @@
  * #%L
  * Liquibase extension for Clickhouse
  * %%
- * Copyright (C) 2020 - 2022 Mediarithmics
+ * Copyright (C) 2020 - 2023 Mediarithmics
  * %%
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/java/liquibase/ext/clickhouse/sqlgenerator/InitializeDatabaseChangeLogLockClickHouse.java

@@ -20,8 +20,6 @@
 package liquibase.ext.clickhouse.sqlgenerator;
 
 import liquibase.ext.clickhouse.database.ClickHouseDatabase;
-import liquibase.ext.clickhouse.params.ClusterConfig;
-import liquibase.ext.clickhouse.params.ParamsLoader;
 
 import liquibase.database.Database;
 import liquibase.sql.Sql;
@@ -48,21 +46,12 @@ public Sql[] generateSql(
       InitializeDatabaseChangeLogLockTableStatement statement,
       Database database,
       SqlGeneratorChain sqlGeneratorChain) {
-    ClusterConfig properties = ParamsLoader.getLiquibaseClickhouseProperties();
 
     String clearDatabaseQuery =
         String.format(
-            "ALTER TABLE %s.%s "
-                + SqlGeneratorUtil.generateSqlOnClusterClause(properties)
-                + "DELETE WHERE 1 SETTINGS mutations_sync = 1",
-            database.getDefaultSchemaName(),
-            database.getDatabaseChangeLogLockTableName());
-
-    String initLockQuery =
-        String.format(
-            "INSERT INTO %s.%s (ID, LOCKED) VALUES (1, 0)",
+            "SET allow_experimental_lightweight_delete = 1; DELETE FROM %s.%s WHERE 1",
             database.getDefaultSchemaName(), database.getDatabaseChangeLogLockTableName());
 
-    return SqlGeneratorUtil.generateSql(database, clearDatabaseQuery, initLockQuery);
+    return SqlGeneratorUtil.generateSql(database, clearDatabaseQuery);
   }
 }

src/main/java/liquibase/ext/clickhouse/sqlgenerator/LockDatabaseChangeLogClickHouse.java

@@ -20,8 +20,6 @@
 package liquibase.ext.clickhouse.sqlgenerator;
 
 import liquibase.ext.clickhouse.database.ClickHouseDatabase;
-import liquibase.ext.clickhouse.params.ClusterConfig;
-import liquibase.ext.clickhouse.params.ParamsLoader;
 
 import liquibase.database.Database;
 import liquibase.sql.Sql;
@@ -46,14 +44,11 @@ public Sql[] generateSql(
       LockDatabaseChangeLogStatement statement,
       Database database,
       SqlGeneratorChain sqlGeneratorChain) {
-    ClusterConfig properties = ParamsLoader.getLiquibaseClickhouseProperties();
 
     String host = String.format("%s %s (%s)", hostname, hostDescription, hostaddress);
     String lockQuery =
         String.format(
-            "ALTER TABLE %s.%s "
-                + SqlGeneratorUtil.generateSqlOnClusterClause(properties)
-                + "UPDATE LOCKED = 1,LOCKEDBY = '%s',LOCKGRANTED = %s WHERE ID = 1 AND LOCKED = 0 SETTINGS mutations_sync = 1",
+            "INSERT INTO %s.%s (ID, LOCKED, LOCKEDBY, LOCKGRANTED) VALUES (1, 1, '%s', %s)",
             database.getDefaultSchemaName(),
             database.getDatabaseChangeLogLockTableName(),
             host,

src/main/java/liquibase/ext/clickhouse/sqlgenerator/RemoveChangeSetRanStatusClickHouse.java

@@ -20,8 +20,6 @@
 package liquibase.ext.clickhouse.sqlgenerator;
 
 import liquibase.ext.clickhouse.database.ClickHouseDatabase;
-import liquibase.ext.clickhouse.params.ClusterConfig;
-import liquibase.ext.clickhouse.params.ParamsLoader;
 
 import liquibase.changelog.ChangeSet;
 import liquibase.database.Database;
@@ -47,14 +45,12 @@ public Sql[] generateSql(
       RemoveChangeSetRanStatusStatement statement,
       Database database,
       SqlGeneratorChain sqlGeneratorChain) {
-    ClusterConfig properties = ParamsLoader.getLiquibaseClickhouseProperties();
 
     ChangeSet changeSet = statement.getChangeSet();
     String unlockQuery =
         String.format(
-            "ALTER TABLE %s.%s "
-                + SqlGeneratorUtil.generateSqlOnClusterClause(properties)
-                + "DELETE WHERE ID = '%s' AND AUTHOR = '%s' AND FILENAME = '%s' SETTINGS mutations_sync = 1",
+            "SET allow_experimental_lightweight_delete = 1;"
+                + "DELETE FROM %s.%s WHERE ID = '%s' AND AUTHOR = '%s' AND FILENAME = '%s'",
             database.getDefaultSchemaName(),
             database.getDatabaseChangeLogTableName(),
             changeSet.getId(),

src/main/java/liquibase/ext/clickhouse/sqlgenerator/UnlockDatabaseChangelogClickHouse.java

@@ -20,8 +20,6 @@
 package liquibase.ext.clickhouse.sqlgenerator;
 
 import liquibase.ext.clickhouse.database.ClickHouseDatabase;
-import liquibase.ext.clickhouse.params.ClusterConfig;
-import liquibase.ext.clickhouse.params.ParamsLoader;
 
 import liquibase.database.Database;
 import liquibase.sql.Sql;
@@ -46,15 +44,11 @@ public Sql[] generateSql(
       UnlockDatabaseChangeLogStatement statement,
       Database database,
       SqlGeneratorChain sqlGeneratorChain) {
-    ClusterConfig properties = ParamsLoader.getLiquibaseClickhouseProperties();
 
     String unlockQuery =
         String.format(
-            "ALTER TABLE %s.%s "
-                + SqlGeneratorUtil.generateSqlOnClusterClause(properties)
-                + "UPDATE LOCKED = 0,LOCKEDBY = null, LOCKGRANTED = null WHERE ID = 1 AND LOCKED = 1 SETTINGS mutations_sync = 1",
-            database.getDefaultSchemaName(),
-            database.getDatabaseChangeLogLockTableName());
+            "SET allow_experimental_lightweight_delete = 1; DELETE FROM %s.%s WHERE ID = 1;",
+            database.getDefaultSchemaName(), database.getDatabaseChangeLogLockTableName());
 
     return SqlGeneratorUtil.generateSql(database, unlockQuery);
   }

src/test/java/liquibase/ClickHouseTest.java

@@ -37,7 +37,7 @@ public class ClickHouseTest {
 
   @Container
   private static ClickHouseContainer clickHouseContainer =
-      new ClickHouseContainer("clickhouse/clickhouse-server:22.3.8.39");
+      new ClickHouseContainer("clickhouse/clickhouse-server:23.3");
 
   @Test
   void canInitializeLiquibaseSchema() {