Include support for git-remote-s3

modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy

@@ -98,6 +98,7 @@ class Launcher {
                 new CmdList(),
                 new CmdLog(),
                 new CmdPull(),
+                new CmdPush(),
                 new CmdRun(),
                 new CmdKubeRun(),
                 new CmdDrop(),

modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy

@@ -112,6 +112,14 @@ class AssetManager {
         build(pipelineName, config)
     }
 
+    AssetManager(File path, String pipelineName, HubOptions cliOpts = null ) {
+        assert path
+        assert pipelineName
+        // build the object
+        def config = ProviderConfig.getDefault()
+        build(path, pipelineName, config, cliOpts)
+    }
+
     /**
      * Build the asset manager internal data structure
      *
@@ -135,6 +143,22 @@ class AssetManager {
         return this
     }
 
+    @PackageScope
+    AssetManager build( File path, String pipelineName, Map config = null, HubOptions cliOpts = null ) {
+
+        this.providerConfigs = ProviderConfig.createFromMap(config)
+
+        this.project = resolveName(pipelineName)
+        this.localPath = path
+        this.hub = checkHubProvider(cliOpts)
+        this.provider = createHubProvider(hub)
+        setupCredentials(cliOpts)
+        validateProjectDir()
+
+        return this
+    }
+
+
     @PackageScope
     File getLocalGitConfig() {
         localPath ? new File(localPath,'.git/config') : null
@@ -307,7 +331,7 @@ class AssetManager {
     @PackageScope
     String resolveNameFromGitUrl( String repository ) {
 
-        final isUrl = repository.startsWith('http://') || repository.startsWith('https://') || repository.startsWith('file:/')
+        final isUrl = repository.startsWith('http://') || repository.startsWith('https://') || repository.startsWith('file:/') || repository.startsWith('s3://')
         if( !isUrl )
             return null
 
@@ -685,6 +709,45 @@ class AssetManager {
 
     }
 
+    /**
+     * Upload a pipeline to a remote repository
+     *
+     * @param revision The revision/branch to upload
+     * @param remoteName The name of the remote (default: origin)
+     * @param isNewRepo Whether this is a new repository initialization
+     * @result A message representing the operation result
+     */
+    String upload(String revision, String remoteName = "origin", boolean isNewRepo = false) {
+        assert project
+        assert localPath
+
+        // Create and checkout branch if it doesn't exist
+        try {
+            git.checkout().setName(revision).call()
+        }
+        catch( Exception ignored ) {
+            // Branch doesn't exist, create it
+            git.checkout()
+                .setCreateBranch(true)
+                .setName(revision)
+                .call()
+        }
+
+
+        def pushCommand = git.push()
+            .setRemote(remoteName)
+
+        pushCommand.add(revision)
+
+        if( provider.hasCredentials() )
+            pushCommand.setCredentialsProvider( provider.getGitCredentials() )
+
+        def result = pushCommand.call()
+        return "pushed to ${remoteName} (${revision})"
+    }
+
+
+
     /**
      * Clone a pipeline from a remote pipeline repository to the specified folder
      *

modules/nextflow/src/main/groovy/nextflow/scm/RepositoryFactory.groovy

@@ -88,6 +88,7 @@ class RepositoryFactory implements ExtensionPoint {
 
     // --==  static definitions ==--
     private static boolean codeCommitLoaded
+    private static boolean s3Loaded
     private static List<RepositoryFactory> factories0
 
     private static List<RepositoryFactory> factories() {
@@ -102,6 +103,11 @@ class RepositoryFactory implements ExtensionPoint {
 
     static RepositoryProvider newRepositoryProvider(ProviderConfig config, String project) {
         // check if it's needed to load new plugins
+        if( config.platform == 's3' && !s3Loaded){
+            Plugins.startIfMissing('nf-amazon')
+            s3Loaded=true
+            factories0=null
+        }
         if( (config.name=='codecommit' || config.platform=='codecommit') && !codeCommitLoaded ) {
             Plugins.startIfMissing('nf-codecommit')
             codeCommitLoaded=true
@@ -120,6 +126,11 @@ class RepositoryFactory implements ExtensionPoint {
 
     static ProviderConfig newProviderConfig(String name, Map<String,Object> attrs) {
         // check if it's needed to load new plugins
+        if( attrs.platform == 's3' && !s3Loaded){
+            Plugins.startIfMissing('nf-amazon')
+            s3Loaded=true
+            factories0=null
+        }
         if( (name=='codecommit' || attrs.platform=='codecommit') && !codeCommitLoaded ) {
             Plugins.startIfMissing('nf-codecommit')
             codeCommitLoaded=true
@@ -134,6 +145,11 @@ class RepositoryFactory implements ExtensionPoint {
     }
 
     static ProviderConfig getProviderConfig(List<ProviderConfig> providers, GitUrl url) {
+        if( url.protocol.equals('s3') && !s3Loaded){
+            Plugins.startIfMissing('nf-amazon')
+            s3Loaded=true
+            factories0=null
+        }
         if( url.domain.startsWith('git-codecommit.') && url.domain.endsWith('.amazonaws.com') && !codeCommitLoaded ) {
             Plugins.startIfMissing('nf-codecommit')
             codeCommitLoaded=true

plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/S3ProviderConfig.groovy

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2013-2025, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.cloud.aws.scm
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import nextflow.Global
+import nextflow.exception.AbortOperationException
+import nextflow.scm.ProviderConfig
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider
+import software.amazon.awssdk.regions.Region
+
+/**
+ * Implements a provider config for git-remote-s3 repositories
+ *
+ * @author Jorge Ejarque <[email protected]>
+ */
+@Slf4j
+@CompileStatic
+class S3ProviderConfig extends ProviderConfig {
+
+    private Region region = Region.US_EAST_1
+
+    private AwsCredentialsProvider awsCredentialsProvider = DefaultCredentialsProvider.builder().build()
+
+    S3ProviderConfig(String name, Map values) {
+        super(name, values)
+        setDefaultsFromAwsConfig()
+        // Override with scm repo attributes
+        setValuesFromMap(values)
+    }
+
+    S3ProviderConfig(String name){
+        super(name,[ platform: 's3', server: "s3://$name"])
+        setDefaultsFromAwsConfig()
+    }
+
+    private void setDefaultsFromAwsConfig() {
+        final config = Global.session?.config?.aws as Map
+        if( config ) {
+            setValuesFromMap(config)
+        }
+    }
+    private void setValuesFromMap(Map values){
+        if( values.region )
+            region = Region.of(values.region as String)
+        if( values.accessKey && values.secretKey ){
+            awsCredentialsProvider = StaticCredentialsProvider.create(
+                AwsBasicCredentials.builder()
+                    .accessKeyId(values.accessKey as String)
+                    .secretAccessKey(values.secretKey as String)
+                    .build())
+        }
+    }
+
+    Region getRegion(){
+        this.region
+    }
+
+    AwsCredentialsProvider getAwsCredentialsProvider(){
+        this.awsCredentialsProvider
+    }
+
+    @Override
+    protected String resolveProjectName(String path){
+        log.debug ("Resolving project name from $path. returning ")
+        if (!server.startsWith('s3://'))
+            new AbortOperationException("S3 project server doesn't start with s3://")
+        return "${server.substring('s3://'.size())}/$path"
+    }
+
+}

plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/S3RepositoryFactory.groovy

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2013-2025, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.cloud.aws.scm
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import nextflow.cloud.aws.scm.jgit.TransportS3
+import nextflow.plugin.Priority
+import nextflow.scm.GitUrl
+import nextflow.scm.ProviderConfig
+import nextflow.scm.RepositoryFactory
+import nextflow.scm.RepositoryProvider
+
+import java.util.concurrent.atomic.AtomicBoolean
+
+/**
+ * Implements a factory to create an instance of {@link S3RepositoryProvider}
+ *
+ * @author Jorge Ejarque <[email protected]>
+ */
+@Slf4j
+@Priority(-10)
+@CompileStatic
+class S3RepositoryFactory extends RepositoryFactory{
+
+    private static AtomicBoolean registered = new AtomicBoolean(false)
+
+    @Override
+    protected RepositoryProvider createProviderInstance(ProviderConfig config, String project) {
+        if (!registered.get()) {
+            registered.set(true)
+            TransportS3.register()
+        }
+
+        return config.platform == 's3'
+            ? new S3RepositoryProvider(project, config)
+            : null
+    }
+
+    @Override
+    protected ProviderConfig getConfig(List<ProviderConfig> providers, GitUrl url) {
+        // do not care about non AWS codecommit url
+        if( url.protocol != 's3' )
+            return null
+
+        // S3 repository config depends on the bucket name stored as domain
+        def config = providers.find( it -> it.domain == url.domain )
+        if( config ) {
+            log.debug "Git url=$url (1) -> config=$config"
+            return config
+        }
+
+        // still nothing, create a new instance
+        config = new S3ProviderConfig(url.domain)
+
+
+        return config
+    }
+
+    @Override
+    protected ProviderConfig createConfigInstance(String name, Map attrs) {
+        final copy = new HashMap(attrs)
+        return copy.platform == 's3'
+            ? new S3ProviderConfig(name, copy)
+            : null
+    }
+
+
+
+
+}