diff --git a/build.gradle b/build.gradle index dade656f78b82..c5611e8b453fb 100644 --- a/build.gradle +++ b/build.gradle @@ -221,14 +221,6 @@ allprojects { "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi', "org.elasticsearch.test:framework:${version}": ':test:framework', "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip', - "downloads.zip:elasticsearch:${version}": ':distribution:archives:zip', - "downloads.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip', - "downloads.tar:elasticsearch:${version}": ':distribution:archives:tar', - "downloads.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar', - "downloads.rpm:elasticsearch:${version}": ':distribution:packages:rpm', - "downloads.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm', - "downloads.deb:elasticsearch:${version}": ':distribution:packages:deb', - "downloads.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb', "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage', "org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware', // for transport client @@ -240,19 +232,6 @@ allprojects { "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator', "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval', ] - // substitute unreleased versions with projects that check out and build locally - bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion -> - Version unreleased = unreleasedVersion.version - String snapshotProject = ":distribution:bwc:${unreleasedVersion.gradleProjectName}" - ext.projectSubstitutions["downloads.deb:elasticsearch:${unreleased}"] = snapshotProject - ext.projectSubstitutions["downloads.rpm:elasticsearch:${unreleased}"] = snapshotProject - ext.projectSubstitutions["downloads.zip:elasticsearch:${unreleased}"] = snapshotProject - if (unreleased.onOrAfter('6.3.0')) { - ext.projectSubstitutions["downloads.deb:elasticsearch-oss:${unreleased}"] = snapshotProject - ext.projectSubstitutions["downloads.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject - ext.projectSubstitutions["downloads.zip:elasticsearch-oss:${unreleased}"] = snapshotProject - } - } /* * Gradle only resolve project substitutions during dependency resolution but diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 72041b4e628d3..e38cb854a10b0 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -23,6 +23,7 @@ import org.apache.tools.ant.taskdefs.condition.Os import org.elasticsearch.gradle.BuildPlugin import org.elasticsearch.gradle.LoggedExec import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.VersionCollection import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.plugin.PluginBuildPlugin import org.elasticsearch.gradle.plugin.PluginPropertiesExtension @@ -171,6 +172,12 @@ class ClusterFormationTasks { /** Adds a dependency on the given distribution */ static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) { + if (distro.equals("integ-test-zip")) { + // short circuit integ test so it doesn't complicate the rest of the distribution setup below + project.dependencies.add(configuration.name, + "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${elasticsearchVersion}@zip") + return + } // TEMP HACK // The oss docs CI build overrides the distro on the command line. This hack handles backcompat until CI is updated. if (distro.equals('oss-zip')) { @@ -180,22 +187,31 @@ class ClusterFormationTasks { distro = 'default' } // END TEMP HACK - if (['integ-test-zip', 'oss', 'default'].contains(distro) == false) { + if (['oss', 'default'].contains(distro) == false) { throw new GradleException("Unknown distribution: ${distro} in project ${project.path}") } Version version = Version.fromString(elasticsearchVersion) - if (version.before('6.3.0') && distro.startsWith('oss-')) { - distro = distro.substring('oss-'.length()) - } - String group = "downloads.zip" - if (distro.equals("integ-test-zip")) { - group = "org.elasticsearch.distribution.integ-test-zip" - } + String group = "downloads.zip" // dummy group, does not matter except for integ-test-zip, it is ignored by the fake ivy repo String artifactName = 'elasticsearch' if (distro.equals('oss') && Version.fromString(elasticsearchVersion).onOrAfter('6.3.0')) { artifactName += '-oss' } - project.dependencies.add(configuration.name, "${group}:${artifactName}:${elasticsearchVersion}@zip") + String snapshotProject = distro == 'oss' ? 'oss-zip' : 'zip' + Object dependency + boolean internalBuild = project.hasProperty('bwcVersions') + VersionCollection.UnreleasedVersionInfo unreleasedInfo = null + if (project.hasProperty('bwcVersions')) { + // NOTE: leniency is needed for external plugin authors using build-tools. maybe build the version compat info into build-tools? + unreleasedInfo = project.bwcVersions.unreleasedInfo(version) + } + if (unreleasedInfo != null) { + dependency = project.dependencies.project(path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject) + } else if (internalBuild && elasticsearchVersion.equals(VersionProperties.elasticsearch)) { + dependency = project.dependencies.project(path: ":distribution:archives:${snapshotProject}") + } else { + dependency = "${group}:${artifactName}:${elasticsearchVersion}@zip" + } + project.dependencies.add(configuration.name, dependency) } /** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */ diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy index dfb825afea255..fec6b2eab7405 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy @@ -4,6 +4,7 @@ import org.apache.tools.ant.taskdefs.condition.Os import org.elasticsearch.gradle.FileContentsTask import org.elasticsearch.gradle.LoggedExec import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.VersionCollection import org.gradle.api.* import org.gradle.api.artifacts.dsl.RepositoryHandler import org.gradle.api.execution.TaskExecutionAdapter @@ -184,22 +185,38 @@ class VagrantTestPlugin implements Plugin { upgradeFromVersion = Version.fromString(upgradeFromVersionRaw) } + List dependencies = new ArrayList<>() DISTRIBUTIONS.each { // Adds a dependency for the current version - project.dependencies.add(PACKAGING_CONFIGURATION, - project.dependencies.project(path: ":distribution:${it}", configuration: 'default')) + dependencies.add(project.dependencies.project(path: ":distribution:${it}", configuration: 'default')) } - UPGRADE_FROM_ARCHIVES.each { - // The version of elasticsearch that we upgrade *from* - project.dependencies.add(PACKAGING_CONFIGURATION, - "downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}") - if (upgradeFromVersion.onOrAfter('6.3.0')) { - project.dependencies.add(PACKAGING_CONFIGURATION, - "downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}") + // The version of elasticsearch that we upgrade *from* + VersionCollection.UnreleasedVersionInfo unreleasedInfo = project.bwcVersions.unreleasedInfo(upgradeFromVersion) + if (unreleasedInfo != null) { + // handle snapshots pointing to bwc build + UPGRADE_FROM_ARCHIVES.each { + dependencies.add(project.dependencies.project( + path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: it)) + if (upgradeFromVersion.onOrAfter('6.3.0')) { + dependencies.add(project.dependencies.project( + path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: "oss-${it}")) + } + } + } else { + UPGRADE_FROM_ARCHIVES.each { + // The version of elasticsearch that we upgrade *from* + dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}") + if (upgradeFromVersion.onOrAfter('6.3.0')) { + dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}") + } } } + for (Object dependency : dependencies) { + project.dependencies.add(PACKAGING_CONFIGURATION, dependency) + } + project.extensions.esvagrant.upgradeFromVersion = upgradeFromVersion } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java b/buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java index 1cf2fd9e1037c..e2af34dbabdc0 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java @@ -20,6 +20,8 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -86,6 +88,7 @@ public class VersionCollection { private final Version currentVersion; private final Map> groupByMajor; + private final Map unreleased; public class UnreleasedVersionInfo { public final Version version; @@ -129,6 +132,16 @@ protected VersionCollection(List versionLines, Version currentVersionPro assertCurrentVersionMatchesParsed(currentVersionProperty); assertNoOlderThanTwoMajors(); + + Map unreleased = new HashMap<>(); + for (Version unreleasedVersion : getUnreleased()) { + if (unreleasedVersion.equals(currentVersion)) { + continue; + } + unreleased.put(unreleasedVersion, + new UnreleasedVersionInfo(unreleasedVersion, getBranchFor(unreleasedVersion), getGradleProjectNameFor(unreleasedVersion))); + } + this.unreleased = Collections.unmodifiableMap(unreleased); } private void assertNoOlderThanTwoMajors() { @@ -150,6 +163,13 @@ private void assertCurrentVersionMatchesParsed(Version currentVersionProperty) { } } + /** + * Returns info about the unreleased version, or {@code null} if the version is released. + */ + public UnreleasedVersionInfo unreleasedInfo(Version version) { + return unreleased.get(version); + } + public void forPreviousUnreleased(Consumer consumer) { getUnreleased().stream() .filter(version -> version.equals(currentVersion) == false) diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle index 47f3b78c43f5d..a0acce1ef64eb 100644 --- a/distribution/bwc/build.gradle +++ b/distribution/bwc/build.gradle @@ -120,18 +120,19 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre } } - List artifactFiles = [] + Map artifactFiles = [:] List projectDirs = [] - for (String project : ['zip', 'deb', 'rpm']) { + List projects = ['zip', 'deb', 'rpm'] + for (String projectName : projects) { String baseDir = "distribution" if (bwcVersion.onOrAfter('6.3.0')) { - baseDir += project == 'zip' ? '/archives' : '/packages' + baseDir += projectName == 'zip' ? '/archives' : '/packages' // add oss variant first - projectDirs.add("${baseDir}/oss-${project}") - artifactFiles.add(file("${checkoutDir}/${baseDir}/oss-${project}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT.${project}")) + projectDirs.add("${baseDir}/oss-${projectName}") + artifactFiles.put("oss-" + projectName, file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT.${projectName}")) } - projectDirs.add("${baseDir}/${project}") - artifactFiles.add(file("${checkoutDir}/${baseDir}/${project}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT.${project}")) + projectDirs.add("${baseDir}/${projectName}") + artifactFiles.put(projectName, file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT.${projectName}")) } task buildBwcVersion(type: Exec) { @@ -187,7 +188,7 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre standardOutput = new IndentingOutputStream(System.out, bwcVersion) errorOutput = new IndentingOutputStream(System.err, bwcVersion) doLast { - List missing = artifactFiles.grep { file -> + List missing = artifactFiles.values().grep { file -> false == file.exists() } if (false == missing.empty) { @@ -197,12 +198,16 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre } } - artifacts { - for (File artifactFile : artifactFiles) { - String artifactName = artifactFile.name.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch' + for (e in artifactFiles) { + String projectName = e.key + File artifactFile = e.value + String artifactFileName = artifactFile.name + String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch' String suffix = artifactFile.toString()[-3..-1] - 'default' file: artifactFile, name: artifactName, type: suffix, builtBy: buildBwcVersion - } + configurations.create(projectName) + artifacts { + it.add(projectName, [file: artifactFile, name: artifactName, type: suffix, builtBy: buildBwcVersion]) + } } // make sure no dependencies were added to assemble; we want it to be a no-op assemble.dependsOn = []