From 4a8fab3dd749561a484c95cdbfda48a469ba66f9 Mon Sep 17 00:00:00 2001 From: Dai MIKURUBE Date: Thu, 30 Nov 2023 17:07:26 +0900 Subject: [PATCH 1/2] Initial work to "install" Maven-based Embulk plugins out of Embulk commands, to replace Bundler --- .github/CODEOWNERS | 1 + .../gradle/runset/EmbulkRunSetPlugin.java | 1 + .../gradle/runset/InstallEmbulkRunSet.java | 199 ++++++++++++++++++ .../gradle/runset/TestEmbulkRunSetPlugin.java | 21 +- .../java/org/embulk/gradle/runset/Util.java | 143 +++++++++++++ src/test/resources/simple/build.gradle | 13 ++ 6 files changed, 375 insertions(+), 3 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 src/main/java/org/embulk/gradle/runset/InstallEmbulkRunSet.java create mode 100644 src/test/java/org/embulk/gradle/runset/Util.java create mode 100644 src/test/resources/simple/build.gradle diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..1b44dee --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @embulk/core-team diff --git a/src/main/java/org/embulk/gradle/runset/EmbulkRunSetPlugin.java b/src/main/java/org/embulk/gradle/runset/EmbulkRunSetPlugin.java index f729aa0..edc1585 100644 --- a/src/main/java/org/embulk/gradle/runset/EmbulkRunSetPlugin.java +++ b/src/main/java/org/embulk/gradle/runset/EmbulkRunSetPlugin.java @@ -25,5 +25,6 @@ public class EmbulkRunSetPlugin implements Plugin { @Override public void apply(final Project project) { + project.getTasks().register("installEmbulkRunSet", InstallEmbulkRunSet.class); } } diff --git a/src/main/java/org/embulk/gradle/runset/InstallEmbulkRunSet.java b/src/main/java/org/embulk/gradle/runset/InstallEmbulkRunSet.java new file mode 100644 index 0000000..02c102c --- /dev/null +++ b/src/main/java/org/embulk/gradle/runset/InstallEmbulkRunSet.java @@ -0,0 +1,199 @@ +/* + * Copyright 2023 The Embulk project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.embulk.gradle.runset; + +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.gradle.api.IllegalDependencyNotation; +import org.gradle.api.Project; +import org.gradle.api.artifacts.ArtifactCollection; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.artifacts.ResolvableDependencies; +import org.gradle.api.artifacts.component.ComponentIdentifier; +import org.gradle.api.artifacts.component.ModuleComponentIdentifier; +import org.gradle.api.artifacts.component.ProjectComponentIdentifier; +import org.gradle.api.artifacts.result.ArtifactResolutionResult; +import org.gradle.api.artifacts.result.ArtifactResult; +import org.gradle.api.artifacts.result.ComponentArtifactsResult; +import org.gradle.api.artifacts.result.ResolvedArtifactResult; +import org.gradle.api.file.DuplicatesStrategy; +import org.gradle.api.logging.Logger; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.tasks.Copy; +import org.gradle.maven.MavenModule; +import org.gradle.maven.MavenPomArtifact; + +/** + * A Gradle Task to set up an environment for running Embulk. + */ +public class InstallEmbulkRunSet extends Copy { + public InstallEmbulkRunSet() { + super(); + + this.project = this.getProject(); + this.logger = this.project.getLogger(); + + final ObjectFactory objectFactory = this.project.getObjects(); + } + + /** + * Adds a Maven artifact to be installed. + * + *

It tries to simulate Gradle's dependency notations, but it is yet far from perfect. + * + * @see org.gradle.api.internal.notations.DependencyNotationParser#create + */ + public void artifact(final Object dependencyNotation) { + final Dependency dependency; + if (dependencyNotation instanceof CharSequence) { + dependency = this.dependencyFromCharSequence((CharSequence) dependencyNotation); + } else if (dependencyNotation instanceof Map) { + dependency = this.dependencyFromMap((Map) dependencyNotation); + } else { + throw new IllegalDependencyNotation("Supplied module notation is invalid."); + } + + // Constructing an independent (detached) Configuration so that its dependencies are not affected by other plugins. + final Configuration configuration = this.project.getConfigurations().detachedConfiguration(dependency); + + final ResolvableDependencies resolvavbleDependencies = configuration.getIncoming(); + final ArtifactCollection artifactCollection = resolvavbleDependencies.getArtifacts(); + + // Getting the JAR files and component IDs. + final ArrayList componentIds = new ArrayList<>(); + for (final ResolvedArtifactResult resolvedArtifactResult : artifactCollection.getArtifacts()) { + componentIds.add(resolvedArtifactResult.getId().getComponentIdentifier()); + this.fromArtifact(resolvedArtifactResult, "jar"); + } + + // Getting the POM files. + final ArtifactResolutionResult artifactResolutionResult = this.project.getDependencies() + .createArtifactResolutionQuery() + .forComponents(componentIds) + .withArtifacts(MavenModule.class, MavenPomArtifact.class) + .execute(); + for (final ComponentArtifactsResult componentArtifactResult : artifactResolutionResult.getResolvedComponents()) { + for (final ArtifactResult artifactResult : componentArtifactResult.getArtifacts(MavenPomArtifact.class)) { + if (artifactResult instanceof ResolvedArtifactResult) { + final ResolvedArtifactResult resolvedArtifactResult = (ResolvedArtifactResult) artifactResult; + this.fromArtifact(resolvedArtifactResult, "pom"); + } + } + } + } + + private void fromArtifact(final ResolvedArtifactResult resolvedArtifactResult, final String artifactType) { + final ComponentIdentifier id = resolvedArtifactResult.getId().getComponentIdentifier(); + final File file = resolvedArtifactResult.getFile(); + + if (id instanceof ModuleComponentIdentifier) { + final Path modulePath = moduleToPath((ModuleComponentIdentifier) id); + this.logger.info("Setting to copy {}:{} into {}", id, artifactType, modulePath); + this.logger.debug("Cached file: {}", file); + this.from(file, copy -> { + copy.into(modulePath.toFile()); + copy.setDuplicatesStrategy(DuplicatesStrategy.EXCLUDE); + }); + } else if (id instanceof ProjectComponentIdentifier) { + throw new IllegalDependencyNotation("Cannot install artifacts for a project component (" + id.getDisplayName() + ")"); + } else { + throw new IllegalDependencyNotation( + "Cannot resolve the artifacts for component " + + id.getDisplayName() + + " with unsupported type " + + id.getClass().getName() + + "."); + } + } + + private static Path moduleToPath(final ModuleComponentIdentifier id) { + final String[] splitGroup = id.getGroup().split("\\."); + if (splitGroup.length <= 0) { + return Paths.get(""); + } + + final String[] more = new String[splitGroup.length + 2 - 1]; + for (int i = 1; i < splitGroup.length; i++) { + more[i - 1] = splitGroup[i]; + } + more[splitGroup.length - 1] = id.getModule(); + more[splitGroup.length] = id.getVersion(); + final Path path = Paths.get(splitGroup[0], more); + assert !path.isAbsolute(); + return path; + } + + // https://github.com/gradle/gradle/blob/v8.4.0/subprojects/dependency-management/src/main/java/org/gradle/api/internal/notations/DependencyStringNotationConverter.java + private Dependency dependencyFromCharSequence(final CharSequence dependencyNotation) { + final String notationString = dependencyNotation.toString(); + this.logger.debug("Artifact: {}", notationString); + return this.project.getDependencies().create(notationString); + } + + // https://github.com/gradle/gradle/blob/v8.4.0/subprojects/core/src/main/java/org/gradle/internal/typeconversion/MapNotationConverter.java + private Dependency dependencyFromMap(final Map dependencyNotation) { + final Map notationMap = validateMap(dependencyNotation); + this.logger.debug("Artifact: {}", notationMap); + return this.project.getDependencies().create(notationMap); + } + + private static Map validateMap(final Map dependencyNotation) { + final LinkedHashMap map = new LinkedHashMap<>(); + for (final Map.Entry entry : castMap(dependencyNotation).entrySet()) { + final Object keyObject = entry.getKey(); + if (!(keyObject instanceof CharSequence)) { + throw new IllegalDependencyNotation("Supplied Map module notation is invalid. Its key must be a String."); + } + final String key = (String) keyObject; + if (!ACCEPTABLE_MAP_KEYS.contains(key)) { + throw new IllegalDependencyNotation( + "Supplied Map module notation is invalid. Its key must be one of: [" + + String.join(", ", ACCEPTABLE_MAP_KEYS) + + "]"); + } + + final Object valueObject = entry.getValue(); + if (!(valueObject instanceof CharSequence)) { + throw new IllegalDependencyNotation("Supplied Map module notation is invalid. Its value must be a String."); + } + final String value = (String) valueObject; + map.put(key, value); + } + return Collections.unmodifiableMap(map); + } + + @SuppressWarnings("unchecked") + private static Map castMap(final Map map) { + return (Map) map; + } + + // https://github.com/gradle/gradle/blob/v8.4.0/subprojects/dependency-management/src/main/java/org/gradle/api/internal/notations/DependencyMapNotationConverter.java#L42-L58 + private static List ACCEPTABLE_MAP_KEYS = + Arrays.asList("group", "name", "version", "configuration", "ext", "classifier"); + + private final Logger logger; + + private final Project project; +} diff --git a/src/test/java/org/embulk/gradle/runset/TestEmbulkRunSetPlugin.java b/src/test/java/org/embulk/gradle/runset/TestEmbulkRunSetPlugin.java index cabbe47..967a593 100644 --- a/src/test/java/org/embulk/gradle/runset/TestEmbulkRunSetPlugin.java +++ b/src/test/java/org/embulk/gradle/runset/TestEmbulkRunSetPlugin.java @@ -16,16 +16,31 @@ package org.embulk.gradle.runset; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.embulk.gradle.runset.Util.prepareProjectDir; +import static org.embulk.gradle.runset.Util.runGradle; import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; public class TestEmbulkRunSetPlugin { @Test - public void test(@TempDir Path tempDir) throws IOException { - assertEquals(true, true); + public void testSimple(@TempDir Path tempDir) throws IOException { + final Path projectDir = prepareProjectDir(tempDir, "simple"); + + runGradle(projectDir, "installEmbulkRunSet"); + + Files.walkFileTree(projectDir.resolve("build/simple"), new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { + System.out.println(projectDir.relativize(file)); + return FileVisitResult.CONTINUE; + } + }); } } diff --git a/src/test/java/org/embulk/gradle/runset/Util.java b/src/test/java/org/embulk/gradle/runset/Util.java new file mode 100644 index 0000000..4bc3045 --- /dev/null +++ b/src/test/java/org/embulk/gradle/runset/Util.java @@ -0,0 +1,143 @@ +/* + * Copyright 2019 The Embulk project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.embulk.gradle.runset; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.JarURLConnection; +import java.net.URL; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; +import org.gradle.testkit.runner.BuildResult; +import org.gradle.testkit.runner.GradleRunner; + +/** + * Utility methods for testing the Embulk plugins Gradle plugin. + */ +class Util { + private Util() { + // No instantiation. + } + + static Path prepareProjectDir(final Path tempDir, final String testProjectName) { + final String resourceName = testProjectName + System.getProperty("file.separator") + "build.gradle"; + final Path resourceDir; + try { + final URL resourceUrl = Util.class.getClassLoader().getResource(resourceName); + if (resourceUrl == null) { + throw new FileNotFoundException(resourceName + " is not found."); + } + resourceDir = Paths.get(resourceUrl.toURI()).getParent(); + } catch (final Exception ex) { + fail("Failed to find a test resource.", ex); + throw new RuntimeException(ex); // Never reaches. + } + + final Path projectDir; + try { + projectDir = Files.createDirectory(tempDir.resolve(testProjectName)); + } catch (final Exception ex) { + fail("Failed to create a test directory.", ex); + throw new RuntimeException(ex); // Never reaches. + } + + try { + copyFilesRecursively(resourceDir, projectDir); + } catch (final Exception ex) { + fail("Failed to copy test files.", ex); + throw new RuntimeException(ex); // Never reaches. + } + + return projectDir; + } + + private static void copyFilesRecursively(final Path source, final Path destination) throws IOException { + Files.walkFileTree(source, new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttributes attrs) throws IOException { + final Path target = destination.resolve(source.relativize(dir)); + Files.createDirectories(target); + System.out.println(target.toString() + System.getProperty("file.separator")); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { + final Path target = destination.resolve(source.relativize(file)); + Files.copy(file, target); + System.out.println(target); + return FileVisitResult.CONTINUE; + } + }); + } + + static BuildResult runGradle(final Path projectDir, final String... args) { + final ArrayList argsList = new ArrayList<>(); + argsList.addAll(Arrays.asList(args)); + argsList.add("--stacktrace"); + argsList.add("--info"); + final BuildResult result = newGradleRunner(projectDir, argsList).build(); + System.out.println("Running 'gradle " + String.join(" ", argsList) + "' :"); + System.out.println("============================================================"); + System.out.print(result.getOutput()); + System.out.println("============================================================"); + return result; + } + + static void assertFileDoesContain(final Path path, final String expected) throws IOException { + try (final Stream lines = Files.newBufferedReader(path).lines()) { + final boolean found = lines.filter(actualLine -> { + return actualLine.contains(expected); + }).findAny().isPresent(); + if (!found) { + fail("\"" + path.toString() + "\" does not contain \"" + expected + "\"."); + } + } + } + + static void assertFileDoesNotContain(final Path path, final String notExpected) throws IOException { + try (final Stream lines = Files.newBufferedReader(path).lines()) { + lines.forEach(actualLine -> { + assertFalse(actualLine.contains(notExpected)); + }); + } + } + + private static GradleRunner newGradleRunner(final Path projectDir, final List args) { + return GradleRunner.create() + .withProjectDir(projectDir.toFile()) + .withArguments(args) + .withDebug(true) + .withPluginClasspath(); + } + + static JarURLConnection openJarUrlConnection(final Path jarPath) throws IOException { + final URL jarUrl = new URL("jar:" + jarPath.toUri().toURL().toString() + "!/"); + return (JarURLConnection) jarUrl.openConnection(); + } +} diff --git a/src/test/resources/simple/build.gradle b/src/test/resources/simple/build.gradle new file mode 100644 index 0000000..b4fca2d --- /dev/null +++ b/src/test/resources/simple/build.gradle @@ -0,0 +1,13 @@ +plugins { + id "org.embulk.runset" +} + +repositories { + mavenCentral() +} + +installEmbulkRunSet { + into "${project.buildDir}/simple" + artifact "org.embulk:embulk-input-postgresql:0.13.2" + artifact group: "org.embulk", name: "embulk-input-s3", version: "0.6.0" +} From 91dae07055478f8725ab4be7ea9b67a02de889cc Mon Sep 17 00:00:00 2001 From: Dai MIKURUBE Date: Fri, 1 Dec 2023 16:34:17 +0900 Subject: [PATCH 2/2] Fix a typo --- .../java/org/embulk/gradle/runset/InstallEmbulkRunSet.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/embulk/gradle/runset/InstallEmbulkRunSet.java b/src/main/java/org/embulk/gradle/runset/InstallEmbulkRunSet.java index 02c102c..f233250 100644 --- a/src/main/java/org/embulk/gradle/runset/InstallEmbulkRunSet.java +++ b/src/main/java/org/embulk/gradle/runset/InstallEmbulkRunSet.java @@ -78,8 +78,8 @@ public void artifact(final Object dependencyNotation) { // Constructing an independent (detached) Configuration so that its dependencies are not affected by other plugins. final Configuration configuration = this.project.getConfigurations().detachedConfiguration(dependency); - final ResolvableDependencies resolvavbleDependencies = configuration.getIncoming(); - final ArtifactCollection artifactCollection = resolvavbleDependencies.getArtifacts(); + final ResolvableDependencies resolvableDependencies = configuration.getIncoming(); + final ArtifactCollection artifactCollection = resolvableDependencies.getArtifacts(); // Getting the JAR files and component IDs. final ArrayList componentIds = new ArrayList<>();