diff --git a/build.gradle b/build.gradle index 22f7cf1..1f97952 100644 --- a/build.gradle +++ b/build.gradle @@ -49,7 +49,7 @@ gradlePlugin { id = "org.embulk.runset" displayName = "A Gradle plugin to prepare an environment for running Embulk" description = "${project.description}" - implementationClass = "org.embulk.gradle.runset.RunsetPlugin" + implementationClass = "org.embulk.gradle.runset.EmbulkRunSetPlugin" tags = ["embulk"] } } diff --git a/src/main/java/org/embulk/gradle/runset/EmbulkRunSetPlugin.java b/src/main/java/org/embulk/gradle/runset/EmbulkRunSetPlugin.java index f729aa0..edc1585 100644 --- a/src/main/java/org/embulk/gradle/runset/EmbulkRunSetPlugin.java +++ b/src/main/java/org/embulk/gradle/runset/EmbulkRunSetPlugin.java @@ -25,5 +25,6 @@ public class EmbulkRunSetPlugin implements Plugin { @Override public void apply(final Project project) { + project.getTasks().register("installEmbulkRunSet", InstallEmbulkRunSet.class); } } diff --git a/src/main/java/org/embulk/gradle/runset/InstallEmbulkRunSet.java b/src/main/java/org/embulk/gradle/runset/InstallEmbulkRunSet.java new file mode 100644 index 0000000..cb5532e --- /dev/null +++ b/src/main/java/org/embulk/gradle/runset/InstallEmbulkRunSet.java @@ -0,0 +1,201 @@ +/* + * Copyright 2022 The Embulk project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.embulk.gradle.runset; + +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.gradle.api.IllegalDependencyNotation; +import org.gradle.api.Project; +import org.gradle.api.artifacts.ArtifactCollection; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.artifacts.ResolvableDependencies; +import org.gradle.api.artifacts.component.ComponentIdentifier; +import org.gradle.api.artifacts.component.ModuleComponentIdentifier; +import org.gradle.api.artifacts.component.ProjectComponentIdentifier; +import org.gradle.api.artifacts.result.ArtifactResolutionResult; +import org.gradle.api.artifacts.result.ArtifactResult; +import org.gradle.api.artifacts.result.ComponentArtifactsResult; +import org.gradle.api.artifacts.result.ResolvedArtifactResult; +import org.gradle.api.file.DuplicatesStrategy; +import org.gradle.api.logging.Logger; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.Copy; +import org.gradle.api.tasks.InputFile; +import org.gradle.maven.MavenModule; +import org.gradle.maven.MavenPomArtifact; + +/** + * A Gradle Task to set up an environment for running Embulk. + */ +public class InstallEmbulkRunSet extends Copy { + public InstallEmbulkRunSet() { + super(); + + this.project = this.getProject(); + this.logger = this.project.getLogger(); + + final ObjectFactory objectFactory = this.project.getObjects(); + } + + /** + * Adds a Maven artifact to be installed. + * + *

It tries to simulate Gradle's dependency notations, but it is yet far from perfect. + * + * @see org.gradle.api.internal.notations.DependencyNotationParser#create + */ + public void artifact(final Object dependencyNotation) { + final Dependency dependency; + if (dependencyNotation instanceof CharSequence) { + dependency = this.dependencyFromCharSequence((CharSequence) dependencyNotation); + } else if (dependencyNotation instanceof Map) { + dependency = this.dependencyFromMap((Map) dependencyNotation); + } else { + throw new IllegalDependencyNotation("Supplied module notation is invalid."); + } + + // Constructing an independent (detached) Configuration so that its dependencies are not affected by other plugins. + final Configuration configuration = this.project.getConfigurations().detachedConfiguration(dependency); + + final ResolvableDependencies resolvavbleDependencies = configuration.getIncoming(); + final ArtifactCollection artifactCollection = resolvavbleDependencies.getArtifacts(); + + // Getting the JAR files and component IDs. + final ArrayList componentIds = new ArrayList<>(); + for (final ResolvedArtifactResult resolvedArtifactResult : artifactCollection.getArtifacts()) { + componentIds.add(resolvedArtifactResult.getId().getComponentIdentifier()); + this.fromArtifact(resolvedArtifactResult, "jar"); + } + + // Getting the POM files. + final ArtifactResolutionResult artifactResolutionResult = this.project.getDependencies() + .createArtifactResolutionQuery() + .forComponents(componentIds) + .withArtifacts(MavenModule.class, MavenPomArtifact.class) + .execute(); + for (final ComponentArtifactsResult componentArtifactResult : artifactResolutionResult.getResolvedComponents()) { + for (final ArtifactResult artifactResult : componentArtifactResult.getArtifacts(MavenPomArtifact.class)) { + if (artifactResult instanceof ResolvedArtifactResult) { + final ResolvedArtifactResult resolvedArtifactResult = (ResolvedArtifactResult) artifactResult; + this.fromArtifact(resolvedArtifactResult, "pom"); + } + } + } + } + + private void fromArtifact(final ResolvedArtifactResult resolvedArtifactResult, final String artifactType) { + final ComponentIdentifier id = resolvedArtifactResult.getId().getComponentIdentifier(); + final File file = resolvedArtifactResult.getFile(); + + if (id instanceof ModuleComponentIdentifier) { + final Path modulePath = moduleToPath((ModuleComponentIdentifier) id); + this.logger.info("Setting to copy {}:{} into {}", id, artifactType, modulePath); + this.logger.debug("Cached file: {}", file); + this.from(file, copy -> { + copy.into(modulePath.toFile()); + copy.setDuplicatesStrategy(DuplicatesStrategy.EXCLUDE); + }); + } else if (id instanceof ProjectComponentIdentifier) { + throw new IllegalDependencyNotation("Cannot install artifacts for a project component (" + id.getDisplayName() + ")"); + } else { + throw new IllegalDependencyNotation( + "Cannot resolve the artifacts for component " + + id.getDisplayName() + + " with unsupported type " + + id.getClass().getName() + + "."); + } + } + + private static Path moduleToPath(final ModuleComponentIdentifier id) { + final String[] splitGroup = id.getGroup().split("\\."); + if (splitGroup.length <= 0) { + return Paths.get(""); + } + + final String[] more = new String[splitGroup.length + 2 - 1]; + for (int i = 1; i < splitGroup.length; i++) { + more[i - 1] = splitGroup[i]; + } + more[splitGroup.length - 1] = id.getModule(); + more[splitGroup.length] = id.getVersion(); + final Path path = Paths.get(splitGroup[0], more); + assert !path.isAbsolute(); + return path; + } + + // https://github.com/gradle/gradle/blob/v8.4.0/subprojects/dependency-management/src/main/java/org/gradle/api/internal/notations/DependencyStringNotationConverter.java + private Dependency dependencyFromCharSequence(final CharSequence dependencyNotation) { + final String notationString = dependencyNotation.toString(); + this.logger.debug("Artifact: {}", notationString); + return this.project.getDependencies().create(notationString); + } + + // https://github.com/gradle/gradle/blob/v8.4.0/subprojects/core/src/main/java/org/gradle/internal/typeconversion/MapNotationConverter.java + private Dependency dependencyFromMap(final Map dependencyNotation) { + final Map notationMap = validateMap(dependencyNotation); + this.logger.debug("Artifact: {}", notationMap); + return this.project.getDependencies().create(notationMap); + } + + private static Map validateMap(final Map dependencyNotation) { + final LinkedHashMap map = new LinkedHashMap<>(); + for (final Map.Entry entry : castMap(dependencyNotation).entrySet()) { + final Object keyObject = entry.getKey(); + if (!(keyObject instanceof CharSequence)) { + throw new IllegalDependencyNotation("Supplied Map module notation is invalid. Its key must be a String."); + } + final String key = (String) keyObject; + if (!ACCEPTABLE_MAP_KEYS.contains(key)) { + throw new IllegalDependencyNotation( + "Supplied Map module notation is invalid. Its key must be one of: [" + + String.join(", ", ACCEPTABLE_MAP_KEYS) + + "]"); + } + + final Object valueObject = entry.getValue(); + if (!(valueObject instanceof CharSequence)) { + throw new IllegalDependencyNotation("Supplied Map module notation is invalid. Its value must be a String."); + } + final String value = (String) valueObject; + map.put(key, value); + } + return Collections.unmodifiableMap(map); + } + + @SuppressWarnings("unchecked") + private static Map castMap(final Map map) { + return (Map) map; + } + + // https://github.com/gradle/gradle/blob/v8.4.0/subprojects/dependency-management/src/main/java/org/gradle/api/internal/notations/DependencyMapNotationConverter.java#L42-L58 + private static List ACCEPTABLE_MAP_KEYS = + Arrays.asList("group", "name", "version", "configuration", "ext", "classifier"); + + private final Logger logger; + + private final Project project; +} diff --git a/src/test/java/org/embulk/gradle/runset/TestEmbulkRunSetPlugin.java b/src/test/java/org/embulk/gradle/runset/TestEmbulkRunSetPlugin.java new file mode 100644 index 0000000..967a593 --- /dev/null +++ b/src/test/java/org/embulk/gradle/runset/TestEmbulkRunSetPlugin.java @@ -0,0 +1,46 @@ +/* + * Copyright 2022 The Embulk project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.embulk.gradle.runset; + +import static org.embulk.gradle.runset.Util.prepareProjectDir; +import static org.embulk.gradle.runset.Util.runGradle; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class TestEmbulkRunSetPlugin { + @Test + public void testSimple(@TempDir Path tempDir) throws IOException { + final Path projectDir = prepareProjectDir(tempDir, "simple"); + + runGradle(projectDir, "installEmbulkRunSet"); + + Files.walkFileTree(projectDir.resolve("build/simple"), new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { + System.out.println(projectDir.relativize(file)); + return FileVisitResult.CONTINUE; + } + }); + } +} diff --git a/src/test/java/org/embulk/gradle/runset/TestRunsetPlugin.java b/src/test/java/org/embulk/gradle/runset/TestRunsetPlugin.java deleted file mode 100644 index 54fbf84..0000000 --- a/src/test/java/org/embulk/gradle/runset/TestRunsetPlugin.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2022 The Embulk project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.embulk.gradle.runset; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.io.IOException; -import java.nio.file.Path; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -public class TestRunsetPlugin { - @Test - public void test(@TempDir Path tempDir) throws IOException { - assertEquals(true, true); - } -} diff --git a/src/test/java/org/embulk/gradle/runset/Util.java b/src/test/java/org/embulk/gradle/runset/Util.java new file mode 100644 index 0000000..4bc3045 --- /dev/null +++ b/src/test/java/org/embulk/gradle/runset/Util.java @@ -0,0 +1,143 @@ +/* + * Copyright 2019 The Embulk project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.embulk.gradle.runset; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.JarURLConnection; +import java.net.URL; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; +import org.gradle.testkit.runner.BuildResult; +import org.gradle.testkit.runner.GradleRunner; + +/** + * Utility methods for testing the Embulk plugins Gradle plugin. + */ +class Util { + private Util() { + // No instantiation. + } + + static Path prepareProjectDir(final Path tempDir, final String testProjectName) { + final String resourceName = testProjectName + System.getProperty("file.separator") + "build.gradle"; + final Path resourceDir; + try { + final URL resourceUrl = Util.class.getClassLoader().getResource(resourceName); + if (resourceUrl == null) { + throw new FileNotFoundException(resourceName + " is not found."); + } + resourceDir = Paths.get(resourceUrl.toURI()).getParent(); + } catch (final Exception ex) { + fail("Failed to find a test resource.", ex); + throw new RuntimeException(ex); // Never reaches. + } + + final Path projectDir; + try { + projectDir = Files.createDirectory(tempDir.resolve(testProjectName)); + } catch (final Exception ex) { + fail("Failed to create a test directory.", ex); + throw new RuntimeException(ex); // Never reaches. + } + + try { + copyFilesRecursively(resourceDir, projectDir); + } catch (final Exception ex) { + fail("Failed to copy test files.", ex); + throw new RuntimeException(ex); // Never reaches. + } + + return projectDir; + } + + private static void copyFilesRecursively(final Path source, final Path destination) throws IOException { + Files.walkFileTree(source, new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttributes attrs) throws IOException { + final Path target = destination.resolve(source.relativize(dir)); + Files.createDirectories(target); + System.out.println(target.toString() + System.getProperty("file.separator")); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { + final Path target = destination.resolve(source.relativize(file)); + Files.copy(file, target); + System.out.println(target); + return FileVisitResult.CONTINUE; + } + }); + } + + static BuildResult runGradle(final Path projectDir, final String... args) { + final ArrayList argsList = new ArrayList<>(); + argsList.addAll(Arrays.asList(args)); + argsList.add("--stacktrace"); + argsList.add("--info"); + final BuildResult result = newGradleRunner(projectDir, argsList).build(); + System.out.println("Running 'gradle " + String.join(" ", argsList) + "' :"); + System.out.println("============================================================"); + System.out.print(result.getOutput()); + System.out.println("============================================================"); + return result; + } + + static void assertFileDoesContain(final Path path, final String expected) throws IOException { + try (final Stream lines = Files.newBufferedReader(path).lines()) { + final boolean found = lines.filter(actualLine -> { + return actualLine.contains(expected); + }).findAny().isPresent(); + if (!found) { + fail("\"" + path.toString() + "\" does not contain \"" + expected + "\"."); + } + } + } + + static void assertFileDoesNotContain(final Path path, final String notExpected) throws IOException { + try (final Stream lines = Files.newBufferedReader(path).lines()) { + lines.forEach(actualLine -> { + assertFalse(actualLine.contains(notExpected)); + }); + } + } + + private static GradleRunner newGradleRunner(final Path projectDir, final List args) { + return GradleRunner.create() + .withProjectDir(projectDir.toFile()) + .withArguments(args) + .withDebug(true) + .withPluginClasspath(); + } + + static JarURLConnection openJarUrlConnection(final Path jarPath) throws IOException { + final URL jarUrl = new URL("jar:" + jarPath.toUri().toURL().toString() + "!/"); + return (JarURLConnection) jarUrl.openConnection(); + } +} diff --git a/src/test/resources/simple/build.gradle b/src/test/resources/simple/build.gradle new file mode 100644 index 0000000..b4fca2d --- /dev/null +++ b/src/test/resources/simple/build.gradle @@ -0,0 +1,13 @@ +plugins { + id "org.embulk.runset" +} + +repositories { + mavenCentral() +} + +installEmbulkRunSet { + into "${project.buildDir}/simple" + artifact "org.embulk:embulk-input-postgresql:0.13.2" + artifact group: "org.embulk", name: "embulk-input-s3", version: "0.6.0" +}