Merge pull request #1 from embulk/initial-work

Initial work to "install" Maven-based Embulk plugins out of Embulk commands, to replace Bundler

.github/CODEOWNERS

@@ -0,0 +1 @@
+*	@embulk/core-team

src/main/java/org/embulk/gradle/runset/EmbulkRunSetPlugin.java

@@ -25,5 +25,6 @@
 public class EmbulkRunSetPlugin implements Plugin<Project> {
     @Override
     public void apply(final Project project) {
+        project.getTasks().register("installEmbulkRunSet", InstallEmbulkRunSet.class);
     }
 }

src/main/java/org/embulk/gradle/runset/InstallEmbulkRunSet.java

@@ -0,0 +1,199 @@
+/*
+ * Copyright 2023 The Embulk project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.embulk.gradle.runset;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import org.gradle.api.IllegalDependencyNotation;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.ArtifactCollection;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.artifacts.ResolvableDependencies;
+import org.gradle.api.artifacts.component.ComponentIdentifier;
+import org.gradle.api.artifacts.component.ModuleComponentIdentifier;
+import org.gradle.api.artifacts.component.ProjectComponentIdentifier;
+import org.gradle.api.artifacts.result.ArtifactResolutionResult;
+import org.gradle.api.artifacts.result.ArtifactResult;
+import org.gradle.api.artifacts.result.ComponentArtifactsResult;
+import org.gradle.api.artifacts.result.ResolvedArtifactResult;
+import org.gradle.api.file.DuplicatesStrategy;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.tasks.Copy;
+import org.gradle.maven.MavenModule;
+import org.gradle.maven.MavenPomArtifact;
+
+/**
+ * A Gradle Task to set up an environment for running Embulk.
+ */
+public class InstallEmbulkRunSet extends Copy {
+    public InstallEmbulkRunSet() {
+        super();
+
+        this.project = this.getProject();
+        this.logger = this.project.getLogger();
+
+        final ObjectFactory objectFactory = this.project.getObjects();
+    }
+
+    /**
+     * Adds a Maven artifact to be installed.
+     *
+     * <p>It tries to simulate Gradle's dependency notations, but it is yet far from perfect.
+     *
+     * @see <a href="https://github.com/gradle/gradle/blob/v8.4.0/subprojects/dependency-management/src/main/java/org/gradle/api/internal/notations/DependencyNotationParser.java#L49-L86">org.gradle.api.internal.notations.DependencyNotationParser#create</a>
+     */
+    public void artifact(final Object dependencyNotation) {
+        final Dependency dependency;
+        if (dependencyNotation instanceof CharSequence) {
+            dependency = this.dependencyFromCharSequence((CharSequence) dependencyNotation);
+        } else if (dependencyNotation instanceof Map) {
+            dependency = this.dependencyFromMap((Map) dependencyNotation);
+        } else {
+            throw new IllegalDependencyNotation("Supplied module notation is invalid.");
+        }
+
+        // Constructing an independent (detached) Configuration so that its dependencies are not affected by other plugins.
+        final Configuration configuration = this.project.getConfigurations().detachedConfiguration(dependency);
+
+        final ResolvableDependencies resolvableDependencies = configuration.getIncoming();
+        final ArtifactCollection artifactCollection = resolvableDependencies.getArtifacts();
+
+        // Getting the JAR files and component IDs.
+        final ArrayList<ComponentIdentifier> componentIds = new ArrayList<>();
+        for (final ResolvedArtifactResult resolvedArtifactResult : artifactCollection.getArtifacts()) {
+            componentIds.add(resolvedArtifactResult.getId().getComponentIdentifier());
+            this.fromArtifact(resolvedArtifactResult, "jar");
+        }
+
+        // Getting the POM files.
+        final ArtifactResolutionResult artifactResolutionResult = this.project.getDependencies()
+                .createArtifactResolutionQuery()
+                .forComponents(componentIds)
+                .withArtifacts(MavenModule.class, MavenPomArtifact.class)
+                .execute();
+        for (final ComponentArtifactsResult componentArtifactResult : artifactResolutionResult.getResolvedComponents()) {
+            for (final ArtifactResult artifactResult : componentArtifactResult.getArtifacts(MavenPomArtifact.class)) {
+                if (artifactResult instanceof ResolvedArtifactResult) {
+                    final ResolvedArtifactResult resolvedArtifactResult = (ResolvedArtifactResult) artifactResult;
+                    this.fromArtifact(resolvedArtifactResult, "pom");
+                }
+            }
+        }
+    }
+
+    private void fromArtifact(final ResolvedArtifactResult resolvedArtifactResult, final String artifactType) {
+        final ComponentIdentifier id = resolvedArtifactResult.getId().getComponentIdentifier();
+        final File file = resolvedArtifactResult.getFile();
+
+        if (id instanceof ModuleComponentIdentifier) {
+            final Path modulePath = moduleToPath((ModuleComponentIdentifier) id);
+            this.logger.info("Setting to copy {}:{} into {}", id, artifactType, modulePath);
+            this.logger.debug("Cached file: {}", file);
+            this.from(file, copy -> {
+                copy.into(modulePath.toFile());
+                copy.setDuplicatesStrategy(DuplicatesStrategy.EXCLUDE);
+            });
+        } else if (id instanceof ProjectComponentIdentifier) {
+            throw new IllegalDependencyNotation("Cannot install artifacts for a project component (" + id.getDisplayName() + ")");
+        } else {
+            throw new IllegalDependencyNotation(
+                    "Cannot resolve the artifacts for component "
+                    + id.getDisplayName()
+                    + " with unsupported type "
+                    + id.getClass().getName()
+                    + ".");
+        }
+    }
+
+    private static Path moduleToPath(final ModuleComponentIdentifier id) {
+        final String[] splitGroup = id.getGroup().split("\\.");
+        if (splitGroup.length <= 0) {
+            return Paths.get("");
+        }
+
+        final String[] more = new String[splitGroup.length + 2 - 1];
+        for (int i = 1; i < splitGroup.length; i++) {
+            more[i - 1] = splitGroup[i];
+        }
+        more[splitGroup.length - 1] = id.getModule();
+        more[splitGroup.length] = id.getVersion();
+        final Path path = Paths.get(splitGroup[0], more);
+        assert !path.isAbsolute();
+        return path;
+    }
+
+    // https://github.com/gradle/gradle/blob/v8.4.0/subprojects/dependency-management/src/main/java/org/gradle/api/internal/notations/DependencyStringNotationConverter.java
+    private Dependency dependencyFromCharSequence(final CharSequence dependencyNotation) {
+        final String notationString = dependencyNotation.toString();
+        this.logger.debug("Artifact: {}", notationString);
+        return this.project.getDependencies().create(notationString);
+    }
+
+    // https://github.com/gradle/gradle/blob/v8.4.0/subprojects/core/src/main/java/org/gradle/internal/typeconversion/MapNotationConverter.java
+    private Dependency dependencyFromMap(final Map dependencyNotation) {
+        final Map<String, String> notationMap = validateMap(dependencyNotation);
+        this.logger.debug("Artifact: {}", notationMap);
+        return this.project.getDependencies().create(notationMap);
+    }
+
+    private static Map<String, String> validateMap(final Map dependencyNotation) {
+        final LinkedHashMap<String, String> map = new LinkedHashMap<>();
+        for (final Map.Entry<Object, Object> entry : castMap(dependencyNotation).entrySet()) {
+            final Object keyObject = entry.getKey();
+            if (!(keyObject instanceof CharSequence)) {
+                throw new IllegalDependencyNotation("Supplied Map module notation is invalid. Its key must be a String.");
+            }
+            final String key = (String) keyObject;
+            if (!ACCEPTABLE_MAP_KEYS.contains(key)) {
+                throw new IllegalDependencyNotation(
+                        "Supplied Map module notation is invalid. Its key must be one of: ["
+                        + String.join(", ", ACCEPTABLE_MAP_KEYS)
+                        + "]");
+            }
+
+            final Object valueObject = entry.getValue();
+            if (!(valueObject instanceof CharSequence)) {
+                throw new IllegalDependencyNotation("Supplied Map module notation is invalid. Its value must be a String.");
+            }
+            final String value = (String) valueObject;
+            map.put(key, value);
+        }
+        return Collections.unmodifiableMap(map);
+    }
+
+    @SuppressWarnings("unchecked")
+    private static Map<Object, Object> castMap(final Map map) {
+        return (Map<Object, Object>) map;
+    }
+
+    // https://github.com/gradle/gradle/blob/v8.4.0/subprojects/dependency-management/src/main/java/org/gradle/api/internal/notations/DependencyMapNotationConverter.java#L42-L58
+    private static List<String> ACCEPTABLE_MAP_KEYS =
+            Arrays.asList("group", "name", "version", "configuration", "ext", "classifier");
+
+    private final Logger logger;
+
+    private final Project project;
+}

src/test/java/org/embulk/gradle/runset/TestEmbulkRunSetPlugin.java

@@ -16,16 +16,31 @@
 
 package org.embulk.gradle.runset;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.embulk.gradle.runset.Util.prepareProjectDir;
+import static org.embulk.gradle.runset.Util.runGradle;
 
 import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 
 public class TestEmbulkRunSetPlugin  {
     @Test
-    public void test(@TempDir Path tempDir) throws IOException {
-        assertEquals(true, true);
+    public void testSimple(@TempDir Path tempDir) throws IOException {
+        final Path projectDir = prepareProjectDir(tempDir, "simple");
+
+        runGradle(projectDir, "installEmbulkRunSet");
+
+        Files.walkFileTree(projectDir.resolve("build/simple"), new SimpleFileVisitor<Path>() {
+                @Override
+                public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
+                    System.out.println(projectDir.relativize(file));
+                    return FileVisitResult.CONTINUE;
+                }
+            });
     }
 }

src/test/java/org/embulk/gradle/runset/Util.java

@@ -0,0 +1,143 @@
+/*
+ * Copyright 2019 The Embulk project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.embulk.gradle.runset;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.fail;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.JarURLConnection;
+import java.net.URL;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Stream;
+import org.gradle.testkit.runner.BuildResult;
+import org.gradle.testkit.runner.GradleRunner;
+
+/**
+ * Utility methods for testing the Embulk plugins Gradle plugin.
+ */
+class Util {
+    private Util() {
+        // No instantiation.
+    }
+
+    static Path prepareProjectDir(final Path tempDir, final String testProjectName) {
+        final String resourceName = testProjectName + System.getProperty("file.separator") + "build.gradle";
+        final Path resourceDir;
+        try {
+            final URL resourceUrl = Util.class.getClassLoader().getResource(resourceName);
+            if (resourceUrl == null) {
+                throw new FileNotFoundException(resourceName + " is not found.");
+            }
+            resourceDir = Paths.get(resourceUrl.toURI()).getParent();
+        } catch (final Exception ex) {
+            fail("Failed to find a test resource.", ex);
+            throw new RuntimeException(ex);  // Never reaches.
+        }
+
+        final Path projectDir;
+        try {
+            projectDir = Files.createDirectory(tempDir.resolve(testProjectName));
+        } catch (final Exception ex) {
+            fail("Failed to create a test directory.", ex);
+            throw new RuntimeException(ex);  // Never reaches.
+        }
+
+        try {
+            copyFilesRecursively(resourceDir, projectDir);
+        } catch (final Exception ex) {
+            fail("Failed to copy test files.", ex);
+            throw new RuntimeException(ex);  // Never reaches.
+        }
+
+        return projectDir;
+    }
+
+    private static void copyFilesRecursively(final Path source, final Path destination) throws IOException {
+        Files.walkFileTree(source, new SimpleFileVisitor<Path>() {
+            @Override
+            public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttributes attrs) throws IOException {
+                final Path target = destination.resolve(source.relativize(dir));
+                Files.createDirectories(target);
+                System.out.println(target.toString() + System.getProperty("file.separator"));
+                return FileVisitResult.CONTINUE;
+            }
+
+            @Override
+            public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
+                final Path target = destination.resolve(source.relativize(file));
+                Files.copy(file, target);
+                System.out.println(target);
+                return FileVisitResult.CONTINUE;
+            }
+        });
+    }
+
+    static BuildResult runGradle(final Path projectDir, final String... args) {
+        final ArrayList<String> argsList = new ArrayList<>();
+        argsList.addAll(Arrays.asList(args));
+        argsList.add("--stacktrace");
+        argsList.add("--info");
+        final BuildResult result = newGradleRunner(projectDir, argsList).build();
+        System.out.println("Running 'gradle " + String.join(" ", argsList) + "' :");
+        System.out.println("============================================================");
+        System.out.print(result.getOutput());
+        System.out.println("============================================================");
+        return result;
+    }
+
+    static void assertFileDoesContain(final Path path, final String expected) throws IOException {
+        try (final Stream<String> lines = Files.newBufferedReader(path).lines()) {
+            final boolean found = lines.filter(actualLine -> {
+                return actualLine.contains(expected);
+            }).findAny().isPresent();
+            if (!found) {
+                fail("\"" + path.toString() + "\" does not contain \"" + expected + "\".");
+            }
+        }
+    }
+
+    static void assertFileDoesNotContain(final Path path, final String notExpected) throws IOException {
+        try (final Stream<String> lines = Files.newBufferedReader(path).lines()) {
+            lines.forEach(actualLine -> {
+                assertFalse(actualLine.contains(notExpected));
+            });
+        }
+    }
+
+    private static GradleRunner newGradleRunner(final Path projectDir, final List<String> args) {
+        return GradleRunner.create()
+                .withProjectDir(projectDir.toFile())
+                .withArguments(args)
+                .withDebug(true)
+                .withPluginClasspath();
+    }
+
+    static JarURLConnection openJarUrlConnection(final Path jarPath) throws IOException {
+        final URL jarUrl = new URL("jar:" + jarPath.toUri().toURL().toString() + "!/");
+        return (JarURLConnection) jarUrl.openConnection();
+    }
+}