Merge branch 'elastic:main' into main

.ci/jobs.t/elastic+elasticsearch+periodic+packaging-tests.yml

@@ -2,7 +2,8 @@
 - job:
     name: elastic+elasticsearch+%BRANCH%+periodic+packaging-tests
     display-name: "elastic / elasticsearch # %BRANCH% - packaging tests"
-    description: "Testing of the Elasticsearch %BRANCH% branch packaging tests.\n"
+    description: "This job has been migrated to Buildkite.\n"
+    disabled: true
     project-type: multijob
     node: master
     vault: []

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java

@@ -17,17 +17,20 @@
 import org.gradle.api.GradleException;
 import org.gradle.api.Project;
 import org.gradle.api.logging.LogLevel;
+import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.TaskProvider;
+import org.gradle.jvm.toolchain.JavaLanguageVersion;
+import org.gradle.jvm.toolchain.JavaToolchainService;
+import org.gradle.jvm.toolchain.JvmVendorSpec;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
 
-import static org.elasticsearch.gradle.internal.util.JavaUtil.getJavaHome;
-
 /**
  * By registering bwc tasks via this extension we can support declaring custom bwc tasks from the build script
  * without relying on groovy closures and sharing common logic for tasks created by the BwcSetup plugin already.
@@ -37,16 +40,22 @@ public class BwcSetupExtension {
     private static final String MINIMUM_COMPILER_VERSION_PATH = "src/main/resources/minimumCompilerVersion";
     private static final Version BUILD_TOOL_MINIMUM_VERSION = Version.fromString("7.14.0");
     private final Project project;
+    private final ObjectFactory objectFactory;
+    private final JavaToolchainService toolChainService;
     private final Provider<BwcVersions.UnreleasedVersionInfo> unreleasedVersionInfo;
 
     private Provider<File> checkoutDir;
 
     public BwcSetupExtension(
         Project project,
+        ObjectFactory objectFactory,
+        JavaToolchainService toolChainService,
         Provider<BwcVersions.UnreleasedVersionInfo> unreleasedVersionInfo,
         Provider<File> checkoutDir
     ) {
         this.project = project;
+        this.objectFactory = objectFactory;
+        this.toolChainService = toolChainService;
         this.unreleasedVersionInfo = unreleasedVersionInfo;
         this.checkoutDir = checkoutDir;
     }
@@ -137,4 +146,14 @@ private static String readFromFile(File file) {
             throw new GradleException("Cannot read java properties file.", ioException);
         }
     }
+
+    /** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */
+    public String getJavaHome(final int version) {
+        Property<JavaLanguageVersion> value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version));
+        return toolChainService.launcherFor(javaToolchainSpec -> {
+            javaToolchainSpec.getLanguageVersion().value(value);
+            javaToolchainSpec.getVendor().set(JvmVendorSpec.ORACLE);
+        }).get().getMetadata().getInstallationPath().getAsFile().getAbsolutePath();
+    }
+
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/GenerateProviderManifest.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal;
+
+import org.elasticsearch.gradle.util.FileUtils;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.file.ConfigurableFileCollection;
+import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.tasks.Classpath;
+import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.TaskAction;
+
+import java.io.File;
+import java.util.stream.Collectors;
+
+import javax.inject.Inject;
+
+abstract class GenerateProviderManifest extends DefaultTask {
+
+    @Inject
+    public GenerateProviderManifest() {}
+
+    @Classpath
+    @InputFiles
+    abstract public ConfigurableFileCollection getProviderImplClasspath();
+
+    @OutputFile
+    abstract RegularFileProperty getManifestFile();
+
+    @TaskAction
+    void generateManifest() {
+        File manifestFile = getManifestFile().get().getAsFile();
+        manifestFile.getParentFile().mkdirs();
+        FileUtils.write(manifestFile, generateManifestContent(), "UTF-8");
+    }
+
+    private String generateManifestContent() {
+        return getProviderImplClasspath().getFiles().stream().map(File::getName).sorted().collect(Collectors.joining("\n"));
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java

@@ -16,10 +16,13 @@
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
+import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.plugins.JvmToolchainsPlugin;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.Copy;
 import org.gradle.api.tasks.TaskProvider;
+import org.gradle.jvm.toolchain.JavaToolchainService;
 import org.gradle.language.base.plugins.LifecycleBasePlugin;
 
 import java.io.File;
@@ -44,16 +47,21 @@
  */
 public class InternalDistributionBwcSetupPlugin implements Plugin<Project> {
 
+    private final ObjectFactory objectFactory;
     private ProviderFactory providerFactory;
+    private JavaToolchainService toolChainService;
 
     @Inject
-    public InternalDistributionBwcSetupPlugin(ProviderFactory providerFactory) {
+    public InternalDistributionBwcSetupPlugin(ObjectFactory objectFactory, ProviderFactory providerFactory) {
+        this.objectFactory = objectFactory;
         this.providerFactory = providerFactory;
     }
 
     @Override
     public void apply(Project project) {
         project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
+        project.getPlugins().apply(JvmToolchainsPlugin.class);
+        toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
         BuildParams.getBwcVersions().forPreviousUnreleased((BwcVersions.UnreleasedVersionInfo unreleasedVersion) -> {
             configureBwcProject(project.project(unreleasedVersion.gradleProjectPath()), unreleasedVersion);
         });
@@ -63,7 +71,7 @@ private void configureBwcProject(Project project, BwcVersions.UnreleasedVersionI
         Provider<BwcVersions.UnreleasedVersionInfo> versionInfoProvider = providerFactory.provider(() -> versionInfo);
         Provider<File> checkoutDir = versionInfoProvider.map(info -> new File(project.getBuildDir(), "bwc/checkout-" + info.branch()));
         BwcSetupExtension bwcSetupExtension = project.getExtensions()
-            .create("bwcSetup", BwcSetupExtension.class, project, versionInfoProvider, checkoutDir);
+            .create("bwcSetup", BwcSetupExtension.class, project, objectFactory, toolChainService, versionInfoProvider, checkoutDir);
         BwcGitExtension gitExtension = project.getPlugins().apply(InternalBwcGitPlugin.class).getGitExtension();
         Provider<Version> bwcVersion = versionInfoProvider.map(info -> info.version());
         gitExtension.setBwcVersion(versionInfoProvider.map(info -> info.version()));

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java

@@ -29,7 +29,7 @@ public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin {
     public TaskProvider<? extends Task> createTask(Project project) {
         project.getPlugins().apply(CompileOnlyResolvePlugin.class);
         project.getConfigurations().create("forbiddenApisCliJar");
-        project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:3.5.1");
+        project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:3.6");
         Configuration jdkJarHellConfig = project.getConfigurations().create(JDK_JAR_HELL_CONFIG_NAME);
         if (project.getPath().equals(LIBS_ELASTICSEARCH_CORE_PROJECT_PATH) == false) {
             // Internal projects are not all plugins, so make sure the check is available

build-tools-internal/version.properties

@@ -2,8 +2,7 @@ elasticsearch     = 8.12.0
 lucene            = 9.8.0
 
 bundled_jdk_vendor = openjdk
-bundled_jdk = 21+35@fd2272bbf8e04c3dbaee13770090416c
-
+bundled_jdk = 21.0.1+12@415e3f918a1f4062a0074a2794853d0d
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0

docs/changelog/101026.yaml

@@ -0,0 +1,5 @@
+pr: 101026
+summary: Remove `auto_configure` privilege for profiling
+area: Authorization
+type: enhancement
+issues: []

docs/changelog/101120.yaml

@@ -0,0 +1,6 @@
+pr: 101120
+summary: "ESQL: Fix escaping of backslash in LIKE operator"
+area: ES|QL
+type: bug
+issues:
+ - 101106

docs/changelog/101126.yaml

@@ -0,0 +1,5 @@
+pr: 101126
+summary: Include totals in flamegraph response
+area: Application
+type: enhancement
+issues: []

docs/changelog/101133.yaml

@@ -0,0 +1,5 @@
+pr: 101133
+summary: Update bundled JDK to 21.0.1
+area: Packaging
+type: upgrade
+issues: []

docs/changelog/101184.yaml

@@ -0,0 +1,6 @@
+pr: 101184
+summary: More robust timeout for repo analysis
+area: Snapshot/Restore
+type: bug
+issues:
+ - 101182

docs/changelog/101185.yaml

@@ -0,0 +1,5 @@
+pr: 101185
+summary: Repo analysis of uncontended register behaviour
+area: Snapshot/Restore
+type: enhancement
+issues: []

docs/changelog/101205.yaml

@@ -0,0 +1,5 @@
+pr: 101205
+summary: Increase K/V look-back time interval
+area: Application
+type: bug
+issues: []

docs/changelog/101212.yaml

@@ -0,0 +1,6 @@
+pr: 101212
+summary: Fix painless execute api and tsdb issue
+area: TSDB
+type: bug
+issues:
+ - 101072

docs/reference/esql/esql-commands.asciidoc

@@ -1,17 +1,63 @@
 [[esql-commands]]
-== {esql} commands
+=== {esql} commands
 
 ++++
 <titleabbrev>Commands</titleabbrev>
 ++++
 
-{esql} provides a comprehensive set of source and processing commands:
+// tag::source_commands[]
+==== Source commands
 
-<<esql-source-commands>>::
-include::source-commands/esql-source-commands.asciidoc[tag=list]
+An {esql} source command produces a table, typically with data from {es}.
 
-<<esql-processing-commands>>::
-include::processing-commands/esql-processing-commands.asciidoc[tag=list]
+image::images/esql/source-command.svg[A source command producing a table from {es},align="center"]
 
-include::source-commands/esql-source-commands.asciidoc[]
-include::processing-commands/esql-processing-commands.asciidoc[]
+{esql} supports these source commands:
+
+* <<esql-from>>
+* <<esql-row>>
+* <<esql-show>>
+
+include::source-commands/from.asciidoc[]
+include::source-commands/row.asciidoc[]
+include::source-commands/show.asciidoc[]
+
+// end::source_command[]
+
+// tag::proc_commands[]
+==== Processing commands
+
+{esql} processing commands change an input table by adding, removing, or changing
+rows and columns.
+
+image::images/esql/processing-command.svg[A processing command changing an input table,align="center"]
+
+{esql} supports these processing commands:
+
+* <<esql-dissect>>
+* <<esql-drop>>
+* <<esql-enrich>>
+* <<esql-eval>>
+* <<esql-grok>>
+* <<esql-keep>>
+* <<esql-limit>>
+* <<esql-mv_expand>>
+* <<esql-rename>>
+* <<esql-sort>>
+* <<esql-stats-by>>
+* <<esql-where>>
+
+include::processing-commands/dissect.asciidoc[]
+include::processing-commands/drop.asciidoc[]
+include::processing-commands/enrich.asciidoc[]
+include::processing-commands/eval.asciidoc[]
+include::processing-commands/grok.asciidoc[]
+include::processing-commands/keep.asciidoc[]
+include::processing-commands/limit.asciidoc[]
+include::processing-commands/mv_expand.asciidoc[]
+include::processing-commands/rename.asciidoc[]
+include::processing-commands/sort.asciidoc[]
+include::processing-commands/stats.asciidoc[]
+include::processing-commands/where.asciidoc[]
+
+// end::proc_command[]