Merge branch 'elastic:main' into main

.buildkite/pipelines/intake.yml

@@ -40,7 +40,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.15", "8.10.4", "8.11.0", "8.12.0"]
+            BWC_VERSION: ["7.17.15", "8.10.5", "8.11.0", "8.12.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1680,6 +1680,22 @@ steps:
         env:
           BWC_VERSION: 8.10.4
 
+      - label: "{{matrix.image}} / 8.10.5 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.10.5
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.5
+
       - label: "{{matrix.image}} / 8.11.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.yml

@@ -1032,6 +1032,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.10.4
+      - label: 8.10.5 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.5#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.5
       - label: 8.11.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest
         timeout_in_minutes: 300

.ci/bwcVersions

@@ -102,5 +102,6 @@ BWC_VERSION:
   - "8.10.2"
   - "8.10.3"
   - "8.10.4"
+  - "8.10.5"
   - "8.11.0"
   - "8.12.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
   - "7.17.15"
-  - "8.10.4"
+  - "8.10.5"
   - "8.11.0"
   - "8.12.0"

build-tools-internal/src/main/groovy/elasticsearch.build-complete.gradle

@@ -13,7 +13,8 @@ import java.nio.file.Files
 String buildNumber = System.getenv('BUILD_NUMBER') ?: System.getenv('BUILDKITE_BUILD_NUMBER')
 String performanceTest = System.getenv('BUILD_PERFORMANCE_TEST')
 if (buildNumber && performanceTest == null && GradleUtils.isIncludedBuild(project) == false) {
-  File uploadFile = file("build/${buildNumber}.tar.bz2")
+  def uploadFilePath = "build/${buildNumber}.tar.bz2"
+  File uploadFile = file(uploadFilePath)
   project.gradle.buildFinished { result ->
     println "build complete, generating: $uploadFile"
     if (uploadFile.exists()) {
@@ -64,5 +65,27 @@ if (buildNumber && performanceTest == null && GradleUtils.isIncludedBuild(projec
     } catch (Exception e) {
       logger.lifecycle("Failed to archive additional logs", e)
     }
+
+    if (uploadFile.exists() && System.getenv("BUILDKITE") == "true") {
+      try {
+        println "Uploading buildkite artifact: ${uploadFilePath}..."
+        new ProcessBuilder("buildkite-agent", "artifact", "upload", uploadFilePath)
+          .start()
+          .waitFor()
+
+        println "Generating buildscan link for artifact..."
+
+        def process = new ProcessBuilder("buildkite-agent", "artifact", "search", uploadFilePath, "--step", System.getenv('BUILDKITE_JOB_ID'), "--format", "%i").start()
+        process.waitFor()
+        def artifactUuid = (process.text ?: "").trim()
+
+        println "Artifact UUID: ${artifactUuid}"
+        if (artifactUuid) {
+          buildScan.link 'Artifact Upload', "https://buildkite.com/organizations/elastic/pipelines/${System.getenv('BUILDKITE_PIPELINE_SLUG')}/builds/${buildNumber}/jobs/${System.getenv('BUILDKITE_JOB_ID')}/artifacts/${artifactUuid}"
+        }
+      } catch (Exception e) {
+        logger.lifecycle("Failed to upload buildkite artifact", e)
+      }
+    }
   }
 }

build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle

@@ -98,10 +98,17 @@ buildScan {
       def branch = System.getenv('BUILDKITE_BRANCH')
       def repoMatcher = System.getenv('BUILDKITE_REPO') =~ /(https:\/\/github\.com\/|git@github\.com:)(\S+)\.git/
       def repository = repoMatcher.matches() ? repoMatcher.group(2) : "<unknown>"
+      def jobName = (System.getenv('BUILDKITE_LABEL') ?: '').replaceAll(/[^a-zA-Z0-9_\-]+/, '_').toLowerCase()
+
       tag 'CI'
       link 'CI Build', buildKiteUrl
       value 'Job Number', System.getenv('BUILDKITE_BUILD_NUMBER')
 
+      value 'Pipeline', System.getenv('BUILDKITE_PIPELINE_SLUG')
+      tag System.getenv('BUILDKITE_PIPELINE_SLUG')
+
+      value 'Job Name', jobName
+      tag jobName
 
       // Add SCM information
       def prId = System.getenv('BUILDKITE_PULL_REQUEST')

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java

@@ -54,6 +54,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         String defaultRouting = request.param("routing");
         String defaultPipeline = request.param("pipeline");
         Boolean defaultRequireAlias = request.paramAsBoolean("require_alias", null);
+        Boolean defaultListExecutedPipelines = request.paramAsBoolean("list_executed_pipelines", null);
 
         String waitForActiveShards = request.param("wait_for_active_shards");
         if (waitForActiveShards != null) {
@@ -68,6 +69,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
             null,
             defaultPipeline,
             defaultRequireAlias,
+            defaultListExecutedPipelines,
             true,
             request.getXContentType(),
             request.getRestApiVersion()

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java

@@ -298,7 +298,7 @@ private static String[] pluginProperties(String name, String[] additionalProps,
                 "version",
                 "1.0",
                 "elasticsearch.version",
-                InstallPluginAction.getSemanticVersion(Build.current().version()),
+                Build.current().version(),
                 "java.version",
                 System.getProperty("java.specification.version")
 
@@ -1110,8 +1110,8 @@ public void testOfficialPluginSnapshot() throws Exception {
         String url = String.format(
             Locale.ROOT,
             "https://snapshots.elastic.co/%s-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-%s.zip",
-            InstallPluginAction.getSemanticVersion(Build.current().version()),
-            Build.current().version()
+            Build.current().version(),
+            Build.current().qualifiedVersion()
         );
         assertInstallPluginFromUrl("analysis-icu", url, "abc123", true);
     }
@@ -1120,8 +1120,8 @@ public void testInstallReleaseBuildOfPluginOnSnapshotBuild() {
         String url = String.format(
             Locale.ROOT,
             "https://snapshots.elastic.co/%s-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-%s.zip",
-            InstallPluginAction.getSemanticVersion(Build.current().version()),
-            Build.current().version()
+            Build.current().version(),
+            Build.current().qualifiedVersion()
         );
         // attempting to install a release build of a plugin (no staging ID) on a snapshot build should throw a user exception
         final UserException e = expectThrows(
@@ -1137,9 +1137,9 @@ public void testInstallReleaseBuildOfPluginOnSnapshotBuild() {
 
     public void testOfficialPluginStaging() throws Exception {
         String url = "https://staging.elastic.co/"
-            + InstallPluginAction.getSemanticVersion(Build.current().version())
-            + "-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-"
             + Build.current().version()
+            + "-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-"
+            + Build.current().qualifiedVersion()
             + ".zip";
         assertInstallPluginFromUrl("analysis-icu", url, "abc123", false);
     }
@@ -1148,7 +1148,7 @@ public void testOfficialPlatformPlugin() throws Exception {
         String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-"
             + Platforms.PLATFORM_NAME
             + "-"
-            + Build.current().version()
+            + Build.current().qualifiedVersion()
             + ".zip";
         assertInstallPluginFromUrl("analysis-icu", url, null, false);
     }
@@ -1157,16 +1157,16 @@ public void testOfficialPlatformPluginSnapshot() throws Exception {
         String url = String.format(
             Locale.ROOT,
             "https://snapshots.elastic.co/%s-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-%s-%s.zip",
-            InstallPluginAction.getSemanticVersion(Build.current().version()),
+            Build.current().version(),
             Platforms.PLATFORM_NAME,
-            Build.current().version()
+            Build.current().qualifiedVersion()
         );
         assertInstallPluginFromUrl("analysis-icu", url, "abc123", true);
     }
 
     public void testOfficialPlatformPluginStaging() throws Exception {
         String url = "https://staging.elastic.co/"
-            + InstallPluginAction.getSemanticVersion(Build.current().version())
+            + Build.current().version()
             + "-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-"
             + Platforms.PLATFORM_NAME
             + "-"

docs/changelog/100031.yaml

@@ -0,0 +1,5 @@
+pr: 100031
+summary: Add executed pipelines to bulk api response
+area: Indices APIs
+type: enhancement
+issues: []

docs/changelog/100033.yaml

@@ -0,0 +1,9 @@
+pr: 100033
+summary: "[Behavioral Analytics] Analytics collections use Data Stream Lifecycle (DSL)\
+  \ instead of Index Lifecycle Management (ILM) for data retention management. Behavioral\
+  \ analytics has traditionally used ILM to manage data retention. Starting with 8.12.0,\
+  \ this will change. Analytics collections created prior to 8.12.0 will continue to use\
+  \ their existing ILM policies, but new analytics collections will be managed using DSL."
+area: Application
+type: feature
+issues: [ ]

docs/changelog/100383.yaml

@@ -0,0 +1,5 @@
+pr: 100383
+summary: Push s3 requests count via metrics API
+area: Distributed
+type: enhancement
+issues: []

docs/changelog/100492.yaml

@@ -0,0 +1,6 @@
+pr: 100492
+summary: Add runtime field of type `geo_shape`
+area: Geo
+type: enhancement
+issues:
+ - 61299

docs/changelog/100779.yaml

@@ -0,0 +1,6 @@
+pr: 100779
+summary: Fix NullPointerException in RotableSecret
+area: Security
+type: bug
+issues:
+ - 99759

docs/changelog/100826.yaml

@@ -0,0 +1,7 @@
+pr: 100826
+summary: Fix geo tile bounding boxes to be consistent with arithmetic method
+area: Geo
+type: bug
+issues:
+ - 92611
+ - 95574

docs/changelog/100828.yaml

@@ -0,0 +1,5 @@
+pr: 100828
+summary: Consider task cancelled exceptions as recoverable
+area: Transform
+type: bug
+issues: []

docs/changelog/100846.yaml

@@ -0,0 +1,6 @@
+pr: 100846
+summary: Consistent scores for multi-term `SourceConfirmedTestQuery`
+area: Search
+type: bug
+issues:
+ - 98712

docs/changelog/100862.yaml

@@ -0,0 +1,5 @@
+pr: 100862
+summary: Sending an index name to `DocumentParsingObserver` that is not ever null
+area: Ingest Node
+type: bug
+issues: []

docs/changelog/100866.yaml

@@ -0,0 +1,6 @@
+pr: 100866
+summary: "ESQL: Preserve intermediate aggregation output in local relation"
+area: ES|QL
+type: bug
+issues:
+ - 100807

docs/changelog/100872.yaml

@@ -0,0 +1,5 @@
+pr: 100872
+summary: Improve painless error wrapping
+area: Infra/Scripting
+type: bug
+issues: []

docs/changelog/100875.yaml

@@ -0,0 +1,6 @@
+pr: 100875
+summary: Preserve subfields for unsupported types
+area: "Query Languages"
+type: bug
+issues:
+ - 100869

docs/changelog/100886.yaml

@@ -0,0 +1,5 @@
+pr: 100886
+summary: Use the correct writable name for model assignment metadata in mixed version clusters. Prevents a node failure due to IllegalArgumentException Unknown NamedWriteable [trained_model_assignment]
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/100911.yaml

@@ -0,0 +1,6 @@
+pr: 100911
+summary: '`WaitForSnapshotStep` verifies if the index belongs to the latest snapshot
+  of that SLM policy'
+area: ILM+SLM
+type: bug
+issues: []

docs/changelog/100986.yaml

@@ -0,0 +1,6 @@
+pr: 100986
+summary: Synchronize Coordinator#onClusterStateApplied
+area: Cluster Coordination
+type: bug
+issues:
+ - 99023

docs/changelog/101001.yaml

@@ -0,0 +1,6 @@
+pr: 101001
+summary: "ESQL: Support date and time intervals as input params"
+area: ES|QL
+type: bug
+issues:
+ - 99570

docs/changelog/101032.yaml

@@ -0,0 +1,5 @@
+pr: 101032
+summary: Throw when wrapping rate agg in `DeferableBucketAggregator`
+area: TSDB
+type: bug
+issues: []

docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc

@@ -138,7 +138,7 @@ To apply synonyms, you will need to include a synonym graph token filter into an
         "my_analyzer": {
           "type": "custom",
           "tokenizer": "standard",
-          "filter": ["lowercase", "synonym_graph"]
+          "filter": ["stemmer", "synonym_graph"]
         }
       }
 ----
@@ -150,8 +150,8 @@ To apply synonyms, you will need to include a synonym graph token filter into an
 Order is important for your token filters.
 Text will be processed first through filters preceding the synonym filter before being processed by the synonym filter.
 
-In the above example, text will be lowercased by the `lowercase` filter before being processed by the `synonyms_filter`.
-This means that all the synonyms defined there needs to be in lowercase, or they won't be found by the synonyms filter.
+{es} will also use the token filters preceding the synonym filter in a tokenizer chain to parse the entries in a synonym file or synonym set.
+In the above example, the synonyms graph token filter is placed after a stemmer. The stemmer will also be applied to the synonym entries.
 
 The synonym rules should not contain words that are removed by a filter that appears later in the chain (like a `stop` filter).
 Removing a term from a synonym rule means there will be no matching for it at query time.

docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc

@@ -128,7 +128,7 @@ To apply synonyms, you will need to include a synonym token filters into an anal
         "my_analyzer": {
           "type": "custom",
           "tokenizer": "standard",
-          "filter": ["lowercase", "synonym"]
+          "filter": ["stemmer", "synonym_graph"]
         }
       }
 ----
@@ -140,8 +140,8 @@ To apply synonyms, you will need to include a synonym token filters into an anal
 Order is important for your token filters.
 Text will be processed first through filters preceding the synonym filter before being processed by the synonym filter.
 
-In the above example, text will be lowercased by the `lowercase` filter before being processed by the `synonyms_filter`.
-This means that all the synonyms defined there needs to be in lowercase, or they won't be found by the synonyms filter.
+{es} will also use the token filters preceding the synonym filter in a tokenizer chain to parse the entries in a synonym file or synonym set.
+In the above example, the synonyms graph token filter is placed after a stemmer. The stemmer will also be applied to the synonym entries.
 
 The synonym rules should not contain words that are removed by a filter that appears later in the chain (like a `stop` filter).
 Removing a term from a synonym rule means there will be no matching for it at query time.