Backport: Onboarding New templates and adding zstd support (#8)

build.gradle

@@ -31,7 +31,7 @@
 buildscript {
     ext {
         opensearch_group = "org.opensearch"
-        opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "2.17.0-SNAPSHOT")
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
     }
@@ -67,6 +67,10 @@ repositories {
     maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
 }
 
+configurations {
+    zipArchive
+}
+
 allprojects {
     group 'org.opensearch'
     version = opensearch_version.tokenize('-')[0] + '.0'
@@ -92,6 +96,7 @@ opensearchplugin {
 }
 
 dependencies {
+    zipArchive group: 'org.opensearch.plugin', name:'opensearch-custom-codecs', version: "2.17.0.0-SNAPSHOT"
 }
 
 allprojects {
@@ -148,6 +153,7 @@ publishing {
                 password "$System.env.SONATYPE_PASSWORD"
             }
         }
+        mavenLocal()
     }
 }
 
@@ -202,14 +208,27 @@ integTest {
     dependsOn "bundlePlugin"
     systemProperty 'tests.security.manager', 'true'
 
+
     systemProperty "https", System.getProperty("https")
     systemProperty "user", System.getProperty("user")
     systemProperty "password", System.getProperty("password")
 }
 
 testClusters.integTest {
-    testDistribution = "ARCHIVE"
+    testDistribution = "INTEG_TEST"
     plugin(project.tasks.bundlePlugin.archiveFile)
+    plugin(provider({
+        new RegularFile() {
+            @Override
+            File getAsFile() {
+                return configurations.zipArchive.asFileTree.matching {
+                    include '**/opensearch-custom-codecs*'
+                }.singleFile
+            }
+        }
+    }))
+    systemProperty 'opensearch.experimental.feature.application_templates.enabled', 'true'
+    setting 'cluster.application_templates.enabled', 'true'
 }
 
 tasks.withType(PublishToMavenRepository) {

src/integTest/java/org/opensearch/system/applicationtemplates/CreateIndexTemplateWithContextTemplateIT.java

@@ -20,10 +20,14 @@
 import org.apache.hc.core5.http.nio.ssl.TlsStrategy;
 import org.apache.hc.core5.reactor.ssl.TlsDetails;
 import org.apache.hc.core5.ssl.SSLContextBuilder;
+import org.opensearch.client.Request;
 import org.opensearch.client.RestClient;
 import org.opensearch.client.RestClientBuilder;
+import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.Strings;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.engine.EngineConfig;
 import org.opensearch.test.rest.OpenSearchRestTestCase;
 
 import javax.net.ssl.SSLEngine;
@@ -32,15 +36,50 @@
 import java.security.KeyManagementException;
 import java.security.KeyStoreException;
 import java.security.NoSuchAlgorithmException;
+import java.util.Map;
 import java.util.Objects;
 
 import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_PER_ROUTE;
 import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_TOTAL;
 
 public class CreateIndexTemplateWithContextTemplateIT extends OpenSearchRestTestCase {
 
-    public void testCreateIndexWithContextBasedTemplate() throws IOException {
-        // TODO: Add E2E test with rest layer here.
+    @SuppressWarnings("unchecked")
+    public void testCreateIndexWithContextBasedTemplate() throws Exception {
+
+        final String indexTemplate = "my-metrics-template";
+        final String index = "my-metrics-1";
+
+        Request request = new Request("PUT", "/_index_template/" + indexTemplate);
+        String content = "{\n"
+            + "    \"index_patterns\": [\n"
+            + "        \"my-metrics-*\"\n"
+            + "    ],\n"
+            + "    \"context\": {\n"
+            + "        \"name\": \"metrics\"\n"
+            + "    }\n"
+            + "}";
+
+        request.setJsonEntity(content);
+
+        // Create index template
+        client().performRequest(request);
+
+        // creating index
+        createIndex(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build());
+
+        Map<String, Object> currentIndexSettings = (Map<String, Object>) ((Map<String, Object>) getIndexSettings(index).get(index)).get(
+            "settings"
+        );
+        assertEquals(currentIndexSettings.get(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey()), "60s");
+        assertEquals(currentIndexSettings.get(IndexSettings.INDEX_MERGE_POLICY.getKey()), "log_byte_size");
+        assertEquals(currentIndexSettings.get(EngineConfig.INDEX_CODEC_SETTING.getKey()), "zstd_no_dict");
+
+        try {
+            ensureGreen(index);
+        } finally {
+            deleteIndex(index);
+        }
     }
 
     @Override

src/main/java/org/opensearch/system/applicationtemplates/LocalSystemTemplateRepository.java

@@ -14,7 +14,6 @@
 import org.opensearch.cluster.applicationtemplates.SystemTemplateMetadata;
 import org.opensearch.cluster.applicationtemplates.SystemTemplateRepository;
 import org.opensearch.cluster.applicationtemplates.TemplateRepositoryMetadata;
-import org.opensearch.common.io.PathUtils;
 import org.opensearch.common.util.io.Streams;
 import org.opensearch.common.xcontent.json.JsonXContent;
 import org.opensearch.core.common.bytes.BytesArray;
@@ -131,7 +130,7 @@ public SystemTemplate getTemplate(SystemTemplateMetadata templateMetadata) throw
     }
 
     static String buildFileName(SystemTemplateMetadata templateMetadata) {
-        return "v" + templateMetadata.version() + PathUtils.getDefaultFileSystem().getSeparator() + templateMetadata.name() + ".json";
+        return "v" + templateMetadata.version() + "/" + templateMetadata.name() + ".json";
     }
 
     // Visible for testing (if we need UTs with mocked resources)

src/main/resources/org/opensearch/system/applicationtemplates/templates.json

@@ -1,15 +1,45 @@
 {
   "repository_schema_version": 1,
   "templates": [
-      {
-        "name": "logs",
-        "version": 1,
-        "type": "@abc_template"
-      },
-      {
-        "name": "metrics",
-        "version": 1,
-        "type": "@abc_template"
-      }
+    {
+      "name": "logs",
+      "version": 1,
+      "type": "@abc_template"
+    },
+    {
+      "name": "metrics",
+      "version": 1,
+      "type": "@abc_template"
+    },
+    {
+      "name": "apache-web-logs",
+      "version": 1,
+      "type": "@abc_template"
+    },
+    {
+      "name": "amazon-cloudtrail-logs",
+      "version": 1,
+      "type": "@abc_template"
+    },
+    {
+      "name": "amazon-elb-logs",
+      "version": 1,
+      "type": "@abc_template"
+    },
+    {
+      "name": "amazon-s3-logs",
+      "version": 1,
+      "type": "@abc_template"
+    },
+    {
+      "name": "nginx-logs",
+      "version": 1,
+      "type": "@abc_template"
+    },
+    {
+      "name": "k8s-logs",
+      "version": 1,
+      "type": "@abc_template"
+    }
   ]
 }