diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..00ca820 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @ankitkala @mgodwan @backslasht @shwetathareja diff --git a/.github/workflows/add-untriaged.yml b/.github/workflows/add-untriaged.yml new file mode 100644 index 0000000..15b9a55 --- /dev/null +++ b/.github/workflows/add-untriaged.yml @@ -0,0 +1,19 @@ +name: Apply 'untriaged' label during issue lifecycle + +on: + issues: + types: [opened, reopened, transferred] + +jobs: + apply-label: + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@v6 + with: + script: | + github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ['untriaged'] + }) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml new file mode 100644 index 0000000..607c6de --- /dev/null +++ b/.github/workflows/backport.yml @@ -0,0 +1,40 @@ +name: Backport +on: + pull_request_target: + types: + - closed + - labeled + +jobs: + backport: + name: Backport + runs-on: ubuntu-latest + # Only react to merged PRs for security reasons. + # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target. + if: > + github.event.pull_request.merged + && ( + github.event.action == 'closed' + || ( + github.event.action == 'labeled' + && contains(github.event.label.name, 'backport') + ) + ) + permissions: + contents: write + pull-requests: write + steps: + - name: GitHub App token + id: github_app_token + uses: tibdex/github-app-token@v1.5.0 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + installation_id: 22958780 + + - name: Backport + uses: VachaShah/backport@v2.2.0 + with: + github_token: ${{ steps.github_app_token.outputs.token }} + head_template: backport/backport-<%= number %>-to-<%= base %> + failure_labels: backport-failed diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 0000000..ff27aa5 --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,59 @@ +name: Gradle Check +on: [pull_request] + +jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + precommit-linux: + needs: Get-CI-Image-Tag + strategy: + matrix: + java: [ 21 ] + if: github.repository == 'opensearch-project/opensearch-system-templates' + runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + + steps: + - uses: actions/checkout@v3 + - name: Run Gradle (check) + run: | + # https://github.com/opensearch-project/opensearch-build/issues/4191 + chown -R ci-runner:ci-runner `pwd` + su ci-runner -c "source /etc/profile.d/java_home.sh && ./gradlew check -Dorg.gradle.java.home=/opt/java/openjdk-${{ matrix.java }}" + - name: Run Gradle (assemble) + run: | + # https://github.com/opensearch-project/opensearch-build/issues/4191 + chown -R ci-runner:ci-runner `pwd` + su ci-runner -c "source /etc/profile.d/java_home.sh && ./gradlew assemble -Dorg.gradle.java.home=/opt/java/openjdk-${{ matrix.java }}" + + precommit-windows-macos: + if: github.repository == 'opensearch-project/opensearch-system-templates' + strategy: + matrix: + java: [ 21 ] + os: [windows-latest, macos-13] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + java-version: ${{ matrix.java }} + distribution: temurin + cache: gradle + - name: Run Gradle (check) + run: | + ./gradlew check + - name: Run Gradle (assemble) + run: | + ./gradlew assemble diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..177205a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,58 @@ +name: Backward Compatibility Checks + +on: + push: + branches: + - main + - 2.x + pull_request: + +env: + GRADLE_OPTS: -Dhttp.keepAlive=true + CI_ENVIRONMENT: normal + +jobs: + + backward-compatibility-build: + runs-on: ubuntu-latest + steps: + - uses: actions/setup-java@v4 + with: + distribution: temurin # Temurin is a distribution of adoptium + java-version: 21 + + - name: Checkout opensearch-system-templates Repo + uses: actions/checkout@v4 + + - name: Build BWC tests + uses: gradle/gradle-build-action@v3 + with: + cache-disabled: true + arguments: | + -p bwc-test build -x test -x integTest + + backward-compatibility: + strategy: + fail-fast: false + matrix: + jdk: [21] + platform: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.platform }} + + steps: + - uses: actions/setup-java@v4 + with: + distribution: temurin # Temurin is a distribution of adoptium + java-version: ${{ matrix.jdk }} + + - name: Checkout opensearch-system-templates Repo + uses: actions/checkout@v4 + + - id: build-previous + uses: ./.github/actions/run-bwc-suite + with: + plugin-previous-branch: "2.x" + plugin-next-branch: "current_branch" + report-artifact-name: bwc-${{ matrix.platform }}-jdk${{ matrix.jdk }} + username: admin + password: admin diff --git a/.github/workflows/delete-backport-branch.yml b/.github/workflows/delete-backport-branch.yml new file mode 100644 index 0000000..f24f022 --- /dev/null +++ b/.github/workflows/delete-backport-branch.yml @@ -0,0 +1,15 @@ +name: Delete merged branch of the backport PRs +on: + pull_request: + types: + - closed + +jobs: + delete-branch: + runs-on: ubuntu-latest + if: startsWith(github.event.pull_request.head.ref,'backport/') + steps: + - name: Delete merged branch + uses: SvanBoxel/delete-merged-branch@main + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish-maven-snapshots.yml b/.github/workflows/publish-maven-snapshots.yml new file mode 100644 index 0000000..6e5db19 --- /dev/null +++ b/.github/workflows/publish-maven-snapshots.yml @@ -0,0 +1,35 @@ +name: Publish snapshots to maven + +on: + workflow_dispatch: + push: + branches: + - main + - '[0-9]+.[0-9]+' + - '[0-9]+.x' + +jobs: + build-and-publish-snapshots: + runs-on: ubuntu-latest + + permissions: + id-token: write + contents: write + + steps: + - uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 21 + - uses: actions/checkout@v3 + - uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }} + aws-region: us-east-1 + - name: publish snapshots to maven + run: | + export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text) + export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text) + echo "::add-mask::$SONATYPE_USERNAME" + echo "::add-mask::$SONATYPE_PASSWORD" + ./gradlew publishPluginZipPublicationToSnapshotsRepository diff --git a/README.md b/README.md index 92616af..a183e99 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ## OpenSearch System Templates The repository is a plugin which acts as a repository for system templates which should be loaded as part of the cluster setup. -These contain application based configuration templates which are used to provide sensible defaults for various use-cases +These contain application based configuration templates which are used to provide sensible defaults for various use-cases for which users may be using OpenSearch. ## Security @@ -11,4 +11,3 @@ See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more inform ## License This project is licensed under the Apache-2.0 License. - diff --git a/build.gradle b/build.gradle index 53cbd85..a07a896 100644 --- a/build.gradle +++ b/build.gradle @@ -92,9 +92,6 @@ opensearchplugin { } dependencies { - implementation "org.opensearch:opensearch-common:${opensearch_version}" - implementation "org.opensearch:opensearch-core:${opensearch_version}" - implementation "org.opensearch:opensearch-x-content:${opensearch_version}" } allprojects { diff --git a/release-notes/opensearch-system-templates-2.16.0.md b/release-notes/opensearch-system-templates-2.16.0.md new file mode 100644 index 0000000..5f8f29f --- /dev/null +++ b/release-notes/opensearch-system-templates-2.16.0.md @@ -0,0 +1,7 @@ +## 2024-06-12 Version 2.16.0.0 + +Compatible with OpenSearch 2.16.0 + +### Features + +* Add new templates to the repository \ No newline at end of file diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100644 index 0000000..41abccd --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +# +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +set -ex + +function usage() { + echo "Usage: $0 [args]" + echo "" + echo "Arguments:" + echo -e "-v VERSION\t[Required] OpenSearch version." + echo -e "-q QUALIFIER\t[Optional] Version qualifier." + echo -e "-s SNAPSHOT\t[Optional] Build a snapshot, default is 'false'." + echo -e "-p PLATFORM\t[Optional] Platform, ignored." + echo -e "-a ARCHITECTURE\t[Optional] Build architecture, ignored." + echo -e "-o OUTPUT\t[Optional] Output path, default is 'artifacts'." + echo -e "-h help" +} + +while getopts ":h:v:q:s:o:p:a:" arg; do + case $arg in + h) + usage + exit 1 + ;; + v) + VERSION=$OPTARG + ;; + q) + QUALIFIER=$OPTARG + ;; + s) + SNAPSHOT=$OPTARG + ;; + o) + OUTPUT=$OPTARG + ;; + p) + PLATFORM=$OPTARG + ;; + a) + ARCHITECTURE=$OPTARG + ;; + :) + echo "Error: -${OPTARG} requires an argument" + usage + exit 1 + ;; + ?) + echo "Invalid option: -${arg}" + exit 1 + ;; + esac +done + +if [ -z "$VERSION" ]; then + echo "Error: You must specify the OpenSearch version" + usage + exit 1 +fi + +[[ ! -z "$QUALIFIER" ]] && VERSION=$VERSION-$QUALIFIER +[[ "$SNAPSHOT" == "true" ]] && VERSION=$VERSION-SNAPSHOT +[ -z "$OUTPUT" ] && OUTPUT=artifacts + +mkdir -p $OUTPUT + +./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER + +zipPath=$(find . -path \*build/distributions/*.zip) +distributions="$(dirname "${zipPath}")" + +echo "COPY ${distributions}/*.zip" +mkdir -p $OUTPUT/plugins +cp ${distributions}/*.zip ./$OUTPUT/plugins + +# Publish plugin zips to maven +./gradlew publishToMavenLocal -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER +./gradlew publishAllPublicationsToStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER +./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER +mkdir -p $OUTPUT/maven/org/opensearch +cp -r ./build/local-staging-repo/org/opensearch/. $OUTPUT/maven/org/opensearch diff --git a/src/integTest/java/org/opensearch/system/applicationtemplates/CreateIndexTemplateWithContextTemplateIT.java b/src/integTest/java/org/opensearch/system/applicationtemplates/CreateIndexTemplateWithContextTemplateIT.java new file mode 100644 index 0000000..5e0df8a --- /dev/null +++ b/src/integTest/java/org/opensearch/system/applicationtemplates/CreateIndexTemplateWithContextTemplateIT.java @@ -0,0 +1,113 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.system.applicationtemplates; + +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.core5.function.Factory; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.nio.ssl.TlsStrategy; +import org.apache.hc.core5.reactor.ssl.TlsDetails; +import org.apache.hc.core5.ssl.SSLContextBuilder; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.Strings; +import org.opensearch.test.rest.OpenSearchRestTestCase; + +import javax.net.ssl.SSLEngine; + +import java.io.IOException; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.Objects; + +import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_PER_ROUTE; +import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_TOTAL; + +public class CreateIndexTemplateWithContextTemplateIT extends OpenSearchRestTestCase { + + public void testCreateIndexWithContextBasedTemplate() throws IOException { + // TODO: Add E2E test with rest layer here. + } + + @Override + protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { + RestClientBuilder builder = RestClient.builder(hosts); + configureHttpOrHttpsClient(builder, settings); + builder.setStrictDeprecationMode(true); + return builder.build(); + } + + protected void configureHttpOrHttpsClient(RestClientBuilder builder, Settings settings) throws IOException { + configureClient(builder, settings); + + if (getProtocol().equalsIgnoreCase("https")) { + final String username = System.getProperty("user"); + if (Strings.isNullOrEmpty(username)) { + throw new RuntimeException("user name is missing"); + } + + final String password = System.getProperty("password"); + if (Strings.isNullOrEmpty(password)) { + throw new RuntimeException("password is missing"); + } + + final BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + final AuthScope anyScope = new AuthScope(null, -1); + credentialsProvider.setCredentials(anyScope, new UsernamePasswordCredentials(username, password.toCharArray())); + + try { + final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create() + .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .setSslContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build()) + // See https://issues.apache.org/jira/browse/HTTPCLIENT-2219 + .setTlsDetailsFactory(new Factory() { + @Override + public TlsDetails create(final SSLEngine sslEngine) { + return new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol()); + } + }) + .build(); + + builder.setHttpClientConfigCallback(httpClientBuilder -> { + final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder.create() + .setMaxConnPerRoute(DEFAULT_MAX_CONN_PER_ROUTE) + .setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL) + .setTlsStrategy(tlsStrategy) + .build(); + + return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setConnectionManager(connectionManager); + }); + } catch (final NoSuchAlgorithmException | KeyManagementException | KeyStoreException ex) { + throw new IOException(ex); + } + + } + } + + @Override + protected String getProtocol() { + return Objects.equals(System.getProperty("https"), "true") ? "https" : "http"; + } + + /** + * wipeAllIndices won't work since it cannot delete security index + */ + @Override + protected boolean preserveIndicesUponCompletion() { + return true; + } +} diff --git a/src/internalClusterTest/java/org/opensearch/system/applicationtemplates/CreateIndexTemplateWithContextIT.java b/src/internalClusterTest/java/org/opensearch/system/applicationtemplates/CreateIndexTemplateWithContextIT.java new file mode 100644 index 0000000..d36b097 --- /dev/null +++ b/src/internalClusterTest/java/org/opensearch/system/applicationtemplates/CreateIndexTemplateWithContextIT.java @@ -0,0 +1,75 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.system.applicationtemplates; + +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.get.GetIndexRequest; +import org.opensearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; +import org.opensearch.cluster.metadata.ComposableIndexTemplate; +import org.opensearch.cluster.metadata.Context; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST) +public class CreateIndexTemplateWithContextIT extends OpenSearchIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Collections.singletonList(ApplicationBasedConfigurationSystemTemplatesPlugin.class); + } + + public void testCreateIndexTemplateWithContext() throws Exception { + internalCluster().ensureAtLeastNumDataNodes(1); + + // Add context to an index template + client().admin() + .indices() + .execute( + PutComposableIndexTemplateAction.INSTANCE, + new PutComposableIndexTemplateAction.Request("my-logs").indexTemplate( + new ComposableIndexTemplate( + List.of("my-logs-*"), + null, + null, + null, + null, + null, + null, + new Context("logs", "_latest", Map.of()) + ) + ) + ) + .actionGet(new TimeValue(30000)); + + String indexName = "my-logs-1"; + client().admin().indices().create(new CreateIndexRequest(indexName)).actionGet(new TimeValue(30000)); + + Map allSettings = client().admin() + .indices() + .getIndex(new GetIndexRequest().indices(indexName)) + .actionGet(new TimeValue(30000)) + .settings(); + + assertEquals("best_compression", allSettings.get(indexName).get("index.codec")); + assertEquals("60s", allSettings.get(indexName).get("index.refresh_interval")); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + Settings baseSettings = super.nodeSettings(nodeOrdinal); + return Settings.builder().put(baseSettings).put("cluster.application_templates.enabled", true).build(); + } +} diff --git a/src/main/java/org/opensearch/system/applicationtemplates/ApplicationBasedConfigurationSystemTemplatesPlugin.java b/src/main/java/org/opensearch/system/applicationtemplates/ApplicationBasedConfigurationSystemTemplatesPlugin.java new file mode 100644 index 0000000..6bd43cf --- /dev/null +++ b/src/main/java/org/opensearch/system/applicationtemplates/ApplicationBasedConfigurationSystemTemplatesPlugin.java @@ -0,0 +1,91 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.system.applicationtemplates; + +import org.opensearch.client.Client; +import org.opensearch.cluster.applicationtemplates.ClusterStateSystemTemplateLoader; +import org.opensearch.cluster.applicationtemplates.SystemTemplateLoader; +import org.opensearch.cluster.applicationtemplates.SystemTemplateMetadata; +import org.opensearch.cluster.applicationtemplates.SystemTemplateRepository; +import org.opensearch.cluster.applicationtemplates.SystemTemplatesPlugin; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.env.NodeEnvironment; +import org.opensearch.plugins.Plugin; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.script.ScriptService; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.watcher.ResourceWatcherService; + +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Supplier; + +import static org.opensearch.cluster.applicationtemplates.SystemTemplateMetadata.COMPONENT_TEMPLATE_TYPE; + +public class ApplicationBasedConfigurationSystemTemplatesPlugin extends Plugin implements SystemTemplatesPlugin { + + private ClusterService clusterService; + private Client client; + + private final Map loaders = new HashMap<>(); + + public ApplicationBasedConfigurationSystemTemplatesPlugin() {} + + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + this.clusterService = clusterService; + this.client = client; + return super.createComponents( + client, + clusterService, + threadPool, + resourceWatcherService, + scriptService, + xContentRegistry, + environment, + nodeEnvironment, + namedWriteableRegistry, + indexNameExpressionResolver, + repositoriesServiceSupplier + ); + } + + @Override + public SystemTemplateRepository loadRepository() throws IOException { + return new LocalSystemTemplateRepository(); + } + + @Override + public SystemTemplateLoader loaderFor(SystemTemplateMetadata templateMetadata) { + return loaders.computeIfAbsent(templateMetadata.type(), k -> { + if (COMPONENT_TEMPLATE_TYPE.equals(templateMetadata.type())) { + return new ClusterStateSystemTemplateLoader(client, () -> clusterService.state()); + } + return null; + }); + } +} diff --git a/src/main/java/org/opensearch/system/applicationtemplates/LocalSystemTemplateRepository.java b/src/main/java/org/opensearch/system/applicationtemplates/LocalSystemTemplateRepository.java new file mode 100644 index 0000000..1c8a09e --- /dev/null +++ b/src/main/java/org/opensearch/system/applicationtemplates/LocalSystemTemplateRepository.java @@ -0,0 +1,144 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.system.applicationtemplates; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.applicationtemplates.SystemTemplate; +import org.opensearch.cluster.applicationtemplates.SystemTemplateMetadata; +import org.opensearch.cluster.applicationtemplates.SystemTemplateRepository; +import org.opensearch.cluster.applicationtemplates.TemplateRepositoryMetadata; +import org.opensearch.common.io.PathUtils; +import org.opensearch.common.util.io.Streams; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +/** + * Repository implementation for this plugin. + */ +public class LocalSystemTemplateRepository implements SystemTemplateRepository { + + private static final Logger logger = LogManager.getLogger(LocalSystemTemplateRepository.class); + + static final String REPOSITORY_ID = "__core__"; + static final long CURRENT_REPO_VERSION = 1L; + + @Override + public TemplateRepositoryMetadata metadata() { + return new TemplateRepositoryMetadata(REPOSITORY_ID, CURRENT_REPO_VERSION); + } + + @Override + public Iterable listTemplates() throws IOException { + List templateMetadataList = new ArrayList<>(); + + try ( + InputStream is = getResourceAsStream("templates.json"); + XContentParser listParser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.IGNORE_DEPRECATIONS, + is + ) + ) { + while (listParser.currentToken() != XContentParser.Token.START_ARRAY) { + listParser.nextToken(); + } + if (!"templates".equals(listParser.currentName())) { + throw new IllegalArgumentException("Format of template metadata file does not match expected format"); + } + + while (listParser.nextToken() != XContentParser.Token.END_ARRAY) { + if (listParser.currentToken() != XContentParser.Token.START_OBJECT) { + throw new IllegalArgumentException( + "Format of template metadata file does not match expected format" + listParser.currentToken() + ); + } + String templateName = null; + String templateType = null; + long templateVersion = 0L; + + String name = null; + while (listParser.nextToken() != XContentParser.Token.END_OBJECT) { + XContentParser.Token currentToken = listParser.currentToken(); + if (currentToken == XContentParser.Token.FIELD_NAME) { + name = listParser.currentName(); + } else if (currentToken == XContentParser.Token.VALUE_STRING) { + if ("name".equals(name)) { + templateName = listParser.text(); + } else if ("type".equals(name)) { + templateType = listParser.text(); + } else { + throw new IllegalArgumentException("Unexpected token " + currentToken); + } + } else if (currentToken == XContentParser.Token.VALUE_NUMBER) { + if ("version".equals(name)) { + templateVersion = listParser.longValue(); + } else { + throw new IllegalArgumentException("Unexpected token " + currentToken); + } + } else { + throw new IllegalArgumentException("Unexpected token " + currentToken); + } + } + if (templateName == null || templateType == null || templateVersion == 0L) { + throw new IllegalArgumentException( + "Could not read template metadata: [name: " + + templateName + + " , type: " + + templateType + + " , version: " + + templateVersion + ); + } + templateMetadataList.add(new SystemTemplateMetadata(templateVersion, templateType, templateName)); + } + } catch (Exception ex) { + throw new IOException("Could not load system templates: ", ex); + } + return templateMetadataList; + } + + @Override + public SystemTemplate getTemplate(SystemTemplateMetadata templateMetadata) throws IOException { + final String fileName = buildFileName(templateMetadata); + logger.debug("Loading {} from file: {}", templateMetadata, fileName); + try (InputStream is = getResourceAsStream(fileName)) { + if (is != null) { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + Streams.copy(is, out); + final BytesReference templateContent = new BytesArray(out.toByteArray()); + return new SystemTemplate(templateContent, templateMetadata, metadata()); + } else { + throw new IOException("Unable to read: " + templateMetadata + " from repository: " + metadata()); + } + } + } + + static String buildFileName(SystemTemplateMetadata templateMetadata) { + return "v" + templateMetadata.version() + PathUtils.getDefaultFileSystem().getSeparator() + templateMetadata.name() + ".json"; + } + + // Visible for testing (if we need UTs with mocked resources) + protected InputStream getResourceAsStream(String name) throws IOException { + return LocalSystemTemplateRepository.class.getResourceAsStream(name); + } + + @Override + public void close() throws IOException {} +} diff --git a/src/main/resources/org/opensearch/system/applicationtemplates/templates.json b/src/main/resources/org/opensearch/system/applicationtemplates/templates.json new file mode 100644 index 0000000..dc28328 --- /dev/null +++ b/src/main/resources/org/opensearch/system/applicationtemplates/templates.json @@ -0,0 +1,15 @@ +{ + "repository_schema_version": 1, + "templates": [ + { + "name": "logs", + "version": 1, + "type": "@abc_template" + }, + { + "name": "metrics", + "version": 1, + "type": "@abc_template" + } + ] +} diff --git a/src/main/resources/org/opensearch/system/applicationtemplates/v1/logs.json b/src/main/resources/org/opensearch/system/applicationtemplates/v1/logs.json new file mode 100644 index 0000000..25e5bf1 --- /dev/null +++ b/src/main/resources/org/opensearch/system/applicationtemplates/v1/logs.json @@ -0,0 +1,14 @@ +{ + "template": { + "settings": { + "codec": "best_compression", + "merge.policy": "log_byte_size", + "refresh_interval": "60s" + } + }, + "_meta": { + "_type": "@abc_template", + "_version": 1 + }, + "version": 1 +} diff --git a/src/main/resources/org/opensearch/system/applicationtemplates/v1/metrics.json b/src/main/resources/org/opensearch/system/applicationtemplates/v1/metrics.json new file mode 100644 index 0000000..25e5bf1 --- /dev/null +++ b/src/main/resources/org/opensearch/system/applicationtemplates/v1/metrics.json @@ -0,0 +1,14 @@ +{ + "template": { + "settings": { + "codec": "best_compression", + "merge.policy": "log_byte_size", + "refresh_interval": "60s" + } + }, + "_meta": { + "_type": "@abc_template", + "_version": 1 + }, + "version": 1 +} diff --git a/src/test/java/org/opensearch/system/applicationtemplates/LocalSystemTemplateRepositoryTests.java b/src/test/java/org/opensearch/system/applicationtemplates/LocalSystemTemplateRepositoryTests.java new file mode 100644 index 0000000..1ddff6f --- /dev/null +++ b/src/test/java/org/opensearch/system/applicationtemplates/LocalSystemTemplateRepositoryTests.java @@ -0,0 +1,55 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.system.applicationtemplates; + +import org.opensearch.cluster.applicationtemplates.SystemTemplate; +import org.opensearch.cluster.applicationtemplates.SystemTemplateMetadata; +import org.opensearch.cluster.applicationtemplates.SystemTemplateRepository; +import org.opensearch.cluster.applicationtemplates.TemplateRepositoryMetadata; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicInteger; + +public class LocalSystemTemplateRepositoryTests extends OpenSearchTestCase { + + public void testRepositoryMetadata() throws Exception { + try (SystemTemplateRepository repository = new LocalSystemTemplateRepository()) { + TemplateRepositoryMetadata repositoryMetadata = repository.metadata(); + assertNotNull(repositoryMetadata); + assertEquals(repositoryMetadata.id(), LocalSystemTemplateRepository.REPOSITORY_ID); + assertEquals(repositoryMetadata.version(), LocalSystemTemplateRepository.CURRENT_REPO_VERSION); + } + } + + public void testRepositoryListTemplates() throws Exception { + try (SystemTemplateRepository repository = new LocalSystemTemplateRepository()) { + AtomicInteger counter = new AtomicInteger(); + repository.listTemplates().forEach(templateMetadata -> { counter.incrementAndGet(); }); + + assertEquals(counter.get(), 2); + } + } + + public void testRepositoryGetTemplate() throws Exception { + try (SystemTemplateRepository repository = new LocalSystemTemplateRepository()) { + for (SystemTemplateMetadata templateMetadata : repository.listTemplates()) { + SystemTemplate template = repository.getTemplate(templateMetadata); + assertEquals(templateMetadata, template.templateMetadata()); + assertNotNull(template.templateContent()); + + try (InputStream is = this.getClass().getResourceAsStream(LocalSystemTemplateRepository.buildFileName(templateMetadata))) { + String expectedTemplateContent = new String(is.readAllBytes(), StandardCharsets.UTF_8); + assertEquals(template.templateContent().utf8ToString(), expectedTemplateContent); + } + } + } + } +}