From 7249b3e7e5aee5383645f024c5e80e1eecaf27f8 Mon Sep 17 00:00:00 2001 From: Mohit Godwani <81609427+mgodwan@users.noreply.github.com> Date: Fri, 30 Aug 2024 22:55:06 +0530 Subject: [PATCH 1/2] Onboarding New templates and adding zstd support (#8) * Add new templates with zstd support Signed-off-by: Mohit Godwani * Address PR comments Signed-off-by: Mohit Godwani * Address PR comments Signed-off-by: Mohit Godwani * Add schema link with version Signed-off-by: Mohit Godwani * Trigger Build Signed-off-by: Mohit Godwani * Trigger Build Signed-off-by: Mohit Godwani --------- Signed-off-by: Mohit Godwani --- .github/workflows/ci.yml | 58 - build.gradle | 21 +- ...ateIndexTemplateWithContextTemplateIT.java | 43 +- .../CreateIndexTemplateWithContextIT.java | 75 - .../LocalSystemTemplateRepository.java | 3 +- .../applicationtemplates/templates.json | 50 +- .../v1/amazon-cloudtrail-logs.json | 231 ++ .../v1/amazon-elb-logs.json | 693 ++++ .../v1/amazon-s3-logs.json | 424 +++ .../v1/apache-web-logs.json | 499 +++ .../applicationtemplates/v1/k8s-logs.json | 2837 +++++++++++++++++ .../system/applicationtemplates/v1/logs.json | 2 +- .../applicationtemplates/v1/metrics.json | 2 +- .../applicationtemplates/v1/nginx-logs.json | 496 +++ .../LocalSystemTemplateRepositoryTests.java | 2 +- 15 files changed, 5285 insertions(+), 151 deletions(-) delete mode 100644 .github/workflows/ci.yml delete mode 100644 src/internalClusterTest/java/org/opensearch/system/applicationtemplates/CreateIndexTemplateWithContextIT.java create mode 100644 src/main/resources/org/opensearch/system/applicationtemplates/v1/amazon-cloudtrail-logs.json create mode 100644 src/main/resources/org/opensearch/system/applicationtemplates/v1/amazon-elb-logs.json create mode 100644 src/main/resources/org/opensearch/system/applicationtemplates/v1/amazon-s3-logs.json create mode 100644 src/main/resources/org/opensearch/system/applicationtemplates/v1/apache-web-logs.json create mode 100644 src/main/resources/org/opensearch/system/applicationtemplates/v1/k8s-logs.json create mode 100644 src/main/resources/org/opensearch/system/applicationtemplates/v1/nginx-logs.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 177205a..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: Backward Compatibility Checks - -on: - push: - branches: - - main - - 2.x - pull_request: - -env: - GRADLE_OPTS: -Dhttp.keepAlive=true - CI_ENVIRONMENT: normal - -jobs: - - backward-compatibility-build: - runs-on: ubuntu-latest - steps: - - uses: actions/setup-java@v4 - with: - distribution: temurin # Temurin is a distribution of adoptium - java-version: 21 - - - name: Checkout opensearch-system-templates Repo - uses: actions/checkout@v4 - - - name: Build BWC tests - uses: gradle/gradle-build-action@v3 - with: - cache-disabled: true - arguments: | - -p bwc-test build -x test -x integTest - - backward-compatibility: - strategy: - fail-fast: false - matrix: - jdk: [21] - platform: [ubuntu-latest, windows-latest] - runs-on: ${{ matrix.platform }} - - steps: - - uses: actions/setup-java@v4 - with: - distribution: temurin # Temurin is a distribution of adoptium - java-version: ${{ matrix.jdk }} - - - name: Checkout opensearch-system-templates Repo - uses: actions/checkout@v4 - - - id: build-previous - uses: ./.github/actions/run-bwc-suite - with: - plugin-previous-branch: "2.x" - plugin-next-branch: "current_branch" - report-artifact-name: bwc-${{ matrix.platform }}-jdk${{ matrix.jdk }} - username: admin - password: admin diff --git a/build.gradle b/build.gradle index a07a896..cdc9b36 100644 --- a/build.gradle +++ b/build.gradle @@ -67,6 +67,10 @@ repositories { maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } } +configurations { + zipArchive +} + allprojects { group 'org.opensearch' version = opensearch_version.tokenize('-')[0] + '.0' @@ -92,6 +96,7 @@ opensearchplugin { } dependencies { + zipArchive group: 'org.opensearch.plugin', name:'opensearch-custom-codecs', version: "3.0.0.0-SNAPSHOT" } allprojects { @@ -148,6 +153,7 @@ publishing { password "$System.env.SONATYPE_PASSWORD" } } + mavenLocal() } } @@ -202,14 +208,27 @@ integTest { dependsOn "bundlePlugin" systemProperty 'tests.security.manager', 'true' + systemProperty "https", System.getProperty("https") systemProperty "user", System.getProperty("user") systemProperty "password", System.getProperty("password") } testClusters.integTest { - testDistribution = "ARCHIVE" + testDistribution = "INTEG_TEST" plugin(project.tasks.bundlePlugin.archiveFile) + plugin(provider({ + new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.matching { + include '**/opensearch-custom-codecs*' + }.singleFile + } + } + })) + systemProperty 'opensearch.experimental.feature.application_templates.enabled', 'true' + setting 'cluster.application_templates.enabled', 'true' } tasks.withType(PublishToMavenRepository) { diff --git a/src/integTest/java/org/opensearch/system/applicationtemplates/CreateIndexTemplateWithContextTemplateIT.java b/src/integTest/java/org/opensearch/system/applicationtemplates/CreateIndexTemplateWithContextTemplateIT.java index 5e0df8a..d4ab3d5 100644 --- a/src/integTest/java/org/opensearch/system/applicationtemplates/CreateIndexTemplateWithContextTemplateIT.java +++ b/src/integTest/java/org/opensearch/system/applicationtemplates/CreateIndexTemplateWithContextTemplateIT.java @@ -20,10 +20,14 @@ import org.apache.hc.core5.http.nio.ssl.TlsStrategy; import org.apache.hc.core5.reactor.ssl.TlsDetails; import org.apache.hc.core5.ssl.SSLContextBuilder; +import org.opensearch.client.Request; import org.opensearch.client.RestClient; import org.opensearch.client.RestClientBuilder; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.EngineConfig; import org.opensearch.test.rest.OpenSearchRestTestCase; import javax.net.ssl.SSLEngine; @@ -32,6 +36,7 @@ import java.security.KeyManagementException; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; +import java.util.Map; import java.util.Objects; import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_PER_ROUTE; @@ -39,8 +44,42 @@ public class CreateIndexTemplateWithContextTemplateIT extends OpenSearchRestTestCase { - public void testCreateIndexWithContextBasedTemplate() throws IOException { - // TODO: Add E2E test with rest layer here. + @SuppressWarnings("unchecked") + public void testCreateIndexWithContextBasedTemplate() throws Exception { + + final String indexTemplate = "my-metrics-template"; + final String index = "my-metrics-1"; + + Request request = new Request("PUT", "/_index_template/" + indexTemplate); + String content = "{\n" + + " \"index_patterns\": [\n" + + " \"my-metrics-*\"\n" + + " ],\n" + + " \"context\": {\n" + + " \"name\": \"metrics\"\n" + + " }\n" + + "}"; + + request.setJsonEntity(content); + + // Create index template + client().performRequest(request); + + // creating index + createIndex(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()); + + Map currentIndexSettings = (Map) ((Map) getIndexSettings(index).get(index)).get( + "settings" + ); + assertEquals(currentIndexSettings.get(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey()), "60s"); + assertEquals(currentIndexSettings.get(IndexSettings.INDEX_MERGE_POLICY.getKey()), "log_byte_size"); + assertEquals(currentIndexSettings.get(EngineConfig.INDEX_CODEC_SETTING.getKey()), "zstd_no_dict"); + + try { + ensureGreen(index); + } finally { + deleteIndex(index); + } } @Override diff --git a/src/internalClusterTest/java/org/opensearch/system/applicationtemplates/CreateIndexTemplateWithContextIT.java b/src/internalClusterTest/java/org/opensearch/system/applicationtemplates/CreateIndexTemplateWithContextIT.java deleted file mode 100644 index d36b097..0000000 --- a/src/internalClusterTest/java/org/opensearch/system/applicationtemplates/CreateIndexTemplateWithContextIT.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.system.applicationtemplates; - -import org.opensearch.action.admin.indices.create.CreateIndexRequest; -import org.opensearch.action.admin.indices.get.GetIndexRequest; -import org.opensearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; -import org.opensearch.cluster.metadata.ComposableIndexTemplate; -import org.opensearch.cluster.metadata.Context; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.plugins.Plugin; -import org.opensearch.test.OpenSearchIntegTestCase; - -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST) -public class CreateIndexTemplateWithContextIT extends OpenSearchIntegTestCase { - - @Override - protected Collection> nodePlugins() { - return Collections.singletonList(ApplicationBasedConfigurationSystemTemplatesPlugin.class); - } - - public void testCreateIndexTemplateWithContext() throws Exception { - internalCluster().ensureAtLeastNumDataNodes(1); - - // Add context to an index template - client().admin() - .indices() - .execute( - PutComposableIndexTemplateAction.INSTANCE, - new PutComposableIndexTemplateAction.Request("my-logs").indexTemplate( - new ComposableIndexTemplate( - List.of("my-logs-*"), - null, - null, - null, - null, - null, - null, - new Context("logs", "_latest", Map.of()) - ) - ) - ) - .actionGet(new TimeValue(30000)); - - String indexName = "my-logs-1"; - client().admin().indices().create(new CreateIndexRequest(indexName)).actionGet(new TimeValue(30000)); - - Map allSettings = client().admin() - .indices() - .getIndex(new GetIndexRequest().indices(indexName)) - .actionGet(new TimeValue(30000)) - .settings(); - - assertEquals("best_compression", allSettings.get(indexName).get("index.codec")); - assertEquals("60s", allSettings.get(indexName).get("index.refresh_interval")); - } - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - Settings baseSettings = super.nodeSettings(nodeOrdinal); - return Settings.builder().put(baseSettings).put("cluster.application_templates.enabled", true).build(); - } -} diff --git a/src/main/java/org/opensearch/system/applicationtemplates/LocalSystemTemplateRepository.java b/src/main/java/org/opensearch/system/applicationtemplates/LocalSystemTemplateRepository.java index 1c8a09e..0346322 100644 --- a/src/main/java/org/opensearch/system/applicationtemplates/LocalSystemTemplateRepository.java +++ b/src/main/java/org/opensearch/system/applicationtemplates/LocalSystemTemplateRepository.java @@ -14,7 +14,6 @@ import org.opensearch.cluster.applicationtemplates.SystemTemplateMetadata; import org.opensearch.cluster.applicationtemplates.SystemTemplateRepository; import org.opensearch.cluster.applicationtemplates.TemplateRepositoryMetadata; -import org.opensearch.common.io.PathUtils; import org.opensearch.common.util.io.Streams; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.bytes.BytesArray; @@ -131,7 +130,7 @@ public SystemTemplate getTemplate(SystemTemplateMetadata templateMetadata) throw } static String buildFileName(SystemTemplateMetadata templateMetadata) { - return "v" + templateMetadata.version() + PathUtils.getDefaultFileSystem().getSeparator() + templateMetadata.name() + ".json"; + return "v" + templateMetadata.version() + "/" + templateMetadata.name() + ".json"; } // Visible for testing (if we need UTs with mocked resources) diff --git a/src/main/resources/org/opensearch/system/applicationtemplates/templates.json b/src/main/resources/org/opensearch/system/applicationtemplates/templates.json index dc28328..1860d33 100644 --- a/src/main/resources/org/opensearch/system/applicationtemplates/templates.json +++ b/src/main/resources/org/opensearch/system/applicationtemplates/templates.json @@ -1,15 +1,45 @@ { "repository_schema_version": 1, "templates": [ - { - "name": "logs", - "version": 1, - "type": "@abc_template" - }, - { - "name": "metrics", - "version": 1, - "type": "@abc_template" - } + { + "name": "logs", + "version": 1, + "type": "@abc_template" + }, + { + "name": "metrics", + "version": 1, + "type": "@abc_template" + }, + { + "name": "apache-web-logs", + "version": 1, + "type": "@abc_template" + }, + { + "name": "amazon-cloudtrail-logs", + "version": 1, + "type": "@abc_template" + }, + { + "name": "amazon-elb-logs", + "version": 1, + "type": "@abc_template" + }, + { + "name": "amazon-s3-logs", + "version": 1, + "type": "@abc_template" + }, + { + "name": "nginx-logs", + "version": 1, + "type": "@abc_template" + }, + { + "name": "k8s-logs", + "version": 1, + "type": "@abc_template" + } ] } diff --git a/src/main/resources/org/opensearch/system/applicationtemplates/v1/amazon-cloudtrail-logs.json b/src/main/resources/org/opensearch/system/applicationtemplates/v1/amazon-cloudtrail-logs.json new file mode 100644 index 0000000..c356fd6 --- /dev/null +++ b/src/main/resources/org/opensearch/system/applicationtemplates/v1/amazon-cloudtrail-logs.json @@ -0,0 +1,231 @@ +{ + "template": { + "settings": { + "index": { + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "codec": "zstd_no_dict", + "merge.policy": "log_byte_size", + "refresh_interval": "60s" + } + }, + "mappings": { + "_meta": { + "correlations": [ + { + "field": "spanId", + "foreign-field": "spanId", + "foreign-schema": "traces" + }, + { + "field": "traceId", + "foreign-field": "traceId", + "foreign-schema": "traces" + } + ], + "labels": [ + "log", + "aws", + "s3", + "cloud", + "cloudtrail" + ] + }, + "_source": { + "enabled": true + }, + "dynamic_templates": [ + { + "resources_map": { + "path_match": "resource.*", + "mapping": { + "type": "keyword" + } + } + }, + { + "attributes_map": { + "path_match": "attributes.*", + "mapping": { + "type": "keyword" + } + } + }, + { + "instrumentation_scope_attributes_map": { + "path_match": "instrumentationScope.attributes.*", + "mapping": { + "type": "keyword" + } + } + } + ], + "properties": { + "severity": { + "properties": { + "number": { + "type": "long" + }, + "text": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + } + } + }, + "traceId": { + "ignore_above": 256, + "type": "keyword" + }, + "spanId": { + "ignore_above": 256, + "type": "keyword" + }, + "@message": { + "path": "body", + "type": "alias" + }, + "@timestamp": { + "type": "date" + }, + "instrumentationScope": { + "properties": { + "dropped_attributes_count": { + "type": "integer" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 128, + "type": "keyword" + } + } + }, + "version": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "schemaUrl": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + } + } + }, + "attributes": { + "type": "object", + "properties": { + "data_stream": { + "properties": { + "namespace": { + "ignore_above": 128, + "type": "keyword" + }, + "type": { + "ignore_above": 56, + "type": "keyword" + }, + "dataset": { + "ignore_above": 128, + "type": "keyword" + } + } + } + } + }, + "body": { + "type": "text" + }, + "event": { + "properties": { + "result": { + "ignore_above": 256, + "type": "keyword" + }, + "exception": { + "properties": { + "stacktrace": { + "type": "text" + }, + "message": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "kind": { + "ignore_above": 256, + "type": "keyword" + }, + "domain": { + "ignore_above": 256, + "type": "keyword" + }, + "name": { + "ignore_above": 256, + "type": "keyword" + }, + "source": { + "ignore_above": 256, + "type": "keyword" + }, + "category": { + "ignore_above": 256, + "type": "keyword" + }, + "type": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "observedTimestamp": { + "type": "date" + }, + "observerTime": { + "path": "observedTimestamp", + "type": "alias" + }, + "schemaUrl": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + } + } + }, + "aliases": { + "logs-cloudtrail": {} + } + }, + "_meta": { + "_type": "@abc_template", + "_version": 1, + "_schema_ref": "https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/aws_cloudtrail/aws_cloudtrail-1.0.0.json" + }, + "version": 1 +} diff --git a/src/main/resources/org/opensearch/system/applicationtemplates/v1/amazon-elb-logs.json b/src/main/resources/org/opensearch/system/applicationtemplates/v1/amazon-elb-logs.json new file mode 100644 index 0000000..c698b78 --- /dev/null +++ b/src/main/resources/org/opensearch/system/applicationtemplates/v1/amazon-elb-logs.json @@ -0,0 +1,693 @@ +{ + "template": { + "settings": { + "index": { + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "codec": "zstd_no_dict", + "merge.policy": "log_byte_size", + "refresh_interval": "60s" + } + }, + "mappings": { + "_meta": { + "correlations": [ + { + "field": "spanId", + "foreign-field": "spanId", + "foreign-schema": "traces" + }, + { + "field": "traceId", + "foreign-field": "traceId", + "foreign-schema": "traces" + } + ], + "labels": [ + "communication", + "http", + "cloud", + "aws", + "elb", + "url", + "log" + ] + }, + "dynamic_templates": [ + { + "request_header_map": { + "path_match": "request.header.*", + "mapping": { + "type": "keyword" + } + } + }, + { + "response_header_map": { + "path_match": "response.header.*", + "mapping": { + "type": "keyword" + } + } + }, + { + "resources_map": { + "path_match": "resource.*", + "mapping": { + "type": "keyword" + } + } + }, + { + "attributes_map": { + "path_match": "attributes.*", + "mapping": { + "type": "keyword" + } + } + }, + { + "instrumentation_scope_attributes_map": { + "path_match": "instrumentationScope.attributes.*", + "mapping": { + "type": "keyword" + } + } + } + ], + "properties": { + "@message": { + "type": "alias", + "path": "body" + }, + "@timestamp": { + "type": "date" + }, + "aws": { + "properties": { + "elb": { + "properties": { + "backend": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "integer" + }, + "processing_time": { + "type": "half_float" + }, + "status_code": { + "type": "short" + } + } + }, + "client": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "integer" + } + } + }, + "connection_time": { + "type": "integer" + }, + "destination": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "integer" + } + } + }, + "elb_status_code": { + "type": "short" + }, + "http": { + "properties": { + "port": { + "type": "integer" + }, + "version": { + "type": "keyword" + } + } + }, + "matched_rule_priority": { + "type": "integer" + }, + "received_bytes": { + "type": "integer" + }, + "request_creation_time": { + "type": "date" + }, + "request_processing_time": { + "type": "half_float" + }, + "response_processing_time": { + "type": "half_float" + }, + "sent_bytes": { + "type": "integer" + }, + "ssl_cipher": { + "type": "keyword" + }, + "ssl_protocol": { + "type": "keyword" + }, + "target_ip": { + "type": "ip" + }, + "target_port": { + "type": "integer" + }, + "target_processing_time": { + "type": "half_float" + }, + "target_status_code": { + "type": "short" + }, + "timestamp": { + "type": "date" + } + } + } + } + }, + "body": { + "type": "text" + }, + "cloud": { + "properties": { + "account": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + }, + "availability_zone": { + "type": "keyword" + }, + "instance": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + }, + "machine": { + "properties": { + "type": { + "type": "keyword" + } + } + }, + "platform": { + "type": "keyword" + }, + "project": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + }, + "provider": { + "type": "keyword" + }, + "region": { + "type": "keyword" + }, + "resource_id": { + "type": "keyword" + }, + "service": { + "properties": { + "name": { + "type": "keyword" + } + } + } + } + }, + "communication": { + "properties": { + "destination": { + "properties": { + "address": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "geo": { + "properties": { + "city_name": { + "type": "keyword" + }, + "country_iso_code": { + "type": "keyword" + }, + "country_name": { + "type": "keyword" + }, + "location": { + "type": "geo_point" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "type": "keyword", + "ignore_above": 1024 + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + }, + "sock": { + "properties": { + "family": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "source": { + "properties": { + "address": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "geo": { + "properties": { + "city_name": { + "type": "keyword" + }, + "country_iso_code": { + "type": "keyword" + }, + "country_name": { + "type": "keyword" + }, + "location": { + "type": "geo_point" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "type": "keyword", + "ignore_above": 1024 + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + } + } + }, + "event": { + "properties": { + "category": { + "type": "keyword", + "ignore_above": 256 + }, + "domain": { + "type": "keyword", + "ignore_above": 256 + }, + "exception": { + "properties": { + "message": { + "type": "keyword", + "ignore_above": 1024 + }, + "stacktrace": { + "type": "text" + }, + "type": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "kind": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "result": { + "type": "keyword", + "ignore_above": 256 + }, + "source": { + "type": "keyword", + "ignore_above": 256 + }, + "type": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "http": { + "properties": { + "client": { + "properties": { + "ip": { + "type": "ip" + } + } + }, + "flavor": { + "type": "keyword", + "ignore_above": 256 + }, + "request": { + "properties": { + "body": { + "properties": { + "content": { + "type": "text" + } + } + }, + "bytes": { + "type": "long" + }, + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "method": { + "type": "keyword", + "ignore_above": 256 + }, + "mime_type": { + "type": "keyword", + "ignore_above": 1024 + }, + "referrer": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "resent_count": { + "type": "integer" + }, + "response": { + "properties": { + "body": { + "properties": { + "content": { + "type": "text" + } + } + }, + "bytes": { + "type": "long" + }, + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "status_code": { + "type": "integer" + } + } + }, + "route": { + "type": "keyword", + "ignore_above": 1024 + }, + "schema": { + "type": "keyword", + "ignore_above": 1024 + }, + "target": { + "type": "keyword", + "ignore_above": 1024 + }, + "url": { + "type": "keyword", + "ignore_above": 2048 + }, + "user_agent": { + "properties": { + "device": { + "properties": { + "name": { + "type": "keyword" + } + } + }, + "name": { + "type": "keyword" + }, + "original": { + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "type": "keyword" + }, + "full": { + "type": "keyword" + }, + "kernel": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "platform": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + } + }, + "version": { + "type": "keyword" + } + } + } + } + }, + "instrumentationScope": { + "properties": { + "dropped_attributes_count": { + "type": "integer" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 128 + } + } + }, + "schemaUrl": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "version": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "observedTimestamp": { + "type": "date" + }, + "observerTime": { + "type": "alias", + "path": "observedTimestamp" + }, + "schemaUrl": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "severity": { + "properties": { + "number": { + "type": "long" + }, + "text": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "spanId": { + "type": "keyword", + "ignore_above": 256 + }, + "traceId": { + "type": "keyword", + "ignore_above": 256 + }, + "url": { + "properties": { + "domain": { + "type": "keyword" + }, + "extension": { + "type": "keyword" + }, + "fragment": { + "type": "keyword" + }, + "full": { + "type": "keyword" + }, + "original": { + "type": "keyword" + }, + "password": { + "type": "keyword" + }, + "path": { + "type": "keyword" + }, + "port": { + "type": "long" + }, + "query": { + "type": "keyword" + }, + "registered_domain": { + "type": "keyword" + }, + "scheme": { + "type": "keyword" + }, + "subdomain": { + "type": "keyword" + }, + "top_level_domain": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + } + } + }, + "aliases": { + "logs-elb": {} + } + }, + "_meta": { + "_type": "@abc_template", + "_version": 1, + "_schema_ref": "https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/amazon_elb/amazon_elb-1.0.0.json" + }, + "version": 1 +} diff --git a/src/main/resources/org/opensearch/system/applicationtemplates/v1/amazon-s3-logs.json b/src/main/resources/org/opensearch/system/applicationtemplates/v1/amazon-s3-logs.json new file mode 100644 index 0000000..648957d --- /dev/null +++ b/src/main/resources/org/opensearch/system/applicationtemplates/v1/amazon-s3-logs.json @@ -0,0 +1,424 @@ +{ + "template": { + "settings": { + "index": { + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "codec": "zstd_no_dict", + "merge.policy": "log_byte_size", + "refresh_interval": "60s" + } + }, + "mappings": { + "_meta": { + "correlations": [ + { + "field": "spanId", + "foreign-field": "spanId", + "foreign-schema": "traces" + }, + { + "field": "traceId", + "foreign-field": "traceId", + "foreign-schema": "traces" + } + ], + "labels": [ + "aws", + "s3", + "cloud", + "log" + ] + }, + "dynamic_templates": [ + { + "resources_map": { + "path_match": "resource.*", + "mapping": { + "type": "keyword" + } + } + }, + { + "attributes_map": { + "path_match": "attributes.*", + "mapping": { + "type": "keyword" + } + } + }, + { + "instrumentation_scope_attributes_map": { + "path_match": "instrumentationScope.attributes.*", + "mapping": { + "type": "keyword" + } + } + } + ], + "properties": { + "@message": { + "type": "alias", + "path": "body" + }, + "@timestamp": { + "type": "date" + }, + "aws": { + "properties": { + "s3": { + "properties": { + "authentication_type": { + "type": "keyword" + }, + "bucket": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "bucket_owner": { + "type": "keyword" + }, + "bytes_sent": { + "type": "long" + }, + "cipher_suite": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "copy_source": { + "type": "keyword" + }, + "delete": { + "type": "keyword" + }, + "error_code": { + "type": "keyword" + }, + "host_header": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "host_id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "http_status": { + "type": "keyword" + }, + "key": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "object_size": { + "type": "long" + }, + "operation": { + "type": "keyword" + }, + "part_number": { + "type": "keyword" + }, + "referrer": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "remote_ip": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "request_id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "request_uri": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "requester": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "signature_version": { + "type": "keyword" + }, + "tls_version": { + "type": "keyword" + }, + "total_time": { + "type": "integer" + }, + "turn_around_time": { + "type": "integer" + }, + "upload_id": { + "type": "keyword" + }, + "user_agent": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "version_id": { + "type": "keyword" + } + } + } + } + }, + "body": { + "type": "text" + }, + "cloud": { + "properties": { + "account": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + }, + "availability_zone": { + "type": "keyword" + }, + "instance": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + }, + "machine": { + "properties": { + "type": { + "type": "keyword" + } + } + }, + "platform": { + "type": "keyword" + }, + "project": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + }, + "provider": { + "type": "keyword" + }, + "region": { + "type": "keyword" + }, + "resource_id": { + "type": "keyword" + }, + "service": { + "properties": { + "name": { + "type": "keyword" + } + } + } + } + }, + "event": { + "properties": { + "category": { + "type": "keyword", + "ignore_above": 256 + }, + "domain": { + "type": "keyword", + "ignore_above": 256 + }, + "exception": { + "properties": { + "message": { + "type": "keyword", + "ignore_above": 1024 + }, + "stacktrace": { + "type": "text" + }, + "type": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "kind": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "result": { + "type": "keyword", + "ignore_above": 256 + }, + "source": { + "type": "keyword", + "ignore_above": 256 + }, + "type": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "instrumentationScope": { + "properties": { + "dropped_attributes_count": { + "type": "integer" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 128 + } + } + }, + "schemaUrl": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "version": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "observedTimestamp": { + "type": "date" + }, + "observerTime": { + "type": "alias", + "path": "observedTimestamp" + }, + "schemaUrl": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "severity": { + "properties": { + "number": { + "type": "long" + }, + "text": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "spanId": { + "type": "keyword", + "ignore_above": 256 + }, + "traceId": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "aliases": { + "logs-aws-s3": {} + } + }, + "_meta": { + "_type": "@abc_template", + "_version": 1, + "_schema_ref": "https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/amazon_s3/amazon_s3-1.0.0.json" + }, + "version": 1 +} diff --git a/src/main/resources/org/opensearch/system/applicationtemplates/v1/apache-web-logs.json b/src/main/resources/org/opensearch/system/applicationtemplates/v1/apache-web-logs.json new file mode 100644 index 0000000..8a5baab --- /dev/null +++ b/src/main/resources/org/opensearch/system/applicationtemplates/v1/apache-web-logs.json @@ -0,0 +1,499 @@ +{ + "template": { + "settings": { + "index": { + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "codec": "zstd_no_dict", + "merge.policy": "log_byte_size", + "refresh_interval": "60s" + } + }, + "mappings": { + "_meta": { + "correlations": [ + { + "field": "spanId", + "foreign-field": "spanId", + "foreign-schema": "traces" + }, + { + "field": "traceId", + "foreign-field": "traceId", + "foreign-schema": "traces" + } + ], + "labels": [ + "communication", + "http", + "log" + ] + }, + "dynamic_templates": [ + { + "request_header_map": { + "path_match": "request.header.*", + "mapping": { + "type": "keyword" + } + } + }, + { + "response_header_map": { + "path_match": "response.header.*", + "mapping": { + "type": "keyword" + } + } + }, + { + "resources_map": { + "path_match": "resource.*", + "mapping": { + "type": "keyword" + } + } + }, + { + "attributes_map": { + "path_match": "attributes.*", + "mapping": { + "type": "keyword" + } + } + }, + { + "instrumentation_scope_attributes_map": { + "path_match": "instrumentationScope.attributes.*", + "mapping": { + "type": "keyword" + } + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "body": { + "type": "text" + }, + "communication": { + "properties": { + "destination": { + "properties": { + "address": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "geo": { + "properties": { + "city_name": { + "type": "keyword" + }, + "country_iso_code": { + "type": "keyword" + }, + "country_name": { + "type": "keyword" + }, + "location": { + "type": "geo_point" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "type": "keyword", + "ignore_above": 1024 + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + }, + "sock": { + "properties": { + "family": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "source": { + "properties": { + "address": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "geo": { + "properties": { + "city_name": { + "type": "keyword" + }, + "country": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "country_iso_code": { + "type": "keyword" + }, + "country_name": { + "type": "keyword" + }, + "location": { + "type": "geo_point" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "type": "keyword", + "ignore_above": 1024 + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + } + } + }, + "event": { + "properties": { + "category": { + "type": "keyword", + "ignore_above": 256 + }, + "domain": { + "type": "keyword", + "ignore_above": 256 + }, + "exception": { + "properties": { + "message": { + "type": "keyword", + "ignore_above": 1024 + }, + "stacktrace": { + "type": "text" + }, + "type": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "kind": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "result": { + "type": "keyword", + "ignore_above": 256 + }, + "source": { + "type": "keyword", + "ignore_above": 256 + }, + "type": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "http": { + "properties": { + "client": { + "properties": { + "ip": { + "type": "ip" + } + } + }, + "flavor": { + "type": "keyword", + "ignore_above": 256 + }, + "request": { + "properties": { + "body": { + "properties": { + "content": { + "type": "text" + } + } + }, + "bytes": { + "type": "long" + }, + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "method": { + "type": "keyword", + "ignore_above": 256 + }, + "mime_type": { + "type": "keyword", + "ignore_above": 1024 + }, + "referrer": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "resent_count": { + "type": "integer" + }, + "response": { + "properties": { + "body": { + "properties": { + "content": { + "type": "text" + } + } + }, + "bytes": { + "type": "long" + }, + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "status_code": { + "type": "integer" + } + } + }, + "route": { + "type": "keyword", + "ignore_above": 1024 + }, + "schema": { + "type": "keyword", + "ignore_above": 1024 + }, + "target": { + "type": "keyword", + "ignore_above": 1024 + }, + "url": { + "type": "keyword", + "ignore_above": 2048 + }, + "user_agent": { + "properties": { + "device": { + "properties": { + "name": { + "type": "keyword" + } + } + }, + "name": { + "type": "keyword" + }, + "original": { + "type": "keyword" + }, + "os": { + "properties": { + "device": { + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "family": { + "type": "keyword" + }, + "full": { + "type": "keyword" + }, + "kernel": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "platform": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + } + }, + "version": { + "type": "keyword" + } + } + } + } + }, + "instrumentationScope": { + "properties": { + "dropped_attributes_count": { + "type": "integer" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 128 + } + } + }, + "schemaUrl": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "version": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "observedTimestamp": { + "type": "date" + }, + "observerTime": { + "type": "alias", + "path": "observedTimestamp" + }, + "schemaUrl": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "severity": { + "properties": { + "number": { + "type": "long" + }, + "text": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "spanId": { + "type": "keyword", + "ignore_above": 256 + }, + "traceId": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "aliases": { + "logs-aws-s3": {} + } + }, + "_meta": { + "_type": "@abc_template", + "_version": 1, + "_schema_ref": "https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/apache/apache-1.0.0.json" + }, + "version": 1 +} diff --git a/src/main/resources/org/opensearch/system/applicationtemplates/v1/k8s-logs.json b/src/main/resources/org/opensearch/system/applicationtemplates/v1/k8s-logs.json new file mode 100644 index 0000000..e5e2840 --- /dev/null +++ b/src/main/resources/org/opensearch/system/applicationtemplates/v1/k8s-logs.json @@ -0,0 +1,2837 @@ +{ + "template": { + "settings": { + "index": { + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "refresh_interval": "60s", + "codec": "zstd_no_dict", + "merge.policy": "log_byte_size" + } + }, + "mappings": { + "_meta": { + "correlations": [ + { + "field": "spanId", + "foreign-field": "spanId", + "foreign-schema": "traces" + }, + { + "field": "traceId", + "foreign-field": "traceId", + "foreign-schema": "traces" + } + ], + "labels": [ + "k8s", + "container", + "cloud" + ] + }, + "dynamic_templates": [ + { + "resources_map": { + "path_match": "resource.*", + "mapping": { + "type": "keyword" + } + } + }, + { + "attributes_map": { + "path_match": "attributes.*", + "mapping": { + "type": "keyword" + } + } + }, + { + "instrumentation_scope_attributes_map": { + "path_match": "instrumentationScope.attributes.*", + "mapping": { + "type": "keyword" + } + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "attributes": { + "properties": { + "k8s": { + "properties": { + "cluster": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "uid": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "container": { + "properties": { + "name": { + "type": "alias", + "path": "kubernetes.container.name" + }, + "restart_count": { + "type": "long" + } + } + }, + "daemonset": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "uid": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "deployment": { + "properties": { + "name": { + "type": "alias", + "path": "kubernetes.deployment.name" + }, + "uid": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "job": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "uid": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "namespace": { + "properties": { + "name": { + "type": "alias", + "path": "kubernetes.namespace" + } + } + }, + "node": { + "properties": { + "name": { + "type": "alias", + "path": "kubernetes.node.name" + }, + "uid": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "pod": { + "properties": { + "name": { + "type": "alias", + "path": "kubernetes.pod.name" + }, + "uid": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "replicaset": { + "properties": { + "name": { + "type": "alias", + "path": "kubernetes.replicaset.name" + }, + "uid": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "statefulset": { + "properties": { + "name": { + "type": "alias", + "path": "kubernetes.statefulset.name" + }, + "uid": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + }, + "body": { + "type": "text" + }, + "cloud": { + "properties": { + "account": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + }, + "availability_zone": { + "type": "keyword" + }, + "instance": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + }, + "machine": { + "properties": { + "type": { + "type": "keyword" + } + } + }, + "platform": { + "type": "keyword" + }, + "project": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + }, + "provider": { + "type": "keyword" + }, + "region": { + "type": "keyword" + }, + "resource_id": { + "type": "keyword" + }, + "service": { + "properties": { + "name": { + "type": "keyword" + } + } + } + } + }, + "container": { + "properties": { + "cpu": { + "properties": { + "usage": { + "type": "float" + } + } + }, + "disk": { + "properties": { + "read": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "write": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "id": { + "type": "keyword" + }, + "image": { + "properties": { + "hash": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "tag": { + "type": "keyword" + } + } + }, + "labels": { + "type": "keyword" + }, + "memory": { + "properties": { + "usage": { + "type": "float" + } + } + }, + "name": { + "type": "keyword" + }, + "network": { + "properties": { + "egress": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "ingress": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "runtime": { + "type": "keyword" + } + } + }, + "event": { + "properties": { + "category": { + "type": "keyword", + "ignore_above": 256 + }, + "dataset": { + "type": "keyword", + "ignore_above": 128 + }, + "domain": { + "type": "keyword", + "ignore_above": 256 + }, + "duration": { + "type": "long" + }, + "exception": { + "properties": { + "message": { + "type": "keyword", + "ignore_above": 1024 + }, + "stacktrace": { + "type": "text" + }, + "type": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "kind": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "result": { + "type": "keyword", + "ignore_above": 256 + }, + "source": { + "type": "keyword", + "ignore_above": 256 + }, + "type": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "instrumentationScope": { + "properties": { + "dropped_attributes_count": { + "type": "integer" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 128 + } + } + }, + "schemaUrl": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "version": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "kubernetes": { + "properties": { + "annotations": { + "type": "object" + }, + "apiserver": { + "properties": { + "request": { + "properties": { + "client": { + "type": "keyword", + "ignore_above": 256 + }, + "count": { + "type": "long" + }, + "latency": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + }, + "resource": { + "type": "keyword", + "ignore_above": 256 + }, + "scope": { + "type": "keyword", + "ignore_above": 256 + }, + "subresource": { + "type": "keyword", + "ignore_above": 256 + }, + "verb": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "container": { + "properties": { + "cpu": { + "properties": { + "limit": { + "properties": { + "cores": { + "type": "float" + }, + "nanocores": { + "type": "long" + } + } + }, + "request": { + "properties": { + "cores": { + "type": "float" + }, + "nanocores": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "core": { + "properties": { + "ns": { + "type": "long" + } + } + }, + "limit": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "nanocores": { + "type": "long" + }, + "node": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + } + } + } + } + }, + "id": { + "type": "keyword", + "ignore_above": 256 + }, + "image": { + "type": "keyword", + "ignore_above": 256 + }, + "logs": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "capacity": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "inodes": { + "properties": { + "count": { + "type": "long" + }, + "free": { + "type": "long" + }, + "used": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "memory": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "majorpagefaults": { + "type": "long" + }, + "pagefaults": { + "type": "long" + }, + "request": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "rss": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "limit": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "node": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + } + } + }, + "workingset": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "rootfs": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "capacity": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "inodes": { + "properties": { + "used": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "start_time": { + "type": "date" + }, + "status": { + "properties": { + "phase": { + "type": "keyword", + "ignore_above": 256 + }, + "ready": { + "type": "boolean" + }, + "reason": { + "type": "keyword", + "ignore_above": 256 + }, + "restarts": { + "type": "long" + } + } + } + } + }, + "controllermanager": { + "properties": { + "client": { + "properties": { + "request": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "code": { + "type": "keyword", + "ignore_above": 256 + }, + "handler": { + "type": "keyword", + "ignore_above": 256 + }, + "host": { + "type": "keyword", + "ignore_above": 256 + }, + "http": { + "properties": { + "request": { + "properties": { + "count": { + "type": "long" + }, + "duration": { + "properties": { + "us": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "double" + } + } + } + } + }, + "size": { + "properties": { + "bytes": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "long" + } + } + } + } + } + } + }, + "response": { + "properties": { + "size": { + "properties": { + "bytes": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "leader": { + "properties": { + "is_master": { + "type": "boolean" + } + } + }, + "method": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "node": { + "properties": { + "collector": { + "properties": { + "count": { + "type": "long" + }, + "eviction": { + "properties": { + "count": { + "type": "long" + } + } + }, + "health": { + "properties": { + "pct": { + "type": "long" + } + } + }, + "unhealthy": { + "properties": { + "count": { + "type": "long" + } + } + } + } + } + } + }, + "process": { + "properties": { + "cpu": { + "properties": { + "sec": { + "type": "double" + } + } + }, + "fds": { + "properties": { + "open": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "memory": { + "properties": { + "resident": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "virtual": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "started": { + "properties": { + "sec": { + "type": "double" + } + } + } + } + }, + "workqueue": { + "properties": { + "adds": { + "properties": { + "count": { + "type": "long" + } + } + }, + "depth": { + "properties": { + "count": { + "type": "long" + } + } + }, + "longestrunning": { + "properties": { + "sec": { + "type": "double" + } + } + }, + "retries": { + "properties": { + "count": { + "type": "long" + } + } + }, + "unfinished": { + "properties": { + "sec": { + "type": "double" + } + } + } + } + }, + "zone": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "deployment": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "paused": { + "type": "boolean" + }, + "replicas": { + "properties": { + "available": { + "type": "long" + }, + "desired": { + "type": "long" + }, + "unavailable": { + "type": "long" + }, + "updated": { + "type": "long" + } + } + } + } + }, + "event": { + "properties": { + "count": { + "type": "long" + }, + "involved_object": { + "properties": { + "api_version": { + "type": "keyword", + "ignore_above": 256 + }, + "kind": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "resource_version": { + "type": "keyword", + "ignore_above": 256 + }, + "uid": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "message": { + "type": "keyword", + "ignore_above": 256, + "copy_to": [ + "message" + ] + }, + "metadata": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "namespace": { + "type": "keyword", + "ignore_above": 256 + }, + "resource_version": { + "type": "keyword", + "ignore_above": 256 + }, + "self_link": { + "type": "keyword", + "ignore_above": 256 + }, + "timestamp": { + "properties": { + "created": { + "type": "date" + } + } + }, + "uid": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "reason": { + "type": "keyword", + "ignore_above": 256 + }, + "timestamp": { + "properties": { + "first_occurrence": { + "type": "date" + }, + "last_occurrence": { + "type": "date" + } + } + }, + "type": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "labels": { + "properties": { + "app": { + "properties": { + "kubernetes": { + "properties": { + "io/component": { + "type": "keyword", + "ignore_above": 256 + }, + "io/instance": { + "type": "keyword", + "ignore_above": 256 + }, + "io/managed-by": { + "type": "keyword", + "ignore_above": 256 + }, + "io/name": { + "type": "keyword", + "ignore_above": 256 + }, + "io/version": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "value": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "controller-revision-hash": { + "type": "keyword", + "ignore_above": 256 + }, + "eks": { + "properties": { + "amazonaws": { + "properties": { + "com/component": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "helm": { + "properties": { + "sh/chart": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "io": { + "properties": { + "kompose": { + "properties": { + "service": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "k8s-app": { + "type": "keyword", + "ignore_above": 256 + }, + "pod-template-generation": { + "type": "keyword", + "ignore_above": 256 + }, + "pod-template-hash": { + "type": "keyword", + "ignore_above": 256 + }, + "statefulset": { + "properties": { + "kubernetes": { + "properties": { + "io/pod-name": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + }, + "namespace": { + "type": "keyword", + "ignore_above": 256 + }, + "node": { + "properties": { + "_domain": { + "properties": { + "labels": { + "properties": { + "beta": { + "properties": { + "kubernetes": { + "properties": { + "io/arch": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "io/instance-type": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "io/os": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + }, + "eks": { + "properties": { + "amazonaws": { + "properties": { + "com/capacityType": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "com/nodegroup": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "com/nodegroup-image": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + }, + "failure-domain": { + "properties": { + "beta": { + "properties": { + "kubernetes": { + "properties": { + "io/region": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "io/zone": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + } + } + }, + "k8s": { + "properties": { + "io/cloud-provider-aws": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "kubernetes": { + "properties": { + "io/arch": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "io/hostname": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "io/os": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "node": { + "properties": { + "kubernetes": { + "properties": { + "io/instance-type": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + }, + "topology": { + "properties": { + "ebs": { + "properties": { + "csi": { + "properties": { + "aws": { + "properties": { + "com/zone": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + } + } + }, + "kubernetes": { + "properties": { + "io/region": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "io/zone": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + } + } + } + } + }, + "_module": { + "properties": { + "labels": { + "properties": { + "beta": { + "properties": { + "kubernetes": { + "properties": { + "io/arch": { + "type": "keyword", + "ignore_above": 256 + }, + "io/instance-type": { + "type": "keyword", + "ignore_above": 256 + }, + "io/os": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "eks": { + "properties": { + "amazonaws": { + "properties": { + "com/capacityType": { + "type": "keyword", + "ignore_above": 256 + }, + "com/nodegroup": { + "type": "keyword", + "ignore_above": 256 + }, + "com/nodegroup-image": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "failure-domain": { + "properties": { + "beta": { + "properties": { + "kubernetes": { + "properties": { + "io/region": { + "type": "keyword", + "ignore_above": 256 + }, + "io/zone": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + }, + "k8s": { + "properties": { + "io/cloud-provider-aws": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "kubernetes": { + "properties": { + "io/arch": { + "type": "keyword", + "ignore_above": 256 + }, + "io/hostname": { + "type": "keyword", + "ignore_above": 256 + }, + "io/os": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "node": { + "properties": { + "kubernetes": { + "properties": { + "io/instance-type": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "topology": { + "properties": { + "ebs": { + "properties": { + "csi": { + "properties": { + "aws": { + "properties": { + "com/zone": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + }, + "kubernetes": { + "properties": { + "io/region": { + "type": "keyword", + "ignore_above": 256 + }, + "io/zone": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + } + } + }, + "cpu": { + "properties": { + "allocatable": { + "properties": { + "cores": { + "type": "float" + } + } + }, + "capacity": { + "properties": { + "cores": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "core": { + "properties": { + "ns": { + "type": "long" + } + } + }, + "nanocores": { + "type": "long" + } + } + } + } + }, + "fs": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "capacity": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "inodes": { + "properties": { + "count": { + "type": "long" + }, + "free": { + "type": "long" + }, + "used": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "memory": { + "properties": { + "allocatable": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "capacity": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "majorpagefaults": { + "type": "long" + }, + "pagefaults": { + "type": "long" + }, + "rss": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "workingset": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "network": { + "properties": { + "rx": { + "properties": { + "bytes": { + "type": "long" + }, + "errors": { + "type": "long" + } + } + }, + "tx": { + "properties": { + "bytes": { + "type": "long" + }, + "errors": { + "type": "long" + } + } + } + } + }, + "pod": { + "properties": { + "allocatable": { + "properties": { + "total": { + "type": "long" + } + } + }, + "capacity": { + "properties": { + "total": { + "type": "long" + } + } + } + } + }, + "runtime": { + "properties": { + "imagefs": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "capacity": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "start_time": { + "type": "date" + }, + "status": { + "properties": { + "ready": { + "type": "keyword", + "ignore_above": 256 + }, + "unschedulable": { + "type": "boolean" + } + } + } + } + }, + "pod": { + "properties": { + "cpu": { + "properties": { + "usage": { + "properties": { + "limit": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "nanocores": { + "type": "long" + }, + "node": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + } + } + } + } + }, + "host_ip": { + "type": "ip" + }, + "ip": { + "type": "ip" + }, + "memory": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "major_page_faults": { + "type": "long" + }, + "page_faults": { + "type": "long" + }, + "rss": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "limit": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + "node": { + "properties": { + "pct": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + } + } + }, + "working_set": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "network": { + "properties": { + "rx": { + "properties": { + "bytes": { + "type": "long" + }, + "errors": { + "type": "long" + } + } + }, + "tx": { + "properties": { + "bytes": { + "type": "long" + }, + "errors": { + "type": "long" + } + } + } + } + }, + "start_time": { + "type": "date" + }, + "status": { + "properties": { + "phase": { + "type": "keyword", + "ignore_above": 256 + }, + "ready": { + "type": "keyword", + "ignore_above": 256 + }, + "scheduled": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "uid": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "proxy": { + "properties": { + "client": { + "properties": { + "request": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "code": { + "type": "keyword", + "ignore_above": 256 + }, + "handler": { + "type": "keyword", + "ignore_above": 256 + }, + "host": { + "type": "keyword", + "ignore_above": 256 + }, + "http": { + "properties": { + "request": { + "properties": { + "count": { + "type": "long" + }, + "duration": { + "properties": { + "us": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "double" + } + } + } + } + }, + "size": { + "properties": { + "bytes": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "long" + } + } + } + } + } + } + }, + "response": { + "properties": { + "size": { + "properties": { + "bytes": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "method": { + "type": "keyword", + "ignore_above": 256 + }, + "process": { + "properties": { + "cpu": { + "properties": { + "sec": { + "type": "double" + } + } + }, + "fds": { + "properties": { + "open": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "memory": { + "properties": { + "resident": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "virtual": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "started": { + "properties": { + "sec": { + "type": "double" + } + } + } + } + }, + "sync": { + "properties": { + "networkprogramming": { + "properties": { + "duration": { + "properties": { + "us": { + "properties": { + "bucket": { + "properties": { + "0": { + "type": "long" + }, + "250000": { + "type": "long" + }, + "500000": { + "type": "long" + }, + "1000000": { + "type": "long" + }, + "2000000": { + "type": "long" + }, + "3000000": { + "type": "long" + }, + "4000000": { + "type": "long" + }, + "5000000": { + "type": "long" + }, + "6000000": { + "type": "long" + }, + "7000000": { + "type": "long" + }, + "8000000": { + "type": "long" + }, + "9000000": { + "type": "long" + }, + "10000000": { + "type": "long" + }, + "11000000": { + "type": "long" + }, + "12000000": { + "type": "long" + }, + "13000000": { + "type": "long" + }, + "14000000": { + "type": "long" + }, + "15000000": { + "type": "long" + }, + "16000000": { + "type": "long" + }, + "17000000": { + "type": "long" + }, + "18000000": { + "type": "long" + }, + "19000000": { + "type": "long" + }, + "20000000": { + "type": "long" + }, + "21000000": { + "type": "long" + }, + "22000000": { + "type": "long" + }, + "23000000": { + "type": "long" + }, + "24000000": { + "type": "long" + }, + "25000000": { + "type": "long" + }, + "26000000": { + "type": "long" + }, + "27000000": { + "type": "long" + }, + "28000000": { + "type": "long" + }, + "29000000": { + "type": "long" + }, + "30000000": { + "type": "long" + }, + "31000000": { + "type": "long" + }, + "32000000": { + "type": "long" + }, + "33000000": { + "type": "long" + }, + "34000000": { + "type": "long" + }, + "35000000": { + "type": "long" + }, + "36000000": { + "type": "long" + }, + "37000000": { + "type": "long" + }, + "38000000": { + "type": "long" + }, + "39000000": { + "type": "long" + }, + "40000000": { + "type": "long" + }, + "41000000": { + "type": "long" + }, + "42000000": { + "type": "long" + }, + "43000000": { + "type": "long" + }, + "44000000": { + "type": "long" + }, + "45000000": { + "type": "long" + }, + "46000000": { + "type": "long" + }, + "47000000": { + "type": "long" + }, + "48000000": { + "type": "long" + }, + "49000000": { + "type": "long" + }, + "50000000": { + "type": "long" + }, + "51000000": { + "type": "long" + }, + "52000000": { + "type": "long" + }, + "53000000": { + "type": "long" + }, + "54000000": { + "type": "long" + }, + "55000000": { + "type": "long" + }, + "56000000": { + "type": "long" + }, + "57000000": { + "type": "long" + }, + "58000000": { + "type": "long" + }, + "59000000": { + "type": "long" + }, + "60000000": { + "type": "long" + }, + "65000000": { + "type": "long" + }, + "70000000": { + "type": "long" + }, + "75000000": { + "type": "long" + }, + "80000000": { + "type": "long" + }, + "85000000": { + "type": "long" + }, + "90000000": { + "type": "long" + }, + "95000000": { + "type": "long" + }, + "100000000": { + "type": "long" + }, + "105000000": { + "type": "long" + }, + "110000000": { + "type": "long" + }, + "115000000": { + "type": "long" + }, + "120000000": { + "type": "long" + }, + "150000000": { + "type": "long" + }, + "180000000": { + "type": "long" + }, + "210000000": { + "type": "long" + }, + "240000000": { + "type": "long" + }, + "270000000": { + "type": "long" + }, + "300000000": { + "type": "long" + }, + "*": { + "type": "object" + }, + "+Inf": { + "type": "long" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + } + } + } + } + }, + "rules": { + "properties": { + "duration": { + "properties": { + "us": { + "properties": { + "bucket": { + "properties": { + "1000": { + "type": "long" + }, + "2000": { + "type": "long" + }, + "4000": { + "type": "long" + }, + "8000": { + "type": "long" + }, + "16000": { + "type": "long" + }, + "32000": { + "type": "long" + }, + "64000": { + "type": "long" + }, + "128000": { + "type": "long" + }, + "256000": { + "type": "long" + }, + "512000": { + "type": "long" + }, + "1024000": { + "type": "long" + }, + "2048000": { + "type": "long" + }, + "4096000": { + "type": "long" + }, + "8192000": { + "type": "long" + }, + "16384000": { + "type": "long" + }, + "*": { + "type": "object" + }, + "+Inf": { + "type": "long" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + } + } + } + } + } + } + } + } + }, + "replicaset": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "replicas": { + "properties": { + "available": { + "type": "long" + }, + "desired": { + "type": "long" + }, + "labeled": { + "type": "long" + }, + "observed": { + "type": "long" + }, + "ready": { + "type": "long" + } + } + } + } + }, + "scheduler": { + "properties": { + "client": { + "properties": { + "request": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "code": { + "type": "keyword", + "ignore_above": 256 + }, + "handler": { + "type": "keyword", + "ignore_above": 256 + }, + "host": { + "type": "keyword", + "ignore_above": 256 + }, + "http": { + "properties": { + "request": { + "properties": { + "count": { + "type": "long" + }, + "duration": { + "properties": { + "us": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "double" + } + } + } + } + }, + "size": { + "properties": { + "bytes": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "long" + } + } + } + } + } + } + }, + "response": { + "properties": { + "size": { + "properties": { + "bytes": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "leader": { + "properties": { + "is_master": { + "type": "boolean" + } + } + }, + "method": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "operation": { + "type": "keyword", + "ignore_above": 256 + }, + "process": { + "properties": { + "cpu": { + "properties": { + "sec": { + "type": "double" + } + } + }, + "fds": { + "properties": { + "open": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "memory": { + "properties": { + "resident": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "virtual": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "started": { + "properties": { + "sec": { + "type": "double" + } + } + } + } + }, + "result": { + "type": "keyword", + "ignore_above": 256 + }, + "scheduling": { + "properties": { + "duration": { + "properties": { + "seconds": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "double" + } + } + } + } + }, + "e2e": { + "properties": { + "duration": { + "properties": { + "us": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + } + } + } + } + }, + "pod": { + "properties": { + "attempts": { + "properties": { + "count": { + "type": "long" + } + } + }, + "preemption": { + "properties": { + "victims": { + "properties": { + "count": { + "type": "long" + } + } + } + } + } + } + } + } + } + } + }, + "statefulset": { + "properties": { + "created": { + "type": "long" + }, + "generation": { + "properties": { + "desired": { + "type": "long" + }, + "observed": { + "type": "long" + } + } + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "replicas": { + "properties": { + "desired": { + "type": "long" + }, + "observed": { + "type": "long" + } + } + } + } + }, + "system": { + "properties": { + "container": { + "type": "keyword", + "ignore_above": 256 + }, + "cpu": { + "properties": { + "usage": { + "properties": { + "core": { + "properties": { + "ns": { + "type": "long" + } + } + }, + "nanocores": { + "type": "long" + } + } + } + } + }, + "memory": { + "properties": { + "majorpagefaults": { + "type": "long" + }, + "pagefaults": { + "type": "long" + }, + "rss": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "workingset": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "start_time": { + "type": "date" + } + } + }, + "volume": { + "properties": { + "fs": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "capacity": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "inodes": { + "properties": { + "count": { + "type": "long" + }, + "free": { + "type": "long" + }, + "used": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "name": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "observedTimestamp": { + "type": "date" + }, + "observerTime": { + "type": "alias", + "path": "observedTimestamp" + }, + "schemaUrl": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "severity": { + "properties": { + "number": { + "type": "long" + }, + "text": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "spanId": { + "type": "keyword", + "ignore_above": 256 + }, + "traceId": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "aliases": { + "logs-k8s": {} + } + }, + "_meta": { + "_type": "@abc_template", + "_version": 1, + "_schema_ref": "https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/k8s/k8s-1.0.0.json" + }, + "version": 1 +} diff --git a/src/main/resources/org/opensearch/system/applicationtemplates/v1/logs.json b/src/main/resources/org/opensearch/system/applicationtemplates/v1/logs.json index 25e5bf1..0a0c6c0 100644 --- a/src/main/resources/org/opensearch/system/applicationtemplates/v1/logs.json +++ b/src/main/resources/org/opensearch/system/applicationtemplates/v1/logs.json @@ -1,7 +1,7 @@ { "template": { "settings": { - "codec": "best_compression", + "codec": "zstd_no_dict", "merge.policy": "log_byte_size", "refresh_interval": "60s" } diff --git a/src/main/resources/org/opensearch/system/applicationtemplates/v1/metrics.json b/src/main/resources/org/opensearch/system/applicationtemplates/v1/metrics.json index 25e5bf1..0a0c6c0 100644 --- a/src/main/resources/org/opensearch/system/applicationtemplates/v1/metrics.json +++ b/src/main/resources/org/opensearch/system/applicationtemplates/v1/metrics.json @@ -1,7 +1,7 @@ { "template": { "settings": { - "codec": "best_compression", + "codec": "zstd_no_dict", "merge.policy": "log_byte_size", "refresh_interval": "60s" } diff --git a/src/main/resources/org/opensearch/system/applicationtemplates/v1/nginx-logs.json b/src/main/resources/org/opensearch/system/applicationtemplates/v1/nginx-logs.json new file mode 100644 index 0000000..a720071 --- /dev/null +++ b/src/main/resources/org/opensearch/system/applicationtemplates/v1/nginx-logs.json @@ -0,0 +1,496 @@ +{ + "template": { + "settings": { + "index": { + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "codec": "zstd_no_dict", + "merge.policy": "log_byte_size", + "refresh_interval": "60s" + } + }, + "mappings": { + "_meta": { + "correlations": [ + { + "field": "spanId", + "foreign-field": "spanId", + "foreign-schema": "traces" + }, + { + "field": "traceId", + "foreign-field": "traceId", + "foreign-schema": "traces" + } + ], + "labels": [ + "communication", + "http", + "log" + ] + }, + "dynamic_templates": [ + { + "request_header_map": { + "path_match": "request.header.*", + "mapping": { + "type": "keyword" + } + } + }, + { + "response_header_map": { + "path_match": "response.header.*", + "mapping": { + "type": "keyword" + } + } + }, + { + "resources_map": { + "path_match": "resource.*", + "mapping": { + "type": "keyword" + } + } + }, + { + "attributes_map": { + "path_match": "attributes.*", + "mapping": { + "type": "keyword" + } + } + }, + { + "instrumentation_scope_attributes_map": { + "path_match": "instrumentationScope.attributes.*", + "mapping": { + "type": "keyword" + } + } + } + ], + "properties": { + "@message": { + "type": "alias", + "path": "body" + }, + "@timestamp": { + "type": "date" + }, + "body": { + "type": "text" + }, + "communication": { + "properties": { + "destination": { + "properties": { + "address": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "geo": { + "properties": { + "city_name": { + "type": "keyword" + }, + "country_iso_code": { + "type": "keyword" + }, + "country_name": { + "type": "keyword" + }, + "location": { + "type": "geo_point" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "type": "keyword", + "ignore_above": 1024 + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + }, + "sock": { + "properties": { + "family": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "source": { + "properties": { + "address": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "geo": { + "properties": { + "city_name": { + "type": "keyword" + }, + "country_iso_code": { + "type": "keyword" + }, + "country_name": { + "type": "keyword" + }, + "location": { + "type": "geo_point" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "type": "keyword", + "ignore_above": 1024 + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + } + } + }, + "event": { + "properties": { + "category": { + "type": "keyword", + "ignore_above": 256 + }, + "domain": { + "type": "keyword", + "ignore_above": 256 + }, + "exception": { + "properties": { + "message": { + "type": "keyword", + "ignore_above": 1024 + }, + "stacktrace": { + "type": "text" + }, + "type": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "kind": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "result": { + "type": "keyword", + "ignore_above": 256 + }, + "source": { + "type": "keyword", + "ignore_above": 256 + }, + "type": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "http": { + "properties": { + "client": { + "properties": { + "ip": { + "type": "ip" + } + } + }, + "flavor": { + "type": "keyword", + "ignore_above": 256 + }, + "request": { + "properties": { + "body": { + "properties": { + "content": { + "type": "text" + } + } + }, + "bytes": { + "type": "long" + }, + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "method": { + "type": "keyword", + "ignore_above": 256 + }, + "mime_type": { + "type": "keyword", + "ignore_above": 1024 + }, + "referrer": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "resent_count": { + "type": "integer" + }, + "response": { + "properties": { + "body": { + "properties": { + "content": { + "type": "text" + } + } + }, + "bytes": { + "type": "long" + }, + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "status_code": { + "type": "integer" + } + } + }, + "route": { + "type": "keyword", + "ignore_above": 1024 + }, + "schema": { + "type": "keyword", + "ignore_above": 1024 + }, + "target": { + "type": "keyword", + "ignore_above": 1024 + }, + "url": { + "type": "keyword", + "ignore_above": 2048 + }, + "user_agent": { + "properties": { + "device": { + "properties": { + "name": { + "type": "keyword" + } + } + }, + "name": { + "type": "keyword" + }, + "original": { + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "type": "keyword" + }, + "full": { + "type": "keyword" + }, + "kernel": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "platform": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + } + }, + "version": { + "type": "keyword" + } + } + } + } + }, + "instrumentationScope": { + "properties": { + "dropped_attributes_count": { + "type": "integer" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 128 + } + } + }, + "schemaUrl": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "version": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "observedTimestamp": { + "type": "date" + }, + "observerTime": { + "type": "alias", + "path": "observedTimestamp" + }, + "schemaUrl": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "severity": { + "properties": { + "number": { + "type": "long" + }, + "text": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "spanId": { + "type": "keyword", + "ignore_above": 256 + }, + "span_id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "traceId": { + "type": "keyword", + "ignore_above": 256 + }, + "trace_id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + }, + "_meta": { + "_type": "@abc_template", + "_version": 1, + "_schema_ref": "https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/nginx/nginx-1.0.0.json" + }, + "version": 1 +} diff --git a/src/test/java/org/opensearch/system/applicationtemplates/LocalSystemTemplateRepositoryTests.java b/src/test/java/org/opensearch/system/applicationtemplates/LocalSystemTemplateRepositoryTests.java index 1ddff6f..4da2cbb 100644 --- a/src/test/java/org/opensearch/system/applicationtemplates/LocalSystemTemplateRepositoryTests.java +++ b/src/test/java/org/opensearch/system/applicationtemplates/LocalSystemTemplateRepositoryTests.java @@ -34,7 +34,7 @@ public void testRepositoryListTemplates() throws Exception { AtomicInteger counter = new AtomicInteger(); repository.listTemplates().forEach(templateMetadata -> { counter.incrementAndGet(); }); - assertEquals(counter.get(), 2); + assertEquals(counter.get(), 8); } } From e1bffe3c713237af5ec2ce1589ce2fa6e0e9bf6a Mon Sep 17 00:00:00 2001 From: Mohit Godwani Date: Tue, 3 Sep 2024 16:17:25 +0530 Subject: [PATCH 2/2] Add version 2.x for artefacts Signed-off-by: Mohit Godwani --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index cdc9b36..1816e9a 100644 --- a/build.gradle +++ b/build.gradle @@ -31,7 +31,7 @@ buildscript { ext { opensearch_group = "org.opensearch" - opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.17.0-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") } @@ -96,7 +96,7 @@ opensearchplugin { } dependencies { - zipArchive group: 'org.opensearch.plugin', name:'opensearch-custom-codecs', version: "3.0.0.0-SNAPSHOT" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-custom-codecs', version: "2.17.0.0-SNAPSHOT" } allprojects {