diff --git a/build.gradle b/build.gradle index 6cdfd7a3f7..6245073ff9 100644 --- a/build.gradle +++ b/build.gradle @@ -15,7 +15,7 @@ buildscript { ext { - es_version = "7.7.0" + es_version = "7.8.0" } // This isn't applying from repositories.gradle so repeating it here repositories { @@ -40,7 +40,7 @@ repositories { } ext { - opendistroVersion = '1.8.0' + opendistroVersion = '1.9.0' isSnapshot = "true" == System.getProperty("build.snapshot", "true") } @@ -97,6 +97,10 @@ licenseHeaders { excludes = ['com/amazon/opendistroforelasticsearch/sql/antlr/parser/**'] } +tasks.withType(licenseHeaders.class) { + additionalLicense 'AL ', 'Apache', 'Licensed under the Apache License, Version 2.0 (the "License")' +} + // TODO: need to fix java doc to enable JavaDoc javadoc.enabled = false esplugin { @@ -272,7 +276,7 @@ dependencies { testCompile group: "org.elasticsearch.client", name: 'transport', version: "${es_version}" // JDBC drivers for comparison test. Somehow Apache Derby throws security permission exception. - testCompile group: 'com.amazon.opendistroforelasticsearch.client', name: 'opendistro-sql-jdbc', version: '1.3.0.0' + testCompile group: 'com.amazon.opendistroforelasticsearch.client', name: 'opendistro-sql-jdbc', version: '1.8.0.0' testCompile group: 'com.h2database', name: 'h2', version: '1.4.200' testCompile group: 'org.xerial', name: 'sqlite-jdbc', version: '3.28.0' //testCompile group: 'org.apache.derby', name: 'derby', version: '10.15.1.3' @@ -280,6 +284,8 @@ dependencies { apply plugin: 'nebula.ospackage' +validateNebulaPom.enabled = false + // This is afterEvaluate because the bundlePlugin ZIP task is updated afterEvaluate and changes the ZIP name to match the plugin name afterEvaluate { ospackage { diff --git a/gradle.properties b/gradle.properties index c740b5d54c..5896cf54df 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,4 +13,4 @@ # permissions and limitations under the License. # -version=1.8.0 +version=1.9.0 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index cc3d06cf2c..75afb6ed35 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -13,7 +13,7 @@ # permissions and limitations under the License. # -distributionUrl=https\://services.gradle.org/distributions/gradle-6.4-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-all.zip distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStorePath=wrapper/dists diff --git a/release-notes/opendistro-elasticsearch-sql.release-notes-1.9.0.md b/release-notes/opendistro-elasticsearch-sql.release-notes-1.9.0.md new file mode 100644 index 0000000000..22322e845b --- /dev/null +++ b/release-notes/opendistro-elasticsearch-sql.release-notes-1.9.0.md @@ -0,0 +1,16 @@ +## 2020-06-23 Version 1.9.0.0 + +### Features +#### Elasticsearch Compatibility +* Feature [#532](https://github.com/opendistro-for-elasticsearch/sql/pull/532): Elasticsearch 7.8.0 compatibility (issue: [#531](https://github.com/opendistro-for-elasticsearch/sql/issues/531)) + +#### Documentation +* Feature [#486](https://github.com/opendistro-for-elasticsearch/sql/pull/486): Add Github badges to README + +### Enhancements +#### SQL Features +* Feature [#473](https://github.com/opendistro-for-elasticsearch/sql/pull/473): Support Integration Tests for Security enabled ODFE cluster + +### Bugfixes +* Bugfix [#489](https://github.com/opendistro-for-elasticsearch/sql/pull/489): Fix ANTLR grammar for negative integer and floating point number (issue: [#488](https://github.com/opendistro-for-elasticsearch/sql/issues/488)) +* Bugfix [#522](https://github.com/opendistro-for-elasticsearch/sql/pull/522): Bug fix, support long type for aggregation (issue: [#521](https://github.com/opendistro-for-elasticsearch/sql/issues/521)) diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java index 2d33ea53e2..44ff0d8670 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java @@ -109,11 +109,11 @@ public void setClusterService(ClusterService clusterService) { this.clusterService = clusterService; clusterService.addListener(event -> { - if (event.metaDataChanged()) { + if (event.metadataChanged()) { // State in cluster service is already changed to event.state() before listener fired if (LOG.isDebugEnabled()) { LOG.debug("Metadata in cluster state changed: {}", - new IndexMappings(clusterService.state().metaData())); + new IndexMappings(clusterService.state().metadata())); } cache.invalidateAll(); } @@ -169,8 +169,8 @@ public IndexMappings getFieldMappings(String[] indices, String[] types) { } /** - * Get field mappings by index expressions, type and field filter. Because IndexMetaData/MappingMetaData - * is hard to convert to FieldMappingMetaData, custom mapping domain objects are being used here. In future, + * Get field mappings by index expressions, type and field filter. Because IndexMetadata/MappingMetadata + * is hard to convert to FieldMappingMetadata, custom mapping domain objects are being used here. In future, * it should be moved to domain model layer for all ES specific knowledge. *

* Note that cluster state may be change inside ES so it's possible to read different state in 2 accesses @@ -222,7 +222,7 @@ private IndexMappings findMappings(ClusterState state, String[] indices, String[ Function> fieldFilter) throws IOException { LOG.debug("Cache didn't help. Load and parse mapping in cluster state"); return new IndexMappings( - state.metaData().findMappings(indices, types, fieldFilter) + state.metadata().findMappings(indices, types, fieldFilter) ); } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMapping.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMapping.java index 0bbacc0327..63ecb50e64 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMapping.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMapping.java @@ -21,7 +21,7 @@ import java.util.Map; import static java.util.Collections.emptyMap; -import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData; +import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetadata; /** * Field mapping that parses native ES mapping. @@ -39,7 +39,7 @@ public class FieldMapping { /** * Native mapping information returned from ES */ - private final Map typeMappings; + private final Map typeMappings; /** * Maps a field name to Field object that specified in query explicitly @@ -51,7 +51,7 @@ public FieldMapping(String fieldName) { } public FieldMapping(String fieldName, - Map typeMappings, + Map typeMappings, Map specifiedFieldByNames) { this.fieldName = fieldName; @@ -119,16 +119,16 @@ public String path() { } /** - * Used to retrieve the type of fields from metaData map structures for both regular and nested fields + * Used to retrieve the type of fields from metadata map structures for both regular and nested fields */ @SuppressWarnings("unchecked") public String type() { - FieldMappingMetaData metaData = typeMappings.get(fieldName); - Map source = metaData.sourceAsMap(); + FieldMappingMetadata metadata = typeMappings.get(fieldName); + Map source = metadata.sourceAsMap(); String[] fieldPath = fieldName.split("\\."); /* - * When field is not nested the metaData source is fieldName -> type + * When field is not nested the metadata source is fieldName -> type * When it is nested or contains "." in general (ex. fieldName.nestedName) the source is nestedName -> type */ String root = (fieldPath.length == 1) ? fieldName : fieldPath[1]; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappings.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappings.java index f21a320d4c..e245a95659 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappings.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappings.java @@ -15,7 +15,7 @@ package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping; -import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.json.JSONObject; import java.util.HashMap; @@ -61,7 +61,7 @@ public class FieldMappings implements Mappings> { */ private final Map fieldMappings; - public FieldMappings(MappingMetaData mappings) { + public FieldMappings(MappingMetadata mappings) { fieldMappings = mappings.sourceAsMap(); } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/IndexMappings.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/IndexMappings.java index 283ba3ff30..09bddf9bda 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/IndexMappings.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/IndexMappings.java @@ -15,8 +15,8 @@ package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.collect.ImmutableOpenMap; import java.util.Map; @@ -55,12 +55,12 @@ public IndexMappings() { this.indexMappings = emptyMap(); } - public IndexMappings(MetaData metaData) { - this.indexMappings = buildMappings(metaData.indices(), - indexMetaData -> new TypeMappings(indexMetaData.getMappings())); + public IndexMappings(Metadata metadata) { + this.indexMappings = buildMappings(metadata.indices(), + indexMetadata -> new TypeMappings(indexMetadata.getMappings())); } - public IndexMappings(ImmutableOpenMap> mappings) { + public IndexMappings(ImmutableOpenMap> mappings) { this.indexMappings = buildMappings(mappings, TypeMappings::new); } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/TypeMappings.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/TypeMappings.java index 00f7b849c0..10f0ce0d8a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/TypeMappings.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/TypeMappings.java @@ -15,7 +15,7 @@ package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping; -import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.collect.ImmutableOpenMap; import java.util.Map; @@ -36,7 +36,7 @@ public class TypeMappings implements Mappings { */ private final Map typeMappings; - public TypeMappings(ImmutableOpenMap mappings) { + public TypeMappings(ImmutableOpenMap mappings) { typeMappings = buildMappings(mappings, FieldMappings::new); } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/AsyncRestExecutor.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/AsyncRestExecutor.java index 708942b137..d670d118fa 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/AsyncRestExecutor.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/AsyncRestExecutor.java @@ -143,7 +143,7 @@ private void async(Client client, Map params, QueryAction queryA // Preserve context of calling thread to ensure headers of requests are forwarded when running blocking actions threadPool.schedule( - threadPool.preserveContext(LogUtils.withCurrentContext(runnable)), + LogUtils.withCurrentContext(runnable), new TimeValue(0L), SQL_WORKER_THREAD_POOL_NAME ); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/GetIndexRequestRestListener.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/GetIndexRequestRestListener.java index a81bcf9429..cb7259dceb 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/GetIndexRequestRestListener.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/GetIndexRequestRestListener.java @@ -19,8 +19,8 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; -import org.elasticsearch.cluster.metadata.AliasMetaData; -import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; @@ -77,22 +77,22 @@ public RestResponse buildResponse(GetIndexResponse getIndexResponse, XContentBui return new BytesRestResponse(RestStatus.OK, builder); } - private void writeAliases(List aliases, XContentBuilder builder, ToXContent.Params params) + private void writeAliases(List aliases, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(Fields.ALIASES); if (aliases != null) { - for (AliasMetaData alias : aliases) { - AliasMetaData.Builder.toXContent(alias, builder, params); + for (AliasMetadata alias : aliases) { + AliasMetadata.Builder.toXContent(alias, builder, params); } } builder.endObject(); } - private void writeMappings(ImmutableOpenMap mappings, + private void writeMappings(ImmutableOpenMap mappings, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(Fields.MAPPINGS); if (mappings != null) { - for (ObjectObjectCursor typeEntry : mappings) { + for (ObjectObjectCursor typeEntry : mappings) { builder.field(typeEntry.key); builder.map(typeEntry.value.sourceAsMap()); } @@ -114,4 +114,4 @@ static class Fields { static final String SETTINGS = "settings"; static final String WARMERS = "warmers"; } -} \ No newline at end of file +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorAsyncRestExecutor.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorAsyncRestExecutor.java index 84d19e945e..8f986f9366 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorAsyncRestExecutor.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorAsyncRestExecutor.java @@ -93,7 +93,7 @@ private void async(Client client, Map params, RestChannel channe // Preserve context of calling thread to ensure headers of requests are forwarded when running blocking actions threadPool.schedule( - threadPool.preserveContext(LogUtils.withCurrentContext(runnable)), + LogUtils.withCurrentContext(runnable), new TimeValue(0L), SQL_WORKER_THREAD_POOL_NAME ); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DescribeResultSet.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DescribeResultSet.java index 04044f8360..9f6b1b7a70 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DescribeResultSet.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DescribeResultSet.java @@ -22,7 +22,7 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.collect.ImmutableOpenMap; import java.util.ArrayList; @@ -89,20 +89,20 @@ private List loadColumns() { private List loadRows() { List rows = new ArrayList<>(); GetIndexResponse indexResponse = (GetIndexResponse) queryResult; - ImmutableOpenMap> indexMappings = indexResponse.getMappings(); + ImmutableOpenMap> indexMappings = indexResponse.getMappings(); // Iterate through indices in indexMappings - for (ObjectObjectCursor> indexCursor : indexMappings) { + for (ObjectObjectCursor> indexCursor : indexMappings) { String index = indexCursor.key; // Check to see if index matches given pattern if (matchesPattern(index, statement.getIndexPattern())) { - ImmutableOpenMap typeMapping = indexCursor.value; + ImmutableOpenMap typeMapping = indexCursor.value; // Assuming ES 6.x, iterate through the only type of the index to get mapping data - for (ObjectObjectCursor typeCursor : typeMapping) { - MappingMetaData mappingMetaData = typeCursor.value; + for (ObjectObjectCursor typeCursor : typeMapping) { + MappingMetadata mappingMetadata = typeCursor.value; // Load rows for each field in the mapping - rows.addAll(loadIndexData(index, mappingMetaData.getSourceAsMap())); + rows.addAll(loadIndexData(index, mappingMetadata.getSourceAsMap())); } } } @@ -110,12 +110,12 @@ private List loadRows() { } @SuppressWarnings("unchecked") - private List loadIndexData(String index, Map mappingMetaData) { + private List loadIndexData(String index, Map mappingMetadata) { List rows = new ArrayList<>(); - Map flattenedMetaData = flattenMappingMetaData(mappingMetaData, "", new HashMap<>()); + Map flattenedMetadata = flattenMappingMetadata(mappingMetadata, "", new HashMap<>()); int position = 1; // Used as an arbitrary ORDINAL_POSITION value for the time being - for (Entry entry : flattenedMetaData.entrySet()) { + for (Entry entry : flattenedMetadata.entrySet()) { String columnPattern = statement.getColumnPattern(); // Check to see if column name matches pattern, if given @@ -153,21 +153,21 @@ private Map loadRowData(String index, String column, String type * 'GetIndexRequestBuilder' that was used in the old ShowQueryAction. Since the format of the resulting meta data * is different, this method is being used to flatten and retrieve types. *

- * In the future, should look for a way to generalize this since Schema is currently using FieldMappingMetaData - * whereas here we are using MappingMetaData. + * In the future, should look for a way to generalize this since Schema is currently using FieldMappingMetadata + * whereas here we are using MappingMetadata. */ @SuppressWarnings("unchecked") - private Map flattenMappingMetaData(Map mappingMetaData, + private Map flattenMappingMetadata(Map mappingMetadata, String currPath, Map flattenedMapping) { - Map properties = (Map) mappingMetaData.get("properties"); + Map properties = (Map) mappingMetadata.get("properties"); for (Entry entry : properties.entrySet()) { - Map metaData = (Map) entry.getValue(); + Map metadata = (Map) entry.getValue(); String fullPath = addToPath(currPath, entry.getKey()); - flattenedMapping.put(fullPath, (String) metaData.getOrDefault("type", DEFAULT_OBJECT_DATATYPE)); - if (metaData.containsKey("properties")) { - flattenedMapping = flattenMappingMetaData(metaData, fullPath, flattenedMapping); + flattenedMapping.put(fullPath, (String) metadata.getOrDefault("type", DEFAULT_OBJECT_DATATYPE)); + if (metadata.containsKey("properties")) { + flattenedMapping = flattenMappingMetadata(metadata, fullPath, flattenedMapping); } } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java index ece165a951..7a7cee6ef3 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java @@ -65,7 +65,7 @@ import static java.util.Collections.unmodifiableMap; import static java.util.stream.Collectors.toSet; -import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData; +import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetadata; public class SelectResultSet extends ResultSet { @@ -188,13 +188,13 @@ private void loadFromEsState(Query query) { .getFieldMappings(request) .actionGet(); - Map>> mappings = response.mappings(); + Map>> mappings = response.mappings(); if (mappings.isEmpty()) { throw new IllegalArgumentException(String.format("Index type %s does not exist", query.getFrom())); } // Assumption is all indices share the same mapping which is validated in TermFieldRewriter. - Map> indexMappings = mappings.values().iterator().next(); + Map> indexMappings = mappings.values().iterator().next(); // if index mappings size is 0 and the expression is a cast: that means that we are casting by alias // if so, add the original field that was being looked at to the mapping (how?) @@ -205,7 +205,7 @@ private void loadFromEsState(Query query) { * 2. If the incorrect type name was given then the response is null * 3. If no type name is given, the indexMapping is searched for a typeMapping */ - Map typeMappings = new HashMap<>(); + Map typeMappings = new HashMap<>(); if (indexMappings.containsKey(typeName)) { typeMappings = indexMappings.get(typeName); } else { @@ -416,7 +416,7 @@ private Schema.Type fetchMethodReturnType(int fieldIndex, MethodField field) { * will be used. */ private List populateColumns(Query query, String[] fieldNames, Map typeMappings) { + FieldMappingMetadata> typeMappings) { List fieldNameList; if (isSelectAll() || containsWildcard(query)) { @@ -911,4 +911,4 @@ private Map addMap(String field, Object term) { private boolean isJoinQuery() { return query instanceof JoinSelect; } -} \ No newline at end of file +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticJoinExecutor.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticJoinExecutor.java index 7dcce0a1bb..00fc4dbe2c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticJoinExecutor.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticJoinExecutor.java @@ -31,6 +31,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -216,8 +217,10 @@ protected SearchHit createUnmachedResult(List secondTableReturnedFields, String unmatchedId = hit.getId() + "|0"; Text unamatchedType = new Text(hit.getType() + "|null"); - SearchHit searchHit = new SearchHit(docId, unmatchedId, unamatchedType, hit.getFields()); - + Map documentFields = new HashMap<>(); + Map metaFields = new HashMap<>(); + SearchHit.splitFieldsByMetadata(hit.getFields(), documentFields, metaFields); + SearchHit searchHit = new SearchHit(docId, unmatchedId, unamatchedType, documentFields, metaFields); searchHit.sourceRef(hit.getSourceRef()); searchHit.getSourceAsMap().clear(); searchHit.getSourceAsMap().putAll(hit.getSourceAsMap()); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/HashJoinElasticExecutor.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/HashJoinElasticExecutor.java index 1d796fa656..57e59ef4fe 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/HashJoinElasticExecutor.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/HashJoinElasticExecutor.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -182,9 +183,12 @@ private List createCombinedResults(TableInJoinRequestBuilder secondTa onlyReturnedFields(copiedSource, secondTableRequest.getReturnedFields(), secondTableRequest.getOriginalSelect().isSelectAll()); + Map documentFields = new HashMap<>(); + Map metaFields = new HashMap<>(); + SearchHit.splitFieldsByMetadata(matchingHit.getFields(), documentFields, metaFields); SearchHit searchHit = new SearchHit(matchingHit.docId(), combinedId, new Text(matchingHit.getType() + "|" + secondTableHit.getType()), - matchingHit.getFields()); + documentFields, metaFields); searchHit.sourceRef(matchingHit.getSourceRef()); searchHit.getSourceAsMap().clear(); searchHit.getSourceAsMap().putAll(matchingHit.getSourceAsMap()); @@ -238,8 +242,12 @@ private void createKeyToResultsAndFillOptimizationStructure( String key = getComparisonKey(t1ToT2FieldsComparison, hit, true, optimizationTermsFilterStructure.get(comparisonID)); + Map documentFields = new HashMap<>(); + Map metaFields = new HashMap<>(); + SearchHit.splitFieldsByMetadata(hit.getFields(), documentFields, metaFields); //int docid , id - SearchHit searchHit = new SearchHit(resultIds, hit.getId(), new Text(hit.getType()), hit.getFields()); + SearchHit searchHit = new SearchHit(resultIds, hit.getId(), new Text(hit.getType()), + documentFields, metaFields); searchHit.sourceRef(hit.getSourceRef()); onlyReturnedFields(searchHit.getSourceAsMap(), firstTableRequest.getReturnedFields(), diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/NestedLoopsElasticExecutor.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/NestedLoopsElasticExecutor.java index badf5642fc..33c3842f46 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/NestedLoopsElasticExecutor.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/NestedLoopsElasticExecutor.java @@ -33,12 +33,14 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -172,9 +174,12 @@ private SearchHit getMergedHit(int currentCombinedResults, String t1Alias, Strin SearchHit hitFromFirstTable, SearchHit matchedHit) { onlyReturnedFields(matchedHit.getSourceAsMap(), nestedLoopsRequest.getSecondTable().getReturnedFields(), nestedLoopsRequest.getSecondTable().getOriginalSelect().isSelectAll()); + Map documentFields = new HashMap<>(); + Map metaFields = new HashMap<>(); + SearchHit.splitFieldsByMetadata(hitFromFirstTable.getFields(), documentFields, metaFields); SearchHit searchHit = new SearchHit(currentCombinedResults, hitFromFirstTable.getId() + "|" + matchedHit.getId(), new Text(hitFromFirstTable.getType() + "|" + matchedHit.getType()), - hitFromFirstTable.getFields()); + documentFields, metaFields); searchHit.sourceRef(hitFromFirstTable.getSourceRef()); searchHit.getSourceAsMap().clear(); searchHit.getSourceAsMap().putAll(hitFromFirstTable.getSourceAsMap()); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/MinusExecutor.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/MinusExecutor.java index 6441024f70..4e72269da1 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/MinusExecutor.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/MinusExecutor.java @@ -134,7 +134,11 @@ private void fillMinusHitsFromOneField(String fieldName, Set fieldValues ArrayList values = new ArrayList<>(); values.add(result); fields.put(fieldName, new DocumentField(fieldName, values)); - SearchHit searchHit = new SearchHit(currentId, currentId + "", new Text(someHit.getType()), fields); + Map documentFields = new HashMap<>(); + Map metaFields = new HashMap<>(); + SearchHit.splitFieldsByMetadata(fields, documentFields, metaFields); + SearchHit searchHit = new SearchHit(currentId, currentId + "", new Text(someHit.getType()), + documentFields, metaFields); searchHit.sourceRef(someHit.getSourceRef()); searchHit.getSourceAsMap().clear(); Map sourceAsMap = new HashMap<>(); @@ -155,8 +159,11 @@ private void fillMinusHitsFromResults(Set comperableHitResu ArrayList values = new ArrayList<>(); values.add(result); SearchHit originalHit = result.getOriginalHit(); + Map documentFields = new HashMap<>(); + Map metaFields = new HashMap<>(); + SearchHit.splitFieldsByMetadata(originalHit.getFields(), documentFields, metaFields); SearchHit searchHit = new SearchHit(currentId, originalHit.getId(), new Text(originalHit.getType()), - originalHit.getFields()); + documentFields, metaFields); searchHit.sourceRef(originalHit.getSourceRef()); searchHit.getSourceAsMap().clear(); Map sourceAsMap = result.getFlattenMap(); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/UnionExecutor.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/UnionExecutor.java index 0e1ed744cb..06e82f431c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/UnionExecutor.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/UnionExecutor.java @@ -22,12 +22,14 @@ import org.apache.lucene.search.TotalHits.Relation; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.text.Text; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -64,7 +66,11 @@ public void run() { private void fillInternalSearchHits(List unionHits, SearchHit[] hits, Map fieldNameToAlias) { for (SearchHit hit : hits) { - SearchHit searchHit = new SearchHit(currentId, hit.getId(), new Text(hit.getType()), hit.getFields()); + Map documentFields = new HashMap<>(); + Map metaFields = new HashMap<>(); + SearchHit.splitFieldsByMetadata(hit.getFields(), documentFields, metaFields); + SearchHit searchHit = new SearchHit(currentId, hit.getId(), new Text(hit.getType()), + documentFields, metaFields); searchHit.sourceRef(hit.getSourceRef()); searchHit.getSourceAsMap().clear(); Map sourceAsMap = hit.getSourceAsMap(); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SqlPlug.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SqlPlug.java index 00e657409a..01f072a186 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SqlPlug.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SqlPlug.java @@ -34,6 +34,7 @@ import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.script.ScriptService; @@ -89,11 +90,13 @@ public Collection createComponents(Client client, ClusterService cluster NamedXContentRegistry xContentRegistry, Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver) { + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier) { LocalClusterState.state().setClusterService(clusterService); LocalClusterState.state().setSqlSettings(sqlSettings); return super.createComponents(client, clusterService, threadPool, resourceWatcherService, scriptService, - xContentRegistry, environment, nodeEnvironment, namedWriteableRegistry, indexNameExpressionResolver); + xContentRegistry, environment, nodeEnvironment, namedWriteableRegistry, indexNameExpressionResolver, + repositoriesServiceSupplier); } @Override @@ -102,7 +105,7 @@ public List> getExecutorBuilders(Settings settings) { new FixedExecutorBuilder( settings, AsyncRestExecutor.SQL_WORKER_THREAD_POOL_NAME, - EsExecutors.numberOfProcessors(settings), + EsExecutors.allocatedProcessors(settings), 1000, null ) diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/scroll/SearchHitRow.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/scroll/SearchHitRow.java index 3ff84fd1c3..a915825516 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/scroll/SearchHitRow.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/scroll/SearchHitRow.java @@ -17,6 +17,7 @@ import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.Row; import com.google.common.base.Strings; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.text.Text; import org.elasticsearch.search.SearchHit; @@ -150,13 +151,17 @@ private Object getValueOfPath(Object source, String path, boolean isIgnoreFirstD } private SearchHit cloneHit(Row other) { + Map documentFields = new HashMap<>(); + Map metaFields = new HashMap<>(); + SearchHit.splitFieldsByMetadata(hit.getFields(), documentFields, metaFields); SearchHit combined = new SearchHit( hit.docId(), hit.getId() + "|" + (other == NULL ? "0" : ((SearchHitRow) other).hit.getId()), new Text( hit.getType() + "|" + (other == NULL ? null : ((SearchHitRow) other).hit.getType()) ), - hit.getFields() + documentFields, + metaFields ); combined.sourceRef(hit.getSourceRef()); combined.getSourceAsMap().clear(); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/matchtoterm/TermFieldScope.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/matchtoterm/TermFieldScope.java index 1286d77966..f11146f9a0 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/matchtoterm/TermFieldScope.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/matchtoterm/TermFieldScope.java @@ -31,7 +31,7 @@ */ public class TermFieldScope { - // mapper => index, type, field_name, FieldMappingMetaData + // mapper => index, type, field_name, FieldMappingMetadata private IndexMappings mapper; private FieldMappings finalMapping; private Map aliases; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/JDBCConnection.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/JDBCConnection.java index 7519a4e0f6..9be0382072 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/JDBCConnection.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/JDBCConnection.java @@ -106,7 +106,7 @@ public DBResult select(String query) { try (Statement stmt = connection.createStatement()) { ResultSet resultSet = stmt.executeQuery(query); DBResult result = new DBResult(databaseName); - populateMetaData(resultSet, result); + populateMetadata(resultSet, result); populateData(resultSet, result); return result; } catch (SQLException e) { @@ -138,16 +138,16 @@ private String getValueList(String[] fieldValues) { collect(joining(",")); } - private void populateMetaData(ResultSet resultSet, DBResult result) throws SQLException { - ResultSetMetaData metaData = resultSet.getMetaData(); - for (int i = 1; i <= metaData.getColumnCount(); i++) { + private void populateMetadata(ResultSet resultSet, DBResult result) throws SQLException { + ResultSetMetaData metadata = resultSet.getMetaData(); + for (int i = 1; i <= metadata.getColumnCount(); i++) { // Use label name (alias) if present. - String colName = metaData.getColumnLabel(i); + String colName = metadata.getColumnLabel(i); if (Strings.isNullOrEmpty(colName)) { - colName = metaData.getColumnName(i); + colName = metadata.getColumnName(i); } - result.addColumn(colName, metaData.getColumnTypeName(i)); + result.addColumn(colName, metadata.getColumnTypeName(i)); } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/JDBCConnectionTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/JDBCConnectionTest.java index bb2cfd32bf..2683a97d91 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/JDBCConnectionTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/JDBCConnectionTest.java @@ -108,10 +108,10 @@ public void testInsertData() throws SQLException { @Test public void testSelectQuery() throws SQLException { - ResultSetMetaData metaData = mockMetaData(ImmutableMap.of("name", "VARCHAR", "age", "INT")); + ResultSetMetaData metadata = mockMetadata(ImmutableMap.of("name", "VARCHAR", "age", "INT")); ResultSet resultSet = mockResultSet(new Object[]{"John", 25}, new Object[]{"Hank", 30}); when(statement.executeQuery(anyString())).thenReturn(resultSet); - when(resultSet.getMetaData()).thenReturn(metaData); + when(resultSet.getMetaData()).thenReturn(metadata); DBResult result = conn.select("SELECT * FROM test"); assertEquals("Test DB", result.getDatabaseName()); @@ -133,10 +133,10 @@ public void testSelectQuery() throws SQLException { @Test public void testSelectQueryWithAlias() throws SQLException { - ResultSetMetaData metaData = mockMetaData(ImmutableMap.of("name", "VARCHAR", "age", "INT"), "n", "a"); + ResultSetMetaData metadata = mockMetadata(ImmutableMap.of("name", "VARCHAR", "age", "INT"), "n", "a"); ResultSet resultSet = mockResultSet(new Object[]{"John", 25}, new Object[]{"Hank", 30}); when(statement.executeQuery(anyString())).thenReturn(resultSet); - when(resultSet.getMetaData()).thenReturn(metaData); + when(resultSet.getMetaData()).thenReturn(metadata); DBResult result = conn.select("SELECT * FROM test"); assertEquals( @@ -150,14 +150,14 @@ public void testSelectQueryWithAlias() throws SQLException { @Test public void testSelectQueryWithFloatInResultSet() throws SQLException { - ResultSetMetaData metaData = mockMetaData(ImmutableMap.of("name", "VARCHAR", "balance", "FLOAT")); + ResultSetMetaData metadata = mockMetadata(ImmutableMap.of("name", "VARCHAR", "balance", "FLOAT")); ResultSet resultSet = mockResultSet( new Object[]{"John", 25.123}, new Object[]{"Hank", 30.456}, new Object[]{"Allen", 15.1} ); when(statement.executeQuery(anyString())).thenReturn(resultSet); - when(resultSet.getMetaData()).thenReturn(metaData); + when(resultSet.getMetaData()).thenReturn(metadata); DBResult result = conn.select("SELECT * FROM test"); assertEquals( @@ -194,28 +194,28 @@ private ResultSet mockResultSet(Object[]... rows) throws SQLException { return resultSet; } - private ResultSetMetaData mockMetaData(Map nameAndTypes, String... aliases) throws SQLException { - ResultSetMetaData metaData = mock(ResultSetMetaData.class); + private ResultSetMetaData mockMetadata(Map nameAndTypes, String... aliases) throws SQLException { + ResultSetMetaData metadata = mock(ResultSetMetaData.class); - OngoingStubbing getColumnName = when(metaData.getColumnName(anyInt())); + OngoingStubbing getColumnName = when(metadata.getColumnName(anyInt())); for (String name : nameAndTypes.keySet()) { getColumnName = getColumnName.thenReturn(name); } - OngoingStubbing getColumnTypeName = when(metaData.getColumnTypeName(anyInt())); + OngoingStubbing getColumnTypeName = when(metadata.getColumnTypeName(anyInt())); for (String value : nameAndTypes.values()) { getColumnTypeName = getColumnTypeName.thenReturn(value); } if (aliases.length > 0) { - OngoingStubbing getColumnLabel = when(metaData.getColumnLabel(anyInt())); + OngoingStubbing getColumnLabel = when(metadata.getColumnLabel(anyInt())); for (String alias : aliases) { getColumnLabel = getColumnLabel.thenReturn(alias); } } - when(metaData.getColumnCount()).thenReturn(nameAndTypes.size()); - return metaData; + when(metadata.getColumnCount()).thenReturn(nameAndTypes.size()); + return metadata; } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/MetaDataQueryIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/MetaDataQueryIT.java index 9850e2397f..92ac51fb7c 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/MetaDataQueryIT.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/MetaDataQueryIT.java @@ -30,7 +30,7 @@ public class MetaDataQueryIT extends DocTest { @Section(1) - public void queryMetaData() { + public void queryMetadata() { section( title("Querying Metadata"), description( diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/CustomExternalTestCluster.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/CustomExternalTestCluster.java index 981be35e76..ae1ef854f6 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/CustomExternalTestCluster.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/CustomExternalTestCluster.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.env.Environment; +import org.elasticsearch.http.HttpInfo; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.TestCluster; @@ -40,6 +41,8 @@ import java.util.Collections; import java.util.concurrent.atomic.AtomicInteger; +import static org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest.Metric.HTTP; +import static org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest.Metric.SETTINGS; import static org.elasticsearch.test.ESTestCase.getTestTransportType; public class CustomExternalTestCluster extends TestCluster { @@ -82,14 +85,16 @@ public CustomExternalTestCluster(Path tempDir, Settings additionalSettings, Coll MockTransportClient client = new MockTransportClient(clientSettings, pluginClasses); try { client.addTransportAddresses(transportAddresses); - NodesInfoResponse nodeInfos = client.admin().cluster().prepareNodesInfo().clear().setSettings(true).setHttp(true).get(); + NodesInfoResponse nodeInfos = client.admin().cluster().prepareNodesInfo().clear() + .addMetrics(SETTINGS.metricName(), HTTP.metricName()) + .get(); httpAddresses = new InetSocketAddress[nodeInfos.getNodes().size()]; this.clusterName = nodeInfos.getClusterName().value(); int dataNodes = 0; int masterAndDataNodes = 0; for (int i = 0; i < nodeInfos.getNodes().size(); i++) { NodeInfo nodeInfo = nodeInfos.getNodes().get(i); - httpAddresses[i] = nodeInfo.getHttp().address().publishAddress().address(); + httpAddresses[i] = nodeInfo.getInfo(HttpInfo.class).address().publishAddress().address(); if (DiscoveryNode.isDataNode(nodeInfo.getSettings())) { dataNodes++; masterAndDataNodes++; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MetaDataQueriesIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MetaDataQueriesIT.java index 221e306760..592414dcc5 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MetaDataQueriesIT.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MetaDataQueriesIT.java @@ -47,7 +47,7 @@ * These are the outputs of "schema" for SHOW and DESCRIBE, the position of the value in "datarows" will match the * position of the field in "schema": * - * 1) SHOW query (based on the getTables() method listed here https://docs.oracle.com/javase/8/docs/api/java/sql/DatabaseMetaData.html) + * 1) SHOW query (based on the getTables() method listed here https://docs.oracle.com/javase/8/docs/api/java/sql/DatabaseMetadata.html) * "schema": [ * { * "name": "TABLE_CAT", @@ -91,7 +91,7 @@ * } * ] * - * 2) DESCRIBE query (based on the getColumns() method listed here https://docs.oracle.com/javase/8/docs/api/java/sql/DatabaseMetaData.html) + * 2) DESCRIBE query (based on the getColumns() method listed here https://docs.oracle.com/javase/8/docs/api/java/sql/DatabaseMetadata.html) * "schema": [ * { * "name": "TABLE_CAT", diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/LocalClusterStateTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/LocalClusterStateTest.java index 5528468fd8..f49f76c077 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/LocalClusterStateTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/LocalClusterStateTest.java @@ -104,7 +104,7 @@ public class LocalClusterStateTest { " }\n" + " }\n" + " },\n" + - // ==== All required by IndexMetaData.fromXContent() ==== + // ==== All required by IndexMetadata.fromXContent() ==== " \"settings\": {\n" + " \"index\": {\n" + " \"number_of_shards\": 5,\n" + @@ -173,19 +173,19 @@ public void getMappingFromCache() throws IOException { for (int i = 0; i < 10; i++) { LocalClusterState.state().getFieldMappings(new String[]{INDEX_NAME}); } - verify(mockService.state().metaData(), times(1)).findMappings(eq(new String[]{INDEX_NAME}), any(), any()); + verify(mockService.state().metadata(), times(1)).findMappings(eq(new String[]{INDEX_NAME}), any(), any()); // 2.Fire cluster state change event Assert.assertNotNull(listener[0]); ClusterChangedEvent mockEvent = mock(ClusterChangedEvent.class); - when(mockEvent.metaDataChanged()).thenReturn(true); + when(mockEvent.metadataChanged()).thenReturn(true); listener[0].clusterChanged(mockEvent); // 3.Cache should be invalidated and call findMapping another time only for (int i = 0; i < 5; i++) { LocalClusterState.state().getFieldMappings(new String[]{INDEX_NAME}); } - verify(mockService.state().metaData(), times(2)).findMappings(eq(new String[]{INDEX_NAME}), any(), any()); + verify(mockService.state().metadata(), times(2)).findMappings(eq(new String[]{INDEX_NAME}), any(), any()); } @Test diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/CheckScriptContents.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/CheckScriptContents.java index 72e6a5809e..0599d5be5d 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/CheckScriptContents.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/CheckScriptContents.java @@ -39,10 +39,10 @@ import org.elasticsearch.client.Client; import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.xcontent.DeprecationHandler; @@ -187,7 +187,7 @@ public static void stubMockClient(Client mockClient) { " }\n" + " }\n" + " },\n" + - // ==== All required by IndexMetaData.fromXContent() ==== + // ==== All required by IndexMetadata.fromXContent() ==== " \"settings\": {\n" + " \"index\": {\n" + " \"number_of_shards\": 5,\n" + @@ -239,14 +239,14 @@ public static void mockLocalClusterState(String mappings) { public static ClusterService mockClusterService(String mappings) { ClusterService mockService = mock(ClusterService.class); ClusterState mockState = mock(ClusterState.class); - MetaData mockMetaData = mock(MetaData.class); + Metadata mockMetadata = mock(Metadata.class); when(mockService.state()).thenReturn(mockState); - when(mockState.metaData()).thenReturn(mockMetaData); + when(mockState.metadata()).thenReturn(mockMetadata); try { - ImmutableOpenMap.Builder> builder = ImmutableOpenMap.builder(); - builder.put(TestsConstants.TEST_INDEX_BANK, IndexMetaData.fromXContent(createParser(mappings)).getMappings()); - when(mockMetaData.findMappings(any(), any(), any())).thenReturn(builder.build()); + ImmutableOpenMap.Builder> builder = ImmutableOpenMap.builder(); + builder.put(TestsConstants.TEST_INDEX_BANK, IndexMetadata.fromXContent(createParser(mappings)).getMappings()); + when(mockMetadata.findMappings(any(), any(), any())).thenReturn(builder.build()); } catch (IOException e) { throw new IllegalStateException(e); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MultipleIndexClusterUtils.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MultipleIndexClusterUtils.java index 0bd2e3a91c..df927a29fb 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MultipleIndexClusterUtils.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MultipleIndexClusterUtils.java @@ -18,9 +18,9 @@ import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; import com.google.common.collect.ImmutableMap; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -149,7 +149,7 @@ public class MultipleIndexClusterUtils { "}"; public static void mockMultipleIndexEnv() { - mockLocalClusterState(new ImmutableMap.Builder>>() + mockLocalClusterState(new ImmutableMap.Builder>>() .put(INDEX_ACCOUNT_1, buildIndexMapping(INDEX_ACCOUNT_1, INDEX_ACCOUNT_1_MAPPING)) .put(INDEX_ACCOUNT_2, buildIndexMapping(INDEX_ACCOUNT_2, INDEX_ACCOUNT_2_MAPPING)) .put(INDEX_ACCOUNT_ALL, buildIndexMapping(new ImmutableMap.Builder() @@ -159,23 +159,23 @@ public static void mockMultipleIndexEnv() { .build()); } - public static void mockLocalClusterState(Map>> indexMapping) { + public static void mockLocalClusterState(Map>> indexMapping) { LocalClusterState.state().setClusterService(mockClusterService(indexMapping)); LocalClusterState.state().setResolver(mockIndexNameExpressionResolver()); LocalClusterState.state().setSqlSettings(mockSqlSettings()); } - public static ClusterService mockClusterService(Map>> indexMapping) { + public static ClusterService mockClusterService(Map>> indexMapping) { ClusterService mockService = mock(ClusterService.class); ClusterState mockState = mock(ClusterState.class); - MetaData mockMetaData = mock(MetaData.class); + Metadata mockMetadata = mock(Metadata.class); when(mockService.state()).thenReturn(mockState); - when(mockState.metaData()).thenReturn(mockMetaData); + when(mockState.metadata()).thenReturn(mockMetadata); try { - for (Map.Entry>> entry : indexMapping.entrySet()) { - when(mockMetaData.findMappings(eq(new String[]{entry.getKey()}), any(), any())).thenReturn(entry.getValue()); + for (Map.Entry>> entry : indexMapping.entrySet()) { + when(mockMetadata.findMappings(eq(new String[]{entry.getKey()}), any(), any())).thenReturn(entry.getValue()); } } catch (IOException e) { throw new IllegalStateException(e); @@ -183,11 +183,11 @@ public static ClusterService mockClusterService(Map> buildIndexMapping(Map indexMapping) { + private static ImmutableOpenMap> buildIndexMapping(Map indexMapping) { try { - ImmutableOpenMap.Builder> builder = ImmutableOpenMap.builder(); + ImmutableOpenMap.Builder> builder = ImmutableOpenMap.builder(); for (Map.Entry entry : indexMapping.entrySet()) { - builder.put(entry.getKey(), IndexMetaData.fromXContent(createParser(entry.getValue())).getMappings()); + builder.put(entry.getKey(), IndexMetadata.fromXContent(createParser(entry.getValue())).getMappings()); } return builder.build(); } catch (IOException e) { @@ -195,11 +195,11 @@ private static ImmutableOpenMap> buildIndexMapping(String index, + private static ImmutableOpenMap> buildIndexMapping(String index, String mapping) { try { - ImmutableOpenMap.Builder> builder = ImmutableOpenMap.builder(); - builder.put(index, IndexMetaData.fromXContent(createParser(mapping)).getMappings()); + ImmutableOpenMap.Builder> builder = ImmutableOpenMap.builder(); + builder.put(index, IndexMetadata.fromXContent(createParser(mapping)).getMappings()); return builder.build(); } catch (IOException e) { throw new IllegalStateException(e);