Adds an auto_schema flag to Avro/Parquet codecs

data-prepper-plugins/avro-codecs/src/main/java/org/opensearch/dataprepper/plugins/codec/avro/AvroOutputCodecConfig.java

@@ -5,6 +5,7 @@
 package org.opensearch.dataprepper.plugins.codec.avro;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import jakarta.validation.constraints.AssertTrue;
 
 /**
  * Configuration class for {@link AvroOutputCodec}.
@@ -13,11 +14,27 @@ public class AvroOutputCodecConfig {
     @JsonProperty("schema")
     private String schema;
 
+    @JsonProperty("auto_schema")
+    private boolean autoSchema;
+
+    @AssertTrue(message = "The Avro codec requires either defining a schema or setting auto_schema to true to automatically generate a schema.")
+    boolean isSchemaOrAutoSchemaDefined() {
+        return schema != null ^ autoSchema;
+    }
+
     public String getSchema() {
         return schema;
     }
 
-    public void setSchema(String schema) {
+    void setSchema(final String schema) {
         this.schema = schema;
     }
+
+    public boolean isAutoSchema() {
+        return autoSchema;
+    }
+
+    void setAutoSchema(final boolean autoSchema) {
+        this.autoSchema = autoSchema;
+    }
 }

data-prepper-plugins/avro-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/avro/AvroOutputCodecConfigTest.java

@@ -0,0 +1,47 @@
+package org.opensearch.dataprepper.plugins.codec.avro;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.UUID;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+
+class AvroOutputCodecConfigTest {
+
+    private AvroOutputCodecConfig createObjectUnderTest() {
+        return new AvroOutputCodecConfig();
+    }
+
+    @Test
+    void isSchemaOrAutoSchemaDefined_returns_true_if_schema_defined() {
+        AvroOutputCodecConfig objectUnderTest = createObjectUnderTest();
+        objectUnderTest.setSchema(UUID.randomUUID().toString());
+
+        assertThat(objectUnderTest.isSchemaOrAutoSchemaDefined(), equalTo(true));
+    }
+
+    @Test
+    void isSchemaOrAutoSchemaDefined_returns_true_if_schema_null_and_autoSchema_true() {
+        AvroOutputCodecConfig objectUnderTest = createObjectUnderTest();
+        objectUnderTest.setAutoSchema(true);
+
+        assertThat(objectUnderTest.isSchemaOrAutoSchemaDefined(), equalTo(true));
+    }
+
+    @Test
+    void isSchemaOrAutoSchemaDefined_returns_false_if_schema_null() {
+        AvroOutputCodecConfig objectUnderTest = createObjectUnderTest();
+
+        assertThat(objectUnderTest.isSchemaOrAutoSchemaDefined(), equalTo(false));
+    }
+
+    @Test
+    void isSchemaOrAutoSchemaDefined_returns_false_if_schema_and_autoSchema() {
+        AvroOutputCodecConfig objectUnderTest = createObjectUnderTest();
+        objectUnderTest.setSchema(UUID.randomUUID().toString());
+        objectUnderTest.setAutoSchema(true);
+
+        assertThat(objectUnderTest.isSchemaOrAutoSchemaDefined(), equalTo(false));
+    }
+}

data-prepper-plugins/s3-sink/src/main/java/org/opensearch/dataprepper/plugins/codec/parquet/ParquetOutputCodecConfig.java

@@ -5,17 +5,34 @@
 package org.opensearch.dataprepper.plugins.codec.parquet;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import jakarta.validation.constraints.AssertTrue;
 
 public class ParquetOutputCodecConfig {
     @JsonProperty("schema")
     private String schema;
 
+    @JsonProperty("auto_schema")
+    private boolean autoSchema;
+
+    @AssertTrue(message = "The Parquet codec requires either defining a schema or setting auto_schema to true to automatically generate a schema.")
+    boolean isSchemaOrAutoSchemaDefined() {
+        return schema != null ^ autoSchema;
+    }
+
     public String getSchema() {
         return schema;
     }
 
-    public void setSchema(String schema) {
+    public void setSchema(final String schema) {
         this.schema = schema;
     }
+
+    public boolean isAutoSchema() {
+        return autoSchema;
+    }
+
+    void setAutoSchema(final boolean autoSchema) {
+        this.autoSchema = autoSchema;
+    }
 }
 

data-prepper-plugins/s3-sink/src/test/java/org/opensearch/dataprepper/plugins/codec/parquet/ParquetOutputCodecConfigTest.java

@@ -0,0 +1,48 @@
+package org.opensearch.dataprepper.plugins.codec.parquet;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.UUID;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+
+class ParquetOutputCodecConfigTest {
+
+    private ParquetOutputCodecConfig createObjectUnderTest() {
+        return new ParquetOutputCodecConfig();
+    }
+
+    @Test
+    void isSchemaOrAutoSchemaDefined_returns_true_if_schema_defined() {
+        ParquetOutputCodecConfig objectUnderTest = createObjectUnderTest();
+        objectUnderTest.setSchema(UUID.randomUUID().toString());
+
+        assertThat(objectUnderTest.isSchemaOrAutoSchemaDefined(), equalTo(true));
+    }
+
+    @Test
+    void isSchemaOrAutoSchemaDefined_returns_true_if_schema_null_and_autoSchema_true() {
+        ParquetOutputCodecConfig objectUnderTest = createObjectUnderTest();
+        objectUnderTest.setAutoSchema(true);
+
+        assertThat(objectUnderTest.isSchemaOrAutoSchemaDefined(), equalTo(true));
+    }
+
+    @Test
+    void isSchemaOrAutoSchemaDefined_returns_false_if_schema_null() {
+        ParquetOutputCodecConfig objectUnderTest = createObjectUnderTest();
+
+        assertThat(objectUnderTest.isSchemaOrAutoSchemaDefined(), equalTo(false));
+    }
+
+    @Test
+    void isSchemaOrAutoSchemaDefined_returns_false_if_schema_and_autoSchema() {
+        ParquetOutputCodecConfig objectUnderTest = createObjectUnderTest();
+        objectUnderTest.setSchema(UUID.randomUUID().toString());
+        objectUnderTest.setAutoSchema(true);
+
+        assertThat(objectUnderTest.isSchemaOrAutoSchemaDefined(), equalTo(false));
+    }
+
+}