Merge branch 'main' into v0.15.x

.git-blame-ignore-revs

@@ -18,3 +18,9 @@ bfe19393b17710f92b61ea80e6e5f4e026ea2676
 
 # Scala Steward: Reformat with scalafmt 3.8.0
 e339d7d594db19fcb7f8909a7c1754269ccaae00
+
+# Scala Steward: Reformat with scalafmt 3.8.1
+58ff23b36468049356b0bcbcacb412c7cf149255
+
+# Scala Steward: Reformat with scalafmt 3.8.3
+d5e9a90c86690605d6572c03f4380b5bd65056d1

.github/workflows/ci.yml

@@ -38,6 +38,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     steps:
+      - name: Install sbt
+        if: contains(runner.os, 'macos')
+        run: brew install sbt
+
       - name: Checkout current branch (full)
         uses: actions/checkout@v4
         with:
@@ -98,7 +102,7 @@ jobs:
 
       - name: Check headers and formatting
         if: matrix.java == 'corretto@11' && matrix.os == 'ubuntu-latest'
-        run: sbt '++ ${{ matrix.scala }}' headerCheckAll scalafmtCheckAll 'project /' scalafmtSbtCheck
+        run: sbt '++ ${{ matrix.scala }}' headerCheckAll javafmtCheckAll scalafmtCheckAll 'project /' scalafmtSbtCheck
 
       - name: Test
         run: sbt '++ ${{ matrix.scala }}' test
@@ -116,11 +120,11 @@ jobs:
 
       - name: Make target directories
         if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
-        run: mkdir -p integration/target scio-tensorflow/target site/target scio-cassandra/cassandra3/target scio-jmh/target scio-elasticsearch/es8/target scio-jdbc/target scio-macros/target scio-grpc/target scio-elasticsearch/common/target scio-test/target scio-avro/target scio-elasticsearch/es7/target scio-redis/target scio-extra/target scio-google-cloud-platform/target scio-smb/target scio-neo4j/target scio-parquet/target scio-core/target scio-repl/target project/target
+        run: mkdir -p scio-bom/target scio-tensorflow/target site/target scio-cassandra/cassandra3/target scio-elasticsearch/es8/target scio-jdbc/target scio-macros/target scio-grpc/target scio-elasticsearch/common/target scio-test/target scio-avro/target scio-elasticsearch/es7/target scio-redis/target scio-extra/target scio-test/parquet/target scio-test/core/target scio-google-cloud-platform/target scio-smb/target scio-test/google-cloud-platform/target scio-neo4j/target scio-parquet/target scio-core/target scio-repl/target project/target
 
       - name: Compress target directories
         if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
-        run: tar cf targets.tar integration/target scio-tensorflow/target site/target scio-cassandra/cassandra3/target scio-jmh/target scio-elasticsearch/es8/target scio-jdbc/target scio-macros/target scio-grpc/target scio-elasticsearch/common/target scio-test/target scio-avro/target scio-elasticsearch/es7/target scio-redis/target scio-extra/target scio-google-cloud-platform/target scio-smb/target scio-neo4j/target scio-parquet/target scio-core/target scio-repl/target project/target
+        run: tar cf targets.tar scio-bom/target scio-tensorflow/target site/target scio-cassandra/cassandra3/target scio-elasticsearch/es8/target scio-jdbc/target scio-macros/target scio-grpc/target scio-elasticsearch/common/target scio-test/target scio-avro/target scio-elasticsearch/es7/target scio-redis/target scio-extra/target scio-test/parquet/target scio-test/core/target scio-google-cloud-platform/target scio-smb/target scio-test/google-cloud-platform/target scio-neo4j/target scio-parquet/target scio-core/target scio-repl/target project/target
 
       - name: Upload target directories
         if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
@@ -139,6 +143,10 @@ jobs:
         java: [corretto@11]
     runs-on: ${{ matrix.os }}
     steps:
+      - name: Install sbt
+        if: contains(runner.os, 'macos')
+        run: brew install sbt
+
       - name: Checkout current branch (full)
         uses: actions/checkout@v4
         with:
@@ -245,6 +253,10 @@ jobs:
         java: [corretto@11]
     runs-on: ${{ matrix.os }}
     steps:
+      - name: Install sbt
+        if: contains(runner.os, 'macos')
+        run: brew install sbt
+
       - name: Checkout current branch (full)
         uses: actions/checkout@v4
         with:
@@ -292,7 +304,7 @@ jobs:
       - name: Submit Dependencies
         uses: scalacenter/sbt-dependency-submission@v2
         with:
-          modules-ignore: scio_2.13 scio_2.12 scio-examples_2.13 scio-examples_2.12
+          modules-ignore: scio_2.13 scio_2.12 scio-examples_2.13 scio-examples_2.12 integration_2.13 integration_2.12 scio-jmh_2.13 scio-jmh_2.12
           configs-ignore: test scala-tool scala-doc-tool test-internal
 
   validate-steward:
@@ -354,7 +366,42 @@ jobs:
         run: sbt '++ ${{ matrix.scala }}' coverage test coverageAggregate
 
       - name: Upload coverage report
-        run: 'bash <(curl -s https://codecov.io/bash)'
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+  avro-legacy:
+    name: Test Legacy Avro
+    if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        scala: [2.13]
+        java: [corretto@11]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout current branch (full)
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Java (corretto@11)
+        id: setup-java-corretto-11
+        if: matrix.java == 'corretto@11'
+        uses: actions/setup-java@v4
+        with:
+          distribution: corretto
+          java-version: 11
+          cache: sbt
+
+      - name: sbt update
+        if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false'
+        run: sbt +update
+
+      - name: Test
+        env:
+          JAVA_OPTS: '-Davro.version=1.8.2'
+        run: sbt '++ ${{ matrix.scala }}' scio-avro/test
 
   it-test:
     name: Integration Test

.gitignore

@@ -33,3 +33,4 @@ metals.sbt
 .bigquery/
 .scio_repl
 *.jfr
+scio-bom/

.scala-steward.conf

@@ -3,6 +3,7 @@ updates.ignore = [
   { groupId = "com.fasterxml.jackson.core" },
   { groupId = "com.fasterxml.jackson.datatype" },
   { groupId = "com.fasterxml.jackson.module" },
+  { groupId = "com.github.luben" },
   { groupId = "com.google.api" },
   { groupId = "com.google.api-client" },
   { groupId = "com.google.api.grpc" },

.scalafmt.conf

@@ -1,4 +1,4 @@
-version = "3.8.0"
+version = "3.8.3"
 runner.dialect = scala213
 
 fileOverride {

README.md

@@ -14,15 +14,13 @@
 
 Scio is a Scala API for [Apache Beam](http://beam.incubator.apache.org/) and [Google Cloud Dataflow](https://github.com/GoogleCloudPlatform/DataflowJavaSDK) inspired by [Apache Spark](http://spark.apache.org/) and [Scalding](https://github.com/twitter/scalding).
 
-Scio 0.3.0 and future versions depend on Apache Beam (`org.apache.beam`) while earlier versions depend on Google Cloud Dataflow SDK (`com.google.cloud.dataflow`). See this [page](https://spotify.github.io/scio/Apache-Beam.html) for a list of breaking changes.
-
 # Features
 
 - Scala API close to that of Spark and Scalding core APIs
 - Unified batch and streaming programming model
 - Fully managed service<sup>\*</sup>
 - Integration with Google Cloud products: Cloud Storage, BigQuery, Pub/Sub, Datastore, Bigtable
-- JDBC, [TensorFlow](http://tensorflow.org/) TFRecords, Cassandra, Elasticsearch and Parquet I/O
+- Avro, Cassandra, Elasticsearch, gRPC, JDBC, neo4j, Parquet, Redis, [TensorFlow](http://tensorflow.org/) IOs
 - Interactive mode with Scio REPL
 - Type safe BigQuery
 - Integration with [Algebird](https://github.com/twitter/algebird) and [Breeze](https://github.com/scalanlp/breeze)
@@ -87,10 +85,13 @@ Scio includes the following artifacts:
 - `scio-repl`: extension of the Scala REPL with Scio specific operations
 - `scio-smb`: add-on for Sort Merge Bucket operations
 - `scio-tensorflow`: add-on for TensorFlow TFRecords IO and prediction
-- `scio-test`: test utilities, add to your project as a "test" dependency
+- `scio-test`: all following test utilities. Add to your project as a "test" dependency
+  - `scio-test-core`: test core utilities
+  - `scio-test-google-cloud-platform`: test utilities for Google Cloud IO's
+  - `scio-test-parquet`: test utilities for Parquet
 
 # License
 
-Copyright 2021 Spotify AB.
+Copyright 2024 Spotify AB.
 
 Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0