Upgrade Hadoop to 2.10.2/3.2.4.

GoogleCloudDataproc · Mar 28, 2023 · dd52c91 · dd52c91
1 parent 16818f4
commit dd52c91
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 2 deletions.
diff --git a/gcs/CHANGES.md b/gcs/CHANGES.md
@@ -2,6 +2,8 @@
 
 ## Next
 
+1.  Upgrade Hadoop to 2.10.2/3.2.4.
+
 ## 2.1.9 - 2022-11-14
 
 1.  Set socket read timeout (`fs.gs.http.read-timeout`) as early as possible on

diff --git a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemBase.java b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemBase.java
@@ -67,6 +67,7 @@
 import com.google.cloud.hadoop.util.HttpTransportFactory;
 import com.google.cloud.hadoop.util.PropertyUtil;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Ascii;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableList;
@@ -611,6 +612,24 @@ protected int getDefaultPort() {
     return result;
   }
 
+  public boolean hasPathCapability(Path path, String capability) throws IOException {
+    switch (validatePathCapabilityArgs(path, capability)) {
+        // TODO: remove string literals in favor of Constants in CommonPathCapabilities.java
+        // from Hadoop 3 when Hadoop 2 is no longer supported
+      case "fs.capability.paths.append":
+      case "fs.capability.paths.concat":
+        return true;
+      default:
+        return false;
+    }
+  }
+
+  private static String validatePathCapabilityArgs(Path path, String capability) {
+    checkNotNull(path);
+    checkArgument(!isNullOrEmpty(capability), "capability parameter is empty string");
+    return Ascii.toLowerCase(capability);
+  }
+
   /**
    * Opens the given file for reading.
    *

diff --git a/pom.xml b/pom.xml
@@ -111,8 +111,8 @@
     <google.protobuf.version>3.13.0</google.protobuf.version>
     <google.truth.version>1.1</google.truth.version>
     <grpc.version>1.33.1</grpc.version>
-    <hadoop.two.version>2.10.1</hadoop.two.version>
-    <hadoop.three.version>3.2.1</hadoop.three.version>
+    <hadoop.two.version>2.10.2</hadoop.two.version>
+    <hadoop.three.version>3.2.4</hadoop.three.version>
     <junit.version>4.13.1</junit.version>
     <junit.system-rules.version>1.19.0</junit.system-rules.version>
     <mockito.version>3.6.0</mockito.version>