GH-35698: [C#] Update FlatBuffers (#35699)

### Rationale for this change

The FlatBuffers definitions in the C# code don't seem to have been updated for a while. Adding the latest FlatBuffers code ensures that we're up to date with both the Arrow definitions and the FlatBuffers compiler.

### What changes are included in this PR?

The latest version of the C# FlatBuffers runtime files were copied into the project (with visibility changed from public to internal).
The latest version of the FlatBuffers compiler was used against the latest version of the Arrow .fbs source. The output of the compiler was edited to more closely match the existing code by moving it into the directory structure and namespaces of the existing classes and by changing class visibility from public to internal.
A few small changes were made to the existing source to accommodate changes in the FlatBuffers runtime, most specifically that the files have moved from the namespace "FlatBuffers" to the namespace "Google.FlatBuffers".

### Are these changes tested?

No substantive product changes were made. All tests still pass.

### Are there any user-facing changes?

No.

Resolves #35698 
* Closes: #35698

Authored-by: Curt Hagenlocher <[email protected]>
Signed-off-by: Weston Pace <[email protected]>

csharp/src/Apache.Arrow.Flight/Internal/FlightDataStream.cs

@@ -22,7 +22,7 @@
 using Apache.Arrow.Flatbuf;
 using Apache.Arrow.Flight.Protocol;
 using Apache.Arrow.Ipc;
-using FlatBuffers;
+using Google.FlatBuffers;
 using Google.Protobuf;
 using Grpc.Core;
 

csharp/src/Apache.Arrow.Flight/Internal/FlightMessageSerializer.cs

@@ -19,7 +19,7 @@
 using System.IO;
 using System.Text;
 using Apache.Arrow.Ipc;
-using FlatBuffers;
+using Google.FlatBuffers;
 
 namespace Apache.Arrow.Flight
 {

csharp/src/Apache.Arrow/Flatbuf/Block.cs

@@ -6,21 +6,22 @@ namespace Apache.Arrow.Flatbuf
 {
 
 using global::System;
-using global::FlatBuffers;
+using global::System.Collections.Generic;
+using global::Google.FlatBuffers;
 
 internal struct Block : IFlatbufferObject
 {
   private Struct __p;
   public ByteBuffer ByteBuffer { get { return __p.bb; } }
-  public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; }
+  public void __init(int _i, ByteBuffer _bb) { __p = new Struct(_i, _bb); }
   public Block __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; }
 
   /// Index to the start of the RecordBlock (note this is past the Message header)
   public long Offset { get { return __p.bb.GetLong(__p.bb_pos + 0); } }
   /// Length of the metadata
   public int MetaDataLength { get { return __p.bb.GetInt(__p.bb_pos + 8); } }
   /// Length of the data (this is aligned so there can be a gap between this and
-  /// the metatdata).
+  /// the metadata).
   public long BodyLength { get { return __p.bb.GetLong(__p.bb_pos + 16); } }
 
   public static Offset<Block> CreateBlock(FlatBufferBuilder builder, long Offset, int MetaDataLength, long BodyLength) {
@@ -31,7 +32,7 @@ public static Offset<Block> CreateBlock(FlatBufferBuilder builder, long Offset,
     builder.PutLong(Offset);
     return new Offset<Block>(builder.Offset);
   }
-};
+}
 
 
 }

csharp/src/Apache.Arrow/Flatbuf/BodyCompression.cs

@@ -6,7 +6,8 @@ namespace Apache.Arrow.Flatbuf
 {
 
 using global::System;
-using global::FlatBuffers;
+using global::System.Collections.Generic;
+using global::Google.FlatBuffers;
 
 /// Optional compression for the memory buffers constituting IPC message
 /// bodies. Intended for use with RecordBatch but could be used for other
@@ -15,33 +16,46 @@ internal struct BodyCompression : IFlatbufferObject
 {
   private Table __p;
   public ByteBuffer ByteBuffer { get { return __p.bb; } }
+  public static void ValidateVersion() { FlatBufferConstants.FLATBUFFERS_23_5_9(); }
   public static BodyCompression GetRootAsBodyCompression(ByteBuffer _bb) { return GetRootAsBodyCompression(_bb, new BodyCompression()); }
   public static BodyCompression GetRootAsBodyCompression(ByteBuffer _bb, BodyCompression obj) { return (obj.__assign(_bb.GetInt(_bb.Position) + _bb.Position, _bb)); }
-  public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; }
+  public void __init(int _i, ByteBuffer _bb) { __p = new Table(_i, _bb); }
   public BodyCompression __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; }
 
-  /// Compressor library
+  /// Compressor library.
+  /// For LZ4_FRAME, each compressed buffer must consist of a single frame.
   public CompressionType Codec { get { int o = __p.__offset(4); return o != 0 ? (CompressionType)__p.bb.GetSbyte(o + __p.bb_pos) : CompressionType.LZ4_FRAME; } }
   /// Indicates the way the record batch body was compressed
   public BodyCompressionMethod Method { get { int o = __p.__offset(6); return o != 0 ? (BodyCompressionMethod)__p.bb.GetSbyte(o + __p.bb_pos) : BodyCompressionMethod.BUFFER; } }
 
   public static Offset<BodyCompression> CreateBodyCompression(FlatBufferBuilder builder,
       CompressionType codec = CompressionType.LZ4_FRAME,
       BodyCompressionMethod method = BodyCompressionMethod.BUFFER) {
-    builder.StartObject(2);
+    builder.StartTable(2);
     BodyCompression.AddMethod(builder, method);
     BodyCompression.AddCodec(builder, codec);
     return BodyCompression.EndBodyCompression(builder);
   }
 
-  public static void StartBodyCompression(FlatBufferBuilder builder) { builder.StartObject(2); }
+  public static void StartBodyCompression(FlatBufferBuilder builder) { builder.StartTable(2); }
   public static void AddCodec(FlatBufferBuilder builder, CompressionType codec) { builder.AddSbyte(0, (sbyte)codec, 0); }
   public static void AddMethod(FlatBufferBuilder builder, BodyCompressionMethod method) { builder.AddSbyte(1, (sbyte)method, 0); }
   public static Offset<BodyCompression> EndBodyCompression(FlatBufferBuilder builder) {
-    int o = builder.EndObject();
+    int o = builder.EndTable();
     return new Offset<BodyCompression>(o);
   }
-};
+}
+
 
+static internal class BodyCompressionVerify
+{
+  static public bool Verify(Google.FlatBuffers.Verifier verifier, uint tablePos)
+  {
+    return verifier.VerifyTableStart(tablePos)
+      && verifier.VerifyField(tablePos, 4 /*Codec*/, 1 /*CompressionType*/, 1, false)
+      && verifier.VerifyField(tablePos, 6 /*Method*/, 1 /*BodyCompressionMethod*/, 1, false)
+      && verifier.VerifyTableEnd(tablePos);
+  }
+}
 
 }

csharp/src/Apache.Arrow/Flatbuf/Buffer.cs

@@ -6,22 +6,26 @@ namespace Apache.Arrow.Flatbuf
 {
 
 using global::System;
-using global::FlatBuffers;
+using global::System.Collections.Generic;
+using global::Google.FlatBuffers;
 
 /// ----------------------------------------------------------------------
 /// A Buffer represents a single contiguous memory segment
 internal struct Buffer : IFlatbufferObject
 {
   private Struct __p;
   public ByteBuffer ByteBuffer { get { return __p.bb; } }
-  public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; }
+  public void __init(int _i, ByteBuffer _bb) { __p = new Struct(_i, _bb); }
   public Buffer __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; }
 
   /// The relative offset into the shared memory page where the bytes for this
   /// buffer starts
   public long Offset { get { return __p.bb.GetLong(__p.bb_pos + 0); } }
   /// The absolute length (in bytes) of the memory buffer. The memory is found
-  /// from offset (inclusive) to offset + length (non-inclusive).
+  /// from offset (inclusive) to offset + length (non-inclusive). When building
+  /// messages using the encapsulated IPC message, padding bytes may be written
+  /// after a buffer, but such padding bytes do not need to be accounted for in
+  /// the size here.
   public long Length { get { return __p.bb.GetLong(__p.bb_pos + 8); } }
 
   public static Offset<Buffer> CreateBuffer(FlatBufferBuilder builder, long Offset, long Length) {
@@ -30,7 +34,7 @@ public static Offset<Buffer> CreateBuffer(FlatBufferBuilder builder, long Offset
     builder.PutLong(Offset);
     return new Offset<Buffer>(builder.Offset);
   }
-};
+}
 
 
 }

csharp/src/Apache.Arrow/Flatbuf/DictionaryBatch.cs

@@ -6,7 +6,8 @@ namespace Apache.Arrow.Flatbuf
 {
 
 using global::System;
-using global::FlatBuffers;
+using global::System.Collections.Generic;
+using global::Google.FlatBuffers;
 
 /// For sending dictionary encoding information. Any Field can be
 /// dictionary-encoded, but in this case none of its children may be
@@ -18,37 +19,51 @@ internal struct DictionaryBatch : IFlatbufferObject
 {
   private Table __p;
   public ByteBuffer ByteBuffer { get { return __p.bb; } }
+  public static void ValidateVersion() { FlatBufferConstants.FLATBUFFERS_23_5_9(); }
   public static DictionaryBatch GetRootAsDictionaryBatch(ByteBuffer _bb) { return GetRootAsDictionaryBatch(_bb, new DictionaryBatch()); }
   public static DictionaryBatch GetRootAsDictionaryBatch(ByteBuffer _bb, DictionaryBatch obj) { return (obj.__assign(_bb.GetInt(_bb.Position) + _bb.Position, _bb)); }
-  public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; }
+  public void __init(int _i, ByteBuffer _bb) { __p = new Table(_i, _bb); }
   public DictionaryBatch __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; }
 
   public long Id { get { int o = __p.__offset(4); return o != 0 ? __p.bb.GetLong(o + __p.bb_pos) : (long)0; } }
   public RecordBatch? Data { get { int o = __p.__offset(6); return o != 0 ? (RecordBatch?)(new RecordBatch()).__assign(__p.__indirect(o + __p.bb_pos), __p.bb) : null; } }
   /// If isDelta is true the values in the dictionary are to be appended to a
-  /// dictionary with the indicated id
+  /// dictionary with the indicated id. If isDelta is false this dictionary
+  /// should replace the existing dictionary.
   public bool IsDelta { get { int o = __p.__offset(8); return o != 0 ? 0!=__p.bb.Get(o + __p.bb_pos) : (bool)false; } }
 
   public static Offset<DictionaryBatch> CreateDictionaryBatch(FlatBufferBuilder builder,
       long id = 0,
       Offset<RecordBatch> dataOffset = default(Offset<RecordBatch>),
       bool isDelta = false) {
-    builder.StartObject(3);
+    builder.StartTable(3);
     DictionaryBatch.AddId(builder, id);
     DictionaryBatch.AddData(builder, dataOffset);
     DictionaryBatch.AddIsDelta(builder, isDelta);
     return DictionaryBatch.EndDictionaryBatch(builder);
   }
 
-  public static void StartDictionaryBatch(FlatBufferBuilder builder) { builder.StartObject(3); }
+  public static void StartDictionaryBatch(FlatBufferBuilder builder) { builder.StartTable(3); }
   public static void AddId(FlatBufferBuilder builder, long id) { builder.AddLong(0, id, 0); }
   public static void AddData(FlatBufferBuilder builder, Offset<RecordBatch> dataOffset) { builder.AddOffset(1, dataOffset.Value, 0); }
   public static void AddIsDelta(FlatBufferBuilder builder, bool isDelta) { builder.AddBool(2, isDelta, false); }
   public static Offset<DictionaryBatch> EndDictionaryBatch(FlatBufferBuilder builder) {
-    int o = builder.EndObject();
+    int o = builder.EndTable();
     return new Offset<DictionaryBatch>(o);
   }
-};
+}
+
 
+static internal class DictionaryBatchVerify
+{
+  static public bool Verify(Google.FlatBuffers.Verifier verifier, uint tablePos)
+  {
+    return verifier.VerifyTableStart(tablePos)
+      && verifier.VerifyField(tablePos, 4 /*Id*/, 8 /*long*/, 8, false)
+      && verifier.VerifyTable(tablePos, 6 /*Data*/, RecordBatchVerify.Verify, false)
+      && verifier.VerifyField(tablePos, 8 /*IsDelta*/, 1 /*bool*/, 1, false)
+      && verifier.VerifyTableEnd(tablePos);
+  }
+}
 
 }

csharp/src/Apache.Arrow/Flatbuf/DictionaryEncoding.cs

@@ -6,52 +6,72 @@ namespace Apache.Arrow.Flatbuf
 {
 
 using global::System;
-using global::FlatBuffers;
+using global::System.Collections.Generic;
+using global::Google.FlatBuffers;
 
-/// ----------------------------------------------------------------------
-/// Dictionary encoding metadata
 internal struct DictionaryEncoding : IFlatbufferObject
 {
   private Table __p;
   public ByteBuffer ByteBuffer { get { return __p.bb; } }
+  public static void ValidateVersion() { FlatBufferConstants.FLATBUFFERS_23_5_9(); }
   public static DictionaryEncoding GetRootAsDictionaryEncoding(ByteBuffer _bb) { return GetRootAsDictionaryEncoding(_bb, new DictionaryEncoding()); }
   public static DictionaryEncoding GetRootAsDictionaryEncoding(ByteBuffer _bb, DictionaryEncoding obj) { return (obj.__assign(_bb.GetInt(_bb.Position) + _bb.Position, _bb)); }
-  public void __init(int _i, ByteBuffer _bb) { __p.bb_pos = _i; __p.bb = _bb; }
+  public void __init(int _i, ByteBuffer _bb) { __p = new Table(_i, _bb); }
   public DictionaryEncoding __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; }
 
   /// The known dictionary id in the application where this data is used. In
   /// the file or streaming formats, the dictionary ids are found in the
   /// DictionaryBatch messages
   public long Id { get { int o = __p.__offset(4); return o != 0 ? __p.bb.GetLong(o + __p.bb_pos) : (long)0; } }
-  /// The dictionary indices are constrained to be positive integers. If this
-  /// field is null, the indices must be signed int32
+  /// The dictionary indices are constrained to be non-negative integers. If
+  /// this field is null, the indices must be signed int32. To maximize
+  /// cross-language compatibility and performance, implementations are
+  /// recommended to prefer signed integer types over unsigned integer types
+  /// and to avoid uint64 indices unless they are required by an application.
   public Int? IndexType { get { int o = __p.__offset(6); return o != 0 ? (Int?)(new Int()).__assign(__p.__indirect(o + __p.bb_pos), __p.bb) : null; } }
   /// By default, dictionaries are not ordered, or the order does not have
   /// semantic meaning. In some statistical, applications, dictionary-encoding
   /// is used to represent ordered categorical data, and we provide a way to
   /// preserve that metadata here
   public bool IsOrdered { get { int o = __p.__offset(8); return o != 0 ? 0!=__p.bb.Get(o + __p.bb_pos) : (bool)false; } }
+  public DictionaryKind DictionaryKind { get { int o = __p.__offset(10); return o != 0 ? (DictionaryKind)__p.bb.GetShort(o + __p.bb_pos) : DictionaryKind.DenseArray; } }
 
   public static Offset<DictionaryEncoding> CreateDictionaryEncoding(FlatBufferBuilder builder,
       long id = 0,
       Offset<Int> indexTypeOffset = default(Offset<Int>),
-      bool isOrdered = false) {
-    builder.StartObject(3);
+      bool isOrdered = false,
+      DictionaryKind dictionaryKind = DictionaryKind.DenseArray) {
+    builder.StartTable(4);
     DictionaryEncoding.AddId(builder, id);
     DictionaryEncoding.AddIndexType(builder, indexTypeOffset);
+    DictionaryEncoding.AddDictionaryKind(builder, dictionaryKind);
     DictionaryEncoding.AddIsOrdered(builder, isOrdered);
     return DictionaryEncoding.EndDictionaryEncoding(builder);
   }
 
-  public static void StartDictionaryEncoding(FlatBufferBuilder builder) { builder.StartObject(3); }
+  public static void StartDictionaryEncoding(FlatBufferBuilder builder) { builder.StartTable(4); }
   public static void AddId(FlatBufferBuilder builder, long id) { builder.AddLong(0, id, 0); }
   public static void AddIndexType(FlatBufferBuilder builder, Offset<Int> indexTypeOffset) { builder.AddOffset(1, indexTypeOffset.Value, 0); }
   public static void AddIsOrdered(FlatBufferBuilder builder, bool isOrdered) { builder.AddBool(2, isOrdered, false); }
+  public static void AddDictionaryKind(FlatBufferBuilder builder, DictionaryKind dictionaryKind) { builder.AddShort(3, (short)dictionaryKind, 0); }
   public static Offset<DictionaryEncoding> EndDictionaryEncoding(FlatBufferBuilder builder) {
-    int o = builder.EndObject();
+    int o = builder.EndTable();
     return new Offset<DictionaryEncoding>(o);
   }
-};
+}
+
 
+static internal class DictionaryEncodingVerify
+{
+  static public bool Verify(Google.FlatBuffers.Verifier verifier, uint tablePos)
+  {
+    return verifier.VerifyTableStart(tablePos)
+      && verifier.VerifyField(tablePos, 4 /*Id*/, 8 /*long*/, 8, false)
+      && verifier.VerifyTable(tablePos, 6 /*IndexType*/, IntVerify.Verify, false)
+      && verifier.VerifyField(tablePos, 8 /*IsOrdered*/, 1 /*bool*/, 1, false)
+      && verifier.VerifyField(tablePos, 10 /*DictionaryKind*/, 2 /*DictionaryKind*/, 2, false)
+      && verifier.VerifyTableEnd(tablePos);
+  }
+}
 
 }

csharp/src/Apache.Arrow/Flatbuf/Enums/BodyCompressionMethod.cs

@@ -17,7 +17,7 @@ internal enum BodyCompressionMethod : sbyte
   /// uncompressed length may be set to -1 to indicate that the data that
   /// follows is not compressed, which can be useful for cases where
   /// compression does not yield appreciable savings.
- BUFFER = 0,
+  BUFFER = 0,
 };
 
 

csharp/src/Apache.Arrow/Flatbuf/Enums/DictionaryKind.cs

@@ -0,0 +1,19 @@
+// <auto-generated>
+//  automatically generated by the FlatBuffers compiler, do not modify
+// </auto-generated>
+
+namespace Apache.Arrow.Flatbuf
+{
+
+/// ----------------------------------------------------------------------
+/// Dictionary encoding metadata
+/// Maintained for forwards compatibility, in the future
+/// Dictionaries might be explicit maps between integers and values
+/// allowing for non-contiguous index values
+internal enum DictionaryKind : short
+{
+  DenseArray = 0,
+};
+
+
+}

csharp/src/Apache.Arrow/Flatbuf/Enums/IntervalUnit.cs

@@ -7,8 +7,9 @@ namespace Apache.Arrow.Flatbuf
 
 internal enum IntervalUnit : short
 {
- YEAR_MONTH = 0,
- DAY_TIME = 1,
+  YEAR_MONTH = 0,
+  DAY_TIME = 1,
+  MONTH_DAY_NANO = 2,
 };
 
 

csharp/src/Apache.Arrow/Flatbuf/Enums/MessageHeader.cs

@@ -15,12 +15,44 @@ namespace Apache.Arrow.Flatbuf
 /// it is best to send data using RecordBatch
 internal enum MessageHeader : byte
 {
- NONE = 0,
- Schema = 1,
- DictionaryBatch = 2,
- RecordBatch = 3,
- Tensor = 4,
+  NONE = 0,
+  Schema = 1,
+  DictionaryBatch = 2,
+  RecordBatch = 3,
+  Tensor = 4,
+  SparseTensor = 5,
 };
 
 
+
+static internal class MessageHeaderVerify
+{
+  static public bool Verify(Google.FlatBuffers.Verifier verifier, byte typeId, uint tablePos)
+  {
+    bool result = true;
+    switch((MessageHeader)typeId)
+    {
+      case MessageHeader.Schema:
+        result = SchemaVerify.Verify(verifier, tablePos);
+        break;
+      case MessageHeader.DictionaryBatch:
+        result = DictionaryBatchVerify.Verify(verifier, tablePos);
+        break;
+      case MessageHeader.RecordBatch:
+        result = RecordBatchVerify.Verify(verifier, tablePos);
+        break;
+      case MessageHeader.Tensor:
+        result = TensorVerify.Verify(verifier, tablePos);
+        break;
+      case MessageHeader.SparseTensor:
+        result = SparseTensorVerify.Verify(verifier, tablePos);
+        break;
+      default: result = true;
+        break;
+    }
+    return result;
+  }
+}
+
+
 }