diff --git a/CHANGELOG.md b/CHANGELOG.md
index b6cfe6dd92dd8..cc66cc82f00af 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -160,6 +160,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Disable concurrent aggs for Diversified Sampler and Sampler aggs ([#11087](https://github.com/opensearch-project/OpenSearch/issues/11087))
 - Made leader/follower check timeout setting dynamic ([#10528](https://github.com/opensearch-project/OpenSearch/pull/10528))
 - Improve boolean parsing performance ([#11308](https://github.com/opensearch-project/OpenSearch/pull/11308))
+- Interpret byte array as primitive using VarHandles ([#11362](https://github.com/opensearch-project/OpenSearch/pull/11362))
 - Change error message when per shard document limit is breached ([#11312](https://github.com/opensearch-project/OpenSearch/pull/11312))
 
 ### Deprecated
diff --git a/libs/core/src/main/java/org/opensearch/core/common/bytes/AbstractBytesReference.java b/libs/core/src/main/java/org/opensearch/core/common/bytes/AbstractBytesReference.java
index 8c1efcd00c24e..a2bf7e499dee8 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/bytes/AbstractBytesReference.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/bytes/AbstractBytesReference.java
@@ -53,11 +53,6 @@ public abstract class AbstractBytesReference implements BytesReference {
     private Integer hash = null;
     private static final int MAX_UTF16_LENGTH = Integer.MAX_VALUE >> 1;
 
-    @Override
-    public int getInt(int index) {
-        return (get(index) & 0xFF) << 24 | (get(index + 1) & 0xFF) << 16 | (get(index + 2) & 0xFF) << 8 | get(index + 3) & 0xFF;
-    }
-
     @Override
     public int indexOf(byte marker, int from) {
         final int to = length();
diff --git a/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesArray.java b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesArray.java
index ae04ddcc19eee..d7a8414935143 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesArray.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesArray.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.core.common.bytes;
 
+import org.apache.lucene.util.BitUtil;
 import org.apache.lucene.util.BytesRef;
 import org.opensearch.core.common.io.stream.StreamInput;
 
@@ -83,6 +84,11 @@ public byte get(int index) {
         return bytes[offset + index];
     }
 
+    @Override
+    public int getInt(int index) {
+        return (int) BitUtil.VH_BE_INT.get(bytes, offset + index);
+    }
+
     @Override
     public int length() {
         return length;
diff --git a/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
index 9d24d3653397b..8cb65c9feb1ca 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/bytes/BytesReference.java
@@ -153,9 +153,11 @@ static BytesReference fromByteArray(ByteArray byteArray, int length) {
     byte get(int index);
 
     /**
-     * Returns the integer read from the 4 bytes (BE) starting at the given index.
+     * Returns the integer read from the 4 bytes (big endian) starting at the given index.
      */
-    int getInt(int index);
+    default int getInt(int index) {
+        return ((get(index) & 0xFF) << 24) | ((get(index + 1) & 0xFF) << 16) | ((get(index + 2) & 0xFF) << 8) | (get(index + 3) & 0xFF);
+    }
 
     /**
      * Finds the index of the first occurrence of the given marker between within the given bounds.
diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesStreamInput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesStreamInput.java
index 30c84708728ef..cad43f817faaf 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesStreamInput.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BytesStreamInput.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.core.common.io.stream;
 
+import org.apache.lucene.util.BitUtil;
 import org.apache.lucene.util.BytesRef;
 
 import java.io.EOFException;
@@ -121,4 +122,33 @@ public int read() throws IOException {
         return bytes[pos++] & 0xFF;
     }
 
+    @Override
+    public short readShort() throws IOException {
+        if (available() < Short.BYTES) {
+            throw new EOFException();
+        }
+        short value = (short) BitUtil.VH_BE_SHORT.get(bytes, pos);
+        pos += Short.BYTES;
+        return value;
+    }
+
+    @Override
+    public int readInt() throws IOException {
+        if (available() < Integer.BYTES) {
+            throw new EOFException();
+        }
+        int value = (int) BitUtil.VH_BE_INT.get(bytes, pos);
+        pos += Integer.BYTES;
+        return value;
+    }
+
+    @Override
+    public long readLong() throws IOException {
+        if (available() < Long.BYTES) {
+            throw new EOFException();
+        }
+        long value = (long) BitUtil.VH_BE_LONG.get(bytes, pos);
+        pos += Long.BYTES;
+        return value;
+    }
 }
diff --git a/libs/core/src/main/java/org/opensearch/core/util/BytesRefUtils.java b/libs/core/src/main/java/org/opensearch/core/util/BytesRefUtils.java
index 30c9f182fcae6..2aad068534b9d 100644
--- a/libs/core/src/main/java/org/opensearch/core/util/BytesRefUtils.java
+++ b/libs/core/src/main/java/org/opensearch/core/util/BytesRefUtils.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.core.util;
 
+import org.apache.lucene.util.BitUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefArray;
 import org.apache.lucene.util.BytesRefBuilder;
@@ -103,12 +104,12 @@ public static int sortAndDedup(final BytesRefArray bytes, final int[] indices) {
         return uniqueCount;
     }
 
+    /**
+     * Decodes a long value written as bytes in big endian order.
+     * @param bytes in big endian order
+     * @return long value
+     */
     public static long bytesToLong(BytesRef bytes) {
-        int high = (bytes.bytes[bytes.offset + 0] << 24) | ((bytes.bytes[bytes.offset + 1] & 0xff) << 16) | ((bytes.bytes[bytes.offset + 2]
-            & 0xff) << 8) | (bytes.bytes[bytes.offset + 3] & 0xff);
-        int low = (bytes.bytes[bytes.offset + 4] << 24) | ((bytes.bytes[bytes.offset + 5] & 0xff) << 16) | ((bytes.bytes[bytes.offset + 6]
-            & 0xff) << 8) | (bytes.bytes[bytes.offset + 7] & 0xff);
-        return (((long) high) << 32) | (low & 0x0ffffffffL);
+        return (long) BitUtil.VH_BE_LONG.get(bytes.bytes, bytes.offset);
     }
-
 }
diff --git a/libs/core/src/test/java/org/opensearch/core/util/BytesRefUtilsTests.java b/libs/core/src/test/java/org/opensearch/core/util/BytesRefUtilsTests.java
index 421263b883f2a..214f9292ae3a5 100644
--- a/libs/core/src/test/java/org/opensearch/core/util/BytesRefUtilsTests.java
+++ b/libs/core/src/test/java/org/opensearch/core/util/BytesRefUtilsTests.java
@@ -12,7 +12,6 @@
 import org.apache.lucene.util.BytesRefArray;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.apache.lucene.util.Counter;
-import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.nio.ByteBuffer;
@@ -90,8 +89,12 @@ public void testSortByteRefArray() {
     }
 
     public void testBytesToLong() {
-        final long value = randomLong();
-        final BytesReference buffer = BytesReference.fromByteBuffer(ByteBuffer.allocate(8).putLong(value).flip());
-        assertThat(BytesRefUtils.bytesToLong(buffer.toBytesRef()), equalTo(value));
+        long value = randomLong();
+        int paddingStart = randomIntBetween(0, 10);
+        int paddingEnd = randomIntBetween(0, 10);
+        byte[] bytes = new byte[paddingStart + Long.BYTES + paddingEnd];
+        ByteBuffer.wrap(bytes).putLong(paddingStart, value);
+        BytesRef bytesRef = new BytesRef(bytes, paddingStart, Long.BYTES);
+        assertThat(BytesRefUtils.bytesToLong(bytesRef), equalTo(value));
     }
 }
diff --git a/server/src/main/java/org/opensearch/common/lucene/store/ByteArrayIndexInput.java b/server/src/main/java/org/opensearch/common/lucene/store/ByteArrayIndexInput.java
index 6eb613daf5133..bb273b14c42e2 100644
--- a/server/src/main/java/org/opensearch/common/lucene/store/ByteArrayIndexInput.java
+++ b/server/src/main/java/org/opensearch/common/lucene/store/ByteArrayIndexInput.java
@@ -33,6 +33,7 @@
 
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.RandomAccessInput;
+import org.apache.lucene.util.BitUtil;
 
 import java.io.EOFException;
 import java.io.IOException;
@@ -121,47 +122,25 @@ public void readBytes(final byte[] b, final int offset, int len) throws IOExcept
     @Override
     public byte readByte(long pos) throws IOException {
         validatePos(pos, Byte.BYTES);
-        return internalReadByte(pos);
+        return bytes[offset + (int) pos];
     }
 
     @Override
     public short readShort(long pos) throws IOException {
         validatePos(pos, Short.BYTES);
-        return internalReadShort(pos);
+        return (short) BitUtil.VH_LE_SHORT.get(bytes, offset + (int) pos);
     }
 
     @Override
     public int readInt(long pos) throws IOException {
         validatePos(pos, Integer.BYTES);
-        return internalReadInt(pos);
+        return (int) BitUtil.VH_LE_INT.get(bytes, offset + (int) pos);
     }
 
     @Override
     public long readLong(long pos) throws IOException {
         validatePos(pos, Long.BYTES);
-        return internalReadLong(pos);
-    }
-
-    private byte internalReadByte(long pos) {
-        return bytes[offset + (int) pos];
-    }
-
-    private short internalReadShort(long pos) {
-        final byte p1 = internalReadByte(pos);
-        final byte p2 = internalReadByte(pos + 1);
-        return (short) (((p2 & 0xFF) << 8) | (p1 & 0xFF));
-    }
-
-    private int internalReadInt(long pos) {
-        final short p1 = internalReadShort(pos);
-        final short p2 = internalReadShort(pos + Short.BYTES);
-        return ((p2 & 0xFFFF) << 16) | (p1 & 0xFFFF);
-    }
-
-    public long internalReadLong(long pos) {
-        final int p1 = internalReadInt(pos);
-        final int p2 = internalReadInt(pos + Integer.BYTES);
-        return (((long) p2) << 32) | (p1 & 0xFFFFFFFFL);
+        return (long) BitUtil.VH_LE_LONG.get(bytes, offset + (int) pos);
     }
 
     private void validatePos(long pos, int len) throws EOFException {
diff --git a/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamInput.java b/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamInput.java
index 8c9ccc3b487df..f75f27b7bcb91 100644
--- a/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamInput.java
+++ b/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamInput.java
@@ -33,6 +33,7 @@
 package org.opensearch.index.translog;
 
 import org.apache.lucene.store.BufferedChecksum;
+import org.apache.lucene.util.BitUtil;
 import org.opensearch.core.common.io.stream.FilterStreamInput;
 import org.opensearch.core.common.io.stream.StreamInput;
 
@@ -92,22 +93,21 @@ public void readBytes(byte[] b, int offset, int len) throws IOException {
     public short readShort() throws IOException {
         final byte[] buf = buffer.get();
         readBytes(buf, 0, 2);
-        return (short) (((buf[0] & 0xFF) << 8) | (buf[1] & 0xFF));
+        return (short) BitUtil.VH_BE_SHORT.get(buf, 0);
     }
 
     @Override
     public int readInt() throws IOException {
         final byte[] buf = buffer.get();
         readBytes(buf, 0, 4);
-        return ((buf[0] & 0xFF) << 24) | ((buf[1] & 0xFF) << 16) | ((buf[2] & 0xFF) << 8) | (buf[3] & 0xFF);
+        return (int) BitUtil.VH_BE_INT.get(buf, 0);
     }
 
     @Override
     public long readLong() throws IOException {
         final byte[] buf = buffer.get();
         readBytes(buf, 0, 8);
-        return (((long) (((buf[0] & 0xFF) << 24) | ((buf[1] & 0xFF) << 16) | ((buf[2] & 0xFF) << 8) | (buf[3] & 0xFF))) << 32) | ((((buf[4]
-            & 0xFF) << 24) | ((buf[5] & 0xFF) << 16) | ((buf[6] & 0xFF) << 8) | (buf[7] & 0xFF)) & 0xFFFFFFFFL);
+        return (long) BitUtil.VH_BE_LONG.get(buf, 0);
     }
 
     @Override