diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/LocalActivityWorker.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/LocalActivityWorker.java
index 69babb1fb..1c3fdbeea 100644
--- a/temporal-sdk/src/main/java/io/temporal/internal/worker/LocalActivityWorker.java
+++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/LocalActivityWorker.java
@@ -127,6 +127,14 @@ private void submitAttempt(
     slotQueue.submitAttempt(reservationDat, isRetry, task);
   }
 
+  private RetryOptions getRetryOptions(PollActivityTaskQueueResponseOrBuilder activityTask) {
+    if (isRetryPolicyNotSet(activityTask)) {
+      return RetryOptions.getDefaultInstance();
+    } else {
+      return RetryOptionsUtils.toRetryOptions(activityTask.getRetryPolicy());
+    }
+  }
+
   /**
    * @param executionContext execution context of the activity
    * @param activityTask activity task
@@ -154,11 +162,7 @@ private RetryDecision shouldRetry(
       throw (Error) attemptThrowable;
     }
 
-    if (isRetryPolicyNotSet(activityTask)) {
-      return new RetryDecision(RetryState.RETRY_STATE_RETRY_POLICY_NOT_SET, null);
-    }
-
-    RetryOptions retryOptions = RetryOptionsUtils.toRetryOptions(activityTask.getRetryPolicy());
+    RetryOptions retryOptions = getRetryOptions(activityTask);
 
     if (RetryOptionsUtils.isNotRetryable(retryOptions, attemptThrowable)) {
       return new RetryDecision(RetryState.RETRY_STATE_NON_RETRYABLE_FAILURE, null);
@@ -368,11 +372,7 @@ private RetryState shouldStillRetry(
         @Nullable Failure previousLocalExecutionFailure) {
       int currentAttempt = activityTask.getAttempt();
 
-      if (isRetryPolicyNotSet(activityTask)) {
-        return RetryState.RETRY_STATE_RETRY_POLICY_NOT_SET;
-      }
-
-      RetryOptions retryOptions = RetryOptionsUtils.toRetryOptions(activityTask.getRetryPolicy());
+      RetryOptions retryOptions = getRetryOptions(activityTask);
 
       if (previousLocalExecutionFailure != null
           && previousLocalExecutionFailure.hasApplicationFailureInfo()