diff --git a/.github/workflows/archery.yml b/.github/workflows/archery.yml
index 87f365b9065c8..b016f7d11b9fa 100644
--- a/.github/workflows/archery.yml
+++ b/.github/workflows/archery.yml
@@ -34,7 +34,6 @@ on:
 env:
   ARCHERY_DEBUG: 1
   ARCHERY_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-  ARCHERY_USE_DOCKER_CLI: 1
 
 concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index b6075746ff40a..25db1c39ad89e 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -25,7 +25,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
   ARROW_ENABLE_TIMING_TESTS: OFF
   DOCKER_VOLUME_PREFIX: ".docker/"
 
diff --git a/.github/workflows/docs_light.yml b/.github/workflows/docs_light.yml
index f66e8473e2516..ea7fe5d02d7b8 100644
--- a/.github/workflows/docs_light.yml
+++ b/.github/workflows/docs_light.yml
@@ -34,7 +34,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
   ARROW_ENABLE_TIMING_TESTS: OFF
   DOCKER_VOLUME_PREFIX: ".docker/"
 
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index 0d369d252b56c..d9c1d28f01503 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -43,7 +43,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
 
 jobs:
 
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 46d422a53ae69..43f8af0a600d8 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -54,7 +54,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
index d4211c2c81cb5..0317879b580ba 100644
--- a/.github/workflows/java.yml
+++ b/.github/workflows/java.yml
@@ -46,7 +46,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml
index 533da7c36be34..c2bc679e681a2 100644
--- a/.github/workflows/java_jni.yml
+++ b/.github/workflows/java_jni.yml
@@ -46,7 +46,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml
index ad22968a3a68b..630bef61105f6 100644
--- a/.github/workflows/js.yml
+++ b/.github/workflows/js.yml
@@ -40,7 +40,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
 
 jobs:
 
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index daadd971f8ac8..916db2580e371 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -42,7 +42,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml
index 0ff7266860f29..544aa9b75c249 100644
--- a/.github/workflows/r.yml
+++ b/.github/workflows/r.yml
@@ -52,7 +52,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml
index ca2305a7f9357..e4d650e74a8ad 100644
--- a/.github/workflows/ruby.yml
+++ b/.github/workflows/ruby.yml
@@ -54,7 +54,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml
index 3f039315b505a..1b3c9eca1814a 100644
--- a/.github/workflows/swift.yml
+++ b/.github/workflows/swift.yml
@@ -42,7 +42,6 @@ permissions:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
diff --git a/dev/archery/archery/docker/cli.py b/dev/archery/archery/docker/cli.py
index 7053db2afccff..c0897e462d557 100644
--- a/dev/archery/archery/docker/cli.py
+++ b/dev/archery/archery/docker/cli.py
@@ -54,7 +54,7 @@ def _execute(self, *args, **kwargs):
 @click.option('--using-docker-buildx', default=False, is_flag=True,
               envvar='ARCHERY_USE_DOCKER_BUILDX',
               help="Use buildx with docker CLI directly for building instead "
-                   "of calling docker-compose or the plain docker build "
+                   "of calling `docker compose` or the plain docker build "
                    "command. This option makes the build cache reusable "
                    "across hosts.")
 @click.pass_context
@@ -78,8 +78,7 @@ def docker(ctx, src, dry_run, using_docker_cli, using_docker_buildx):
                             using_docker=using_docker_cli,
                             using_buildx=using_docker_buildx,
                             debug=ctx.obj.get('debug', False),
-                            compose_bin=("docker compose" if using_docker_cli
-                                         else "docker-compose"))
+                            compose_bin="docker compose")
     if dry_run:
         _mock_compose_calls(compose)
     ctx.obj['compose'] = compose
diff --git a/dev/archery/archery/docker/core.py b/dev/archery/archery/docker/core.py
index 5be4887ea4f63..4de75c68242d5 100644
--- a/dev/archery/archery/docker/core.py
+++ b/dev/archery/archery/docker/core.py
@@ -135,9 +135,9 @@ def _read_config(self, config_path, compose_bin):
             compose = Docker()
             args = ['compose']
         else:
-            compose = Command('docker-compose')
+            compose = Command(compose_bin)
             args = []
-        args += ['--file', str(config_path), 'config']
+        args += [f'--file={config_path}', 'config']
         result = compose.run(*args, env=self.env, check=False,
                              stderr=subprocess.PIPE, stdout=subprocess.PIPE)
 
@@ -193,7 +193,7 @@ def clear_pull_memory(self):
     def _execute_compose(self, *args, **kwargs):
         # execute as a docker compose command
         try:
-            result = super().run('--file', str(self.config.path), *args,
+            result = super().run(f'--file={self.config.path}', *args,
                                  env=self.config.env, **kwargs)
             result.check_returncode()
         except subprocess.CalledProcessError as e:
diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml
index 77e8867652e65..bdbed1bd678e6 100644
--- a/dev/tasks/java-jars/github.yml
+++ b/dev/tasks/java-jars/github.yml
@@ -30,7 +30,6 @@ jobs:
       ARCH: {{ '${{ matrix.platform.archery_arch }}' }}
       ARCH_ALIAS: {{ '${{ matrix.platform.archery_arch_alias }}' }}
       ARCH_SHORT: {{ '${{ matrix.platform.archery_arch_short }}' }}
-      ARCHERY_USE_DOCKER_CLI: {{ "${{matrix.platform.archery_use_docker_cli || '1'}}" }}
     strategy:
       fail-fast: false
       matrix:
@@ -45,7 +44,6 @@ jobs:
             archery_arch: "arm64v8"
             archery_arch_alias: "aarch64"
             archery_arch_short: "arm64"
-            archery_use_docker_cli: "0"
     steps:
       {{ macros.github_checkout_arrow()|indent }}
       {{ macros.github_free_space()|indent }}
diff --git a/dev/tasks/linux-packages/github.linux.yml b/dev/tasks/linux-packages/github.linux.yml
index 891682c4358d8..cce976cd60e4e 100644
--- a/dev/tasks/linux-packages/github.linux.yml
+++ b/dev/tasks/linux-packages/github.linux.yml
@@ -29,7 +29,6 @@ jobs:
     {% endif %}
     env:
       ARCHITECTURE: {{ architecture }}
-      ARCHERY_USE_DOCKER_CLI: {{ '0' if architecture == 'arm64' else '1' }}
     steps:
       {{ macros.github_checkout_arrow()|indent }}
       {{ macros.github_login_dockerhub()|indent }}
diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja
index b225109050aae..6423ca0e9efda 100644
--- a/dev/tasks/macros.jinja
+++ b/dev/tasks/macros.jinja
@@ -26,7 +26,6 @@ on:
 
 env:
   ARCHERY_DEBUG: 1
-  ARCHERY_USE_DOCKER_CLI: 1
 {% endmacro %}
 
 {%- macro github_checkout_arrow(fetch_depth=1, submodules="recursive", action_v="4") -%}
diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml
index 5c82bf74b30b7..38edf1537d7bd 100644
--- a/dev/tasks/python-wheels/github.linux.yml
+++ b/dev/tasks/python-wheels/github.linux.yml
@@ -31,10 +31,8 @@ jobs:
       # archery uses these environment variables
       {% if arch == "amd64" %}
       ARCH: amd64
-      ARCHERY_USE_DOCKER_CLI: 1
       {% else %}
       ARCH: arm64v8
-      ARCHERY_USE_DOCKER_CLI: 0
       {% endif %}
       PYTHON: "{{ python_version }}"
 
diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml
index 01f4977a9b0b1..0c3e2bb4bf20b 100644
--- a/dev/tasks/python-wheels/github.windows.yml
+++ b/dev/tasks/python-wheels/github.windows.yml
@@ -33,8 +33,6 @@ jobs:
       # note that we don't run docker build since there wouldn't be a cache hit
       # and rebuilding the dependencies takes a fair amount of time
       REPO: ghcr.io/ursacomputing/arrow
-      # prefer the docker cli over docker-compose
-      ARCHERY_USE_DOCKER_CLI: 1
 
     steps:
       {{ macros.github_checkout_arrow()|indent }}
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 07a4d638f1291..78dff68968fcb 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -1110,7 +1110,6 @@ tasks:
     template: docker-tests/github.linux.yml
     params:
       env:
-        ARCHERY_USE_DOCKER_CLI: 0
         UBUNTU: 20.04
       flags: -e ARROW_SKYHOOK=ON
       image: ubuntu-cpp
@@ -1494,16 +1493,12 @@ tasks:
     ci: github
     template: docker-tests/github.cuda.yml
     params:
-      env:
-        ARCHERY_USE_DOCKER_CLI: 0
       image: ubuntu-cuda-cpp
 
   test-cuda-python:
     ci: github
     template: docker-tests/github.cuda.yml
     params:
-      env:
-        ARCHERY_USE_DOCKER_CLI: 0
       image: ubuntu-cuda-python
 
   ############################## Fuzz tests #################################
@@ -1565,7 +1560,6 @@ tasks:
     template: docker-tests/github.linux.yml
     params:
       env:
-        ARCHERY_USE_DOCKER_CLI: 0
         HDFS: "{{ hdfs_version }}"
         PYTHON: "3.10"
       image: conda-python-hdfs
diff --git a/docker-compose.yml b/docker-compose.yml
index daa5c74bcb969..67b5e245056f7 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -203,7 +203,6 @@ volumes:
     name: ${ARCH}-ubuntu-${UBUNTU}-ccache
 
 services:
-
   ################################# C++ #######################################
   # Release build:
   #   docker-compose run -e ARROW_BUILD_TYPE=release conda-cpp|debian-cpp|...