diff --git a/testing/cpm/cpm_generate_pins-nested/CMakeLists.txt b/testing/cpm/cpm_generate_pins-nested/CMakeLists.txt
index 5151d285..6320281e 100644
--- a/testing/cpm/cpm_generate_pins-nested/CMakeLists.txt
+++ b/testing/cpm/cpm_generate_pins-nested/CMakeLists.txt
@@ -24,8 +24,15 @@ add_subdirectory(b)
 include(${rapids-cmake-dir}/cpm/init.cmake)
 rapids_cpm_init(GENERATE_PINNED_VERSIONS)
 
+# only check projects that were downloaded by CPM (ignore those already in the build environment)
+#
 # don't check cuco here as it uses hashes and that breaks our verify script
-set(projects-to-verify rmm fmt spdlog CCCL)
+foreach(proj IN ITEMS rmm fmt spdlog CCCL)
+  if(${proj}_SOURCE_DIR)
+    list(APPEND projects-to-verify ${proj})
+  endif()
+endforeach()
+
 add_custom_target(verify_generated_pins ALL
   COMMAND ${CMAKE_COMMAND} -S="${CMAKE_SOURCE_DIR}/verify/" -B"${CMAKE_BINARY_DIR}/verify_build"
   -D"rapids-cmake-dir=${rapids-cmake-dir}"
diff --git a/testing/cpm/cpm_generate_pins-nested/verify/CMakeLists.txt b/testing/cpm/cpm_generate_pins-nested/verify/CMakeLists.txt
index 918846b0..aea5c4b7 100644
--- a/testing/cpm/cpm_generate_pins-nested/verify/CMakeLists.txt
+++ b/testing/cpm/cpm_generate_pins-nested/verify/CMakeLists.txt
@@ -32,22 +32,29 @@ foreach(proj IN LISTS projects-to-verify)
   rapids_cpm_package_details(${proj} ${proj}_version ${proj}_repository ${proj}_tag ${proj}_shallow ${proj}_exclude)
 endforeach()
 
-
-file(READ "${CMAKE_CURRENT_BINARY_DIR}/../rapids-cmake/pinned_versions.json" json_data)
-
 # Verify that the pinned_versions is valid json by using it
 rapids_cpm_package_override("${CMAKE_CURRENT_BINARY_DIR}/../rapids-cmake/pinned_versions.json")
 
+# helper macro for gathering versions.json and pinned_versions.json content to be printed in error messages
+macro(_get_json_data)
+  include("${rapids-cmake-dir}/cpm/detail/get_default_json.cmake")
+  include("${rapids-cmake-dir}/cpm/detail/get_override_json.cmake")
+  get_default_json(${proj} default_versions)
+  get_override_json(${proj} pinned_versions)
+endmacro()
+
 foreach(proj IN LISTS projects-to-verify)
   # Verify that each git_tag is now different.
   rapids_cpm_package_details(${proj} ${proj}_version ${proj}_repository pin_${proj}_tag pin_${proj}_shallow ${proj}_exclude)
   if(pin_${proj}_tag STREQUAL ${proj}_tag)
-    message(FATAL_ERROR "pinned ${proj} tag (${pin_${proj}_tag}) should differ compared to baseline ${${proj}_tag}")
+    _get_json_data()
+    message(FATAL_ERROR "pinned ${proj} tag (${pin_${proj}_tag}) should differ compared to baseline ${${proj}_tag}\npinned_versions.json:\n${pinned_versions}\nversions.json:\n${default_versions}")
   endif()
 
   # Everything should have shallow marked as false
   # so that clones by SHA1 work
   if(pin_${proj}_shallow)
-    message(FATAL_ERROR "pin_${proj}_shallow is expected to be false, but got ${pin_${proj}_shallow}")
+    _get_json_data()
+    message(FATAL_ERROR "pin_${proj}_shallow is expected to be false, but got ${pin_${proj}_shallow}\npinned_versions.json:\n${pinned_versions}\nversions.json:\n${default_versions}")
   endif()
 endforeach()
diff --git a/testing/cpm/cpm_generate_pins-simple/CMakeLists.txt b/testing/cpm/cpm_generate_pins-simple/CMakeLists.txt
index 6be74385..72c98cd9 100644
--- a/testing/cpm/cpm_generate_pins-simple/CMakeLists.txt
+++ b/testing/cpm/cpm_generate_pins-simple/CMakeLists.txt
@@ -22,7 +22,14 @@ include(${rapids-cmake-dir}/cpm/rmm.cmake)
 
 rapids_cpm_init(GENERATE_PINNED_VERSIONS)
 rapids_cpm_rmm()
-set(projects-to-verify rmm fmt spdlog CCCL)
+
+# only check projects that were downloaded by CPM (ignore those already in the build environment)
+foreach(proj IN ITEMS rmm fmt spdlog CCCL)
+  if(${proj}_SOURCE_DIR)
+    list(APPEND projects-to-verify ${proj})
+  endif()
+endforeach()
+
 add_custom_target(verify_generated_pins ALL
   COMMAND ${CMAKE_COMMAND} -S="${CMAKE_SOURCE_DIR}/verify/" -B"${CMAKE_BINARY_DIR}/verify_build"
   -D"rapids-cmake-dir=${rapids-cmake-dir}"
diff --git a/testing/cpm/cpm_generate_pins-simple/verify/CMakeLists.txt b/testing/cpm/cpm_generate_pins-simple/verify/CMakeLists.txt
index 736662b7..aea5c4b7 100644
--- a/testing/cpm/cpm_generate_pins-simple/verify/CMakeLists.txt
+++ b/testing/cpm/cpm_generate_pins-simple/verify/CMakeLists.txt
@@ -35,16 +35,26 @@ endforeach()
 # Verify that the pinned_versions is valid json by using it
 rapids_cpm_package_override("${CMAKE_CURRENT_BINARY_DIR}/../rapids-cmake/pinned_versions.json")
 
+# helper macro for gathering versions.json and pinned_versions.json content to be printed in error messages
+macro(_get_json_data)
+  include("${rapids-cmake-dir}/cpm/detail/get_default_json.cmake")
+  include("${rapids-cmake-dir}/cpm/detail/get_override_json.cmake")
+  get_default_json(${proj} default_versions)
+  get_override_json(${proj} pinned_versions)
+endmacro()
+
 foreach(proj IN LISTS projects-to-verify)
   # Verify that each git_tag is now different.
   rapids_cpm_package_details(${proj} ${proj}_version ${proj}_repository pin_${proj}_tag pin_${proj}_shallow ${proj}_exclude)
   if(pin_${proj}_tag STREQUAL ${proj}_tag)
-    message(FATAL_ERROR "pinned ${proj} tag (${pin_${proj}_tag}) should differ compared to baseline ${${proj}_tag}")
+    _get_json_data()
+    message(FATAL_ERROR "pinned ${proj} tag (${pin_${proj}_tag}) should differ compared to baseline ${${proj}_tag}\npinned_versions.json:\n${pinned_versions}\nversions.json:\n${default_versions}")
   endif()
 
   # Everything should have shallow marked as false
   # so that clones by SHA1 work
   if(pin_${proj}_shallow)
-    message(FATAL_ERROR "pin_${proj}_shallow is expected to be false, but got ${pin_${proj}_shallow}")
+    _get_json_data()
+    message(FATAL_ERROR "pin_${proj}_shallow is expected to be false, but got ${pin_${proj}_shallow}\npinned_versions.json:\n${pinned_versions}\nversions.json:\n${default_versions}")
   endif()
 endforeach()