From 0df6d8a1d8d6fd530ae2fd68625541d436e009c2 Mon Sep 17 00:00:00 2001 From: Ben Andersen Date: Thu, 7 Mar 2024 11:56:53 +1100 Subject: [PATCH] added test cases and rez env flag Signed-off-by: Ben Andersen --- src/rez/cli/env.py | 7 +++- src/rez/package_cache.py | 12 +++---- src/rez/resolved_context.py | 10 ++++-- src/rez/rezconfig.py | 7 ++-- src/rez/tests/test_package_cache.py | 56 +++++++++++++++++++++++++++-- 5 files changed, 75 insertions(+), 17 deletions(-) diff --git a/src/rez/cli/env.py b/src/rez/cli/env.py index 20cf68edff..a917830db5 100644 --- a/src/rez/cli/env.py +++ b/src/rez/cli/env.py @@ -116,6 +116,10 @@ def setup_parser(parser, completions=False): parser.add_argument( "--no-pkg-cache", action="store_true", help="Disable package caching") + parser.add_argument( + "--pkg-cache-sync", action="store_true", + help="Disable asynchronous package caching. " + "Process will block until packages are cached.") parser.add_argument( "--pre-command", type=str, help=SUPPRESS) PKG_action = parser.add_argument( @@ -212,7 +216,8 @@ def command(opts, parser, extra_arg_groups=None): caching=(not opts.no_cache), suppress_passive=opts.no_passive, print_stats=opts.stats, - package_caching=(not opts.no_pkg_cache) + package_caching=(not opts.no_pkg_cache), + package_cache_async=(not opts.pkg_cache_sync), ) success = (context.status == ResolverStatus.solved) diff --git a/src/rez/package_cache.py b/src/rez/package_cache.py index 95e6cf8081..8e85ca7a1c 100644 --- a/src/rez/package_cache.py +++ b/src/rez/package_cache.py @@ -366,7 +366,7 @@ def remove_variant(self, variant): return self.VARIANT_REMOVED - def add_variants(self, variants, _async=True): + def add_variants(self, variants, package_cache_async=True): """Update the package cache by adding some or all of the given variants. This method is called when a context is created or sourced. Variants @@ -460,18 +460,14 @@ def add_variants(self, variants, _async=True): else: out_target = devnull - func = subprocess.Popen - - # use subprocess.call blocks where subprocess.Popen doesn't - if not _async: - func = subprocess.call - - func( + process = subprocess.Popen( args, stdout=out_target, stderr=out_target, **kwargs ) + if not package_cache_async: + process.wait() except Exception as e: print_warning( diff --git a/src/rez/resolved_context.py b/src/rez/resolved_context.py index b7a82dcc89..937ec33939 100644 --- a/src/rez/resolved_context.py +++ b/src/rez/resolved_context.py @@ -167,7 +167,7 @@ def __init__(self, package_requests, verbosity=0, timestamp=None, package_filter=None, package_orderers=None, max_fails=-1, add_implicit_packages=True, time_limit=-1, callback=None, package_load_callback=None, buf=None, suppress_passive=False, - print_stats=False, package_caching=None): + print_stats=False, package_caching=None, package_cache_async=None): """Perform a package resolve, and store the result. Args: @@ -205,6 +205,8 @@ def __init__(self, package_requests, verbosity=0, timestamp=None, package_caching (bool|None): If True, apply package caching settings as per the config. If None, enable as determined by config setting :data:`package_cache_during_build`. + package_cache_async (Optional[bool]): If True, cache packages asynchronously. + If None, use the config setting :data:`package_cache_async` """ self.load_path = None @@ -249,6 +251,10 @@ def __init__(self, package_requests, verbosity=0, timestamp=None, self.package_caching = package_caching + if package_cache_async is None: + package_cache_async = config.package_cache_async + self.package_cache_async = package_cache_async + # patch settings self.default_patch_lock = PatchLock.no_lock self.patch_locks = {} @@ -1846,7 +1852,7 @@ def _update_package_cache(self): if pkgcache: pkgcache.add_variants( self.resolved_packages, - config.package_cache_async + self.package_cache_async ) @classmethod diff --git a/src/rez/rezconfig.py b/src/rez/rezconfig.py index 38ea03b279..59151c0adb 100644 --- a/src/rez/rezconfig.py +++ b/src/rez/rezconfig.py @@ -279,7 +279,8 @@ package_cache_during_build = False # Enable package caching to run asynchronously during a resolve. -package_cache_async = True +# If this is false, a resolve will block until all packages are cached. +package_cache_async = True # Allow caching of local packages. You would only want to set this True for # testing purposes. @@ -316,7 +317,7 @@ # This is useful as Platform.os might show different # values depending on the availability of ``lsb-release`` on the system. # The map supports regular expression, e.g. to keep versions. -# +# # .. note:: # The following examples are not necessarily recommendations. # @@ -1122,7 +1123,7 @@ # Enables/disables colorization globally. # -# .. warning:: +# .. warning:: # Turned off for Windows currently as there seems to be a problem with the colorama module. # # May also set to the string ``force``, which will make rez output color styling diff --git a/src/rez/tests/test_package_cache.py b/src/rez/tests/test_package_cache.py index c340dbfe7b..ee527ae359 100644 --- a/src/rez/tests/test_package_cache.py +++ b/src/rez/tests/test_package_cache.py @@ -134,7 +134,6 @@ def test_caching_on_resolve(self): # Creating the context will asynchronously add variants to the cache # in a separate proc. - # c = ResolvedContext([ "timestamped-1.2.0", "pyfoo-3.1.0" # won't cache, see earlier test @@ -144,14 +143,65 @@ def test_caching_on_resolve(self): # Retry 50 times with 0.1 sec interval, 5 secs is more than enough for # the very small variant to be copied to cache. - # cached_root = None + resolve_not_always_cached = False for _ in range(50): - time.sleep(0.1) cached_root = pkgcache.get_cached_root(variant) if cached_root: break + resolve_not_always_cached = True + time.sleep(0.1) + + self.assertNotEqual(cached_root, None) + + # Test that the package is not immediately cached, since it is asynchronous + # WARNING: This is dangerous since it does open the test to a race condition and + # will fail if the cache happens faster than the resolve. + self.assertNotEqual(resolve_not_always_cached, False) + + expected_payload_file = os.path.join(cached_root, "stuff.txt") + self.assertTrue(os.path.exists(expected_payload_file)) + + # check that refs to root point to cache location in rex code + for ref in ("resolve.timestamped.root", "'{resolve.timestamped.root}'"): + proc = c.execute_rex_code( + code="info(%s)" % ref, + stdout=subprocess.PIPE, + universal_newlines=True + ) + + out, _ = proc.communicate() + root = out.strip() + + self.assertEqual( + root, cached_root, + "Reference %r should resolve to %s, but resolves to %s" + % (ref, cached_root, root) + ) + + @install_dependent() + def test_caching_on_resolve_synchronous(self): + """Test that cache is updated as expected on + resolved env using syncrhonous package caching.""" + pkgcache = self._pkgcache() + + with restore_os_environ(): + # set config settings into env so rez-pkg-cache proc sees them + os.environ.update(self.get_settings_env()) + + # Creating the context will asynchronously add variants to the cache + # in a separate proc. + c = ResolvedContext([ + "timestamped-1.2.0", + "pyfoo-3.1.0", + ], + package_cache_async=False, + ) + + variant = c.get_resolved_package("timestamped") + # The first time we try to access it will be cached, because the cache is blocking + cached_root = pkgcache.get_cached_root(variant) self.assertNotEqual(cached_root, None) expected_payload_file = os.path.join(cached_root, "stuff.txt")