Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add package_cache_async flag. #1452

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/rez/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -419,6 +419,7 @@ def _parse_env_var(self, value):
"package_cache_during_build": Bool,
"package_cache_local": Bool,
"package_cache_same_device": Bool,
"package_cache_async": Bool,
"color_enabled": ForceOrBool,
"resolve_caching": Bool,
"cache_package_files": Bool,
Expand Down
16 changes: 11 additions & 5 deletions src/rez/package_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -371,21 +371,21 @@ def remove_variant(self, variant):

return self.VARIANT_REMOVED

def add_variants_async(self, variants):
def add_variants(self, variants, _async=True):
"""Update the package cache by adding some or all of the given variants.

This method is called when a context is created or sourced. Variants
are then added to the cache in a separate process.
"""

# A prod install is necessary because add_variants_async works by
# A prod install is necessary because add_variants works by
# starting a rez-pkg-cache proc, and this can only be done reliably in
# a prod install. On non-windows we could fork instead, but there would
# remain no good solution on windows.
#
if not system.is_production_rez_install:
raise PackageCacheError(
"PackageCache.add_variants_async is only supported in a "
"PackageCache.add_variants is only supported in a "
"production rez installation."
)

Expand Down Expand Up @@ -465,8 +465,14 @@ def add_variants_async(self, variants):
else:
out_target = devnull

subprocess.Popen(
[exe, "--daemon", self.path],
JeanChristopheMorinPerso marked this conversation as resolved.
Show resolved Hide resolved
func = subprocess.Popen

# use subprocess.call blocks where subprocess.Popen doesn't
if not _async:
func = subprocess.call

func(
args,
stdout=out_target,
stderr=out_target,
**kwargs
Expand Down
7 changes: 5 additions & 2 deletions src/rez/resolved_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -1832,13 +1832,16 @@ def _update_package_cache(self):
not self.success:
return

# see PackageCache.add_variants_async
# see PackageCache.add_variants
if not system.is_production_rez_install:
return

pkgcache = self._get_package_cache()
if pkgcache:
pkgcache.add_variants_async(self.resolved_packages)
pkgcache.add_variants(
self.resolved_packages,
config.package_cache_async
)

@classmethod
def _init_context_tracking_payload_base(cls):
Expand Down
3 changes: 3 additions & 0 deletions src/rez/rezconfig.py
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,9 @@
# Enable package caching during a package build.
package_cache_during_build = False

# Enable package caching to run asynchronously during a resolve.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I find this hard to read. How about "set to false to always cache package payloads synchronously"?

package_cache_async = True

# Allow caching of local packages. You would only want to set this True for
# testing purposes.
package_cache_local = False
Expand Down