diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d5cf2c4d28..de583720082 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -135,7 +135,7 @@ ___
Bugfix: houdini switching context doesnt update variables #5651 -Allows admins to have a list of vars (e.g. JOB) with (dynamic) values that will be updated on context changes, e.g. when switching to another asset or task.Using template keys is supported but formatting keys capitalization variants is not, e.g. {Asset} and {ASSET} won't workDisabling Update Houdini vars on context change feature will leave all Houdini vars unmanaged and thus no context update changes will occur.Also, this PR adds a new button in menu to update vars on demand. +Allows admins to have a list of vars (e.g. JOB) with (dynamic) values that will be updated on context changes, e.g. when switching to another asset or task.Using template keys is supported but formatting keys capitalization variants is not, e.g. {Asset} and {ASSET} won't workDisabling Update Houdini vars on context change feature will leave all Houdini vars unmanaged and thus no context update changes will occur.Also, this PR adds a new button in menu to update vars on demand. ___ @@ -976,7 +976,7 @@ ___
General: Create a desktop icon is checked #5636 -In OP Installer `Create a desktop icon` is checked by default. +In OP Installer `Create a desktop icon` is checked by default. ___
@@ -2278,7 +2278,7 @@ Linux / Centos ### Current Behavior: -the previous behavior (bug) : +the previous behavior (bug) : ![image](https://github.com/quadproduction/OpenPype/assets/135602303/09bff9d5-3f8b-4339-a1e5-30c04ade828c) @@ -2293,11 +2293,11 @@ Happened only once in a particular configuration ### Which project / workfile / asset / ... -open settings with 3.14.7 +open settings with 3.14.7 ### Steps To Reproduce: -1. Run openpype on the 3.15.11-nightly.3 version +1. Run openpype on the 3.15.11-nightly.3 version 2. Open settings in 3.14.7 version ### Relevant log output: @@ -2915,7 +2915,7 @@ ___
Nuke: returned not cleaning of renders folder on the farm #5374 -Previous PR enabled explicit cleanup of `renders` folder after farm publishing. This is not matching customer's workflows. Customer wants to have access to files in `renders` folder and potentially redo some frames for long frame sequences.This PR extends logic of marking rendered files for deletion only if instance doesn't have `stagingDir_persistent`.For backwards compatibility all Nuke instances have `stagingDir_persistent` set to True, eg. `renders` folder won't be cleaned after farm publish. +Previous PR enabled explicit cleanup of `renders` folder after farm publishing. This is not matching customer's workflows. Customer wants to have access to files in `renders` folder and potentially redo some frames for long frame sequences.This PR extends logic of marking rendered files for deletion only if instance doesn't have `stagingDirPersistence`.For backwards compatibility all Nuke instances have `stagingDirPersistence` set to True, eg. `renders` folder won't be cleaned after farm publish. ___ diff --git a/openpype/hosts/maya/plugins/publish/extract_pointcache.py b/openpype/hosts/maya/plugins/publish/extract_pointcache.py index 0cc802fa7aa..6d880e13cdd 100644 --- a/openpype/hosts/maya/plugins/publish/extract_pointcache.py +++ b/openpype/hosts/maya/plugins/publish/extract_pointcache.py @@ -107,7 +107,7 @@ def process(self, instance): } instance.data["representations"].append(representation) - if not instance.data.get("stagingDir_persistent", False): + if not instance.data.get("stagingDirPersistence", False): instance.context.data["cleanupFullPaths"].append(path) self.log.debug("Extracted {} to {}".format(instance, dirname)) diff --git a/openpype/hosts/maya/plugins/publish/extract_proxy_abc.py b/openpype/hosts/maya/plugins/publish/extract_proxy_abc.py index d9bec87cfd3..b565d312107 100644 --- a/openpype/hosts/maya/plugins/publish/extract_proxy_abc.py +++ b/openpype/hosts/maya/plugins/publish/extract_proxy_abc.py @@ -80,7 +80,7 @@ def process(self, instance): } instance.data["representations"].append(representation) - if not instance.data.get("stagingDir_persistent", False): + if not instance.data.get("stagingDirPersistence", False): instance.context.data["cleanupFullPaths"].append(path) self.log.debug("Extracted {} to {}".format(instance, dirname)) diff --git a/openpype/hosts/nuke/plugins/publish/collect_writes.py b/openpype/hosts/nuke/plugins/publish/collect_writes.py index 6f9245f5b96..eddc7efad4c 100644 --- a/openpype/hosts/nuke/plugins/publish/collect_writes.py +++ b/openpype/hosts/nuke/plugins/publish/collect_writes.py @@ -199,7 +199,7 @@ def _set_additional_instance_data( # compatibility. This is mainly focused on `renders`folders which # were previously not cleaned up (and could be used in read notes) # this logic should be removed and replaced with custom staging dir - instance.data["stagingDir_persistent"] = True + instance.data["stagingDirPersistence"] = True def _write_node_helper(self, instance): """Helper function to get write node from instance. @@ -266,6 +266,8 @@ def _get_existing_frames_representation( "name": ext, "ext": ext, "stagingDir": output_dir, + # TODO: do we need to add persistance to representation? + # "stagingDirPersistence": True, "tags": [] } diff --git a/openpype/hosts/photoshop/plugins/publish/extract_image.py b/openpype/hosts/photoshop/plugins/publish/extract_image.py index 680f580cc0e..d3d917c3d60 100644 --- a/openpype/hosts/photoshop/plugins/publish/extract_image.py +++ b/openpype/hosts/photoshop/plugins/publish/extract_image.py @@ -1,11 +1,10 @@ import os -import pyblish.api from openpype.pipeline import publish from openpype.hosts.photoshop import api as photoshop -class ExtractImage(pyblish.api.ContextPlugin): +class ExtractImage(publish.Extractor): """Extract all layers (groups) marked for publish. Usually publishable instance is created as a wrapper of layer(s). For each @@ -88,14 +87,3 @@ def process(self, context): for extracted_id in extract_ids: stub.set_visible(extracted_id, False) - - def staging_dir(self, instance): - """Provide a temporary directory in which to store extracted files - - Upon calling this method the staging directory is stored inside - the instance.data['stagingDir'] - """ - - from openpype.pipeline.publish import get_instance_staging_dir - - return get_instance_staging_dir(instance) diff --git a/openpype/hosts/substancepainter/plugins/publish/collect_textureset_images.py b/openpype/hosts/substancepainter/plugins/publish/collect_textureset_images.py index 316f72509ef..abe77917ac7 100644 --- a/openpype/hosts/substancepainter/plugins/publish/collect_textureset_images.py +++ b/openpype/hosts/substancepainter/plugins/publish/collect_textureset_images.py @@ -2,7 +2,7 @@ import copy import pyblish.api -from openpype.pipeline import publish +from openpype.pipeline.publish import get_instance_staging_dir import substance_painter.textureset from openpype.hosts.substancepainter.api.lib import ( @@ -165,7 +165,7 @@ def get_export_config(self, instance): # See: https://substance3d.adobe.com/documentation/ptpy/api/substance_painter/export # noqa config = { # noqa "exportShaderParams": True, - "exportPath": publish.get_instance_staging_dir(instance), + "exportPath": get_instance_staging_dir(instance), "defaultExportPreset": preset_url, # Custom overrides to the exporter diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 6ed5819f2bf..b0de86c9e58 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -439,8 +439,8 @@ def process(self, instance): # map inputVersions `ObjectId` -> `str` so json supports it "inputVersions": list(map(str, data.get("inputVersions", []))), "colorspace": instance.data.get("colorspace"), - "stagingDir_persistent": instance.data.get( - "stagingDir_persistent", False + "stagingDirPersistence": instance.data.get( + "stagingDirPersistence", False ) } diff --git a/openpype/pipeline/__init__.py b/openpype/pipeline/__init__.py index 8f370d389bf..04799d7a75f 100644 --- a/openpype/pipeline/__init__.py +++ b/openpype/pipeline/__init__.py @@ -9,6 +9,12 @@ ) from .anatomy import Anatomy +from .tempdir import get_temp_dir + +from .stagingdir import ( + get_staging_dir +) + from .create import ( BaseCreator, Creator, @@ -111,6 +117,12 @@ # --- Anatomy --- "Anatomy", + # --- Temp dir --- + "get_temp_dir", + + # --- Staging dir --- + "get_staging_dir", + # --- Create --- "BaseCreator", "Creator", diff --git a/openpype/pipeline/create/creator_plugins.py b/openpype/pipeline/create/creator_plugins.py index 6aa08cae705..649581f434d 100644 --- a/openpype/pipeline/create/creator_plugins.py +++ b/openpype/pipeline/create/creator_plugins.py @@ -1,3 +1,4 @@ +import os import copy import collections @@ -14,6 +15,7 @@ deregister_plugin, deregister_plugin_path ) +from openpype.pipeline import get_staging_dir from .constants import DEFAULT_VARIANT_VALUE from .subset_name import get_subset_name @@ -199,6 +201,7 @@ def __init__( # Reference to CreateContext self.create_context = create_context self.project_settings = project_settings + self.system_settings = system_settings # Creator is running in headless mode (without UI elemets) # - we may use UI inside processing this attribute should be checked @@ -723,6 +726,58 @@ def get_pre_create_attr_defs(self): return self.pre_create_attr_defs + def apply_staging_dir(self, instance): + """Apply staging dir with persistence to instance's transient data. + + Method is called on instance creation and on instance update. + + Args: + instance (CreatedInstance): Instance for which should be staging + dir applied. + + Returns: + str: Path to staging dir. + """ + create_ctx = self.create_context + asset_name = instance.get("asset") + subset = instance.get("subset") + if not asset_name or not subset: + return None + + version = instance.get("version") + if version is not None: + formatting_data = {"version": version} + + project_name = create_ctx.get_current_project_name() + host_name = create_ctx.host_name + task_name = instance.get("task") + + dir_data = get_staging_dir( + project_name, asset_name, host_name, + self.family, task_name, subset, self.project_anatomy, + project_settings=self.project_settings, + system_settings=self.system_settings, + always_return_path=False, + log=self.log, + formatting_data=formatting_data, + ) + + if not dir_data: + return None + + staging_dir_path = dir_data["stagingDir"] + + if not os.path.exists(staging_dir_path): + os.makedirs(staging_dir_path) + + instance.transient_data.update(dir_data) + + self.log.info( + "Applied staging dir to instance: {}".format(staging_dir_path) + ) + + return staging_dir_path + class HiddenCreator(BaseCreator): @abstractmethod def create(self, instance_data, source_data): diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index 7ef3439dbd1..3f88819a909 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -268,8 +268,8 @@ def create_skeleton_instance( representations = get_transferable_representations(instance) instance_skeleton_data["representations"] = representations - persistent = instance.data.get("stagingDir_persistent") is True - instance_skeleton_data["stagingDir_persistent"] = persistent + persistent = instance.data.get("stagingDirPersistence") is True + instance_skeleton_data["stagingDirPersistence"] = persistent return instance_skeleton_data diff --git a/openpype/pipeline/publish/lib.py b/openpype/pipeline/publish/lib.py index 4d9443f6351..d72e94a8ba8 100644 --- a/openpype/pipeline/publish/lib.py +++ b/openpype/pipeline/publish/lib.py @@ -2,7 +2,6 @@ import sys import inspect import copy -import tempfile import xml.etree.ElementTree import pyblish.util @@ -20,18 +19,16 @@ get_system_settings, ) from openpype.pipeline import ( - tempdir, - Anatomy + get_staging_dir ) + from openpype.pipeline.plugin_discover import DiscoverResult from .contants import ( DEFAULT_PUBLISH_TEMPLATE, - DEFAULT_HERO_PUBLISH_TEMPLATE, - TRANSIENT_DIR_TEMPLATE + DEFAULT_HERO_PUBLISH_TEMPLATE ) - def get_template_name_profiles( project_name, project_settings=None, logger=None ): @@ -668,56 +665,75 @@ def context_plugin_should_run(plugin, context): return False -def get_instance_staging_dir(instance): - """Unified way how staging dir is stored and created on instances. +# deprecated: backward compatibility only +# TODO: remove in the future +def get_custom_staging_dir_info(*args, **kwargs): + from openpype.pipeline.stagingdir import get_staging_dir_config - First check if 'stagingDir' is already set in instance data. - In case there already is new tempdir will not be created. + tr_data = get_staging_dir_config(*args, **kwargs) - It also supports `OPENPYPE_TMPDIR`, so studio can define own temp - shared repository per project or even per more granular context. - Template formatting is supported also with optional keys. Folder is - created in case it doesn't exists. + if not tr_data: + return None, None - Available anatomy formatting keys: - - root[work | ] - - project[name | code] + return tr_data["template"], tr_data["persistence"] - Note: - Staging dir does not have to be necessarily in tempdir so be careful - about its usage. - Args: - instance (pyblish.lib.Instance): Instance for which we want to get - staging dir. +def get_instance_staging_dir(instance): + """Unified way how staging dir is stored and created on instances. + + First check if 'stagingDir' is already set in instance data. + In case there already is new tempdir will not be created. Returns: - str: Path to staging dir of instance. + str: Path to staging dir """ staging_dir = instance.data.get('stagingDir') + if staging_dir: return staging_dir - anatomy = instance.context.data.get("anatomy") + anatomy_data = instance.data["anatomyData"] + formatting_data = copy.deepcopy(anatomy_data) + + # anatomy data based variables + family = anatomy_data["family"] + subset = anatomy_data["subset"] + asset_name = anatomy_data["asset"] + project_name = anatomy_data["project"]["name"] + task = anatomy_data.get("task", {}) + + # context data based variables + project_entity = instance.context.data["projectEntity"] + asset_entity = instance.context.data["assetEntity"] + host_name = instance.context.data["hostName"] + project_settings = instance.context.data["project_settings"] + system_settings = instance.context.data["system_settings"] + anatomy = instance.context.data["anatomy"] + current_file = instance.context.data.get("currentFile") + + # add current file as workfile name into formatting data + if current_file: + workfile = os.path.basename(current_file) + workfile_name, _ = os.path.splitext(workfile) + formatting_data["workfile_name"] = workfile_name + + dir_data = get_staging_dir( + project_name, asset_name, host_name, family, + task.get("name"), subset, anatomy, + project_doc=project_entity, asset_doc=asset_entity, + project_settings=project_settings, + system_settings=system_settings, + formatting_data=formatting_data + ) - # get customized tempdir path from `OPENPYPE_TMPDIR` env var - custom_temp_dir = tempdir.create_custom_tempdir( - anatomy.project_name, anatomy) + staging_dir_path = dir_data["stagingDir"] - if custom_temp_dir: - staging_dir = os.path.normpath( - tempfile.mkdtemp( - prefix="pyblish_tmp_", - dir=custom_temp_dir - ) - ) - else: - staging_dir = os.path.normpath( - tempfile.mkdtemp(prefix="pyblish_tmp_") - ) - instance.data['stagingDir'] = staging_dir + if not os.path.exists(staging_dir_path): + os.makedirs(staging_dir_path) - return staging_dir + instance.data.update(dir_data) + + return staging_dir_path def get_publish_repre_path(instance, repre, only_published=False): @@ -772,82 +788,6 @@ def get_publish_repre_path(instance, repre, only_published=False): return None -def get_custom_staging_dir_info(project_name, host_name, family, task_name, - task_type, subset_name, - project_settings=None, - anatomy=None, log=None): - """Checks profiles if context should use special custom dir as staging. - - Args: - project_name (str) - host_name (str) - family (str) - task_name (str) - task_type (str) - subset_name (str) - project_settings(Dict[str, Any]): Prepared project settings. - anatomy (Dict[str, Any]) - log (Logger) (optional) - - Returns: - (tuple) - Raises: - ValueError - if misconfigured template should be used - """ - settings = project_settings or get_project_settings(project_name) - custom_staging_dir_profiles = (settings["global"] - ["tools"] - ["publish"] - ["custom_staging_dir_profiles"]) - if not custom_staging_dir_profiles: - return None, None - - if not log: - log = Logger.get_logger("get_custom_staging_dir_info") - - filtering_criteria = { - "hosts": host_name, - "families": family, - "task_names": task_name, - "task_types": task_type, - "subsets": subset_name - } - profile = filter_profiles(custom_staging_dir_profiles, - filtering_criteria, - logger=log) - - if not profile or not profile["active"]: - return None, None - - if not anatomy: - anatomy = Anatomy(project_name) - - template_name = profile["template_name"] or TRANSIENT_DIR_TEMPLATE - _validate_transient_template(project_name, template_name, anatomy) - - custom_staging_dir = anatomy.templates[template_name]["folder"] - is_persistent = profile["custom_staging_dir_persistent"] - - return custom_staging_dir, is_persistent - - -def _validate_transient_template(project_name, template_name, anatomy): - """Check that transient template is correctly configured. - - Raises: - ValueError - if misconfigured template - """ - if template_name not in anatomy.templates: - raise ValueError(("Anatomy of project \"{}\" does not have set" - " \"{}\" template key!" - ).format(project_name, template_name)) - - if "folder" not in anatomy.templates[template_name]: - raise ValueError(("There is not set \"folder\" template in \"{}\" anatomy" # noqa - " for project \"{}\"." - ).format(template_name, project_name)) - - def get_published_workfile_instance(context): """Find workfile instance in context""" for i in context: @@ -965,12 +905,12 @@ def add_repre_files_for_cleanup(instance, repre): # first make sure representation level is not persistent if ( not staging_dir - or repre.get("stagingDir_persistent") + or repre.get("stagingDirPersistence") ): return # then look into instance level if it's not persistent - if instance.data.get("stagingDir_persistent"): + if instance.data.get("stagingDirPersistence"): return if isinstance(files, str): diff --git a/openpype/pipeline/publish/publish_plugins.py b/openpype/pipeline/publish/publish_plugins.py index ae6cbc42d13..b53e206c826 100644 --- a/openpype/pipeline/publish/publish_plugins.py +++ b/openpype/pipeline/publish/publish_plugins.py @@ -266,7 +266,7 @@ def process(self, context, plugin): plugin.repair(context) -class Extractor(pyblish.api.InstancePlugin): +class Extractor(pyblish.api.Plugin): """Extractor base class. The extractor base class implements a "staging_dir" function used to diff --git a/openpype/pipeline/stagingdir.py b/openpype/pipeline/stagingdir.py new file mode 100644 index 00000000000..995b6ddc422 --- /dev/null +++ b/openpype/pipeline/stagingdir.py @@ -0,0 +1,230 @@ +from openpype.lib import ( + Logger, + filter_profiles, + StringTemplate +) +from openpype.settings import get_project_settings +from .anatomy import Anatomy +from .tempdir import get_temp_dir +from openpype.pipeline.template_data import ( + get_template_data, + get_template_data_with_names +) + + +STAGING_DIR_TEMPLATES = "staging_directories" + + +def get_staging_dir_config( + project_name, host_name, family, task_name, + task_type, subset_name, + project_settings=None, + anatomy=None, log=None +): + """Get matching staging dir profile. + + Args: + project_name (str) + host_name (str) + family (str) + task_name (str) + task_type (str) + subset_name (str) + project_settings(Dict[str, Any]): Prepared project settings. + anatomy (Dict[str, Any]) + log (Optional[logging.Logger]) + + Returns: + Dict or None: Data with directory template and is_persistent or None + Raises: + ValueError - if misconfigured template should be used + """ + settings = project_settings or get_project_settings(project_name) + + staging_dir_profiles = ( + settings["global"]["tools"]["publish"]["custom_staging_dir_profiles"] + ) + + if not staging_dir_profiles: + return None + + if not log: + log = Logger.get_logger("get_staging_dir_config") + + filtering_criteria = { + "hosts": host_name, + "families": family, + "task_names": task_name, + "task_types": task_type, + "subsets": subset_name + } + profile = filter_profiles( + staging_dir_profiles, filtering_criteria, logger=log) + + if not profile or not profile["active"]: + return None + + if not anatomy: + anatomy = Anatomy(project_name) + + if not profile.get("template"): + template_name = profile["template_name"] + _validate_template_name(project_name, template_name, anatomy) + + template = ( + anatomy.templates[STAGING_DIR_TEMPLATES][template_name]) + else: + template = profile["template"] + + if not template: + # template should always be found either from anatomy or from profile + raise ValueError( + "Staging dir profile is misconfigured! " + "No template was found for profile! " + "Check your project settings at: " + "'project_settings/global/tools/publish/" + "custom_staging_dir_profiles'" + ) + + data_persistence = ( + # TODO: make this compulsory in the future + profile.get("data_persistence") + # maintain backwards compatibility + or profile.get("custom_staging_dir_persistent") + ) + + return { + "template": template, + "persistence": data_persistence + } + + +def _validate_template_name(project_name, template_name, anatomy): + """Check that staging dir section with appropriate template exist. + + Raises: + ValueError - if misconfigured template + """ + # TODO: only for backward compatibility of anatomy for older projects + if STAGING_DIR_TEMPLATES not in anatomy.templates: + raise ValueError(( + "Anatomy of project \"{}\" does not have set" + " \"{}\" template section!").format(project_name, template_name) + ) + + if template_name not in anatomy.templates[STAGING_DIR_TEMPLATES]: + raise ValueError(( + "Anatomy of project \"{}\" does not have set" + " \"{}\" template key at Staging Dir section!").format( + project_name, template_name) + ) + + +def get_staging_dir( + project_name, asset_name, host_name, + family, task_name, subset, anatomy, + project_doc=None, asset_doc=None, + project_settings=None, + system_settings=None, + **kwargs +): + """Get staging dir data. + + If `force_temp` is set, staging dir will be created as tempdir. + If `always_get_some_dir` is set, staging dir will be created as tempdir if + no staging dir profile is found. + If `prefix` or `suffix` is not set, default values will be used. + + Arguments: + project_name (str): Name of project. + asset_name (str): Name of asset. + host_name (str): Name of host. + family (str): Name of family. + task_name (str): Name of task. + subset (str): Name of subset. + anatomy (openpype.pipeline.Anatomy): Anatomy object. + project_doc (Optional[Dict[str, Any]]): Prepared project document. + asset_doc (Optional[Dict[str, Any]]): Prepared asset document. + project_settings (Optional[Dict[str, Any]]): Prepared project settings. + system_settings (Optional[Dict[str, Any]]): Prepared system settings. + **kwargs: Arbitrary keyword arguments. See below. + + Keyword Arguments: + force_temp (bool): If True, staging dir will be created as tempdir. + always_return_path (bool): If True, staging dir will be created as + tempdir if no staging dir profile is found. + prefix (str): Prefix for staging dir. + suffix (str): Suffix for staging dir. + formatting_data (Dict[str, Any]): Data for formatting staging dir + template. + + Returns: + Dict[str, Any]: Staging dir data + """ + + log = kwargs.get("log") or Logger.get_logger("get_staging_dir") + always_return_path = kwargs.get("always_return_path") + + # make sure always_return_path is set to true by default + if always_return_path is None: + always_return_path = True + + if kwargs.get("force_temp"): + return get_temp_dir( + project_name=project_name, + anatomy=anatomy, + prefix=kwargs.get("prefix"), + suffix=kwargs.get("suffix"), + ) + + # first try to get template data from documents then from names + if all([project_doc, asset_doc]): + # making fewer queries to database + ctx_data = get_template_data( + project_doc, asset_doc, task_name, host_name, system_settings + ) + else: + ctx_data = get_template_data_with_names( + project_name, asset_name, task_name, system_settings + ) + + # add additional data + ctx_data.update({ + "subset": subset, + "host": host_name, + "family": family + }) + ctx_data["root"] = anatomy.roots + + # add additional data from kwargs + if kwargs.get("formatting_data"): + ctx_data.update(kwargs.get("formatting_data")) + + # get staging dir config + staging_dir_config = get_staging_dir_config( + project_name, host_name, family, task_name, + ctx_data.get("task", {}).get("type"), subset, + project_settings=project_settings, + anatomy=anatomy, log=log + ) + + # if no preset matching and always_get_some_dir is set, return tempdir + if not staging_dir_config and always_return_path: + return { + "stagingDir": get_temp_dir( + project_name=project_name, + anatomy=anatomy, + prefix=kwargs.get("prefix"), + suffix=kwargs.get("suffix"), + ), + "stagingDirPersistence": False + } + elif not staging_dir_config: + return None + + return { + "stagingDir": StringTemplate.format_template( + staging_dir_config["template"], ctx_data + ), + "stagingDirPersistence": staging_dir_config["persistence"] + } diff --git a/openpype/pipeline/tempdir.py b/openpype/pipeline/tempdir.py index 55a1346b085..c97b0cacaf7 100644 --- a/openpype/pipeline/tempdir.py +++ b/openpype/pipeline/tempdir.py @@ -3,11 +3,83 @@ """ import os +import tempfile from openpype.lib import StringTemplate from openpype.pipeline import Anatomy -def create_custom_tempdir(project_name, anatomy=None): +def get_temp_dir( + project_name=None, + anatomy=None, + prefix=None, suffix=None, + make_local=False +): + """Get temporary dir path. + + If `make_local` is set, tempdir will be created in local tempdir. + If `anatomy` is not set, default anatomy will be used. + If `prefix` or `suffix` is not set, default values will be used. + + It also supports `OPENPYPE_TMPDIR`, so studio can define own temp + shared repository per project or even per more granular context. + Template formatting is supported also with optional keys. Folder is + created in case it doesn't exists. + + Available anatomy formatting keys: + - root[work | ] + - project[name | code] + + Note: + Staging dir does not have to be necessarily in tempdir so be careful + about its usage. + + Args: + project_name (str)[optional]: Name of project. + anatomy (openpype.pipeline.Anatomy)[optional]: Anatomy object. + make_local (bool)[optional]: If True, temp dir will be created in + local tempdir. + suffix (str)[optional]: Suffix for tempdir. + prefix (str)[optional]: Prefix for tempdir. + + Returns: + str: Path to staging dir of instance. + """ + prefix = prefix or "op_tmp_" + suffix = suffix or "" + + if make_local: + return _create_local_staging_dir(prefix, suffix) + + # make sure anatomy is set + if not anatomy: + anatomy = Anatomy(project_name) + + # get customized tempdir path from `OPENPYPE_TMPDIR` env var + custom_temp_dir = _create_custom_tempdir( + anatomy.project_name, anatomy) + + if custom_temp_dir: + return os.path.normpath( + tempfile.mkdtemp( + prefix=prefix, + suffix=suffix, + dir=custom_temp_dir + ) + ) + else: + return _create_local_staging_dir(prefix, suffix) + + +def _create_local_staging_dir(prefix, suffix): + return os.path.normpath( + tempfile.mkdtemp( + prefix=prefix, + suffix=suffix + ) + ) + + +def _create_custom_tempdir(project_name, anatomy=None): """ Create custom tempdir Template path formatting is supporting: @@ -32,7 +104,7 @@ def create_custom_tempdir(project_name, anatomy=None): if anatomy is None: anatomy = Anatomy(project_name) # create base formate data - data = { + formatting_data = { "root": anatomy.roots, "project": { "name": anatomy.project_name, @@ -41,7 +113,7 @@ def create_custom_tempdir(project_name, anatomy=None): } # path is anatomy template custom_tempdir = StringTemplate.format_template( - openpype_tempdir, data).normalized() + openpype_tempdir, formatting_data).normalized() else: # path is absolute diff --git a/openpype/plugins/publish/cleanup.py b/openpype/plugins/publish/cleanup.py index 6c122ddf096..f99bbc5ae3c 100644 --- a/openpype/plugins/publish/cleanup.py +++ b/openpype/plugins/publish/cleanup.py @@ -94,7 +94,7 @@ def process(self, instance): self.log.debug("No staging directory found at: %s" % staging_dir) return - if instance.data.get("stagingDir_persistent"): + if instance.data.get("stagingDirPersistence"): self.log.debug( "Staging dir {} should be persistent".format(staging_dir) ) diff --git a/openpype/plugins/publish/cleanup_farm.py b/openpype/plugins/publish/cleanup_farm.py index e655437ced7..4622e5a60ce 100644 --- a/openpype/plugins/publish/cleanup_farm.py +++ b/openpype/plugins/publish/cleanup_farm.py @@ -37,7 +37,7 @@ def process(self, context): dirpaths_to_remove = set() for instance in context: staging_dir = instance.data.get("stagingDir") - if staging_dir and not instance.data.get("stagingDir_persistent"): + if staging_dir and not instance.data.get("stagingDirPersistence"): dirpaths_to_remove.add(os.path.normpath(staging_dir)) if "representations" in instance.data: diff --git a/openpype/plugins/publish/collect_custom_staging_dir.py b/openpype/plugins/publish/collect_custom_staging_dir.py deleted file mode 100644 index 669c4873e04..00000000000 --- a/openpype/plugins/publish/collect_custom_staging_dir.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -Requires: - anatomy - - -Provides: - instance.data -> stagingDir (folder path) - -> stagingDir_persistent (bool) -""" -import copy -import os.path - -import pyblish.api - -from openpype.pipeline.publish.lib import get_custom_staging_dir_info - - -class CollectCustomStagingDir(pyblish.api.InstancePlugin): - """Looks through profiles if stagingDir should be persistent and in special - location. - - Transient staging dir could be useful in specific use cases where is - desirable to have temporary renders in specific, persistent folders, could - be on disks optimized for speed for example. - - It is studio responsibility to clean up obsolete folders with data. - - Location of the folder is configured in `project_anatomy/templates/others`. - ('transient' key is expected, with 'folder' key) - - Which family/task type/subset is applicable is configured in: - `project_settings/global/tools/publish/custom_staging_dir_profiles` - - """ - label = "Collect Custom Staging Directory" - order = pyblish.api.CollectorOrder + 0.4990 - - template_key = "transient" - - def process(self, instance): - family = instance.data["family"] - subset_name = instance.data["subset"] - host_name = instance.context.data["hostName"] - project_name = instance.context.data["projectName"] - project_settings = instance.context.data["project_settings"] - anatomy = instance.context.data["anatomy"] - task = instance.data["anatomyData"].get("task", {}) - - transient_tml, is_persistent = get_custom_staging_dir_info( - project_name, host_name, family, task.get("name"), - task.get("type"), subset_name, project_settings=project_settings, - anatomy=anatomy, log=self.log) - - if transient_tml: - anatomy_data = copy.deepcopy(instance.data["anatomyData"]) - anatomy_data["root"] = anatomy.roots - scene_name = instance.context.data.get("currentFile") - if scene_name: - anatomy_data["scene_name"] = os.path.basename(scene_name) - transient_dir = transient_tml.format(**anatomy_data) - instance.data["stagingDir"] = transient_dir - - instance.data["stagingDir_persistent"] = is_persistent - result_str = "Adding '{}' as".format(transient_dir) - else: - result_str = "Not adding" - - self.log.debug("{} custom staging dir for instance with '{}'".format( - result_str, family - )) diff --git a/openpype/plugins/publish/collect_managed_staging_dir.py b/openpype/plugins/publish/collect_managed_staging_dir.py new file mode 100644 index 00000000000..32fcbd966c7 --- /dev/null +++ b/openpype/plugins/publish/collect_managed_staging_dir.py @@ -0,0 +1,45 @@ +""" +Requires: + anatomy + + +Provides: + instance.data -> stagingDir (folder path) + -> stagingDirPersistence (bool) +""" +import pyblish.api + +from openpype.pipeline.publish import get_instance_staging_dir + + +class CollectManagedStagingDir(pyblish.api.InstancePlugin): + """Apply matching Staging Dir profile to a instance. + + Apply Staging dir via profiles could be useful in specific use cases + where is desirable to have temporary renders in specific, + persistent folders, could be on disks optimized for speed for example. + + It is studio's responsibility to clean up obsolete folders with data. + + Location of the folder is configured in: + `project_anatomy/templates/staging_dir`. + + Which family/task type/subset is applicable is configured in: + `project_settings/global/tools/publish/custom_staging_dir_profiles` + """ + label = "Collect Managed Staging Directory" + order = pyblish.api.CollectorOrder + 0.4990 + + def process(self, instance): + + staging_dir_path = get_instance_staging_dir(instance) + + self.log.info( + ( + "Instance staging dir was set to `{}` " + "and persistence is set to `{}`" + ).format( + staging_dir_path, + instance.data.get("stagingDirPersistence") + ) + ) diff --git a/openpype/plugins/publish/collect_rendered_files.py b/openpype/plugins/publish/collect_rendered_files.py index 8a5a5a83f14..d901f562b21 100644 --- a/openpype/plugins/publish/collect_rendered_files.py +++ b/openpype/plugins/publish/collect_rendered_files.py @@ -104,7 +104,7 @@ def _process_path(self, data, anatomy): # stash render job id for later validation instance.data["render_job_id"] = data.get("job").get("_id") staging_dir_persistent = instance.data.get( - "stagingDir_persistent", False + "stagingDirPersistence", False ) representations = [] for repre_data in instance_data.get("representations") or []: diff --git a/openpype/plugins/publish/extract_burnin.py b/openpype/plugins/publish/extract_burnin.py index dc8aab6ce4a..0dd5e21dfba 100644 --- a/openpype/plugins/publish/extract_burnin.py +++ b/openpype/plugins/publish/extract_burnin.py @@ -10,11 +10,13 @@ import pyblish.api from openpype import resources, PACKAGE_DIR -from openpype.pipeline import publish +from openpype.pipeline import ( + publish, + get_temp_dir +) from openpype.lib import ( run_openpype_process, - get_transcode_temp_directory, convert_input_paths_for_ffmpeg, should_convert_for_ffmpeg ) @@ -233,7 +235,11 @@ def main_process(self, instance): # - change staging dir of source representation # - must be set back after output definitions processing if do_convert: - new_staging_dir = get_transcode_temp_directory() + new_staging_dir = get_temp_dir( + project_name=instance.context.data["projectName"], + make_local=True, + prefix="op_transcoding_" + ) repre["stagingDir"] = new_staging_dir convert_input_paths_for_ffmpeg( diff --git a/openpype/plugins/publish/extract_color_transcode.py b/openpype/plugins/publish/extract_color_transcode.py index dbf1b6c8a6b..1990d21fd9c 100644 --- a/openpype/plugins/publish/extract_color_transcode.py +++ b/openpype/plugins/publish/extract_color_transcode.py @@ -3,15 +3,16 @@ import clique import pyblish.api -from openpype.pipeline import publish +from openpype.pipeline import ( + publish, + get_temp_dir +) from openpype.lib import ( - is_oiio_supported, ) from openpype.lib.transcoding import ( - convert_colorspace, - get_transcode_temp_directory, + convert_colorspace ) from openpype.lib.profiles_filtering import filter_profiles @@ -51,7 +52,7 @@ class ExtractOIIOTranscode(publish.Extractor): empty if transcoding should be only into display and viewer colorspace. (In that case both 'display' and 'view' must be filled.) """ - + # TODO: this is not only for color transcoding label = "Transcode color spaces" order = pyblish.api.ExtractorOrder + 0.019 @@ -102,7 +103,11 @@ def process(self, instance): new_repre = copy.deepcopy(repre) original_staging_dir = new_repre["stagingDir"] - new_staging_dir = get_transcode_temp_directory() + new_staging_dir = get_temp_dir( + project_name=instance.context.data["projectName"], + make_local=True, + prefix="op_transcoding_" + ) new_repre["stagingDir"] = new_staging_dir if isinstance(new_repre["files"], list): @@ -271,7 +276,7 @@ def _translate_to_sequence(self, files_to_convert): (list) of [file.1001-1010#.exr] or [fileA.exr, fileB.exr] """ pattern = [clique.PATTERNS["frames"]] - collections, remainder = clique.assemble( + collections, _ = clique.assemble( files_to_convert, patterns=pattern, assume_padded_when_ambiguous=True) diff --git a/openpype/plugins/publish/extract_review.py b/openpype/plugins/publish/extract_review.py index 0ae941511c8..c928a68c9d8 100644 --- a/openpype/plugins/publish/extract_review.py +++ b/openpype/plugins/publish/extract_review.py @@ -23,8 +23,8 @@ should_convert_for_ffmpeg, get_review_layer_name, convert_input_paths_for_ffmpeg, - get_transcode_temp_directory, ) +from openpype.pipeline import get_temp_dir from openpype.pipeline.publish import ( KnownPublishError, get_publish_instance_label, @@ -273,7 +273,11 @@ def main_process(self, instance): # - change staging dir of source representation # - must be set back after output definitions processing if do_convert: - new_staging_dir = get_transcode_temp_directory() + new_staging_dir = get_temp_dir( + project_name=instance.context.data["projectName"], + make_local=True, + prefix="op_transcoding_" + ) repre["stagingDir"] = new_staging_dir convert_input_paths_for_ffmpeg( diff --git a/openpype/settings/defaults/project_anatomy/templates.json b/openpype/settings/defaults/project_anatomy/templates.json index e5e535bf193..4169c556dfc 100644 --- a/openpype/settings/defaults/project_anatomy/templates.json +++ b/openpype/settings/defaults/project_anatomy/templates.json @@ -27,6 +27,7 @@ "path": "{@folder}/{@file}" }, "delivery": {}, + "staging_directories": {}, "unreal": { "folder": "{root[work]}/{project[name]}/unreal/{task[name]}", "file": "{project[code]}_{asset}.{ext}", @@ -58,9 +59,6 @@ "file": "{originalBasename}.{ext}", "path": "{@folder}/{@file}" }, - "transient": { - "folder": "{root[work]}/{project[name]}/{hierarchy}/{asset}/work/{family}/{subset}" - }, "__dynamic_keys_labels__": { "maya2unreal": "Maya to Unreal", "simpleUnrealTextureHero": "Simple Unreal Texture - Hero", diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_anatomy_templates.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_anatomy_templates.json index 0548824ee14..d7bfa846c0c 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_anatomy_templates.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_anatomy_templates.json @@ -143,6 +143,12 @@ "label": "Delivery", "object_type": "text" }, + { + "type": "dict-modifiable", + "key": "staging_directories", + "label": "Staging directories", + "object_type": "text" + }, { "type": "dict", "key": "unreal", diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_tools.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_tools.json index 23fc7c9351b..1275b0dc996 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_tools.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_tools.json @@ -408,7 +408,7 @@ { "type": "list", "key": "custom_staging_dir_profiles", - "label": "Custom Staging Dir Profiles", + "label": "Staging Dir Profiles", "use_label_wrap": true, "docstring": "Profiles to specify special location and persistence for staging dir. Could be used in Creators and Publish phase!", "object_type": { @@ -456,16 +456,26 @@ "type": "separator" }, { - "key": "custom_staging_dir_persistent", - "label": "Custom Staging Folder Persistent", + "key": "data_persistence", + "label": "Folder is persistent", "type": "boolean", "default": false }, + { + "key": "template", + "label": "Template", + "type": "text", + "placeholder": "{root[work]}/{staging_dir}/{asset}/{subset}/{version}" + }, + { + "type": "label", + "label": "Template name: usually used only for templates which needs to be shared
between multiple profiles. Set name of previously defined template within
this section project_anatomy/templates/staging_dir." + }, { "key": "template_name", "label": "Template Name", "type": "text", - "placeholder": "transient" + "placeholder": "Shared template: use project_anatomy/templates/staging_dir" } ] } diff --git a/server_addon/core/server/settings/tools.py b/server_addon/core/server/settings/tools.py index 7befc795e41..95a9dcaddaf 100644 --- a/server_addon/core/server/settings/tools.py +++ b/server_addon/core/server/settings/tools.py @@ -229,10 +229,19 @@ class CustomStagingDirProfileModel(BaseSettingsModel): product_names: list[str] = Field( default_factory=list, title="Product names" ) - custom_staging_dir_persistent: bool = Field( - False, title="Custom Staging Folder Persistent" + data_persistence: bool = Field( + False, title="Folder is persistent" + ) + template: str = Field( + "", + title="Template", + placeholder="{root[work]}/{staging_dir}/{asset}/{subset}/{version}" + ) + template_name: str = Field( + "", + title="Template Name", + placeholder="Shared templates: ayon+anatomy://templates/staging_directories" ) - template_name: str = Field("", title="Template Name") class PublishToolModel(BaseSettingsModel): @@ -246,7 +255,7 @@ class PublishToolModel(BaseSettingsModel): ) custom_staging_dir_profiles: list[CustomStagingDirProfileModel] = Field( default_factory=list, - title="Custom Staging Dir Profiles" + title="Staging Dir Profiles" ) diff --git a/server_addon/core/server/version.py b/server_addon/core/server/version.py index b3f4756216d..ae7362549b3 100644 --- a/server_addon/core/server/version.py +++ b/server_addon/core/server/version.py @@ -1 +1 @@ -__version__ = "0.1.2" +__version__ = "0.1.3"