General: custom staging dir functionality migration

CHANGELOG.md

@@ -135,7 +135,7 @@ ___
 <details>
 <summary>Bugfix: houdini switching context doesnt update variables <a href="https://github.com/ynput/OpenPype/pull/5651">#5651</a></summary>
 
-Allows admins to have a list of vars (e.g. JOB) with (dynamic) values that will be updated on context changes, e.g. when switching to another asset or task.Using template keys is supported but formatting keys capitalization variants is not, e.g. {Asset} and {ASSET} won't workDisabling Update Houdini vars on context change feature will leave all Houdini vars unmanaged and thus no context update changes will occur.Also, this PR adds a new button in menu to update vars on demand. 
+Allows admins to have a list of vars (e.g. JOB) with (dynamic) values that will be updated on context changes, e.g. when switching to another asset or task.Using template keys is supported but formatting keys capitalization variants is not, e.g. {Asset} and {ASSET} won't workDisabling Update Houdini vars on context change feature will leave all Houdini vars unmanaged and thus no context update changes will occur.Also, this PR adds a new button in menu to update vars on demand.
 
 
 ___
@@ -976,7 +976,7 @@ ___
 <details>
 <summary>General: Create a desktop icon is checked <a href="https://github.com/ynput/OpenPype/pull/5636">#5636</a></summary>
 
-In OP Installer `Create a desktop icon` is checked by default. 
+In OP Installer `Create a desktop icon` is checked by default.
 ___
 
 </details>
@@ -2278,7 +2278,7 @@ Linux / Centos
 
 ### Current Behavior:
 
-the previous behavior (bug) : 
+the previous behavior (bug) :
 ![image](https://github.com/quadproduction/OpenPype/assets/135602303/09bff9d5-3f8b-4339-a1e5-30c04ade828c)
 
 
@@ -2293,11 +2293,11 @@ Happened only once in a particular configuration
 
 ### Which project / workfile / asset / ...
 
-open settings with 3.14.7 
+open settings with 3.14.7
 
 ### Steps To Reproduce:
 
-1. Run openpype on the 3.15.11-nightly.3 version 
+1. Run openpype on the 3.15.11-nightly.3 version
 2. Open settings in 3.14.7 version
 
 ### Relevant log output:
@@ -2915,7 +2915,7 @@ ___
 <details>
 <summary>Nuke: returned not cleaning of renders folder on the farm <a href="https://github.com/ynput/OpenPype/pull/5374">#5374</a></summary>
 
-Previous PR enabled explicit cleanup of `renders` folder after farm publishing. This is not matching customer's workflows. Customer wants to have access to files in `renders` folder and potentially redo some frames for long frame sequences.This PR extends logic of marking rendered files for deletion only if instance doesn't have `stagingDir_persistent`.For backwards compatibility all Nuke instances have `stagingDir_persistent` set to True, eg. `renders` folder won't be cleaned after farm publish.
+Previous PR enabled explicit cleanup of `renders` folder after farm publishing. This is not matching customer's workflows. Customer wants to have access to files in `renders` folder and potentially redo some frames for long frame sequences.This PR extends logic of marking rendered files for deletion only if instance doesn't have `stagingDirPersistence`.For backwards compatibility all Nuke instances have `stagingDirPersistence` set to True, eg. `renders` folder won't be cleaned after farm publish.
 
 
 ___

openpype/hosts/maya/plugins/publish/extract_pointcache.py

@@ -107,7 +107,7 @@ def process(self, instance):
         }
         instance.data["representations"].append(representation)
 
-        if not instance.data.get("stagingDir_persistent", False):
+        if not instance.data.get("stagingDirPersistence", False):
             instance.context.data["cleanupFullPaths"].append(path)
 
         self.log.debug("Extracted {} to {}".format(instance, dirname))

openpype/hosts/maya/plugins/publish/extract_proxy_abc.py

@@ -80,7 +80,7 @@ def process(self, instance):
         }
         instance.data["representations"].append(representation)
 
-        if not instance.data.get("stagingDir_persistent", False):
+        if not instance.data.get("stagingDirPersistence", False):
             instance.context.data["cleanupFullPaths"].append(path)
 
         self.log.debug("Extracted {} to {}".format(instance, dirname))

openpype/hosts/nuke/plugins/publish/collect_writes.py

@@ -199,7 +199,7 @@ def _set_additional_instance_data(
         # compatibility. This is mainly focused on `renders`folders which
         # were previously not cleaned up (and could be used in read notes)
         # this logic should be removed and replaced with custom staging dir
-        instance.data["stagingDir_persistent"] = True
+        instance.data["stagingDirPersistence"] = True
 
     def _write_node_helper(self, instance):
         """Helper function to get write node from instance.
@@ -266,6 +266,8 @@ def _get_existing_frames_representation(
             "name": ext,
             "ext": ext,
             "stagingDir": output_dir,
+            # TODO: do we need to add persistance to representation?
+            # "stagingDirPersistence": True,
             "tags": []
         }
 

openpype/hosts/photoshop/plugins/publish/extract_image.py

@@ -1,11 +1,10 @@
 import os
 
-import pyblish.api
 from openpype.pipeline import publish
 from openpype.hosts.photoshop import api as photoshop
 
 
-class ExtractImage(pyblish.api.ContextPlugin):
+class ExtractImage(publish.Extractor):
     """Extract all layers (groups) marked for publish.
 
     Usually publishable instance is created as a wrapper of layer(s). For each
@@ -88,14 +87,3 @@ def process(self, context):
 
                     for extracted_id in extract_ids:
                         stub.set_visible(extracted_id, False)
-
-    def staging_dir(self, instance):
-        """Provide a temporary directory in which to store extracted files
-
-        Upon calling this method the staging directory is stored inside
-        the instance.data['stagingDir']
-        """
-
-        from openpype.pipeline.publish import get_instance_staging_dir
-
-        return get_instance_staging_dir(instance)

openpype/hosts/substancepainter/plugins/publish/collect_textureset_images.py

@@ -2,7 +2,7 @@
 import copy
 import pyblish.api
 
-from openpype.pipeline import publish
+from openpype.pipeline.publish import get_instance_staging_dir
 
 import substance_painter.textureset
 from openpype.hosts.substancepainter.api.lib import (
@@ -165,7 +165,7 @@ def get_export_config(self, instance):
         # See: https://substance3d.adobe.com/documentation/ptpy/api/substance_painter/export  # noqa
         config = {  # noqa
             "exportShaderParams": True,
-            "exportPath": publish.get_instance_staging_dir(instance),
+            "exportPath": get_instance_staging_dir(instance),
             "defaultExportPreset": preset_url,
 
             # Custom overrides to the exporter

openpype/modules/deadline/plugins/publish/submit_publish_job.py

@@ -439,8 +439,8 @@ def process(self, instance):
             # map inputVersions `ObjectId` -> `str` so json supports it
             "inputVersions": list(map(str, data.get("inputVersions", []))),
             "colorspace": instance.data.get("colorspace"),
-            "stagingDir_persistent": instance.data.get(
-                "stagingDir_persistent", False
+            "stagingDirPersistence": instance.data.get(
+                "stagingDirPersistence", False
             )
         }
 

openpype/pipeline/__init__.py

@@ -9,6 +9,12 @@
 )
 from .anatomy import Anatomy
 
+from .tempdir import get_temp_dir
+
+from .stagingdir import (
+    get_staging_dir
+)
+
 from .create import (
     BaseCreator,
     Creator,
@@ -111,6 +117,12 @@
     # --- Anatomy ---
     "Anatomy",
 
+    # --- Temp dir ---
+    "get_temp_dir",
+
+    # --- Staging dir ---
+    "get_staging_dir",
+
     # --- Create ---
     "BaseCreator",
     "Creator",

openpype/pipeline/create/creator_plugins.py

@@ -14,6 +14,7 @@
     deregister_plugin,
     deregister_plugin_path
 )
+from openpype.pipeline import get_staging_dir
 
 from .constants import DEFAULT_VARIANT_VALUE
 from .subset_name import get_subset_name
@@ -199,6 +200,7 @@ def __init__(
         # Reference to CreateContext
         self.create_context = create_context
         self.project_settings = project_settings
+        self.system_settings = system_settings
 
         # Creator is running in headless mode (without UI elemets)
         # - we may use UI inside processing this attribute should be checked
@@ -723,6 +725,58 @@ def get_pre_create_attr_defs(self):
         return self.pre_create_attr_defs
 
 
+    def apply_staging_dir(self, instance):
+        """Apply staging dir with persistence to instance's transient data.
+
+        Method is called on instance creation and on instance update.
+
+        Args:
+            instance (CreatedInstance): Instance for which should be staging
+                dir applied.
+
+        Returns:
+            str: Path to staging dir.
+        """
+        create_ctx = self.create_context
+        asset_name = instance.get("asset")
+        subset = instance.get("subset")
+        if not asset_name or not subset:
+            return None
+
+        version = instance.get("version")
+        if version is not None:
+            formatting_data = {"version": version}
+
+        project_name = create_ctx.get_current_project_name()
+        host_name = create_ctx.host_name
+        task_name = instance.get("task")
+
+        dir_data = get_staging_dir(
+            project_name, asset_name, host_name,
+            self.family, task_name, subset, self.project_anatomy,
+            project_settings=self.project_settings,
+            system_settings=self.system_settings,
+            always_return_path=False,
+            log=self.log,
+            formatting_data=formatting_data,
+        )
+
+        if not dir_data:
+            return None
+
+        staging_dir_path = dir_data["stagingDir"]
+
+        if not os.path.exists(staging_dir_path):
+            os.makedirs(staging_dir_path)
+
+        instance.transient_data.update(dir_data)
+
+        self.log.info(
+            "Applied staging dir to instance: {}".format(staging_dir_path)
+        )
+
+        return staging_dir_path
+
 class HiddenCreator(BaseCreator):
     @abstractmethod
     def create(self, instance_data, source_data):

openpype/pipeline/farm/pyblish_functions.py

@@ -268,8 +268,8 @@ def create_skeleton_instance(
     representations = get_transferable_representations(instance)
     instance_skeleton_data["representations"] = representations
 
-    persistent = instance.data.get("stagingDir_persistent") is True
-    instance_skeleton_data["stagingDir_persistent"] = persistent
+    persistent = instance.data.get("stagingDirPersistence") is True
+    instance_skeleton_data["stagingDirPersistence"] = persistent
 
     return instance_skeleton_data