From 450c0270fa335cdf09e6364912b4dcd4c540c574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Wed, 26 Oct 2022 18:37:44 +0200 Subject: [PATCH 001/104] :construction: wip on maya royalrender submit plugin --- openpype/modules/royalrender/api.py | 20 ++- .../publish/submit_maya_royalrender.py | 150 ++++++++++++++++++ openpype/modules/royalrender/rr_job.py | 8 +- 3 files changed, 163 insertions(+), 15 deletions(-) create mode 100644 openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py diff --git a/openpype/modules/royalrender/api.py b/openpype/modules/royalrender/api.py index de1dba8724b..c47d50b62b5 100644 --- a/openpype/modules/royalrender/api.py +++ b/openpype/modules/royalrender/api.py @@ -15,9 +15,8 @@ class Api: RR_SUBMIT_CONSOLE = 1 RR_SUBMIT_API = 2 - def __init__(self, settings, project=None): + def __init__(self, project=None): self.log = Logger.get_logger("RoyalRender") - self._settings = settings self._initialize_rr(project) def _initialize_rr(self, project=None): @@ -91,21 +90,21 @@ def _initialize_module_path(self): sys.path.append(os.path.join(self._rr_path, rr_module_path)) - def create_submission(self, jobs, submitter_attributes, file_name=None): - # type: (list[RRJob], list[SubmitterParameter], str) -> SubmitFile + @staticmethod + def create_submission(jobs, submitter_attributes): + # type: (list[RRJob], list[SubmitterParameter]) -> SubmitFile """Create jobs submission file. Args: jobs (list): List of :class:`RRJob` submitter_attributes (list): List of submitter attributes :class:`SubmitterParameter` for whole submission batch. - file_name (str), optional): File path to write data to. Returns: str: XML data of job submission files. """ - raise NotImplementedError + return SubmitFile(SubmitterParameters=submitter_attributes, Jobs=jobs) def submit_file(self, file, mode=RR_SUBMIT_CONSOLE): # type: (SubmitFile, int) -> None @@ -119,15 +118,14 @@ def submit_file(self, file, mode=RR_SUBMIT_CONSOLE): # self._submit_using_api(file) def _submit_using_console(self, file): - # type: (SubmitFile) -> bool + # type: (SubmitFile) -> None rr_console = os.path.join( self._get_rr_bin_path(), - "rrSubmitterconsole" + "rrSubmitterConsole" ) - if sys.platform.lower() == "darwin": - if "/bin/mac64" in rr_console: - rr_console = rr_console.replace("/bin/mac64", "/bin/mac") + if sys.platform.lower() == "darwin" and "/bin/mac64" in rr_console: + rr_console = rr_console.replace("/bin/mac64", "/bin/mac") if sys.platform.lower() == "win32": if "/bin/win64" in rr_console: diff --git a/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py new file mode 100644 index 00000000000..c354cc80a0c --- /dev/null +++ b/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py @@ -0,0 +1,150 @@ +# -*- coding: utf-8 -*- +"""Submitting render job to RoyalRender.""" +import os +import sys +import tempfile + +from maya.OpenMaya import MGlobal # noqa +from pyblish.api import InstancePlugin, IntegratorOrder +from openpype.hosts.maya.api.lib import get_attr_in_layer +from openpype.pipeline.farm.tools import get_published_workfile_instance +from openpype.pipeline.publish import KnownPublishError +from openpype.modules.royalrender.api import Api as rr_api +from openpype.modules.royalrender.rr_job import RRJob, SubmitterParameter + + +class MayaSubmitRoyalRender(InstancePlugin): + label = "Submit to RoyalRender" + order = IntegratorOrder + 0.1 + use_published = True + + def __init__(self, *args, **kwargs): + self._instance = None + self._rrRoot = None + self.scene_path = None + self.job = None + self.submission_parameters = None + self.rr_api = None + + def get_job(self): + """Prepare job payload. + + Returns: + RRJob: RoyalRender job payload. + + """ + def get_rr_platform(): + if sys.platform.lower() in ["win32", "win64"]: + return "win" + elif sys.platform.lower() == "darwin": + return "mac" + else: + return "lx" + + expected_files = self._instance.data["expectedFiles"] + first_file = next(self._iter_expected_files(expected_files)) + output_dir = os.path.dirname(first_file) + self._instance.data["outputDir"] = output_dir + workspace = self._instance.context.data["workspaceDir"] + default_render_file = self._instance.context.data.get('project_settings') \ + .get('maya') \ + .get('RenderSettings') \ + .get('default_render_image_folder') + filename = os.path.basename(self.scene_path) + dirname = os.path.join(workspace, default_render_file) + + job = RRJob( + Software="Maya", + Renderer=self._instance.data["renderer"], + SeqStart=int(self._instance.data["frameStartHandle"]), + SeqEnd=int(self._instance.data["frameEndHandle"]), + SeqStep=int(self._instance.data["byFrameStep"]), + SeqFileOffset=0, + Version="{0:.2f}".format(MGlobal.apiVersion() / 10000), + SceneName=os.path.basename(self.scene_path), + IsActive=True, + ImageDir=dirname, + ImageFilename=filename, + ImageExtension="." + os.path.splitext(filename)[1], + ImagePreNumberLetter=".", + ImageSingleOutputFile=False, + SceneOS=get_rr_platform(), + Camera=self._instance.data["cameras"][0], + Layer=self._instance.data["layer"], + SceneDatabaseDir=workspace, + ImageFramePadding=get_attr_in_layer( + "defaultRenderGlobals.extensionPadding", + self._instance.data["layer"]), + ImageWidth=self._instance.data["resolutionWidth"], + ImageHeight=self._instance.data["resolutionHeight"] + ) + return job + + @staticmethod + def get_submission_parameters(): + return [] + + def create_file(self, name, ext, contents=None): + temp = tempfile.NamedTemporaryFile( + dir=self.tempdir, + suffix=ext, + prefix=name + '.', + delete=False, + ) + + if contents: + with open(temp.name, 'w') as f: + f.write(contents) + + return temp.name + + def process(self, instance): + """Plugin entry point.""" + self._instance = instance + context = instance.context + self.rr_api = rr_api(context.data["project"]) + + # get royalrender module + """ + try: + rr_module = context.data.get( + "openPypeModules")["royalrender"] + except AttributeError: + self.log.error("Cannot get OpenPype RoyalRender module.") + raise AssertionError("OpenPype RoyalRender module not found.") + """ + + self._rrRoot = instance.data["rrPath"] or context.data["defaultRRPath"] # noqa + if not self._rrRoot: + raise KnownPublishError( + ("Missing RoyalRender root. " + "You need to configure RoyalRender module.")) + file_path = None + if self.use_published: + file_path = get_published_workfile_instance() + + # fallback if nothing was set + if not file_path: + self.log.warning("Falling back to workfile") + file_path = context.data["currentFile"] + + self.scene_path = file_path + self.job = self.get_job() + self.log.info(self.job) + self.submission_parameters = self.get_submission_parameters() + + self.process_submission() + + def process_submission(self): + submission = rr_api.create_submission( + [self.job], + self.submission_parameters) + + self.log.debug(submission) + xml = tempfile.NamedTemporaryFile(suffix=".xml", delete=False) + with open(xml.name, "w") as f: + f.write(submission.serialize()) + + self.rr_api.submit_file(file=xml) + + diff --git a/openpype/modules/royalrender/rr_job.py b/openpype/modules/royalrender/rr_job.py index c660eceac76..beb8c17187e 100644 --- a/openpype/modules/royalrender/rr_job.py +++ b/openpype/modules/royalrender/rr_job.py @@ -35,7 +35,7 @@ class RRJob: # Is the job enabled for submission? # enabled by default - IsActive = attr.ib() # type: str + IsActive = attr.ib() # type: bool # Sequence settings of this job SeqStart = attr.ib() # type: int @@ -60,7 +60,7 @@ class RRJob: # If you render a single file, e.g. Quicktime or Avi, then you have to # set this value. Videos have to be rendered at once on one client. - ImageSingleOutputFile = attr.ib(default="false") # type: str + ImageSingleOutputFile = attr.ib(default=False) # type: bool # Semi-Required (required for some render applications) # ----------------------------------------------------- @@ -169,11 +169,11 @@ class SubmitFile: # Delete submission file after processing DeleteXML = attr.ib(default=1) # type: int - # List of submitter options per job + # List of the submitter options per job. # list item must be of `SubmitterParameter` type SubmitterParameters = attr.ib(factory=list) # type: list - # List of job is submission batch. + # List of the jobs in submission batch. # list item must be of type `RRJob` Jobs = attr.ib(factory=list) # type: list From 635fe48edb835b1748daf3023bac7781f44d2f53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Wed, 26 Oct 2022 18:38:11 +0200 Subject: [PATCH 002/104] :recycle: move functions to common lib --- .../deadline/abstract_submit_deadline.py | 112 +----------------- openpype/pipeline/farm/tools.py | 109 +++++++++++++++++ 2 files changed, 111 insertions(+), 110 deletions(-) create mode 100644 openpype/pipeline/farm/tools.py diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 512ff800ee6..0299b3ff189 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -21,6 +21,7 @@ AbstractMetaInstancePlugin, KnownPublishError ) +from openpype.pipeline.farm.tools import get_published_workfile_instance JSONDecodeError = getattr(json.decoder, "JSONDecodeError", ValueError) @@ -424,7 +425,7 @@ def process(self, instance): file_path = None if self.use_published: - file_path = self.from_published_scene() + file_path = get_published_workfile_instance(instance) # fallback if nothing was set if not file_path: @@ -495,96 +496,6 @@ def get_aux_files(self): """ return [] - def from_published_scene(self, replace_in_path=True): - """Switch work scene for published scene. - - If rendering/exporting from published scenes is enabled, this will - replace paths from working scene to published scene. - - Args: - replace_in_path (bool): if True, it will try to find - old scene name in path of expected files and replace it - with name of published scene. - - Returns: - str: Published scene path. - None: if no published scene is found. - - Note: - Published scene path is actually determined from project Anatomy - as at the time this plugin is running scene can still no be - published. - - """ - - instance = self._instance - workfile_instance = self._get_workfile_instance(instance.context) - if workfile_instance is None: - return - - # determine published path from Anatomy. - template_data = workfile_instance.data.get("anatomyData") - rep = workfile_instance.data.get("representations")[0] - template_data["representation"] = rep.get("name") - template_data["ext"] = rep.get("ext") - template_data["comment"] = None - - anatomy = instance.context.data['anatomy'] - anatomy_filled = anatomy.format(template_data) - template_filled = anatomy_filled["publish"]["path"] - file_path = os.path.normpath(template_filled) - - self.log.info("Using published scene for render {}".format(file_path)) - - if not os.path.exists(file_path): - self.log.error("published scene does not exist!") - raise - - if not replace_in_path: - return file_path - - # now we need to switch scene in expected files - # because token will now point to published - # scene file and that might differ from current one - def _clean_name(path): - return os.path.splitext(os.path.basename(path))[0] - - new_scene = _clean_name(file_path) - orig_scene = _clean_name(instance.context.data["currentFile"]) - expected_files = instance.data.get("expectedFiles") - - if isinstance(expected_files[0], dict): - # we have aovs and we need to iterate over them - new_exp = {} - for aov, files in expected_files[0].items(): - replaced_files = [] - for f in files: - replaced_files.append( - str(f).replace(orig_scene, new_scene) - ) - new_exp[aov] = replaced_files - # [] might be too much here, TODO - instance.data["expectedFiles"] = [new_exp] - else: - new_exp = [] - for f in expected_files: - new_exp.append( - str(f).replace(orig_scene, new_scene) - ) - instance.data["expectedFiles"] = new_exp - - metadata_folder = instance.data.get("publishRenderMetadataFolder") - if metadata_folder: - metadata_folder = metadata_folder.replace(orig_scene, - new_scene) - instance.data["publishRenderMetadataFolder"] = metadata_folder - - self.log.info("Scene name was switched {} -> {}".format( - orig_scene, new_scene - )) - - return file_path - def assemble_payload( self, job_info=None, plugin_info=None, aux_files=None): """Assemble payload data from its various parts. @@ -644,22 +555,3 @@ def submit(self, payload): self._instance.data["deadlineSubmissionJob"] = result return result["_id"] - - @staticmethod - def _get_workfile_instance(context): - """Find workfile instance in context""" - for i in context: - - is_workfile = ( - "workfile" in i.data.get("families", []) or - i.data["family"] == "workfile" - ) - if not is_workfile: - continue - - # test if there is instance of workfile waiting - # to be published. - assert i.data["publish"] is True, ( - "Workfile (scene) must be published along") - - return i diff --git a/openpype/pipeline/farm/tools.py b/openpype/pipeline/farm/tools.py new file mode 100644 index 00000000000..8cf1af399ea --- /dev/null +++ b/openpype/pipeline/farm/tools.py @@ -0,0 +1,109 @@ +import os + + +def get_published_workfile_instance(context): + """Find workfile instance in context""" + for i in context: + is_workfile = ( + "workfile" in i.data.get("families", []) or + i.data["family"] == "workfile" + ) + if not is_workfile: + continue + + # test if there is instance of workfile waiting + # to be published. + if i.data["publish"] is not True: + continue + + return i + + +def from_published_scene(instance, replace_in_path=True): + """Switch work scene for published scene. + + If rendering/exporting from published scenes is enabled, this will + replace paths from working scene to published scene. + + Args: + instance (pyblish.api.Instance): Instance data to process. + replace_in_path (bool): if True, it will try to find + old scene name in path of expected files and replace it + with name of published scene. + + Returns: + str: Published scene path. + None: if no published scene is found. + + Note: + Published scene path is actually determined from project Anatomy + as at the time this plugin is running the scene can be still + un-published. + + """ + workfile_instance = get_published_workfile_instance(instance.context) + if workfile_instance is None: + return + + # determine published path from Anatomy. + template_data = workfile_instance.data.get("anatomyData") + rep = workfile_instance.data.get("representations")[0] + template_data["representation"] = rep.get("name") + template_data["ext"] = rep.get("ext") + template_data["comment"] = None + + anatomy = instance.context.data['anatomy'] + anatomy_filled = anatomy.format(template_data) + template_filled = anatomy_filled["publish"]["path"] + file_path = os.path.normpath(template_filled) + + self.log.info("Using published scene for render {}".format(file_path)) + + if not os.path.exists(file_path): + self.log.error("published scene does not exist!") + raise + + if not replace_in_path: + return file_path + + # now we need to switch scene in expected files + # because token will now point to published + # scene file and that might differ from current one + def _clean_name(path): + return os.path.splitext(os.path.basename(path))[0] + + new_scene = _clean_name(file_path) + orig_scene = _clean_name(instance.context.data["currentFile"]) + expected_files = instance.data.get("expectedFiles") + + if isinstance(expected_files[0], dict): + # we have aovs and we need to iterate over them + new_exp = {} + for aov, files in expected_files[0].items(): + replaced_files = [] + for f in files: + replaced_files.append( + str(f).replace(orig_scene, new_scene) + ) + new_exp[aov] = replaced_files + # [] might be too much here, TODO + instance.data["expectedFiles"] = [new_exp] + else: + new_exp = [] + for f in expected_files: + new_exp.append( + str(f).replace(orig_scene, new_scene) + ) + instance.data["expectedFiles"] = new_exp + + metadata_folder = instance.data.get("publishRenderMetadataFolder") + if metadata_folder: + metadata_folder = metadata_folder.replace(orig_scene, + new_scene) + instance.data["publishRenderMetadataFolder"] = metadata_folder + + self.log.info("Scene name was switched {} -> {}".format( + orig_scene, new_scene + )) + + return file_path From aaafcd633341e774944b7fb865acc3f790a60f70 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 11 Jan 2023 19:13:28 +0100 Subject: [PATCH 003/104] :construction: redoing publishing flow for multiple rr roots --- .../maya/plugins/create/create_render.py | 89 +++++++++++++++---- .../maya/plugins/publish/collect_render.py | 7 ++ openpype/modules/royalrender/api.py | 30 +------ .../publish/collect_default_rr_path.py | 23 ----- .../publish/collect_rr_path_from_instance.py | 16 ++-- .../publish/collect_sequences_from_job.py | 2 +- .../publish/submit_maya_royalrender.py | 51 +++++++++-- .../project_settings/royalrender.json | 3 + .../defaults/system_settings/modules.json | 6 +- 9 files changed, 138 insertions(+), 89 deletions(-) delete mode 100644 openpype/modules/royalrender/plugins/publish/collect_default_rr_path.py diff --git a/openpype/hosts/maya/plugins/create/create_render.py b/openpype/hosts/maya/plugins/create/create_render.py index 83751494427..dedb057fb7e 100644 --- a/openpype/hosts/maya/plugins/create/create_render.py +++ b/openpype/hosts/maya/plugins/create/create_render.py @@ -79,31 +79,58 @@ def __init__(self, *args, **kwargs): if self._project_settings["maya"]["RenderSettings"]["apply_render_settings"]: # noqa lib_rendersettings.RenderSettings().set_default_renderer_settings() - # Deadline-only + # Handling farms manager = ModulesManager() deadline_settings = get_system_settings()["modules"]["deadline"] - if not deadline_settings["enabled"]: - self.deadline_servers = {} + rr_settings = get_system_settings()["modules"]["royalrender"] + + self.deadline_servers = {} + self.rr_paths = {} + + if deadline_settings["enabled"]: + self.deadline_module = manager.modules_by_name["deadline"] + try: + default_servers = deadline_settings["deadline_urls"] + project_servers = ( + self._project_settings["deadline"]["deadline_servers"] + ) + self.deadline_servers = { + k: default_servers[k] + for k in project_servers + if k in default_servers + } + + if not self.deadline_servers: + self.deadline_servers = default_servers + + except AttributeError: + # Handle situation were we had only one url for deadline. + # get default deadline webservice url from deadline module + self.deadline_servers = self.deadline_module.deadline_urls + + # RoyalRender only + if not rr_settings["enabled"]: return - self.deadline_module = manager.modules_by_name["deadline"] + + self.rr_module = manager.modules_by_name["royalrender"] try: - default_servers = deadline_settings["deadline_urls"] - project_servers = ( - self._project_settings["deadline"]["deadline_servers"] + default_paths = rr_settings["rr_paths"] + project_paths = ( + self._project_settings["royalrender"]["rr_paths"] ) - self.deadline_servers = { - k: default_servers[k] - for k in project_servers - if k in default_servers + self.rr_paths = { + k: default_paths[k] + for k in project_paths + if k in default_paths } - if not self.deadline_servers: - self.deadline_servers = default_servers + if not self.rr_paths: + self.rr_paths = default_paths except AttributeError: - # Handle situation were we had only one url for deadline. - # get default deadline webservice url from deadline module - self.deadline_servers = self.deadline_module.deadline_urls + # Handle situation were we had only one path for royalrender. + # Get default royalrender root path from the rr module. + self.rr_paths = self.rr_module.rr_paths def process(self): """Entry point.""" @@ -139,6 +166,14 @@ def process(self): self._deadline_webservice_changed ]) + # add RoyalRender root path selection list + if self.rr_paths: + cmds.scriptJob( + attributeChange=[ + "{}.rrPaths".format(self.instance), + self._rr_path_changed + ]) + cmds.setAttr("{}.machineList".format(self.instance), lock=True) rs = renderSetup.instance() layers = rs.getRenderLayers() @@ -191,6 +226,18 @@ def _deadline_webservice_changed(self): attributeType="enum", enumName=":".join(sorted_pools)) + @staticmethod + def _rr_path_changed(): + """Unused callback to pull information from RR.""" + """ + _ = self.rr_paths[ + self.server_aliases[ + cmds.getAttr("{}.rrPaths".format(self.instance)) + ] + ] + """ + pass + def _create_render_settings(self): """Create instance settings.""" # get pools (slave machines of the render farm) @@ -225,15 +272,21 @@ def _create_render_settings(self): system_settings = get_system_settings()["modules"] deadline_enabled = system_settings["deadline"]["enabled"] + royalrender_enabled = system_settings["royalrender"]["enabled"] muster_enabled = system_settings["muster"]["enabled"] muster_url = system_settings["muster"]["MUSTER_REST_URL"] - if deadline_enabled and muster_enabled: + if deadline_enabled and muster_enabled and royalrender_enabled: self.log.error( - "Both Deadline and Muster are enabled. " "Cannot support both." + ("Multiple render farm support (Deadline/RoyalRender/Muster) " + "is enabled. We support only one at time.") ) raise RuntimeError("Both Deadline and Muster are enabled") + if royalrender_enabled: + self.server_aliases = list(self.rr_paths.keys()) + self.data["rrPaths"] = self.server_aliases + if deadline_enabled: self.server_aliases = list(self.deadline_servers.keys()) self.data["deadlineServers"] = self.server_aliases diff --git a/openpype/hosts/maya/plugins/publish/collect_render.py b/openpype/hosts/maya/plugins/publish/collect_render.py index b1ad3ca58ed..efae830b643 100644 --- a/openpype/hosts/maya/plugins/publish/collect_render.py +++ b/openpype/hosts/maya/plugins/publish/collect_render.py @@ -328,6 +328,13 @@ def process(self, context): if deadline_settings["enabled"]: data["deadlineUrl"] = render_instance.data.get("deadlineUrl") + rr_settings = ( + context.data["system_settings"]["modules"]["royalrender"] + ) + if rr_settings["enabled"]: + data["rrPathName"] = render_instance.data.get("rrPathName") + self.log.info(data["rrPathName"]) + if self.sync_workfile_version: data["version"] = context.data["version"] diff --git a/openpype/modules/royalrender/api.py b/openpype/modules/royalrender/api.py index c47d50b62b5..dcb518deb1c 100644 --- a/openpype/modules/royalrender/api.py +++ b/openpype/modules/royalrender/api.py @@ -15,36 +15,10 @@ class Api: RR_SUBMIT_CONSOLE = 1 RR_SUBMIT_API = 2 - def __init__(self, project=None): + def __init__(self, rr_path=None): self.log = Logger.get_logger("RoyalRender") - self._initialize_rr(project) - - def _initialize_rr(self, project=None): - # type: (str) -> None - """Initialize RR Path. - - Args: - project (str, Optional): Project name to set RR api in - context. - - """ - if project: - project_settings = get_project_settings(project) - rr_path = ( - project_settings - ["royalrender"] - ["rr_paths"] - ) - else: - rr_path = ( - self._settings - ["modules"] - ["royalrender"] - ["rr_path"] - ["default"] - ) - os.environ["RR_ROOT"] = rr_path self._rr_path = rr_path + os.environ["RR_ROOT"] = rr_path def _get_rr_bin_path(self, rr_root=None): # type: (str) -> str diff --git a/openpype/modules/royalrender/plugins/publish/collect_default_rr_path.py b/openpype/modules/royalrender/plugins/publish/collect_default_rr_path.py deleted file mode 100644 index 3ce95e0c50e..00000000000 --- a/openpype/modules/royalrender/plugins/publish/collect_default_rr_path.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- -"""Collect default Deadline server.""" -import pyblish.api - - -class CollectDefaultRRPath(pyblish.api.ContextPlugin): - """Collect default Royal Render path.""" - - order = pyblish.api.CollectorOrder - label = "Default Royal Render Path" - - def process(self, context): - try: - rr_module = context.data.get( - "openPypeModules")["royalrender"] - except AttributeError: - msg = "Cannot get OpenPype Royal Render module." - self.log.error(msg) - raise AssertionError(msg) - - # get default deadline webservice url from deadline module - self.log.debug(rr_module.rr_paths) - context.data["defaultRRPath"] = rr_module.rr_paths["default"] # noqa: E501 diff --git a/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py b/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py index 6a3dc276f37..187e2b9c446 100644 --- a/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py +++ b/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py @@ -5,19 +5,19 @@ class CollectRRPathFromInstance(pyblish.api.InstancePlugin): """Collect RR Path from instance.""" - order = pyblish.api.CollectorOrder + 0.01 - label = "Royal Render Path from the Instance" + order = pyblish.api.CollectorOrder + label = "Collect Royal Render path name from the Instance" families = ["rendering"] def process(self, instance): - instance.data["rrPath"] = self._collect_rr_path(instance) + instance.data["rrPathName"] = self._collect_rr_path_name(instance) self.log.info( - "Using {} for submission.".format(instance.data["rrPath"])) + "Using '{}' for submission.".format(instance.data["rrPathName"])) @staticmethod - def _collect_rr_path(render_instance): + def _collect_rr_path_name(render_instance): # type: (pyblish.api.Instance) -> str - """Get Royal Render path from render instance.""" + """Get Royal Render pat name from render instance.""" rr_settings = ( render_instance.context.data ["system_settings"] @@ -42,8 +42,6 @@ def _collect_rr_path(render_instance): # Handle situation were we had only one url for royal render. return render_instance.context.data["defaultRRPath"] - return rr_servers[ - list(rr_servers.keys())[ + return list(rr_servers.keys())[ int(render_instance.data.get("rrPaths")) ] - ] diff --git a/openpype/modules/royalrender/plugins/publish/collect_sequences_from_job.py b/openpype/modules/royalrender/plugins/publish/collect_sequences_from_job.py index 65af90e8a61..4c123e4134b 100644 --- a/openpype/modules/royalrender/plugins/publish/collect_sequences_from_job.py +++ b/openpype/modules/royalrender/plugins/publish/collect_sequences_from_job.py @@ -71,7 +71,7 @@ class CollectSequencesFromJob(pyblish.api.ContextPlugin): """Gather file sequences from job directory. When "OPENPYPE_PUBLISH_DATA" environment variable is set these paths - (folders or .json files) are parsed for image sequences. Otherwise the + (folders or .json files) are parsed for image sequences. Otherwise, the current working directory is searched for file sequences. """ diff --git a/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py index f1ba5e62ef6..ef98bcf74c3 100644 --- a/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py +++ b/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py @@ -3,9 +3,10 @@ import os import sys import tempfile +import platform from maya.OpenMaya import MGlobal # noqa -from pyblish.api import InstancePlugin, IntegratorOrder +from pyblish.api import InstancePlugin, IntegratorOrder, Context from openpype.hosts.maya.api.lib import get_attr_in_layer from openpype.pipeline.publish.lib import get_published_workfile_instance from openpype.pipeline.publish import KnownPublishError @@ -16,6 +17,8 @@ class MayaSubmitRoyalRender(InstancePlugin): label = "Submit to RoyalRender" order = IntegratorOrder + 0.1 + families = ["renderlayer"] + targets = ["local"] use_published = True def __init__(self, *args, **kwargs): @@ -102,7 +105,16 @@ def process(self, instance): """Plugin entry point.""" self._instance = instance context = instance.context - self.rr_api = rr_api(context.data["project"]) + from pprint import pformat + + self._rr_root = self._resolve_rr_path(context, instance.data.get("rrPathName")) # noqa + self.log.debug(self._rr_root) + if not self._rr_root: + raise KnownPublishError( + ("Missing RoyalRender root. " + "You need to configure RoyalRender module.")) + + self.rr_api = rr_api(self._rr_root) # get royalrender module """ @@ -114,11 +126,7 @@ def process(self, instance): raise AssertionError("OpenPype RoyalRender module not found.") """ - self._rrRoot = instance.data["rrPath"] or context.data["defaultRRPath"] # noqa - if not self._rrRoot: - raise KnownPublishError( - ("Missing RoyalRender root. " - "You need to configure RoyalRender module.")) + file_path = None if self.use_published: file_path = get_published_workfile_instance() @@ -147,4 +155,33 @@ def process_submission(self): self.rr_api.submit_file(file=xml) + @staticmethod + def _resolve_rr_path(context, rr_path_name): + # type: (Context, str) -> str + rr_settings = ( + context.data + ["system_settings"] + ["modules"] + ["royalrender"] + ) + try: + default_servers = rr_settings["rr_paths"] + project_servers = ( + context.data + ["project_settings"] + ["royalrender"] + ["rr_paths"] + ) + rr_servers = { + k: default_servers[k] + for k in project_servers + if k in default_servers + } + + except (AttributeError, KeyError): + # Handle situation were we had only one url for royal render. + return context.data["defaultRRPath"][platform.system().lower()] + + return rr_servers[rr_path_name][platform.system().lower()] + diff --git a/openpype/settings/defaults/project_settings/royalrender.json b/openpype/settings/defaults/project_settings/royalrender.json index be267b11d8e..dc7d1574a10 100644 --- a/openpype/settings/defaults/project_settings/royalrender.json +++ b/openpype/settings/defaults/project_settings/royalrender.json @@ -1,4 +1,7 @@ { + "rr_paths": [ + "default" + ], "publish": { "CollectSequencesFromJob": { "review": true diff --git a/openpype/settings/defaults/system_settings/modules.json b/openpype/settings/defaults/system_settings/modules.json index c84d23d3fcb..b5f7784663f 100644 --- a/openpype/settings/defaults/system_settings/modules.json +++ b/openpype/settings/defaults/system_settings/modules.json @@ -185,9 +185,9 @@ "enabled": false, "rr_paths": { "default": { - "windows": "", - "darwin": "", - "linux": "" + "windows": "C:\\RR8", + "darwin": "/Volumes/share/RR8", + "linux": "/mnt/studio/RR8" } } }, From 0eb7da3b93b7e54f9461575751c87672a2fb8982 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 11 Jan 2023 19:13:54 +0100 Subject: [PATCH 004/104] :recycle: optimizing enum classes --- openpype/settings/entities/__init__.py | 4 +- openpype/settings/entities/enum_entity.py | 81 ++++++++++--------- .../schema_project_royalrender.json | 6 ++ 3 files changed, 53 insertions(+), 38 deletions(-) diff --git a/openpype/settings/entities/__init__.py b/openpype/settings/entities/__init__.py index b2cb2204f44..dab17c84a3e 100644 --- a/openpype/settings/entities/__init__.py +++ b/openpype/settings/entities/__init__.py @@ -107,7 +107,8 @@ TaskTypeEnumEntity, DeadlineUrlEnumEntity, AnatomyTemplatesEnumEntity, - ShotgridUrlEnumEntity + ShotgridUrlEnumEntity, + RoyalRenderRootEnumEntity ) from .list_entity import ListEntity @@ -173,6 +174,7 @@ "TaskTypeEnumEntity", "DeadlineUrlEnumEntity", "ShotgridUrlEnumEntity", + "RoyalRenderRootEnumEntity", "AnatomyTemplatesEnumEntity", "ListEntity", diff --git a/openpype/settings/entities/enum_entity.py b/openpype/settings/entities/enum_entity.py index c0c103ea10e..9f9ae930264 100644 --- a/openpype/settings/entities/enum_entity.py +++ b/openpype/settings/entities/enum_entity.py @@ -1,3 +1,5 @@ +import abc +import six import copy from .input_entities import InputEntity from .exceptions import EntitySchemaError @@ -476,8 +478,9 @@ def set_override_state(self, *args, **kwargs): self.set(value_on_not_set) -class DeadlineUrlEnumEntity(BaseEnumEntity): - schema_types = ["deadline_url-enum"] +@six.add_metaclass(abc.ABCMeta) +class FarmRootEnumEntity(BaseEnumEntity): + schema_types = [] def _item_initialization(self): self.multiselection = self.schema_data.get("multiselection", True) @@ -495,22 +498,8 @@ def _item_initialization(self): # GUI attribute self.placeholder = self.schema_data.get("placeholder") - def _get_enum_values(self): - deadline_urls_entity = self.get_entity_from_path( - "system_settings/modules/deadline/deadline_urls" - ) - - valid_keys = set() - enum_items_list = [] - for server_name, url_entity in deadline_urls_entity.items(): - enum_items_list.append( - {server_name: "{}: {}".format(server_name, url_entity.value)} - ) - valid_keys.add(server_name) - return enum_items_list, valid_keys - def set_override_state(self, *args, **kwargs): - super(DeadlineUrlEnumEntity, self).set_override_state(*args, **kwargs) + super(FarmRootEnumEntity, self).set_override_state(*args, **kwargs) self.enum_items, self.valid_keys = self._get_enum_values() if self.multiselection: @@ -527,21 +516,49 @@ def set_override_state(self, *args, **kwargs): elif self._current_value not in self.valid_keys: self._current_value = tuple(self.valid_keys)[0] + @abc.abstractmethod + def _get_enum_values(self): + pass -class ShotgridUrlEnumEntity(BaseEnumEntity): - schema_types = ["shotgrid_url-enum"] - def _item_initialization(self): - self.multiselection = False +class DeadlineUrlEnumEntity(FarmRootEnumEntity): + schema_types = ["deadline_url-enum"] - self.enum_items = [] - self.valid_keys = set() + def _get_enum_values(self): + deadline_urls_entity = self.get_entity_from_path( + "system_settings/modules/deadline/deadline_urls" + ) - self.valid_value_types = (STRING_TYPE,) - self.value_on_not_set = "" + valid_keys = set() + enum_items_list = [] + for server_name, url_entity in deadline_urls_entity.items(): + enum_items_list.append( + {server_name: "{}: {}".format(server_name, url_entity.value)} + ) + valid_keys.add(server_name) + return enum_items_list, valid_keys - # GUI attribute - self.placeholder = self.schema_data.get("placeholder") + +class RoyalRenderRootEnumEntity(FarmRootEnumEntity): + schema_types = ["rr_root-enum"] + + def _get_enum_values(self): + rr_root_entity = self.get_entity_from_path( + "system_settings/modules/royalrender/rr_paths" + ) + + valid_keys = set() + enum_items_list = [] + for server_name, url_entity in rr_root_entity.items(): + enum_items_list.append( + {server_name: "{}: {}".format(server_name, url_entity.value)} + ) + valid_keys.add(server_name) + return enum_items_list, valid_keys + + +class ShotgridUrlEnumEntity(FarmRootEnumEntity): + schema_types = ["shotgrid_url-enum"] def _get_enum_values(self): shotgrid_settings = self.get_entity_from_path( @@ -561,16 +578,6 @@ def _get_enum_values(self): valid_keys.add(server_name) return enum_items_list, valid_keys - def set_override_state(self, *args, **kwargs): - super(ShotgridUrlEnumEntity, self).set_override_state(*args, **kwargs) - - self.enum_items, self.valid_keys = self._get_enum_values() - if not self.valid_keys: - self._current_value = "" - - elif self._current_value not in self.valid_keys: - self._current_value = tuple(self.valid_keys)[0] - class AnatomyTemplatesEnumEntity(BaseEnumEntity): schema_types = ["anatomy-templates-enum"] diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_royalrender.json b/openpype/settings/entities/schemas/projects_schema/schema_project_royalrender.json index cabb4747d5e..f4bf2f51ba6 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_royalrender.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_royalrender.json @@ -5,6 +5,12 @@ "collapsible": true, "is_file": true, "children": [ + { + "type": "rr_root-enum", + "key": "rr_paths", + "label": "Royal Render Roots", + "multiselect": true + }, { "type": "dict", "collapsible": true, From 2cc688fb72dd9d04bbe76beab2ca3d04c4c9df1d Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Mon, 16 Jan 2023 10:40:05 +0100 Subject: [PATCH 005/104] :construction: render job submission --- openpype/modules/royalrender/api.py | 76 +++++++++++-------- .../publish/submit_maya_royalrender.py | 45 ++++++----- openpype/modules/royalrender/rr_job.py | 10 ++- 3 files changed, 77 insertions(+), 54 deletions(-) diff --git a/openpype/modules/royalrender/api.py b/openpype/modules/royalrender/api.py index dcb518deb1c..8b13b9781f1 100644 --- a/openpype/modules/royalrender/api.py +++ b/openpype/modules/royalrender/api.py @@ -2,11 +2,13 @@ """Wrapper around Royal Render API.""" import sys import os +import platform from openpype.settings import get_project_settings from openpype.lib.local_settings import OpenPypeSettingsRegistry from openpype.lib import Logger, run_subprocess from .rr_job import RRJob, SubmitFile, SubmitterParameter +from openpype.lib.vendor_bin_utils import find_tool_in_custom_paths class Api: @@ -20,31 +22,46 @@ def __init__(self, rr_path=None): self._rr_path = rr_path os.environ["RR_ROOT"] = rr_path - def _get_rr_bin_path(self, rr_root=None): - # type: (str) -> str - """Get path to RR bin folder.""" + def _get_rr_bin_path(self, tool_name=None, rr_root=None): + # type: (str, str) -> str + """Get path to RR bin folder. + + Args: + tool_name (str): Name of RR executable you want. + rr_root (str, Optional): Custom RR root if needed. + + Returns: + str: Path to the tool based on current platform. + + """ rr_root = rr_root or self._rr_path is_64bit_python = sys.maxsize > 2 ** 32 - rr_bin_path = "" + rr_bin_parts = [rr_root, "bin"] if sys.platform.lower() == "win32": - rr_bin_path = "/bin/win64" - if not is_64bit_python: - # we are using 64bit python - rr_bin_path = "/bin/win" - rr_bin_path = rr_bin_path.replace( - "/", os.path.sep - ) + rr_bin_parts.append("win") if sys.platform.lower() == "darwin": - rr_bin_path = "/bin/mac64" - if not is_64bit_python: - rr_bin_path = "/bin/mac" + rr_bin_parts.append("mac") + + if sys.platform.lower().startswith("linux"): + rr_bin_parts.append("lx") + + rr_bin_path = os.sep.join(rr_bin_parts) - if sys.platform.lower() == "linux": - rr_bin_path = "/bin/lx64" + paths_to_check = [] + # if we use 64bit python, append 64bit specific path first + if is_64bit_python: + if not tool_name: + return rr_bin_path + "64" + paths_to_check.append(rr_bin_path + "64") - return os.path.join(rr_root, rr_bin_path) + # otherwise use 32bit + if not tool_name: + return rr_bin_path + paths_to_check.append(rr_bin_path) + + return find_tool_in_custom_paths(paths_to_check, tool_name) def _initialize_module_path(self): # type: () -> None @@ -84,30 +101,25 @@ def submit_file(self, file, mode=RR_SUBMIT_CONSOLE): # type: (SubmitFile, int) -> None if mode == self.RR_SUBMIT_CONSOLE: self._submit_using_console(file) + return - # RR v7 supports only Python 2.7 so we bail out in fear + # RR v7 supports only Python 2.7, so we bail out in fear # until there is support for Python 3 😰 raise NotImplementedError( "Submission via RoyalRender API is not supported yet") # self._submit_using_api(file) - def _submit_using_console(self, file): + def _submit_using_console(self, job_file): # type: (SubmitFile) -> None - rr_console = os.path.join( - self._get_rr_bin_path(), - "rrSubmitterConsole" - ) - - if sys.platform.lower() == "darwin" and "/bin/mac64" in rr_console: - rr_console = rr_console.replace("/bin/mac64", "/bin/mac") + rr_start_local = self._get_rr_bin_path("rrStartLocal") - if sys.platform.lower() == "win32": - if "/bin/win64" in rr_console: - rr_console = rr_console.replace("/bin/win64", "/bin/win") - rr_console += ".exe" + self.log.info("rr_console: {}".format(rr_start_local)) - args = [rr_console, file] - run_subprocess(" ".join(args), logger=self.log) + args = [rr_start_local, "rrSubmitterconsole", job_file] + self.log.info("Executing: {}".format(" ".join(args))) + env = os.environ + env["RR_ROOT"] = self._rr_path + run_subprocess(args, logger=self.log, env=env) def _submit_using_api(self, file): # type: (SubmitFile) -> None diff --git a/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py index ef98bcf74c3..9303bad8954 100644 --- a/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py +++ b/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py @@ -38,11 +38,11 @@ def get_job(self): """ def get_rr_platform(): if sys.platform.lower() in ["win32", "win64"]: - return "win" + return "windows" elif sys.platform.lower() == "darwin": return "mac" else: - return "lx" + return "linux" expected_files = self._instance.data["expectedFiles"] first_file = next(self._iter_expected_files(expected_files)) @@ -53,8 +53,10 @@ def get_rr_platform(): .get('maya') \ .get('RenderSettings') \ .get('default_render_image_folder') - filename = os.path.basename(self.scene_path) - dirname = os.path.join(workspace, default_render_file) + file_name = os.path.basename(self.scene_path) + dir_name = os.path.join(workspace, default_render_file) + layer = self._instance.data["setMembers"] # type: str + layer_name = layer.removeprefix("rs_") job = RRJob( Software="Maya", @@ -64,22 +66,22 @@ def get_rr_platform(): SeqStep=int(self._instance.data["byFrameStep"]), SeqFileOffset=0, Version="{0:.2f}".format(MGlobal.apiVersion() / 10000), - SceneName=os.path.basename(self.scene_path), + SceneName=self.scene_path, IsActive=True, - ImageDir=dirname, - ImageFilename=filename, - ImageExtension="." + os.path.splitext(filename)[1], + ImageDir=dir_name, + ImageFilename="{}.".format(layer_name), + ImageExtension=os.path.splitext(first_file)[1], ImagePreNumberLetter=".", ImageSingleOutputFile=False, SceneOS=get_rr_platform(), Camera=self._instance.data["cameras"][0], - Layer=self._instance.data["layer"], + Layer=layer_name, SceneDatabaseDir=workspace, - ImageFramePadding=get_attr_in_layer( - "defaultRenderGlobals.extensionPadding", - self._instance.data["layer"]), + CustomSHotName=self._instance.context.data["asset"], + CompanyProjectName=self._instance.context.data["projectName"], ImageWidth=self._instance.data["resolutionWidth"], - ImageHeight=self._instance.data["resolutionHeight"] + ImageHeight=self._instance.data["resolutionHeight"], + PreID=1 ) return job @@ -125,11 +127,9 @@ def process(self, instance): self.log.error("Cannot get OpenPype RoyalRender module.") raise AssertionError("OpenPype RoyalRender module not found.") """ - - file_path = None if self.use_published: - file_path = get_published_workfile_instance() + file_path = get_published_workfile_instance(context) # fallback if nothing was set if not file_path: @@ -153,7 +153,8 @@ def process_submission(self): with open(xml.name, "w") as f: f.write(submission.serialize()) - self.rr_api.submit_file(file=xml) + self.log.info("submitting job file: {}".format(xml.name)) + self.rr_api.submit_file(file=xml.name) @staticmethod def _resolve_rr_path(context, rr_path_name): @@ -184,4 +185,12 @@ def _resolve_rr_path(context, rr_path_name): return rr_servers[rr_path_name][platform.system().lower()] - + @staticmethod + def _iter_expected_files(exp): + if isinstance(exp[0], dict): + for _aov, files in exp[0].items(): + for file in files: + yield file + else: + for file in exp: + yield file diff --git a/openpype/modules/royalrender/rr_job.py b/openpype/modules/royalrender/rr_job.py index beb8c17187e..f5c7033b625 100644 --- a/openpype/modules/royalrender/rr_job.py +++ b/openpype/modules/royalrender/rr_job.py @@ -87,7 +87,7 @@ class RRJob: # Frame Padding of the frame number in the rendered filename. # Some render config files are setting the padding at render time. - ImageFramePadding = attr.ib(default=None) # type: str + ImageFramePadding = attr.ib(default=None) # type: int # Some render applications support overriding the image format at # the render commandline. @@ -129,6 +129,7 @@ class RRJob: CustomUserInfo = attr.ib(default=None) # type: str SubmitMachine = attr.ib(default=None) # type: str Color_ID = attr.ib(default=2) # type: int + CompanyProjectName = attr.ib(default=None) # type: str RequiredLicenses = attr.ib(default=None) # type: str @@ -225,7 +226,7 @@ def filter_data(a, v): # foo=bar~baz~goo self._process_submitter_parameters( self.SubmitterParameters, root, job_file) - + root.appendChild(job_file) for job in self.Jobs: # type: RRJob if not isinstance(job, RRJob): raise AttributeError( @@ -247,10 +248,11 @@ def filter_data(a, v): custom_attr.name)] = custom_attr.value for item, value in serialized_job.items(): - xml_attr = root.create(item) + xml_attr = root.createElement(item) xml_attr.appendChild( - root.createTextNode(value) + root.createTextNode(str(value)) ) xml_job.appendChild(xml_attr) + job_file.appendChild(xml_job) return root.toprettyxml(indent="\t") From b3b266752bc2396fa7d68637f25e5725283e54d6 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 18 Jan 2023 17:06:05 +0100 Subject: [PATCH 006/104] :construction: refactor RR job flow --- .../publish/collect_rr_path_from_instance.py | 12 +- ...nder.py => create_maya_royalrender_job.py} | 76 ++------ .../publish/create_publish_royalrender_job.py | 178 ++++++++++++++++++ .../publish/submit_jobs_to_royalrender.py | 115 +++++++++++ openpype/pipeline/farm/pyblish.py | 49 +++++ 5 files changed, 364 insertions(+), 66 deletions(-) rename openpype/modules/royalrender/plugins/publish/{submit_maya_royalrender.py => create_maya_royalrender_job.py} (74%) create mode 100644 openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py create mode 100644 openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py create mode 100644 openpype/pipeline/farm/pyblish.py diff --git a/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py b/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py index 187e2b9c446..40f34561fa9 100644 --- a/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py +++ b/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py @@ -15,19 +15,21 @@ def process(self, instance): "Using '{}' for submission.".format(instance.data["rrPathName"])) @staticmethod - def _collect_rr_path_name(render_instance): + def _collect_rr_path_name(instance): # type: (pyblish.api.Instance) -> str """Get Royal Render pat name from render instance.""" rr_settings = ( - render_instance.context.data + instance.context.data ["system_settings"] ["modules"] ["royalrender"] ) + if not instance.data.get("rrPaths"): + return "default" try: default_servers = rr_settings["rr_paths"] project_servers = ( - render_instance.context.data + instance.context.data ["project_settings"] ["royalrender"] ["rr_paths"] @@ -40,8 +42,8 @@ def _collect_rr_path_name(render_instance): except (AttributeError, KeyError): # Handle situation were we had only one url for royal render. - return render_instance.context.data["defaultRRPath"] + return rr_settings["rr_paths"]["default"] return list(rr_servers.keys())[ - int(render_instance.data.get("rrPaths")) + int(instance.data.get("rrPaths")) ] diff --git a/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py similarity index 74% rename from openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py rename to openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py index 9303bad8954..e194a7edc60 100644 --- a/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py +++ b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py @@ -2,20 +2,18 @@ """Submitting render job to RoyalRender.""" import os import sys -import tempfile import platform from maya.OpenMaya import MGlobal # noqa from pyblish.api import InstancePlugin, IntegratorOrder, Context -from openpype.hosts.maya.api.lib import get_attr_in_layer from openpype.pipeline.publish.lib import get_published_workfile_instance from openpype.pipeline.publish import KnownPublishError -from openpype.modules.royalrender.api import Api as rr_api -from openpype.modules.royalrender.rr_job import RRJob, SubmitterParameter +from openpype.modules.royalrender.api import Api as rrApi +from openpype.modules.royalrender.rr_job import RRJob -class MayaSubmitRoyalRender(InstancePlugin): - label = "Submit to RoyalRender" +class CreateMayaRoyalRenderJob(InstancePlugin): + label = "Create Maya Render job in RR" order = IntegratorOrder + 0.1 families = ["renderlayer"] targets = ["local"] @@ -81,28 +79,9 @@ def get_rr_platform(): CompanyProjectName=self._instance.context.data["projectName"], ImageWidth=self._instance.data["resolutionWidth"], ImageHeight=self._instance.data["resolutionHeight"], - PreID=1 ) return job - @staticmethod - def get_submission_parameters(): - return [] - - def create_file(self, name, ext, contents=None): - temp = tempfile.NamedTemporaryFile( - dir=self.tempdir, - suffix=ext, - prefix=name + '.', - delete=False, - ) - - if contents: - with open(temp.name, 'w') as f: - f.write(contents) - - return temp.name - def process(self, instance): """Plugin entry point.""" self._instance = instance @@ -116,17 +95,8 @@ def process(self, instance): ("Missing RoyalRender root. " "You need to configure RoyalRender module.")) - self.rr_api = rr_api(self._rr_root) + self.rr_api = rrApi(self._rr_root) - # get royalrender module - """ - try: - rr_module = context.data.get( - "openPypeModules")["royalrender"] - except AttributeError: - self.log.error("Cannot get OpenPype RoyalRender module.") - raise AssertionError("OpenPype RoyalRender module not found.") - """ file_path = None if self.use_published: file_path = get_published_workfile_instance(context) @@ -137,24 +107,18 @@ def process(self, instance): file_path = context.data["currentFile"] self.scene_path = file_path - self.job = self.get_job() - self.log.info(self.job) - self.submission_parameters = self.get_submission_parameters() - self.process_submission() + self._instance.data["rrJobs"] = [self.get_job()] - def process_submission(self): - submission = rr_api.create_submission( - [self.job], - self.submission_parameters) - - self.log.debug(submission) - xml = tempfile.NamedTemporaryFile(suffix=".xml", delete=False) - with open(xml.name, "w") as f: - f.write(submission.serialize()) - - self.log.info("submitting job file: {}".format(xml.name)) - self.rr_api.submit_file(file=xml.name) + @staticmethod + def _iter_expected_files(exp): + if isinstance(exp[0], dict): + for _aov, files in exp[0].items(): + for file in files: + yield file + else: + for file in exp: + yield file @staticmethod def _resolve_rr_path(context, rr_path_name): @@ -184,13 +148,3 @@ def _resolve_rr_path(context, rr_path_name): return context.data["defaultRRPath"][platform.system().lower()] return rr_servers[rr_path_name][platform.system().lower()] - - @staticmethod - def _iter_expected_files(exp): - if isinstance(exp[0], dict): - for _aov, files in exp[0].items(): - for file in files: - yield file - else: - for file in exp: - yield file diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py new file mode 100644 index 00000000000..9d8cc602c52 --- /dev/null +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +"""Create publishing job on RoyalRender.""" +from pyblish.api import InstancePlugin, IntegratorOrder +from copy import deepcopy +from openpype.pipeline import legacy_io +import requests +import os + + +class CreatePublishRoyalRenderJob(InstancePlugin): + label = "Create publish job in RR" + order = IntegratorOrder + 0.2 + icon = "tractor" + targets = ["local"] + hosts = ["fusion", "maya", "nuke", "celaction", "aftereffects", "harmony"] + families = ["render.farm", "prerender.farm", + "renderlayer", "imagesequence", "vrayscene"] + aov_filter = {"maya": [r".*([Bb]eauty).*"], + "aftereffects": [r".*"], # for everything from AE + "harmony": [r".*"], # for everything from AE + "celaction": [r".*"]} + + def process(self, instance): + data = instance.data.copy() + context = instance.context + self.context = context + self.anatomy = instance.context.data["anatomy"] + + asset = data.get("asset") + subset = data.get("subset") + source = self._remap_source( + data.get("source") or context.data["source"]) + + + + def _remap_source(self, source): + success, rootless_path = ( + self.anatomy.find_root_template_from_path(source) + ) + if success: + source = rootless_path + else: + # `rootless_path` is not set to `source` if none of roots match + self.log.warning(( + "Could not find root path for remapping \"{}\"." + " This may cause issues." + ).format(source)) + return source + + def _submit_post_job(self, instance, job, instances): + """Submit publish job to RoyalRender.""" + data = instance.data.copy() + subset = data["subset"] + job_name = "Publish - {subset}".format(subset=subset) + + # instance.data.get("subset") != instances[0]["subset"] + # 'Main' vs 'renderMain' + override_version = None + instance_version = instance.data.get("version") # take this if exists + if instance_version != 1: + override_version = instance_version + output_dir = self._get_publish_folder( + instance.context.data['anatomy'], + deepcopy(instance.data["anatomyData"]), + instance.data.get("asset"), + instances[0]["subset"], + 'render', + override_version + ) + + # Transfer the environment from the original job to this dependent + # job, so they use the same environment + metadata_path, roothless_metadata_path = \ + self._create_metadata_path(instance) + + environment = { + "AVALON_PROJECT": legacy_io.Session["AVALON_PROJECT"], + "AVALON_ASSET": legacy_io.Session["AVALON_ASSET"], + "AVALON_TASK": legacy_io.Session["AVALON_TASK"], + "OPENPYPE_USERNAME": instance.context.data["user"], + "OPENPYPE_PUBLISH_JOB": "1", + "OPENPYPE_RENDER_JOB": "0", + "OPENPYPE_REMOTE_JOB": "0", + "OPENPYPE_LOG_NO_COLORS": "1" + } + + # add environments from self.environ_keys + for env_key in self.environ_keys: + if os.getenv(env_key): + environment[env_key] = os.environ[env_key] + + # pass environment keys from self.environ_job_filter + job_environ = job["Props"].get("Env", {}) + for env_j_key in self.environ_job_filter: + if job_environ.get(env_j_key): + environment[env_j_key] = job_environ[env_j_key] + + # Add mongo url if it's enabled + if instance.context.data.get("deadlinePassMongoUrl"): + mongo_url = os.environ.get("OPENPYPE_MONGO") + if mongo_url: + environment["OPENPYPE_MONGO"] = mongo_url + + priority = self.deadline_priority or instance.data.get("priority", 50) + + args = [ + "--headless", + 'publish', + roothless_metadata_path, + "--targets", "deadline", + "--targets", "farm" + ] + + # Generate the payload for Deadline submission + payload = { + "JobInfo": { + "Plugin": self.deadline_plugin, + "BatchName": job["Props"]["Batch"], + "Name": job_name, + "UserName": job["Props"]["User"], + "Comment": instance.context.data.get("comment", ""), + + "Department": self.deadline_department, + "ChunkSize": self.deadline_chunk_size, + "Priority": priority, + + "Group": self.deadline_group, + "Pool": instance.data.get("primaryPool"), + "SecondaryPool": instance.data.get("secondaryPool"), + + "OutputDirectory0": output_dir + }, + "PluginInfo": { + "Version": self.plugin_pype_version, + "Arguments": " ".join(args), + "SingleFrameOnly": "True", + }, + # Mandatory for Deadline, may be empty + "AuxFiles": [], + } + + # add assembly jobs as dependencies + if instance.data.get("tileRendering"): + self.log.info("Adding tile assembly jobs as dependencies...") + job_index = 0 + for assembly_id in instance.data.get("assemblySubmissionJobs"): + payload["JobInfo"]["JobDependency{}".format(job_index)] = assembly_id # noqa: E501 + job_index += 1 + elif instance.data.get("bakingSubmissionJobs"): + self.log.info("Adding baking submission jobs as dependencies...") + job_index = 0 + for assembly_id in instance.data["bakingSubmissionJobs"]: + payload["JobInfo"]["JobDependency{}".format(job_index)] = assembly_id # noqa: E501 + job_index += 1 + else: + payload["JobInfo"]["JobDependency0"] = job["_id"] + + if instance.data.get("suspend_publish"): + payload["JobInfo"]["InitialStatus"] = "Suspended" + + for index, (key_, value_) in enumerate(environment.items()): + payload["JobInfo"].update( + { + "EnvironmentKeyValue%d" + % index: "{key}={value}".format( + key=key_, value=value_ + ) + } + ) + # remove secondary pool + payload["JobInfo"].pop("SecondaryPool", None) + + self.log.info("Submitting Deadline job ...") + + url = "{}/api/jobs".format(self.deadline_url) + response = requests.post(url, json=payload, timeout=10) + if not response.ok: + raise Exception(response.text) \ No newline at end of file diff --git a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py new file mode 100644 index 00000000000..4fcf3a08bd8 --- /dev/null +++ b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +"""Submit jobs to RoyalRender.""" +import tempfile +import platform + +from pyblish.api import IntegratorOrder, ContextPlugin, Context +from openpype.modules.royalrender.api import RRJob, Api as rrApi +from openpype.pipeline.publish import KnownPublishError + + +class SubmitJobsToRoyalRender(ContextPlugin): + """Find all jobs, create submission XML and submit it to RoyalRender.""" + label = "Submit jobs to RoyalRender" + order = IntegratorOrder + 0.1 + targets = ["local"] + + def __init__(self): + super(SubmitJobsToRoyalRender, self).__init__() + self._rr_root = None + self._rr_api = None + self._submission_parameters = [] + + def process(self, context): + rr_settings = ( + context.data + ["system_settings"] + ["modules"] + ["royalrender"] + ) + + if rr_settings["enabled"] is not True: + self.log.warning("RoyalRender modules is disabled.") + return + + # iterate over all instances and try to find RRJobs + jobs = [] + for instance in context: + if isinstance(instance.data.get("rrJob"), RRJob): + jobs.append(instance.data.get("rrJob")) + if instance.data.get("rrJobs"): + if all(isinstance(job, RRJob) for job in instance.data.get("rrJobs")): + jobs += instance.data.get("rrJobs") + + if jobs: + self._rr_root = self._resolve_rr_path( + context, instance.data.get("rrPathName")) # noqa + if not self._rr_root: + raise KnownPublishError( + ("Missing RoyalRender root. " + "You need to configure RoyalRender module.")) + self._rr_api = rrApi(self._rr_root) + self._submission_parameters = self.get_submission_parameters() + self.process_submission(jobs) + return + + self.log.info("No RoyalRender jobs found") + + def process_submission(self, jobs): + # type: ([RRJob]) -> None + submission = rrApi.create_submission( + jobs, + self._submission_parameters) + + xml = tempfile.NamedTemporaryFile(suffix=".xml", delete=False) + with open(xml.name, "w") as f: + f.write(submission.serialize()) + + self.log.info("submitting job(s) file: {}".format(xml.name)) + self._rr_api.submit_file(file=xml.name) + + def create_file(self, name, ext, contents=None): + temp = tempfile.NamedTemporaryFile( + dir=self.tempdir, + suffix=ext, + prefix=name + '.', + delete=False, + ) + + if contents: + with open(temp.name, 'w') as f: + f.write(contents) + + return temp.name + + def get_submission_parameters(self): + return [] + + @staticmethod + def _resolve_rr_path(context, rr_path_name): + # type: (Context, str) -> str + rr_settings = ( + context.data + ["system_settings"] + ["modules"] + ["royalrender"] + ) + try: + default_servers = rr_settings["rr_paths"] + project_servers = ( + context.data + ["project_settings"] + ["royalrender"] + ["rr_paths"] + ) + rr_servers = { + k: default_servers[k] + for k in project_servers + if k in default_servers + } + + except (AttributeError, KeyError): + # Handle situation were we had only one url for royal render. + return context.data["defaultRRPath"][platform.system().lower()] + + return rr_servers[rr_path_name][platform.system().lower()] diff --git a/openpype/pipeline/farm/pyblish.py b/openpype/pipeline/farm/pyblish.py new file mode 100644 index 00000000000..02535f40903 --- /dev/null +++ b/openpype/pipeline/farm/pyblish.py @@ -0,0 +1,49 @@ +from openpype.lib import Logger +import attr + + +@attr.s +class InstanceSkeleton(object): + family = attr.ib(factory=) + +def remap_source(source, anatomy): + success, rootless_path = ( + anatomy.find_root_template_from_path(source) + ) + if success: + source = rootless_path + else: + # `rootless_path` is not set to `source` if none of roots match + log = Logger.get_logger("farm_publishing") + log.warning(( + "Could not find root path for remapping \"{}\"." + " This may cause issues." + ).format(source)) + return source + +def get_skeleton_instance() + instance_skeleton_data = { + "family": family, + "subset": subset, + "families": families, + "asset": asset, + "frameStart": start, + "frameEnd": end, + "handleStart": handle_start, + "handleEnd": handle_end, + "frameStartHandle": start - handle_start, + "frameEndHandle": end + handle_end, + "comment": instance.data["comment"], + "fps": fps, + "source": source, + "extendFrames": data.get("extendFrames"), + "overrideExistingFrame": data.get("overrideExistingFrame"), + "pixelAspect": data.get("pixelAspect", 1), + "resolutionWidth": data.get("resolutionWidth", 1920), + "resolutionHeight": data.get("resolutionHeight", 1080), + "multipartExr": data.get("multipartExr", False), + "jobBatchName": data.get("jobBatchName", ""), + "useSequenceForReview": data.get("useSequenceForReview", True), + # map inputVersions `ObjectId` -> `str` so json supports it + "inputVersions": list(map(str, data.get("inputVersions", []))) + } \ No newline at end of file From 60eaf283ed0621aab5c289e5bd6537a80a9af0f2 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 18 Jan 2023 17:06:53 +0100 Subject: [PATCH 007/104] :bug: fix default for ShotGrid this is fixing default value for ShotGrid server enumerator after code refactor done in this branch --- openpype/settings/defaults/project_settings/shotgrid.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/settings/defaults/project_settings/shotgrid.json b/openpype/settings/defaults/project_settings/shotgrid.json index 774bce714b0..143a474b803 100644 --- a/openpype/settings/defaults/project_settings/shotgrid.json +++ b/openpype/settings/defaults/project_settings/shotgrid.json @@ -1,6 +1,6 @@ { "shotgrid_project_id": 0, - "shotgrid_server": "", + "shotgrid_server": [], "event": { "enabled": false }, From b4f1574d443f0abc963a946dcaa287faf8c3deac Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 18 Jan 2023 17:19:18 +0100 Subject: [PATCH 008/104] :rotating_light: hound fixes --- .../hosts/maya/plugins/create/create_render.py | 2 +- openpype/modules/royalrender/api.py | 2 -- .../publish/collect_rr_path_from_instance.py | 4 +--- .../plugins/publish/create_maya_royalrender_job.py | 14 ++++++++------ .../publish/create_publish_royalrender_job.py | 12 +++++------- .../plugins/publish/submit_jobs_to_royalrender.py | 4 +++- openpype/pipeline/farm/pyblish.py | 12 +++++++++--- 7 files changed, 27 insertions(+), 23 deletions(-) diff --git a/openpype/hosts/maya/plugins/create/create_render.py b/openpype/hosts/maya/plugins/create/create_render.py index dedb057fb7e..41893512383 100644 --- a/openpype/hosts/maya/plugins/create/create_render.py +++ b/openpype/hosts/maya/plugins/create/create_render.py @@ -278,7 +278,7 @@ def _create_render_settings(self): if deadline_enabled and muster_enabled and royalrender_enabled: self.log.error( - ("Multiple render farm support (Deadline/RoyalRender/Muster) " + ("Multiple render farm support (Deadline/RoyalRender/Muster) " "is enabled. We support only one at time.") ) raise RuntimeError("Both Deadline and Muster are enabled") diff --git a/openpype/modules/royalrender/api.py b/openpype/modules/royalrender/api.py index 8b13b9781f1..86d27ccc6cf 100644 --- a/openpype/modules/royalrender/api.py +++ b/openpype/modules/royalrender/api.py @@ -2,9 +2,7 @@ """Wrapper around Royal Render API.""" import sys import os -import platform -from openpype.settings import get_project_settings from openpype.lib.local_settings import OpenPypeSettingsRegistry from openpype.lib import Logger, run_subprocess from .rr_job import RRJob, SubmitFile, SubmitterParameter diff --git a/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py b/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py index 40f34561fa9..cfb5b78077f 100644 --- a/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py +++ b/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py @@ -44,6 +44,4 @@ def _collect_rr_path_name(instance): # Handle situation were we had only one url for royal render. return rr_settings["rr_paths"]["default"] - return list(rr_servers.keys())[ - int(instance.data.get("rrPaths")) - ] + return list(rr_servers.keys())[int(instance.data.get("rrPaths"))] diff --git a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py index e194a7edc60..5f427353ac8 100644 --- a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py @@ -47,11 +47,14 @@ def get_rr_platform(): output_dir = os.path.dirname(first_file) self._instance.data["outputDir"] = output_dir workspace = self._instance.context.data["workspaceDir"] - default_render_file = self._instance.context.data.get('project_settings') \ - .get('maya') \ - .get('RenderSettings') \ - .get('default_render_image_folder') - file_name = os.path.basename(self.scene_path) + default_render_file = ( + self._instance.context.data + ['project_settings'] + ['maya'] + ['RenderSettings'] + ['default_render_image_folder'] + ) + # file_name = os.path.basename(self.scene_path) dir_name = os.path.join(workspace, default_render_file) layer = self._instance.data["setMembers"] # type: str layer_name = layer.removeprefix("rs_") @@ -86,7 +89,6 @@ def process(self, instance): """Plugin entry point.""" self._instance = instance context = instance.context - from pprint import pformat self._rr_root = self._resolve_rr_path(context, instance.data.get("rrPathName")) # noqa self.log.debug(self._rr_root) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 9d8cc602c52..e62289641b3 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -26,12 +26,10 @@ def process(self, instance): self.context = context self.anatomy = instance.context.data["anatomy"] - asset = data.get("asset") - subset = data.get("subset") - source = self._remap_source( - data.get("source") or context.data["source"]) - - + # asset = data.get("asset") + # subset = data.get("subset") + # source = self._remap_source( + # data.get("source") or context.data["source"]) def _remap_source(self, source): success, rootless_path = ( @@ -175,4 +173,4 @@ def _submit_post_job(self, instance, job, instances): url = "{}/api/jobs".format(self.deadline_url) response = requests.post(url, json=payload, timeout=10) if not response.ok: - raise Exception(response.text) \ No newline at end of file + raise Exception(response.text) diff --git a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py index 4fcf3a08bd8..325fb36993d 100644 --- a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py +++ b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py @@ -38,7 +38,9 @@ def process(self, context): if isinstance(instance.data.get("rrJob"), RRJob): jobs.append(instance.data.get("rrJob")) if instance.data.get("rrJobs"): - if all(isinstance(job, RRJob) for job in instance.data.get("rrJobs")): + if all( + isinstance(job, RRJob) + for job in instance.data.get("rrJobs")): jobs += instance.data.get("rrJobs") if jobs: diff --git a/openpype/pipeline/farm/pyblish.py b/openpype/pipeline/farm/pyblish.py index 02535f40903..15f4356b860 100644 --- a/openpype/pipeline/farm/pyblish.py +++ b/openpype/pipeline/farm/pyblish.py @@ -4,7 +4,9 @@ @attr.s class InstanceSkeleton(object): - family = attr.ib(factory=) + # family = attr.ib(factory=) + pass + def remap_source(source, anatomy): success, rootless_path = ( @@ -21,7 +23,9 @@ def remap_source(source, anatomy): ).format(source)) return source -def get_skeleton_instance() + +def get_skeleton_instance(): + """ instance_skeleton_data = { "family": family, "subset": subset, @@ -46,4 +50,6 @@ def get_skeleton_instance() "useSequenceForReview": data.get("useSequenceForReview", True), # map inputVersions `ObjectId` -> `str` so json supports it "inputVersions": list(map(str, data.get("inputVersions", []))) - } \ No newline at end of file + } + """ + pass From 56f404e8f9a841d67c9af478ca3aa1ea8df98337 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 18 Jan 2023 17:21:28 +0100 Subject: [PATCH 009/104] :rotating_light: hound fixes 2 --- .../plugins/publish/create_publish_royalrender_job.py | 2 +- openpype/pipeline/farm/pyblish.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index e62289641b3..65de600bfae 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -21,7 +21,7 @@ class CreatePublishRoyalRenderJob(InstancePlugin): "celaction": [r".*"]} def process(self, instance): - data = instance.data.copy() + # data = instance.data.copy() context = instance.context self.context = context self.anatomy = instance.context.data["anatomy"] diff --git a/openpype/pipeline/farm/pyblish.py b/openpype/pipeline/farm/pyblish.py index 15f4356b860..436ad7f195e 100644 --- a/openpype/pipeline/farm/pyblish.py +++ b/openpype/pipeline/farm/pyblish.py @@ -17,10 +17,9 @@ def remap_source(source, anatomy): else: # `rootless_path` is not set to `source` if none of roots match log = Logger.get_logger("farm_publishing") - log.warning(( - "Could not find root path for remapping \"{}\"." - " This may cause issues." - ).format(source)) + log.warning( + ("Could not find root path for remapping \"{}\"." + " This may cause issues.").format(source)) return source From 61fe2ac36b2b8167554239c41d6e23eac051a638 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Tue, 7 Feb 2023 19:22:54 +0100 Subject: [PATCH 010/104] :construction: work on publish job --- openpype/modules/royalrender/api.py | 11 +- .../publish/create_maya_royalrender_job.py | 1 - .../publish/create_publish_royalrender_job.py | 104 ++++++++---------- openpype/modules/royalrender/rr_job.py | 46 +++++++- 4 files changed, 95 insertions(+), 67 deletions(-) diff --git a/openpype/modules/royalrender/api.py b/openpype/modules/royalrender/api.py index 86d27ccc6cf..e610a0c8a8e 100644 --- a/openpype/modules/royalrender/api.py +++ b/openpype/modules/royalrender/api.py @@ -20,19 +20,19 @@ def __init__(self, rr_path=None): self._rr_path = rr_path os.environ["RR_ROOT"] = rr_path - def _get_rr_bin_path(self, tool_name=None, rr_root=None): + @staticmethod + def get_rr_bin_path(rr_root, tool_name=None): # type: (str, str) -> str """Get path to RR bin folder. Args: tool_name (str): Name of RR executable you want. - rr_root (str, Optional): Custom RR root if needed. + rr_root (str): Custom RR root if needed. Returns: str: Path to the tool based on current platform. """ - rr_root = rr_root or self._rr_path is_64bit_python = sys.maxsize > 2 ** 32 rr_bin_parts = [rr_root, "bin"] @@ -65,7 +65,7 @@ def _initialize_module_path(self): # type: () -> None """Set RR modules for Python.""" # default for linux - rr_bin = self._get_rr_bin_path() + rr_bin = self.get_rr_bin_path(self._rr_path) rr_module_path = os.path.join(rr_bin, "lx64/lib") if sys.platform.lower() == "win32": @@ -109,7 +109,8 @@ def submit_file(self, file, mode=RR_SUBMIT_CONSOLE): def _submit_using_console(self, job_file): # type: (SubmitFile) -> None - rr_start_local = self._get_rr_bin_path("rrStartLocal") + rr_start_local = self.get_rr_bin_path( + self._rr_path, "rrStartLocal") self.log.info("rr_console: {}".format(rr_start_local)) diff --git a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py index 5f427353ac8..0b257d8b7a9 100644 --- a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py @@ -32,7 +32,6 @@ def get_job(self): Returns: RRJob: RoyalRender job payload. - """ def get_rr_platform(): if sys.platform.lower() in ["win32", "win64"]: diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 65de600bfae..f87ee589b64 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -6,6 +6,10 @@ import requests import os +from openpype.modules.royalrender.rr_job import RRJob, RREnvList +from openpype.pipeline.publish import KnownPublishError +from openpype.modules.royalrender.api import Api as rrApi + class CreatePublishRoyalRenderJob(InstancePlugin): label = "Create publish job in RR" @@ -45,14 +49,12 @@ def _remap_source(self, source): ).format(source)) return source - def _submit_post_job(self, instance, job, instances): + def get_job(self, instance, job, instances): """Submit publish job to RoyalRender.""" data = instance.data.copy() subset = data["subset"] job_name = "Publish - {subset}".format(subset=subset) - # instance.data.get("subset") != instances[0]["subset"] - # 'Main' vs 'renderMain' override_version = None instance_version = instance.data.get("version") # take this if exists if instance_version != 1: @@ -62,6 +64,8 @@ def _submit_post_job(self, instance, job, instances): deepcopy(instance.data["anatomyData"]), instance.data.get("asset"), instances[0]["subset"], + # TODO: this shouldn't be hardcoded and is in fact settable by + # Settings. 'render', override_version ) @@ -71,7 +75,7 @@ def _submit_post_job(self, instance, job, instances): metadata_path, roothless_metadata_path = \ self._create_metadata_path(instance) - environment = { + environment = RREnvList({ "AVALON_PROJECT": legacy_io.Session["AVALON_PROJECT"], "AVALON_ASSET": legacy_io.Session["AVALON_ASSET"], "AVALON_TASK": legacy_io.Session["AVALON_TASK"], @@ -80,7 +84,7 @@ def _submit_post_job(self, instance, job, instances): "OPENPYPE_RENDER_JOB": "0", "OPENPYPE_REMOTE_JOB": "0", "OPENPYPE_LOG_NO_COLORS": "1" - } + }) # add environments from self.environ_keys for env_key in self.environ_keys: @@ -88,7 +92,16 @@ def _submit_post_job(self, instance, job, instances): environment[env_key] = os.environ[env_key] # pass environment keys from self.environ_job_filter - job_environ = job["Props"].get("Env", {}) + # and collect all pre_ids to wait for + job_environ = {} + jobs_pre_ids = [] + for job in instance["rrJobs"]: # type: RRJob + if job.rrEnvList: + job_environ.update( + dict(RREnvList.parse(job.rrEnvList)) + ) + jobs_pre_ids.append(job.PreID) + for env_j_key in self.environ_job_filter: if job_environ.get(env_j_key): environment[env_j_key] = job_environ[env_j_key] @@ -99,7 +112,7 @@ def _submit_post_job(self, instance, job, instances): if mongo_url: environment["OPENPYPE_MONGO"] = mongo_url - priority = self.deadline_priority or instance.data.get("priority", 50) + priority = self.priority or instance.data.get("priority", 50) args = [ "--headless", @@ -109,66 +122,37 @@ def _submit_post_job(self, instance, job, instances): "--targets", "farm" ] - # Generate the payload for Deadline submission - payload = { - "JobInfo": { - "Plugin": self.deadline_plugin, - "BatchName": job["Props"]["Batch"], - "Name": job_name, - "UserName": job["Props"]["User"], - "Comment": instance.context.data.get("comment", ""), - - "Department": self.deadline_department, - "ChunkSize": self.deadline_chunk_size, - "Priority": priority, - - "Group": self.deadline_group, - "Pool": instance.data.get("primaryPool"), - "SecondaryPool": instance.data.get("secondaryPool"), - - "OutputDirectory0": output_dir - }, - "PluginInfo": { - "Version": self.plugin_pype_version, - "Arguments": " ".join(args), - "SingleFrameOnly": "True", - }, - # Mandatory for Deadline, may be empty - "AuxFiles": [], - } + job = RRJob( + Software="Execute", + Renderer="Once", + # path to OpenPype + SeqStart=1, + SeqEnd=1, + SeqStep=1, + SeqFileOffset=0, + Version="1.0", + SceneName="", + IsActive=True, + ImageFilename="execOnce.file", + ImageDir="", + ImageExtension="", + ImagePreNumberLetter="", + SceneOS=RRJob.get_rr_platform(), + rrEnvList=environment.serialize(), + Priority=priority + ) # add assembly jobs as dependencies if instance.data.get("tileRendering"): self.log.info("Adding tile assembly jobs as dependencies...") - job_index = 0 - for assembly_id in instance.data.get("assemblySubmissionJobs"): - payload["JobInfo"]["JobDependency{}".format(job_index)] = assembly_id # noqa: E501 - job_index += 1 + job.WaitForPreIDs += instance.data.get("assemblySubmissionJobs") elif instance.data.get("bakingSubmissionJobs"): self.log.info("Adding baking submission jobs as dependencies...") - job_index = 0 - for assembly_id in instance.data["bakingSubmissionJobs"]: - payload["JobInfo"]["JobDependency{}".format(job_index)] = assembly_id # noqa: E501 - job_index += 1 + job.WaitForPreIDs += instance.data["bakingSubmissionJobs"] else: - payload["JobInfo"]["JobDependency0"] = job["_id"] - - if instance.data.get("suspend_publish"): - payload["JobInfo"]["InitialStatus"] = "Suspended" - - for index, (key_, value_) in enumerate(environment.items()): - payload["JobInfo"].update( - { - "EnvironmentKeyValue%d" - % index: "{key}={value}".format( - key=key_, value=value_ - ) - } - ) - # remove secondary pool - payload["JobInfo"].pop("SecondaryPool", None) - - self.log.info("Submitting Deadline job ...") + job.WaitForPreIDs += jobs_pre_ids + + self.log.info("Creating RoyalRender Publish job ...") url = "{}/api/jobs".format(self.deadline_url) response = requests.post(url, json=payload, timeout=10) diff --git a/openpype/modules/royalrender/rr_job.py b/openpype/modules/royalrender/rr_job.py index f5c7033b625..21e5291bc3c 100644 --- a/openpype/modules/royalrender/rr_job.py +++ b/openpype/modules/royalrender/rr_job.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- """Python wrapper for RoyalRender XML job file.""" +import sys from xml.dom import minidom as md import attr from collections import namedtuple, OrderedDict @@ -8,6 +9,23 @@ CustomAttribute = namedtuple("CustomAttribute", ["name", "value"]) +class RREnvList(dict): + def serialize(self): + # VariableA=ValueA~~~VariableB=ValueB + return "~~~".join( + ["{}={}".format(k, v) for k, v in sorted(self.items())]) + + @staticmethod + def parse(data): + # type: (str) -> RREnvList + """Parse rrEnvList string and return it as RREnvList object.""" + out = RREnvList() + for var in data.split("~~~"): + k, v = data.split("=") + out[k] = v + return out + + @attr.s class RRJob: """Mapping of Royal Render job file to a data class.""" @@ -108,7 +126,7 @@ class RRJob: # jobs send from this machine. If a job with the PreID was found, then # this jobs waits for the other job. Note: This flag can be used multiple # times to wait for multiple jobs. - WaitForPreID = attr.ib(default=None) # type: int + WaitForPreIDs = attr.ib(factory=list) # type: list # List of submitter options per job # list item must be of `SubmitterParameter` type @@ -138,6 +156,21 @@ class RRJob: TotalFrames = attr.ib(default=None) # type: int Tiled = attr.ib(default=None) # type: str + # Environment + # only used in RR 8.3 and newer + rrEnvList = attr.ib(default=None) # type: str + + @staticmethod + def get_rr_platform(): + # type: () -> str + """Returns name of platform used in rr jobs.""" + if sys.platform.lower() in ["win32", "win64"]: + return "windows" + elif sys.platform.lower() == "darwin": + return "mac" + else: + return "linux" + class SubmitterParameter: """Wrapper for Submitter Parameters.""" @@ -242,6 +275,8 @@ def filter_data(a, v): job, dict_factory=OrderedDict, filter=filter_data) serialized_job.pop("CustomAttributes") serialized_job.pop("SubmitterParameters") + # we are handling `WaitForPreIDs` separately. + wait_pre_ids = serialized_job.pop("WaitForPreIDs", []) for custom_attr in job_custom_attributes: # type: CustomAttribute serialized_job["Custom{}".format( @@ -253,6 +288,15 @@ def filter_data(a, v): root.createTextNode(str(value)) ) xml_job.appendChild(xml_attr) + + # WaitForPreID - can be used multiple times + for pre_id in wait_pre_ids: + xml_attr = root.createElement("WaitForPreID") + xml_attr.appendChild( + root.createTextNode(str(pre_id)) + ) + xml_job.appendChild(xml_attr) + job_file.appendChild(xml_job) return root.toprettyxml(indent="\t") From 085d803558f894261b5cf24c19b95a2f8a4557d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Wed, 26 Oct 2022 18:37:44 +0200 Subject: [PATCH 011/104] :construction: wip on maya royalrender submit plugin --- openpype/modules/royalrender/api.py | 20 ++- .../publish/submit_maya_royalrender.py | 150 ++++++++++++++++++ openpype/modules/royalrender/rr_job.py | 8 +- 3 files changed, 163 insertions(+), 15 deletions(-) create mode 100644 openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py diff --git a/openpype/modules/royalrender/api.py b/openpype/modules/royalrender/api.py index de1dba8724b..c47d50b62b5 100644 --- a/openpype/modules/royalrender/api.py +++ b/openpype/modules/royalrender/api.py @@ -15,9 +15,8 @@ class Api: RR_SUBMIT_CONSOLE = 1 RR_SUBMIT_API = 2 - def __init__(self, settings, project=None): + def __init__(self, project=None): self.log = Logger.get_logger("RoyalRender") - self._settings = settings self._initialize_rr(project) def _initialize_rr(self, project=None): @@ -91,21 +90,21 @@ def _initialize_module_path(self): sys.path.append(os.path.join(self._rr_path, rr_module_path)) - def create_submission(self, jobs, submitter_attributes, file_name=None): - # type: (list[RRJob], list[SubmitterParameter], str) -> SubmitFile + @staticmethod + def create_submission(jobs, submitter_attributes): + # type: (list[RRJob], list[SubmitterParameter]) -> SubmitFile """Create jobs submission file. Args: jobs (list): List of :class:`RRJob` submitter_attributes (list): List of submitter attributes :class:`SubmitterParameter` for whole submission batch. - file_name (str), optional): File path to write data to. Returns: str: XML data of job submission files. """ - raise NotImplementedError + return SubmitFile(SubmitterParameters=submitter_attributes, Jobs=jobs) def submit_file(self, file, mode=RR_SUBMIT_CONSOLE): # type: (SubmitFile, int) -> None @@ -119,15 +118,14 @@ def submit_file(self, file, mode=RR_SUBMIT_CONSOLE): # self._submit_using_api(file) def _submit_using_console(self, file): - # type: (SubmitFile) -> bool + # type: (SubmitFile) -> None rr_console = os.path.join( self._get_rr_bin_path(), - "rrSubmitterconsole" + "rrSubmitterConsole" ) - if sys.platform.lower() == "darwin": - if "/bin/mac64" in rr_console: - rr_console = rr_console.replace("/bin/mac64", "/bin/mac") + if sys.platform.lower() == "darwin" and "/bin/mac64" in rr_console: + rr_console = rr_console.replace("/bin/mac64", "/bin/mac") if sys.platform.lower() == "win32": if "/bin/win64" in rr_console: diff --git a/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py new file mode 100644 index 00000000000..c354cc80a0c --- /dev/null +++ b/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py @@ -0,0 +1,150 @@ +# -*- coding: utf-8 -*- +"""Submitting render job to RoyalRender.""" +import os +import sys +import tempfile + +from maya.OpenMaya import MGlobal # noqa +from pyblish.api import InstancePlugin, IntegratorOrder +from openpype.hosts.maya.api.lib import get_attr_in_layer +from openpype.pipeline.farm.tools import get_published_workfile_instance +from openpype.pipeline.publish import KnownPublishError +from openpype.modules.royalrender.api import Api as rr_api +from openpype.modules.royalrender.rr_job import RRJob, SubmitterParameter + + +class MayaSubmitRoyalRender(InstancePlugin): + label = "Submit to RoyalRender" + order = IntegratorOrder + 0.1 + use_published = True + + def __init__(self, *args, **kwargs): + self._instance = None + self._rrRoot = None + self.scene_path = None + self.job = None + self.submission_parameters = None + self.rr_api = None + + def get_job(self): + """Prepare job payload. + + Returns: + RRJob: RoyalRender job payload. + + """ + def get_rr_platform(): + if sys.platform.lower() in ["win32", "win64"]: + return "win" + elif sys.platform.lower() == "darwin": + return "mac" + else: + return "lx" + + expected_files = self._instance.data["expectedFiles"] + first_file = next(self._iter_expected_files(expected_files)) + output_dir = os.path.dirname(first_file) + self._instance.data["outputDir"] = output_dir + workspace = self._instance.context.data["workspaceDir"] + default_render_file = self._instance.context.data.get('project_settings') \ + .get('maya') \ + .get('RenderSettings') \ + .get('default_render_image_folder') + filename = os.path.basename(self.scene_path) + dirname = os.path.join(workspace, default_render_file) + + job = RRJob( + Software="Maya", + Renderer=self._instance.data["renderer"], + SeqStart=int(self._instance.data["frameStartHandle"]), + SeqEnd=int(self._instance.data["frameEndHandle"]), + SeqStep=int(self._instance.data["byFrameStep"]), + SeqFileOffset=0, + Version="{0:.2f}".format(MGlobal.apiVersion() / 10000), + SceneName=os.path.basename(self.scene_path), + IsActive=True, + ImageDir=dirname, + ImageFilename=filename, + ImageExtension="." + os.path.splitext(filename)[1], + ImagePreNumberLetter=".", + ImageSingleOutputFile=False, + SceneOS=get_rr_platform(), + Camera=self._instance.data["cameras"][0], + Layer=self._instance.data["layer"], + SceneDatabaseDir=workspace, + ImageFramePadding=get_attr_in_layer( + "defaultRenderGlobals.extensionPadding", + self._instance.data["layer"]), + ImageWidth=self._instance.data["resolutionWidth"], + ImageHeight=self._instance.data["resolutionHeight"] + ) + return job + + @staticmethod + def get_submission_parameters(): + return [] + + def create_file(self, name, ext, contents=None): + temp = tempfile.NamedTemporaryFile( + dir=self.tempdir, + suffix=ext, + prefix=name + '.', + delete=False, + ) + + if contents: + with open(temp.name, 'w') as f: + f.write(contents) + + return temp.name + + def process(self, instance): + """Plugin entry point.""" + self._instance = instance + context = instance.context + self.rr_api = rr_api(context.data["project"]) + + # get royalrender module + """ + try: + rr_module = context.data.get( + "openPypeModules")["royalrender"] + except AttributeError: + self.log.error("Cannot get OpenPype RoyalRender module.") + raise AssertionError("OpenPype RoyalRender module not found.") + """ + + self._rrRoot = instance.data["rrPath"] or context.data["defaultRRPath"] # noqa + if not self._rrRoot: + raise KnownPublishError( + ("Missing RoyalRender root. " + "You need to configure RoyalRender module.")) + file_path = None + if self.use_published: + file_path = get_published_workfile_instance() + + # fallback if nothing was set + if not file_path: + self.log.warning("Falling back to workfile") + file_path = context.data["currentFile"] + + self.scene_path = file_path + self.job = self.get_job() + self.log.info(self.job) + self.submission_parameters = self.get_submission_parameters() + + self.process_submission() + + def process_submission(self): + submission = rr_api.create_submission( + [self.job], + self.submission_parameters) + + self.log.debug(submission) + xml = tempfile.NamedTemporaryFile(suffix=".xml", delete=False) + with open(xml.name, "w") as f: + f.write(submission.serialize()) + + self.rr_api.submit_file(file=xml) + + diff --git a/openpype/modules/royalrender/rr_job.py b/openpype/modules/royalrender/rr_job.py index c660eceac76..beb8c17187e 100644 --- a/openpype/modules/royalrender/rr_job.py +++ b/openpype/modules/royalrender/rr_job.py @@ -35,7 +35,7 @@ class RRJob: # Is the job enabled for submission? # enabled by default - IsActive = attr.ib() # type: str + IsActive = attr.ib() # type: bool # Sequence settings of this job SeqStart = attr.ib() # type: int @@ -60,7 +60,7 @@ class RRJob: # If you render a single file, e.g. Quicktime or Avi, then you have to # set this value. Videos have to be rendered at once on one client. - ImageSingleOutputFile = attr.ib(default="false") # type: str + ImageSingleOutputFile = attr.ib(default=False) # type: bool # Semi-Required (required for some render applications) # ----------------------------------------------------- @@ -169,11 +169,11 @@ class SubmitFile: # Delete submission file after processing DeleteXML = attr.ib(default=1) # type: int - # List of submitter options per job + # List of the submitter options per job. # list item must be of `SubmitterParameter` type SubmitterParameters = attr.ib(factory=list) # type: list - # List of job is submission batch. + # List of the jobs in submission batch. # list item must be of type `RRJob` Jobs = attr.ib(factory=list) # type: list From 77315d301c79fe0125bcaab624a8e0ef405f312c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Wed, 26 Oct 2022 18:38:11 +0200 Subject: [PATCH 012/104] :recycle: move functions to common lib --- .../deadline/abstract_submit_deadline.py | 111 +----------------- openpype/pipeline/farm/tools.py | 109 +++++++++++++++++ 2 files changed, 111 insertions(+), 109 deletions(-) create mode 100644 openpype/pipeline/farm/tools.py diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 648eb77007b..5d1b703b773 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -21,6 +21,7 @@ AbstractMetaInstancePlugin, KnownPublishError ) +from openpype.pipeline.farm.tools import get_published_workfile_instance JSONDecodeError = getattr(json.decoder, "JSONDecodeError", ValueError) @@ -426,7 +427,7 @@ def process(self, instance): file_path = None if self.use_published: if not self.import_reference: - file_path = self.from_published_scene() + file_path = get_published_workfile_instance(instance) else: self.log.info("use the scene with imported reference for rendering") # noqa file_path = context.data["currentFile"] @@ -500,95 +501,6 @@ def get_aux_files(self): """ return [] - def from_published_scene(self, replace_in_path=True): - """Switch work scene for published scene. - - If rendering/exporting from published scenes is enabled, this will - replace paths from working scene to published scene. - - Args: - replace_in_path (bool): if True, it will try to find - old scene name in path of expected files and replace it - with name of published scene. - - Returns: - str: Published scene path. - None: if no published scene is found. - - Note: - Published scene path is actually determined from project Anatomy - as at the time this plugin is running scene can still no be - published. - - """ - instance = self._instance - workfile_instance = self._get_workfile_instance(instance.context) - if workfile_instance is None: - return - - # determine published path from Anatomy. - template_data = workfile_instance.data.get("anatomyData") - rep = workfile_instance.data["representations"][0] - template_data["representation"] = rep.get("name") - template_data["ext"] = rep.get("ext") - template_data["comment"] = None - - anatomy = instance.context.data['anatomy'] - anatomy_filled = anatomy.format(template_data) - template_filled = anatomy_filled["publish"]["path"] - file_path = os.path.normpath(template_filled) - - self.log.info("Using published scene for render {}".format(file_path)) - - if not os.path.exists(file_path): - self.log.error("published scene does not exist!") - raise - - if not replace_in_path: - return file_path - - # now we need to switch scene in expected files - # because token will now point to published - # scene file and that might differ from current one - def _clean_name(path): - return os.path.splitext(os.path.basename(path))[0] - - new_scene = _clean_name(file_path) - orig_scene = _clean_name(instance.context.data["currentFile"]) - expected_files = instance.data.get("expectedFiles") - - if isinstance(expected_files[0], dict): - # we have aovs and we need to iterate over them - new_exp = {} - for aov, files in expected_files[0].items(): - replaced_files = [] - for f in files: - replaced_files.append( - str(f).replace(orig_scene, new_scene) - ) - new_exp[aov] = replaced_files - # [] might be too much here, TODO - instance.data["expectedFiles"] = [new_exp] - else: - new_exp = [] - for f in expected_files: - new_exp.append( - str(f).replace(orig_scene, new_scene) - ) - instance.data["expectedFiles"] = new_exp - - metadata_folder = instance.data.get("publishRenderMetadataFolder") - if metadata_folder: - metadata_folder = metadata_folder.replace(orig_scene, - new_scene) - instance.data["publishRenderMetadataFolder"] = metadata_folder - - self.log.info("Scene name was switched {} -> {}".format( - orig_scene, new_scene - )) - - return file_path - def assemble_payload( self, job_info=None, plugin_info=None, aux_files=None): """Assemble payload data from its various parts. @@ -648,22 +560,3 @@ def submit(self, payload): self._instance.data["deadlineSubmissionJob"] = result return result["_id"] - - @staticmethod - def _get_workfile_instance(context): - """Find workfile instance in context""" - for i in context: - - is_workfile = ( - "workfile" in i.data.get("families", []) or - i.data["family"] == "workfile" - ) - if not is_workfile: - continue - - # test if there is instance of workfile waiting - # to be published. - assert i.data["publish"] is True, ( - "Workfile (scene) must be published along") - - return i diff --git a/openpype/pipeline/farm/tools.py b/openpype/pipeline/farm/tools.py new file mode 100644 index 00000000000..8cf1af399ea --- /dev/null +++ b/openpype/pipeline/farm/tools.py @@ -0,0 +1,109 @@ +import os + + +def get_published_workfile_instance(context): + """Find workfile instance in context""" + for i in context: + is_workfile = ( + "workfile" in i.data.get("families", []) or + i.data["family"] == "workfile" + ) + if not is_workfile: + continue + + # test if there is instance of workfile waiting + # to be published. + if i.data["publish"] is not True: + continue + + return i + + +def from_published_scene(instance, replace_in_path=True): + """Switch work scene for published scene. + + If rendering/exporting from published scenes is enabled, this will + replace paths from working scene to published scene. + + Args: + instance (pyblish.api.Instance): Instance data to process. + replace_in_path (bool): if True, it will try to find + old scene name in path of expected files and replace it + with name of published scene. + + Returns: + str: Published scene path. + None: if no published scene is found. + + Note: + Published scene path is actually determined from project Anatomy + as at the time this plugin is running the scene can be still + un-published. + + """ + workfile_instance = get_published_workfile_instance(instance.context) + if workfile_instance is None: + return + + # determine published path from Anatomy. + template_data = workfile_instance.data.get("anatomyData") + rep = workfile_instance.data.get("representations")[0] + template_data["representation"] = rep.get("name") + template_data["ext"] = rep.get("ext") + template_data["comment"] = None + + anatomy = instance.context.data['anatomy'] + anatomy_filled = anatomy.format(template_data) + template_filled = anatomy_filled["publish"]["path"] + file_path = os.path.normpath(template_filled) + + self.log.info("Using published scene for render {}".format(file_path)) + + if not os.path.exists(file_path): + self.log.error("published scene does not exist!") + raise + + if not replace_in_path: + return file_path + + # now we need to switch scene in expected files + # because token will now point to published + # scene file and that might differ from current one + def _clean_name(path): + return os.path.splitext(os.path.basename(path))[0] + + new_scene = _clean_name(file_path) + orig_scene = _clean_name(instance.context.data["currentFile"]) + expected_files = instance.data.get("expectedFiles") + + if isinstance(expected_files[0], dict): + # we have aovs and we need to iterate over them + new_exp = {} + for aov, files in expected_files[0].items(): + replaced_files = [] + for f in files: + replaced_files.append( + str(f).replace(orig_scene, new_scene) + ) + new_exp[aov] = replaced_files + # [] might be too much here, TODO + instance.data["expectedFiles"] = [new_exp] + else: + new_exp = [] + for f in expected_files: + new_exp.append( + str(f).replace(orig_scene, new_scene) + ) + instance.data["expectedFiles"] = new_exp + + metadata_folder = instance.data.get("publishRenderMetadataFolder") + if metadata_folder: + metadata_folder = metadata_folder.replace(orig_scene, + new_scene) + instance.data["publishRenderMetadataFolder"] = metadata_folder + + self.log.info("Scene name was switched {} -> {}".format( + orig_scene, new_scene + )) + + return file_path From ca552a701816b2d0fec6c1ee282fbb89f79b5db9 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 11 Jan 2023 19:13:28 +0100 Subject: [PATCH 013/104] :construction: redoing publishing flow for multiple rr roots --- .../maya/plugins/create/create_render.py | 89 +++++++++++++++---- .../maya/plugins/publish/collect_render.py | 7 ++ openpype/modules/royalrender/api.py | 30 +------ .../publish/collect_default_rr_path.py | 23 ----- .../publish/collect_rr_path_from_instance.py | 16 ++-- .../publish/collect_sequences_from_job.py | 2 +- .../publish/submit_maya_royalrender.py | 51 +++++++++-- .../project_settings/royalrender.json | 3 + .../defaults/system_settings/modules.json | 6 +- 9 files changed, 138 insertions(+), 89 deletions(-) delete mode 100644 openpype/modules/royalrender/plugins/publish/collect_default_rr_path.py diff --git a/openpype/hosts/maya/plugins/create/create_render.py b/openpype/hosts/maya/plugins/create/create_render.py index 387b7321b9f..337868d47d2 100644 --- a/openpype/hosts/maya/plugins/create/create_render.py +++ b/openpype/hosts/maya/plugins/create/create_render.py @@ -80,31 +80,58 @@ def __init__(self, *args, **kwargs): if self._project_settings["maya"]["RenderSettings"]["apply_render_settings"]: # noqa lib_rendersettings.RenderSettings().set_default_renderer_settings() - # Deadline-only + # Handling farms manager = ModulesManager() deadline_settings = get_system_settings()["modules"]["deadline"] - if not deadline_settings["enabled"]: - self.deadline_servers = {} + rr_settings = get_system_settings()["modules"]["royalrender"] + + self.deadline_servers = {} + self.rr_paths = {} + + if deadline_settings["enabled"]: + self.deadline_module = manager.modules_by_name["deadline"] + try: + default_servers = deadline_settings["deadline_urls"] + project_servers = ( + self._project_settings["deadline"]["deadline_servers"] + ) + self.deadline_servers = { + k: default_servers[k] + for k in project_servers + if k in default_servers + } + + if not self.deadline_servers: + self.deadline_servers = default_servers + + except AttributeError: + # Handle situation were we had only one url for deadline. + # get default deadline webservice url from deadline module + self.deadline_servers = self.deadline_module.deadline_urls + + # RoyalRender only + if not rr_settings["enabled"]: return - self.deadline_module = manager.modules_by_name["deadline"] + + self.rr_module = manager.modules_by_name["royalrender"] try: - default_servers = deadline_settings["deadline_urls"] - project_servers = ( - self._project_settings["deadline"]["deadline_servers"] + default_paths = rr_settings["rr_paths"] + project_paths = ( + self._project_settings["royalrender"]["rr_paths"] ) - self.deadline_servers = { - k: default_servers[k] - for k in project_servers - if k in default_servers + self.rr_paths = { + k: default_paths[k] + for k in project_paths + if k in default_paths } - if not self.deadline_servers: - self.deadline_servers = default_servers + if not self.rr_paths: + self.rr_paths = default_paths except AttributeError: - # Handle situation were we had only one url for deadline. - # get default deadline webservice url from deadline module - self.deadline_servers = self.deadline_module.deadline_urls + # Handle situation were we had only one path for royalrender. + # Get default royalrender root path from the rr module. + self.rr_paths = self.rr_module.rr_paths def process(self): """Entry point.""" @@ -140,6 +167,14 @@ def process(self): self._deadline_webservice_changed ]) + # add RoyalRender root path selection list + if self.rr_paths: + cmds.scriptJob( + attributeChange=[ + "{}.rrPaths".format(self.instance), + self._rr_path_changed + ]) + cmds.setAttr("{}.machineList".format(self.instance), lock=True) rs = renderSetup.instance() layers = rs.getRenderLayers() @@ -192,6 +227,18 @@ def _deadline_webservice_changed(self): attributeType="enum", enumName=":".join(sorted_pools)) + @staticmethod + def _rr_path_changed(): + """Unused callback to pull information from RR.""" + """ + _ = self.rr_paths[ + self.server_aliases[ + cmds.getAttr("{}.rrPaths".format(self.instance)) + ] + ] + """ + pass + def _create_render_settings(self): """Create instance settings.""" # get pools (slave machines of the render farm) @@ -226,15 +273,21 @@ def _create_render_settings(self): system_settings = get_system_settings()["modules"] deadline_enabled = system_settings["deadline"]["enabled"] + royalrender_enabled = system_settings["royalrender"]["enabled"] muster_enabled = system_settings["muster"]["enabled"] muster_url = system_settings["muster"]["MUSTER_REST_URL"] - if deadline_enabled and muster_enabled: + if deadline_enabled and muster_enabled and royalrender_enabled: self.log.error( - "Both Deadline and Muster are enabled. " "Cannot support both." + ("Multiple render farm support (Deadline/RoyalRender/Muster) " + "is enabled. We support only one at time.") ) raise RuntimeError("Both Deadline and Muster are enabled") + if royalrender_enabled: + self.server_aliases = list(self.rr_paths.keys()) + self.data["rrPaths"] = self.server_aliases + if deadline_enabled: self.server_aliases = list(self.deadline_servers.keys()) self.data["deadlineServers"] = self.server_aliases diff --git a/openpype/hosts/maya/plugins/publish/collect_render.py b/openpype/hosts/maya/plugins/publish/collect_render.py index 7c47f17acba..2fb55782d2c 100644 --- a/openpype/hosts/maya/plugins/publish/collect_render.py +++ b/openpype/hosts/maya/plugins/publish/collect_render.py @@ -338,6 +338,13 @@ def process(self, context): if deadline_settings["enabled"]: data["deadlineUrl"] = render_instance.data.get("deadlineUrl") + rr_settings = ( + context.data["system_settings"]["modules"]["royalrender"] + ) + if rr_settings["enabled"]: + data["rrPathName"] = render_instance.data.get("rrPathName") + self.log.info(data["rrPathName"]) + if self.sync_workfile_version: data["version"] = context.data["version"] diff --git a/openpype/modules/royalrender/api.py b/openpype/modules/royalrender/api.py index c47d50b62b5..dcb518deb1c 100644 --- a/openpype/modules/royalrender/api.py +++ b/openpype/modules/royalrender/api.py @@ -15,36 +15,10 @@ class Api: RR_SUBMIT_CONSOLE = 1 RR_SUBMIT_API = 2 - def __init__(self, project=None): + def __init__(self, rr_path=None): self.log = Logger.get_logger("RoyalRender") - self._initialize_rr(project) - - def _initialize_rr(self, project=None): - # type: (str) -> None - """Initialize RR Path. - - Args: - project (str, Optional): Project name to set RR api in - context. - - """ - if project: - project_settings = get_project_settings(project) - rr_path = ( - project_settings - ["royalrender"] - ["rr_paths"] - ) - else: - rr_path = ( - self._settings - ["modules"] - ["royalrender"] - ["rr_path"] - ["default"] - ) - os.environ["RR_ROOT"] = rr_path self._rr_path = rr_path + os.environ["RR_ROOT"] = rr_path def _get_rr_bin_path(self, rr_root=None): # type: (str) -> str diff --git a/openpype/modules/royalrender/plugins/publish/collect_default_rr_path.py b/openpype/modules/royalrender/plugins/publish/collect_default_rr_path.py deleted file mode 100644 index 3ce95e0c50e..00000000000 --- a/openpype/modules/royalrender/plugins/publish/collect_default_rr_path.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- -"""Collect default Deadline server.""" -import pyblish.api - - -class CollectDefaultRRPath(pyblish.api.ContextPlugin): - """Collect default Royal Render path.""" - - order = pyblish.api.CollectorOrder - label = "Default Royal Render Path" - - def process(self, context): - try: - rr_module = context.data.get( - "openPypeModules")["royalrender"] - except AttributeError: - msg = "Cannot get OpenPype Royal Render module." - self.log.error(msg) - raise AssertionError(msg) - - # get default deadline webservice url from deadline module - self.log.debug(rr_module.rr_paths) - context.data["defaultRRPath"] = rr_module.rr_paths["default"] # noqa: E501 diff --git a/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py b/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py index 6a3dc276f37..187e2b9c446 100644 --- a/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py +++ b/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py @@ -5,19 +5,19 @@ class CollectRRPathFromInstance(pyblish.api.InstancePlugin): """Collect RR Path from instance.""" - order = pyblish.api.CollectorOrder + 0.01 - label = "Royal Render Path from the Instance" + order = pyblish.api.CollectorOrder + label = "Collect Royal Render path name from the Instance" families = ["rendering"] def process(self, instance): - instance.data["rrPath"] = self._collect_rr_path(instance) + instance.data["rrPathName"] = self._collect_rr_path_name(instance) self.log.info( - "Using {} for submission.".format(instance.data["rrPath"])) + "Using '{}' for submission.".format(instance.data["rrPathName"])) @staticmethod - def _collect_rr_path(render_instance): + def _collect_rr_path_name(render_instance): # type: (pyblish.api.Instance) -> str - """Get Royal Render path from render instance.""" + """Get Royal Render pat name from render instance.""" rr_settings = ( render_instance.context.data ["system_settings"] @@ -42,8 +42,6 @@ def _collect_rr_path(render_instance): # Handle situation were we had only one url for royal render. return render_instance.context.data["defaultRRPath"] - return rr_servers[ - list(rr_servers.keys())[ + return list(rr_servers.keys())[ int(render_instance.data.get("rrPaths")) ] - ] diff --git a/openpype/modules/royalrender/plugins/publish/collect_sequences_from_job.py b/openpype/modules/royalrender/plugins/publish/collect_sequences_from_job.py index 65af90e8a61..4c123e4134b 100644 --- a/openpype/modules/royalrender/plugins/publish/collect_sequences_from_job.py +++ b/openpype/modules/royalrender/plugins/publish/collect_sequences_from_job.py @@ -71,7 +71,7 @@ class CollectSequencesFromJob(pyblish.api.ContextPlugin): """Gather file sequences from job directory. When "OPENPYPE_PUBLISH_DATA" environment variable is set these paths - (folders or .json files) are parsed for image sequences. Otherwise the + (folders or .json files) are parsed for image sequences. Otherwise, the current working directory is searched for file sequences. """ diff --git a/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py index c354cc80a0c..784e4c5ff9f 100644 --- a/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py +++ b/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py @@ -3,9 +3,10 @@ import os import sys import tempfile +import platform from maya.OpenMaya import MGlobal # noqa -from pyblish.api import InstancePlugin, IntegratorOrder +from pyblish.api import InstancePlugin, IntegratorOrder, Context from openpype.hosts.maya.api.lib import get_attr_in_layer from openpype.pipeline.farm.tools import get_published_workfile_instance from openpype.pipeline.publish import KnownPublishError @@ -16,6 +17,8 @@ class MayaSubmitRoyalRender(InstancePlugin): label = "Submit to RoyalRender" order = IntegratorOrder + 0.1 + families = ["renderlayer"] + targets = ["local"] use_published = True def __init__(self, *args, **kwargs): @@ -102,7 +105,16 @@ def process(self, instance): """Plugin entry point.""" self._instance = instance context = instance.context - self.rr_api = rr_api(context.data["project"]) + from pprint import pformat + + self._rr_root = self._resolve_rr_path(context, instance.data.get("rrPathName")) # noqa + self.log.debug(self._rr_root) + if not self._rr_root: + raise KnownPublishError( + ("Missing RoyalRender root. " + "You need to configure RoyalRender module.")) + + self.rr_api = rr_api(self._rr_root) # get royalrender module """ @@ -114,11 +126,7 @@ def process(self, instance): raise AssertionError("OpenPype RoyalRender module not found.") """ - self._rrRoot = instance.data["rrPath"] or context.data["defaultRRPath"] # noqa - if not self._rrRoot: - raise KnownPublishError( - ("Missing RoyalRender root. " - "You need to configure RoyalRender module.")) + file_path = None if self.use_published: file_path = get_published_workfile_instance() @@ -147,4 +155,33 @@ def process_submission(self): self.rr_api.submit_file(file=xml) + @staticmethod + def _resolve_rr_path(context, rr_path_name): + # type: (Context, str) -> str + rr_settings = ( + context.data + ["system_settings"] + ["modules"] + ["royalrender"] + ) + try: + default_servers = rr_settings["rr_paths"] + project_servers = ( + context.data + ["project_settings"] + ["royalrender"] + ["rr_paths"] + ) + rr_servers = { + k: default_servers[k] + for k in project_servers + if k in default_servers + } + + except (AttributeError, KeyError): + # Handle situation were we had only one url for royal render. + return context.data["defaultRRPath"][platform.system().lower()] + + return rr_servers[rr_path_name][platform.system().lower()] + diff --git a/openpype/settings/defaults/project_settings/royalrender.json b/openpype/settings/defaults/project_settings/royalrender.json index b72fed84745..14e36058aaf 100644 --- a/openpype/settings/defaults/project_settings/royalrender.json +++ b/openpype/settings/defaults/project_settings/royalrender.json @@ -1,4 +1,7 @@ { + "rr_paths": [ + "default" + ], "publish": { "CollectSequencesFromJob": { "review": true diff --git a/openpype/settings/defaults/system_settings/modules.json b/openpype/settings/defaults/system_settings/modules.json index 1ddbfd2726c..f524f01d455 100644 --- a/openpype/settings/defaults/system_settings/modules.json +++ b/openpype/settings/defaults/system_settings/modules.json @@ -185,9 +185,9 @@ "enabled": false, "rr_paths": { "default": { - "windows": "", - "darwin": "", - "linux": "" + "windows": "C:\\RR8", + "darwin": "/Volumes/share/RR8", + "linux": "/mnt/studio/RR8" } } }, From 7e4b3cb3af9041c6e43d329410469fc4dbae1632 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 11 Jan 2023 19:13:54 +0100 Subject: [PATCH 014/104] :recycle: optimizing enum classes --- openpype/settings/entities/__init__.py | 4 +- openpype/settings/entities/enum_entity.py | 81 ++++++++++--------- .../schema_project_royalrender.json | 6 ++ 3 files changed, 53 insertions(+), 38 deletions(-) diff --git a/openpype/settings/entities/__init__.py b/openpype/settings/entities/__init__.py index 5e3a76094e4..00db2b33a7d 100644 --- a/openpype/settings/entities/__init__.py +++ b/openpype/settings/entities/__init__.py @@ -107,7 +107,8 @@ TaskTypeEnumEntity, DeadlineUrlEnumEntity, AnatomyTemplatesEnumEntity, - ShotgridUrlEnumEntity + ShotgridUrlEnumEntity, + RoyalRenderRootEnumEntity ) from .list_entity import ListEntity @@ -170,6 +171,7 @@ "TaskTypeEnumEntity", "DeadlineUrlEnumEntity", "ShotgridUrlEnumEntity", + "RoyalRenderRootEnumEntity", "AnatomyTemplatesEnumEntity", "ListEntity", diff --git a/openpype/settings/entities/enum_entity.py b/openpype/settings/entities/enum_entity.py index c0c103ea10e..9f9ae930264 100644 --- a/openpype/settings/entities/enum_entity.py +++ b/openpype/settings/entities/enum_entity.py @@ -1,3 +1,5 @@ +import abc +import six import copy from .input_entities import InputEntity from .exceptions import EntitySchemaError @@ -476,8 +478,9 @@ def set_override_state(self, *args, **kwargs): self.set(value_on_not_set) -class DeadlineUrlEnumEntity(BaseEnumEntity): - schema_types = ["deadline_url-enum"] +@six.add_metaclass(abc.ABCMeta) +class FarmRootEnumEntity(BaseEnumEntity): + schema_types = [] def _item_initialization(self): self.multiselection = self.schema_data.get("multiselection", True) @@ -495,22 +498,8 @@ def _item_initialization(self): # GUI attribute self.placeholder = self.schema_data.get("placeholder") - def _get_enum_values(self): - deadline_urls_entity = self.get_entity_from_path( - "system_settings/modules/deadline/deadline_urls" - ) - - valid_keys = set() - enum_items_list = [] - for server_name, url_entity in deadline_urls_entity.items(): - enum_items_list.append( - {server_name: "{}: {}".format(server_name, url_entity.value)} - ) - valid_keys.add(server_name) - return enum_items_list, valid_keys - def set_override_state(self, *args, **kwargs): - super(DeadlineUrlEnumEntity, self).set_override_state(*args, **kwargs) + super(FarmRootEnumEntity, self).set_override_state(*args, **kwargs) self.enum_items, self.valid_keys = self._get_enum_values() if self.multiselection: @@ -527,21 +516,49 @@ def set_override_state(self, *args, **kwargs): elif self._current_value not in self.valid_keys: self._current_value = tuple(self.valid_keys)[0] + @abc.abstractmethod + def _get_enum_values(self): + pass -class ShotgridUrlEnumEntity(BaseEnumEntity): - schema_types = ["shotgrid_url-enum"] - def _item_initialization(self): - self.multiselection = False +class DeadlineUrlEnumEntity(FarmRootEnumEntity): + schema_types = ["deadline_url-enum"] - self.enum_items = [] - self.valid_keys = set() + def _get_enum_values(self): + deadline_urls_entity = self.get_entity_from_path( + "system_settings/modules/deadline/deadline_urls" + ) - self.valid_value_types = (STRING_TYPE,) - self.value_on_not_set = "" + valid_keys = set() + enum_items_list = [] + for server_name, url_entity in deadline_urls_entity.items(): + enum_items_list.append( + {server_name: "{}: {}".format(server_name, url_entity.value)} + ) + valid_keys.add(server_name) + return enum_items_list, valid_keys - # GUI attribute - self.placeholder = self.schema_data.get("placeholder") + +class RoyalRenderRootEnumEntity(FarmRootEnumEntity): + schema_types = ["rr_root-enum"] + + def _get_enum_values(self): + rr_root_entity = self.get_entity_from_path( + "system_settings/modules/royalrender/rr_paths" + ) + + valid_keys = set() + enum_items_list = [] + for server_name, url_entity in rr_root_entity.items(): + enum_items_list.append( + {server_name: "{}: {}".format(server_name, url_entity.value)} + ) + valid_keys.add(server_name) + return enum_items_list, valid_keys + + +class ShotgridUrlEnumEntity(FarmRootEnumEntity): + schema_types = ["shotgrid_url-enum"] def _get_enum_values(self): shotgrid_settings = self.get_entity_from_path( @@ -561,16 +578,6 @@ def _get_enum_values(self): valid_keys.add(server_name) return enum_items_list, valid_keys - def set_override_state(self, *args, **kwargs): - super(ShotgridUrlEnumEntity, self).set_override_state(*args, **kwargs) - - self.enum_items, self.valid_keys = self._get_enum_values() - if not self.valid_keys: - self._current_value = "" - - elif self._current_value not in self.valid_keys: - self._current_value = tuple(self.valid_keys)[0] - class AnatomyTemplatesEnumEntity(BaseEnumEntity): schema_types = ["anatomy-templates-enum"] diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_royalrender.json b/openpype/settings/entities/schemas/projects_schema/schema_project_royalrender.json index cabb4747d5e..f4bf2f51ba6 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_royalrender.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_royalrender.json @@ -5,6 +5,12 @@ "collapsible": true, "is_file": true, "children": [ + { + "type": "rr_root-enum", + "key": "rr_paths", + "label": "Royal Render Roots", + "multiselect": true + }, { "type": "dict", "collapsible": true, From 7d212c051b34ad505dae4132903bc4b2f94998fd Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Mon, 16 Jan 2023 10:40:05 +0100 Subject: [PATCH 015/104] :construction: render job submission --- openpype/modules/royalrender/api.py | 76 +++++++++++-------- .../publish/submit_maya_royalrender.py | 45 ++++++----- openpype/modules/royalrender/rr_job.py | 10 ++- 3 files changed, 77 insertions(+), 54 deletions(-) diff --git a/openpype/modules/royalrender/api.py b/openpype/modules/royalrender/api.py index dcb518deb1c..8b13b9781f1 100644 --- a/openpype/modules/royalrender/api.py +++ b/openpype/modules/royalrender/api.py @@ -2,11 +2,13 @@ """Wrapper around Royal Render API.""" import sys import os +import platform from openpype.settings import get_project_settings from openpype.lib.local_settings import OpenPypeSettingsRegistry from openpype.lib import Logger, run_subprocess from .rr_job import RRJob, SubmitFile, SubmitterParameter +from openpype.lib.vendor_bin_utils import find_tool_in_custom_paths class Api: @@ -20,31 +22,46 @@ def __init__(self, rr_path=None): self._rr_path = rr_path os.environ["RR_ROOT"] = rr_path - def _get_rr_bin_path(self, rr_root=None): - # type: (str) -> str - """Get path to RR bin folder.""" + def _get_rr_bin_path(self, tool_name=None, rr_root=None): + # type: (str, str) -> str + """Get path to RR bin folder. + + Args: + tool_name (str): Name of RR executable you want. + rr_root (str, Optional): Custom RR root if needed. + + Returns: + str: Path to the tool based on current platform. + + """ rr_root = rr_root or self._rr_path is_64bit_python = sys.maxsize > 2 ** 32 - rr_bin_path = "" + rr_bin_parts = [rr_root, "bin"] if sys.platform.lower() == "win32": - rr_bin_path = "/bin/win64" - if not is_64bit_python: - # we are using 64bit python - rr_bin_path = "/bin/win" - rr_bin_path = rr_bin_path.replace( - "/", os.path.sep - ) + rr_bin_parts.append("win") if sys.platform.lower() == "darwin": - rr_bin_path = "/bin/mac64" - if not is_64bit_python: - rr_bin_path = "/bin/mac" + rr_bin_parts.append("mac") + + if sys.platform.lower().startswith("linux"): + rr_bin_parts.append("lx") + + rr_bin_path = os.sep.join(rr_bin_parts) - if sys.platform.lower() == "linux": - rr_bin_path = "/bin/lx64" + paths_to_check = [] + # if we use 64bit python, append 64bit specific path first + if is_64bit_python: + if not tool_name: + return rr_bin_path + "64" + paths_to_check.append(rr_bin_path + "64") - return os.path.join(rr_root, rr_bin_path) + # otherwise use 32bit + if not tool_name: + return rr_bin_path + paths_to_check.append(rr_bin_path) + + return find_tool_in_custom_paths(paths_to_check, tool_name) def _initialize_module_path(self): # type: () -> None @@ -84,30 +101,25 @@ def submit_file(self, file, mode=RR_SUBMIT_CONSOLE): # type: (SubmitFile, int) -> None if mode == self.RR_SUBMIT_CONSOLE: self._submit_using_console(file) + return - # RR v7 supports only Python 2.7 so we bail out in fear + # RR v7 supports only Python 2.7, so we bail out in fear # until there is support for Python 3 😰 raise NotImplementedError( "Submission via RoyalRender API is not supported yet") # self._submit_using_api(file) - def _submit_using_console(self, file): + def _submit_using_console(self, job_file): # type: (SubmitFile) -> None - rr_console = os.path.join( - self._get_rr_bin_path(), - "rrSubmitterConsole" - ) - - if sys.platform.lower() == "darwin" and "/bin/mac64" in rr_console: - rr_console = rr_console.replace("/bin/mac64", "/bin/mac") + rr_start_local = self._get_rr_bin_path("rrStartLocal") - if sys.platform.lower() == "win32": - if "/bin/win64" in rr_console: - rr_console = rr_console.replace("/bin/win64", "/bin/win") - rr_console += ".exe" + self.log.info("rr_console: {}".format(rr_start_local)) - args = [rr_console, file] - run_subprocess(" ".join(args), logger=self.log) + args = [rr_start_local, "rrSubmitterconsole", job_file] + self.log.info("Executing: {}".format(" ".join(args))) + env = os.environ + env["RR_ROOT"] = self._rr_path + run_subprocess(args, logger=self.log, env=env) def _submit_using_api(self, file): # type: (SubmitFile) -> None diff --git a/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py index 784e4c5ff9f..82236386b7d 100644 --- a/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py +++ b/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py @@ -38,11 +38,11 @@ def get_job(self): """ def get_rr_platform(): if sys.platform.lower() in ["win32", "win64"]: - return "win" + return "windows" elif sys.platform.lower() == "darwin": return "mac" else: - return "lx" + return "linux" expected_files = self._instance.data["expectedFiles"] first_file = next(self._iter_expected_files(expected_files)) @@ -53,8 +53,10 @@ def get_rr_platform(): .get('maya') \ .get('RenderSettings') \ .get('default_render_image_folder') - filename = os.path.basename(self.scene_path) - dirname = os.path.join(workspace, default_render_file) + file_name = os.path.basename(self.scene_path) + dir_name = os.path.join(workspace, default_render_file) + layer = self._instance.data["setMembers"] # type: str + layer_name = layer.removeprefix("rs_") job = RRJob( Software="Maya", @@ -64,22 +66,22 @@ def get_rr_platform(): SeqStep=int(self._instance.data["byFrameStep"]), SeqFileOffset=0, Version="{0:.2f}".format(MGlobal.apiVersion() / 10000), - SceneName=os.path.basename(self.scene_path), + SceneName=self.scene_path, IsActive=True, - ImageDir=dirname, - ImageFilename=filename, - ImageExtension="." + os.path.splitext(filename)[1], + ImageDir=dir_name, + ImageFilename="{}.".format(layer_name), + ImageExtension=os.path.splitext(first_file)[1], ImagePreNumberLetter=".", ImageSingleOutputFile=False, SceneOS=get_rr_platform(), Camera=self._instance.data["cameras"][0], - Layer=self._instance.data["layer"], + Layer=layer_name, SceneDatabaseDir=workspace, - ImageFramePadding=get_attr_in_layer( - "defaultRenderGlobals.extensionPadding", - self._instance.data["layer"]), + CustomSHotName=self._instance.context.data["asset"], + CompanyProjectName=self._instance.context.data["projectName"], ImageWidth=self._instance.data["resolutionWidth"], - ImageHeight=self._instance.data["resolutionHeight"] + ImageHeight=self._instance.data["resolutionHeight"], + PreID=1 ) return job @@ -125,11 +127,9 @@ def process(self, instance): self.log.error("Cannot get OpenPype RoyalRender module.") raise AssertionError("OpenPype RoyalRender module not found.") """ - - file_path = None if self.use_published: - file_path = get_published_workfile_instance() + file_path = get_published_workfile_instance(context) # fallback if nothing was set if not file_path: @@ -153,7 +153,8 @@ def process_submission(self): with open(xml.name, "w") as f: f.write(submission.serialize()) - self.rr_api.submit_file(file=xml) + self.log.info("submitting job file: {}".format(xml.name)) + self.rr_api.submit_file(file=xml.name) @staticmethod def _resolve_rr_path(context, rr_path_name): @@ -184,4 +185,12 @@ def _resolve_rr_path(context, rr_path_name): return rr_servers[rr_path_name][platform.system().lower()] - + @staticmethod + def _iter_expected_files(exp): + if isinstance(exp[0], dict): + for _aov, files in exp[0].items(): + for file in files: + yield file + else: + for file in exp: + yield file diff --git a/openpype/modules/royalrender/rr_job.py b/openpype/modules/royalrender/rr_job.py index beb8c17187e..f5c7033b625 100644 --- a/openpype/modules/royalrender/rr_job.py +++ b/openpype/modules/royalrender/rr_job.py @@ -87,7 +87,7 @@ class RRJob: # Frame Padding of the frame number in the rendered filename. # Some render config files are setting the padding at render time. - ImageFramePadding = attr.ib(default=None) # type: str + ImageFramePadding = attr.ib(default=None) # type: int # Some render applications support overriding the image format at # the render commandline. @@ -129,6 +129,7 @@ class RRJob: CustomUserInfo = attr.ib(default=None) # type: str SubmitMachine = attr.ib(default=None) # type: str Color_ID = attr.ib(default=2) # type: int + CompanyProjectName = attr.ib(default=None) # type: str RequiredLicenses = attr.ib(default=None) # type: str @@ -225,7 +226,7 @@ def filter_data(a, v): # foo=bar~baz~goo self._process_submitter_parameters( self.SubmitterParameters, root, job_file) - + root.appendChild(job_file) for job in self.Jobs: # type: RRJob if not isinstance(job, RRJob): raise AttributeError( @@ -247,10 +248,11 @@ def filter_data(a, v): custom_attr.name)] = custom_attr.value for item, value in serialized_job.items(): - xml_attr = root.create(item) + xml_attr = root.createElement(item) xml_attr.appendChild( - root.createTextNode(value) + root.createTextNode(str(value)) ) xml_job.appendChild(xml_attr) + job_file.appendChild(xml_job) return root.toprettyxml(indent="\t") From 7ab4e0f77362e36226acd6e340c1c924c4c4773c Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 18 Jan 2023 17:06:05 +0100 Subject: [PATCH 016/104] :construction: refactor RR job flow --- .../publish/collect_rr_path_from_instance.py | 12 +- .../publish/create_maya_royalrender_job.py | 150 ++++++++++++++ .../publish/create_publish_royalrender_job.py | 178 ++++++++++++++++ .../publish/submit_jobs_to_royalrender.py | 115 ++++++++++ .../publish/submit_maya_royalrender.py | 196 ------------------ openpype/pipeline/farm/pyblish.py | 49 +++++ 6 files changed, 499 insertions(+), 201 deletions(-) create mode 100644 openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py create mode 100644 openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py create mode 100644 openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py create mode 100644 openpype/pipeline/farm/pyblish.py diff --git a/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py b/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py index 187e2b9c446..40f34561fa9 100644 --- a/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py +++ b/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py @@ -15,19 +15,21 @@ def process(self, instance): "Using '{}' for submission.".format(instance.data["rrPathName"])) @staticmethod - def _collect_rr_path_name(render_instance): + def _collect_rr_path_name(instance): # type: (pyblish.api.Instance) -> str """Get Royal Render pat name from render instance.""" rr_settings = ( - render_instance.context.data + instance.context.data ["system_settings"] ["modules"] ["royalrender"] ) + if not instance.data.get("rrPaths"): + return "default" try: default_servers = rr_settings["rr_paths"] project_servers = ( - render_instance.context.data + instance.context.data ["project_settings"] ["royalrender"] ["rr_paths"] @@ -40,8 +42,8 @@ def _collect_rr_path_name(render_instance): except (AttributeError, KeyError): # Handle situation were we had only one url for royal render. - return render_instance.context.data["defaultRRPath"] + return rr_settings["rr_paths"]["default"] return list(rr_servers.keys())[ - int(render_instance.data.get("rrPaths")) + int(instance.data.get("rrPaths")) ] diff --git a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py new file mode 100644 index 00000000000..e194a7edc60 --- /dev/null +++ b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py @@ -0,0 +1,150 @@ +# -*- coding: utf-8 -*- +"""Submitting render job to RoyalRender.""" +import os +import sys +import platform + +from maya.OpenMaya import MGlobal # noqa +from pyblish.api import InstancePlugin, IntegratorOrder, Context +from openpype.pipeline.publish.lib import get_published_workfile_instance +from openpype.pipeline.publish import KnownPublishError +from openpype.modules.royalrender.api import Api as rrApi +from openpype.modules.royalrender.rr_job import RRJob + + +class CreateMayaRoyalRenderJob(InstancePlugin): + label = "Create Maya Render job in RR" + order = IntegratorOrder + 0.1 + families = ["renderlayer"] + targets = ["local"] + use_published = True + + def __init__(self, *args, **kwargs): + self._instance = None + self._rrRoot = None + self.scene_path = None + self.job = None + self.submission_parameters = None + self.rr_api = None + + def get_job(self): + """Prepare job payload. + + Returns: + RRJob: RoyalRender job payload. + + """ + def get_rr_platform(): + if sys.platform.lower() in ["win32", "win64"]: + return "windows" + elif sys.platform.lower() == "darwin": + return "mac" + else: + return "linux" + + expected_files = self._instance.data["expectedFiles"] + first_file = next(self._iter_expected_files(expected_files)) + output_dir = os.path.dirname(first_file) + self._instance.data["outputDir"] = output_dir + workspace = self._instance.context.data["workspaceDir"] + default_render_file = self._instance.context.data.get('project_settings') \ + .get('maya') \ + .get('RenderSettings') \ + .get('default_render_image_folder') + file_name = os.path.basename(self.scene_path) + dir_name = os.path.join(workspace, default_render_file) + layer = self._instance.data["setMembers"] # type: str + layer_name = layer.removeprefix("rs_") + + job = RRJob( + Software="Maya", + Renderer=self._instance.data["renderer"], + SeqStart=int(self._instance.data["frameStartHandle"]), + SeqEnd=int(self._instance.data["frameEndHandle"]), + SeqStep=int(self._instance.data["byFrameStep"]), + SeqFileOffset=0, + Version="{0:.2f}".format(MGlobal.apiVersion() / 10000), + SceneName=self.scene_path, + IsActive=True, + ImageDir=dir_name, + ImageFilename="{}.".format(layer_name), + ImageExtension=os.path.splitext(first_file)[1], + ImagePreNumberLetter=".", + ImageSingleOutputFile=False, + SceneOS=get_rr_platform(), + Camera=self._instance.data["cameras"][0], + Layer=layer_name, + SceneDatabaseDir=workspace, + CustomSHotName=self._instance.context.data["asset"], + CompanyProjectName=self._instance.context.data["projectName"], + ImageWidth=self._instance.data["resolutionWidth"], + ImageHeight=self._instance.data["resolutionHeight"], + ) + return job + + def process(self, instance): + """Plugin entry point.""" + self._instance = instance + context = instance.context + from pprint import pformat + + self._rr_root = self._resolve_rr_path(context, instance.data.get("rrPathName")) # noqa + self.log.debug(self._rr_root) + if not self._rr_root: + raise KnownPublishError( + ("Missing RoyalRender root. " + "You need to configure RoyalRender module.")) + + self.rr_api = rrApi(self._rr_root) + + file_path = None + if self.use_published: + file_path = get_published_workfile_instance(context) + + # fallback if nothing was set + if not file_path: + self.log.warning("Falling back to workfile") + file_path = context.data["currentFile"] + + self.scene_path = file_path + + self._instance.data["rrJobs"] = [self.get_job()] + + @staticmethod + def _iter_expected_files(exp): + if isinstance(exp[0], dict): + for _aov, files in exp[0].items(): + for file in files: + yield file + else: + for file in exp: + yield file + + @staticmethod + def _resolve_rr_path(context, rr_path_name): + # type: (Context, str) -> str + rr_settings = ( + context.data + ["system_settings"] + ["modules"] + ["royalrender"] + ) + try: + default_servers = rr_settings["rr_paths"] + project_servers = ( + context.data + ["project_settings"] + ["royalrender"] + ["rr_paths"] + ) + rr_servers = { + k: default_servers[k] + for k in project_servers + if k in default_servers + } + + except (AttributeError, KeyError): + # Handle situation were we had only one url for royal render. + return context.data["defaultRRPath"][platform.system().lower()] + + return rr_servers[rr_path_name][platform.system().lower()] diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py new file mode 100644 index 00000000000..9d8cc602c52 --- /dev/null +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +"""Create publishing job on RoyalRender.""" +from pyblish.api import InstancePlugin, IntegratorOrder +from copy import deepcopy +from openpype.pipeline import legacy_io +import requests +import os + + +class CreatePublishRoyalRenderJob(InstancePlugin): + label = "Create publish job in RR" + order = IntegratorOrder + 0.2 + icon = "tractor" + targets = ["local"] + hosts = ["fusion", "maya", "nuke", "celaction", "aftereffects", "harmony"] + families = ["render.farm", "prerender.farm", + "renderlayer", "imagesequence", "vrayscene"] + aov_filter = {"maya": [r".*([Bb]eauty).*"], + "aftereffects": [r".*"], # for everything from AE + "harmony": [r".*"], # for everything from AE + "celaction": [r".*"]} + + def process(self, instance): + data = instance.data.copy() + context = instance.context + self.context = context + self.anatomy = instance.context.data["anatomy"] + + asset = data.get("asset") + subset = data.get("subset") + source = self._remap_source( + data.get("source") or context.data["source"]) + + + + def _remap_source(self, source): + success, rootless_path = ( + self.anatomy.find_root_template_from_path(source) + ) + if success: + source = rootless_path + else: + # `rootless_path` is not set to `source` if none of roots match + self.log.warning(( + "Could not find root path for remapping \"{}\"." + " This may cause issues." + ).format(source)) + return source + + def _submit_post_job(self, instance, job, instances): + """Submit publish job to RoyalRender.""" + data = instance.data.copy() + subset = data["subset"] + job_name = "Publish - {subset}".format(subset=subset) + + # instance.data.get("subset") != instances[0]["subset"] + # 'Main' vs 'renderMain' + override_version = None + instance_version = instance.data.get("version") # take this if exists + if instance_version != 1: + override_version = instance_version + output_dir = self._get_publish_folder( + instance.context.data['anatomy'], + deepcopy(instance.data["anatomyData"]), + instance.data.get("asset"), + instances[0]["subset"], + 'render', + override_version + ) + + # Transfer the environment from the original job to this dependent + # job, so they use the same environment + metadata_path, roothless_metadata_path = \ + self._create_metadata_path(instance) + + environment = { + "AVALON_PROJECT": legacy_io.Session["AVALON_PROJECT"], + "AVALON_ASSET": legacy_io.Session["AVALON_ASSET"], + "AVALON_TASK": legacy_io.Session["AVALON_TASK"], + "OPENPYPE_USERNAME": instance.context.data["user"], + "OPENPYPE_PUBLISH_JOB": "1", + "OPENPYPE_RENDER_JOB": "0", + "OPENPYPE_REMOTE_JOB": "0", + "OPENPYPE_LOG_NO_COLORS": "1" + } + + # add environments from self.environ_keys + for env_key in self.environ_keys: + if os.getenv(env_key): + environment[env_key] = os.environ[env_key] + + # pass environment keys from self.environ_job_filter + job_environ = job["Props"].get("Env", {}) + for env_j_key in self.environ_job_filter: + if job_environ.get(env_j_key): + environment[env_j_key] = job_environ[env_j_key] + + # Add mongo url if it's enabled + if instance.context.data.get("deadlinePassMongoUrl"): + mongo_url = os.environ.get("OPENPYPE_MONGO") + if mongo_url: + environment["OPENPYPE_MONGO"] = mongo_url + + priority = self.deadline_priority or instance.data.get("priority", 50) + + args = [ + "--headless", + 'publish', + roothless_metadata_path, + "--targets", "deadline", + "--targets", "farm" + ] + + # Generate the payload for Deadline submission + payload = { + "JobInfo": { + "Plugin": self.deadline_plugin, + "BatchName": job["Props"]["Batch"], + "Name": job_name, + "UserName": job["Props"]["User"], + "Comment": instance.context.data.get("comment", ""), + + "Department": self.deadline_department, + "ChunkSize": self.deadline_chunk_size, + "Priority": priority, + + "Group": self.deadline_group, + "Pool": instance.data.get("primaryPool"), + "SecondaryPool": instance.data.get("secondaryPool"), + + "OutputDirectory0": output_dir + }, + "PluginInfo": { + "Version": self.plugin_pype_version, + "Arguments": " ".join(args), + "SingleFrameOnly": "True", + }, + # Mandatory for Deadline, may be empty + "AuxFiles": [], + } + + # add assembly jobs as dependencies + if instance.data.get("tileRendering"): + self.log.info("Adding tile assembly jobs as dependencies...") + job_index = 0 + for assembly_id in instance.data.get("assemblySubmissionJobs"): + payload["JobInfo"]["JobDependency{}".format(job_index)] = assembly_id # noqa: E501 + job_index += 1 + elif instance.data.get("bakingSubmissionJobs"): + self.log.info("Adding baking submission jobs as dependencies...") + job_index = 0 + for assembly_id in instance.data["bakingSubmissionJobs"]: + payload["JobInfo"]["JobDependency{}".format(job_index)] = assembly_id # noqa: E501 + job_index += 1 + else: + payload["JobInfo"]["JobDependency0"] = job["_id"] + + if instance.data.get("suspend_publish"): + payload["JobInfo"]["InitialStatus"] = "Suspended" + + for index, (key_, value_) in enumerate(environment.items()): + payload["JobInfo"].update( + { + "EnvironmentKeyValue%d" + % index: "{key}={value}".format( + key=key_, value=value_ + ) + } + ) + # remove secondary pool + payload["JobInfo"].pop("SecondaryPool", None) + + self.log.info("Submitting Deadline job ...") + + url = "{}/api/jobs".format(self.deadline_url) + response = requests.post(url, json=payload, timeout=10) + if not response.ok: + raise Exception(response.text) \ No newline at end of file diff --git a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py new file mode 100644 index 00000000000..4fcf3a08bd8 --- /dev/null +++ b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +"""Submit jobs to RoyalRender.""" +import tempfile +import platform + +from pyblish.api import IntegratorOrder, ContextPlugin, Context +from openpype.modules.royalrender.api import RRJob, Api as rrApi +from openpype.pipeline.publish import KnownPublishError + + +class SubmitJobsToRoyalRender(ContextPlugin): + """Find all jobs, create submission XML and submit it to RoyalRender.""" + label = "Submit jobs to RoyalRender" + order = IntegratorOrder + 0.1 + targets = ["local"] + + def __init__(self): + super(SubmitJobsToRoyalRender, self).__init__() + self._rr_root = None + self._rr_api = None + self._submission_parameters = [] + + def process(self, context): + rr_settings = ( + context.data + ["system_settings"] + ["modules"] + ["royalrender"] + ) + + if rr_settings["enabled"] is not True: + self.log.warning("RoyalRender modules is disabled.") + return + + # iterate over all instances and try to find RRJobs + jobs = [] + for instance in context: + if isinstance(instance.data.get("rrJob"), RRJob): + jobs.append(instance.data.get("rrJob")) + if instance.data.get("rrJobs"): + if all(isinstance(job, RRJob) for job in instance.data.get("rrJobs")): + jobs += instance.data.get("rrJobs") + + if jobs: + self._rr_root = self._resolve_rr_path( + context, instance.data.get("rrPathName")) # noqa + if not self._rr_root: + raise KnownPublishError( + ("Missing RoyalRender root. " + "You need to configure RoyalRender module.")) + self._rr_api = rrApi(self._rr_root) + self._submission_parameters = self.get_submission_parameters() + self.process_submission(jobs) + return + + self.log.info("No RoyalRender jobs found") + + def process_submission(self, jobs): + # type: ([RRJob]) -> None + submission = rrApi.create_submission( + jobs, + self._submission_parameters) + + xml = tempfile.NamedTemporaryFile(suffix=".xml", delete=False) + with open(xml.name, "w") as f: + f.write(submission.serialize()) + + self.log.info("submitting job(s) file: {}".format(xml.name)) + self._rr_api.submit_file(file=xml.name) + + def create_file(self, name, ext, contents=None): + temp = tempfile.NamedTemporaryFile( + dir=self.tempdir, + suffix=ext, + prefix=name + '.', + delete=False, + ) + + if contents: + with open(temp.name, 'w') as f: + f.write(contents) + + return temp.name + + def get_submission_parameters(self): + return [] + + @staticmethod + def _resolve_rr_path(context, rr_path_name): + # type: (Context, str) -> str + rr_settings = ( + context.data + ["system_settings"] + ["modules"] + ["royalrender"] + ) + try: + default_servers = rr_settings["rr_paths"] + project_servers = ( + context.data + ["project_settings"] + ["royalrender"] + ["rr_paths"] + ) + rr_servers = { + k: default_servers[k] + for k in project_servers + if k in default_servers + } + + except (AttributeError, KeyError): + # Handle situation were we had only one url for royal render. + return context.data["defaultRRPath"][platform.system().lower()] + + return rr_servers[rr_path_name][platform.system().lower()] diff --git a/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py index 82236386b7d..e69de29bb2d 100644 --- a/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py +++ b/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py @@ -1,196 +0,0 @@ -# -*- coding: utf-8 -*- -"""Submitting render job to RoyalRender.""" -import os -import sys -import tempfile -import platform - -from maya.OpenMaya import MGlobal # noqa -from pyblish.api import InstancePlugin, IntegratorOrder, Context -from openpype.hosts.maya.api.lib import get_attr_in_layer -from openpype.pipeline.farm.tools import get_published_workfile_instance -from openpype.pipeline.publish import KnownPublishError -from openpype.modules.royalrender.api import Api as rr_api -from openpype.modules.royalrender.rr_job import RRJob, SubmitterParameter - - -class MayaSubmitRoyalRender(InstancePlugin): - label = "Submit to RoyalRender" - order = IntegratorOrder + 0.1 - families = ["renderlayer"] - targets = ["local"] - use_published = True - - def __init__(self, *args, **kwargs): - self._instance = None - self._rrRoot = None - self.scene_path = None - self.job = None - self.submission_parameters = None - self.rr_api = None - - def get_job(self): - """Prepare job payload. - - Returns: - RRJob: RoyalRender job payload. - - """ - def get_rr_platform(): - if sys.platform.lower() in ["win32", "win64"]: - return "windows" - elif sys.platform.lower() == "darwin": - return "mac" - else: - return "linux" - - expected_files = self._instance.data["expectedFiles"] - first_file = next(self._iter_expected_files(expected_files)) - output_dir = os.path.dirname(first_file) - self._instance.data["outputDir"] = output_dir - workspace = self._instance.context.data["workspaceDir"] - default_render_file = self._instance.context.data.get('project_settings') \ - .get('maya') \ - .get('RenderSettings') \ - .get('default_render_image_folder') - file_name = os.path.basename(self.scene_path) - dir_name = os.path.join(workspace, default_render_file) - layer = self._instance.data["setMembers"] # type: str - layer_name = layer.removeprefix("rs_") - - job = RRJob( - Software="Maya", - Renderer=self._instance.data["renderer"], - SeqStart=int(self._instance.data["frameStartHandle"]), - SeqEnd=int(self._instance.data["frameEndHandle"]), - SeqStep=int(self._instance.data["byFrameStep"]), - SeqFileOffset=0, - Version="{0:.2f}".format(MGlobal.apiVersion() / 10000), - SceneName=self.scene_path, - IsActive=True, - ImageDir=dir_name, - ImageFilename="{}.".format(layer_name), - ImageExtension=os.path.splitext(first_file)[1], - ImagePreNumberLetter=".", - ImageSingleOutputFile=False, - SceneOS=get_rr_platform(), - Camera=self._instance.data["cameras"][0], - Layer=layer_name, - SceneDatabaseDir=workspace, - CustomSHotName=self._instance.context.data["asset"], - CompanyProjectName=self._instance.context.data["projectName"], - ImageWidth=self._instance.data["resolutionWidth"], - ImageHeight=self._instance.data["resolutionHeight"], - PreID=1 - ) - return job - - @staticmethod - def get_submission_parameters(): - return [] - - def create_file(self, name, ext, contents=None): - temp = tempfile.NamedTemporaryFile( - dir=self.tempdir, - suffix=ext, - prefix=name + '.', - delete=False, - ) - - if contents: - with open(temp.name, 'w') as f: - f.write(contents) - - return temp.name - - def process(self, instance): - """Plugin entry point.""" - self._instance = instance - context = instance.context - from pprint import pformat - - self._rr_root = self._resolve_rr_path(context, instance.data.get("rrPathName")) # noqa - self.log.debug(self._rr_root) - if not self._rr_root: - raise KnownPublishError( - ("Missing RoyalRender root. " - "You need to configure RoyalRender module.")) - - self.rr_api = rr_api(self._rr_root) - - # get royalrender module - """ - try: - rr_module = context.data.get( - "openPypeModules")["royalrender"] - except AttributeError: - self.log.error("Cannot get OpenPype RoyalRender module.") - raise AssertionError("OpenPype RoyalRender module not found.") - """ - file_path = None - if self.use_published: - file_path = get_published_workfile_instance(context) - - # fallback if nothing was set - if not file_path: - self.log.warning("Falling back to workfile") - file_path = context.data["currentFile"] - - self.scene_path = file_path - self.job = self.get_job() - self.log.info(self.job) - self.submission_parameters = self.get_submission_parameters() - - self.process_submission() - - def process_submission(self): - submission = rr_api.create_submission( - [self.job], - self.submission_parameters) - - self.log.debug(submission) - xml = tempfile.NamedTemporaryFile(suffix=".xml", delete=False) - with open(xml.name, "w") as f: - f.write(submission.serialize()) - - self.log.info("submitting job file: {}".format(xml.name)) - self.rr_api.submit_file(file=xml.name) - - @staticmethod - def _resolve_rr_path(context, rr_path_name): - # type: (Context, str) -> str - rr_settings = ( - context.data - ["system_settings"] - ["modules"] - ["royalrender"] - ) - try: - default_servers = rr_settings["rr_paths"] - project_servers = ( - context.data - ["project_settings"] - ["royalrender"] - ["rr_paths"] - ) - rr_servers = { - k: default_servers[k] - for k in project_servers - if k in default_servers - } - - except (AttributeError, KeyError): - # Handle situation were we had only one url for royal render. - return context.data["defaultRRPath"][platform.system().lower()] - - return rr_servers[rr_path_name][platform.system().lower()] - - @staticmethod - def _iter_expected_files(exp): - if isinstance(exp[0], dict): - for _aov, files in exp[0].items(): - for file in files: - yield file - else: - for file in exp: - yield file diff --git a/openpype/pipeline/farm/pyblish.py b/openpype/pipeline/farm/pyblish.py new file mode 100644 index 00000000000..02535f40903 --- /dev/null +++ b/openpype/pipeline/farm/pyblish.py @@ -0,0 +1,49 @@ +from openpype.lib import Logger +import attr + + +@attr.s +class InstanceSkeleton(object): + family = attr.ib(factory=) + +def remap_source(source, anatomy): + success, rootless_path = ( + anatomy.find_root_template_from_path(source) + ) + if success: + source = rootless_path + else: + # `rootless_path` is not set to `source` if none of roots match + log = Logger.get_logger("farm_publishing") + log.warning(( + "Could not find root path for remapping \"{}\"." + " This may cause issues." + ).format(source)) + return source + +def get_skeleton_instance() + instance_skeleton_data = { + "family": family, + "subset": subset, + "families": families, + "asset": asset, + "frameStart": start, + "frameEnd": end, + "handleStart": handle_start, + "handleEnd": handle_end, + "frameStartHandle": start - handle_start, + "frameEndHandle": end + handle_end, + "comment": instance.data["comment"], + "fps": fps, + "source": source, + "extendFrames": data.get("extendFrames"), + "overrideExistingFrame": data.get("overrideExistingFrame"), + "pixelAspect": data.get("pixelAspect", 1), + "resolutionWidth": data.get("resolutionWidth", 1920), + "resolutionHeight": data.get("resolutionHeight", 1080), + "multipartExr": data.get("multipartExr", False), + "jobBatchName": data.get("jobBatchName", ""), + "useSequenceForReview": data.get("useSequenceForReview", True), + # map inputVersions `ObjectId` -> `str` so json supports it + "inputVersions": list(map(str, data.get("inputVersions", []))) + } \ No newline at end of file From e49cbf34f64260bd3e9960338d84a7be8589af9e Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 18 Jan 2023 17:06:53 +0100 Subject: [PATCH 017/104] :bug: fix default for ShotGrid this is fixing default value for ShotGrid server enumerator after code refactor done in this branch --- openpype/settings/defaults/project_settings/shotgrid.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/settings/defaults/project_settings/shotgrid.json b/openpype/settings/defaults/project_settings/shotgrid.json index 83b6f690741..0dcffed28a2 100644 --- a/openpype/settings/defaults/project_settings/shotgrid.json +++ b/openpype/settings/defaults/project_settings/shotgrid.json @@ -1,6 +1,6 @@ { "shotgrid_project_id": 0, - "shotgrid_server": "", + "shotgrid_server": [], "event": { "enabled": false }, From 8ebef523bc2689b3b56fe5bbd947fdfec4285eb9 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 18 Jan 2023 17:19:18 +0100 Subject: [PATCH 018/104] :rotating_light: hound fixes --- .../hosts/maya/plugins/create/create_render.py | 2 +- openpype/modules/royalrender/api.py | 2 -- .../publish/collect_rr_path_from_instance.py | 4 +--- .../plugins/publish/create_maya_royalrender_job.py | 14 ++++++++------ .../publish/create_publish_royalrender_job.py | 12 +++++------- .../plugins/publish/submit_jobs_to_royalrender.py | 4 +++- openpype/pipeline/farm/pyblish.py | 12 +++++++++--- 7 files changed, 27 insertions(+), 23 deletions(-) diff --git a/openpype/hosts/maya/plugins/create/create_render.py b/openpype/hosts/maya/plugins/create/create_render.py index 337868d47d2..71d42cc82f8 100644 --- a/openpype/hosts/maya/plugins/create/create_render.py +++ b/openpype/hosts/maya/plugins/create/create_render.py @@ -279,7 +279,7 @@ def _create_render_settings(self): if deadline_enabled and muster_enabled and royalrender_enabled: self.log.error( - ("Multiple render farm support (Deadline/RoyalRender/Muster) " + ("Multiple render farm support (Deadline/RoyalRender/Muster) " "is enabled. We support only one at time.") ) raise RuntimeError("Both Deadline and Muster are enabled") diff --git a/openpype/modules/royalrender/api.py b/openpype/modules/royalrender/api.py index 8b13b9781f1..86d27ccc6cf 100644 --- a/openpype/modules/royalrender/api.py +++ b/openpype/modules/royalrender/api.py @@ -2,9 +2,7 @@ """Wrapper around Royal Render API.""" import sys import os -import platform -from openpype.settings import get_project_settings from openpype.lib.local_settings import OpenPypeSettingsRegistry from openpype.lib import Logger, run_subprocess from .rr_job import RRJob, SubmitFile, SubmitterParameter diff --git a/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py b/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py index 40f34561fa9..cfb5b78077f 100644 --- a/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py +++ b/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py @@ -44,6 +44,4 @@ def _collect_rr_path_name(instance): # Handle situation were we had only one url for royal render. return rr_settings["rr_paths"]["default"] - return list(rr_servers.keys())[ - int(instance.data.get("rrPaths")) - ] + return list(rr_servers.keys())[int(instance.data.get("rrPaths"))] diff --git a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py index e194a7edc60..5f427353ac8 100644 --- a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py @@ -47,11 +47,14 @@ def get_rr_platform(): output_dir = os.path.dirname(first_file) self._instance.data["outputDir"] = output_dir workspace = self._instance.context.data["workspaceDir"] - default_render_file = self._instance.context.data.get('project_settings') \ - .get('maya') \ - .get('RenderSettings') \ - .get('default_render_image_folder') - file_name = os.path.basename(self.scene_path) + default_render_file = ( + self._instance.context.data + ['project_settings'] + ['maya'] + ['RenderSettings'] + ['default_render_image_folder'] + ) + # file_name = os.path.basename(self.scene_path) dir_name = os.path.join(workspace, default_render_file) layer = self._instance.data["setMembers"] # type: str layer_name = layer.removeprefix("rs_") @@ -86,7 +89,6 @@ def process(self, instance): """Plugin entry point.""" self._instance = instance context = instance.context - from pprint import pformat self._rr_root = self._resolve_rr_path(context, instance.data.get("rrPathName")) # noqa self.log.debug(self._rr_root) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 9d8cc602c52..e62289641b3 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -26,12 +26,10 @@ def process(self, instance): self.context = context self.anatomy = instance.context.data["anatomy"] - asset = data.get("asset") - subset = data.get("subset") - source = self._remap_source( - data.get("source") or context.data["source"]) - - + # asset = data.get("asset") + # subset = data.get("subset") + # source = self._remap_source( + # data.get("source") or context.data["source"]) def _remap_source(self, source): success, rootless_path = ( @@ -175,4 +173,4 @@ def _submit_post_job(self, instance, job, instances): url = "{}/api/jobs".format(self.deadline_url) response = requests.post(url, json=payload, timeout=10) if not response.ok: - raise Exception(response.text) \ No newline at end of file + raise Exception(response.text) diff --git a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py index 4fcf3a08bd8..325fb36993d 100644 --- a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py +++ b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py @@ -38,7 +38,9 @@ def process(self, context): if isinstance(instance.data.get("rrJob"), RRJob): jobs.append(instance.data.get("rrJob")) if instance.data.get("rrJobs"): - if all(isinstance(job, RRJob) for job in instance.data.get("rrJobs")): + if all( + isinstance(job, RRJob) + for job in instance.data.get("rrJobs")): jobs += instance.data.get("rrJobs") if jobs: diff --git a/openpype/pipeline/farm/pyblish.py b/openpype/pipeline/farm/pyblish.py index 02535f40903..15f4356b860 100644 --- a/openpype/pipeline/farm/pyblish.py +++ b/openpype/pipeline/farm/pyblish.py @@ -4,7 +4,9 @@ @attr.s class InstanceSkeleton(object): - family = attr.ib(factory=) + # family = attr.ib(factory=) + pass + def remap_source(source, anatomy): success, rootless_path = ( @@ -21,7 +23,9 @@ def remap_source(source, anatomy): ).format(source)) return source -def get_skeleton_instance() + +def get_skeleton_instance(): + """ instance_skeleton_data = { "family": family, "subset": subset, @@ -46,4 +50,6 @@ def get_skeleton_instance() "useSequenceForReview": data.get("useSequenceForReview", True), # map inputVersions `ObjectId` -> `str` so json supports it "inputVersions": list(map(str, data.get("inputVersions", []))) - } \ No newline at end of file + } + """ + pass From 5dda835a0dba359c70512e7c21a36548eb4072a0 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 18 Jan 2023 17:21:28 +0100 Subject: [PATCH 019/104] :rotating_light: hound fixes 2 --- .../plugins/publish/create_publish_royalrender_job.py | 2 +- openpype/pipeline/farm/pyblish.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index e62289641b3..65de600bfae 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -21,7 +21,7 @@ class CreatePublishRoyalRenderJob(InstancePlugin): "celaction": [r".*"]} def process(self, instance): - data = instance.data.copy() + # data = instance.data.copy() context = instance.context self.context = context self.anatomy = instance.context.data["anatomy"] diff --git a/openpype/pipeline/farm/pyblish.py b/openpype/pipeline/farm/pyblish.py index 15f4356b860..436ad7f195e 100644 --- a/openpype/pipeline/farm/pyblish.py +++ b/openpype/pipeline/farm/pyblish.py @@ -17,10 +17,9 @@ def remap_source(source, anatomy): else: # `rootless_path` is not set to `source` if none of roots match log = Logger.get_logger("farm_publishing") - log.warning(( - "Could not find root path for remapping \"{}\"." - " This may cause issues." - ).format(source)) + log.warning( + ("Could not find root path for remapping \"{}\"." + " This may cause issues.").format(source)) return source From cc8732aa9d05476fb3162be176b16edacf833b0f Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Tue, 7 Feb 2023 19:22:54 +0100 Subject: [PATCH 020/104] :construction: work on publish job --- openpype/modules/royalrender/api.py | 11 +- .../publish/create_maya_royalrender_job.py | 1 - .../publish/create_publish_royalrender_job.py | 104 ++++++++---------- openpype/modules/royalrender/rr_job.py | 46 +++++++- 4 files changed, 95 insertions(+), 67 deletions(-) diff --git a/openpype/modules/royalrender/api.py b/openpype/modules/royalrender/api.py index 86d27ccc6cf..e610a0c8a8e 100644 --- a/openpype/modules/royalrender/api.py +++ b/openpype/modules/royalrender/api.py @@ -20,19 +20,19 @@ def __init__(self, rr_path=None): self._rr_path = rr_path os.environ["RR_ROOT"] = rr_path - def _get_rr_bin_path(self, tool_name=None, rr_root=None): + @staticmethod + def get_rr_bin_path(rr_root, tool_name=None): # type: (str, str) -> str """Get path to RR bin folder. Args: tool_name (str): Name of RR executable you want. - rr_root (str, Optional): Custom RR root if needed. + rr_root (str): Custom RR root if needed. Returns: str: Path to the tool based on current platform. """ - rr_root = rr_root or self._rr_path is_64bit_python = sys.maxsize > 2 ** 32 rr_bin_parts = [rr_root, "bin"] @@ -65,7 +65,7 @@ def _initialize_module_path(self): # type: () -> None """Set RR modules for Python.""" # default for linux - rr_bin = self._get_rr_bin_path() + rr_bin = self.get_rr_bin_path(self._rr_path) rr_module_path = os.path.join(rr_bin, "lx64/lib") if sys.platform.lower() == "win32": @@ -109,7 +109,8 @@ def submit_file(self, file, mode=RR_SUBMIT_CONSOLE): def _submit_using_console(self, job_file): # type: (SubmitFile) -> None - rr_start_local = self._get_rr_bin_path("rrStartLocal") + rr_start_local = self.get_rr_bin_path( + self._rr_path, "rrStartLocal") self.log.info("rr_console: {}".format(rr_start_local)) diff --git a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py index 5f427353ac8..0b257d8b7a9 100644 --- a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py @@ -32,7 +32,6 @@ def get_job(self): Returns: RRJob: RoyalRender job payload. - """ def get_rr_platform(): if sys.platform.lower() in ["win32", "win64"]: diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 65de600bfae..f87ee589b64 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -6,6 +6,10 @@ import requests import os +from openpype.modules.royalrender.rr_job import RRJob, RREnvList +from openpype.pipeline.publish import KnownPublishError +from openpype.modules.royalrender.api import Api as rrApi + class CreatePublishRoyalRenderJob(InstancePlugin): label = "Create publish job in RR" @@ -45,14 +49,12 @@ def _remap_source(self, source): ).format(source)) return source - def _submit_post_job(self, instance, job, instances): + def get_job(self, instance, job, instances): """Submit publish job to RoyalRender.""" data = instance.data.copy() subset = data["subset"] job_name = "Publish - {subset}".format(subset=subset) - # instance.data.get("subset") != instances[0]["subset"] - # 'Main' vs 'renderMain' override_version = None instance_version = instance.data.get("version") # take this if exists if instance_version != 1: @@ -62,6 +64,8 @@ def _submit_post_job(self, instance, job, instances): deepcopy(instance.data["anatomyData"]), instance.data.get("asset"), instances[0]["subset"], + # TODO: this shouldn't be hardcoded and is in fact settable by + # Settings. 'render', override_version ) @@ -71,7 +75,7 @@ def _submit_post_job(self, instance, job, instances): metadata_path, roothless_metadata_path = \ self._create_metadata_path(instance) - environment = { + environment = RREnvList({ "AVALON_PROJECT": legacy_io.Session["AVALON_PROJECT"], "AVALON_ASSET": legacy_io.Session["AVALON_ASSET"], "AVALON_TASK": legacy_io.Session["AVALON_TASK"], @@ -80,7 +84,7 @@ def _submit_post_job(self, instance, job, instances): "OPENPYPE_RENDER_JOB": "0", "OPENPYPE_REMOTE_JOB": "0", "OPENPYPE_LOG_NO_COLORS": "1" - } + }) # add environments from self.environ_keys for env_key in self.environ_keys: @@ -88,7 +92,16 @@ def _submit_post_job(self, instance, job, instances): environment[env_key] = os.environ[env_key] # pass environment keys from self.environ_job_filter - job_environ = job["Props"].get("Env", {}) + # and collect all pre_ids to wait for + job_environ = {} + jobs_pre_ids = [] + for job in instance["rrJobs"]: # type: RRJob + if job.rrEnvList: + job_environ.update( + dict(RREnvList.parse(job.rrEnvList)) + ) + jobs_pre_ids.append(job.PreID) + for env_j_key in self.environ_job_filter: if job_environ.get(env_j_key): environment[env_j_key] = job_environ[env_j_key] @@ -99,7 +112,7 @@ def _submit_post_job(self, instance, job, instances): if mongo_url: environment["OPENPYPE_MONGO"] = mongo_url - priority = self.deadline_priority or instance.data.get("priority", 50) + priority = self.priority or instance.data.get("priority", 50) args = [ "--headless", @@ -109,66 +122,37 @@ def _submit_post_job(self, instance, job, instances): "--targets", "farm" ] - # Generate the payload for Deadline submission - payload = { - "JobInfo": { - "Plugin": self.deadline_plugin, - "BatchName": job["Props"]["Batch"], - "Name": job_name, - "UserName": job["Props"]["User"], - "Comment": instance.context.data.get("comment", ""), - - "Department": self.deadline_department, - "ChunkSize": self.deadline_chunk_size, - "Priority": priority, - - "Group": self.deadline_group, - "Pool": instance.data.get("primaryPool"), - "SecondaryPool": instance.data.get("secondaryPool"), - - "OutputDirectory0": output_dir - }, - "PluginInfo": { - "Version": self.plugin_pype_version, - "Arguments": " ".join(args), - "SingleFrameOnly": "True", - }, - # Mandatory for Deadline, may be empty - "AuxFiles": [], - } + job = RRJob( + Software="Execute", + Renderer="Once", + # path to OpenPype + SeqStart=1, + SeqEnd=1, + SeqStep=1, + SeqFileOffset=0, + Version="1.0", + SceneName="", + IsActive=True, + ImageFilename="execOnce.file", + ImageDir="", + ImageExtension="", + ImagePreNumberLetter="", + SceneOS=RRJob.get_rr_platform(), + rrEnvList=environment.serialize(), + Priority=priority + ) # add assembly jobs as dependencies if instance.data.get("tileRendering"): self.log.info("Adding tile assembly jobs as dependencies...") - job_index = 0 - for assembly_id in instance.data.get("assemblySubmissionJobs"): - payload["JobInfo"]["JobDependency{}".format(job_index)] = assembly_id # noqa: E501 - job_index += 1 + job.WaitForPreIDs += instance.data.get("assemblySubmissionJobs") elif instance.data.get("bakingSubmissionJobs"): self.log.info("Adding baking submission jobs as dependencies...") - job_index = 0 - for assembly_id in instance.data["bakingSubmissionJobs"]: - payload["JobInfo"]["JobDependency{}".format(job_index)] = assembly_id # noqa: E501 - job_index += 1 + job.WaitForPreIDs += instance.data["bakingSubmissionJobs"] else: - payload["JobInfo"]["JobDependency0"] = job["_id"] - - if instance.data.get("suspend_publish"): - payload["JobInfo"]["InitialStatus"] = "Suspended" - - for index, (key_, value_) in enumerate(environment.items()): - payload["JobInfo"].update( - { - "EnvironmentKeyValue%d" - % index: "{key}={value}".format( - key=key_, value=value_ - ) - } - ) - # remove secondary pool - payload["JobInfo"].pop("SecondaryPool", None) - - self.log.info("Submitting Deadline job ...") + job.WaitForPreIDs += jobs_pre_ids + + self.log.info("Creating RoyalRender Publish job ...") url = "{}/api/jobs".format(self.deadline_url) response = requests.post(url, json=payload, timeout=10) diff --git a/openpype/modules/royalrender/rr_job.py b/openpype/modules/royalrender/rr_job.py index f5c7033b625..21e5291bc3c 100644 --- a/openpype/modules/royalrender/rr_job.py +++ b/openpype/modules/royalrender/rr_job.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- """Python wrapper for RoyalRender XML job file.""" +import sys from xml.dom import minidom as md import attr from collections import namedtuple, OrderedDict @@ -8,6 +9,23 @@ CustomAttribute = namedtuple("CustomAttribute", ["name", "value"]) +class RREnvList(dict): + def serialize(self): + # VariableA=ValueA~~~VariableB=ValueB + return "~~~".join( + ["{}={}".format(k, v) for k, v in sorted(self.items())]) + + @staticmethod + def parse(data): + # type: (str) -> RREnvList + """Parse rrEnvList string and return it as RREnvList object.""" + out = RREnvList() + for var in data.split("~~~"): + k, v = data.split("=") + out[k] = v + return out + + @attr.s class RRJob: """Mapping of Royal Render job file to a data class.""" @@ -108,7 +126,7 @@ class RRJob: # jobs send from this machine. If a job with the PreID was found, then # this jobs waits for the other job. Note: This flag can be used multiple # times to wait for multiple jobs. - WaitForPreID = attr.ib(default=None) # type: int + WaitForPreIDs = attr.ib(factory=list) # type: list # List of submitter options per job # list item must be of `SubmitterParameter` type @@ -138,6 +156,21 @@ class RRJob: TotalFrames = attr.ib(default=None) # type: int Tiled = attr.ib(default=None) # type: str + # Environment + # only used in RR 8.3 and newer + rrEnvList = attr.ib(default=None) # type: str + + @staticmethod + def get_rr_platform(): + # type: () -> str + """Returns name of platform used in rr jobs.""" + if sys.platform.lower() in ["win32", "win64"]: + return "windows" + elif sys.platform.lower() == "darwin": + return "mac" + else: + return "linux" + class SubmitterParameter: """Wrapper for Submitter Parameters.""" @@ -242,6 +275,8 @@ def filter_data(a, v): job, dict_factory=OrderedDict, filter=filter_data) serialized_job.pop("CustomAttributes") serialized_job.pop("SubmitterParameters") + # we are handling `WaitForPreIDs` separately. + wait_pre_ids = serialized_job.pop("WaitForPreIDs", []) for custom_attr in job_custom_attributes: # type: CustomAttribute serialized_job["Custom{}".format( @@ -253,6 +288,15 @@ def filter_data(a, v): root.createTextNode(str(value)) ) xml_job.appendChild(xml_attr) + + # WaitForPreID - can be used multiple times + for pre_id in wait_pre_ids: + xml_attr = root.createElement("WaitForPreID") + xml_attr.appendChild( + root.createTextNode(str(pre_id)) + ) + xml_job.appendChild(xml_attr) + job_file.appendChild(xml_job) return root.toprettyxml(indent="\t") From 5595712e810f4a6c22480f8a8fc054d426c49856 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Thu, 23 Mar 2023 16:09:25 +0100 Subject: [PATCH 021/104] :art: add cmd line arguments attribute to job --- .../plugins/publish/create_publish_royalrender_job.py | 3 +++ openpype/modules/royalrender/rr_job.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index f87ee589b64..167a97cbe42 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -131,7 +131,10 @@ def get_job(self, instance, job, instances): SeqStep=1, SeqFileOffset=0, Version="1.0", + # executable SceneName="", + # command line arguments + CustomAddCmdFlags=" ".join(args), IsActive=True, ImageFilename="execOnce.file", ImageDir="", diff --git a/openpype/modules/royalrender/rr_job.py b/openpype/modules/royalrender/rr_job.py index 21e5291bc3c..689a488a5c3 100644 --- a/openpype/modules/royalrender/rr_job.py +++ b/openpype/modules/royalrender/rr_job.py @@ -138,6 +138,9 @@ class RRJob: # list item must be of `CustomAttribute` named tuple CustomAttributes = attr.ib(factory=list) # type: list + # This is used to hold command line arguments for Execute job + CustomAddCmdFlags = attr.ib(default=None) # type: str + # Additional information for subsequent publish script and # for better display in rrControl UserName = attr.ib(default=None) # type: str From 4e14e0b56b1ad2021cd057a3cc5f939a7f334f6e Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Fri, 24 Mar 2023 15:41:13 +0100 Subject: [PATCH 022/104] :art: add render configuration --- .../render_apps/_config/E01__OpenPype.png | Bin 0 -> 2075 bytes .../_config/E01__OpenPype__PublishJob.cfg | 4 ++++ .../_config/E01__OpenPype___global.inc | 2 ++ .../render_apps/_install_paths/OpenPype.cfg | 12 ++++++++++++ .../_prepost_scripts/OpenPypeEnvironment.cfg | 11 +++++++++++ .../PreOpenPypeInjectEnvironments.py | 4 ++++ 6 files changed, 33 insertions(+) create mode 100644 openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype.png create mode 100644 openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg create mode 100644 openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype___global.inc create mode 100644 openpype/modules/royalrender/rr_root/render_apps/_install_paths/OpenPype.cfg create mode 100644 openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/OpenPypeEnvironment.cfg create mode 100644 openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/PreOpenPypeInjectEnvironments.py diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype.png b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype.png new file mode 100644 index 0000000000000000000000000000000000000000..68c5aec1174c9ee4b0f4ec21fe0aea8c2ac48545 GIT binary patch literal 2075 zcmah~c~BE)6pvFuK|w{7O6zo8IviqlmmpYH0z$$i1Y-yqnOY6W29_Kfb{9wxMNwL$ z9u>uds#Qlv1;wK}B9=0O6to~k1rHDhhkBrRAXXW~_S=9^ulA2k_WORn_j~Vq?|pAq zOQXZ=Y@Ka+Jf2-dxL5}M?YU>9HTeA^oZrCX4gZ;xD_Dgj3Rh8jM5(6Y3B;r~0-VS5 z4>TE-DlNf6@k9b?5WBb_y)#>ljzqF6O4)`jVwtSAWl;m zPo=Oz7zQ{rAkLDSA$0>YaD0#mltWN21VjH#Crso==p6DM-Iz$R6q8692M_R;i$VB3 zLy*1?o0ycw50NF3|EBBpZykeSLsCH^29o+#Om7@8a@WduW|&?+J%l`ya_mP~MY z!6Wp_1R{zsB(RA>*lYSbz0TcVGBNtammJ| zq>0ce#H5}uFhF-Ojv;WtM?ev!qv#mW*aR@LI2)`4Zowp!8bIFfKoKI5l%_PK4q%Kd zEEtLi5&3%g`TFt&NWTmy8xCwqjajT@0ZV`hy!n_nU*IkG^2G%{xDV!udEpplQMRzb zZBQ#&<^S=yo`(E^g+zjflMGAIX3JK8qsI`*{j2_^jf= z6PrK!UiAJJ!oO}Qudd z_Lr)XZa@SyX$#-n*n$1A$%mg}_zveWB(9KzrkPMfc} zw62ZLftGz8?5n~8(sw71$hE3`8Q{C-w#~JjZByshEfQvJnA0#Swr=Uj zi~N$3;-q83l9q3a-Fyl|BJ(6I1yL2flTIMBdim4Q{3Y|Q-|Zd;|JW5f<7g&tDcySd z)u`UGgp(Z}TV{jrg^e+M0mI`VF5ENBv@eh=P*#LENbbI2Z9|#%veL2YY5lXvOV$Ox zHylSq?h6_#m@Z4+xX$^BRaKTK#J$urXV>dbWR5Los;c0xQjBAVI|cobEB&&m;ebdV zm;LLxt**#iZJcaOOIs^b?(hF%_{@XyHB%Zpn%Z3-bb2<}OJ@zEu=Ck+nv%6P_>rK;E zRmRW7(tpU1h*pyt;Ye)y`YJnE`$3lNlM%?C8LSy?JHjT^H$;$?7~|ac7;! zUhBYR&4(5ud#!)!vx=-*TEZMCdVHa?Hb=MTYIc3S)!E8E)rIFNG5O}l+FRe3c+5Ec zY23CQLF4Y5az2-6YGXa&_Kkh5^S1Qqd>2pHf_+xaD~9c!Ybt+VdnEKsZ&mP~WYwPN(dhL@)b>RjtOnMl*vYE|tG dhq}yr)u@6>KAYxurHcEeM}$O+i-Q+u{R76+2nzrJ literal 0 HcmV?d00001 diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg new file mode 100644 index 00000000000..3f0f8285fd3 --- /dev/null +++ b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg @@ -0,0 +1,4 @@ +rendererName= PublishJob + +::include(E01__OpenPype____global.inc) +::include(E01__OpenPype____global__inhouse.inc) diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype___global.inc b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype___global.inc new file mode 100644 index 00000000000..ba38337340d --- /dev/null +++ b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype___global.inc @@ -0,0 +1,2 @@ +IconApp= E01__OpenPype.png +Name= OpenPype diff --git a/openpype/modules/royalrender/rr_root/render_apps/_install_paths/OpenPype.cfg b/openpype/modules/royalrender/rr_root/render_apps/_install_paths/OpenPype.cfg new file mode 100644 index 00000000000..07f7547d297 --- /dev/null +++ b/openpype/modules/royalrender/rr_root/render_apps/_install_paths/OpenPype.cfg @@ -0,0 +1,12 @@ +[Windows] +Executable= openpype_console.exe +Path= OS; \OpenPype\*\openpype_console.exe +Path= 32; \OpenPype\*\openpype_console.exe + +[Linux] +Executable= openpype_console +Path= OS; /opt/openpype/*/openpype_console + +[Mac] +Executable= openpype_console +Path= OS; /Applications/OpenPype*/Content/MacOS/openpype_console diff --git a/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/OpenPypeEnvironment.cfg b/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/OpenPypeEnvironment.cfg new file mode 100644 index 00000000000..70f0bc2e244 --- /dev/null +++ b/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/OpenPypeEnvironment.cfg @@ -0,0 +1,11 @@ +PrePostType= pre +CommandLine= + +CommandLine= rrPythonconsole" > "render_apps/_prepost_scripts/PreOpenPypeInjectEnvironments.py" + +CommandLine= + + +CommandLine= "" +CommandLine= + diff --git a/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/PreOpenPypeInjectEnvironments.py b/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/PreOpenPypeInjectEnvironments.py new file mode 100644 index 00000000000..891de9594c5 --- /dev/null +++ b/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/PreOpenPypeInjectEnvironments.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +import os + +os.environ["OPENYPYPE_TESTVAR"] = "OpenPype was here" From f328c9848948eac8d9d8897c4bbdafb3b8981bde Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Fri, 24 Mar 2023 15:42:06 +0100 Subject: [PATCH 023/104] :heavy_minus_sign: remove old plugin --- .../royalrender/plugins/publish/submit_maya_royalrender.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py diff --git a/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_maya_royalrender.py deleted file mode 100644 index e69de29bb2d..00000000000 From de5bca48fa0f54b71783832bf16d521f6ce6a792 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Fri, 24 Mar 2023 15:49:27 +0100 Subject: [PATCH 024/104] :art: add openpype version --- .../plugins/publish/create_maya_royalrender_job.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py index 0b257d8b7a9..8b6beba031e 100644 --- a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py @@ -6,10 +6,11 @@ from maya.OpenMaya import MGlobal # noqa from pyblish.api import InstancePlugin, IntegratorOrder, Context +from openpype.lib import is_running_from_build from openpype.pipeline.publish.lib import get_published_workfile_instance from openpype.pipeline.publish import KnownPublishError from openpype.modules.royalrender.api import Api as rrApi -from openpype.modules.royalrender.rr_job import RRJob +from openpype.modules.royalrender.rr_job import RRJob, CustomAttribute class CreateMayaRoyalRenderJob(InstancePlugin): @@ -58,6 +59,14 @@ def get_rr_platform(): layer = self._instance.data["setMembers"] # type: str layer_name = layer.removeprefix("rs_") + custom_attributes = [] + if is_running_from_build(): + custom_attributes = [ + CustomAttribute( + name="OpenPypeVersion", + value=os.environ.get("OPENPYPE_VERSION")) + ] + job = RRJob( Software="Maya", Renderer=self._instance.data["renderer"], @@ -81,7 +90,9 @@ def get_rr_platform(): CompanyProjectName=self._instance.context.data["projectName"], ImageWidth=self._instance.data["resolutionWidth"], ImageHeight=self._instance.data["resolutionHeight"], + CustomAttributes=custom_attributes ) + return job def process(self, instance): From 226039f3c5fb753a935af66aefa378d797200701 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Fri, 24 Mar 2023 18:30:49 +0100 Subject: [PATCH 025/104] :recycle: handling of jobs --- .../publish/create_maya_royalrender_job.py | 5 ++++- .../publish/create_publish_royalrender_job.py | 16 +++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py index 8b6beba031e..6e23fb0b749 100644 --- a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py @@ -120,7 +120,10 @@ def process(self, instance): self.scene_path = file_path - self._instance.data["rrJobs"] = [self.get_job()] + if not self._instance.data.get("rrJobs"): + self._instance.data["rrJobs"] = [] + + self._instance.data["rrJobs"] += self.get_job() @staticmethod def _iter_expected_files(exp): diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 167a97cbe42..03d176a324b 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -123,16 +123,16 @@ def get_job(self, instance, job, instances): ] job = RRJob( - Software="Execute", + Software="OpenPype", Renderer="Once", # path to OpenPype SeqStart=1, SeqEnd=1, SeqStep=1, SeqFileOffset=0, - Version="1.0", + Version=os.environ.get("OPENPYPE_VERSION"), # executable - SceneName="", + SceneName=roothless_metadata_path, # command line arguments CustomAddCmdFlags=" ".join(args), IsActive=True, @@ -157,7 +157,9 @@ def get_job(self, instance, job, instances): self.log.info("Creating RoyalRender Publish job ...") - url = "{}/api/jobs".format(self.deadline_url) - response = requests.post(url, json=payload, timeout=10) - if not response.ok: - raise Exception(response.text) + if not instance.data.get("rrJobs"): + self.log.error("There is no RoyalRender job on the instance.") + raise KnownPublishError( + "Can't create publish job without producing jobs") + + instance.data["rrJobs"] += job From 03b3d2f37654e0b514568e99ea48e0422c82111b Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Fri, 24 Mar 2023 18:31:22 +0100 Subject: [PATCH 026/104] :art: better publish job render config --- .../_config/E01__OpenPype__PublishJob.cfg | 73 ++++++++++++++++++- 1 file changed, 70 insertions(+), 3 deletions(-) diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg index 3f0f8285fd3..6414ae45a87 100644 --- a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg +++ b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg @@ -1,4 +1,71 @@ -rendererName= PublishJob +IconApp= E01__OpenPype.png +Name= OpenPype +rendererName= Once +Version= 1 +Version_Minor= 0 +Type=Execute +TYPEv9=Execute +ExecuteJobType=Once -::include(E01__OpenPype____global.inc) -::include(E01__OpenPype____global__inhouse.inc) + +################################# [Windows] [Linux] [Osx] ################################## + + +CommandLine=> + +CommandLine= + + +::win CommandLine= set "CUDA_VISIBLE_DEVICES=" +::lx CommandLine= setenv CUDA_VISIBLE_DEVICES +::osx CommandLine= setenv CUDA_VISIBLE_DEVICES + + +CommandLine= + + +CommandLine= + + +CommandLine= + + +CommandLine= "" --headless publish + --targets royalrender + --targets farm + + + +CommandLine= + + + + +################################## Render Settings ################################## + + + +################################## Submitter Settings ################################## +StartMultipleInstances= 0~0 +SceneFileExtension= *.json +AllowImageNameChange= 0 +AllowImageDirChange= 0 +SequenceDivide= 0~1 +PPSequenceCheck=0~0 +PPCreateSmallVideo=0~0 +PPCreateFullVideo=0~0 +AllowLocalRenderOut= 0~0 + + +################################## Client Settings ################################## + +IconApp=E01__OpenPype.png + +licenseFailLine= + +errorSearchLine= + +permanentErrorSearchLine = + +Frozen_MinCoreUsage=0.3 +Frozen_Minutes=30 From 8adb0534bf6cf36a3705b79aebc90a3894e23e63 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 29 Mar 2023 18:44:04 +0200 Subject: [PATCH 027/104] :recycle: handle OP versions better --- .../publish/create_publish_royalrender_job.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 03d176a324b..9c74bdcea09 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -1,14 +1,14 @@ # -*- coding: utf-8 -*- """Create publishing job on RoyalRender.""" -from pyblish.api import InstancePlugin, IntegratorOrder -from copy import deepcopy -from openpype.pipeline import legacy_io -import requests import os +from copy import deepcopy +from pyblish.api import InstancePlugin, IntegratorOrder + +from openpype.pipeline import legacy_io from openpype.modules.royalrender.rr_job import RRJob, RREnvList from openpype.pipeline.publish import KnownPublishError -from openpype.modules.royalrender.api import Api as rrApi +from openpype.lib.openpype_version import get_OpenPypeVersion, get_openpype_version class CreatePublishRoyalRenderJob(InstancePlugin): @@ -122,6 +122,8 @@ def get_job(self, instance, job, instances): "--targets", "farm" ] + openpype_version = get_OpenPypeVersion() + current_version = openpype_version(version=get_openpype_version()) job = RRJob( Software="OpenPype", Renderer="Once", @@ -130,7 +132,8 @@ def get_job(self, instance, job, instances): SeqEnd=1, SeqStep=1, SeqFileOffset=0, - Version=os.environ.get("OPENPYPE_VERSION"), + Version="{}.{}".format( + current_version.major(), current_version.minor()), # executable SceneName=roothless_metadata_path, # command line arguments From 547fe50ffca326937f5ed85352cda333d93a6db6 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Thu, 13 Apr 2023 17:14:36 +0200 Subject: [PATCH 028/104] :rotating_light: fix hound :dog: --- .../plugins/publish/create_publish_royalrender_job.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 9c74bdcea09..a5493dd0614 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -8,7 +8,8 @@ from openpype.pipeline import legacy_io from openpype.modules.royalrender.rr_job import RRJob, RREnvList from openpype.pipeline.publish import KnownPublishError -from openpype.lib.openpype_version import get_OpenPypeVersion, get_openpype_version +from openpype.lib.openpype_version import ( + get_OpenPypeVersion, get_openpype_version) class CreatePublishRoyalRenderJob(InstancePlugin): From 23f90ec355ff4d2ce58cad7d4185ec38fcd9efc0 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Mon, 17 Apr 2023 12:24:42 +0200 Subject: [PATCH 029/104] :construction: wip on skeleton instance --- openpype/pipeline/farm/pyblish.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/openpype/pipeline/farm/pyblish.py b/openpype/pipeline/farm/pyblish.py index 436ad7f195e..3ad62962d2b 100644 --- a/openpype/pipeline/farm/pyblish.py +++ b/openpype/pipeline/farm/pyblish.py @@ -1,5 +1,6 @@ from openpype.lib import Logger import attr +import pyblish.api @attr.s @@ -23,10 +24,27 @@ def remap_source(source, anatomy): return source -def get_skeleton_instance(): +def create_skeleton_instance(instance): + # type: (pyblish.api.Instance) -> dict + """Create skelenton instance from original instance data. + + This will create dictionary containing skeleton + - common - data used for publishing rendered instances. + This skeleton instance is then extended with additional data + and serialized to be processed by farm job. + + Args: + instance (pyblish.api.Instance): Original instance to + be used as a source of data. + + Returns: + dict: Dictionary with skeleton instance data. + """ - instance_skeleton_data = { - "family": family, + context = instance.context + + return { + "family": "render" if "prerender" not in instance.data["families"] else "prerender", # noqa: E401 "subset": subset, "families": families, "asset": asset, @@ -50,5 +68,3 @@ def get_skeleton_instance(): # map inputVersions `ObjectId` -> `str` so json supports it "inputVersions": list(map(str, data.get("inputVersions", []))) } - """ - pass From c9b6179cc84aa8347f661bfcac9d7f4fae61dcaa Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Fri, 21 Apr 2023 02:53:06 +0200 Subject: [PATCH 030/104] :construction: move common code out from deadline --- .../plugins/publish/submit_publish_job.py | 492 +-------------- openpype/pipeline/farm/pyblish.py | 576 +++++++++++++++++- openpype/pipeline/farm/pyblish.pyi | 23 + 3 files changed, 606 insertions(+), 485 deletions(-) create mode 100644 openpype/pipeline/farm/pyblish.pyi diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 4765772bcff..fd0d87a0828 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -21,35 +21,10 @@ from openpype.tests.lib import is_in_tests from openpype.pipeline.farm.patterning import match_aov_pattern from openpype.lib import is_running_from_build - - -def get_resources(project_name, version, extension=None): - """Get the files from the specific version.""" - - # TODO this functions seems to be weird - # - it's looking for representation with one extension or first (any) - # representation from a version? - # - not sure how this should work, maybe it does for specific use cases - # but probably can't be used for all resources from 2D workflows - extensions = None - if extension: - extensions = [extension] - repre_docs = list(get_representations( - project_name, version_ids=[version["_id"]], extensions=extensions - )) - assert repre_docs, "This is a bug" - - representation = repre_docs[0] - directory = get_representation_path(representation) - print("Source: ", directory) - resources = sorted( - [ - os.path.normpath(os.path.join(directory, fname)) - for fname in os.listdir(directory) - ] - ) - - return resources +from openpype.pipeline.farm.pyblish import ( + create_skeleton_instance, + create_instances_for_aov +) def get_resource_files(resources, frame_range=None): @@ -356,241 +331,6 @@ def _submit_deadline_post_job(self, instance, job, instances): return deadline_publish_job_id - def _copy_extend_frames(self, instance, representation): - """Copy existing frames from latest version. - - This will copy all existing frames from subset's latest version back - to render directory and rename them to what renderer is expecting. - - Arguments: - instance (pyblish.plugin.Instance): instance to get required - data from - representation (dict): presentation to operate on - - """ - import speedcopy - - self.log.info("Preparing to copy ...") - start = instance.data.get("frameStart") - end = instance.data.get("frameEnd") - project_name = legacy_io.active_project() - - # get latest version of subset - # this will stop if subset wasn't published yet - project_name = legacy_io.active_project() - version = get_last_version_by_subset_name( - project_name, - instance.data.get("subset"), - asset_name=instance.data.get("asset") - ) - - # get its files based on extension - subset_resources = get_resources( - project_name, version, representation.get("ext") - ) - r_col, _ = clique.assemble(subset_resources) - - # if override remove all frames we are expecting to be rendered - # so we'll copy only those missing from current render - if instance.data.get("overrideExistingFrame"): - for frame in range(start, end + 1): - if frame not in r_col.indexes: - continue - r_col.indexes.remove(frame) - - # now we need to translate published names from representation - # back. This is tricky, right now we'll just use same naming - # and only switch frame numbers - resource_files = [] - r_filename = os.path.basename( - representation.get("files")[0]) # first file - op = re.search(self.R_FRAME_NUMBER, r_filename) - pre = r_filename[:op.start("frame")] - post = r_filename[op.end("frame"):] - assert op is not None, "padding string wasn't found" - for frame in list(r_col): - fn = re.search(self.R_FRAME_NUMBER, frame) - # silencing linter as we need to compare to True, not to - # type - assert fn is not None, "padding string wasn't found" - # list of tuples (source, destination) - staging = representation.get("stagingDir") - staging = self.anatomy.fill_root(staging) - resource_files.append( - (frame, - os.path.join(staging, - "{}{}{}".format(pre, - fn.group("frame"), - post))) - ) - - # test if destination dir exists and create it if not - output_dir = os.path.dirname(representation.get("files")[0]) - if not os.path.isdir(output_dir): - os.makedirs(output_dir) - - # copy files - for source in resource_files: - speedcopy.copy(source[0], source[1]) - self.log.info(" > {}".format(source[1])) - - self.log.info( - "Finished copying %i files" % len(resource_files)) - - def _create_instances_for_aov( - self, instance_data, exp_files, additional_data - ): - """Create instance for each AOV found. - - This will create new instance for every aov it can detect in expected - files list. - - Arguments: - instance_data (pyblish.plugin.Instance): skeleton data for instance - (those needed) later by collector - exp_files (list): list of expected files divided by aovs - - Returns: - list of instances - - """ - task = os.environ["AVALON_TASK"] - subset = instance_data["subset"] - cameras = instance_data.get("cameras", []) - instances = [] - # go through aovs in expected files - for aov, files in exp_files[0].items(): - cols, rem = clique.assemble(files) - # we shouldn't have any reminders. And if we do, it should - # be just one item for single frame renders. - if not cols and rem: - assert len(rem) == 1, ("Found multiple non related files " - "to render, don't know what to do " - "with them.") - col = rem[0] - ext = os.path.splitext(col)[1].lstrip(".") - else: - # but we really expect only one collection. - # Nothing else make sense. - assert len(cols) == 1, "only one image sequence type is expected" # noqa: E501 - ext = cols[0].tail.lstrip(".") - col = list(cols[0]) - - self.log.debug(col) - # create subset name `familyTaskSubset_AOV` - group_name = 'render{}{}{}{}'.format( - task[0].upper(), task[1:], - subset[0].upper(), subset[1:]) - - cam = [c for c in cameras if c in col.head] - if cam: - if aov: - subset_name = '{}_{}_{}'.format(group_name, cam, aov) - else: - subset_name = '{}_{}'.format(group_name, cam) - else: - if aov: - subset_name = '{}_{}'.format(group_name, aov) - else: - subset_name = '{}'.format(group_name) - - if isinstance(col, (list, tuple)): - staging = os.path.dirname(col[0]) - else: - staging = os.path.dirname(col) - - success, rootless_staging_dir = ( - self.anatomy.find_root_template_from_path(staging) - ) - if success: - staging = rootless_staging_dir - else: - self.log.warning(( - "Could not find root path for remapping \"{}\"." - " This may cause issues on farm." - ).format(staging)) - - self.log.info("Creating data for: {}".format(subset_name)) - - app = os.environ.get("AVALON_APP", "") - - preview = False - - if isinstance(col, list): - render_file_name = os.path.basename(col[0]) - else: - render_file_name = os.path.basename(col) - aov_patterns = self.aov_filter - - preview = match_aov_pattern(app, aov_patterns, render_file_name) - # toggle preview on if multipart is on - - if instance_data.get("multipartExr"): - self.log.debug("Adding preview tag because its multipartExr") - preview = True - self.log.debug("preview:{}".format(preview)) - new_instance = deepcopy(instance_data) - new_instance["subset"] = subset_name - new_instance["subsetGroup"] = group_name - if preview: - new_instance["review"] = True - - # create representation - if isinstance(col, (list, tuple)): - files = [os.path.basename(f) for f in col] - else: - files = os.path.basename(col) - - # Copy render product "colorspace" data to representation. - colorspace = "" - products = additional_data["renderProducts"].layer_data.products - for product in products: - if product.productName == aov: - colorspace = product.colorspace - break - - rep = { - "name": ext, - "ext": ext, - "files": files, - "frameStart": int(instance_data.get("frameStartHandle")), - "frameEnd": int(instance_data.get("frameEndHandle")), - # If expectedFile are absolute, we need only filenames - "stagingDir": staging, - "fps": new_instance.get("fps"), - "tags": ["review"] if preview else [], - "colorspaceData": { - "colorspace": colorspace, - "config": { - "path": additional_data["colorspaceConfig"], - "template": additional_data["colorspaceTemplate"] - }, - "display": additional_data["display"], - "view": additional_data["view"] - } - } - - # support conversion from tiled to scanline - if instance_data.get("convertToScanline"): - self.log.info("Adding scanline conversion.") - rep["tags"].append("toScanline") - - # poor man exclusion - if ext in self.skip_integration_repre_list: - rep["tags"].append("delete") - - self._solve_families(new_instance, preview) - - new_instance["representations"] = [rep] - - # if extending frames from existing version, copy files from there - # into our destination directory - if new_instance.get("extendFrames", False): - self._copy_extend_frames(new_instance, rep) - instances.append(new_instance) - self.log.debug("instances:{}".format(instances)) - return instances - def _get_representations(self, instance, exp_files): """Create representations for file sequences. @@ -748,8 +488,8 @@ def process(self, instance): # type: (pyblish.api.Instance) -> None """Process plugin. - Detect type of renderfarm submission and create and post dependent job - in case of Deadline. It creates json file with metadata needed for + Detect type of render farm submission and create and post dependent + job in case of Deadline. It creates json file with metadata needed for publishing in directory of render. Args: @@ -760,145 +500,16 @@ def process(self, instance): self.log.info("Skipping local instance.") return - data = instance.data.copy() - context = instance.context - self.context = context - self.anatomy = instance.context.data["anatomy"] - - asset = data.get("asset") or legacy_io.Session["AVALON_ASSET"] - subset = data.get("subset") - - start = instance.data.get("frameStart") - if start is None: - start = context.data["frameStart"] - - end = instance.data.get("frameEnd") - if end is None: - end = context.data["frameEnd"] - - handle_start = instance.data.get("handleStart") - if handle_start is None: - handle_start = context.data["handleStart"] - - handle_end = instance.data.get("handleEnd") - if handle_end is None: - handle_end = context.data["handleEnd"] - - fps = instance.data.get("fps") - if fps is None: - fps = context.data["fps"] - - if data.get("extendFrames", False): - start, end = self._extend_frames( - asset, - subset, - start, - end, - data["overrideExistingFrame"]) - - try: - source = data["source"] - except KeyError: - source = context.data["currentFile"] - - success, rootless_path = ( - self.anatomy.find_root_template_from_path(source) - ) - if success: - source = rootless_path - - else: - # `rootless_path` is not set to `source` if none of roots match - self.log.warning(( - "Could not find root path for remapping \"{}\"." - " This may cause issues." - ).format(source)) - - family = "render" - if "prerender" in instance.data["families"]: - family = "prerender" - families = [family] - - # pass review to families if marked as review - if data.get("review"): - families.append("review") - - instance_skeleton_data = { - "family": family, - "subset": subset, - "families": families, - "asset": asset, - "frameStart": start, - "frameEnd": end, - "handleStart": handle_start, - "handleEnd": handle_end, - "frameStartHandle": start - handle_start, - "frameEndHandle": end + handle_end, - "comment": instance.data["comment"], - "fps": fps, - "source": source, - "extendFrames": data.get("extendFrames"), - "overrideExistingFrame": data.get("overrideExistingFrame"), - "pixelAspect": data.get("pixelAspect", 1), - "resolutionWidth": data.get("resolutionWidth", 1920), - "resolutionHeight": data.get("resolutionHeight", 1080), - "multipartExr": data.get("multipartExr", False), - "jobBatchName": data.get("jobBatchName", ""), - "useSequenceForReview": data.get("useSequenceForReview", True), - # map inputVersions `ObjectId` -> `str` so json supports it - "inputVersions": list(map(str, data.get("inputVersions", []))) - } - - # skip locking version if we are creating v01 - instance_version = instance.data.get("version") # take this if exists - if instance_version != 1: - instance_skeleton_data["version"] = instance_version - - # transfer specific families from original instance to new render - for item in self.families_transfer: - if item in instance.data.get("families", []): - instance_skeleton_data["families"] += [item] - - # transfer specific properties from original instance based on - # mapping dictionary `instance_transfer` - for key, values in self.instance_transfer.items(): - if key in instance.data.get("families", []): - for v in values: - instance_skeleton_data[v] = instance.data.get(v) - - # look into instance data if representations are not having any - # which are having tag `publish_on_farm` and include them - for repre in instance.data.get("representations", []): - staging_dir = repre.get("stagingDir") - if staging_dir: - success, rootless_staging_dir = ( - self.anatomy.find_root_template_from_path( - staging_dir - ) - ) - if success: - repre["stagingDir"] = rootless_staging_dir - else: - self.log.warning(( - "Could not find root path for remapping \"{}\"." - " This may cause issues on farm." - ).format(staging_dir)) - repre["stagingDir"] = staging_dir - - if "publish_on_farm" in repre.get("tags"): - # create representations attribute of not there - if "representations" not in instance_skeleton_data.keys(): - instance_skeleton_data["representations"] = [] - - instance_skeleton_data["representations"].append(repre) + instance_skeleton_data = create_skeleton_instance( + instance, + families_transfer=self.families_transfer, + instance_transfer=self.instance_transfer) instances = None - assert data.get("expectedFiles"), ("Submission from old Pype version" - " - missing expectedFiles") """ - if content of `expectedFiles` are dictionaries, we will handle - it as list of AOVs, creating instance from every one of them. + if content of `expectedFiles` list are dictionaries, we will handle + it as list of AOVs, creating instance for every one of them. Example: -------- @@ -920,7 +531,7 @@ def process(self, instance): This will create instances for `beauty` and `Z` subset adding those files to their respective representations. - If we've got only list of files, we collect all filesequences. + If we have only list of files, we collect all file sequences. More then one doesn't probably make sense, but we'll handle it like creating one instance with multiple representations. @@ -938,55 +549,14 @@ def process(self, instance): `foo` and `xxx` """ - self.log.info(data.get("expectedFiles")) - - if isinstance(data.get("expectedFiles")[0], dict): - # we cannot attach AOVs to other subsets as we consider every - # AOV subset of its own. - - additional_data = { - "renderProducts": instance.data["renderProducts"], - "colorspaceConfig": instance.data["colorspaceConfig"], - "display": instance.data["colorspaceDisplay"], - "view": instance.data["colorspaceView"] - } - - # Get templated path from absolute config path. - anatomy = instance.context.data["anatomy"] - colorspaceTemplate = instance.data["colorspaceConfig"] - success, rootless_staging_dir = ( - anatomy.find_root_template_from_path(colorspaceTemplate) - ) - if success: - colorspaceTemplate = rootless_staging_dir - else: - self.log.warning(( - "Could not find root path for remapping \"{}\"." - " This may cause issues on farm." - ).format(colorspaceTemplate)) - additional_data["colorspaceTemplate"] = colorspaceTemplate - - if len(data.get("attachTo")) > 0: - assert len(data.get("expectedFiles")[0].keys()) == 1, ( - "attaching multiple AOVs or renderable cameras to " - "subset is not supported") - - # create instances for every AOV we found in expected files. - # note: this is done for every AOV and every render camere (if - # there are multiple renderable cameras in scene) - instances = self._create_instances_for_aov( - instance_skeleton_data, - data.get("expectedFiles"), - additional_data - ) - self.log.info("got {} instance{}".format( - len(instances), - "s" if len(instances) > 1 else "")) + if isinstance(instance.data.get("expectedFiles")[0], dict): + instances = create_instances_for_aov( + instance, instance_skeleton_data) else: representations = self._get_representations( instance_skeleton_data, - data.get("expectedFiles") + instance.data.get("expectedFiles") ) if "representations" not in instance_skeleton_data.keys(): @@ -1029,11 +599,11 @@ def process(self, instance): render_job = None submission_type = "" if instance.data.get("toBeRenderedOn") == "deadline": - render_job = data.pop("deadlineSubmissionJob", None) + render_job = instance.data.pop("deadlineSubmissionJob", None) submission_type = "deadline" if instance.data.get("toBeRenderedOn") == "muster": - render_job = data.pop("musterSubmissionJob", None) + render_job = instance.data.pop("musterSubmissionJob", None) submission_type = "muster" if not render_job and instance.data.get("tileRendering") is False: @@ -1056,9 +626,9 @@ def process(self, instance): "jobBatchName") else: render_job["Props"]["Batch"] = os.path.splitext( - os.path.basename(context.data.get("currentFile")))[0] + os.path.basename(instance.context.data.get("currentFile")))[0] # User is deadline user - render_job["Props"]["User"] = context.data.get( + render_job["Props"]["User"] = instance.context.data.get( "deadlineUser", getpass.getuser()) render_job["Props"]["Env"] = { @@ -1080,15 +650,15 @@ def process(self, instance): # publish job file publish_job = { - "asset": asset, - "frameStart": start, - "frameEnd": end, - "fps": context.data.get("fps", None), - "source": source, - "user": context.data["user"], - "version": context.data["version"], # this is workfile version - "intent": context.data.get("intent"), - "comment": context.data.get("comment"), + "asset": instance_skeleton_data["asset"], + "frameStart": instance_skeleton_data["frameStart"], + "frameEnd": instance_skeleton_data["frameEnd"], + "fps": instance_skeleton_data["fps"], + "source": instance_skeleton_data["source"], + "user": instance.context.data["user"], + "version": instance.context.data["version"], # this is workfile version + "intent": instance.context.data.get("intent"), + "comment": instance.context.data.get("comment"), "job": render_job or None, "session": legacy_io.Session.copy(), "instances": instances @@ -1098,7 +668,7 @@ def process(self, instance): publish_job["deadline_publish_job_id"] = deadline_publish_job_id # add audio to metadata file if available - audio_file = context.data.get("audioFile") + audio_file = instance.context.data.get("audioFile") if audio_file and os.path.isfile(audio_file): publish_job.update({"audio": audio_file}) diff --git a/openpype/pipeline/farm/pyblish.py b/openpype/pipeline/farm/pyblish.py index 3ad62962d2b..35a944444fa 100644 --- a/openpype/pipeline/farm/pyblish.py +++ b/openpype/pipeline/farm/pyblish.py @@ -1,32 +1,165 @@ +from openpype.pipeline import ( + get_current_project_name, + get_representation_path, + Anatomy, +) +from openpype.client import ( + get_last_version_by_subset_name, + get_representations +) from openpype.lib import Logger import attr import pyblish.api +from openpype.pipeline.publish import KnownPublishError +import os +import clique +from copy import deepcopy +import re @attr.s -class InstanceSkeleton(object): - # family = attr.ib(factory=) - pass +class TimeData: + """Structure used to handle time related data.""" + start = attr.ib(type=int) + end = attr.ib(type=int) + fps = attr.ib() + step = attr.ib(default=1, type=int) + handle_start = attr.ib(default=0, type=int) + handle_end = attr.ib(default=0, type=int) -def remap_source(source, anatomy): +def remap_source(path, anatomy): + """Try to remap path to rootless path. + + Args: + path (str): Path to be remapped to rootless. + anatomy (Anatomy): Anatomy object to handle remapping + itself. + + Returns: + str: Remapped path. + + Throws: + ValueError: if the root cannot be found. + + """ success, rootless_path = ( - anatomy.find_root_template_from_path(source) + anatomy.find_root_template_from_path(path) ) if success: source = rootless_path else: - # `rootless_path` is not set to `source` if none of roots match - log = Logger.get_logger("farm_publishing") - log.warning( - ("Could not find root path for remapping \"{}\"." - " This may cause issues.").format(source)) + raise ValueError( + "Root from template path cannot be found: {}".format(path)) return source -def create_skeleton_instance(instance): - # type: (pyblish.api.Instance) -> dict - """Create skelenton instance from original instance data. +def extend_frames(asset, subset, start, end): + """Get latest version of asset nad update frame range. + + Based on minimum and maximum values. + + Arguments: + asset (str): asset name + subset (str): subset name + start (int): start frame + end (int): end frame + + Returns: + (int, int): update frame start/end + + """ + # Frame comparison + prev_start = None + prev_end = None + + project_name = get_current_project_name() + version = get_last_version_by_subset_name( + project_name, + subset, + asset_name=asset + ) + + # Set prev start / end frames for comparison + if not prev_start and not prev_end: + prev_start = version["data"]["frameStart"] + prev_end = version["data"]["frameEnd"] + + updated_start = min(start, prev_start) + updated_end = max(end, prev_end) + + + return updated_start, updated_end + + +def get_time_data_from_instance_or_context(instance): + """Get time data from instance (or context). + + If time data is not found on instance, data from context will be used. + + Args: + instance (pyblish.api.Instance): Source instance. + + Returns: + TimeData: dataclass holding time information. + + """ + return TimeData( + start=instance.data.get["frameStart"] or \ + instance.context.data.get("frameStart"), + end=instance.data.get("frameEnd") or \ + instance.context.data.get("frameEnd"), + fps=instance.data.get("fps") or \ + instance.context.data.get("fps"), + handle_start=instance.data.get("handleStart") or \ + instance.context.data.get("handleStart"), # noqa: E501 + handle_end=instance.data.get("handleStart") or \ + instance.context.data.get("handleStart") + ) + + +def get_transferable_representations(instance): + """Transfer representations from original instance. + + This will get all representations on the original instance that + are flagged with `publish_on_farm` and return them to be included + on skeleton instance if needed. + + Args: + instance (pyblish.api.Instance): Original instance to be processed. + + Return: + list of dicts: List of transferable representations. + + """ + anatomy = instance.data.get("anatomy") # type: Anatomy + to_transfer = [] + + for representation in instance.data.get("representations", []): + if "publish_on_farm" not in representation.get("tags"): + continue + + trans_rep = representation.copy() + + staging_dir = trans_rep.get("stagingDir") + + if staging_dir: + try: + trans_rep["stagingDir"] = remap_source(staging_dir, anatomy) + except ValueError: + log = Logger.get_logger("farm_publishing") + log.warning( + ("Could not find root path for remapping \"{}\". " + "This may cause issues on farm.").format(staging_dir)) + + to_transfer.append(trans_rep) + return to_transfer + + +def create_skeleton_instance( + instance, families_transfer=None, instance_transfer=None): + # type: (pyblish.api.Instance, list, dict) -> dict + """Create skeleton instance from original instance data. This will create dictionary containing skeleton - common - data used for publishing rendered instances. @@ -36,26 +169,64 @@ def create_skeleton_instance(instance): Args: instance (pyblish.api.Instance): Original instance to be used as a source of data. + families_transfer (list): List of family names to transfer + from the original instance to the skeleton. + instance_transfer (dict): Dict with keys as families and + values as a list of property names to transfer to the + new skeleton. Returns: dict: Dictionary with skeleton instance data. """ + # list of family names to transfer to new family if present + context = instance.context + data = instance.data.copy() + anatomy = data["anatomy"] # type: Anatomy + + families = [data["family"]] + + # pass review to families if marked as review + if data.get("review"): + families.append("review") + + # get time related data from instance (or context) + time_data = get_time_data_from_instance_or_context(instance) + + if data.get("extendFrames", False): + time_data.start, time_data.end = extend_frames( + data["asset"], + data["subset"], + time_data.start, + time_data.end, + ) - return { + source = data.get("source") or context.data.get("currentFile") + success, rootless_path = ( + anatomy.find_root_template_from_path(source) + ) + if success: + source = rootless_path + else: + # `rootless_path` is not set to `source` if none of roots match + log = Logger.get_logger("farm_publishing") + log.warning(("Could not find root path for remapping \"{}\". " + "This may cause issues.").format(source)) + + instance_skeleton_data = { "family": "render" if "prerender" not in instance.data["families"] else "prerender", # noqa: E401 - "subset": subset, + "subset": data["subset"], "families": families, - "asset": asset, - "frameStart": start, - "frameEnd": end, - "handleStart": handle_start, - "handleEnd": handle_end, - "frameStartHandle": start - handle_start, - "frameEndHandle": end + handle_end, - "comment": instance.data["comment"], - "fps": fps, + "asset": data["asset"], + "frameStart": time_data.start, + "frameEnd": time_data.end, + "handleStart": time_data.handle_start, + "handleEnd": time_data.handle_end, + "frameStartHandle": time_data.start - time_data.handle_start, + "frameEndHandle": time_data.end + time_data.handle_end, + "comment": data.get("comment"), + "fps": time_data.fps, "source": source, "extendFrames": data.get("extendFrames"), "overrideExistingFrame": data.get("overrideExistingFrame"), @@ -68,3 +239,360 @@ def create_skeleton_instance(instance): # map inputVersions `ObjectId` -> `str` so json supports it "inputVersions": list(map(str, data.get("inputVersions", []))) } + + # skip locking version if we are creating v01 + instance_version = data.get("version") # take this if exists + if instance_version != 1: + instance_skeleton_data["version"] = instance_version + + # transfer specific families from original instance to new render + for item in families_transfer: + if item in instance.data.get("families", []): + instance_skeleton_data["families"] += [item] + + # transfer specific properties from original instance based on + # mapping dictionary `instance_transfer` + for key, values in instance_transfer.items(): + if key in instance.data.get("families", []): + for v in values: + instance_skeleton_data[v] = instance.data.get(v) + + representations = get_transferable_representations(instance) + instance_skeleton_data["representations"] = [] + instance_skeleton_data["representations"] += representations + + return instance_skeleton_data + +def _solve_families(families): + """Solve families. + + TODO: This is ugly and needs to be refactored. Ftrack family should be + added in different way (based on if the module is enabled?) + + """ + # if we have one representation with preview tag + # flag whole instance for review and for ftrack + if "ftrack" not in families and os.environ.get("FTRACK_SERVER"): + families.append("ftrack") + if "review" not in families: + families.append("review") + return families +def create_instances_for_aov(instance, skeleton): + """Create instances from AOVs. + + This will create new pyblish.api.Instances by going over expected + files defined on original instance. + + Args: + instance (pyblish.api.Instance): Original instance. + skeleton (dict): Skeleton instance data. + + Returns: + list of pyblish.api.Instance: Instances created from + expected files. + + """ + # we cannot attach AOVs to other subsets as we consider every + # AOV subset of its own. + + log = Logger.get_logger("farm_publishing") + additional_color_data = { + "renderProducts": instance.data["renderProducts"], + "colorspaceConfig": instance.data["colorspaceConfig"], + "display": instance.data["colorspaceDisplay"], + "view": instance.data["colorspaceView"] + } + + # Get templated path from absolute config path. + anatomy = instance.context.data["anatomy"] + colorspace_template = instance.data["colorspaceConfig"] + try: + additional_color_data["colorspaceTemplate"] = remap_source( + colorspace_template, anatomy) + except ValueError as e: + log.warning(e) + + # if there are subset to attach to and more than one AOV, + # we cannot proceed. + if ( + len(instance.data.get("attachTo", [])) > 0 + and len(instance.data.get("expectedFiles")[0].keys()) != 1 + ): + raise KnownPublishError( + "attaching multiple AOVs or renderable cameras to " + "subset is not supported yet.") + + # create instances for every AOV we found in expected files. + # NOTE: this is done for every AOV and every render camera (if + # there are multiple renderable cameras in scene) + return _create_instances_for_aov( + instance, + skeleton, + additional_color_data + ) + + +def _create_instances_for_aov(instance, skeleton, additional_data): + """Create instance for each AOV found. + + This will create new instance for every AOV it can detect in expected + files list. + + Args: + instance (pyblish.api.Instance): Original instance. + skeleton (dict): Skeleton data for instance (those needed) later + by collector. + additional_data (dict): ... + + + Returns: + list of instances + + Throws: + ValueError: + + """ + # TODO: this needs to be taking the task from context or instance + task = os.environ["AVALON_TASK"] + + anatomy = instance.data["anatomy"] + subset = skeleton["subset"] + cameras = instance.data.get("cameras", []) + exp_files = instance.data["expectedFiles"] + log = Logger.get_logger("farm_publishing") + + instances = [] + # go through AOVs in expected files + for aov, files in exp_files[0].items(): + cols, rem = clique.assemble(files) + # we shouldn't have any reminders. And if we do, it should + # be just one item for single frame renders. + if not cols and rem: + if len(rem) != 1: + raise ValueError("Found multiple non related files " + "to render, don't know what to do " + "with them.") + col = rem[0] + ext = os.path.splitext(col)[1].lstrip(".") + else: + # but we really expect only one collection. + # Nothing else make sense. + if len(cols) != 1: + raise ValueError("Only one image sequence type is expected.") # noqa: E501 + ext = cols[0].tail.lstrip(".") + col = list(cols[0]) + + # create subset name `familyTaskSubset_AOV` + group_name = 'render{}{}{}{}'.format( + task[0].upper(), task[1:], + subset[0].upper(), subset[1:]) + + cam = [c for c in cameras if c in col.head] + if cam: + if aov: + subset_name = '{}_{}_{}'.format(group_name, cam, aov) + else: + subset_name = '{}_{}'.format(group_name, cam) + else: + if aov: + subset_name = '{}_{}'.format(group_name, aov) + else: + subset_name = '{}'.format(group_name) + + if isinstance(col, (list, tuple)): + staging = os.path.dirname(col[0]) + else: + staging = os.path.dirname(col) + + try: + staging = remap_source(staging, anatomy) + except ValueError as e: + log.warning(e) + + log.info("Creating data for: {}".format(subset_name)) + + app = os.environ.get("AVALON_APP", "") + + preview = False + + if isinstance(col, list): + render_file_name = os.path.basename(col[0]) + else: + render_file_name = os.path.basename(col) + aov_patterns = self.aov_filter + + preview = match_aov_pattern(app, aov_patterns, render_file_name) + # toggle preview on if multipart is on + + if instance.data.get("multipartExr"): + log.debug("Adding preview tag because its multipartExr") + preview = True + + + new_instance = deepcopy(skeleton) + new_instance["subsetGroup"] = group_name + if preview: + new_instance["review"] = True + + # create representation + if isinstance(col, (list, tuple)): + files = [os.path.basename(f) for f in col] + else: + files = os.path.basename(col) + + # Copy render product "colorspace" data to representation. + colorspace = "" + products = additional_data["renderProducts"].layer_data.products + for product in products: + if product.productName == aov: + colorspace = product.colorspace + break + + rep = { + "name": ext, + "ext": ext, + "files": files, + "frameStart": int(skeleton["frameStartHandle"]), + "frameEnd": int(skeleton["frameEndHandle"]), + # If expectedFile are absolute, we need only filenames + "stagingDir": staging, + "fps": new_instance.get("fps"), + "tags": ["review"] if preview else [], + "colorspaceData": { + "colorspace": colorspace, + "config": { + "path": additional_data["colorspaceConfig"], + "template": additional_data["colorspaceTemplate"] + }, + "display": additional_data["display"], + "view": additional_data["view"] + } + } + + # support conversion from tiled to scanline + if instance.data.get("convertToScanline"): + log.info("Adding scanline conversion.") + rep["tags"].append("toScanline") + + # poor man exclusion + if ext in self.skip_integration_repre_list: + rep["tags"].append("delete") + + if preview: + new_instance["families"] = _solve_families(new_instance) + + new_instance["representations"] = [rep] + + # if extending frames from existing version, copy files from there + # into our destination directory + if new_instance.get("extendFrames", False): + copy_extend_frames(new_instance, rep) + instances.append(new_instance) + log.debug("instances:{}".format(instances)) + return instances + +def get_resources(project_name, version, extension=None): + """Get the files from the specific version.""" + + # TODO this functions seems to be weird + # - it's looking for representation with one extension or first (any) + # representation from a version? + # - not sure how this should work, maybe it does for specific use cases + # but probably can't be used for all resources from 2D workflows + extensions = None + if extension: + extensions = [extension] + repre_docs = list(get_representations( + project_name, version_ids=[version["_id"]], extensions=extensions + )) + assert repre_docs, "This is a bug" + + representation = repre_docs[0] + directory = get_representation_path(representation) + print("Source: ", directory) + resources = sorted( + [ + os.path.normpath(os.path.join(directory, fname)) + for fname in os.listdir(directory) + ] + ) + + return resources + +def copy_extend_frames(self, instance, representation): + """Copy existing frames from latest version. + + This will copy all existing frames from subset's latest version back + to render directory and rename them to what renderer is expecting. + + Arguments: + instance (pyblish.plugin.Instance): instance to get required + data from + representation (dict): presentation to operate on + + """ + import speedcopy + + log = Logger.get_logger("farm_publishing") + log.info("Preparing to copy ...") + start = instance.data.get("frameStart") + end = instance.data.get("frameEnd") + project_name = instance.context.data["project"] + + # get latest version of subset + # this will stop if subset wasn't published yet + + version = get_last_version_by_subset_name( + project_name, + instance.data.get("subset"), + asset_name=instance.data.get("asset") + ) + + # get its files based on extension + subset_resources = get_resources( + project_name, version, representation.get("ext") + ) + r_col, _ = clique.assemble(subset_resources) + + # if override remove all frames we are expecting to be rendered + # so we'll copy only those missing from current render + if instance.data.get("overrideExistingFrame"): + for frame in range(start, end + 1): + if frame not in r_col.indexes: + continue + r_col.indexes.remove(frame) + + # now we need to translate published names from representation + # back. This is tricky, right now we'll just use same naming + # and only switch frame numbers + resource_files = [] + r_filename = os.path.basename( + representation.get("files")[0]) # first file + op = re.search(self.R_FRAME_NUMBER, r_filename) + pre = r_filename[:op.start("frame")] + post = r_filename[op.end("frame"):] + assert op is not None, "padding string wasn't found" + for frame in list(r_col): + fn = re.search(self.R_FRAME_NUMBER, frame) + # silencing linter as we need to compare to True, not to + # type + assert fn is not None, "padding string wasn't found" + # list of tuples (source, destination) + staging = representation.get("stagingDir") + staging = self.anatomy.fill_root(staging) + resource_files.append( + (frame, os.path.join( + staging, "{}{}{}".format(pre, fn["frame"], post))) + ) + + # test if destination dir exists and create it if not + output_dir = os.path.dirname(representation.get("files")[0]) + if not os.path.isdir(output_dir): + os.makedirs(output_dir) + + # copy files + for source in resource_files: + speedcopy.copy(source[0], source[1]) + log.info(" > {}".format(source[1])) + + log.info("Finished copying %i files" % len(resource_files)) diff --git a/openpype/pipeline/farm/pyblish.pyi b/openpype/pipeline/farm/pyblish.pyi new file mode 100644 index 00000000000..3667f2d8a5d --- /dev/null +++ b/openpype/pipeline/farm/pyblish.pyi @@ -0,0 +1,23 @@ +import pyblish.api +from openpype.pipeline import Anatomy +from typing import Tuple, Union, List + + +class TimeData: + start: int + end: int + fps: float | int + step: int + handle_start: int + handle_end: int + + def __init__(self, start: int, end: int, fps: float | int, step: int, handle_start: int, handle_end: int): + ... + ... + +def remap_source(source: str, anatomy: Anatomy): ... +def extend_frames(asset: str, subset: str, start: int, end: int) -> Tuple[int, int]: ... +def get_time_data_from_instance_or_context(instance: pyblish.api.Instance) -> TimeData: ... +def get_transferable_representations(instance: pyblish.api.Instance) -> list: ... +def create_skeleton_instance(instance: pyblish.api.Instance, families_transfer: list = ..., instance_transfer: dict = ...) -> dict: ... +def create_instances_for_aov(instance: pyblish.api.Instance, skeleton: dict) -> List[pyblish.api.Instance]: ... From 7f588ee35674c364100acaa7e5b80a4364ed697a Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Fri, 21 Apr 2023 02:54:38 +0200 Subject: [PATCH 031/104] :heavy_plus_sign: add mypy --- poetry.lock | 68 +++++++++++++++++++++++++++++++++++++++++++++++--- pyproject.toml | 1 + 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index f71611cb6f7..d5b80d0f0a3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand. [[package]] name = "acre" @@ -1456,11 +1456,13 @@ python-versions = ">=3.6" files = [ {file = "lief-0.12.3-cp310-cp310-macosx_10_14_arm64.whl", hash = "sha256:66724f337e6a36cea1a9380f13b59923f276c49ca837becae2e7be93a2e245d9"}, {file = "lief-0.12.3-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:6d18aafa2028587c98f6d4387bec94346e92f2b5a8a5002f70b1cf35b1c045cc"}, + {file = "lief-0.12.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d4f69d125caaa8d5ddb574f29cc83101e165ebea1a9f18ad042eb3544081a797"}, {file = "lief-0.12.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c078d6230279ffd3bca717c79664fb8368666f610b577deb24b374607936e9c1"}, {file = "lief-0.12.3-cp310-cp310-win32.whl", hash = "sha256:e3a6af926532d0aac9e7501946134513d63217bacba666e6f7f5a0b7e15ba236"}, {file = "lief-0.12.3-cp310-cp310-win_amd64.whl", hash = "sha256:0750b72e3aa161e1fb0e2e7f571121ae05d2428aafd742ff05a7656ad2288447"}, {file = "lief-0.12.3-cp311-cp311-macosx_10_14_arm64.whl", hash = "sha256:b5c123cb99a7879d754c059e299198b34e7e30e3b64cf22e8962013db0099f47"}, {file = "lief-0.12.3-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:8bc58fa26a830df6178e36f112cb2bbdd65deff593f066d2d51434ff78386ba5"}, + {file = "lief-0.12.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:74ac6143ac6ccd813c9b068d9c5f1f9d55c8813c8b407387eb57de01c3db2d74"}, {file = "lief-0.12.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:04eb6b70d646fb5bd6183575928ee23715550f161f2832cbcd8c6ff2071fb408"}, {file = "lief-0.12.3-cp311-cp311-win32.whl", hash = "sha256:7e2d0a53c403769b04adcf8df92e83c5e25f9103a052aa7f17b0a9cf057735fb"}, {file = "lief-0.12.3-cp311-cp311-win_amd64.whl", hash = "sha256:7f6395c12ee1bc4a5162f567cba96d0c72dfb660e7902e84d4f3029daf14fe33"}, @@ -1480,6 +1482,7 @@ files = [ {file = "lief-0.12.3-cp38-cp38-win_amd64.whl", hash = "sha256:b00667257b43e93d94166c959055b6147d46d302598f3ee55c194b40414c89cc"}, {file = "lief-0.12.3-cp39-cp39-macosx_10_14_arm64.whl", hash = "sha256:e6a1b5b389090d524621c2455795e1262f62dc9381bedd96f0cd72b878c4066d"}, {file = "lief-0.12.3-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:ae773196df814202c0c51056163a1478941b299512b09660a3c37be3c7fac81e"}, + {file = "lief-0.12.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:66ddf88917ec7b00752687c476bb2771dc8ec19bd7e4c0dcff1f8ef774cad4e9"}, {file = "lief-0.12.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:4a47f410032c63ac3be051d963d0337d6b47f0e94bfe8e946ab4b6c428f4d0f8"}, {file = "lief-0.12.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dbd11367c2259bd1131a6c8755dcde33314324de5ea029227bfbc7d3755871e6"}, {file = "lief-0.12.3-cp39-cp39-win32.whl", hash = "sha256:2ce53e311918c3e5b54c815ef420a747208d2a88200c41cd476f3dd1eb876bcf"}, @@ -1676,6 +1679,65 @@ files = [ {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, ] +[[package]] +name = "mypy" +version = "1.2.0" +description = "Optional static typing for Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mypy-1.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:701189408b460a2ff42b984e6bd45c3f41f0ac9f5f58b8873bbedc511900086d"}, + {file = "mypy-1.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fe91be1c51c90e2afe6827601ca14353bbf3953f343c2129fa1e247d55fd95ba"}, + {file = "mypy-1.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d26b513225ffd3eacece727f4387bdce6469192ef029ca9dd469940158bc89e"}, + {file = "mypy-1.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3a2d219775a120581a0ae8ca392b31f238d452729adbcb6892fa89688cb8306a"}, + {file = "mypy-1.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:2e93a8a553e0394b26c4ca683923b85a69f7ccdc0139e6acd1354cc884fe0128"}, + {file = "mypy-1.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3efde4af6f2d3ccf58ae825495dbb8d74abd6d176ee686ce2ab19bd025273f41"}, + {file = "mypy-1.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:695c45cea7e8abb6f088a34a6034b1d273122e5530aeebb9c09626cea6dca4cb"}, + {file = "mypy-1.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0e9464a0af6715852267bf29c9553e4555b61f5904a4fc538547a4d67617937"}, + {file = "mypy-1.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8293a216e902ac12779eb7a08f2bc39ec6c878d7c6025aa59464e0c4c16f7eb9"}, + {file = "mypy-1.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:f46af8d162f3d470d8ffc997aaf7a269996d205f9d746124a179d3abe05ac602"}, + {file = "mypy-1.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:031fc69c9a7e12bcc5660b74122ed84b3f1c505e762cc4296884096c6d8ee140"}, + {file = "mypy-1.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:390bc685ec209ada4e9d35068ac6988c60160b2b703072d2850457b62499e336"}, + {file = "mypy-1.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4b41412df69ec06ab141808d12e0bf2823717b1c363bd77b4c0820feaa37249e"}, + {file = "mypy-1.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:4e4a682b3f2489d218751981639cffc4e281d548f9d517addfd5a2917ac78119"}, + {file = "mypy-1.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a197ad3a774f8e74f21e428f0de7f60ad26a8d23437b69638aac2764d1e06a6a"}, + {file = "mypy-1.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c9a084bce1061e55cdc0493a2ad890375af359c766b8ac311ac8120d3a472950"}, + {file = "mypy-1.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eaeaa0888b7f3ccb7bcd40b50497ca30923dba14f385bde4af78fac713d6d6f6"}, + {file = "mypy-1.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bea55fc25b96c53affab852ad94bf111a3083bc1d8b0c76a61dd101d8a388cf5"}, + {file = "mypy-1.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:4c8d8c6b80aa4a1689f2a179d31d86ae1367ea4a12855cc13aa3ba24bb36b2d8"}, + {file = "mypy-1.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:70894c5345bea98321a2fe84df35f43ee7bb0feec117a71420c60459fc3e1eed"}, + {file = "mypy-1.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4a99fe1768925e4a139aace8f3fb66db3576ee1c30b9c0f70f744ead7e329c9f"}, + {file = "mypy-1.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:023fe9e618182ca6317ae89833ba422c411469156b690fde6a315ad10695a521"}, + {file = "mypy-1.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4d19f1a239d59f10fdc31263d48b7937c585810288376671eaf75380b074f238"}, + {file = "mypy-1.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:2de7babe398cb7a85ac7f1fd5c42f396c215ab3eff731b4d761d68d0f6a80f48"}, + {file = "mypy-1.2.0-py3-none-any.whl", hash = "sha256:d8e9187bfcd5ffedbe87403195e1fc340189a68463903c39e2b63307c9fa0394"}, + {file = "mypy-1.2.0.tar.gz", hash = "sha256:f70a40410d774ae23fcb4afbbeca652905a04de7948eaf0b1789c8d1426b72d1"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=3.10" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +python2 = ["typed-ast (>=1.4.0,<2)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +category = "dev" +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + [[package]] name = "nodeenv" version = "1.7.0" @@ -2352,7 +2414,7 @@ files = [ cffi = ">=1.4.1" [package.extras] -docs = ["sphinx (>=1.6.5)", "sphinx_rtd_theme"] +docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] [[package]] @@ -3462,4 +3524,4 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [metadata] lock-version = "2.0" python-versions = ">=3.9.1,<3.10" -content-hash = "02daca205796a0f29a0d9f50707544e6804f32027eba493cd2aa7f175a00dcea" +content-hash = "9d3a574b1b6f42ae05d4f0fa6d65677ee54a51c53d984dd3f44d02f234962dbb" diff --git a/pyproject.toml b/pyproject.toml index 42ce5aa32c7..afc6cc45d99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,6 +94,7 @@ wheel = "*" enlighten = "*" # cool terminal progress bars toml = "^0.10.2" # for parsing pyproject.toml pre-commit = "*" +mypy = "*" # for better types [tool.poetry.urls] "Bug Tracker" = "https://github.com/pypeclub/openpype/issues" From 3a9109422d499f9ab843e1fa26f8a02c2d463f55 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Fri, 28 Apr 2023 15:17:47 +0200 Subject: [PATCH 032/104] :art: update code from upstream --- openpype/pipeline/farm/tools.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/openpype/pipeline/farm/tools.py b/openpype/pipeline/farm/tools.py index 8cf1af399ea..506f95d6b27 100644 --- a/openpype/pipeline/farm/tools.py +++ b/openpype/pipeline/farm/tools.py @@ -53,14 +53,11 @@ def from_published_scene(instance, replace_in_path=True): template_data["comment"] = None anatomy = instance.context.data['anatomy'] - anatomy_filled = anatomy.format(template_data) - template_filled = anatomy_filled["publish"]["path"] + template_obj = anatomy.templates_obj["publish"]["path"] + template_filled = template_obj.format_strict(template_data) file_path = os.path.normpath(template_filled) - self.log.info("Using published scene for render {}".format(file_path)) - if not os.path.exists(file_path): - self.log.error("published scene does not exist!") raise if not replace_in_path: @@ -102,8 +99,4 @@ def _clean_name(path): new_scene) instance.data["publishRenderMetadataFolder"] = metadata_folder - self.log.info("Scene name was switched {} -> {}".format( - orig_scene, new_scene - )) - return file_path From 92bb47ed231216ea51f4cdb70084358190082d12 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Fri, 28 Apr 2023 18:37:26 +0200 Subject: [PATCH 033/104] :art: add nuke job creator --- .../publish/create_nuke_deadline_job.py | 307 ++++++++++++++++++ .../publish/create_publish_royalrender_job.py | 119 ++++--- openpype/modules/royalrender/rr_job.py | 22 +- openpype/pipeline/farm/pyblish.py | 112 +++++-- openpype/pipeline/farm/pyblish.pyi | 3 +- 5 files changed, 487 insertions(+), 76 deletions(-) create mode 100644 openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py diff --git a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py new file mode 100644 index 00000000000..9f49294459d --- /dev/null +++ b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py @@ -0,0 +1,307 @@ +# -*- coding: utf-8 -*- +"""Submitting render job to RoyalRender.""" +import os +import sys +import re +import platform +from datetime import datetime + +from pyblish.api import InstancePlugin, IntegratorOrder, Context +from openpype.tests.lib import is_in_tests +from openpype.lib import is_running_from_build +from openpype.pipeline.publish.lib import get_published_workfile_instance +from openpype.pipeline.publish import KnownPublishError +from openpype.modules.royalrender.api import Api as rrApi +from openpype.modules.royalrender.rr_job import ( + RRJob, CustomAttribute, get_rr_platform) +from openpype.lib import ( + is_running_from_build, + BoolDef, + NumberDef +) + + +class CreateNukeRoyalRenderJob(InstancePlugin): + label = "Create Nuke Render job in RR" + order = IntegratorOrder + 0.1 + hosts = ["nuke"] + families = ["render", "prerender"] + targets = ["local"] + optional = True + + priority = 50 + chunk_size = 1 + concurrent_tasks = 1 + + @classmethod + def get_attribute_defs(cls): + return [ + NumberDef( + "priority", + label="Priority", + default=cls.priority, + decimals=0 + ), + NumberDef( + "chunk", + label="Frames Per Task", + default=cls.chunk_size, + decimals=0, + minimum=1, + maximum=1000 + ), + NumberDef( + "concurrency", + label="Concurrency", + default=cls.concurrent_tasks, + decimals=0, + minimum=1, + maximum=10 + ), + BoolDef( + "use_gpu", + default=cls.use_gpu, + label="Use GPU" + ), + BoolDef( + "suspend_publish", + default=False, + label="Suspend publish" + ) + ] + + def __init__(self, *args, **kwargs): + self._instance = None + self._rr_root = None + self.scene_path = None + self.job = None + self.submission_parameters = None + self.rr_api = None + + def process(self, instance): + if not instance.data.get("farm"): + self.log.info("Skipping local instance.") + return + + instance.data["attributeValues"] = self.get_attr_values_from_data( + instance.data) + + # add suspend_publish attributeValue to instance data + instance.data["suspend_publish"] = instance.data["attributeValues"][ + "suspend_publish"] + + context = instance.context + + self._rr_root = self._resolve_rr_path(context, instance.data.get( + "rrPathName")) # noqa + self.log.debug(self._rr_root) + if not self._rr_root: + raise KnownPublishError( + ("Missing RoyalRender root. " + "You need to configure RoyalRender module.")) + + self.rr_api = rrApi(self._rr_root) + + self.scene_path = context.data["currentFile"] + if self.use_published: + file_path = get_published_workfile_instance(context) + + # fallback if nothing was set + if not file_path: + self.log.warning("Falling back to workfile") + file_path = context.data["currentFile"] + + self.scene_path = file_path + self.log.info( + "Using published scene for render {}".format(self.scene_path) + ) + + if not self._instance.data.get("expectedFiles"): + self._instance.data["expectedFiles"] = [] + + if not self._instance.data.get("rrJobs"): + self._instance.data["rrJobs"] = [] + + self._instance.data["rrJobs"] += self.create_jobs() + + # redefinition of families + if "render" in self._instance.data["family"]: + self._instance.data["family"] = "write" + self._instance.data["families"].insert(0, "render2d") + elif "prerender" in self._instance.data["family"]: + self._instance.data["family"] = "write" + self._instance.data["families"].insert(0, "prerender") + + self._instance.data["outputDir"] = os.path.dirname( + self._instance.data["path"]).replace("\\", "/") + + + def create_jobs(self): + submit_frame_start = int(self._instance.data["frameStartHandle"]) + submit_frame_end = int(self._instance.data["frameEndHandle"]) + + # get output path + render_path = self._instance.data['path'] + script_path = self.scene_path + node = self._instance.data["transientData"]["node"] + + # main job + jobs = [ + self.get_job( + script_path, + render_path, + node.name(), + submit_frame_start, + submit_frame_end, + ) + ] + + for baking_script in self._instance.data.get("bakingNukeScripts", []): + render_path = baking_script["bakeRenderPath"] + script_path = baking_script["bakeScriptPath"] + exe_node_name = baking_script["bakeWriteNodeName"] + + jobs.append(self.get_job( + script_path, + render_path, + exe_node_name, + submit_frame_start, + submit_frame_end + )) + + return jobs + + def get_job(self, script_path, render_path, + node_name, start_frame, end_frame): + """Get RR job based on current instance. + + Args: + script_path (str): Path to Nuke script. + render_path (str): Output path. + node_name (str): Name of the render node. + start_frame (int): Start frame. + end_frame (int): End frame. + + Returns: + RRJob: RoyalRender Job instance. + + """ + render_dir = os.path.normpath(os.path.dirname(render_path)) + batch_name = os.path.basename(script_path) + jobname = "%s - %s" % (batch_name, self._instance.name) + if is_in_tests(): + batch_name += datetime.now().strftime("%d%m%Y%H%M%S") + + output_filename_0 = self.preview_fname(render_path) + + custom_attributes = [] + if is_running_from_build(): + custom_attributes = [ + CustomAttribute( + name="OpenPypeVersion", + value=os.environ.get("OPENPYPE_VERSION")) + ] + + nuke_version = re.search( + r"\d+\.\d+", self._instance.context.data.get("hostVersion")) + + # this will append expected files to instance as needed. + expected_files = self.expected_files( + render_path, start_frame, end_frame) + first_file = next(self._iter_expected_files(expected_files)) + + job = RRJob( + Software="Nuke", + Renderer="", + SeqStart=int(start_frame), + SeqEnd=int(end_frame), + SeqStep=int(self._instance.data.get("byFrameStep"), 1), + SeqFileOffset=0, + Version=nuke_version.group(), + SceneName=script_path, + IsActive=True, + ImageDir=render_dir.replace("\\", "/"), + ImageFilename="{}.".format(os.path.splitext(first_file)[0]), + ImageExtension=os.path.splitext(first_file)[1], + ImagePreNumberLetter=".", + ImageSingleOutputFile=False, + SceneOS=get_rr_platform(), + Layer=node_name, + SceneDatabaseDir=script_path, + CustomSHotName=self._instance.context.data["asset"], + CompanyProjectName=self._instance.context.data["projectName"], + ImageWidth=self._instance.data["resolutionWidth"], + ImageHeight=self._instance.data["resolutionHeight"], + CustomAttributes=custom_attributes + ) + + @staticmethod + def _resolve_rr_path(context, rr_path_name): + # type: (Context, str) -> str + rr_settings = ( + context.data + ["system_settings"] + ["modules"] + ["royalrender"] + ) + try: + default_servers = rr_settings["rr_paths"] + project_servers = ( + context.data + ["project_settings"] + ["royalrender"] + ["rr_paths"] + ) + rr_servers = { + k: default_servers[k] + for k in project_servers + if k in default_servers + } + + except (AttributeError, KeyError): + # Handle situation were we had only one url for royal render. + return context.data["defaultRRPath"][platform.system().lower()] + + return rr_servers[rr_path_name][platform.system().lower()] + + def expected_files(self, path, start_frame, end_frame): + """Get expected files. + + This function generate expected files from provided + path and start/end frames. + + It was taken from Deadline module, but this should be + probably handled better in collector to support more + flexible scenarios. + + Args: + path (str): Output path. + start_frame (int): Start frame. + end_frame (int): End frame. + + Returns: + list: List of expected files. + + """ + dir_name = os.path.dirname(path) + file = os.path.basename(path) + + expected_files = [] + + if "#" in file: + pparts = file.split("#") + padding = "%0{}d".format(len(pparts) - 1) + file = pparts[0] + padding + pparts[-1] + + if "%" not in file: + expected_files.append(path) + return + + if self._instance.data.get("slate"): + start_frame -= 1 + + expected_files.extend( + os.path.join(dir_name, (file % i)).replace("\\", "/") + for i in range(start_frame, (end_frame + 1)) + ) + return expected_files diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index a5493dd0614..b1c84c87b96 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -2,14 +2,20 @@ """Create publishing job on RoyalRender.""" import os from copy import deepcopy +import json -from pyblish.api import InstancePlugin, IntegratorOrder +from pyblish.api import InstancePlugin, IntegratorOrder, Instance from openpype.pipeline import legacy_io from openpype.modules.royalrender.rr_job import RRJob, RREnvList from openpype.pipeline.publish import KnownPublishError from openpype.lib.openpype_version import ( get_OpenPypeVersion, get_openpype_version) +from openpype.pipeline.farm.pyblish import ( + create_skeleton_instance, + create_instances_for_aov, + attach_instances_to_subset +) class CreatePublishRoyalRenderJob(InstancePlugin): @@ -31,50 +37,84 @@ def process(self, instance): self.context = context self.anatomy = instance.context.data["anatomy"] - # asset = data.get("asset") - # subset = data.get("subset") - # source = self._remap_source( - # data.get("source") or context.data["source"]) + if not instance.data.get("farm"): + self.log.info("Skipping local instance.") + return + + instance_skeleton_data = create_skeleton_instance( + instance, + families_transfer=self.families_transfer, + instance_transfer=self.instance_transfer) + + instances = None + if isinstance(instance.data.get("expectedFiles")[0], dict): + instances = create_instances_for_aov( + instance, instance_skeleton_data, self.aov_filter) - def _remap_source(self, source): - success, rootless_path = ( - self.anatomy.find_root_template_from_path(source) - ) - if success: - source = rootless_path else: - # `rootless_path` is not set to `source` if none of roots match - self.log.warning(( - "Could not find root path for remapping \"{}\"." - " This may cause issues." - ).format(source)) - return source - - def get_job(self, instance, job, instances): - """Submit publish job to RoyalRender.""" + representations = self._get_representations( + instance_skeleton_data, + instance.data.get("expectedFiles") + ) + + if "representations" not in instance_skeleton_data.keys(): + instance_skeleton_data["representations"] = [] + + # add representation + instance_skeleton_data["representations"] += representations + instances = [instance_skeleton_data] + + # attach instances to subset + if instance.data.get("attachTo"): + instances = attach_instances_to_subset( + instance.data.get("attachTo"), instances + ) + + self.log.info("Creating RoyalRender Publish job ...") + + if not instance.data.get("rrJobs"): + self.log.error(("There is no prior RoyalRender " + "job on the instance.")) + raise KnownPublishError( + "Can't create publish job without prior ppducing jobs first") + + publish_job = self.get_job(instance, instances) + + instance.data["rrJobs"] += publish_job + + metadata_path, rootless_metadata_path = self._create_metadata_path( + instance) + + self.log.info("Writing json file: {}".format(metadata_path)) + with open(metadata_path, "w") as f: + json.dump(publish_job, f, indent=4, sort_keys=True) + + def get_job(self, instance, instances): + """Create RR publishing job. + + Based on provided original instance and additional instances, + create publishing job and return it to be submitted to farm. + + Args: + instance (Instance): Original instance. + instances (list of Instance): List of instances to + be published on farm. + + Returns: + RRJob: RoyalRender publish job. + + """ data = instance.data.copy() subset = data["subset"] job_name = "Publish - {subset}".format(subset=subset) - override_version = None instance_version = instance.data.get("version") # take this if exists - if instance_version != 1: - override_version = instance_version - output_dir = self._get_publish_folder( - instance.context.data['anatomy'], - deepcopy(instance.data["anatomyData"]), - instance.data.get("asset"), - instances[0]["subset"], - # TODO: this shouldn't be hardcoded and is in fact settable by - # Settings. - 'render', - override_version - ) + override_version = instance_version if instance_version != 1 else None # Transfer the environment from the original job to this dependent # job, so they use the same environment metadata_path, roothless_metadata_path = \ - self._create_metadata_path(instance) + self._create_metadata_path(instance) environment = RREnvList({ "AVALON_PROJECT": legacy_io.Session["AVALON_PROJECT"], @@ -96,7 +136,7 @@ def get_job(self, instance, job, instances): # and collect all pre_ids to wait for job_environ = {} jobs_pre_ids = [] - for job in instance["rrJobs"]: # type: RRJob + for job in instance.data["rrJobs"]: # type: RRJob if job.rrEnvList: job_environ.update( dict(RREnvList.parse(job.rrEnvList)) @@ -159,11 +199,4 @@ def get_job(self, instance, job, instances): else: job.WaitForPreIDs += jobs_pre_ids - self.log.info("Creating RoyalRender Publish job ...") - - if not instance.data.get("rrJobs"): - self.log.error("There is no RoyalRender job on the instance.") - raise KnownPublishError( - "Can't create publish job without producing jobs") - - instance.data["rrJobs"] += job + return job diff --git a/openpype/modules/royalrender/rr_job.py b/openpype/modules/royalrender/rr_job.py index 689a488a5c3..5f034e74a1b 100644 --- a/openpype/modules/royalrender/rr_job.py +++ b/openpype/modules/royalrender/rr_job.py @@ -9,6 +9,17 @@ CustomAttribute = namedtuple("CustomAttribute", ["name", "value"]) +def get_rr_platform(): + # type: () -> str + """Returns name of platform used in rr jobs.""" + if sys.platform.lower() in ["win32", "win64"]: + return "windows" + elif sys.platform.lower() == "darwin": + return "mac" + else: + return "linux" + + class RREnvList(dict): def serialize(self): # VariableA=ValueA~~~VariableB=ValueB @@ -163,17 +174,6 @@ class RRJob: # only used in RR 8.3 and newer rrEnvList = attr.ib(default=None) # type: str - @staticmethod - def get_rr_platform(): - # type: () -> str - """Returns name of platform used in rr jobs.""" - if sys.platform.lower() in ["win32", "win64"]: - return "windows" - elif sys.platform.lower() == "darwin": - return "mac" - else: - return "linux" - class SubmitterParameter: """Wrapper for Submitter Parameters.""" diff --git a/openpype/pipeline/farm/pyblish.py b/openpype/pipeline/farm/pyblish.py index 35a944444fa..e5ebd3666c0 100644 --- a/openpype/pipeline/farm/pyblish.py +++ b/openpype/pipeline/farm/pyblish.py @@ -11,10 +11,12 @@ import attr import pyblish.api from openpype.pipeline.publish import KnownPublishError +from openpype.pipeline.farm.patterning import match_aov_pattern import os import clique from copy import deepcopy import re +import warnings @attr.s @@ -263,6 +265,7 @@ def create_skeleton_instance( return instance_skeleton_data + def _solve_families(families): """Solve families. @@ -277,7 +280,9 @@ def _solve_families(families): if "review" not in families: families.append("review") return families -def create_instances_for_aov(instance, skeleton): + + +def create_instances_for_aov(instance, skeleton, aov_filter): """Create instances from AOVs. This will create new pyblish.api.Instances by going over expected @@ -328,11 +333,12 @@ def create_instances_for_aov(instance, skeleton): return _create_instances_for_aov( instance, skeleton, + aov_filter, additional_color_data ) -def _create_instances_for_aov(instance, skeleton, additional_data): +def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data): """Create instance for each AOV found. This will create new instance for every AOV it can detect in expected @@ -491,35 +497,64 @@ def _create_instances_for_aov(instance, skeleton, additional_data): log.debug("instances:{}".format(instances)) return instances + def get_resources(project_name, version, extension=None): - """Get the files from the specific version.""" - - # TODO this functions seems to be weird - # - it's looking for representation with one extension or first (any) - # representation from a version? - # - not sure how this should work, maybe it does for specific use cases - # but probably can't be used for all resources from 2D workflows - extensions = None + """Get the files from the specific version. + + This will return all get all files from representation. + + Todo: + This is really weird function, and it's use is + highly controversial. First, it will not probably work + ar all in final release of AYON, second, the logic isn't sound. + It should try to find representation matching the current one - + because it is used to pull out files from previous version to + be included in this one. + + .. deprecated:: 3.15.5 + This won't work in AYON and even the logic must be refactored. + + Args: + project_name (str): Name of the project. + version (dict): Version document. + extension (str): extension used to filter + representations. + + Returns: + list: of files + + """ + warnings.warn(( + "This won't work in AYON and even " + "the logic must be refactored."), DeprecationWarning) + extensions = [] if extension: extensions = [extension] + + # there is a `context_filter` argument that won't probably work in + # final release of AYON. SO we'll rather not use it repre_docs = list(get_representations( - project_name, version_ids=[version["_id"]], extensions=extensions - )) - assert repre_docs, "This is a bug" + project_name, version_ids=[version["_id"]])) + + filtered = [] + for doc in repre_docs: + if doc["context"]["ext"] in extensions: + filtered.append(doc) - representation = repre_docs[0] + representation = filtered[0] directory = get_representation_path(representation) print("Source: ", directory) resources = sorted( [ - os.path.normpath(os.path.join(directory, fname)) - for fname in os.listdir(directory) + os.path.normpath(os.path.join(directory, file_name)) + for file_name in os.listdir(directory) ] ) return resources -def copy_extend_frames(self, instance, representation): + +def copy_extend_frames(instance, representation): """Copy existing frames from latest version. This will copy all existing frames from subset's latest version back @@ -533,11 +568,15 @@ def copy_extend_frames(self, instance, representation): """ import speedcopy + R_FRAME_NUMBER = re.compile( + r".+\.(?P[0-9]+)\..+") + log = Logger.get_logger("farm_publishing") log.info("Preparing to copy ...") start = instance.data.get("frameStart") end = instance.data.get("frameEnd") project_name = instance.context.data["project"] + anatomy = instance.data["anatomy"] # type: Anatomy # get latest version of subset # this will stop if subset wasn't published yet @@ -554,7 +593,7 @@ def copy_extend_frames(self, instance, representation): ) r_col, _ = clique.assemble(subset_resources) - # if override remove all frames we are expecting to be rendered + # if override remove all frames we are expecting to be rendered, # so we'll copy only those missing from current render if instance.data.get("overrideExistingFrame"): for frame in range(start, end + 1): @@ -568,18 +607,18 @@ def copy_extend_frames(self, instance, representation): resource_files = [] r_filename = os.path.basename( representation.get("files")[0]) # first file - op = re.search(self.R_FRAME_NUMBER, r_filename) + op = re.search(R_FRAME_NUMBER, r_filename) pre = r_filename[:op.start("frame")] post = r_filename[op.end("frame"):] assert op is not None, "padding string wasn't found" for frame in list(r_col): - fn = re.search(self.R_FRAME_NUMBER, frame) + fn = re.search(R_FRAME_NUMBER, frame) # silencing linter as we need to compare to True, not to # type assert fn is not None, "padding string wasn't found" # list of tuples (source, destination) staging = representation.get("stagingDir") - staging = self.anatomy.fill_root(staging) + staging = anatomy.fill_root(staging) resource_files.append( (frame, os.path.join( staging, "{}{}{}".format(pre, fn["frame"], post))) @@ -596,3 +635,34 @@ def copy_extend_frames(self, instance, representation): log.info(" > {}".format(source[1])) log.info("Finished copying %i files" % len(resource_files)) + + +def attach_instances_to_subset(attach_to, instances): + """Attach instance to subset. + + If we are attaching to other subsets, create copy of existing + instances, change data to match its subset and replace + existing instances with modified data. + + Args: + attach_to (list): List of instances to attach to. + instances (list): List of instances to attach. + + Returns: + list: List of attached instances. + + """ + # + + new_instances = [] + for attach_instance in attach_to: + for i in instances: + new_inst = copy(i) + new_inst["version"] = attach_instance.get("version") + new_inst["subset"] = attach_instance.get("subset") + new_inst["family"] = attach_instance.get("family") + new_inst["append"] = True + # don't set subsetGroup if we are attaching + new_inst.pop("subsetGroup") + new_instances.append(new_inst) + return new_instances diff --git a/openpype/pipeline/farm/pyblish.pyi b/openpype/pipeline/farm/pyblish.pyi index 3667f2d8a5d..76f7c34dcd6 100644 --- a/openpype/pipeline/farm/pyblish.pyi +++ b/openpype/pipeline/farm/pyblish.pyi @@ -20,4 +20,5 @@ def extend_frames(asset: str, subset: str, start: int, end: int) -> Tuple[int, i def get_time_data_from_instance_or_context(instance: pyblish.api.Instance) -> TimeData: ... def get_transferable_representations(instance: pyblish.api.Instance) -> list: ... def create_skeleton_instance(instance: pyblish.api.Instance, families_transfer: list = ..., instance_transfer: dict = ...) -> dict: ... -def create_instances_for_aov(instance: pyblish.api.Instance, skeleton: dict) -> List[pyblish.api.Instance]: ... +def create_instances_for_aov(instance: pyblish.api.Instance, skeleton: dict, aov_filter: dict) -> List[pyblish.api.Instance]: ... +def attach_instances_to_subset(attach_to: list, instances: list) -> list: ... From 1f0572aaa0c1e69f8a6b5cf33d20546029bf6f53 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Fri, 28 Apr 2023 18:37:56 +0200 Subject: [PATCH 034/104] :recycle: refactor deadline code to make use of abstracted code --- .../plugins/publish/submit_publish_job.py | 27 +++++-------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 58f34c24b17..86f647dd1bd 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -23,7 +23,8 @@ from openpype.lib import is_running_from_build from openpype.pipeline.farm.pyblish import ( create_skeleton_instance, - create_instances_for_aov + create_instances_for_aov, + attach_instances_to_subset ) @@ -551,7 +552,7 @@ def process(self, instance): if isinstance(instance.data.get("expectedFiles")[0], dict): instances = create_instances_for_aov( - instance, instance_skeleton_data) + instance, instance_skeleton_data, self.aov_filter) else: representations = self._get_representations( @@ -566,25 +567,11 @@ def process(self, instance): instance_skeleton_data["representations"] += representations instances = [instance_skeleton_data] - # if we are attaching to other subsets, create copy of existing - # instances, change data to match its subset and replace - # existing instances with modified data + # attach instances to subset if instance.data.get("attachTo"): - self.log.info("Attaching render to subset:") - new_instances = [] - for at in instance.data.get("attachTo"): - for i in instances: - new_i = copy(i) - new_i["version"] = at.get("version") - new_i["subset"] = at.get("subset") - new_i["family"] = at.get("family") - new_i["append"] = True - # don't set subsetGroup if we are attaching - new_i.pop("subsetGroup") - new_instances.append(new_i) - self.log.info(" - {} / v{}".format( - at.get("subset"), at.get("version"))) - instances = new_instances + instances = attach_instances_to_subset( + instance.data.get("attachTo"), instances + ) r''' SUBMiT PUBLiSH JOB 2 D34DLiN3 ____ From 806b9250c89838b84a33b8fd24849a62dd1f35b7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 3 May 2023 18:32:46 +0200 Subject: [PATCH 035/104] Fix wrong family 'rendering' is obsolete --- .../plugins/publish/collect_rr_path_from_instance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py b/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py index cfb5b78077f..e21cd7c39f9 100644 --- a/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py +++ b/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py @@ -7,7 +7,7 @@ class CollectRRPathFromInstance(pyblish.api.InstancePlugin): order = pyblish.api.CollectorOrder label = "Collect Royal Render path name from the Instance" - families = ["rendering"] + families = ["render"] def process(self, instance): instance.data["rrPathName"] = self._collect_rr_path_name(instance) From ddab9240a1d1c51c06787ae24f9d17767bdb7a28 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 3 May 2023 18:35:22 +0200 Subject: [PATCH 036/104] Extract iter_expected_files --- .../plugins/publish/submit_maya_deadline.py | 15 +++------------ .../publish/create_maya_royalrender_job.py | 13 ++----------- .../plugins/publish/create_nuke_deadline_job.py | 7 +++++-- openpype/pipeline/farm/tools.py | 10 ++++++++++ 4 files changed, 20 insertions(+), 25 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index a6cdcb7e711..16fba382e46 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -39,6 +39,7 @@ from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo from openpype.tests.lib import is_in_tests from openpype.lib import is_running_from_build +from openpype.pipeline.farm.tools import iter_expected_files def _validate_deadline_bool_value(instance, attribute, value): @@ -198,7 +199,7 @@ def get_job_info(self): # Add list of expected files to job # --------------------------------- exp = instance.data.get("expectedFiles") - for filepath in self._iter_expected_files(exp): + for filepath in iter_expected_files(exp): job_info.OutputDirectory += os.path.dirname(filepath) job_info.OutputFilename += os.path.basename(filepath) @@ -254,7 +255,7 @@ def process_submission(self): # TODO: Avoid the need for this logic here, needed for submit publish # Store output dir for unified publisher (filesequence) expected_files = instance.data["expectedFiles"] - first_file = next(self._iter_expected_files(expected_files)) + first_file = next(iter_expected_files(expected_files)) output_dir = os.path.dirname(first_file) instance.data["outputDir"] = output_dir instance.data["toBeRenderedOn"] = "deadline" @@ -772,16 +773,6 @@ def _job_info_label(self, label): end=int(self._instance.data["frameEndHandle"]), ) - @staticmethod - def _iter_expected_files(exp): - if isinstance(exp[0], dict): - for _aov, files in exp[0].items(): - for file in files: - yield file - else: - for file in exp: - yield file - def _format_tiles( filename, diff --git a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py index 6e23fb0b749..8461e74d6d9 100644 --- a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py @@ -11,6 +11,7 @@ from openpype.pipeline.publish import KnownPublishError from openpype.modules.royalrender.api import Api as rrApi from openpype.modules.royalrender.rr_job import RRJob, CustomAttribute +from openpype.pipeline.farm.tools import iter_expected_files class CreateMayaRoyalRenderJob(InstancePlugin): @@ -43,7 +44,7 @@ def get_rr_platform(): return "linux" expected_files = self._instance.data["expectedFiles"] - first_file = next(self._iter_expected_files(expected_files)) + first_file = next(iter_expected_files(expected_files)) output_dir = os.path.dirname(first_file) self._instance.data["outputDir"] = output_dir workspace = self._instance.context.data["workspaceDir"] @@ -125,16 +126,6 @@ def process(self, instance): self._instance.data["rrJobs"] += self.get_job() - @staticmethod - def _iter_expected_files(exp): - if isinstance(exp[0], dict): - for _aov, files in exp[0].items(): - for file in files: - yield file - else: - for file in exp: - yield file - @staticmethod def _resolve_rr_path(context, rr_path_name): # type: (Context, str) -> str diff --git a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py index 9f49294459d..217ebed057b 100644 --- a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py @@ -19,9 +19,11 @@ BoolDef, NumberDef ) +from openpype.pipeline import OpenPypePyblishPluginMixin +from openpype.pipeline.farm.tools import iter_expected_files -class CreateNukeRoyalRenderJob(InstancePlugin): +class CreateNukeRoyalRenderJob(InstancePlugin, OpenPypePyblishPluginMixin): label = "Create Nuke Render job in RR" order = IntegratorOrder + 0.1 hosts = ["nuke"] @@ -208,7 +210,8 @@ def get_job(self, script_path, render_path, # this will append expected files to instance as needed. expected_files = self.expected_files( render_path, start_frame, end_frame) - first_file = next(self._iter_expected_files(expected_files)) + self._instance.data["expectedFiles"].extend(expected_files) + first_file = next(iter_expected_files(expected_files)) job = RRJob( Software="Nuke", diff --git a/openpype/pipeline/farm/tools.py b/openpype/pipeline/farm/tools.py index 506f95d6b27..6f9e0ac3935 100644 --- a/openpype/pipeline/farm/tools.py +++ b/openpype/pipeline/farm/tools.py @@ -100,3 +100,13 @@ def _clean_name(path): instance.data["publishRenderMetadataFolder"] = metadata_folder return file_path + + +def iter_expected_files(exp): + if isinstance(exp[0], dict): + for _aov, files in exp[0].items(): + for file in files: + yield file + else: + for file in exp: + yield file From 7fe4820bec138f5f740793f3d51a346176dece49 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 3 May 2023 18:37:49 +0200 Subject: [PATCH 037/104] Renamed file Original name collided in Nuke (Python2). --- .../farm/{pyblish.py => pyblish_functions.py} | 20 ++++++++++--------- .../{pyblish.pyi => pyblish_functions.pyi} | 0 2 files changed, 11 insertions(+), 9 deletions(-) rename openpype/pipeline/farm/{pyblish.py => pyblish_functions.py} (99%) rename openpype/pipeline/farm/{pyblish.pyi => pyblish_functions.pyi} (100%) diff --git a/openpype/pipeline/farm/pyblish.py b/openpype/pipeline/farm/pyblish_functions.py similarity index 99% rename from openpype/pipeline/farm/pyblish.py rename to openpype/pipeline/farm/pyblish_functions.py index e5ebd3666c0..84d958bea14 100644 --- a/openpype/pipeline/farm/pyblish.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -1,3 +1,12 @@ +import copy +import attr +from pyblish.api import Instance +import os +import clique +from copy import deepcopy +import re +import warnings + from openpype.pipeline import ( get_current_project_name, get_representation_path, @@ -8,19 +17,12 @@ get_representations ) from openpype.lib import Logger -import attr -import pyblish.api from openpype.pipeline.publish import KnownPublishError from openpype.pipeline.farm.patterning import match_aov_pattern -import os -import clique -from copy import deepcopy -import re -import warnings @attr.s -class TimeData: +class TimeData(object): """Structure used to handle time related data.""" start = attr.ib(type=int) end = attr.ib(type=int) @@ -160,7 +162,7 @@ def get_transferable_representations(instance): def create_skeleton_instance( instance, families_transfer=None, instance_transfer=None): - # type: (pyblish.api.Instance, list, dict) -> dict + # type: (Instance, list, dict) -> dict """Create skeleton instance from original instance data. This will create dictionary containing skeleton diff --git a/openpype/pipeline/farm/pyblish.pyi b/openpype/pipeline/farm/pyblish_functions.pyi similarity index 100% rename from openpype/pipeline/farm/pyblish.pyi rename to openpype/pipeline/farm/pyblish_functions.pyi From 266d34bebb1e1ab07554edadee556da14a2d84a5 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 3 May 2023 18:38:55 +0200 Subject: [PATCH 038/104] Fix querying anatomy Should be from context, not on instance anymore. --- openpype/pipeline/farm/pyblish_functions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index 84d958bea14..a2eaddbba67 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -136,7 +136,7 @@ def get_transferable_representations(instance): list of dicts: List of transferable representations. """ - anatomy = instance.data.get("anatomy") # type: Anatomy + anatomy = instance.context.data["anatomy"] # type: Anatomy to_transfer = [] for representation in instance.data.get("representations", []): @@ -187,7 +187,7 @@ def create_skeleton_instance( context = instance.context data = instance.data.copy() - anatomy = data["anatomy"] # type: Anatomy + anatomy = instance.context.data["anatomy"] # type: Anatomy families = [data["family"]] @@ -363,7 +363,7 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data): # TODO: this needs to be taking the task from context or instance task = os.environ["AVALON_TASK"] - anatomy = instance.data["anatomy"] + anatomy = instance.context.data["anatomy"] subset = skeleton["subset"] cameras = instance.data.get("cameras", []) exp_files = instance.data["expectedFiles"] @@ -578,7 +578,7 @@ def copy_extend_frames(instance, representation): start = instance.data.get("frameStart") end = instance.data.get("frameEnd") project_name = instance.context.data["project"] - anatomy = instance.data["anatomy"] # type: Anatomy + anatomy = instance.context.data["anatomy"] # type: Anatomy # get latest version of subset # this will stop if subset wasn't published yet From 79132763ab9bd4cb00b1e21bb8ace6381120d01a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 3 May 2023 18:39:29 +0200 Subject: [PATCH 039/104] Fix typo, wrong parentheses --- openpype/pipeline/farm/pyblish_functions.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index a2eaddbba67..f4c1ea2f27f 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -109,16 +109,16 @@ def get_time_data_from_instance_or_context(instance): """ return TimeData( - start=instance.data.get["frameStart"] or \ - instance.context.data.get("frameStart"), - end=instance.data.get("frameEnd") or \ - instance.context.data.get("frameEnd"), - fps=instance.data.get("fps") or \ - instance.context.data.get("fps"), - handle_start=instance.data.get("handleStart") or \ - instance.context.data.get("handleStart"), # noqa: E501 - handle_end=instance.data.get("handleStart") or \ - instance.context.data.get("handleStart") + start=(instance.data.get("frameStart") or + instance.context.data.get("frameStart")), + end=(instance.data.get("frameEnd") or + instance.context.data.get("frameEnd")), + fps=(instance.data.get("fps") or + instance.context.data.get("fps")), + handle_start=(instance.data.get("handleStart") or + instance.context.data.get("handleStart")), # noqa: E501 + handle_end=(instance.data.get("handleStart") or + instance.context.data.get("handleStart")) ) From 87eb1edf891fb8f8cfe968654dc65ace58afbd84 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 3 May 2023 18:40:47 +0200 Subject: [PATCH 040/104] Fix wrong method --- openpype/pipeline/farm/pyblish_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index f4c1ea2f27f..645b31b2de9 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -659,7 +659,7 @@ def attach_instances_to_subset(attach_to, instances): new_instances = [] for attach_instance in attach_to: for i in instances: - new_inst = copy(i) + new_inst = copy.deepcopy(i) new_inst["version"] = attach_instance.get("version") new_inst["subset"] = attach_instance.get("subset") new_inst["family"] = attach_instance.get("family") From c284908cb42244236a8d439b6e7178c101b715f3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 3 May 2023 18:48:36 +0200 Subject: [PATCH 041/104] Fix attr.s for Python2 --- openpype/modules/royalrender/rr_job.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/modules/royalrender/rr_job.py b/openpype/modules/royalrender/rr_job.py index 5f034e74a1b..8d96b8ff4a2 100644 --- a/openpype/modules/royalrender/rr_job.py +++ b/openpype/modules/royalrender/rr_job.py @@ -38,7 +38,7 @@ def parse(data): @attr.s -class RRJob: +class RRJob(object): """Mapping of Royal Render job file to a data class.""" # Required @@ -197,7 +197,7 @@ def serialize(self): @attr.s -class SubmitFile: +class SubmitFile(object): """Class wrapping Royal Render submission XML file.""" # Syntax version of the submission file. From 1b4f452301436131c0024e80b5b80538861ceba8 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 3 May 2023 19:23:58 +0200 Subject: [PATCH 042/104] Extracted prepare_representations Various fixes. WIP --- .../plugins/publish/submit_publish_job.py | 149 ++--------------- .../publish/create_nuke_deadline_job.py | 48 +++++- .../publish/create_publish_royalrender_job.py | 32 +++- .../publish/submit_jobs_to_royalrender.py | 2 +- openpype/pipeline/farm/pyblish_functions.py | 156 +++++++++++++++++- 5 files changed, 230 insertions(+), 157 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 86f647dd1bd..58a0cd72199 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -21,10 +21,11 @@ from openpype.tests.lib import is_in_tests from openpype.pipeline.farm.patterning import match_aov_pattern from openpype.lib import is_running_from_build -from openpype.pipeline.farm.pyblish import ( +from openpype.pipeline.farm.pyblish_functions import ( create_skeleton_instance, create_instances_for_aov, - attach_instances_to_subset + attach_instances_to_subset, + prepare_representations ) @@ -332,140 +333,6 @@ def _submit_deadline_post_job(self, instance, job, instances): return deadline_publish_job_id - def _get_representations(self, instance, exp_files): - """Create representations for file sequences. - - This will return representations of expected files if they are not - in hierarchy of aovs. There should be only one sequence of files for - most cases, but if not - we create representation from each of them. - - Arguments: - instance (dict): instance data for which we are - setting representations - exp_files (list): list of expected files - - Returns: - list of representations - - """ - representations = [] - host_name = os.environ.get("AVALON_APP", "") - collections, remainders = clique.assemble(exp_files) - - # create representation for every collected sequence - for collection in collections: - ext = collection.tail.lstrip(".") - preview = False - # TODO 'useSequenceForReview' is temporary solution which does - # not work for 100% of cases. We must be able to tell what - # expected files contains more explicitly and from what - # should be review made. - # - "review" tag is never added when is set to 'False' - if instance["useSequenceForReview"]: - # toggle preview on if multipart is on - if instance.get("multipartExr", False): - self.log.debug( - "Adding preview tag because its multipartExr" - ) - preview = True - else: - render_file_name = list(collection)[0] - # if filtered aov name is found in filename, toggle it for - # preview video rendering - preview = match_aov_pattern( - host_name, self.aov_filter, render_file_name - ) - - staging = os.path.dirname(list(collection)[0]) - success, rootless_staging_dir = ( - self.anatomy.find_root_template_from_path(staging) - ) - if success: - staging = rootless_staging_dir - else: - self.log.warning(( - "Could not find root path for remapping \"{}\"." - " This may cause issues on farm." - ).format(staging)) - - frame_start = int(instance.get("frameStartHandle")) - if instance.get("slate"): - frame_start -= 1 - - rep = { - "name": ext, - "ext": ext, - "files": [os.path.basename(f) for f in list(collection)], - "frameStart": frame_start, - "frameEnd": int(instance.get("frameEndHandle")), - # If expectedFile are absolute, we need only filenames - "stagingDir": staging, - "fps": instance.get("fps"), - "tags": ["review"] if preview else [], - } - - # poor man exclusion - if ext in self.skip_integration_repre_list: - rep["tags"].append("delete") - - if instance.get("multipartExr", False): - rep["tags"].append("multipartExr") - - # support conversion from tiled to scanline - if instance.get("convertToScanline"): - self.log.info("Adding scanline conversion.") - rep["tags"].append("toScanline") - - representations.append(rep) - - self._solve_families(instance, preview) - - # add remainders as representations - for remainder in remainders: - ext = remainder.split(".")[-1] - - staging = os.path.dirname(remainder) - success, rootless_staging_dir = ( - self.anatomy.find_root_template_from_path(staging) - ) - if success: - staging = rootless_staging_dir - else: - self.log.warning(( - "Could not find root path for remapping \"{}\"." - " This may cause issues on farm." - ).format(staging)) - - rep = { - "name": ext, - "ext": ext, - "files": os.path.basename(remainder), - "stagingDir": staging, - } - - preview = match_aov_pattern( - host_name, self.aov_filter, remainder - ) - if preview: - rep.update({ - "fps": instance.get("fps"), - "tags": ["review"] - }) - self._solve_families(instance, preview) - - already_there = False - for repre in instance.get("representations", []): - # might be added explicitly before by publish_on_farm - already_there = repre.get("files") == rep["files"] - if already_there: - self.log.debug("repre {} already_there".format(repre)) - break - - if not already_there: - representations.append(rep) - - return representations - def _solve_families(self, instance, preview=False): families = instance.get("families") @@ -552,12 +419,16 @@ def process(self, instance): if isinstance(instance.data.get("expectedFiles")[0], dict): instances = create_instances_for_aov( - instance, instance_skeleton_data, self.aov_filter) + instance, instance_skeleton_data, + self.aov_filter, self.skip_integration_repre_list) else: - representations = self._get_representations( + representations = prepare_representations( instance_skeleton_data, - instance.data.get("expectedFiles") + instance.data.get("expectedFiles"), + self.anatomy, + self.aov_filter, + self.skip_integration_repre_list ) if "representations" not in instance_skeleton_data.keys(): diff --git a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py index 217ebed057b..0472f2ea80d 100644 --- a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py @@ -1,14 +1,13 @@ # -*- coding: utf-8 -*- """Submitting render job to RoyalRender.""" +import copy import os -import sys import re import platform from datetime import datetime from pyblish.api import InstancePlugin, IntegratorOrder, Context from openpype.tests.lib import is_in_tests -from openpype.lib import is_running_from_build from openpype.pipeline.publish.lib import get_published_workfile_instance from openpype.pipeline.publish import KnownPublishError from openpype.modules.royalrender.api import Api as rrApi @@ -34,6 +33,8 @@ class CreateNukeRoyalRenderJob(InstancePlugin, OpenPypePyblishPluginMixin): priority = 50 chunk_size = 1 concurrent_tasks = 1 + use_gpu = True + use_published = True @classmethod def get_attribute_defs(cls): @@ -69,6 +70,11 @@ def get_attribute_defs(cls): "suspend_publish", default=False, label="Suspend publish" + ), + BoolDef( + "use_published", + default=cls.use_published, + label="Use published workfile" ) ] @@ -81,6 +87,17 @@ def __init__(self, *args, **kwargs): self.rr_api = None def process(self, instance): + # import json + # def _default_json(value): + # return str(value) + # filepath = "C:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype\\tests\\unit\\openpype\\modules\\royalrender\\plugins\\publish\\resources\\instance.json" + # with open(filepath, "w") as f: + # f.write(json.dumps(instance.data, indent=4, default=_default_json)) + # + # filepath = "C:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype\\tests\\unit\\openpype\\modules\\royalrender\\plugins\\publish\\resources\\context.json" + # with open(filepath, "w") as f: + # f.write(json.dumps(instance.context.data, indent=4, default=_default_json)) + if not instance.data.get("farm"): self.log.info("Skipping local instance.") return @@ -93,6 +110,7 @@ def process(self, instance): "suspend_publish"] context = instance.context + self._instance = instance self._rr_root = self._resolve_rr_path(context, instance.data.get( "rrPathName")) # noqa @@ -218,7 +236,7 @@ def get_job(self, script_path, render_path, Renderer="", SeqStart=int(start_frame), SeqEnd=int(end_frame), - SeqStep=int(self._instance.data.get("byFrameStep"), 1), + SeqStep=int(self._instance.data.get("byFrameStep", 1)), SeqFileOffset=0, Version=nuke_version.group(), SceneName=script_path, @@ -298,7 +316,7 @@ def expected_files(self, path, start_frame, end_frame): if "%" not in file: expected_files.append(path) - return + return expected_files if self._instance.data.get("slate"): start_frame -= 1 @@ -308,3 +326,25 @@ def expected_files(self, path, start_frame, end_frame): for i in range(start_frame, (end_frame + 1)) ) return expected_files + + def preview_fname(self, path): + """Return output file path with #### for padding. + + Deadline requires the path to be formatted with # in place of numbers. + For example `/path/to/render.####.png` + + Args: + path (str): path to rendered images + + Returns: + str + + """ + self.log.debug("_ path: `{}`".format(path)) + if "%" in path: + search_results = re.search(r"(%0)(\d)(d.)", path).groups() + self.log.debug("_ search_results: `{}`".format(search_results)) + return int(search_results[1]) + if "#" in path: + self.log.debug("_ path: `{}`".format(path)) + return path diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index b1c84c87b96..6f0bc995d00 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -11,10 +11,11 @@ from openpype.pipeline.publish import KnownPublishError from openpype.lib.openpype_version import ( get_OpenPypeVersion, get_openpype_version) -from openpype.pipeline.farm.pyblish import ( +from openpype.pipeline.farm.pyblish_functions import ( create_skeleton_instance, create_instances_for_aov, - attach_instances_to_subset + attach_instances_to_subset, + prepare_representations ) @@ -31,6 +32,20 @@ class CreatePublishRoyalRenderJob(InstancePlugin): "harmony": [r".*"], # for everything from AE "celaction": [r".*"]} + skip_integration_repre_list = [] + + # mapping of instance properties to be transferred to new instance + # for every specified family + instance_transfer = { + "slate": ["slateFrames", "slate"], + "review": ["lutPath"], + "render2d": ["bakingNukeScripts", "version"], + "renderlayer": ["convertToScanline"] + } + + # list of family names to transfer to new family if present + families_transfer = ["render3d", "render2d", "ftrack", "slate"] + def process(self, instance): # data = instance.data.copy() context = instance.context @@ -46,15 +61,18 @@ def process(self, instance): families_transfer=self.families_transfer, instance_transfer=self.instance_transfer) - instances = None if isinstance(instance.data.get("expectedFiles")[0], dict): instances = create_instances_for_aov( - instance, instance_skeleton_data, self.aov_filter) + instance, instance_skeleton_data, + self.aov_filter, self.skip_integration_repre_list) else: - representations = self._get_representations( + representations = prepare_representations( instance_skeleton_data, - instance.data.get("expectedFiles") + instance.data.get("expectedFiles"), + self.anatomy, + self.aov_filter, + self.skip_integration_repre_list ) if "representations" not in instance_skeleton_data.keys(): @@ -104,7 +122,7 @@ def get_job(self, instance, instances): RRJob: RoyalRender publish job. """ - data = instance.data.copy() + data = deepcopy(instance.data) subset = data["subset"] job_name = "Publish - {subset}".format(subset=subset) diff --git a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py index 325fb36993d..8546554372e 100644 --- a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py +++ b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py @@ -11,7 +11,7 @@ class SubmitJobsToRoyalRender(ContextPlugin): """Find all jobs, create submission XML and submit it to RoyalRender.""" label = "Submit jobs to RoyalRender" - order = IntegratorOrder + 0.1 + order = IntegratorOrder + 0.3 targets = ["local"] def __init__(self): diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index 645b31b2de9..792cc07f02c 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -284,7 +284,150 @@ def _solve_families(families): return families -def create_instances_for_aov(instance, skeleton, aov_filter): +def prepare_representations(instance, exp_files, anatomy, aov_filter, + skip_integration_repre_list): + """Create representations for file sequences. + + This will return representations of expected files if they are not + in hierarchy of aovs. There should be only one sequence of files for + most cases, but if not - we create representation from each of them. + + Arguments: + instance (dict): instance data for which we are + setting representations + exp_files (list): list of expected files + anatomy (Anatomy): + aov_filter (dict): add review for specific aov names + skip_integration_repre_list (list): exclude specific extensions + + Returns: + list of representations + + """ + representations = [] + host_name = os.environ.get("AVALON_APP", "") + collections, remainders = clique.assemble(exp_files) + + log = Logger.get_logger("farm_publishing") + + # create representation for every collected sequence + for collection in collections: + ext = collection.tail.lstrip(".") + preview = False + # TODO 'useSequenceForReview' is temporary solution which does + # not work for 100% of cases. We must be able to tell what + # expected files contains more explicitly and from what + # should be review made. + # - "review" tag is never added when is set to 'False' + if instance["useSequenceForReview"]: + # toggle preview on if multipart is on + if instance.get("multipartExr", False): + log.debug( + "Adding preview tag because its multipartExr" + ) + preview = True + else: + render_file_name = list(collection)[0] + # if filtered aov name is found in filename, toggle it for + # preview video rendering + preview = match_aov_pattern( + host_name, aov_filter, render_file_name + ) + + staging = os.path.dirname(list(collection)[0]) + success, rootless_staging_dir = ( + anatomy.find_root_template_from_path(staging) + ) + if success: + staging = rootless_staging_dir + else: + log.warning(( + "Could not find root path for remapping \"{}\"." + " This may cause issues on farm." + ).format(staging)) + + frame_start = int(instance.get("frameStartHandle")) + if instance.get("slate"): + frame_start -= 1 + + rep = { + "name": ext, + "ext": ext, + "files": [os.path.basename(f) for f in list(collection)], + "frameStart": frame_start, + "frameEnd": int(instance.get("frameEndHandle")), + # If expectedFile are absolute, we need only filenames + "stagingDir": staging, + "fps": instance.get("fps"), + "tags": ["review"] if preview else [], + } + + # poor man exclusion + if ext in skip_integration_repre_list: + rep["tags"].append("delete") + + if instance.get("multipartExr", False): + rep["tags"].append("multipartExr") + + # support conversion from tiled to scanline + if instance.get("convertToScanline"): + log.info("Adding scanline conversion.") + rep["tags"].append("toScanline") + + representations.append(rep) + + if preview: + instance["families"] = _solve_families(instance["families"]) + + # add remainders as representations + for remainder in remainders: + ext = remainder.split(".")[-1] + + staging = os.path.dirname(remainder) + success, rootless_staging_dir = ( + anatomy.find_root_template_from_path(staging) + ) + if success: + staging = rootless_staging_dir + else: + log.warning(( + "Could not find root path for remapping \"{}\"." + " This may cause issues on farm." + ).format(staging)) + + rep = { + "name": ext, + "ext": ext, + "files": os.path.basename(remainder), + "stagingDir": staging, + } + + preview = match_aov_pattern( + host_name, aov_filter, remainder + ) + if preview: + rep.update({ + "fps": instance.get("fps"), + "tags": ["review"] + }) + instance["families"] = _solve_families(instance["families"]) + + already_there = False + for repre in instance.get("representations", []): + # might be added explicitly before by publish_on_farm + already_there = repre.get("files") == rep["files"] + if already_there: + log.debug("repre {} already_there".format(repre)) + break + + if not already_there: + representations.append(rep) + + return representations + + +def create_instances_for_aov(instance, skeleton, aov_filter, + skip_integration_repre_list): """Create instances from AOVs. This will create new pyblish.api.Instances by going over expected @@ -336,11 +479,13 @@ def create_instances_for_aov(instance, skeleton, aov_filter): instance, skeleton, aov_filter, - additional_color_data + additional_color_data, + skip_integration_repre_list ) -def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data): +def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, + skip_integration_repre_list): """Create instance for each AOV found. This will create new instance for every AOV it can detect in expected @@ -427,7 +572,7 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data): render_file_name = os.path.basename(col[0]) else: render_file_name = os.path.basename(col) - aov_patterns = self.aov_filter + aov_patterns = aov_filter preview = match_aov_pattern(app, aov_patterns, render_file_name) # toggle preview on if multipart is on @@ -436,7 +581,6 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data): log.debug("Adding preview tag because its multipartExr") preview = True - new_instance = deepcopy(skeleton) new_instance["subsetGroup"] = group_name if preview: @@ -483,7 +627,7 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data): rep["tags"].append("toScanline") # poor man exclusion - if ext in self.skip_integration_repre_list: + if ext in skip_integration_repre_list: rep["tags"].append("delete") if preview: From fb06a2e6819ae1e7696306c48457c9d6f681ab07 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 May 2023 12:59:46 +0200 Subject: [PATCH 043/104] Extracted create_metadata_path --- .../plugins/publish/submit_publish_job.py | 43 +++---------------- .../publish/create_publish_royalrender_job.py | 12 +++--- openpype/pipeline/farm/pyblish_functions.py | 33 ++++++++++++++ 3 files changed, 45 insertions(+), 43 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 58a0cd72199..ff6bcf1801d 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- """Submit publishing job to farm.""" - import os import json import re @@ -12,20 +11,18 @@ from openpype.client import ( get_last_version_by_subset_name, - get_representations, ) from openpype.pipeline import ( - get_representation_path, legacy_io, ) from openpype.tests.lib import is_in_tests -from openpype.pipeline.farm.patterning import match_aov_pattern from openpype.lib import is_running_from_build from openpype.pipeline.farm.pyblish_functions import ( create_skeleton_instance, create_instances_for_aov, attach_instances_to_subset, - prepare_representations + prepare_representations, + create_metadata_path ) @@ -154,36 +151,6 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): # poor man exclusion skip_integration_repre_list = [] - def _create_metadata_path(self, instance): - ins_data = instance.data - # Ensure output dir exists - output_dir = ins_data.get( - "publishRenderMetadataFolder", ins_data["outputDir"]) - - try: - if not os.path.isdir(output_dir): - os.makedirs(output_dir) - except OSError: - # directory is not available - self.log.warning("Path is unreachable: `{}`".format(output_dir)) - - metadata_filename = "{}_metadata.json".format(ins_data["subset"]) - - metadata_path = os.path.join(output_dir, metadata_filename) - - # Convert output dir to `{root}/rest/of/path/...` with Anatomy - success, rootless_mtdt_p = self.anatomy.find_root_template_from_path( - metadata_path) - if not success: - # `rootless_path` is not set to `output_dir` if none of roots match - self.log.warning(( - "Could not find root path for remapping \"{}\"." - " This may cause issues on farm." - ).format(output_dir)) - rootless_mtdt_p = metadata_path - - return metadata_path, rootless_mtdt_p - def _submit_deadline_post_job(self, instance, job, instances): """Submit publish job to Deadline. @@ -216,7 +183,7 @@ def _submit_deadline_post_job(self, instance, job, instances): # Transfer the environment from the original job to this dependent # job so they use the same environment metadata_path, rootless_metadata_path = \ - self._create_metadata_path(instance) + create_metadata_path(instance, self.anatomy) environment = { "AVALON_PROJECT": legacy_io.Session["AVALON_PROJECT"], @@ -539,8 +506,8 @@ def process(self, instance): } publish_job.update({"ftrack": ftrack}) - metadata_path, rootless_metadata_path = self._create_metadata_path( - instance) + metadata_path, rootless_metadata_path = \ + create_metadata_path(instance, self.anatomy) self.log.info("Writing json file: {}".format(metadata_path)) with open(metadata_path, "w") as f: diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 6f0bc995d00..5a64e5a9b71 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -15,7 +15,8 @@ create_skeleton_instance, create_instances_for_aov, attach_instances_to_subset, - prepare_representations + prepare_representations, + create_metadata_path ) @@ -100,8 +101,8 @@ def process(self, instance): instance.data["rrJobs"] += publish_job - metadata_path, rootless_metadata_path = self._create_metadata_path( - instance) + metadata_path, rootless_metadata_path = \ + create_metadata_path(instance, self.anatomy) self.log.info("Writing json file: {}".format(metadata_path)) with open(metadata_path, "w") as f: @@ -122,7 +123,8 @@ def get_job(self, instance, instances): RRJob: RoyalRender publish job. """ - data = deepcopy(instance.data) + # data = deepcopy(instance.data) + data = instance.data subset = data["subset"] job_name = "Publish - {subset}".format(subset=subset) @@ -132,7 +134,7 @@ def get_job(self, instance, instances): # Transfer the environment from the original job to this dependent # job, so they use the same environment metadata_path, roothless_metadata_path = \ - self._create_metadata_path(instance) + create_metadata_path(instance, self.anatomy) environment = RREnvList({ "AVALON_PROJECT": legacy_io.Session["AVALON_PROJECT"], diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index 792cc07f02c..6c08545b1b1 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -812,3 +812,36 @@ def attach_instances_to_subset(attach_to, instances): new_inst.pop("subsetGroup") new_instances.append(new_inst) return new_instances + + +def create_metadata_path(instance, anatomy): + ins_data = instance.data + # Ensure output dir exists + output_dir = ins_data.get( + "publishRenderMetadataFolder", ins_data["outputDir"]) + + log = Logger.get_logger("farm_publishing") + + try: + if not os.path.isdir(output_dir): + os.makedirs(output_dir) + except OSError: + # directory is not available + log.warning("Path is unreachable: `{}`".format(output_dir)) + + metadata_filename = "{}_metadata.json".format(ins_data["subset"]) + + metadata_path = os.path.join(output_dir, metadata_filename) + + # Convert output dir to `{root}/rest/of/path/...` with Anatomy + success, rootless_mtdt_p = anatomy.find_root_template_from_path( + metadata_path) + if not success: + # `rootless_path` is not set to `output_dir` if none of roots match + log.warning(( + "Could not find root path for remapping \"{}\"." + " This may cause issues on farm." + ).format(output_dir)) + rootless_mtdt_p = metadata_path + + return metadata_path, rootless_mtdt_p From dded3e1369c365e13035484209336232b35c11ff Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 May 2023 14:01:11 +0200 Subject: [PATCH 044/104] Added missing variables --- .../publish/create_publish_royalrender_job.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 5a64e5a9b71..334f3a5718d 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -47,6 +47,20 @@ class CreatePublishRoyalRenderJob(InstancePlugin): # list of family names to transfer to new family if present families_transfer = ["render3d", "render2d", "ftrack", "slate"] + environ_job_filter = [ + "OPENPYPE_METADATA_FILE" + ] + + environ_keys = [ + "FTRACK_API_USER", + "FTRACK_API_KEY", + "FTRACK_SERVER", + "AVALON_APP_NAME", + "OPENPYPE_USERNAME", + "OPENPYPE_SG_USER", + ] + priority = 50 + def process(self, instance): # data = instance.data.copy() context = instance.context From 3bbab4511fd5f9f30dc46ab542e79dd92e96af14 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 May 2023 14:01:58 +0200 Subject: [PATCH 045/104] Removed usage of legacy_io --- .../plugins/publish/create_publish_royalrender_job.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 334f3a5718d..0f0001aced4 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -150,11 +150,13 @@ def get_job(self, instance, instances): metadata_path, roothless_metadata_path = \ create_metadata_path(instance, self.anatomy) + anatomy_data = instance.context.data["anatomyData"] + environment = RREnvList({ - "AVALON_PROJECT": legacy_io.Session["AVALON_PROJECT"], - "AVALON_ASSET": legacy_io.Session["AVALON_ASSET"], - "AVALON_TASK": legacy_io.Session["AVALON_TASK"], - "OPENPYPE_USERNAME": instance.context.data["user"], + "AVALON_PROJECT": anatomy_data["project"]["name"], + "AVALON_ASSET": anatomy_data["asset"], + "AVALON_TASK": anatomy_data["task"]["name"], + "OPENPYPE_USERNAME": anatomy_data["user"], "OPENPYPE_PUBLISH_JOB": "1", "OPENPYPE_RENDER_JOB": "0", "OPENPYPE_REMOTE_JOB": "0", From 9a95fac77c29d052facb3fce78915a85704a7eba Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 May 2023 14:03:13 +0200 Subject: [PATCH 046/104] Changed to use os env OPENPYPE_VERSION get_OpenpypeVersion() returns None in hosts(or at least in Nuke) --- .../plugins/publish/create_publish_royalrender_job.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 0f0001aced4..4a889ebea60 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -6,7 +6,6 @@ from pyblish.api import InstancePlugin, IntegratorOrder, Instance -from openpype.pipeline import legacy_io from openpype.modules.royalrender.rr_job import RRJob, RREnvList from openpype.pipeline.publish import KnownPublishError from openpype.lib.openpype_version import ( @@ -199,8 +198,6 @@ def get_job(self, instance, instances): "--targets", "farm" ] - openpype_version = get_OpenPypeVersion() - current_version = openpype_version(version=get_openpype_version()) job = RRJob( Software="OpenPype", Renderer="Once", @@ -209,8 +206,7 @@ def get_job(self, instance, instances): SeqEnd=1, SeqStep=1, SeqFileOffset=0, - Version="{}.{}".format( - current_version.major(), current_version.minor()), + Version=os.environ.get("OPENPYPE_VERSION"), # executable SceneName=roothless_metadata_path, # command line arguments From cace4f23772cab9c0d0ad3c667a702d93b27fd93 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 May 2023 14:09:50 +0200 Subject: [PATCH 047/104] Used correct function --- .../plugins/publish/create_publish_royalrender_job.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 4a889ebea60..ab1b1e881e4 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -6,7 +6,11 @@ from pyblish.api import InstancePlugin, IntegratorOrder, Instance -from openpype.modules.royalrender.rr_job import RRJob, RREnvList +from openpype.modules.royalrender.rr_job import ( + RRJob, + RREnvList, + get_rr_platform +) from openpype.pipeline.publish import KnownPublishError from openpype.lib.openpype_version import ( get_OpenPypeVersion, get_openpype_version) @@ -216,7 +220,7 @@ def get_job(self, instance, instances): ImageDir="", ImageExtension="", ImagePreNumberLetter="", - SceneOS=RRJob.get_rr_platform(), + SceneOS=get_rr_platform(), rrEnvList=environment.serialize(), Priority=priority ) From b18e6ad431022af757a06e989205cb23fb99e761 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 May 2023 14:50:42 +0200 Subject: [PATCH 048/104] Proper serialization Without it json would complain. --- .../plugins/publish/create_publish_royalrender_job.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index ab1b1e881e4..602d578a4be 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """Create publishing job on RoyalRender.""" import os +import attr from copy import deepcopy import json @@ -123,7 +124,7 @@ def process(self, instance): self.log.info("Writing json file: {}".format(metadata_path)) with open(metadata_path, "w") as f: - json.dump(publish_job, f, indent=4, sort_keys=True) + json.dump(attr.asdict(publish_job), f, indent=4, sort_keys=True) def get_job(self, instance, instances): """Create RR publishing job. From 050c11ee2b30861d77805de76f5aa75b25824f14 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 May 2023 14:57:53 +0200 Subject: [PATCH 049/104] Fix adding --- .../plugins/publish/create_publish_royalrender_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 602d578a4be..a74c528676e 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -117,7 +117,7 @@ def process(self, instance): publish_job = self.get_job(instance, instances) - instance.data["rrJobs"] += publish_job + instance.data["rrJobs"].append(publish_job) metadata_path, rootless_metadata_path = \ create_metadata_path(instance, self.anatomy) From c8451d142939a5b78a5ee7642768e955654d3a30 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 May 2023 15:26:57 +0200 Subject: [PATCH 050/104] Fix copy Not sure if it shouldn't be deepcopy, but that doesn't work as something cannot be pickled. --- .../plugins/publish/create_publish_royalrender_job.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index a74c528676e..2acd3ba4a50 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -141,8 +141,7 @@ def get_job(self, instance, instances): RRJob: RoyalRender publish job. """ - # data = deepcopy(instance.data) - data = instance.data + data = instance.data.copy() subset = data["subset"] job_name = "Publish - {subset}".format(subset=subset) From ac02eac861838435b1a752af89de551ebf534316 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 May 2023 15:32:48 +0200 Subject: [PATCH 051/104] Small fixes --- .../royalrender/plugins/publish/create_nuke_deadline_job.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py index 0472f2ea80d..4a2821e195a 100644 --- a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py @@ -142,7 +142,7 @@ def process(self, instance): if not self._instance.data.get("rrJobs"): self._instance.data["rrJobs"] = [] - self._instance.data["rrJobs"] += self.create_jobs() + self._instance.data["rrJobs"].extend(self.create_jobs()) # redefinition of families if "render" in self._instance.data["family"]: @@ -155,7 +155,6 @@ def process(self, instance): self._instance.data["outputDir"] = os.path.dirname( self._instance.data["path"]).replace("\\", "/") - def create_jobs(self): submit_frame_start = int(self._instance.data["frameStartHandle"]) submit_frame_end = int(self._instance.data["frameEndHandle"]) @@ -256,6 +255,8 @@ def get_job(self, script_path, render_path, CustomAttributes=custom_attributes ) + return job + @staticmethod def _resolve_rr_path(context, rr_path_name): # type: (Context, str) -> str From 4e455ead316fbc334502be51bde30e5eca0f9e3c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 5 May 2023 13:37:55 +0200 Subject: [PATCH 052/104] Instance data might not contain "publish" key In that case publish is implicit, more likely it will be set explicitly to False than not existing at all. --- openpype/pipeline/publish/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/pipeline/publish/lib.py b/openpype/pipeline/publish/lib.py index 58d9fd8a051..cf52c113c31 100644 --- a/openpype/pipeline/publish/lib.py +++ b/openpype/pipeline/publish/lib.py @@ -804,7 +804,7 @@ def get_published_workfile_instance(context): # test if there is instance of workfile waiting # to be published. - if i.data["publish"] is not True: + if not i.data.get("publish", True): continue return i From 8858c0c0dab5622be8816a169d785a7227e266b6 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 5 May 2023 14:09:13 +0200 Subject: [PATCH 053/104] Get proper rrPathName Instance could be picked non-deterministically, without rrPathName value. --- .../plugins/publish/submit_jobs_to_royalrender.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py index 8546554372e..29d5fe6d727 100644 --- a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py +++ b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py @@ -34,6 +34,7 @@ def process(self, context): # iterate over all instances and try to find RRJobs jobs = [] + instance_rr_path = None for instance in context: if isinstance(instance.data.get("rrJob"), RRJob): jobs.append(instance.data.get("rrJob")) @@ -42,10 +43,11 @@ def process(self, context): isinstance(job, RRJob) for job in instance.data.get("rrJobs")): jobs += instance.data.get("rrJobs") + if instance.data.get("rrPathName"): + instance_rr_path = instance.data["rrPathName"] if jobs: - self._rr_root = self._resolve_rr_path( - context, instance.data.get("rrPathName")) # noqa + self._rr_root = self._resolve_rr_path(context, instance_rr_path) if not self._rr_root: raise KnownPublishError( ("Missing RoyalRender root. " From d8e8e10806d7428aefd7f808f53c4f3ca7449f64 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 9 May 2023 16:38:40 +0200 Subject: [PATCH 054/104] Remove unwanted logging code --- .../plugins/publish/create_nuke_deadline_job.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py index 4a2821e195a..80493988456 100644 --- a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py @@ -87,17 +87,6 @@ def __init__(self, *args, **kwargs): self.rr_api = None def process(self, instance): - # import json - # def _default_json(value): - # return str(value) - # filepath = "C:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype\\tests\\unit\\openpype\\modules\\royalrender\\plugins\\publish\\resources\\instance.json" - # with open(filepath, "w") as f: - # f.write(json.dumps(instance.data, indent=4, default=_default_json)) - # - # filepath = "C:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype\\tests\\unit\\openpype\\modules\\royalrender\\plugins\\publish\\resources\\context.json" - # with open(filepath, "w") as f: - # f.write(json.dumps(instance.context.data, indent=4, default=_default_json)) - if not instance.data.get("farm"): self.log.info("Skipping local instance.") return From 45894da8dcecc49d06c4c582f52dcbcf4b9190a4 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 9 May 2023 18:46:10 +0200 Subject: [PATCH 055/104] Fix frame placeholder Without it RR won't find rendered files. --- .../plugins/publish/create_nuke_deadline_job.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py index 80493988456..699f6654686 100644 --- a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py @@ -16,7 +16,8 @@ from openpype.lib import ( is_running_from_build, BoolDef, - NumberDef + NumberDef, + collect_frames ) from openpype.pipeline import OpenPypePyblishPluginMixin from openpype.pipeline.farm.tools import iter_expected_files @@ -219,6 +220,10 @@ def get_job(self, script_path, render_path, self._instance.data["expectedFiles"].extend(expected_files) first_file = next(iter_expected_files(expected_files)) + file_name, file_ext = os.path.splitext(os.path.basename(first_file)) + frame_pattern = ".{}".format(start_frame) + file_name = file_name.replace(frame_pattern, '.#') + job = RRJob( Software="Nuke", Renderer="", @@ -230,9 +235,9 @@ def get_job(self, script_path, render_path, SceneName=script_path, IsActive=True, ImageDir=render_dir.replace("\\", "/"), - ImageFilename="{}.".format(os.path.splitext(first_file)[0]), - ImageExtension=os.path.splitext(first_file)[1], - ImagePreNumberLetter=".", + ImageFilename="{}".format(file_name), + ImageExtension=file_ext, + ImagePreNumberLetter="", ImageSingleOutputFile=False, SceneOS=get_rr_platform(), Layer=node_name, From 97043fe3107c6c8d15982b80563d6466af6bbfc7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 9 May 2023 18:47:16 +0200 Subject: [PATCH 056/104] Use absolute path instead of rootless Rootless path will result jobs won't show up in rrControl. --- .../plugins/publish/create_publish_royalrender_job.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 2acd3ba4a50..29e467e46a5 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -66,7 +66,6 @@ class CreatePublishRoyalRenderJob(InstancePlugin): priority = 50 def process(self, instance): - # data = instance.data.copy() context = instance.context self.context = context self.anatomy = instance.context.data["anatomy"] @@ -150,7 +149,7 @@ def get_job(self, instance, instances): # Transfer the environment from the original job to this dependent # job, so they use the same environment - metadata_path, roothless_metadata_path = \ + metadata_path, rootless_metadata_path = \ create_metadata_path(instance, self.anatomy) anatomy_data = instance.context.data["anatomyData"] @@ -194,10 +193,13 @@ def get_job(self, instance, instances): priority = self.priority or instance.data.get("priority", 50) + ## rr requires absolut path or all jobs won't show up in rControl + abs_metadata_path = self.anatomy.fill_root(rootless_metadata_path) + args = [ "--headless", 'publish', - roothless_metadata_path, + abs_metadata_path, "--targets", "deadline", "--targets", "farm" ] @@ -212,7 +214,7 @@ def get_job(self, instance, instances): SeqFileOffset=0, Version=os.environ.get("OPENPYPE_VERSION"), # executable - SceneName=roothless_metadata_path, + SceneName=abs_metadata_path, # command line arguments CustomAddCmdFlags=" ".join(args), IsActive=True, From 07139db41c002bf686d44c2951e99a43bbe5ee7f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 May 2023 11:37:58 +0200 Subject: [PATCH 057/104] Implemented waiting on job id --- .../plugins/publish/submit_jobs_to_royalrender.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py index 29d5fe6d727..689fe098d9d 100644 --- a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py +++ b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py @@ -61,6 +61,14 @@ def process(self, context): def process_submission(self, jobs): # type: ([RRJob]) -> None + + idx_pre_id = 0 + for job in jobs: + job.PreID = idx_pre_id + if idx_pre_id > 0: + job.WaitForPreIDs.append(idx_pre_id - 1) + idx_pre_id += 1 + submission = rrApi.create_submission( jobs, self._submission_parameters) From ddf0b3b3ec9a7c63813f3b0bd12d42df72d7fb33 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 May 2023 11:38:52 +0200 Subject: [PATCH 058/104] Added RequiredMemory Without it default (4GB) is used. --- .../plugins/publish/submit_jobs_to_royalrender.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py index 689fe098d9d..6e91dff6ac8 100644 --- a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py +++ b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py @@ -4,7 +4,11 @@ import platform from pyblish.api import IntegratorOrder, ContextPlugin, Context -from openpype.modules.royalrender.api import RRJob, Api as rrApi +from openpype.modules.royalrender.api import ( + RRJob, + Api as rrApi, + SubmitterParameter +) from openpype.pipeline.publish import KnownPublishError @@ -95,7 +99,7 @@ def create_file(self, name, ext, contents=None): return temp.name def get_submission_parameters(self): - return [] + return [SubmitterParameter("RequiredMemory", "0")] @staticmethod def _resolve_rr_path(context, rr_path_name): From 449157fd73002fc2e4c14ebbde3adf9b84319074 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 May 2023 16:37:22 +0200 Subject: [PATCH 059/104] Fix executable placeholder --- .../rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg index 6414ae45a87..864eeaf15aa 100644 --- a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg +++ b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg @@ -30,7 +30,7 @@ CommandLine= CommandLine= -CommandLine= "" --headless publish +CommandLine= "" --headless publish --targets royalrender --targets farm From edcc1cc5f34cf366ff221742d743e89a71ba6368 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 May 2023 16:39:17 +0200 Subject: [PATCH 060/104] Fix content of metadata file Must contain metadata useful for publish not only properties of RR job. --- .../publish/create_publish_royalrender_job.py | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 29e467e46a5..db7118103d3 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -13,8 +13,9 @@ get_rr_platform ) from openpype.pipeline.publish import KnownPublishError -from openpype.lib.openpype_version import ( - get_OpenPypeVersion, get_openpype_version) +from openpype.pipeline import ( + legacy_io, +) from openpype.pipeline.farm.pyblish_functions import ( create_skeleton_instance, create_instances_for_aov, @@ -114,16 +115,31 @@ def process(self, instance): raise KnownPublishError( "Can't create publish job without prior ppducing jobs first") - publish_job = self.get_job(instance, instances) - - instance.data["rrJobs"].append(publish_job) + rr_job = self.get_job(instance, instances) + instance.data["rrJobs"].append(rr_job) + + # publish job file + publish_job = { + "asset": instance_skeleton_data["asset"], + "frameStart": instance_skeleton_data["frameStart"], + "frameEnd": instance_skeleton_data["frameEnd"], + "fps": instance_skeleton_data["fps"], + "source": instance_skeleton_data["source"], + "user": instance.context.data["user"], + "version": instance.context.data["version"], # this is workfile version + "intent": instance.context.data.get("intent"), + "comment": instance.context.data.get("comment"), + "job": attr.asdict(rr_job), + "session": legacy_io.Session.copy(), + "instances": instances + } metadata_path, rootless_metadata_path = \ create_metadata_path(instance, self.anatomy) self.log.info("Writing json file: {}".format(metadata_path)) with open(metadata_path, "w") as f: - json.dump(attr.asdict(publish_job), f, indent=4, sort_keys=True) + json.dump(publish_job, f, indent=4, sort_keys=True) def get_job(self, instance, instances): """Create RR publishing job. From 0ad2e216d4582901a14954234d67766956b05fea Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 May 2023 16:39:58 +0200 Subject: [PATCH 061/104] Add logging to log file --- .../plugins/publish/create_publish_royalrender_job.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index db7118103d3..28d6faecc0e 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -212,12 +212,12 @@ def get_job(self, instance, instances): ## rr requires absolut path or all jobs won't show up in rControl abs_metadata_path = self.anatomy.fill_root(rootless_metadata_path) + # command line set in E01__OpenPype__PublishJob.cfg, here only + # additional logging args = [ - "--headless", - 'publish', - abs_metadata_path, - "--targets", "deadline", - "--targets", "farm" + ">", os.path.join(os.path.dirname(abs_metadata_path), + "rr_out.log"), + "2>&1" ] job = RRJob( From a10bf73c725fdd3ef3ff718f454227981dfa422a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 May 2023 16:40:28 +0200 Subject: [PATCH 062/104] Fix batching of publish job --- .../plugins/publish/create_publish_royalrender_job.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 28d6faecc0e..101234f9f2a 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -240,7 +240,8 @@ def get_job(self, instance, instances): ImagePreNumberLetter="", SceneOS=get_rr_platform(), rrEnvList=environment.serialize(), - Priority=priority + Priority=priority, + CompanyProjectName=instance.context.data["projectName"] ) # add assembly jobs as dependencies From 0a6fd30d03a9de544848ec08a24b05b30bd0d24e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 May 2023 16:40:57 +0200 Subject: [PATCH 063/104] Remove unneeded import --- .../plugins/publish/create_publish_royalrender_job.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 101234f9f2a..4ce96a908d0 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -2,7 +2,6 @@ """Create publishing job on RoyalRender.""" import os import attr -from copy import deepcopy import json from pyblish.api import InstancePlugin, IntegratorOrder, Instance From b00126677550ce85e02d794af30715d81e5d4fb7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 May 2023 17:14:25 +0200 Subject: [PATCH 064/104] Clean up --- .../publish/create_publish_royalrender_job.py | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 4ce96a908d0..e6836ad4e7a 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -25,6 +25,14 @@ class CreatePublishRoyalRenderJob(InstancePlugin): + """Creates job which publishes rendered files to publish area. + + Job waits until all rendering jobs are finished, triggers `publish` command + where it reads from prepared .json file with metadata about what should + be published, renames prepared images and publishes them. + + When triggered it produces .log file next to .json file in work area. + """ label = "Create publish job in RR" order = IntegratorOrder + 0.2 icon = "tractor" @@ -62,6 +70,7 @@ class CreatePublishRoyalRenderJob(InstancePlugin): "AVALON_APP_NAME", "OPENPYPE_USERNAME", "OPENPYPE_SG_USER", + "OPENPYPE_MONGO" ] priority = 50 @@ -112,7 +121,7 @@ def process(self, instance): self.log.error(("There is no prior RoyalRender " "job on the instance.")) raise KnownPublishError( - "Can't create publish job without prior ppducing jobs first") + "Can't create publish job without prior rendering jobs first") rr_job = self.get_job(instance, instances) instance.data["rrJobs"].append(rr_job) @@ -125,7 +134,7 @@ def process(self, instance): "fps": instance_skeleton_data["fps"], "source": instance_skeleton_data["source"], "user": instance.context.data["user"], - "version": instance.context.data["version"], # this is workfile version + "version": instance.context.data["version"], # workfile version "intent": instance.context.data.get("intent"), "comment": instance.context.data.get("comment"), "job": attr.asdict(rr_job), @@ -157,7 +166,7 @@ def get_job(self, instance, instances): """ data = instance.data.copy() subset = data["subset"] - job_name = "Publish - {subset}".format(subset=subset) + jobname = "Publish - {subset}".format(subset=subset) instance_version = instance.data.get("version") # take this if exists override_version = instance_version if instance_version != 1 else None @@ -173,11 +182,7 @@ def get_job(self, instance, instances): "AVALON_PROJECT": anatomy_data["project"]["name"], "AVALON_ASSET": anatomy_data["asset"], "AVALON_TASK": anatomy_data["task"]["name"], - "OPENPYPE_USERNAME": anatomy_data["user"], - "OPENPYPE_PUBLISH_JOB": "1", - "OPENPYPE_RENDER_JOB": "0", - "OPENPYPE_REMOTE_JOB": "0", - "OPENPYPE_LOG_NO_COLORS": "1" + "OPENPYPE_USERNAME": anatomy_data["user"] }) # add environments from self.environ_keys @@ -200,12 +205,6 @@ def get_job(self, instance, instances): if job_environ.get(env_j_key): environment[env_j_key] = job_environ[env_j_key] - # Add mongo url if it's enabled - if instance.context.data.get("deadlinePassMongoUrl"): - mongo_url = os.environ.get("OPENPYPE_MONGO") - if mongo_url: - environment["OPENPYPE_MONGO"] = mongo_url - priority = self.priority or instance.data.get("priority", 50) ## rr requires absolut path or all jobs won't show up in rControl @@ -222,13 +221,11 @@ def get_job(self, instance, instances): job = RRJob( Software="OpenPype", Renderer="Once", - # path to OpenPype SeqStart=1, SeqEnd=1, SeqStep=1, SeqFileOffset=0, Version=os.environ.get("OPENPYPE_VERSION"), - # executable SceneName=abs_metadata_path, # command line arguments CustomAddCmdFlags=" ".join(args), @@ -240,6 +237,7 @@ def get_job(self, instance, instances): SceneOS=get_rr_platform(), rrEnvList=environment.serialize(), Priority=priority, + CustomSHotName=jobname, CompanyProjectName=instance.context.data["projectName"] ) From 540981da6ab257c5a6cfd16ae723c6a94520c938 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 May 2023 17:19:33 +0200 Subject: [PATCH 065/104] Clean up --- .../plugins/publish/create_nuke_deadline_job.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py index 699f6654686..8d05373e32f 100644 --- a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- """Submitting render job to RoyalRender.""" -import copy import os import re import platform @@ -17,13 +16,12 @@ is_running_from_build, BoolDef, NumberDef, - collect_frames ) from openpype.pipeline import OpenPypePyblishPluginMixin -from openpype.pipeline.farm.tools import iter_expected_files class CreateNukeRoyalRenderJob(InstancePlugin, OpenPypePyblishPluginMixin): + """Creates separate rendering job for Royal Render""" label = "Create Nuke Render job in RR" order = IntegratorOrder + 0.1 hosts = ["nuke"] @@ -202,6 +200,7 @@ def get_job(self, script_path, render_path, batch_name += datetime.now().strftime("%d%m%Y%H%M%S") output_filename_0 = self.preview_fname(render_path) + _, file_ext = os.path.splitext(os.path.basename(render_path)) custom_attributes = [] if is_running_from_build(): @@ -218,11 +217,6 @@ def get_job(self, script_path, render_path, expected_files = self.expected_files( render_path, start_frame, end_frame) self._instance.data["expectedFiles"].extend(expected_files) - first_file = next(iter_expected_files(expected_files)) - - file_name, file_ext = os.path.splitext(os.path.basename(first_file)) - frame_pattern = ".{}".format(start_frame) - file_name = file_name.replace(frame_pattern, '.#') job = RRJob( Software="Nuke", @@ -235,14 +229,14 @@ def get_job(self, script_path, render_path, SceneName=script_path, IsActive=True, ImageDir=render_dir.replace("\\", "/"), - ImageFilename="{}".format(file_name), + ImageFilename="{}".format(output_filename_0), ImageExtension=file_ext, ImagePreNumberLetter="", ImageSingleOutputFile=False, SceneOS=get_rr_platform(), Layer=node_name, SceneDatabaseDir=script_path, - CustomSHotName=self._instance.context.data["asset"], + CustomSHotName=jobname, CompanyProjectName=self._instance.context.data["projectName"], ImageWidth=self._instance.data["resolutionWidth"], ImageHeight=self._instance.data["resolutionHeight"], From ddb227bd959991cae8a51faa04e9aee28af7e381 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 May 2023 17:25:13 +0200 Subject: [PATCH 066/104] Remove unfinished file --- .../default_modules/royal_render/test_rr_job.py | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 tests/unit/openpype/default_modules/royal_render/test_rr_job.py diff --git a/tests/unit/openpype/default_modules/royal_render/test_rr_job.py b/tests/unit/openpype/default_modules/royal_render/test_rr_job.py deleted file mode 100644 index ab8b1bfd50b..00000000000 --- a/tests/unit/openpype/default_modules/royal_render/test_rr_job.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- -"""Test suite for User Settings.""" -# import pytest -# from openpype.modules import ModulesManager - - -def test_rr_job(): - # manager = ModulesManager() - # rr_module = manager.modules_by_name["royalrender"] - ... From f3ce4c6b71b3f0aec014b78452ee6f489023c4a1 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 May 2023 18:33:13 +0200 Subject: [PATCH 067/104] Hound --- .../deadline/plugins/publish/submit_publish_job.py | 11 ++++------- .../plugins/publish/create_nuke_deadline_job.py | 5 +++-- .../plugins/publish/create_publish_royalrender_job.py | 7 ++----- openpype/modules/royalrender/rr_job.py | 2 +- openpype/pipeline/farm/pyblish_functions.py | 7 ++++--- 5 files changed, 14 insertions(+), 18 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index ff6bcf1801d..be0863f9b2b 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -336,12 +336,9 @@ def process(self, instance): return instance_skeleton_data = create_skeleton_instance( - instance, - families_transfer=self.families_transfer, - instance_transfer=self.instance_transfer) - - instances = None - + instance, + families_transfer=self.families_transfer, + instance_transfer=self.instance_transfer) """ if content of `expectedFiles` list are dictionaries, we will handle it as list of AOVs, creating instance for every one of them. @@ -481,7 +478,7 @@ def process(self, instance): "fps": instance_skeleton_data["fps"], "source": instance_skeleton_data["source"], "user": instance.context.data["user"], - "version": instance.context.data["version"], # this is workfile version + "version": instance.context.data["version"], # workfile version "intent": instance.context.data.get("intent"), "comment": instance.context.data.get("comment"), "job": render_job or None, diff --git a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py index 8d05373e32f..0d5e440c90f 100644 --- a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py @@ -200,7 +200,8 @@ def get_job(self, script_path, render_path, batch_name += datetime.now().strftime("%d%m%Y%H%M%S") output_filename_0 = self.preview_fname(render_path) - _, file_ext = os.path.splitext(os.path.basename(render_path)) + file_name, file_ext = os.path.splitext( + os.path.basename(output_filename_0)) custom_attributes = [] if is_running_from_build(): @@ -229,7 +230,7 @@ def get_job(self, script_path, render_path, SceneName=script_path, IsActive=True, ImageDir=render_dir.replace("\\", "/"), - ImageFilename="{}".format(output_filename_0), + ImageFilename=file_name, ImageExtension=file_ext, ImagePreNumberLetter="", ImageSingleOutputFile=False, diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index e6836ad4e7a..3cc63db377e 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -4,7 +4,7 @@ import attr import json -from pyblish.api import InstancePlugin, IntegratorOrder, Instance +from pyblish.api import InstancePlugin, IntegratorOrder from openpype.modules.royalrender.rr_job import ( RRJob, @@ -168,9 +168,6 @@ def get_job(self, instance, instances): subset = data["subset"] jobname = "Publish - {subset}".format(subset=subset) - instance_version = instance.data.get("version") # take this if exists - override_version = instance_version if instance_version != 1 else None - # Transfer the environment from the original job to this dependent # job, so they use the same environment metadata_path, rootless_metadata_path = \ @@ -207,7 +204,7 @@ def get_job(self, instance, instances): priority = self.priority or instance.data.get("priority", 50) - ## rr requires absolut path or all jobs won't show up in rControl + # rr requires absolut path or all jobs won't show up in rControl abs_metadata_path = self.anatomy.fill_root(rootless_metadata_path) # command line set in E01__OpenPype__PublishJob.cfg, here only diff --git a/openpype/modules/royalrender/rr_job.py b/openpype/modules/royalrender/rr_job.py index 8d96b8ff4a2..b85ac592f88 100644 --- a/openpype/modules/royalrender/rr_job.py +++ b/openpype/modules/royalrender/rr_job.py @@ -32,7 +32,7 @@ def parse(data): """Parse rrEnvList string and return it as RREnvList object.""" out = RREnvList() for var in data.split("~~~"): - k, v = data.split("=") + k, v = var.split("=") out[k] = v return out diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index 6c08545b1b1..dbba9f8a9a7 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -92,7 +92,6 @@ def extend_frames(asset, subset, start, end): updated_start = min(start, prev_start) updated_end = max(end, prev_end) - return updated_start, updated_end @@ -217,9 +216,11 @@ def create_skeleton_instance( log = Logger.get_logger("farm_publishing") log.warning(("Could not find root path for remapping \"{}\". " "This may cause issues.").format(source)) - + family = ("render" + if "prerender" not in instance.data["families"] + else "prerender") instance_skeleton_data = { - "family": "render" if "prerender" not in instance.data["families"] else "prerender", # noqa: E401 + "family": family, "subset": data["subset"], "families": families, "asset": data["asset"], From 36ef2b5b5f8712764bc2e2219d452838fafdc999 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 May 2023 18:39:40 +0200 Subject: [PATCH 068/104] Hound --- .../deadline/plugins/publish/submit_publish_job.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index be0863f9b2b..2b5916cea5b 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -336,9 +336,8 @@ def process(self, instance): return instance_skeleton_data = create_skeleton_instance( - instance, - families_transfer=self.families_transfer, - instance_transfer=self.instance_transfer) + instance, families_transfer=self.families_transfer, + instance_transfer=self.instance_transfer) """ if content of `expectedFiles` list are dictionaries, we will handle it as list of AOVs, creating instance for every one of them. @@ -447,8 +446,9 @@ def process(self, instance): render_job["Props"]["Batch"] = instance.data.get( "jobBatchName") else: - render_job["Props"]["Batch"] = os.path.splitext( - os.path.basename(instance.context.data.get("currentFile")))[0] + batch = os.path.splitext(os.path.basename( + instance.context.data.get("currentFile")))[0] + render_job["Props"]["Batch"] = batch # User is deadline user render_job["Props"]["User"] = instance.context.data.get( "deadlineUser", getpass.getuser()) From b654b10e049a9a18860c431568d9cd645f60daf7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 May 2023 11:10:42 +0200 Subject: [PATCH 069/104] Handle conflict with recent develop Added explicit disabling of adding review. --- .../plugins/publish/submit_publish_job.py | 13 ++++++---- .../publish/create_publish_royalrender_job.py | 11 +++++++-- openpype/pipeline/farm/pyblish_functions.py | 24 +++++++++++++------ 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 71a609997f7..acd47e32243 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -299,7 +299,7 @@ def _submit_deadline_post_job(self, instance, job, instances): deadline_publish_job_id = response.json()["_id"] return deadline_publish_job_id - + def _solve_families(self, instance, preview=False): families = instance.get("families") @@ -379,19 +379,24 @@ def process(self, instance): This will result in one instance with two representations: `foo` and `xxx` """ - + do_not_add_review = False + if instance.data.get("review") is False: + self.log.debug("Instance has review explicitly disabled.") + do_not_add_review = True if isinstance(instance.data.get("expectedFiles")[0], dict): instances = create_instances_for_aov( instance, instance_skeleton_data, - self.aov_filter, self.skip_integration_repre_list) + self.aov_filter, self.skip_integration_repre_list, + do_not_add_review) else: representations = prepare_representations( instance_skeleton_data, instance.data.get("expectedFiles"), self.anatomy, self.aov_filter, - self.skip_integration_repre_list + self.skip_integration_repre_list, + do_not_add_review ) if "representations" not in instance_skeleton_data.keys(): diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 3cc63db377e..af5bdfd5e6a 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -88,10 +88,16 @@ def process(self, instance): families_transfer=self.families_transfer, instance_transfer=self.instance_transfer) + do_not_add_review = False + if instance.data.get("review") is False: + self.log.debug("Instance has review explicitly disabled.") + do_not_add_review = True + if isinstance(instance.data.get("expectedFiles")[0], dict): instances = create_instances_for_aov( instance, instance_skeleton_data, - self.aov_filter, self.skip_integration_repre_list) + self.aov_filter, self.skip_integration_repre_list, + do_not_add_review) else: representations = prepare_representations( @@ -99,7 +105,8 @@ def process(self, instance): instance.data.get("expectedFiles"), self.anatomy, self.aov_filter, - self.skip_integration_repre_list + self.skip_integration_repre_list, + do_not_add_review ) if "representations" not in instance_skeleton_data.keys(): diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index dbba9f8a9a7..be031b1fde6 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -286,7 +286,8 @@ def _solve_families(families): def prepare_representations(instance, exp_files, anatomy, aov_filter, - skip_integration_repre_list): + skip_integration_repre_list, + do_not_add_review): """Create representations for file sequences. This will return representations of expected files if they are not @@ -299,7 +300,8 @@ def prepare_representations(instance, exp_files, anatomy, aov_filter, exp_files (list): list of expected files anatomy (Anatomy): aov_filter (dict): add review for specific aov names - skip_integration_repre_list (list): exclude specific extensions + skip_integration_repre_list (list): exclude specific extensions, + do_not_add_review (bool): explicitly skip review Returns: list of representations @@ -351,6 +353,7 @@ def prepare_representations(instance, exp_files, anatomy, aov_filter, if instance.get("slate"): frame_start -= 1 + preview = preview and not do_not_add_review rep = { "name": ext, "ext": ext, @@ -406,6 +409,7 @@ def prepare_representations(instance, exp_files, anatomy, aov_filter, preview = match_aov_pattern( host_name, aov_filter, remainder ) + preview = preview and not do_not_add_review if preview: rep.update({ "fps": instance.get("fps"), @@ -428,7 +432,8 @@ def prepare_representations(instance, exp_files, anatomy, aov_filter, def create_instances_for_aov(instance, skeleton, aov_filter, - skip_integration_repre_list): + skip_integration_repre_list, + do_not_add_review): """Create instances from AOVs. This will create new pyblish.api.Instances by going over expected @@ -437,6 +442,7 @@ def create_instances_for_aov(instance, skeleton, aov_filter, Args: instance (pyblish.api.Instance): Original instance. skeleton (dict): Skeleton instance data. + skip_integration_repre_list (list): skip Returns: list of pyblish.api.Instance: Instances created from @@ -481,12 +487,13 @@ def create_instances_for_aov(instance, skeleton, aov_filter, skeleton, aov_filter, additional_color_data, - skip_integration_repre_list + skip_integration_repre_list, + do_not_add_review ) def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, - skip_integration_repre_list): + skip_integration_repre_list, do_not_add_review): """Create instance for each AOV found. This will create new instance for every AOV it can detect in expected @@ -496,7 +503,10 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, instance (pyblish.api.Instance): Original instance. skeleton (dict): Skeleton data for instance (those needed) later by collector. - additional_data (dict): ... + additional_data (dict): .. + skip_integration_repre_list (list): list of extensions that shouldn't + be published + do_not_addbe _review (bool): explicitly disable review Returns: @@ -631,7 +641,7 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, if ext in skip_integration_repre_list: rep["tags"].append("delete") - if preview: + if preview and not do_not_add_review: new_instance["families"] = _solve_families(new_instance) new_instance["representations"] = [rep] From c302b61d729826a223651f0fa7bc36992601030c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 May 2023 11:11:22 +0200 Subject: [PATCH 070/104] Removed _extend_frames Moved to publish_functions --- .../plugins/publish/submit_publish_job.py | 41 ------------------- 1 file changed, 41 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index acd47e32243..853a3983e7e 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -515,47 +515,6 @@ def process(self, instance): with open(metadata_path, "w") as f: json.dump(publish_job, f, indent=4, sort_keys=True) - def _extend_frames(self, asset, subset, start, end): - """Get latest version of asset nad update frame range. - - Based on minimum and maximuma values. - - Arguments: - asset (str): asset name - subset (str): subset name - start (int): start frame - end (int): end frame - - Returns: - (int, int): upddate frame start/end - - """ - # Frame comparison - prev_start = None - prev_end = None - - project_name = legacy_io.active_project() - version = get_last_version_by_subset_name( - project_name, - subset, - asset_name=asset - ) - - # Set prev start / end frames for comparison - if not prev_start and not prev_end: - prev_start = version["data"]["frameStart"] - prev_end = version["data"]["frameEnd"] - - updated_start = min(start, prev_start) - updated_end = max(end, prev_end) - - self.log.info( - "Updating start / end frame : " - "{} - {}".format(updated_start, updated_end) - ) - - return updated_start, updated_end - def _get_publish_folder(self, anatomy, template_data, asset, subset, family='render', version=None): From 1aef4c57f3d5d0f3fd70035ff3b6f452b43aecbc Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 May 2023 12:23:48 +0200 Subject: [PATCH 071/104] Fix missing import --- openpype/modules/deadline/abstract_submit_deadline.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index f6696b2c053..c2cc518e403 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -21,7 +21,10 @@ AbstractMetaInstancePlugin, KnownPublishError ) -from openpype.pipeline.publish.lib import replace_published_scene +from openpype.pipeline.publish.lib import ( + replace_published_scene, + get_published_workfile_instance +) JSONDecodeError = getattr(json.decoder, "JSONDecodeError", ValueError) From 78f9a68b35027daa1218491240be90f117b2486c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 May 2023 13:10:02 +0200 Subject: [PATCH 072/104] Fix aov handling Use col.head directly as rem in second logic path has no .head, it is only str. Fix review handling --- openpype/pipeline/farm/pyblish_functions.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index be031b1fde6..7444c221987 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -353,6 +353,7 @@ def prepare_representations(instance, exp_files, anatomy, aov_filter, if instance.get("slate"): frame_start -= 1 + # explicitly disable review by user preview = preview and not do_not_add_review rep = { "name": ext, @@ -544,14 +545,14 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, if len(cols) != 1: raise ValueError("Only one image sequence type is expected.") # noqa: E501 ext = cols[0].tail.lstrip(".") - col = list(cols[0]) + col = cols[0].head # create subset name `familyTaskSubset_AOV` group_name = 'render{}{}{}{}'.format( task[0].upper(), task[1:], subset[0].upper(), subset[1:]) - cam = [c for c in cameras if c in col.head] + cam = [c for c in cameras if c in col] if cam: if aov: subset_name = '{}_{}_{}'.format(group_name, cam, aov) @@ -577,8 +578,6 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, app = os.environ.get("AVALON_APP", "") - preview = False - if isinstance(col, list): render_file_name = os.path.basename(col[0]) else: @@ -587,11 +586,13 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, preview = match_aov_pattern(app, aov_patterns, render_file_name) # toggle preview on if multipart is on - if instance.data.get("multipartExr"): log.debug("Adding preview tag because its multipartExr") preview = True + # explicitly disable review by user + preview = preview and not do_not_add_review + new_instance = deepcopy(skeleton) new_instance["subsetGroup"] = group_name if preview: @@ -641,7 +642,7 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, if ext in skip_integration_repre_list: rep["tags"].append("delete") - if preview and not do_not_add_review: + if preview: new_instance["families"] = _solve_families(new_instance) new_instance["representations"] = [rep] From a4434d932b5462d5e4255152c27d3e233466467d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 May 2023 14:54:25 +0200 Subject: [PATCH 073/104] Fix adding review to families Renamed function to (bit) more reasonable name. --- openpype/pipeline/farm/pyblish_functions.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index 7444c221987..09cb50941a6 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -269,8 +269,11 @@ def create_skeleton_instance( return instance_skeleton_data -def _solve_families(families): - """Solve families. +def _add_review_families(families): + """Adds review flag to families. + + Handles situation when new instances are created which should have review + in families. In that case they should have 'ftrack' too. TODO: This is ugly and needs to be refactored. Ftrack family should be added in different way (based on if the module is enabled?) @@ -382,7 +385,7 @@ def prepare_representations(instance, exp_files, anatomy, aov_filter, representations.append(rep) if preview: - instance["families"] = _solve_families(instance["families"]) + instance["families"] = _add_review_families(instance["families"]) # add remainders as representations for remainder in remainders: @@ -416,7 +419,7 @@ def prepare_representations(instance, exp_files, anatomy, aov_filter, "fps": instance.get("fps"), "tags": ["review"] }) - instance["families"] = _solve_families(instance["families"]) + instance["families"] = _add_review_families(instance["families"]) already_there = False for repre in instance.get("representations", []): @@ -643,7 +646,8 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, rep["tags"].append("delete") if preview: - new_instance["families"] = _solve_families(new_instance) + new_instance["families"] = _add_review_families( + new_instance["families"]) new_instance["representations"] = [rep] From d745ebce16a2d64a5b3739c3aaa795d69726379b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 May 2023 14:54:50 +0200 Subject: [PATCH 074/104] Fix missing colorspaceTemplate key --- openpype/pipeline/farm/pyblish_functions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index 09cb50941a6..0b3e4acaaed 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -472,6 +472,7 @@ def create_instances_for_aov(instance, skeleton, aov_filter, colorspace_template, anatomy) except ValueError as e: log.warning(e) + additional_color_data["colorspaceTemplate"] = colorspace_template # if there are subset to attach to and more than one AOV, # we cannot proceed. From 162b58ce7a574d8d9ea5efde58c01e2ec9c99a29 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 May 2023 15:00:58 +0200 Subject: [PATCH 075/104] Fix access to anatomy --- .../plugins/publish/submit_publish_job.py | 38 ++++++------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 853a3983e7e..4f12552d341 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -165,6 +165,8 @@ def _submit_deadline_post_job(self, instance, job, instances): subset = data["subset"] job_name = "Publish - {subset}".format(subset=subset) + anatomy = instance.context.data['anatomy'] + # instance.data.get("subset") != instances[0]["subset"] # 'Main' vs 'renderMain' override_version = None @@ -172,7 +174,7 @@ def _submit_deadline_post_job(self, instance, job, instances): if instance_version != 1: override_version = instance_version output_dir = self._get_publish_folder( - instance.context.data['anatomy'], + anatomy, deepcopy(instance.data["anatomyData"]), instance.data.get("asset"), instances[0]["subset"], @@ -183,7 +185,7 @@ def _submit_deadline_post_job(self, instance, job, instances): # Transfer the environment from the original job to this dependent # job so they use the same environment metadata_path, rootless_metadata_path = \ - create_metadata_path(instance, self.anatomy) + create_metadata_path(instance, anatomy) environment = { "AVALON_PROJECT": legacy_io.Session["AVALON_PROJECT"], @@ -263,13 +265,15 @@ def _submit_deadline_post_job(self, instance, job, instances): self.log.info("Adding tile assembly jobs as dependencies...") job_index = 0 for assembly_id in instance.data.get("assemblySubmissionJobs"): - payload["JobInfo"]["JobDependency{}".format(job_index)] = assembly_id # noqa: E501 + payload["JobInfo"]["JobDependency{}".format( + job_index)] = assembly_id # noqa: E501 job_index += 1 elif instance.data.get("bakingSubmissionJobs"): self.log.info("Adding baking submission jobs as dependencies...") job_index = 0 for assembly_id in instance.data["bakingSubmissionJobs"]: - payload["JobInfo"]["JobDependency{}".format(job_index)] = assembly_id # noqa: E501 + payload["JobInfo"]["JobDependency{}".format( + job_index)] = assembly_id # noqa: E501 job_index += 1 else: payload["JobInfo"]["JobDependency0"] = job["_id"] @@ -300,25 +304,6 @@ def _submit_deadline_post_job(self, instance, job, instances): return deadline_publish_job_id - def _solve_families(self, instance, preview=False): - families = instance.get("families") - - # if we have one representation with preview tag - # flag whole instance for review and for ftrack - if preview: - if "ftrack" not in families: - if os.environ.get("FTRACK_SERVER"): - self.log.debug( - "Adding \"ftrack\" to families because of preview tag." - ) - families.append("ftrack") - if "review" not in families: - self.log.debug( - "Adding \"review\" to families because of preview tag." - ) - families.append("review") - instance["families"] = families - def process(self, instance): # type: (pyblish.api.Instance) -> None """Process plugin. @@ -335,6 +320,8 @@ def process(self, instance): self.log.info("Skipping local instance.") return + anatomy = instance.context.data["anatomy"] + instance_skeleton_data = create_skeleton_instance( instance, families_transfer=self.families_transfer, instance_transfer=self.instance_transfer) @@ -393,7 +380,7 @@ def process(self, instance): representations = prepare_representations( instance_skeleton_data, instance.data.get("expectedFiles"), - self.anatomy, + anatomy, self.aov_filter, self.skip_integration_repre_list, do_not_add_review @@ -509,9 +496,8 @@ def process(self, instance): publish_job.update({"ftrack": ftrack}) metadata_path, rootless_metadata_path = \ - create_metadata_path(instance, self.anatomy) + create_metadata_path(instance, anatomy) - self.log.info("Writing json file: {}".format(metadata_path)) with open(metadata_path, "w") as f: json.dump(publish_job, f, indent=4, sort_keys=True) From ad488c7ee4f54a209c617d3a6043122d41109deb Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 May 2023 16:50:52 +0200 Subject: [PATCH 076/104] Fix use proper families renderLayer is only for processing, final family is 'render'/'prerender'. --- openpype/pipeline/farm/pyblish_functions.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index 0b3e4acaaed..1bfc5e3dc23 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -188,12 +188,6 @@ def create_skeleton_instance( data = instance.data.copy() anatomy = instance.context.data["anatomy"] # type: Anatomy - families = [data["family"]] - - # pass review to families if marked as review - if data.get("review"): - families.append("review") - # get time related data from instance (or context) time_data = get_time_data_from_instance_or_context(instance) @@ -219,6 +213,12 @@ def create_skeleton_instance( family = ("render" if "prerender" not in instance.data["families"] else "prerender") + families = [family] + + # pass review to families if marked as review + if data.get("review"): + families.append("review") + instance_skeleton_data = { "family": family, "subset": data["subset"], From c1f3201316ab938ec2d97e65ca32a0021818d0f2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 May 2023 16:51:41 +0200 Subject: [PATCH 077/104] Fix use proper subset name Subset name should contain task, not only 'Main' --- openpype/pipeline/farm/pyblish_functions.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index 1bfc5e3dc23..89bff6dfeef 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -210,6 +210,7 @@ def create_skeleton_instance( log = Logger.get_logger("farm_publishing") log.warning(("Could not find root path for remapping \"{}\". " "This may cause issues.").format(source)) + family = ("render" if "prerender" not in instance.data["families"] else "prerender") @@ -594,11 +595,12 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, log.debug("Adding preview tag because its multipartExr") preview = True - # explicitly disable review by user - preview = preview and not do_not_add_review - new_instance = deepcopy(skeleton) + new_instance["subset"] = subset_name new_instance["subsetGroup"] = group_name + + # explicitly disable review by user + preview = preview and not do_not_add_review if preview: new_instance["review"] = True From 6df0fbccf77680b2621ccbea52703a980c8a9548 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 12 May 2023 15:33:33 +0200 Subject: [PATCH 078/104] OP-1066 - fix changed imports from pyblish.api import breaks Python3 hosts --- openpype/hosts/maya/api/fbx.py | 4 ++-- .../plugins/publish/create_maya_royalrender_job.py | 8 ++++---- .../plugins/publish/create_nuke_deadline_job.py | 9 +++++---- .../plugins/publish/create_publish_royalrender_job.py | 6 +++--- .../plugins/publish/submit_jobs_to_royalrender.py | 8 ++++---- openpype/pipeline/farm/pyblish_functions.py | 4 ++-- 6 files changed, 20 insertions(+), 19 deletions(-) diff --git a/openpype/hosts/maya/api/fbx.py b/openpype/hosts/maya/api/fbx.py index 260241f5fc3..e9cf8af491b 100644 --- a/openpype/hosts/maya/api/fbx.py +++ b/openpype/hosts/maya/api/fbx.py @@ -2,7 +2,7 @@ """Tools to work with FBX.""" import logging -from pyblish.api import Instance +import pyblish.api from maya import cmds # noqa import maya.mel as mel # noqa @@ -141,7 +141,7 @@ def parse_overrides(self, instance, options): return options def set_options_from_instance(self, instance): - # type: (Instance) -> None + # type: (pyblish.api.Instance) -> None """Sets FBX export options from data in the instance. Args: diff --git a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py index 8461e74d6d9..1bb2785dd68 100644 --- a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py @@ -5,7 +5,7 @@ import platform from maya.OpenMaya import MGlobal # noqa -from pyblish.api import InstancePlugin, IntegratorOrder, Context +import pyblish.api from openpype.lib import is_running_from_build from openpype.pipeline.publish.lib import get_published_workfile_instance from openpype.pipeline.publish import KnownPublishError @@ -14,9 +14,9 @@ from openpype.pipeline.farm.tools import iter_expected_files -class CreateMayaRoyalRenderJob(InstancePlugin): +class CreateMayaRoyalRenderJob(pyblish.api.InstancePlugin): label = "Create Maya Render job in RR" - order = IntegratorOrder + 0.1 + order = pyblish.api.IntegratorOrder + 0.1 families = ["renderlayer"] targets = ["local"] use_published = True @@ -128,7 +128,7 @@ def process(self, instance): @staticmethod def _resolve_rr_path(context, rr_path_name): - # type: (Context, str) -> str + # type: (pyblish.api.Context, str) -> str rr_settings = ( context.data ["system_settings"] diff --git a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py index 0d5e440c90f..a90c4c4f835 100644 --- a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py @@ -5,7 +5,7 @@ import platform from datetime import datetime -from pyblish.api import InstancePlugin, IntegratorOrder, Context +import pyblish.api from openpype.tests.lib import is_in_tests from openpype.pipeline.publish.lib import get_published_workfile_instance from openpype.pipeline.publish import KnownPublishError @@ -20,10 +20,11 @@ from openpype.pipeline import OpenPypePyblishPluginMixin -class CreateNukeRoyalRenderJob(InstancePlugin, OpenPypePyblishPluginMixin): +class CreateNukeRoyalRenderJob(pyblish.api.InstancePlugin, + OpenPypePyblishPluginMixin): """Creates separate rendering job for Royal Render""" label = "Create Nuke Render job in RR" - order = IntegratorOrder + 0.1 + order = pyblish.api.IntegratorOrder + 0.1 hosts = ["nuke"] families = ["render", "prerender"] targets = ["local"] @@ -248,7 +249,7 @@ def get_job(self, script_path, render_path, @staticmethod def _resolve_rr_path(context, rr_path_name): - # type: (Context, str) -> str + # type: (pyblish.api.Context, str) -> str rr_settings = ( context.data ["system_settings"] diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index af5bdfd5e6a..c652509373a 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -4,7 +4,7 @@ import attr import json -from pyblish.api import InstancePlugin, IntegratorOrder +import pyblish.api from openpype.modules.royalrender.rr_job import ( RRJob, @@ -24,7 +24,7 @@ ) -class CreatePublishRoyalRenderJob(InstancePlugin): +class CreatePublishRoyalRenderJob(pyblish.api.InstancePlugin): """Creates job which publishes rendered files to publish area. Job waits until all rendering jobs are finished, triggers `publish` command @@ -34,7 +34,7 @@ class CreatePublishRoyalRenderJob(InstancePlugin): When triggered it produces .log file next to .json file in work area. """ label = "Create publish job in RR" - order = IntegratorOrder + 0.2 + order = pyblish.api.IntegratorOrder + 0.2 icon = "tractor" targets = ["local"] hosts = ["fusion", "maya", "nuke", "celaction", "aftereffects", "harmony"] diff --git a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py index 6e91dff6ac8..8fc8604b839 100644 --- a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py +++ b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py @@ -3,7 +3,7 @@ import tempfile import platform -from pyblish.api import IntegratorOrder, ContextPlugin, Context +import pyblish.api from openpype.modules.royalrender.api import ( RRJob, Api as rrApi, @@ -12,10 +12,10 @@ from openpype.pipeline.publish import KnownPublishError -class SubmitJobsToRoyalRender(ContextPlugin): +class SubmitJobsToRoyalRender(pyblish.api.ContextPlugin): """Find all jobs, create submission XML and submit it to RoyalRender.""" label = "Submit jobs to RoyalRender" - order = IntegratorOrder + 0.3 + order = pyblish.api.IntegratorOrder + 0.3 targets = ["local"] def __init__(self): @@ -103,7 +103,7 @@ def get_submission_parameters(self): @staticmethod def _resolve_rr_path(context, rr_path_name): - # type: (Context, str) -> str + # type: (pyblish.api.Context, str) -> str rr_settings = ( context.data ["system_settings"] diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index 89bff6dfeef..8be0887b0d8 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -1,6 +1,6 @@ import copy import attr -from pyblish.api import Instance +import pyblish.api import os import clique from copy import deepcopy @@ -161,7 +161,7 @@ def get_transferable_representations(instance): def create_skeleton_instance( instance, families_transfer=None, instance_transfer=None): - # type: (Instance, list, dict) -> dict + # type: (pyblish.api.Instance, list, dict) -> dict """Create skeleton instance from original instance data. This will create dictionary containing skeleton From 2c3cd1c630e75292b9b32676ce2fb7638fcdb94d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 12 May 2023 15:34:32 +0200 Subject: [PATCH 079/104] OP-1066 - renamed file --- ...create_nuke_deadline_job.py => create_nuke_royalrender_job.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename openpype/modules/royalrender/plugins/publish/{create_nuke_deadline_job.py => create_nuke_royalrender_job.py} (100%) diff --git a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py b/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py similarity index 100% rename from openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py rename to openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py From 86a1357033c18a82f76385df8ae8531ae446f48e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 12 May 2023 18:23:38 +0200 Subject: [PATCH 080/104] OP-1066 - sanitize version RR expect version in format 3.157 instead proper 3.15.7-nightly.2 --- .../publish/create_publish_royalrender_job.py | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index c652509373a..6eb8f2649e3 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -3,6 +3,7 @@ import os import attr import json +import re import pyblish.api @@ -229,7 +230,7 @@ def get_job(self, instance, instances): SeqEnd=1, SeqStep=1, SeqFileOffset=0, - Version=os.environ.get("OPENPYPE_VERSION"), + Version=self._sanitize_version(os.environ.get("OPENPYPE_VERSION")), SceneName=abs_metadata_path, # command line arguments CustomAddCmdFlags=" ".join(args), @@ -256,3 +257,26 @@ def get_job(self, instance, instances): job.WaitForPreIDs += jobs_pre_ids return job + + def _sanitize_version(self, version): + """Returns version in format MAJOR.MINORPATCH + + 3.15.7-nightly.2 >> 3.157 + """ + VERSION_REGEX = re.compile( + r"(?P0|[1-9]\d*)" + r"\.(?P0|[1-9]\d*)" + r"\.(?P0|[1-9]\d*)" + r"(?:-(?P[a-zA-Z\d\-.]*))?" + r"(?:\+(?P[a-zA-Z\d\-.]*))?" + ) + + valid_parts = VERSION_REGEX.findall(version) + if len(valid_parts) != 1: + # Return invalid version with filled 'origin' attribute + return version + + # Unpack found version + major, minor, patch, pre, post = valid_parts[0] + + return "{}.{}{}".format(major, minor, patch) From 56f2edfa53a7cf4bde01cc69b8c1ae28514d90bd Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 17 May 2023 16:56:59 +0200 Subject: [PATCH 081/104] Add prerender family to collector for RR path --- .../plugins/publish/collect_rr_path_from_instance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py b/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py index e21cd7c39f9..ff5040c8df1 100644 --- a/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py +++ b/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py @@ -7,7 +7,7 @@ class CollectRRPathFromInstance(pyblish.api.InstancePlugin): order = pyblish.api.CollectorOrder label = "Collect Royal Render path name from the Instance" - families = ["render"] + families = ["render", "prerender"] def process(self, instance): instance.data["rrPathName"] = self._collect_rr_path_name(instance) From f6118ed6a8c79c9b6a89890e610771521d0f62e2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 18 May 2023 17:42:37 +0200 Subject: [PATCH 082/104] Added docuumentation --- website/docs/admin_settings_system.md | 4 +++ website/docs/module_royalrender.md | 37 +++++++++++++++++++++++++++ website/sidebars.js | 1 + 3 files changed, 42 insertions(+) create mode 100644 website/docs/module_royalrender.md diff --git a/website/docs/admin_settings_system.md b/website/docs/admin_settings_system.md index d61713ccd55..8abcefd24d6 100644 --- a/website/docs/admin_settings_system.md +++ b/website/docs/admin_settings_system.md @@ -102,6 +102,10 @@ workstation that should be submitting render jobs to muster via OpenPype. **`templates mapping`** - you can customize Muster templates to match your existing setup here. +### Royal Render + +**`Royal Render Root Paths`** - multi platform paths to Royal Render installation. + ### Clockify **`Workspace Name`** - name of the clockify workspace where you would like to be sending all the timelogs. diff --git a/website/docs/module_royalrender.md b/website/docs/module_royalrender.md new file mode 100644 index 00000000000..2b75fbefef8 --- /dev/null +++ b/website/docs/module_royalrender.md @@ -0,0 +1,37 @@ +--- +id: module_royalrender +title: Royal Render Administration +sidebar_label: Royal Render +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +## Preparation + +For [Royal Render](hhttps://www.royalrender.de/) support you need to set a few things up in both OpenPype and Royal Render itself + +1. Deploy OpenPype executable to all nodes of Royal Render farm. See [Install & Run](admin_use.md) + +2. Enable Royal Render Module in the [OpenPype Admin Settings](admin_settings_system.md#royal-render). + +3. Point OpenPype to your Royal Render installation in the [OpenPype Admin Settings](admin_settings_system.md#royal-render). + +4. Install our custom plugin and scripts to your RR repository. It should be as simple as copying content of `openpype/modules/royalrender/rr_root` to `path/to/your/royalrender/repository`. + + +## Configuration + +OpenPype integration for Royal Render consists of pointing RR to location of Openpype executable. That is being done by copying `_install_paths/OpenPype.cfg` to +RR root folder. This file contains reasonable defaults. They could be changed in this file or modified Render apps in `rrControl`. + + +## Debugging + +Current implementation uses dynamically build '.xml' file which is stored in temporary folder accessible by RR. It might make sense to +use this Openpype built file and try to run it via `*__rrServerConsole` executable from command line in case of unforeseeable issues. + +## Known issues + +Currently environment values set in Openpype are not propagated into render jobs on RR. It is studio responsibility to synchronize environment variables from Openpype with all render nodes for now. diff --git a/website/sidebars.js b/website/sidebars.js index 4874782197f..c846b04ca7f 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -111,6 +111,7 @@ module.exports = { "module_site_sync", "module_deadline", "module_muster", + "module_royalrender", "module_clockify", "module_slack" ], From d7fd9c8e18b7f47fa43c559038d08d71e54ab87c Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Mon, 5 Jun 2023 12:36:18 +0200 Subject: [PATCH 083/104] :recycle: non-optional access to rep data --- openpype/pipeline/farm/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/pipeline/farm/tools.py b/openpype/pipeline/farm/tools.py index 6f9e0ac3935..f3acac7a323 100644 --- a/openpype/pipeline/farm/tools.py +++ b/openpype/pipeline/farm/tools.py @@ -47,7 +47,7 @@ def from_published_scene(instance, replace_in_path=True): # determine published path from Anatomy. template_data = workfile_instance.data.get("anatomyData") - rep = workfile_instance.data.get("representations")[0] + rep = workfile_instance.data["representations"][0] template_data["representation"] = rep.get("name") template_data["ext"] = rep.get("ext") template_data["comment"] = None From 705368897936811a33f18d92b33939028cf0f85b Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 7 Jun 2023 16:20:30 +0200 Subject: [PATCH 084/104] :bug: fix rendering from published file in Deadline --- openpype/modules/deadline/abstract_submit_deadline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index c2cc518e403..525e112b710 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -430,7 +430,7 @@ def process(self, instance): file_path = None if self.use_published: if not self.import_reference: - file_path = get_published_workfile_instance(instance) + file_path = self.from_published_scene(context) else: self.log.info("use the scene with imported reference for rendering") # noqa file_path = context.data["currentFile"] From 2851b3230ec4ae332a0106913491fcbadfd6a179 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 7 Jun 2023 17:29:23 +0200 Subject: [PATCH 085/104] :bug: fix collections files must be lists, fixed camera handling of single frame renders --- openpype/pipeline/farm/pyblish_functions.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index 8be0887b0d8..0ace02edb90 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -550,14 +550,19 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, if len(cols) != 1: raise ValueError("Only one image sequence type is expected.") # noqa: E501 ext = cols[0].tail.lstrip(".") - col = cols[0].head + col = list(cols[0]) # create subset name `familyTaskSubset_AOV` group_name = 'render{}{}{}{}'.format( task[0].upper(), task[1:], subset[0].upper(), subset[1:]) - cam = [c for c in cameras if c in col] + # if there are multiple cameras, we need to add camera name + if isinstance(col, (list, tuple)): + cam = [c for c in cameras if c in col[0]] + else: + # in case of single frame + cam = [c for c in cameras if c in col] if cam: if aov: subset_name = '{}_{}_{}'.format(group_name, cam, aov) From 01b5e6b9500a8b4460429427ee06f6021208be97 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 9 Jun 2023 17:30:35 +0200 Subject: [PATCH 086/104] OP-6037 - extracted generic logic from working Nuke implementation Will be used as a base for Maya impl too. --- openpype/modules/royalrender/lib.py | 301 ++++++++++++++++ .../publish/create_nuke_royalrender_job.py | 320 ++---------------- 2 files changed, 326 insertions(+), 295 deletions(-) create mode 100644 openpype/modules/royalrender/lib.py diff --git a/openpype/modules/royalrender/lib.py b/openpype/modules/royalrender/lib.py new file mode 100644 index 00000000000..ce78b1a7386 --- /dev/null +++ b/openpype/modules/royalrender/lib.py @@ -0,0 +1,301 @@ +# -*- coding: utf-8 -*- +"""Submitting render job to RoyalRender.""" +import os +import re +import platform +from datetime import datetime + +import pyblish.api +from openpype.tests.lib import is_in_tests +from openpype.pipeline.publish.lib import get_published_workfile_instance +from openpype.pipeline.publish import KnownPublishError +from openpype.modules.royalrender.api import Api as rrApi +from openpype.modules.royalrender.rr_job import ( + RRJob, CustomAttribute, get_rr_platform) +from openpype.lib import ( + is_running_from_build, + BoolDef, + NumberDef, +) +from openpype.pipeline import OpenPypePyblishPluginMixin + + +class BaseCreateRoyalRenderJob(pyblish.api.InstancePlugin, + OpenPypePyblishPluginMixin): + """Creates separate rendering job for Royal Render""" + label = "Create Nuke Render job in RR" + order = pyblish.api.IntegratorOrder + 0.1 + hosts = ["nuke"] + families = ["render", "prerender"] + targets = ["local"] + optional = True + + priority = 50 + chunk_size = 1 + concurrent_tasks = 1 + use_gpu = True + use_published = True + + @classmethod + def get_attribute_defs(cls): + return [ + NumberDef( + "priority", + label="Priority", + default=cls.priority, + decimals=0 + ), + NumberDef( + "chunk", + label="Frames Per Task", + default=cls.chunk_size, + decimals=0, + minimum=1, + maximum=1000 + ), + NumberDef( + "concurrency", + label="Concurrency", + default=cls.concurrent_tasks, + decimals=0, + minimum=1, + maximum=10 + ), + BoolDef( + "use_gpu", + default=cls.use_gpu, + label="Use GPU" + ), + BoolDef( + "suspend_publish", + default=False, + label="Suspend publish" + ), + BoolDef( + "use_published", + default=cls.use_published, + label="Use published workfile" + ) + ] + + def __init__(self, *args, **kwargs): + self._instance = None + self._rr_root = None + self.scene_path = None + self.job = None + self.submission_parameters = None + self.rr_api = None + + def process(self, instance): + if not instance.data.get("farm"): + self.log.info("Skipping local instance.") + return + + instance.data["attributeValues"] = self.get_attr_values_from_data( + instance.data) + + # add suspend_publish attributeValue to instance data + instance.data["suspend_publish"] = instance.data["attributeValues"][ + "suspend_publish"] + + context = instance.context + self._instance = instance + + self._rr_root = self._resolve_rr_path(context, instance.data.get( + "rrPathName")) # noqa + self.log.debug(self._rr_root) + if not self._rr_root: + raise KnownPublishError( + ("Missing RoyalRender root. " + "You need to configure RoyalRender module.")) + + self.rr_api = rrApi(self._rr_root) + + self.scene_path = context.data["currentFile"] + if self.use_published: + file_path = get_published_workfile_instance(context) + + # fallback if nothing was set + if not file_path: + self.log.warning("Falling back to workfile") + file_path = context.data["currentFile"] + + self.scene_path = file_path + self.log.info( + "Using published scene for render {}".format(self.scene_path) + ) + + if not self._instance.data.get("expectedFiles"): + self._instance.data["expectedFiles"] = [] + + if not self._instance.data.get("rrJobs"): + self._instance.data["rrJobs"] = [] + + self._instance.data["outputDir"] = os.path.dirname( + self._instance.data["path"]).replace("\\", "/") + + def get_job(self, instance, script_path, render_path, node_name): + """Get RR job based on current instance. + + Args: + script_path (str): Path to Nuke script. + render_path (str): Output path. + node_name (str): Name of the render node. + + Returns: + RRJob: RoyalRender Job instance. + + """ + start_frame = int(instance.data["frameStartHandle"]) + end_frame = int(instance.data["frameEndHandle"]) + + batch_name = os.path.basename(script_path) + jobname = "%s - %s" % (batch_name, self._instance.name) + if is_in_tests(): + batch_name += datetime.now().strftime("%d%m%Y%H%M%S") + + render_dir = os.path.normpath(os.path.dirname(render_path)) + output_filename_0 = self.preview_fname(render_path) + file_name, file_ext = os.path.splitext( + os.path.basename(output_filename_0)) + + custom_attributes = [] + if is_running_from_build(): + custom_attributes = [ + CustomAttribute( + name="OpenPypeVersion", + value=os.environ.get("OPENPYPE_VERSION")) + ] + + # this will append expected files to instance as needed. + expected_files = self.expected_files( + instance, render_path, start_frame, end_frame) + instance.data["expectedFiles"].extend(expected_files) + + job = RRJob( + Software="", + Renderer="", + SeqStart=int(start_frame), + SeqEnd=int(end_frame), + SeqStep=int(instance.data.get("byFrameStep", 1)), + SeqFileOffset=0, + Version=0, + SceneName=script_path, + IsActive=True, + ImageDir=render_dir.replace("\\", "/"), + ImageFilename=file_name, + ImageExtension=file_ext, + ImagePreNumberLetter="", + ImageSingleOutputFile=False, + SceneOS=get_rr_platform(), + Layer=node_name, + SceneDatabaseDir=script_path, + CustomSHotName=jobname, + CompanyProjectName=instance.context.data["projectName"], + ImageWidth=instance.data["resolutionWidth"], + ImageHeight=instance.data["resolutionHeight"], + CustomAttributes=custom_attributes + ) + + return job + + def update_job_with_host_specific(self, instance, job): + """Host specific mapping for RRJob""" + raise NotImplementedError + + @staticmethod + def _resolve_rr_path(context, rr_path_name): + # type: (pyblish.api.Context, str) -> str + rr_settings = ( + context.data + ["system_settings"] + ["modules"] + ["royalrender"] + ) + try: + default_servers = rr_settings["rr_paths"] + project_servers = ( + context.data + ["project_settings"] + ["royalrender"] + ["rr_paths"] + ) + rr_servers = { + k: default_servers[k] + for k in project_servers + if k in default_servers + } + + except (AttributeError, KeyError): + # Handle situation were we had only one url for royal render. + return context.data["defaultRRPath"][platform.system().lower()] + + return rr_servers[rr_path_name][platform.system().lower()] + + def expected_files(self, instance, path, start_frame, end_frame): + """Get expected files. + + This function generate expected files from provided + path and start/end frames. + + It was taken from Deadline module, but this should be + probably handled better in collector to support more + flexible scenarios. + + Args: + instance (Instance) + path (str): Output path. + start_frame (int): Start frame. + end_frame (int): End frame. + + Returns: + list: List of expected files. + + """ + if instance.data.get("expectedFiles"): + return instance.data["expectedFiles"] + + dir_name = os.path.dirname(path) + file = os.path.basename(path) + + expected_files = [] + + if "#" in file: + pparts = file.split("#") + padding = "%0{}d".format(len(pparts) - 1) + file = pparts[0] + padding + pparts[-1] + + if "%" not in file: + expected_files.append(path) + return expected_files + + if self._instance.data.get("slate"): + start_frame -= 1 + + expected_files.extend( + os.path.join(dir_name, (file % i)).replace("\\", "/") + for i in range(start_frame, (end_frame + 1)) + ) + return expected_files + + def preview_fname(self, path): + """Return output file path with #### for padding. + + RR requires the path to be formatted with # in place of numbers. + For example `/path/to/render.####.png` + + Args: + path (str): path to rendered images + + Returns: + str + + """ + self.log.debug("_ path: `{}`".format(path)) + if "%" in path: + search_results = re.search(r"(%0)(\d)(d.)", path).groups() + self.log.debug("_ search_results: `{}`".format(search_results)) + return int(search_results[1]) + if "#" in path: + self.log.debug("_ path: `{}`".format(path)) + return path diff --git a/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py index a90c4c4f835..62636a6744f 100644 --- a/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py @@ -1,137 +1,18 @@ # -*- coding: utf-8 -*- """Submitting render job to RoyalRender.""" -import os import re -import platform -from datetime import datetime -import pyblish.api -from openpype.tests.lib import is_in_tests -from openpype.pipeline.publish.lib import get_published_workfile_instance -from openpype.pipeline.publish import KnownPublishError -from openpype.modules.royalrender.api import Api as rrApi -from openpype.modules.royalrender.rr_job import ( - RRJob, CustomAttribute, get_rr_platform) -from openpype.lib import ( - is_running_from_build, - BoolDef, - NumberDef, -) -from openpype.pipeline import OpenPypePyblishPluginMixin +from openpype.modules.royalrender import lib -class CreateNukeRoyalRenderJob(pyblish.api.InstancePlugin, - OpenPypePyblishPluginMixin): +class CreateNukeRoyalRenderJob(lib.BaseCreateRoyalRenderJob): """Creates separate rendering job for Royal Render""" label = "Create Nuke Render job in RR" - order = pyblish.api.IntegratorOrder + 0.1 hosts = ["nuke"] families = ["render", "prerender"] - targets = ["local"] - optional = True - - priority = 50 - chunk_size = 1 - concurrent_tasks = 1 - use_gpu = True - use_published = True - - @classmethod - def get_attribute_defs(cls): - return [ - NumberDef( - "priority", - label="Priority", - default=cls.priority, - decimals=0 - ), - NumberDef( - "chunk", - label="Frames Per Task", - default=cls.chunk_size, - decimals=0, - minimum=1, - maximum=1000 - ), - NumberDef( - "concurrency", - label="Concurrency", - default=cls.concurrent_tasks, - decimals=0, - minimum=1, - maximum=10 - ), - BoolDef( - "use_gpu", - default=cls.use_gpu, - label="Use GPU" - ), - BoolDef( - "suspend_publish", - default=False, - label="Suspend publish" - ), - BoolDef( - "use_published", - default=cls.use_published, - label="Use published workfile" - ) - ] - - def __init__(self, *args, **kwargs): - self._instance = None - self._rr_root = None - self.scene_path = None - self.job = None - self.submission_parameters = None - self.rr_api = None def process(self, instance): - if not instance.data.get("farm"): - self.log.info("Skipping local instance.") - return - - instance.data["attributeValues"] = self.get_attr_values_from_data( - instance.data) - - # add suspend_publish attributeValue to instance data - instance.data["suspend_publish"] = instance.data["attributeValues"][ - "suspend_publish"] - - context = instance.context - self._instance = instance - - self._rr_root = self._resolve_rr_path(context, instance.data.get( - "rrPathName")) # noqa - self.log.debug(self._rr_root) - if not self._rr_root: - raise KnownPublishError( - ("Missing RoyalRender root. " - "You need to configure RoyalRender module.")) - - self.rr_api = rrApi(self._rr_root) - - self.scene_path = context.data["currentFile"] - if self.use_published: - file_path = get_published_workfile_instance(context) - - # fallback if nothing was set - if not file_path: - self.log.warning("Falling back to workfile") - file_path = context.data["currentFile"] - - self.scene_path = file_path - self.log.info( - "Using published scene for render {}".format(self.scene_path) - ) - - if not self._instance.data.get("expectedFiles"): - self._instance.data["expectedFiles"] = [] - - if not self._instance.data.get("rrJobs"): - self._instance.data["rrJobs"] = [] - - self._instance.data["rrJobs"].extend(self.create_jobs()) + super(CreateNukeRoyalRenderJob, self).process(instance) # redefinition of families if "render" in self._instance.data["family"]: @@ -141,26 +22,37 @@ def process(self, instance): self._instance.data["family"] = "write" self._instance.data["families"].insert(0, "prerender") - self._instance.data["outputDir"] = os.path.dirname( - self._instance.data["path"]).replace("\\", "/") + jobs = self.create_jobs(self._instance) + for job in jobs: + job = self.update_job_with_host_specific(instance, job) + + instance.data["rrJobs"] += jobs + + def update_job_with_host_specific(self, instance, job): + nuke_version = re.search( + r"\d+\.\d+", self._instance.context.data.get("hostVersion")) + + job.Software = "Nuke" + job.Version = nuke_version.group() - def create_jobs(self): - submit_frame_start = int(self._instance.data["frameStartHandle"]) - submit_frame_end = int(self._instance.data["frameEndHandle"]) + return job + def create_jobs(self, instance): + """Nuke creates multiple RR jobs - for baking etc.""" # get output path - render_path = self._instance.data['path'] + render_path = instance.data['path'] + self.log.info("render::{}".format(render_path)) + self.log.info("expected::{}".format(instance.data.get("expectedFiles"))) script_path = self.scene_path node = self._instance.data["transientData"]["node"] # main job jobs = [ self.get_job( + instance, script_path, render_path, - node.name(), - submit_frame_start, - submit_frame_end, + node.name() ) ] @@ -170,172 +62,10 @@ def create_jobs(self): exe_node_name = baking_script["bakeWriteNodeName"] jobs.append(self.get_job( + instance, script_path, render_path, - exe_node_name, - submit_frame_start, - submit_frame_end + exe_node_name )) return jobs - - def get_job(self, script_path, render_path, - node_name, start_frame, end_frame): - """Get RR job based on current instance. - - Args: - script_path (str): Path to Nuke script. - render_path (str): Output path. - node_name (str): Name of the render node. - start_frame (int): Start frame. - end_frame (int): End frame. - - Returns: - RRJob: RoyalRender Job instance. - - """ - render_dir = os.path.normpath(os.path.dirname(render_path)) - batch_name = os.path.basename(script_path) - jobname = "%s - %s" % (batch_name, self._instance.name) - if is_in_tests(): - batch_name += datetime.now().strftime("%d%m%Y%H%M%S") - - output_filename_0 = self.preview_fname(render_path) - file_name, file_ext = os.path.splitext( - os.path.basename(output_filename_0)) - - custom_attributes = [] - if is_running_from_build(): - custom_attributes = [ - CustomAttribute( - name="OpenPypeVersion", - value=os.environ.get("OPENPYPE_VERSION")) - ] - - nuke_version = re.search( - r"\d+\.\d+", self._instance.context.data.get("hostVersion")) - - # this will append expected files to instance as needed. - expected_files = self.expected_files( - render_path, start_frame, end_frame) - self._instance.data["expectedFiles"].extend(expected_files) - - job = RRJob( - Software="Nuke", - Renderer="", - SeqStart=int(start_frame), - SeqEnd=int(end_frame), - SeqStep=int(self._instance.data.get("byFrameStep", 1)), - SeqFileOffset=0, - Version=nuke_version.group(), - SceneName=script_path, - IsActive=True, - ImageDir=render_dir.replace("\\", "/"), - ImageFilename=file_name, - ImageExtension=file_ext, - ImagePreNumberLetter="", - ImageSingleOutputFile=False, - SceneOS=get_rr_platform(), - Layer=node_name, - SceneDatabaseDir=script_path, - CustomSHotName=jobname, - CompanyProjectName=self._instance.context.data["projectName"], - ImageWidth=self._instance.data["resolutionWidth"], - ImageHeight=self._instance.data["resolutionHeight"], - CustomAttributes=custom_attributes - ) - - return job - - @staticmethod - def _resolve_rr_path(context, rr_path_name): - # type: (pyblish.api.Context, str) -> str - rr_settings = ( - context.data - ["system_settings"] - ["modules"] - ["royalrender"] - ) - try: - default_servers = rr_settings["rr_paths"] - project_servers = ( - context.data - ["project_settings"] - ["royalrender"] - ["rr_paths"] - ) - rr_servers = { - k: default_servers[k] - for k in project_servers - if k in default_servers - } - - except (AttributeError, KeyError): - # Handle situation were we had only one url for royal render. - return context.data["defaultRRPath"][platform.system().lower()] - - return rr_servers[rr_path_name][platform.system().lower()] - - def expected_files(self, path, start_frame, end_frame): - """Get expected files. - - This function generate expected files from provided - path and start/end frames. - - It was taken from Deadline module, but this should be - probably handled better in collector to support more - flexible scenarios. - - Args: - path (str): Output path. - start_frame (int): Start frame. - end_frame (int): End frame. - - Returns: - list: List of expected files. - - """ - dir_name = os.path.dirname(path) - file = os.path.basename(path) - - expected_files = [] - - if "#" in file: - pparts = file.split("#") - padding = "%0{}d".format(len(pparts) - 1) - file = pparts[0] + padding + pparts[-1] - - if "%" not in file: - expected_files.append(path) - return expected_files - - if self._instance.data.get("slate"): - start_frame -= 1 - - expected_files.extend( - os.path.join(dir_name, (file % i)).replace("\\", "/") - for i in range(start_frame, (end_frame + 1)) - ) - return expected_files - - def preview_fname(self, path): - """Return output file path with #### for padding. - - Deadline requires the path to be formatted with # in place of numbers. - For example `/path/to/render.####.png` - - Args: - path (str): path to rendered images - - Returns: - str - - """ - self.log.debug("_ path: `{}`".format(path)) - if "%" in path: - search_results = re.search(r"(%0)(\d)(d.)", path).groups() - self.log.debug("_ search_results: `{}`".format(search_results)) - return int(search_results[1]) - if "#" in path: - self.log.debug("_ path: `{}`".format(path)) - return path From 644b734585cadecb32f953b4efa3c4859150bebb Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 9 Jun 2023 17:32:11 +0200 Subject: [PATCH 087/104] OP-6037 - first iteration of Maya to RR Uses generic logic, probably needs fine tuning. --- .../publish/create_maya_royalrender_job.py | 160 +++--------------- 1 file changed, 23 insertions(+), 137 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py index 1bb2785dd68..5ad3c90bdc3 100644 --- a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py @@ -1,156 +1,42 @@ # -*- coding: utf-8 -*- """Submitting render job to RoyalRender.""" +# -*- coding: utf-8 -*- +"""Submitting render job to RoyalRender.""" import os -import sys -import platform -from maya.OpenMaya import MGlobal # noqa -import pyblish.api -from openpype.lib import is_running_from_build -from openpype.pipeline.publish.lib import get_published_workfile_instance -from openpype.pipeline.publish import KnownPublishError -from openpype.modules.royalrender.api import Api as rrApi -from openpype.modules.royalrender.rr_job import RRJob, CustomAttribute +from maya.OpenMaya import MGlobal + +from openpype.modules.royalrender import lib from openpype.pipeline.farm.tools import iter_expected_files -class CreateMayaRoyalRenderJob(pyblish.api.InstancePlugin): +class CreateMayaRoyalRenderJob(lib.BaseCreateRoyalRenderJob): label = "Create Maya Render job in RR" - order = pyblish.api.IntegratorOrder + 0.1 families = ["renderlayer"] - targets = ["local"] - use_published = True - - def __init__(self, *args, **kwargs): - self._instance = None - self._rrRoot = None - self.scene_path = None - self.job = None - self.submission_parameters = None - self.rr_api = None - - def get_job(self): - """Prepare job payload. - - Returns: - RRJob: RoyalRender job payload. - """ - def get_rr_platform(): - if sys.platform.lower() in ["win32", "win64"]: - return "windows" - elif sys.platform.lower() == "darwin": - return "mac" - else: - return "linux" - - expected_files = self._instance.data["expectedFiles"] - first_file = next(iter_expected_files(expected_files)) - output_dir = os.path.dirname(first_file) - self._instance.data["outputDir"] = output_dir - workspace = self._instance.context.data["workspaceDir"] - default_render_file = ( - self._instance.context.data - ['project_settings'] - ['maya'] - ['RenderSettings'] - ['default_render_image_folder'] - ) - # file_name = os.path.basename(self.scene_path) - dir_name = os.path.join(workspace, default_render_file) - layer = self._instance.data["setMembers"] # type: str - layer_name = layer.removeprefix("rs_") - custom_attributes = [] - if is_running_from_build(): - custom_attributes = [ - CustomAttribute( - name="OpenPypeVersion", - value=os.environ.get("OPENPYPE_VERSION")) - ] - - job = RRJob( - Software="Maya", - Renderer=self._instance.data["renderer"], - SeqStart=int(self._instance.data["frameStartHandle"]), - SeqEnd=int(self._instance.data["frameEndHandle"]), - SeqStep=int(self._instance.data["byFrameStep"]), - SeqFileOffset=0, - Version="{0:.2f}".format(MGlobal.apiVersion() / 10000), - SceneName=self.scene_path, - IsActive=True, - ImageDir=dir_name, - ImageFilename="{}.".format(layer_name), - ImageExtension=os.path.splitext(first_file)[1], - ImagePreNumberLetter=".", - ImageSingleOutputFile=False, - SceneOS=get_rr_platform(), - Camera=self._instance.data["cameras"][0], - Layer=layer_name, - SceneDatabaseDir=workspace, - CustomSHotName=self._instance.context.data["asset"], - CompanyProjectName=self._instance.context.data["projectName"], - ImageWidth=self._instance.data["resolutionWidth"], - ImageHeight=self._instance.data["resolutionHeight"], - CustomAttributes=custom_attributes - ) + def update_job_with_host_specific(self, instance, job): + job.Software = "Maya" + job.Version = "{0:.2f}".format(MGlobal.apiVersion() / 10000) + job.Camera = instance.data["cameras"][0], + workspace = instance.context.data["workspaceDir"] + job.SceneDatabaseDir = workspace return job def process(self, instance): """Plugin entry point.""" - self._instance = instance - context = instance.context - - self._rr_root = self._resolve_rr_path(context, instance.data.get("rrPathName")) # noqa - self.log.debug(self._rr_root) - if not self._rr_root: - raise KnownPublishError( - ("Missing RoyalRender root. " - "You need to configure RoyalRender module.")) - - self.rr_api = rrApi(self._rr_root) - - file_path = None - if self.use_published: - file_path = get_published_workfile_instance(context) - - # fallback if nothing was set - if not file_path: - self.log.warning("Falling back to workfile") - file_path = context.data["currentFile"] - - self.scene_path = file_path + super(CreateMayaRoyalRenderJob, self).process(instance) - if not self._instance.data.get("rrJobs"): - self._instance.data["rrJobs"] = [] - - self._instance.data["rrJobs"] += self.get_job() + expected_files = self._instance.data["expectedFiles"] + first_file_path = next(iter_expected_files(expected_files)) + output_dir = os.path.dirname(first_file_path) + self._instance.data["outputDir"] = output_dir - @staticmethod - def _resolve_rr_path(context, rr_path_name): - # type: (pyblish.api.Context, str) -> str - rr_settings = ( - context.data - ["system_settings"] - ["modules"] - ["royalrender"] - ) - try: - default_servers = rr_settings["rr_paths"] - project_servers = ( - context.data - ["project_settings"] - ["royalrender"] - ["rr_paths"] - ) - rr_servers = { - k: default_servers[k] - for k in project_servers - if k in default_servers - } + layer = self._instance.data["setMembers"] # type: str + layer_name = layer.removeprefix("rs_") - except (AttributeError, KeyError): - # Handle situation were we had only one url for royal render. - return context.data["defaultRRPath"][platform.system().lower()] + job = self.get_job(instance, self.scene_path, first_file_path, + layer_name) + job = self.update_job_with_host_specific(instance, job) - return rr_servers[rr_path_name][platform.system().lower()] + instance.data["rrJobs"] += job From 9cb121601a27727dd73df8b3241bd72c666edda0 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 14 Jun 2023 10:13:48 +0200 Subject: [PATCH 088/104] Resolved conflict --- .../plugins/publish/collect_rr_path_from_instance.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py b/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py index 6a3dc276f37..c2ba1e2c192 100644 --- a/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py +++ b/openpype/modules/royalrender/plugins/publish/collect_rr_path_from_instance.py @@ -5,9 +5,9 @@ class CollectRRPathFromInstance(pyblish.api.InstancePlugin): """Collect RR Path from instance.""" - order = pyblish.api.CollectorOrder + 0.01 - label = "Royal Render Path from the Instance" - families = ["rendering"] + order = pyblish.api.CollectorOrder + label = "Collect Royal Render path name from the Instance" + families = ["render", "prerender", "renderlayer"] def process(self, instance): instance.data["rrPath"] = self._collect_rr_path(instance) From 6619be9bc72159aa0cfe60d63403bfa28b903a5a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 27 Jun 2023 11:49:29 +0200 Subject: [PATCH 089/104] Fix append job --- .../plugins/publish/create_maya_royalrender_job.py | 5 ++--- .../plugins/publish/create_nuke_royalrender_job.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py index 5ad3c90bdc3..695f33cc359 100644 --- a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- """Submitting render job to RoyalRender.""" -# -*- coding: utf-8 -*- -"""Submitting render job to RoyalRender.""" import os from maya.OpenMaya import MGlobal @@ -12,6 +10,7 @@ class CreateMayaRoyalRenderJob(lib.BaseCreateRoyalRenderJob): label = "Create Maya Render job in RR" + hosts = ["maya"] families = ["renderlayer"] def update_job_with_host_specific(self, instance, job): @@ -39,4 +38,4 @@ def process(self, instance): layer_name) job = self.update_job_with_host_specific(instance, job) - instance.data["rrJobs"] += job + instance.data["rrJobs"].append(job) diff --git a/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py index 62636a6744f..7b4a66a920d 100644 --- a/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py @@ -26,7 +26,7 @@ def process(self, instance): for job in jobs: job = self.update_job_with_host_specific(instance, job) - instance.data["rrJobs"] += jobs + instance.data["rrJobs"].append(job) def update_job_with_host_specific(self, instance, job): nuke_version = re.search( From d6b0b9c74c70d33f57374229a1c3352237da53a7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 27 Jun 2023 11:50:25 +0200 Subject: [PATCH 090/104] Remove outputDir --- openpype/modules/royalrender/lib.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/openpype/modules/royalrender/lib.py b/openpype/modules/royalrender/lib.py index ce78b1a7386..1433bac6a27 100644 --- a/openpype/modules/royalrender/lib.py +++ b/openpype/modules/royalrender/lib.py @@ -131,9 +131,6 @@ def process(self, instance): if not self._instance.data.get("rrJobs"): self._instance.data["rrJobs"] = [] - self._instance.data["outputDir"] = os.path.dirname( - self._instance.data["path"]).replace("\\", "/") - def get_job(self, instance, script_path, render_path, node_name): """Get RR job based on current instance. From d0e1d5c36fbaa96d12bb0b372aa207e7ec000be4 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 27 Jun 2023 18:29:33 +0200 Subject: [PATCH 091/104] Fix cleanup of camera info Without it .mel script in RR would fail with syntax error. --- .../royalrender/plugins/publish/create_maya_royalrender_job.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py index 695f33cc359..69e54f54b7e 100644 --- a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py @@ -16,7 +16,8 @@ class CreateMayaRoyalRenderJob(lib.BaseCreateRoyalRenderJob): def update_job_with_host_specific(self, instance, job): job.Software = "Maya" job.Version = "{0:.2f}".format(MGlobal.apiVersion() / 10000) - job.Camera = instance.data["cameras"][0], + if instance.data.get("cameras"): + job.Camera = instance.data["cameras"][0].replace("'", '"') workspace = instance.context.data["workspaceDir"] job.SceneDatabaseDir = workspace From ec54515b8caf08773ef721b6a9e7a6edc3e4ecb5 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 28 Jun 2023 12:53:28 +0200 Subject: [PATCH 092/104] Fix - correct resolution of published workile path --- openpype/modules/royalrender/lib.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/openpype/modules/royalrender/lib.py b/openpype/modules/royalrender/lib.py index 1433bac6a27..cbddb22b4ef 100644 --- a/openpype/modules/royalrender/lib.py +++ b/openpype/modules/royalrender/lib.py @@ -113,12 +113,15 @@ def process(self, instance): self.scene_path = context.data["currentFile"] if self.use_published: - file_path = get_published_workfile_instance(context) + published_workfile = get_published_workfile_instance(context) # fallback if nothing was set - if not file_path: + if published_workfile is None: self.log.warning("Falling back to workfile") file_path = context.data["currentFile"] + else: + workfile_repre = published_workfile.data["representations"][0] + file_path = workfile_repre["published_path"] self.scene_path = file_path self.log.info( From 0df104be54146921944fa32a02f98fac4db455d1 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 28 Jun 2023 15:10:57 +0200 Subject: [PATCH 093/104] Fix - proper padding of file name RR requires replacing frame value with # properly padded. --- openpype/modules/royalrender/lib.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/openpype/modules/royalrender/lib.py b/openpype/modules/royalrender/lib.py index cbddb22b4ef..006ebc5bfd2 100644 --- a/openpype/modules/royalrender/lib.py +++ b/openpype/modules/royalrender/lib.py @@ -155,7 +155,7 @@ def get_job(self, instance, script_path, render_path, node_name): batch_name += datetime.now().strftime("%d%m%Y%H%M%S") render_dir = os.path.normpath(os.path.dirname(render_path)) - output_filename_0 = self.preview_fname(render_path) + output_filename_0 = self.pad_file_name(render_path, str(start_frame)) file_name, file_ext = os.path.splitext( os.path.basename(output_filename_0)) @@ -278,14 +278,16 @@ def expected_files(self, instance, path, start_frame, end_frame): ) return expected_files - def preview_fname(self, path): + def pad_file_name(self, path, first_frame): """Return output file path with #### for padding. RR requires the path to be formatted with # in place of numbers. For example `/path/to/render.####.png` Args: - path (str): path to rendered images + path (str): path to rendered image + first_frame (str): from representation to cleany replace with # + padding Returns: str @@ -298,4 +300,10 @@ def preview_fname(self, path): return int(search_results[1]) if "#" in path: self.log.debug("_ path: `{}`".format(path)) + return path + + if first_frame: + padding = len(first_frame) + path = path.replace(first_frame, "#" * padding) + return path From 4a672c5c12ee8d8b3659f08d1a581f31f272ca20 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 28 Jun 2023 15:15:36 +0200 Subject: [PATCH 094/104] Hound --- openpype/modules/deadline/abstract_submit_deadline.py | 3 +-- .../royalrender/plugins/publish/create_nuke_royalrender_job.py | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index b63a43598d0..6ff9afbc427 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -23,8 +23,7 @@ OpenPypePyblishPluginMixin ) from openpype.pipeline.publish.lib import ( - replace_published_scene, - get_published_workfile_instance + replace_published_scene ) JSONDecodeError = getattr(json.decoder, "JSONDecodeError", ValueError) diff --git a/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py index 7b4a66a920d..763792bdc22 100644 --- a/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py @@ -41,8 +41,6 @@ def create_jobs(self, instance): """Nuke creates multiple RR jobs - for baking etc.""" # get output path render_path = instance.data['path'] - self.log.info("render::{}".format(render_path)) - self.log.info("expected::{}".format(instance.data.get("expectedFiles"))) script_path = self.scene_path node = self._instance.data["transientData"]["node"] From b9e362d1e8a0edb4c7c2bc44d5af8be8a94a6b37 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 3 Jul 2023 17:15:59 +0200 Subject: [PATCH 095/104] Fix - Nuke needs to have multiple jobs First job is pure rendering, then there could be baking etc. scripts. --- .../royalrender/plugins/publish/create_nuke_royalrender_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py index 763792bdc22..7f503c80a9d 100644 --- a/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py @@ -26,7 +26,7 @@ def process(self, instance): for job in jobs: job = self.update_job_with_host_specific(instance, job) - instance.data["rrJobs"].append(job) + instance.data["rrJobs"].append(job) def update_job_with_host_specific(self, instance, job): nuke_version = re.search( From 614d600564643fdd4ebcc543a2571425611ea385 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 4 Jul 2023 17:21:09 +0200 Subject: [PATCH 096/104] Refactor - removed unnecessary variable Fixed expectedFiles, they need to extended (at least in Nuke, first renders, then baking etc.) --- openpype/modules/royalrender/lib.py | 21 +++++++----------- .../publish/create_maya_royalrender_job.py | 6 ++--- .../publish/create_nuke_royalrender_job.py | 22 +++++++++---------- 3 files changed, 22 insertions(+), 27 deletions(-) diff --git a/openpype/modules/royalrender/lib.py b/openpype/modules/royalrender/lib.py index 006ebc5bfd2..4708d25eed3 100644 --- a/openpype/modules/royalrender/lib.py +++ b/openpype/modules/royalrender/lib.py @@ -79,7 +79,6 @@ def get_attribute_defs(cls): ] def __init__(self, *args, **kwargs): - self._instance = None self._rr_root = None self.scene_path = None self.job = None @@ -99,7 +98,6 @@ def process(self, instance): "suspend_publish"] context = instance.context - self._instance = instance self._rr_root = self._resolve_rr_path(context, instance.data.get( "rrPathName")) # noqa @@ -128,11 +126,11 @@ def process(self, instance): "Using published scene for render {}".format(self.scene_path) ) - if not self._instance.data.get("expectedFiles"): - self._instance.data["expectedFiles"] = [] + if not instance.data.get("expectedFiles"): + instance.data["expectedFiles"] = [] - if not self._instance.data.get("rrJobs"): - self._instance.data["rrJobs"] = [] + if not instance.data.get("rrJobs"): + instance.data["rrJobs"] = [] def get_job(self, instance, script_path, render_path, node_name): """Get RR job based on current instance. @@ -150,7 +148,7 @@ def get_job(self, instance, script_path, render_path, node_name): end_frame = int(instance.data["frameEndHandle"]) batch_name = os.path.basename(script_path) - jobname = "%s - %s" % (batch_name, self._instance.name) + jobname = "%s - %s" % (batch_name, instance.name) if is_in_tests(): batch_name += datetime.now().strftime("%d%m%Y%H%M%S") @@ -252,9 +250,6 @@ def expected_files(self, instance, path, start_frame, end_frame): list: List of expected files. """ - if instance.data.get("expectedFiles"): - return instance.data["expectedFiles"] - dir_name = os.path.dirname(path) file = os.path.basename(path) @@ -269,7 +264,7 @@ def expected_files(self, instance, path, start_frame, end_frame): expected_files.append(path) return expected_files - if self._instance.data.get("slate"): + if instance.data.get("slate"): start_frame -= 1 expected_files.extend( @@ -293,13 +288,13 @@ def pad_file_name(self, path, first_frame): str """ - self.log.debug("_ path: `{}`".format(path)) + self.log.debug("pad_file_name path: `{}`".format(path)) if "%" in path: search_results = re.search(r"(%0)(\d)(d.)", path).groups() self.log.debug("_ search_results: `{}`".format(search_results)) return int(search_results[1]) if "#" in path: - self.log.debug("_ path: `{}`".format(path)) + self.log.debug("already padded: `{}`".format(path)) return path if first_frame: diff --git a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py index 69e54f54b7e..22d910b7cd0 100644 --- a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py @@ -27,12 +27,12 @@ def process(self, instance): """Plugin entry point.""" super(CreateMayaRoyalRenderJob, self).process(instance) - expected_files = self._instance.data["expectedFiles"] + expected_files = instance.data["expectedFiles"] first_file_path = next(iter_expected_files(expected_files)) output_dir = os.path.dirname(first_file_path) - self._instance.data["outputDir"] = output_dir + instance.data["outputDir"] = output_dir - layer = self._instance.data["setMembers"] # type: str + layer = instance.data["setMembers"] # type: str layer_name = layer.removeprefix("rs_") job = self.get_job(instance, self.scene_path, first_file_path, diff --git a/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py index 7f503c80a9d..6db0fe959ee 100644 --- a/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py @@ -15,22 +15,22 @@ def process(self, instance): super(CreateNukeRoyalRenderJob, self).process(instance) # redefinition of families - if "render" in self._instance.data["family"]: - self._instance.data["family"] = "write" - self._instance.data["families"].insert(0, "render2d") - elif "prerender" in self._instance.data["family"]: - self._instance.data["family"] = "write" - self._instance.data["families"].insert(0, "prerender") + if "render" in instance.data["family"]: + instance.data["family"] = "write" + instance.data["families"].insert(0, "render2d") + elif "prerender" in instance.data["family"]: + instance.data["family"] = "write" + instance.data["families"].insert(0, "prerender") - jobs = self.create_jobs(self._instance) + jobs = self.create_jobs(instance) for job in jobs: job = self.update_job_with_host_specific(instance, job) - instance.data["rrJobs"].append(job) + instance.data["rrJobs"].append(job) def update_job_with_host_specific(self, instance, job): nuke_version = re.search( - r"\d+\.\d+", self._instance.context.data.get("hostVersion")) + r"\d+\.\d+", instance.context.data.get("hostVersion")) job.Software = "Nuke" job.Version = nuke_version.group() @@ -42,7 +42,7 @@ def create_jobs(self, instance): # get output path render_path = instance.data['path'] script_path = self.scene_path - node = self._instance.data["transientData"]["node"] + node = instance.data["transientData"]["node"] # main job jobs = [ @@ -54,7 +54,7 @@ def create_jobs(self, instance): ) ] - for baking_script in self._instance.data.get("bakingNukeScripts", []): + for baking_script in instance.data.get("bakingNukeScripts", []): render_path = baking_script["bakeRenderPath"] script_path = baking_script["bakeScriptPath"] exe_node_name = baking_script["bakeWriteNodeName"] From 8754345b44ebb7747a96bcf6386b702596cf9429 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 4 Jul 2023 17:35:42 +0200 Subject: [PATCH 097/104] Fix - Nuke needs to have multiple jobs got accidentally reverted It needs to loop through jobs to add them all. --- .../royalrender/plugins/publish/create_nuke_royalrender_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py index 6db0fe959ee..71daa6edf80 100644 --- a/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py @@ -26,7 +26,7 @@ def process(self, instance): for job in jobs: job = self.update_job_with_host_specific(instance, job) - instance.data["rrJobs"].append(job) + instance.data["rrJobs"].append(job) def update_job_with_host_specific(self, instance, job): nuke_version = re.search( From 708819f8b13980ebacea6468f15e6b4e2cf9051c Mon Sep 17 00:00:00 2001 From: Pype Club Date: Wed, 5 Jul 2023 16:50:58 +0200 Subject: [PATCH 098/104] Refactor - renamed replace_published_scene Former name pointed to replacing of whole file. This function is only about using path to published workfile instead of work workfile. --- openpype/modules/deadline/abstract_submit_deadline.py | 5 +++-- openpype/pipeline/publish/lib.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 6ff9afbc427..d9d250fe9e2 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -23,7 +23,7 @@ OpenPypePyblishPluginMixin ) from openpype.pipeline.publish.lib import ( - replace_published_scene + replace_with_published_scene_path ) JSONDecodeError = getattr(json.decoder, "JSONDecodeError", ValueError) @@ -528,7 +528,8 @@ def from_published_scene(self, replace_in_path=True): published. """ - return replace_published_scene(self._instance, replace_in_path=True) + return replace_with_published_scene_path( + self._instance, replace_in_path=replace_in_path) def assemble_payload( self, job_info=None, plugin_info=None, aux_files=None): diff --git a/openpype/pipeline/publish/lib.py b/openpype/pipeline/publish/lib.py index a03303bc394..5d2a88fa3b7 100644 --- a/openpype/pipeline/publish/lib.py +++ b/openpype/pipeline/publish/lib.py @@ -882,10 +882,11 @@ def get_published_workfile_instance(context): return i -def replace_published_scene(instance, replace_in_path=True): - """Switch work scene for published scene. +def replace_with_published_scene_path(instance, replace_in_path=True): + """Switch work scene path for published scene. If rendering/exporting from published scenes is enabled, this will replace paths from working scene to published scene. + This only works if publish contains workfile instance! Args: instance (pyblish.api.Instance): Pyblish instance. replace_in_path (bool): if True, it will try to find From 26cd4f0029473aa361bfe0b46139a1e0b0f30109 Mon Sep 17 00:00:00 2001 From: Pype Club Date: Wed, 5 Jul 2023 16:51:51 +0200 Subject: [PATCH 099/104] Hound --- openpype/pipeline/farm/pyblish_functions.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index 0ace02edb90..afd2ef3eeb7 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -822,8 +822,6 @@ def attach_instances_to_subset(attach_to, instances): list: List of attached instances. """ - # - new_instances = [] for attach_instance in attach_to: for i in instances: From 892758b4d02eaf80ea9cc0918a7920d31e0f4961 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 12 Jul 2023 17:05:24 +0200 Subject: [PATCH 100/104] Added missed colorspace injection for image sequences Develop contains this information, was missed during shuffling things around. --- .../plugins/publish/submit_publish_job.py | 7 ++++--- .../publish/create_publish_royalrender_job.py | 7 +++++-- openpype/pipeline/farm/pyblish_functions.py | 15 ++++++++++++--- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index ea9c296732d..1bc7ea0161c 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -15,7 +15,7 @@ from openpype.pipeline import ( legacy_io, ) -from openpype.pipeline.publish import OpenPypePyblishPluginMixin +from openpype.pipeline import publish from openpype.lib import EnumDef from openpype.tests.lib import is_in_tests from openpype.lib import is_running_from_build @@ -57,7 +57,7 @@ def get_resource_files(resources, frame_range=None): class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin, - OpenPypePyblishPluginMixin): + publish.ColormanagedPyblishPluginMixin): """Process Job submitted on farm. These jobs are dependent on a deadline or muster job @@ -396,7 +396,8 @@ def process(self, instance): anatomy, self.aov_filter, self.skip_integration_repre_list, - do_not_add_review + do_not_add_review, + self ) if "representations" not in instance_skeleton_data.keys(): diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 6eb8f2649e3..5f75bbc3f1a 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -23,9 +23,11 @@ prepare_representations, create_metadata_path ) +from openpype.pipeline import publish -class CreatePublishRoyalRenderJob(pyblish.api.InstancePlugin): +class CreatePublishRoyalRenderJob(pyblish.api.InstancePlugin, + publish.ColormanagedPyblishPluginMixin): """Creates job which publishes rendered files to publish area. Job waits until all rendering jobs are finished, triggers `publish` command @@ -107,7 +109,8 @@ def process(self, instance): self.anatomy, self.aov_filter, self.skip_integration_repre_list, - do_not_add_review + do_not_add_review, + self ) if "representations" not in instance_skeleton_data.keys(): diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index 0ace02edb90..3834524faaf 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -243,7 +243,8 @@ def create_skeleton_instance( "jobBatchName": data.get("jobBatchName", ""), "useSequenceForReview": data.get("useSequenceForReview", True), # map inputVersions `ObjectId` -> `str` so json supports it - "inputVersions": list(map(str, data.get("inputVersions", []))) + "inputVersions": list(map(str, data.get("inputVersions", []))), + "colorspace": data.get("colorspace") } # skip locking version if we are creating v01 @@ -291,7 +292,8 @@ def _add_review_families(families): def prepare_representations(instance, exp_files, anatomy, aov_filter, skip_integration_repre_list, - do_not_add_review): + do_not_add_review, + color_managed_plugin): """Create representations for file sequences. This will return representations of expected files if they are not @@ -306,7 +308,7 @@ def prepare_representations(instance, exp_files, anatomy, aov_filter, aov_filter (dict): add review for specific aov names skip_integration_repre_list (list): exclude specific extensions, do_not_add_review (bool): explicitly skip review - + color_managed_plugin (publish.ColormanagedPyblishPluginMixin) Returns: list of representations @@ -433,6 +435,13 @@ def prepare_representations(instance, exp_files, anatomy, aov_filter, if not already_there: representations.append(rep) + for rep in representations: + # inject colorspace data + color_managed_plugin.set_representation_colorspace( + rep, instance.context, + colorspace=instance.data["colorspace"] + ) + return representations From 48f06ba414f33f028d7cfbab6eef4fdf261137df Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 12 Jul 2023 18:01:50 +0200 Subject: [PATCH 101/104] Fix - removed wrong argument Probably incorrectly merged from develop where it is already fixed. --- openpype/modules/deadline/abstract_submit_deadline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index d9d250fe9e2..3fa427204b3 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -433,7 +433,7 @@ def process(self, instance): file_path = None if self.use_published: if not self.import_reference: - file_path = self.from_published_scene(context) + file_path = self.from_published_scene() else: self.log.info("use the scene with imported reference for rendering") # noqa file_path = context.data["currentFile"] From abc6a1eb27d0e574c9f6d934d25f67ad7be681db Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 12 Jul 2023 18:06:17 +0200 Subject: [PATCH 102/104] Fix - context must be passe to set_representation_colorspace --- .../plugins/publish/submit_publish_job.py | 2 ++ .../publish/create_publish_royalrender_job.py | 1 + openpype/pipeline/farm/pyblish_functions.py | 35 ++++++++++--------- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 9e3ad20b44d..457ebfd0fef 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -57,6 +57,7 @@ def get_resource_files(resources, frame_range=None): class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin, + publish.OpenPypePyblishPluginMixin, publish.ColormanagedPyblishPluginMixin): """Process Job submitted on farm. @@ -396,6 +397,7 @@ def process(self, instance): self.aov_filter, self.skip_integration_repre_list, do_not_add_review, + instance.context, self ) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 5f75bbc3f1a..3eb49a39ee4 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -110,6 +110,7 @@ def process(self, instance): self.aov_filter, self.skip_integration_repre_list, do_not_add_review, + instance.context, self ) diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index 76ee5832d62..2df8269d792 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -290,9 +290,10 @@ def _add_review_families(families): return families -def prepare_representations(instance, exp_files, anatomy, aov_filter, +def prepare_representations(skeleton_data, exp_files, anatomy, aov_filter, skip_integration_repre_list, do_not_add_review, + context, color_managed_plugin): """Create representations for file sequences. @@ -301,7 +302,7 @@ def prepare_representations(instance, exp_files, anatomy, aov_filter, most cases, but if not - we create representation from each of them. Arguments: - instance (dict): instance data for which we are + skeleton_data (dict): instance data for which we are setting representations exp_files (list): list of expected files anatomy (Anatomy): @@ -328,9 +329,9 @@ def prepare_representations(instance, exp_files, anatomy, aov_filter, # expected files contains more explicitly and from what # should be review made. # - "review" tag is never added when is set to 'False' - if instance["useSequenceForReview"]: + if skeleton_data["useSequenceForReview"]: # toggle preview on if multipart is on - if instance.get("multipartExr", False): + if skeleton_data.get("multipartExr", False): log.debug( "Adding preview tag because its multipartExr" ) @@ -355,8 +356,8 @@ def prepare_representations(instance, exp_files, anatomy, aov_filter, " This may cause issues on farm." ).format(staging)) - frame_start = int(instance.get("frameStartHandle")) - if instance.get("slate"): + frame_start = int(skeleton_data.get("frameStartHandle")) + if skeleton_data.get("slate"): frame_start -= 1 # explicitly disable review by user @@ -366,10 +367,10 @@ def prepare_representations(instance, exp_files, anatomy, aov_filter, "ext": ext, "files": [os.path.basename(f) for f in list(collection)], "frameStart": frame_start, - "frameEnd": int(instance.get("frameEndHandle")), + "frameEnd": int(skeleton_data.get("frameEndHandle")), # If expectedFile are absolute, we need only filenames "stagingDir": staging, - "fps": instance.get("fps"), + "fps": skeleton_data.get("fps"), "tags": ["review"] if preview else [], } @@ -377,18 +378,19 @@ def prepare_representations(instance, exp_files, anatomy, aov_filter, if ext in skip_integration_repre_list: rep["tags"].append("delete") - if instance.get("multipartExr", False): + if skeleton_data.get("multipartExr", False): rep["tags"].append("multipartExr") # support conversion from tiled to scanline - if instance.get("convertToScanline"): + if skeleton_data.get("convertToScanline"): log.info("Adding scanline conversion.") rep["tags"].append("toScanline") representations.append(rep) if preview: - instance["families"] = _add_review_families(instance["families"]) + skeleton_data["families"] = _add_review_families( + skeleton_data["families"]) # add remainders as representations for remainder in remainders: @@ -419,13 +421,14 @@ def prepare_representations(instance, exp_files, anatomy, aov_filter, preview = preview and not do_not_add_review if preview: rep.update({ - "fps": instance.get("fps"), + "fps": skeleton_data.get("fps"), "tags": ["review"] }) - instance["families"] = _add_review_families(instance["families"]) + skeleton_data["families"] = \ + _add_review_families(skeleton_data["families"]) already_there = False - for repre in instance.get("representations", []): + for repre in skeleton_data.get("representations", []): # might be added explicitly before by publish_on_farm already_there = repre.get("files") == rep["files"] if already_there: @@ -438,8 +441,8 @@ def prepare_representations(instance, exp_files, anatomy, aov_filter, for rep in representations: # inject colorspace data color_managed_plugin.set_representation_colorspace( - rep, instance.context, - colorspace=instance.data["colorspace"] + rep, context, + colorspace=skeleton_data["colorspace"] ) return representations From 2be5b33510a869d560bb95e0fb3e33ed93400f03 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 17 Jul 2023 13:14:18 +0200 Subject: [PATCH 103/104] Fix wrong merge --- .../plugins/publish/validate_primitive_hierarchy_paths.py | 8 -------- openpype/hosts/max/api/plugin.py | 2 -- 2 files changed, 10 deletions(-) diff --git a/openpype/hosts/houdini/plugins/publish/validate_primitive_hierarchy_paths.py b/openpype/hosts/houdini/plugins/publish/validate_primitive_hierarchy_paths.py index 3da5665f589..0d84aa7db81 100644 --- a/openpype/hosts/houdini/plugins/publish/validate_primitive_hierarchy_paths.py +++ b/openpype/hosts/houdini/plugins/publish/validate_primitive_hierarchy_paths.py @@ -70,14 +70,6 @@ def get_invalid(cls, instance): cls.log.debug("Checking for attribute: %s", path_attr) - if not hasattr(output_node, "geometry"): - # In the case someone has explicitly set an Object - # node instead of a SOP node in Geometry context - # then for now we ignore - this allows us to also - # export object transforms. - cls.log.warning("No geometry output node found, skipping check..") - return - if not hasattr(output_node, "geometry"): # In the case someone has explicitly set an Object # node instead of a SOP node in Geometry context diff --git a/openpype/hosts/max/api/plugin.py b/openpype/hosts/max/api/plugin.py index 36b4ea32d48..d8db716e6d5 100644 --- a/openpype/hosts/max/api/plugin.py +++ b/openpype/hosts/max/api/plugin.py @@ -78,9 +78,7 @@ if idx do ( continue ) - name = c as string - append temp_arr handle_name append i_node_arr node_ref append sel_list name From b90d1c9e653cf9badfbce7de7f5442f55cb6eeef Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 17 Jul 2023 13:22:04 +0200 Subject: [PATCH 104/104] Fix wrong merge --- openpype/hosts/maya/api/fbx.py | 4 ++-- openpype/settings/defaults/project_settings/shotgrid.json | 2 +- poetry.lock | 2 +- pyproject.toml | 1 - 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/maya/api/fbx.py b/openpype/hosts/maya/api/fbx.py index e9cf8af491b..260241f5fc3 100644 --- a/openpype/hosts/maya/api/fbx.py +++ b/openpype/hosts/maya/api/fbx.py @@ -2,7 +2,7 @@ """Tools to work with FBX.""" import logging -import pyblish.api +from pyblish.api import Instance from maya import cmds # noqa import maya.mel as mel # noqa @@ -141,7 +141,7 @@ def parse_overrides(self, instance, options): return options def set_options_from_instance(self, instance): - # type: (pyblish.api.Instance) -> None + # type: (Instance) -> None """Sets FBX export options from data in the instance. Args: diff --git a/openpype/settings/defaults/project_settings/shotgrid.json b/openpype/settings/defaults/project_settings/shotgrid.json index 0dcffed28a2..83b6f690741 100644 --- a/openpype/settings/defaults/project_settings/shotgrid.json +++ b/openpype/settings/defaults/project_settings/shotgrid.json @@ -1,6 +1,6 @@ { "shotgrid_project_id": 0, - "shotgrid_server": [], + "shotgrid_server": "", "event": { "enabled": false }, diff --git a/poetry.lock b/poetry.lock index 50f81506381..5621d39988a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "acre" diff --git a/pyproject.toml b/pyproject.toml index 5b582573103..fe5477fb003 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,7 +100,6 @@ wheel = "*" enlighten = "*" # cool terminal progress bars toml = "^0.10.2" # for parsing pyproject.toml pre-commit = "*" -mypy = "*" # for better types [tool.poetry.urls] "Bug Tracker" = "https://github.com/pypeclub/openpype/issues"