diff --git a/openpype/lib/__init__.py b/openpype/lib/__init__.py index b3b12ac2507..f964b208ee4 100644 --- a/openpype/lib/__init__.py +++ b/openpype/lib/__init__.py @@ -149,7 +149,8 @@ from .path_tools import ( format_file_size, collect_frames, - create_hard_link, + create_hardlink, + create_symlink, version_up, get_version_from_path, get_last_version_from_path, @@ -265,7 +266,8 @@ "format_file_size", "collect_frames", - "create_hard_link", + "create_hardlink", + "create_symlink", "version_up", "get_version_from_path", "get_last_version_from_path", diff --git a/openpype/lib/file_transaction.py b/openpype/lib/file_transaction.py index 80f4e81f2c3..e350c2d2fc6 100644 --- a/openpype/lib/file_transaction.py +++ b/openpype/lib/file_transaction.py @@ -4,7 +4,7 @@ import errno import six -from openpype.lib import create_hard_link +from openpype.lib import create_hardlink, create_symlink # this is needed until speedcopy for linux is fixed if sys.platform == "win32": @@ -53,6 +53,7 @@ class FileTransaction(object): MODE_COPY = 0 MODE_HARDLINK = 1 + MODE_SYMLINK = 2 def __init__(self, log=None, allow_queue_replacements=False): if log is None: @@ -78,7 +79,7 @@ def add(self, src, dst, mode=MODE_COPY): Args: src (str): Source path. dst (str): Destination path. - mode (MODE_COPY, MODE_HARDLINK): Transfer mode. + mode (MODE_COPY, MODE_HARDLINK, MODE_SYMLINK): Transfer mode. """ opts = {"mode": mode} @@ -142,7 +143,11 @@ def process(self): elif opts["mode"] == self.MODE_HARDLINK: self.log.debug("Hardlinking file ... {} -> {}".format( src, dst)) - create_hard_link(src, dst) + create_hardlink(src, dst) + elif opts["mode"] == self.MODE_SYMLINK: + self.log.debug("Symlinking file ... {} -> {}".format( + src, dst)) + create_symlink(src, dst) self._transferred.append(dst) diff --git a/openpype/lib/path_tools.py b/openpype/lib/path_tools.py index fec6a0c47dc..a425a8b7cd3 100644 --- a/openpype/lib/path_tools.py +++ b/openpype/lib/path_tools.py @@ -29,7 +29,7 @@ def format_file_size(file_size, suffix=None): return "%.1f%s%s" % (file_size, "Yi", suffix) -def create_hard_link(src_path, dst_path): +def create_hardlink(src_path, dst_path): """Create hardlink of file. Args: @@ -65,6 +65,41 @@ def create_hard_link(src_path, dst_path): ) +def create_symlink(src_path, dst_path): + """Create symlink of file. + Args: + src_path(str): Full path to a file which is used as source for + symlink. + dst_path(str): Full path to a file where a link of source will be + added. + """ + # Use `os.symlink` if is available + # - should be for all platforms with newer python versions + if hasattr(os, "symlink"): + os.symlink(src_path, dst_path) + return + + # Windows implementation of symlinks ( + # - for older versions of python + if platform.system().lower() == "windows": + import ctypes + from ctypes.wintypes import BOOL + CreateSymLink = ctypes.windll.kernel32.CreateSymbolicLinkW + CreateSymLink.argtypes = [ + ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p + ] + CreateSymLink.restype = BOOL + + res = CreateSymLink(dst_path, src_path, None) + if res == 0: + raise ctypes.WinError() + return + # Raises not implemented error if gets here + raise NotImplementedError( + "Implementation of symlink for current environment is missing." + ) + + def collect_frames(files): """Returns dict of source path and its frame, if from sequence diff --git a/openpype/pipeline/delivery.py b/openpype/pipeline/delivery.py index bbd01f7a4ee..045f527e414 100644 --- a/openpype/pipeline/delivery.py +++ b/openpype/pipeline/delivery.py @@ -6,7 +6,7 @@ import clique import collections -from openpype.lib import create_hard_link +from openpype.lib import create_hardlink def _copy_file(src_path, dst_path): @@ -19,7 +19,7 @@ def _copy_file(src_path, dst_path): if os.path.exists(dst_path): return try: - create_hard_link( + create_hardlink( src_path, dst_path ) diff --git a/openpype/plugins/publish/integrate.py b/openpype/plugins/publish/integrate.py index 3735cae5f2b..59b6a90b064 100644 --- a/openpype/plugins/publish/integrate.py +++ b/openpype/plugins/publish/integrate.py @@ -1,4 +1,5 @@ import os +import re import logging import sys import copy @@ -281,8 +282,8 @@ def register(self, instance, file_transactions, filtered_repres): instance) for src, dst in prepared["transfers"]: - # todo: add support for hardlink transfers - file_transactions.add(src, dst) + file_transaction_mode = self.get_file_transaction_mode(instance, src) + file_transactions.add(src, dst, mode=file_transaction_mode) prepared_representations.append(prepared) @@ -294,7 +295,8 @@ def register(self, instance, file_transactions, filtered_repres): file_copy_modes = [ ("transfers", FileTransaction.MODE_COPY), - ("hardlinks", FileTransaction.MODE_HARDLINK) + ("hardlinks", FileTransaction.MODE_HARDLINK), + ("symlinks", FileTransaction.MODE_SYMLINK) ] for files_type, copy_mode in file_copy_modes: for src, dst in instance.data.get(files_type, []): @@ -404,6 +406,26 @@ def register(self, instance, file_transactions, filtered_repres): ) ) + @staticmethod + def get_file_transaction_mode(instance, src): + transaction_mode = FileTransaction.MODE_COPY + + is_symlink_requested = False + hierarchy_data = instance.data.get("hierarchyData") + if hierarchy_data and hierarchy_data.get("symlink") == "True": + is_symlink_requested = True + + if not is_symlink_requested: + return transaction_mode + + pattern = instance.context.data["project_settings"]["global"]["tools"]["publish"]["symlink"][ + "file_regex_pattern"] + + if not pattern or bool(re.match(pattern, src)): + transaction_mode = FileTransaction.MODE_SYMLINK + + return transaction_mode + def prepare_subset(self, instance, op_session, project_name): asset_doc = instance.data["assetEntity"] subset_name = instance.data["subset"] diff --git a/openpype/plugins/publish/integrate_hero_version.py b/openpype/plugins/publish/integrate_hero_version.py index 59dc6b5c641..545484ed094 100644 --- a/openpype/plugins/publish/integrate_hero_version.py +++ b/openpype/plugins/publish/integrate_hero_version.py @@ -19,7 +19,7 @@ prepare_hero_version_update_data, prepare_representation_update_data, ) -from openpype.lib import create_hard_link +from openpype.lib import create_hardlink from openpype.pipeline import ( schema ) @@ -608,7 +608,7 @@ def copy_file(self, src_path, dst_path): # First try hardlink and copy if paths are cross drive try: - create_hard_link(src_path, dst_path) + create_hardlink(src_path, dst_path) # Return when successful return diff --git a/openpype/settings/defaults/project_settings/global.json b/openpype/settings/defaults/project_settings/global.json index 782fff1052f..aa729a54502 100644 --- a/openpype/settings/defaults/project_settings/global.json +++ b/openpype/settings/defaults/project_settings/global.json @@ -556,7 +556,10 @@ } ], "hero_template_name_profiles": [], - "custom_staging_dir_profiles": [] + "custom_staging_dir_profiles": [], + "symlink": { + "file_regex_pattern": "" + } } }, "project_folder_structure": "{\"__project_root__\": {\"prod\": {}, \"resources\": {\"footage\": {\"plates\": {}, \"offline\": {}}, \"audio\": {}, \"art_dept\": {}}, \"editorial\": {}, \"assets\": {\"characters\": {}, \"locations\": {}}, \"shots\": {}}}", diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_tools.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_tools.json index 23fc7c9351b..ea886662b4c 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_tools.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_tools.json @@ -469,6 +469,20 @@ } ] } + }, + { + "type": "dict", + "collapsible": true, + "key": "symlink", + "label": "Symlink", + "is_group": true, + "children": [ + { + "type": "text", + "key": "file_regex_pattern", + "label": "File Regex Pattern" + } + ] } ] }