[QUAD] File transfer / Copy: Add ability to create symlink

openpype/lib/__init__.py

@@ -149,7 +149,8 @@
 from .path_tools import (
     format_file_size,
     collect_frames,
-    create_hard_link,
+    create_hardlink,
+    create_symlink,
     version_up,
     get_version_from_path,
     get_last_version_from_path,
@@ -265,7 +266,8 @@
 
     "format_file_size",
     "collect_frames",
-    "create_hard_link",
+    "create_hardlink",
+    "create_symlink",
     "version_up",
     "get_version_from_path",
     "get_last_version_from_path",

openpype/lib/file_transaction.py

@@ -4,7 +4,7 @@
 import errno
 import six
 
-from openpype.lib import create_hard_link
+from openpype.lib import create_hardlink, create_symlink
 
 # this is needed until speedcopy for linux is fixed
 if sys.platform == "win32":
@@ -53,6 +53,7 @@ class FileTransaction(object):
 
     MODE_COPY = 0
     MODE_HARDLINK = 1
+    MODE_SYMLINK = 2
 
     def __init__(self, log=None, allow_queue_replacements=False):
         if log is None:
@@ -78,7 +79,7 @@ def add(self, src, dst, mode=MODE_COPY):
         Args:
             src (str): Source path.
             dst (str): Destination path.
-            mode (MODE_COPY, MODE_HARDLINK): Transfer mode.
+            mode (MODE_COPY, MODE_HARDLINK, MODE_SYMLINK): Transfer mode.
         """
 
         opts = {"mode": mode}
@@ -142,7 +143,11 @@ def process(self):
             elif opts["mode"] == self.MODE_HARDLINK:
                 self.log.debug("Hardlinking file ... {} -> {}".format(
                     src, dst))
-                create_hard_link(src, dst)
+                create_hardlink(src, dst)
+            elif opts["mode"] == self.MODE_SYMLINK:
+                self.log.debug("Symlinking file ... {} -> {}".format(
+                    src, dst))
+                create_symlink(src, dst)
 
             self._transferred.append(dst)
 

openpype/lib/path_tools.py

@@ -29,7 +29,7 @@ def format_file_size(file_size, suffix=None):
     return "%.1f%s%s" % (file_size, "Yi", suffix)
 
 
-def create_hard_link(src_path, dst_path):
+def create_hardlink(src_path, dst_path):
     """Create hardlink of file.
 
     Args:
@@ -65,6 +65,41 @@ def create_hard_link(src_path, dst_path):
     )
 
 
+def create_symlink(src_path, dst_path):
+    """Create symlink of file.
+    Args:
+        src_path(str): Full path to a file which is used as source for
+            symlink.
+        dst_path(str): Full path to a file where a link of source will be
+            added.
+    """
+    # Use `os.symlink` if is available
+    #   - should be for all platforms with newer python versions
+    if hasattr(os, "symlink"):
+        os.symlink(src_path, dst_path)
+        return
+
+    # Windows implementation of symlinks (
+    #  - for older versions of python
+    if platform.system().lower() == "windows":
+        import ctypes
+        from ctypes.wintypes import BOOL
+        CreateSymLink = ctypes.windll.kernel32.CreateSymbolicLinkW
+        CreateSymLink.argtypes = [
+            ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p
+        ]
+        CreateSymLink.restype = BOOL
+
+        res = CreateSymLink(dst_path, src_path, None)
+        if res == 0:
+            raise ctypes.WinError()
+        return
+    # Raises not implemented error if gets here
+    raise NotImplementedError(
+        "Implementation of symlink for current environment is missing."
+    )
+
+
 def collect_frames(files):
     """Returns dict of source path and its frame, if from sequence
 

openpype/pipeline/delivery.py

@@ -6,7 +6,7 @@
 import clique
 import collections
 
-from openpype.lib import create_hard_link
+from openpype.lib import create_hardlink
 
 
 def _copy_file(src_path, dst_path):
@@ -19,7 +19,7 @@ def _copy_file(src_path, dst_path):
     if os.path.exists(dst_path):
         return
     try:
-        create_hard_link(
+        create_hardlink(
             src_path,
             dst_path
         )

openpype/plugins/publish/integrate.py

@@ -1,4 +1,5 @@
 import os
+import re
 import logging
 import sys
 import copy
@@ -281,8 +282,8 @@ def register(self, instance, file_transactions, filtered_repres):
                 instance)
 
             for src, dst in prepared["transfers"]:
-                # todo: add support for hardlink transfers
-                file_transactions.add(src, dst)
+                file_transaction_mode = self.get_file_transaction_mode(instance, src)
+                file_transactions.add(src, dst, mode=file_transaction_mode)
 
             prepared_representations.append(prepared)
 
@@ -294,7 +295,8 @@ def register(self, instance, file_transactions, filtered_repres):
 
         file_copy_modes = [
             ("transfers", FileTransaction.MODE_COPY),
-            ("hardlinks", FileTransaction.MODE_HARDLINK)
+            ("hardlinks", FileTransaction.MODE_HARDLINK),
+            ("symlinks", FileTransaction.MODE_SYMLINK)
         ]
         for files_type, copy_mode in file_copy_modes:
             for src, dst in instance.data.get(files_type, []):
@@ -404,6 +406,26 @@ def register(self, instance, file_transactions, filtered_repres):
             )
         )
 
+    @staticmethod
+    def get_file_transaction_mode(instance, src):
+        transaction_mode = FileTransaction.MODE_COPY
+
+        is_symlink_requested = False
+        hierarchy_data = instance.data.get("hierarchyData")
+        if hierarchy_data and hierarchy_data.get("symlink") == "True":
+            is_symlink_requested = True
+
+        if not is_symlink_requested:
+            return transaction_mode
+
+        pattern = instance.context.data["project_settings"]["global"]["tools"]["publish"]["symlink"][
+            "file_regex_pattern"]
+
+        if not pattern or bool(re.match(pattern, src)):
+            transaction_mode = FileTransaction.MODE_SYMLINK
+
+        return transaction_mode
+
     def prepare_subset(self, instance, op_session, project_name):
         asset_doc = instance.data["assetEntity"]
         subset_name = instance.data["subset"]

openpype/plugins/publish/integrate_hero_version.py

@@ -19,7 +19,7 @@
     prepare_hero_version_update_data,
     prepare_representation_update_data,
 )
-from openpype.lib import create_hard_link
+from openpype.lib import create_hardlink
 from openpype.pipeline import (
     schema
 )
@@ -608,7 +608,7 @@ def copy_file(self, src_path, dst_path):
 
         # First try hardlink and copy if paths are cross drive
         try:
-            create_hard_link(src_path, dst_path)
+            create_hardlink(src_path, dst_path)
             # Return when successful
             return
 

openpype/settings/defaults/project_settings/global.json

@@ -556,7 +556,10 @@
                 }
             ],
             "hero_template_name_profiles": [],
-            "custom_staging_dir_profiles": []
+            "custom_staging_dir_profiles": [],
+            "symlink": {
+                "file_regex_pattern": ""
+            }
         }
     },
     "project_folder_structure": "{\"__project_root__\": {\"prod\": {}, \"resources\": {\"footage\": {\"plates\": {}, \"offline\": {}}, \"audio\": {}, \"art_dept\": {}}, \"editorial\": {}, \"assets\": {\"characters\": {}, \"locations\": {}}, \"shots\": {}}}",

openpype/settings/entities/schemas/projects_schema/schemas/schema_global_tools.json

@@ -469,6 +469,20 @@
                             }
                         ]
                     }
+                },
+                {
+                    "type": "dict",
+                    "collapsible": true,
+                    "key": "symlink",
+                    "label": "Symlink",
+                    "is_group": true,
+                    "children": [
+                        {
+                            "type": "text",
+                            "key": "file_regex_pattern",
+                            "label": "File Regex Pattern"
+                        }
+                    ]
                 }
             ]
         }