From f24398fe11156363d47b0eb0b9ec2de5725de48b Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 13 Mar 2018 12:28:26 +0100 Subject: [PATCH 01/81] Draft implementation for a "launch" script using the env_prototype --- colorbleed/environments/global.json | 9 ++++ colorbleed/environments/maya2018.json | 58 +++++++++++++++++++++++++ colorbleed/environments/yeti.2.2.4.json | 28 ++++++++++++ colorbleed/scripts/launch.py | 57 ++++++++++++++++++++++++ 4 files changed, 152 insertions(+) create mode 100644 colorbleed/environments/global.json create mode 100644 colorbleed/environments/maya2018.json create mode 100644 colorbleed/environments/yeti.2.2.4.json create mode 100644 colorbleed/scripts/launch.py diff --git a/colorbleed/environments/global.json b/colorbleed/environments/global.json new file mode 100644 index 00000000..ea519691 --- /dev/null +++ b/colorbleed/environments/global.json @@ -0,0 +1,9 @@ +{ + "CB_PIPELINE": "P:/pipeline", + + "PYTHONPATH": [ + "{CB_PIPELINE}/dev/git/Qt.py", + "{CB_PIPELINE}/dev/git/cb", + "{PYTHONPATH}" + ] +} diff --git a/colorbleed/environments/maya2018.json b/colorbleed/environments/maya2018.json new file mode 100644 index 00000000..65fba760 --- /dev/null +++ b/colorbleed/environments/maya2018.json @@ -0,0 +1,58 @@ +{ + "MAYA_VERSION": "2018", + "MAYA_LOCATION": { + "darwin": "/Applications/Autodesk/maya${MAYA_VERSION}/Maya.app/Contents", + "linux": "/usr/autodesk/maya${MAYA_VERSION}", + "windows": "C:/Program Files/Autodesk/Maya${MAYA_VERSION}" + }, + "PATH": { + "darwin": "${MAYA_LOCATION}/bin;{PATH}", + "linux": "${MAYA_LOCATION}/bin;{PATH}", + "windows": [ + "${MAYA_LOCATION}/bin", + "C:/Program Files/Common Files/Autodesk Shared/", + "C:/Program Files (x86)/Autodesk/Backburner/", + "{PATH}" + ] + }, + "DYLD_LIBRARY_PATH": { + "darwin": "${MAYA_LOCATION}/MacOS" + }, + + "MAYA_DISABLE_CLIC_IPM": "Yes", + "MAYA_DISABLE_CIP": "Yes", + "MAYA_DISABLE_CER": "Yes", + "PYMEL_SKIP_MEL_INIT": "Yes", + "LC_ALL": "C", + + "CB_MAYA_SHARED": "{CB_PIPELINE}/dev/apps/maya_shared/{MAYA_VERSION}", + "MAYA_SCRIPT_PATH": [ + "{CB_MAYA_SHARED}/scripts", + "{MAYA_SCRIPT_PATH}" + ], + "MAYA_MODULE_PATH": [ + "{CB_MAYA_SHARED}/modules/cvshapeinverter", + "{MAYA_MODULE_PATH}" + ], + "XBMLANGPATH": [ + "{CB_MAYA_SHARED}/prefs/icons", + "{XBMLANGPATH}" + ], + + "PYTHONPATH": [ + "{CB_PIPELINE}/2.1_dev/git/scriptsmenu/python", + "{CB_PIPELINE}/2.1_dev/git/maya-look-assigner", + "{CB_PIPELINE}/2.1_dev/git/maya_deadline_submission_settings/python", + "{CB_PIPELINE}/dev/git/maya-capture", + "{CB_PIPELINE}/dev/git/maya-capture-gui", + "{CB_PIPELINE}/dev/git/maya-capture-gui-cb", + "{CB_PIPELINE}/dev/apps/maya_shared/2018/scripts", + "{AVALON_CORE}/setup/maya", + "{PYTHONPATH}" + ], + + "FFMPEG_PATH": "{CB_PIPELINE}/dev/apps/ffmpeg/bin/ffmpeg.exe", + "COLORBLEED_SCRIPTS": "{CB_PIPELINE}/dev/git/cbMayaScripts/cbMayaScripts", + + "VRAY_USE_THREAD_AFFINITY": "0" +} \ No newline at end of file diff --git a/colorbleed/environments/yeti.2.2.4.json b/colorbleed/environments/yeti.2.2.4.json new file mode 100644 index 00000000..0f3c4ba4 --- /dev/null +++ b/colorbleed/environments/yeti.2.2.4.json @@ -0,0 +1,28 @@ +{ + "PEREGRINEL_LICENSE": "5053@CBserver", + + "YETI_ROOT": "{CB_MAYA_SHARED}/modules/Yeti-v2.2.4_Maya2018-windows64", + "MAYA_MODULE_PATH": "{MAYA_MODULE_PATH};{YETI_ROOT}", + + "PATH": [ + "{YETI_ROOT}/bin", + "{PATH}" + ], + + "VRAY_PLUGINS_X64": [ + "{YETI_ROOT}/bin", + "{VRAY_PLUGINS_X64}" + ], + "VRAY_FOR_MAYA2018_PLUGINS_X64": [ + "{YETI_ROOT}/bin", + "{VRAY_FOR_MAYA2018_PLUGINS_X64}" + ], + + "REDSHIFT_MAYAEXTENSIONSPATH": [ + "{YETI_ROOT}/plug-ins", + "{REDSHIFT_MAYAEXTENSIONSPATH}" + ], + + "MTOA_EXTENSIONS_PATH": "{YETI_ROOT}/plug-ins;{MTOA_EXTENSIONS_PATH}", + "ARNOLD_PLUGIN_PATH": "{YETI_ROOT}/bin;{ARNOLD_PLUGIN_PATH}" +} \ No newline at end of file diff --git a/colorbleed/scripts/launch.py b/colorbleed/scripts/launch.py new file mode 100644 index 00000000..a9c3f5a2 --- /dev/null +++ b/colorbleed/scripts/launch.py @@ -0,0 +1,57 @@ +import os +import pprint + +import avalon.lib as lib + + +# TODO: Remove redundant hack +import sys +sys.path.append(r"P:\pipeline\dev\git\env_prototype") +import env_prototype.api as api + + +config_root = os.path.dirname(os.path.dirname(__file__)) +os.environ["TOOL_ENV"] = os.path.join(config_root, "environments") + + +def launch(tools, executable, args): + + tools_env = api.get_tools(tools.split(";")) + env = api.compute(tools_env) + + env = api.merge(env, current_env=dict(os.environ)) + print("Environment:\n%s" % pprint.pformat(env, indent=4)) + + # Search for the executable within the tool's environment + # by temporarily taking on its `PATH` settings + original = os.environ["PATH"] + os.environ["PATH"] = env.get("PATH", os.environ.get("PATH", "")) + exe = lib.which(executable) + os.environ["PATH"] = original + + if not exe: + raise ValueError("Unable to find executable: %s" % executable) + + print("Launching: %s" % exe) + lib.launch(exe, + environment=env, + args=args) + + +if __name__ == '__main__': + + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--tools", + help="The tool environments to include. " + "These should be separated by `;`", + required=True) + parser.add_argument("--executable", + help="The executable to run. ", + required=True) + + kwargs, args = parser.parse_known_args() + + launch(tools=kwargs.tools, + executable=kwargs.executable, + args=args) From 7620fd3901a8bb6daeab25c063e5da527fc18aeb Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 16 Mar 2018 14:00:09 +0100 Subject: [PATCH 02/81] Created fusion dir for scripts, moves switch_and_submit in dir --- colorbleed/scripts/fusion/__init__.py | 0 .../switch_and_submit.py} | 105 ++++++++++++------ 2 files changed, 70 insertions(+), 35 deletions(-) create mode 100644 colorbleed/scripts/fusion/__init__.py rename colorbleed/scripts/{fusion_switch_shot.py => fusion/switch_and_submit.py} (75%) diff --git a/colorbleed/scripts/fusion/__init__.py b/colorbleed/scripts/fusion/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/colorbleed/scripts/fusion_switch_shot.py b/colorbleed/scripts/fusion/switch_and_submit.py similarity index 75% rename from colorbleed/scripts/fusion_switch_shot.py rename to colorbleed/scripts/fusion/switch_and_submit.py index 07bbafa0..3eb77506 100644 --- a/colorbleed/scripts/fusion_switch_shot.py +++ b/colorbleed/scripts/fusion/switch_and_submit.py @@ -16,8 +16,22 @@ self = sys.modules[__name__] self._project = None +error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" -def _format_version_folder(folder): + +def get_fusion_instance(): + fusion = getattr(sys.modules["__main__"], "fusion", None) + if fusion is None: + try: + # Support for FuScript.exe, BlackmagicFusion module for py2 only + import BlackmagicFusion as bmf + fusion = bmf.scriptapp("Fusion") + except ImportError: + raise RuntimeError("Could not find a Fusion instance") + return fusion + + +def format_version_folder(folder): """Format a version folder based on the filepath Assumption here is made that, if the path does not exists the folder @@ -44,7 +58,7 @@ def _format_version_folder(folder): return version_folder -def _get_work_folder(session): +def get_work_folder(session): """Convenience function to get the work folder path of the current asset""" # Get new filename, create path based on asset and work template @@ -54,25 +68,13 @@ def _get_work_folder(session): return os.path.normpath(work_path) -def _get_fusion_instance(): - fusion = getattr(sys.modules["__main__"], "fusion", None) - if fusion is None: - try: - # Support for FuScript.exe, BlackmagicFusion module for py2 only - import BlackmagicFusion as bmf - fusion = bmf.scriptapp("Fusion") - except ImportError: - raise RuntimeError("Could not find a Fusion instance") - return fusion - - -def _format_filepath(session): +def format_filepath(session): project = session["AVALON_PROJECT"] asset = session["AVALON_ASSET"] # Save updated slap comp - work_path = _get_work_folder(session) + work_path = get_work_folder(session) walk_to_dir = os.path.join(work_path, "scenes", "slapcomp") slapcomp_dir = os.path.abspath(walk_to_dir) @@ -92,7 +94,7 @@ def _format_filepath(session): return new_filepath -def _update_savers(comp, session): +def update_savers(comp, session): """Update all savers of the current comp to ensure the output is correct Args: @@ -103,9 +105,9 @@ def _update_savers(comp, session): None """ - new_work = _get_work_folder(session) + new_work = get_work_folder(session) renders = os.path.join(new_work, "renders") - version_folder = _format_version_folder(renders) + version_folder = format_version_folder(renders) renders_version = os.path.join(renders, version_folder) comp.Print("New renders to: %s\n" % renders) @@ -144,13 +146,33 @@ def update_frame_range(comp, representations): fusion_lib.update_frame_range(start, end, comp=comp) -def switch(asset_name, filepath=None, new=True): +def submit_deadline(): + """Work around method to ensure everything works with said context""" + + import pyblish.util + context = pyblish.util.publish() + + if not context: + log.warning("Nothing collected.") + sys.exit(1) + + # Collect errors, {plugin name: error} + error_results = [r for r in context.data["results"] if r["error"]] + if error_results: + log.error(" Errors occurred ...") + for result in error_results: + log.error(error_format.format(**result)) + sys.exit(2) + + +def switch(asset_name, file_path=None, new=True, deadline=True): """Switch the current containers of the file to the other asset (shot) Args: - filepath (str): file path of the comp file + file_path (str): file path of the comp file asset_name (str): name of the asset (shot) new (bool): Save updated comp under a different name + deadline (bool): if set to true Returns: comp path (str): new filepath of the updated comp @@ -158,8 +180,8 @@ def switch(asset_name, filepath=None, new=True): """ # Ensure filename is absolute - if not os.path.abspath(filepath): - filepath = os.path.abspath(filepath) + if not os.path.abspath(file_path): + file_path = os.path.abspath(file_path) # Get current project self._project = io.find_one({"type": "project", @@ -171,13 +193,13 @@ def switch(asset_name, filepath=None, new=True): assert asset, "Could not find '%s' in the database" % asset_name # Go to comp - if not filepath: + if not file_path: current_comp = avalon.fusion.get_current_comp() assert current_comp is not None, "Could not find current comp" else: - fusion = _get_fusion_instance() - current_comp = fusion.LoadComp(filepath, quiet=True) - assert current_comp is not None, "Fusion could not load '%s'" % filepath + fusion = get_fusion_instance() + current_comp = fusion.LoadComp(file_path, quiet=True) + assert current_comp is not None, "Fusion could not load '%s'" % file_path host = api.registered_host() containers = list(host.ls()) @@ -202,23 +224,31 @@ def switch(asset_name, filepath=None, new=True): switch_to_session["AVALON_ASSET"] = asset['name'] if new: - comp_path = _format_filepath(switch_to_session) + comp_path = format_filepath(switch_to_session) # Update savers output based on new session - _update_savers(current_comp, switch_to_session) + update_savers(current_comp, switch_to_session) else: - comp_path = colorbleed.version_up(filepath) + comp_path = colorbleed.version_up(file_path) - current_comp.Print(comp_path) - - current_comp.Print("\nUpdating frame range") + current_comp.Print("\nNew path: %s" % comp_path) + current_comp.Print("\nUpdating frame range ..") update_frame_range(current_comp, representations) current_comp.Save(comp_path) - return comp_path + if deadline: + # Update session with correct asset name + api.Session.update(**switch_to_session) + + # Submit to deadline render + publish + current_comp.SetData("colorbleed.rendermode", "deadline") + submit_deadline() + return current_comp + +# Usability for deadline job submission if __name__ == '__main__': import argparse @@ -236,9 +266,14 @@ def switch(asset_name, filepath=None, new=True): default=True, help="Name of the asset (shot) to switch") + parser.add_argument("--render", + default=False, + help="If set True the new composition file will be used" + "to render") + args, unknown = parser.parse_args() api.install(avalon.fusion) - switch(args.asset_name, args.file_path) + switch(args.asset_name, args.file_path, deadline=args.render) sys.exit(0) From d33e4715de2b39bff627606209609e63d7de3b20 Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 16 Mar 2018 14:06:29 +0100 Subject: [PATCH 03/81] Added assertion, changed signature order --- colorbleed/scripts/fusion/switch_and_submit.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/colorbleed/scripts/fusion/switch_and_submit.py b/colorbleed/scripts/fusion/switch_and_submit.py index 3eb77506..8a0f39ea 100644 --- a/colorbleed/scripts/fusion/switch_and_submit.py +++ b/colorbleed/scripts/fusion/switch_and_submit.py @@ -165,7 +165,7 @@ def submit_deadline(): sys.exit(2) -def switch(asset_name, file_path=None, new=True, deadline=True): +def switch(file_path=None, asset_name=None, new=True, deadline=True): """Switch the current containers of the file to the other asset (shot) Args: @@ -179,6 +179,8 @@ def switch(asset_name, file_path=None, new=True, deadline=True): """ + assert asset_name, "Function requires at least an asset name" + # Ensure filename is absolute if not os.path.abspath(file_path): file_path = os.path.abspath(file_path) @@ -274,6 +276,8 @@ def switch(asset_name, file_path=None, new=True, deadline=True): args, unknown = parser.parse_args() api.install(avalon.fusion) - switch(args.asset_name, args.file_path, deadline=args.render) + switch(file_path=args.file_path, + asset_name=args.asset_name, + deadline=args.render) sys.exit(0) From 25331664202526efa9b6fb801a89236e9e43bbde Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 16 Mar 2018 14:08:14 +0100 Subject: [PATCH 04/81] initial draft of logic for submit_switch_job --- .../plugins/publish/submit_publish_job.py | 4 + .../plugins/publish/submit_switch_job.py | 81 +++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 colorbleed/plugins/publish/submit_switch_job.py diff --git a/colorbleed/plugins/publish/submit_publish_job.py b/colorbleed/plugins/publish/submit_publish_job.py index c13f7e3d..63a24218 100644 --- a/colorbleed/plugins/publish/submit_publish_job.py +++ b/colorbleed/plugins/publish/submit_publish_job.py @@ -159,3 +159,7 @@ def process(self, instance): response = requests.post(url, json=payload) if not response.ok: raise Exception(response.text) + + # Temporary key name, deadlineSubmissionJob was already taken + if instance.data("runSlapComp", False): + instance.data["deadlineDependJob"] = response.json() diff --git a/colorbleed/plugins/publish/submit_switch_job.py b/colorbleed/plugins/publish/submit_switch_job.py new file mode 100644 index 00000000..88a4b611 --- /dev/null +++ b/colorbleed/plugins/publish/submit_switch_job.py @@ -0,0 +1,81 @@ +import os + +from avalon import api +from avalon.vendor import requests + +import pyblish.api + + +def _get_script(): + """Get path to the image sequence script""" + try: + from colorbleed.scripts.fusion import switch_and_submit + except Exception as e: + raise RuntimeError("Expected module 'fusion_switch_shot'" + "to be available") + + module_path = switch_and_submit.__file__ + if module_path.endswith(".pyc"): + module_path = module_path[:-len(".pyc")] + ".py" + + return module_path + + +class SubmitDependentSwitchJobDeadline(pyblish.api.InstancePlugin): + """Run Switch Shot on specified comp as depending job + + """ + + label = "Submit Switch Jobs to Deadline" + order = pyblish.api.IntegratorOrder + 0.2 + hosts = ["maya"] + families = ["colorbleed.renderlayer"] + + def process(self, instance): + + AVALON_DEADLINE = api.Session.get("AVALON_DEADLINE", + "http://localhost:8082") + assert AVALON_DEADLINE, "Requires AVALON_DEADLINE" + + job = instance.data.get("deadlineDependJob", {}) + + filepath = instance.data("flowFile", "") + if not filepath: + raise RuntimeError("No flow file (comp) chosen") + + shot = api.Session["AVALON_ASSET"] + comment = instance.context.data["comment"] + + args = '--file_path "{}" --asset_name "{}" --render 1'.format( + filepath, shot) + payload_name = "{} SWITCH".format(os.path.basename(filepath)) + + payload = { + "JobInfo": { + "BatchName": payload_name, + "Name": payload_name, + "JobType": "Normal", + "JobDependency0": job["_id"], + "UserName": job["Props"]["User"], + "Comment": comment, + "InitialStatus": "Pending"}, + "PluginInfo": { + "Version": "3.6", + "ScriptFile": _get_script(), + "Arguments": args, + "SingleFrameOnly": "True" + } + } + + session = {} + for index, key in enumerate(api.Session): + dl_key = "EnvironmentKeyValue%d" % index + value = "{key}={value}".format(key=key, value=api.Session[key]) + session[dl_key] = value + + payload.update(session) + + url = "{}/api/jobs".format(AVALON_DEADLINE) + response = requests.post(url, json=payload) + if not response.ok: + raise Exception(response.text) From 7d6754a74f2099b0b84b283b54ce266a60080410 Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 19 Mar 2018 11:37:01 +0100 Subject: [PATCH 05/81] Added slapcomp support --- colorbleed/plugins/maya/create/colorbleed_renderglobals.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/colorbleed/plugins/maya/create/colorbleed_renderglobals.py b/colorbleed/plugins/maya/create/colorbleed_renderglobals.py index 3ca857a1..108f3d46 100644 --- a/colorbleed/plugins/maya/create/colorbleed_renderglobals.py +++ b/colorbleed/plugins/maya/create/colorbleed_renderglobals.py @@ -30,6 +30,8 @@ def __init__(self, *args, **kwargs): data["priority"] = 50 data["whitelist"] = False data["machineList"] = "" + data["runSlapComp"] = False + data["flowFile"] = "" self.data = data self.options = {"useSelection": False} # Force no content From e683347485de88202cb77c530361800769bafdb3 Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 19 Mar 2018 15:10:19 +0100 Subject: [PATCH 06/81] changed to context plugin, fixed missing keys for job --- .../{ => maya}/publish/submit_switch_job.py | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) rename colorbleed/plugins/{ => maya}/publish/submit_switch_job.py (75%) diff --git a/colorbleed/plugins/publish/submit_switch_job.py b/colorbleed/plugins/maya/publish/submit_switch_job.py similarity index 75% rename from colorbleed/plugins/publish/submit_switch_job.py rename to colorbleed/plugins/maya/publish/submit_switch_job.py index 88a4b611..29b6e4ae 100644 --- a/colorbleed/plugins/publish/submit_switch_job.py +++ b/colorbleed/plugins/maya/publish/submit_switch_job.py @@ -1,4 +1,5 @@ import os +import pprint from avalon import api from avalon.vendor import requests @@ -21,7 +22,7 @@ def _get_script(): return module_path -class SubmitDependentSwitchJobDeadline(pyblish.api.InstancePlugin): +class SubmitDependentSwitchJobDeadline(pyblish.api.ContextPlugin): """Run Switch Shot on specified comp as depending job """ @@ -31,13 +32,18 @@ class SubmitDependentSwitchJobDeadline(pyblish.api.InstancePlugin): hosts = ["maya"] families = ["colorbleed.renderlayer"] - def process(self, instance): + def process(self, context): + + instance = context[0] AVALON_DEADLINE = api.Session.get("AVALON_DEADLINE", "http://localhost:8082") assert AVALON_DEADLINE, "Requires AVALON_DEADLINE" - job = instance.data.get("deadlineDependJob", {}) + job = instance.data.get("deadlineDependJob", None) + if not job: + self.log.warning("No dependent Job found") + return True filepath = instance.data("flowFile", "") if not filepath: @@ -52,19 +58,21 @@ def process(self, instance): payload = { "JobInfo": { - "BatchName": payload_name, + "Plugin": "Python", + "BatchName": job["Props"]["Batch"], "Name": payload_name, "JobType": "Normal", "JobDependency0": job["_id"], "UserName": job["Props"]["User"], "Comment": comment, - "InitialStatus": "Pending"}, + "InitialStatus": "Suspended"}, "PluginInfo": { "Version": "3.6", "ScriptFile": _get_script(), "Arguments": args, "SingleFrameOnly": "True" - } + }, + "AuxFiles": [] } session = {} @@ -79,3 +87,9 @@ def process(self, instance): response = requests.post(url, json=payload) if not response.ok: raise Exception(response.text) + + # Temporary key name, deadlineSubmissionJob was already taken + if instance.data("runSlapComp", False): + instance.data["deadlineDependJob"] = response.json() + + self.log.info("Slap comp arguments: %s" % args) From b240c4111af119177daf187a3f19878cb62eb571 Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 19 Mar 2018 15:10:47 +0100 Subject: [PATCH 07/81] Added support for slap comp --- colorbleed/plugins/maya/publish/collect_renderlayers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/colorbleed/plugins/maya/publish/collect_renderlayers.py b/colorbleed/plugins/maya/publish/collect_renderlayers.py index c196415a..48ecc1cf 100644 --- a/colorbleed/plugins/maya/publish/collect_renderlayers.py +++ b/colorbleed/plugins/maya/publish/collect_renderlayers.py @@ -135,4 +135,10 @@ def parse_options(self, render_globals): state = "Suspended" if attributes["suspendPublishJob"] else "Active" options["suspendPublishJob"] = state + # Check if the run slap comp + if attributes["runSlapComp"]: + self.log.info("Running render through slap comp as post ..") + options["runSlapComp"] = True + options["flowFile"] = attributes["flowFile"] + return options From 77dab4618eb9fb73e4b0e811a175b037f73a2370 Mon Sep 17 00:00:00 2001 From: wijnand Date: Wed, 21 Mar 2018 15:11:50 +0100 Subject: [PATCH 08/81] Removed unused import --- colorbleed/plugins/fusion/publish/collect_comp.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/colorbleed/plugins/fusion/publish/collect_comp.py b/colorbleed/plugins/fusion/publish/collect_comp.py index 1cf182c8..ed7166e6 100644 --- a/colorbleed/plugins/fusion/publish/collect_comp.py +++ b/colorbleed/plugins/fusion/publish/collect_comp.py @@ -1,5 +1,3 @@ -import os - import pyblish.api from avalon import fusion From f12152bea4f3b9e6635edb20ead88700b537eefd Mon Sep 17 00:00:00 2001 From: wijnand Date: Wed, 21 Mar 2018 15:17:14 +0100 Subject: [PATCH 09/81] Improved switch_and_submit, always abspath, if no path fetch from comp --- .../scripts/fusion/switch_and_submit.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/colorbleed/scripts/fusion/switch_and_submit.py b/colorbleed/scripts/fusion/switch_and_submit.py index 8a0f39ea..fb02ed05 100644 --- a/colorbleed/scripts/fusion/switch_and_submit.py +++ b/colorbleed/scripts/fusion/switch_and_submit.py @@ -17,6 +17,7 @@ self._project = None error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" +FUSION_EXE = "C:/Program File/Blackmagic Design/Fusion 9/Fusion.exe" def get_fusion_instance(): @@ -25,7 +26,7 @@ def get_fusion_instance(): try: # Support for FuScript.exe, BlackmagicFusion module for py2 only import BlackmagicFusion as bmf - fusion = bmf.scriptapp("Fusion") + fusion = bmf.scriptapp("Fusion", "localhost", 2) except ImportError: raise RuntimeError("Could not find a Fusion instance") return fusion @@ -182,9 +183,12 @@ def switch(file_path=None, asset_name=None, new=True, deadline=True): assert asset_name, "Function requires at least an asset name" # Ensure filename is absolute - if not os.path.abspath(file_path): + if file_path: file_path = os.path.abspath(file_path) + api.install(avalon.fusion) + host = api.registered_host() + # Get current project self._project = io.find_one({"type": "project", "name": api.Session["AVALON_PROJECT"]}) @@ -194,16 +198,17 @@ def switch(file_path=None, asset_name=None, new=True, deadline=True): asset = io.find_one({"type": "asset", "name": asset_name}) assert asset, "Could not find '%s' in the database" % asset_name - # Go to comp if not file_path: + # Assuming we use the current open comp current_comp = avalon.fusion.get_current_comp() assert current_comp is not None, "Could not find current comp" + file_path = current_comp.GetAttrs("COMPS_FileName") else: fusion = get_fusion_instance() - current_comp = fusion.LoadComp(file_path, quiet=True) - assert current_comp is not None, "Fusion could not load '%s'" % file_path + current_comp = fusion.LoadComp(file_path) + assert current_comp is not None, ("Fusion could not load '%s'" + % file_path) - host = api.registered_host() containers = list(host.ls()) assert containers, "Nothing to update" @@ -240,7 +245,7 @@ def switch(file_path=None, asset_name=None, new=True, deadline=True): current_comp.Save(comp_path) if deadline: - # Update session with correct asset name + # Update session with correct asset name and comp path api.Session.update(**switch_to_session) # Submit to deadline render + publish @@ -273,9 +278,8 @@ def switch(file_path=None, asset_name=None, new=True, deadline=True): help="If set True the new composition file will be used" "to render") - args, unknown = parser.parse_args() + args = parser.parse_args() - api.install(avalon.fusion) switch(file_path=args.file_path, asset_name=args.asset_name, deadline=args.render) From cb30677f56de886bfb8e8f6f34b33c2ff698add4 Mon Sep 17 00:00:00 2001 From: wijnand Date: Wed, 21 Mar 2018 16:02:52 +0100 Subject: [PATCH 10/81] Removed constant, fixed format_version_folder, renamed function --- .../scripts/fusion/switch_and_submit.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/colorbleed/scripts/fusion/switch_and_submit.py b/colorbleed/scripts/fusion/switch_and_submit.py index fb02ed05..3f02c683 100644 --- a/colorbleed/scripts/fusion/switch_and_submit.py +++ b/colorbleed/scripts/fusion/switch_and_submit.py @@ -16,9 +16,6 @@ self = sys.modules[__name__] self._project = None -error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" -FUSION_EXE = "C:/Program File/Blackmagic Design/Fusion 9/Fusion.exe" - def get_fusion_instance(): fusion = getattr(sys.modules["__main__"], "fusion", None) @@ -47,16 +44,14 @@ def format_version_folder(folder): new_version = 1 if os.path.isdir(folder): - re_version = re.compile("v\d+$") - versions = [i for i in os.listdir(folder) if os.path.isdir(i) - and re_version.match(i)] + re_version = re.compile("v\d+") + versions = [i for i in os.listdir(folder) if re_version.match(i) + and os.path.isdir(os.path.join(folder, i))] if versions: # ensure the "v" is not included new_version = int(max(versions)[1:]) + 1 - version_folder = "v{:03d}".format(new_version) - - return version_folder + return "v{:03d}".format(new_version) def get_work_folder(session): @@ -147,9 +142,12 @@ def update_frame_range(comp, representations): fusion_lib.update_frame_range(start, end, comp=comp) -def submit_deadline(): +# NOT FUSION RELATED +def publish_local(): """Work around method to ensure everything works with said context""" + error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" + import pyblish.util context = pyblish.util.publish() @@ -250,7 +248,7 @@ def switch(file_path=None, asset_name=None, new=True, deadline=True): # Submit to deadline render + publish current_comp.SetData("colorbleed.rendermode", "deadline") - submit_deadline() + publish_local() return current_comp From 55d58ab236891affeac079006baca76fa6fb89da Mon Sep 17 00:00:00 2001 From: wijnand Date: Wed, 21 Mar 2018 16:25:08 +0100 Subject: [PATCH 11/81] Merged publish_local logic in switch, use api.publish for publish --- .../scripts/fusion/switch_and_submit.py | 63 +++++++++---------- 1 file changed, 29 insertions(+), 34 deletions(-) diff --git a/colorbleed/scripts/fusion/switch_and_submit.py b/colorbleed/scripts/fusion/switch_and_submit.py index 3f02c683..a5cd649d 100644 --- a/colorbleed/scripts/fusion/switch_and_submit.py +++ b/colorbleed/scripts/fusion/switch_and_submit.py @@ -16,8 +16,11 @@ self = sys.modules[__name__] self._project = None +error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" + def get_fusion_instance(): + """Try to get an instance of Fusion""" fusion = getattr(sys.modules["__main__"], "fusion", None) if fusion is None: try: @@ -54,16 +57,6 @@ def format_version_folder(folder): return "v{:03d}".format(new_version) -def get_work_folder(session): - """Convenience function to get the work folder path of the current asset""" - - # Get new filename, create path based on asset and work template - template_work = self._project["config"]["template"]["work"] - work_path = pipeline._format_work_template(template_work, session) - - return os.path.normpath(work_path) - - def format_filepath(session): project = session["AVALON_PROJECT"] @@ -90,6 +83,16 @@ def format_filepath(session): return new_filepath +def get_work_folder(session): + """Convenience function to get the work folder path of the current asset""" + + # Get new filename, create path based on asset and work template + template_work = self._project["config"]["template"]["work"] + work_path = pipeline._format_work_template(template_work, session) + + return os.path.normpath(work_path) + + def update_savers(comp, session): """Update all savers of the current comp to ensure the output is correct @@ -142,28 +145,6 @@ def update_frame_range(comp, representations): fusion_lib.update_frame_range(start, end, comp=comp) -# NOT FUSION RELATED -def publish_local(): - """Work around method to ensure everything works with said context""" - - error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" - - import pyblish.util - context = pyblish.util.publish() - - if not context: - log.warning("Nothing collected.") - sys.exit(1) - - # Collect errors, {plugin name: error} - error_results = [r for r in context.data["results"] if r["error"]] - if error_results: - log.error(" Errors occurred ...") - for result in error_results: - log.error(error_format.format(**result)) - sys.exit(2) - - def switch(file_path=None, asset_name=None, new=True, deadline=True): """Switch the current containers of the file to the other asset (shot) @@ -240,15 +221,29 @@ def switch(file_path=None, asset_name=None, new=True, deadline=True): current_comp.Print("\nUpdating frame range ..") update_frame_range(current_comp, representations) + # Save changes current_comp.Save(comp_path) if deadline: # Update session with correct asset name and comp path api.Session.update(**switch_to_session) - # Submit to deadline render + publish + # Set render mode to deadline current_comp.SetData("colorbleed.rendermode", "deadline") - publish_local() + + # Run publisher + context = api.publish() + if not context: + log.warning("Nothing collected.") + sys.exit(1) + + # Collect errors, {plugin name: error} + error_results = [r for r in context.data["results"] if r["error"]] + if error_results: + log.error(" Errors occurred ...") + for result in error_results: + log.error(error_format.format(**result)) + sys.exit(2) return current_comp From e53c2d403529be7f019b9baf6efbfa75796fdf96 Mon Sep 17 00:00:00 2001 From: wijnand Date: Wed, 21 Mar 2018 17:34:11 +0100 Subject: [PATCH 12/81] Removed argparse from module --- .../scripts/fusion/switch_and_submit.py | 34 +------------------ 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/colorbleed/scripts/fusion/switch_and_submit.py b/colorbleed/scripts/fusion/switch_and_submit.py index a5cd649d..f41a4dee 100644 --- a/colorbleed/scripts/fusion/switch_and_submit.py +++ b/colorbleed/scripts/fusion/switch_and_submit.py @@ -237,7 +237,7 @@ def switch(file_path=None, asset_name=None, new=True, deadline=True): log.warning("Nothing collected.") sys.exit(1) - # Collect errors, {plugin name: error} + # Collect errors, {plugin name: error}, if any error_results = [r for r in context.data["results"] if r["error"]] if error_results: log.error(" Errors occurred ...") @@ -246,35 +246,3 @@ def switch(file_path=None, asset_name=None, new=True, deadline=True): sys.exit(2) return current_comp - - -# Usability for deadline job submission -if __name__ == '__main__': - - import argparse - - parser = argparse.ArgumentParser(description="Switch to a shot within an" - "existing comp file") - - parser.add_argument("--file_path", - type=str, - default=True, - help="File path of the comp to use") - - parser.add_argument("--asset_name", - type=str, - default=True, - help="Name of the asset (shot) to switch") - - parser.add_argument("--render", - default=False, - help="If set True the new composition file will be used" - "to render") - - args = parser.parse_args() - - switch(file_path=args.file_path, - asset_name=args.asset_name, - deadline=args.render) - - sys.exit(0) From 1208f8db4fda295184059cd98c17de22b180133a Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 22 Mar 2018 10:51:14 +0100 Subject: [PATCH 13/81] Cosmetics --- .../plugins/maya/publish/collect_renderlayers.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/colorbleed/plugins/maya/publish/collect_renderlayers.py b/colorbleed/plugins/maya/publish/collect_renderlayers.py index 48ecc1cf..4292ec0f 100644 --- a/colorbleed/plugins/maya/publish/collect_renderlayers.py +++ b/colorbleed/plugins/maya/publish/collect_renderlayers.py @@ -120,10 +120,13 @@ def parse_options(self, render_globals): attributes = maya.read(render_globals) - options = {"renderGlobals": {}} - options["renderGlobals"]["Priority"] = attributes["priority"] - legacy = attributes["useLegacyRenderLayers"] - options["renderGlobals"]["UseLegacyRenderLayers"] = legacy + options = { + "renderGlobals": + { + "Priority": attributes["priority"], + "UseLegacyRenderLayers": attributes["useLegacyRenderLayers"] + } + } # Machine list machine_list = attributes["machineList"] From 5dbf143494d3160ac943c0a8d6a2d701fd3e0338 Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 22 Mar 2018 10:54:20 +0100 Subject: [PATCH 14/81] Job dependend on last submitted, changed get_script function, update env --- .../plugins/maya/publish/submit_switch_job.py | 43 +++++++++++-------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/colorbleed/plugins/maya/publish/submit_switch_job.py b/colorbleed/plugins/maya/publish/submit_switch_job.py index 29b6e4ae..5a83a716 100644 --- a/colorbleed/plugins/maya/publish/submit_switch_job.py +++ b/colorbleed/plugins/maya/publish/submit_switch_job.py @@ -1,5 +1,4 @@ import os -import pprint from avalon import api from avalon.vendor import requests @@ -7,19 +6,21 @@ import pyblish.api -def _get_script(): +def _get_script_dir(): """Get path to the image sequence script""" try: - from colorbleed.scripts.fusion import switch_and_submit - except Exception as e: - raise RuntimeError("Expected module 'fusion_switch_shot'" - "to be available") + import colorbleed + scriptdir = os.path.dirname(colorbleed.__file__) + fusion_scripts = os.path.join(scriptdir, + "scripts", + "fusion") + except: + raise RuntimeError("This is a bug") - module_path = switch_and_submit.__file__ - if module_path.endswith(".pyc"): - module_path = module_path[:-len(".pyc")] + ".py" + assert os.path.isdir(fusion_scripts), "Config is incomplete" + fusion_scripts = fusion_scripts.replace(os.sep, "/") - return module_path + return fusion_scripts class SubmitDependentSwitchJobDeadline(pyblish.api.ContextPlugin): @@ -34,7 +35,8 @@ class SubmitDependentSwitchJobDeadline(pyblish.api.ContextPlugin): def process(self, context): - instance = context[0] + # Run it as depend on the last submitted instance + instance = context[-1] AVALON_DEADLINE = api.Session.get("AVALON_DEADLINE", "http://localhost:8082") @@ -52,6 +54,9 @@ def process(self, context): shot = api.Session["AVALON_ASSET"] comment = instance.context.data["comment"] + scriptdir = _get_script_dir() + scriptfile = os.path.join(scriptdir, "deadline_swith_and_submi.py") + args = '--file_path "{}" --asset_name "{}" --render 1'.format( filepath, shot) payload_name = "{} SWITCH".format(os.path.basename(filepath)) @@ -68,20 +73,20 @@ def process(self, context): "InitialStatus": "Suspended"}, "PluginInfo": { "Version": "3.6", - "ScriptFile": _get_script(), + "ScriptFile": scriptfile, "Arguments": args, "SingleFrameOnly": "True" }, "AuxFiles": [] } - session = {} - for index, key in enumerate(api.Session): - dl_key = "EnvironmentKeyValue%d" % index - value = "{key}={value}".format(key=key, value=api.Session[key]) - session[dl_key] = value - - payload.update(session) + environment = job["Props"].get("Env", {}) + payload["JobInfo"].update({ + "EnvironmentKeyValue%d" % index: "{key}={value}".format( + key=key, + value=environment[key] + ) for index, key in enumerate(environment) + }) url = "{}/api/jobs".format(AVALON_DEADLINE) response = requests.post(url, json=payload) From 8de8984582b8b83d5051511a607a7a115230579f Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 22 Mar 2018 11:35:19 +0100 Subject: [PATCH 15/81] Refactored __main__ --- colorbleed/scripts/publish_filesequence.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/colorbleed/scripts/publish_filesequence.py b/colorbleed/scripts/publish_filesequence.py index c37ceee0..f026cfc4 100644 --- a/colorbleed/scripts/publish_filesequence.py +++ b/colorbleed/scripts/publish_filesequence.py @@ -62,8 +62,9 @@ def publish(paths, gui=False): sys.exit(2) -def __main__(): +if __name__ == '__main__': import argparse + parser = argparse.ArgumentParser() parser.add_argument("--paths", nargs="*", @@ -81,7 +82,3 @@ def __main__(): print("Running publish imagesequence...") print("Paths: {}".format(kwargs.paths or [os.getcwd()])) publish(kwargs.paths, gui=kwargs.gui) - - -if __name__ == '__main__': - __main__() From 1e0d4b7a061d766e86ae8652eaa05d8ad420f804 Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 22 Mar 2018 12:36:09 +0100 Subject: [PATCH 16/81] Improved format_version_folder logic, explicit error print in publishing --- colorbleed/scripts/fusion/switch_and_submit.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/colorbleed/scripts/fusion/switch_and_submit.py b/colorbleed/scripts/fusion/switch_and_submit.py index f41a4dee..987a8af6 100644 --- a/colorbleed/scripts/fusion/switch_and_submit.py +++ b/colorbleed/scripts/fusion/switch_and_submit.py @@ -45,16 +45,17 @@ def format_version_folder(folder): str: new version folder name """ - new_version = 1 + version_int = 1 if os.path.isdir(folder): re_version = re.compile("v\d+") versions = [i for i in os.listdir(folder) if re_version.match(i) and os.path.isdir(os.path.join(folder, i))] if versions: - # ensure the "v" is not included - new_version = int(max(versions)[1:]) + 1 + # ensure the "v" is not included and convert to ints + int_versions = [int(v[1:]) for v in versions] + version_int += max(int_versions) - return "v{:03d}".format(new_version) + return "v{:03d}".format(version_int) def format_filepath(session): @@ -240,7 +241,7 @@ def switch(file_path=None, asset_name=None, new=True, deadline=True): # Collect errors, {plugin name: error}, if any error_results = [r for r in context.data["results"] if r["error"]] if error_results: - log.error(" Errors occurred ...") + log.error("{} Errors occurred ...".format(__name__)) for result in error_results: log.error(error_format.format(**result)) sys.exit(2) From e7bf90ee943adb4bc5caf63ef113e141778efdf2 Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 22 Mar 2018 14:18:52 +0100 Subject: [PATCH 17/81] Removed debug print, removed astrixes from update() --- colorbleed/scripts/fusion/switch_and_submit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/colorbleed/scripts/fusion/switch_and_submit.py b/colorbleed/scripts/fusion/switch_and_submit.py index 987a8af6..b7bb6c45 100644 --- a/colorbleed/scripts/fusion/switch_and_submit.py +++ b/colorbleed/scripts/fusion/switch_and_submit.py @@ -227,7 +227,7 @@ def switch(file_path=None, asset_name=None, new=True, deadline=True): if deadline: # Update session with correct asset name and comp path - api.Session.update(**switch_to_session) + api.Session.update(switch_to_session) # Set render mode to deadline current_comp.SetData("colorbleed.rendermode", "deadline") @@ -241,7 +241,7 @@ def switch(file_path=None, asset_name=None, new=True, deadline=True): # Collect errors, {plugin name: error}, if any error_results = [r for r in context.data["results"] if r["error"]] if error_results: - log.error("{} Errors occurred ...".format(__name__)) + log.error("Errors occurred ...") for result in error_results: log.error(error_format.format(**result)) sys.exit(2) From abd6ff95aef561b52a4afa4ce64d3e8811261ef8 Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 22 Mar 2018 14:25:22 +0100 Subject: [PATCH 18/81] Extended docstrings --- colorbleed/scripts/fusion/switch_and_submit.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/colorbleed/scripts/fusion/switch_and_submit.py b/colorbleed/scripts/fusion/switch_and_submit.py index b7bb6c45..f92e420d 100644 --- a/colorbleed/scripts/fusion/switch_and_submit.py +++ b/colorbleed/scripts/fusion/switch_and_submit.py @@ -149,6 +149,9 @@ def update_frame_range(comp, representations): def switch(file_path=None, asset_name=None, new=True, deadline=True): """Switch the current containers of the file to the other asset (shot) + Exitcode 1: No item found to publish + Exitcode 2: Errors during publish process + Args: file_path (str): file path of the comp file asset_name (str): name of the asset (shot) From 7c5c2954246ac72cd13ffa97725af9ec3c9ecee0 Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 22 Mar 2018 14:35:33 +0100 Subject: [PATCH 19/81] Implemented get_script_dir method --- .../scripts/Comp/colorbleed/switch_ui.py | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/setup/fusion/scripts/Comp/colorbleed/switch_ui.py b/setup/fusion/scripts/Comp/colorbleed/switch_ui.py index 8f1466ab..f5ab6dd3 100644 --- a/setup/fusion/scripts/Comp/colorbleed/switch_ui.py +++ b/setup/fusion/scripts/Comp/colorbleed/switch_ui.py @@ -1,4 +1,5 @@ import os +import site import glob import logging @@ -146,6 +147,22 @@ def _refresh(self): def _on_switch(self): + def _get_script_dir(): + """Get path to the image sequence script""" + try: + import colorbleed + scriptdir = os.path.dirname(colorbleed.__file__) + fusion_scripts = os.path.join(scriptdir, + "scripts", + "fusion") + except: + raise RuntimeError("This is a bug") + + assert os.path.isdir(fusion_scripts), "Config is incomplete" + fusion_scripts = fusion_scripts.replace(os.sep, "/") + + return fusion_scripts + if not self._use_current.isChecked(): file_name = self._comps.itemData(self._comps.currentIndex()) else: @@ -154,8 +171,12 @@ def _on_switch(self): asset = self._assets.currentText() - import colorbleed.scripts.fusion_switch_shot as switch_shot - switch_shot.switch(asset_name=asset, filepath=file_name, new=True) + # Get switch and submit module + script_dir = _get_script_dir() + site.addsitedir(script_dir) + + import switch_and_submit as switch_shot + switch_shot.switch(asset_name=asset, file_path=file_name, new=True) def _get_context_directory(self): From 7af871d9f268d5436cc9f4ac0691daa897abcfa1 Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 22 Mar 2018 14:47:44 +0100 Subject: [PATCH 20/81] refactored context[:] to list(context) due to slice issue in Py36 --- colorbleed/plugins/fusion/publish/submit_deadline.py | 2 +- colorbleed/plugins/fusion/publish/validate_unique_subsets.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/colorbleed/plugins/fusion/publish/submit_deadline.py b/colorbleed/plugins/fusion/publish/submit_deadline.py index 5afb484e..bb69cc8a 100644 --- a/colorbleed/plugins/fusion/publish/submit_deadline.py +++ b/colorbleed/plugins/fusion/publish/submit_deadline.py @@ -39,7 +39,7 @@ def process(self, instance): # Collect all saver instances in context that are to be rendered saver_instances = [] - for instance in context[:]: + for instance in list(context): if not self.families[0] in instance.data.get("families"): # Allow only saver family instances continue diff --git a/colorbleed/plugins/fusion/publish/validate_unique_subsets.py b/colorbleed/plugins/fusion/publish/validate_unique_subsets.py index 527b4acc..bbea7cba 100644 --- a/colorbleed/plugins/fusion/publish/validate_unique_subsets.py +++ b/colorbleed/plugins/fusion/publish/validate_unique_subsets.py @@ -11,10 +11,9 @@ class ValidateUniqueSubsets(pyblish.api.InstancePlugin): @classmethod def get_invalid(cls, instance): - context = instance.context subset = instance.data["subset"] - for other_instance in context[:]: + for other_instance in list(context): if other_instance == instance: continue @@ -26,4 +25,4 @@ def get_invalid(cls, instance): def process(self, instance): invalid = self.get_invalid(instance) if invalid: - raise RuntimeError("Animation content is invalid. See log.") + raise RuntimeError("Content is invalid. See log.") From 4c9a6f51688556d356da98574e3347891cd9ec59 Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 22 Mar 2018 14:48:28 +0100 Subject: [PATCH 21/81] Added module for deadline switching of comps --- .../fusion/deadline_switch_and_submit.py | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 colorbleed/scripts/fusion/deadline_switch_and_submit.py diff --git a/colorbleed/scripts/fusion/deadline_switch_and_submit.py b/colorbleed/scripts/fusion/deadline_switch_and_submit.py new file mode 100644 index 00000000..8eaf3d0c --- /dev/null +++ b/colorbleed/scripts/fusion/deadline_switch_and_submit.py @@ -0,0 +1,200 @@ +""" +This module is for a standalone approach for Fusion similar to Maya. +Note that this will require FusionConsoleNode.exe and the BlackmagicFusion +module, + +Deadline runs a python process, lets call it P + +P will start the FusionConsoleNode in a new SUBPROCESS +This SUBPROCESS will need to have the same environment as P to ensure it can +use AVALON + + P --> SUBPROCESS (FusionConsoleNode.EXE /listen) + +From the SUBPROCESS comes a Fusion Console Node which will be used as the Fusion +instance to work in. In order to get the correct Fusion instance we use a +ScriptServer to get all Fusion programs which are running. +This is done by comparing the process ids with the subprocess.pid. + + See `get_fusion_instance` function for more details + +In `avalon.fusion.pipeline` we have create a work around to get the fusion +instance. This is done through + + getattr(sys.module["__main__"], "fusion", None) + +Because we do this we can also allow to set correct fusion module, this is done +by using the setattr. This will ensure that all other functions which are run +within `process()` can find `fusion`. + +""" + + +import subprocess +import traceback +import site +import time +import sys +import os + +# This script only works with Python 2.7 and 3.6 +version = "{0}{1}".format(*sys.version_info) # {major}{minor} +assert version in ["27", "36"], "Script only works in Python 2.7 or 3.6" +key = "FUSION_PYTHON{0}_HOME".format(version) + +# Set Python 3.6 home for fusion, debug +print("Settings FUSION_PYTHON36_HOME ..") +os.environ[key] = os.path.dirname(sys.executable) + +FUSCRIPT_EXE = r"C:/Program Files/Blackmagic Design/Fusion9/FuScript.exe" +FUSION_CONSOLE_EXE = r"C:/Program Files/Blackmagic Design/Fusion Render Node 9/FusionConsoleNode.exe" + +import BlackmagicFusion as bmf + + +def _get_script_dir(): + """Get path to the image sequence script""" + try: + import colorbleed + scriptdir = os.path.dirname(colorbleed.__file__) + fusion_scripts = os.path.join(scriptdir, + "scripts", + "fusion") + except: + raise RuntimeError("This is a bug") + + assert os.path.isdir(fusion_scripts), "Config is incomplete" + fusion_scripts = fusion_scripts.replace(os.sep, "/") + + return fusion_scripts + + +def start_server(): + bmf.startserver() + return get_server() + + +def get_server(tries=10, timeout=0.5): + + count = 0 + srv = None + + while not srv: + count += 1 + print("Connecting to ScriptServer (try: %s)" % count) + srv = bmf.scriptapp("", "localhost", timeout) # Runs script server + if count > tries: + break + + return srv + + +def get_fusion_instance(pid, srv, timeout=10): + """Get the fusion instance which has been launched""" + + count = 0 + host = None + while not host: + if count > timeout: + break + fusion_hosts = srv.GetHostList().values() + host = next((i for i in fusion_hosts if int(i["ProcessID"]) == pid), + None) + if not host: + print("Find Fusion host... (%ss)" % count) + time.sleep(0.5) + count += 0.5 + + assert host, "Fusion not found with pid: %s" % pid + + return bmf.scriptapp(host["Name"], "localhost", 2, host["UUID"]) + + +def process(file_path, asset_name, deadline=False): + """Run switch in a Fusion Console Node (cmd) + + Args: + file_path (str): File path of the comp to use + asset_name (str): Name of the asset (shot) to switch + deadline (bool, optional): If set True the new composition file will be + used to render + Returns: + None + + """ + + # Start a fusion console node in "listen" mode + proc = subprocess.Popen([FUSION_CONSOLE_EXE, "/listen"]) + + srv = get_server() + if not srv: + print("No server found, starting server ..") + srv = start_server() + + # Force fusion into main magical module so that host.ls() works + fusion = get_fusion_instance(proc.pid, srv) + assert fusion + print("Connected to: %s" % fusion) + setattr(sys.modules["__main__"], "fusion", fusion) + + # Get fusion.pipeline + from avalon.fusion import pipeline + + # This does not set + loaded_comp = fusion.LoadComp(file_path) + if not loaded_comp: + raise RuntimeError("Comp could not be loaded. File '%s'" % file_path) + pipeline.force_current_comp(loaded_comp) + current_comp = pipeline.get_current_comp() + + assert loaded_comp == current_comp, "Could not find the correct comp" + + print("Loaded comp name: %s" % current_comp.GetAttrs("COMPS_FileName")) + + # Get switch and submit script + scriptdir = _get_script_dir() + site.addsitedir(scriptdir) + import switch_and_submit as switch + + # Fusion host + try: + # Execute script in comp + result = switch.switch(asset_name=asset_name, deadline=deadline) + except: + tb = traceback.format_exc() + proc.terminate() # Ensure process closes when failing + raise RuntimeError(tb) + + print("Success:", result is not None) + print("Closing all running process ..") + proc.terminate() + + +# Usability for deadline job submission +if __name__ == '__main__': + + import argparse + + parser = argparse.ArgumentParser(description="Switch to a shot within an" + "existing comp file") + + parser.add_argument("--file_path", + type=str, + default=True, + help="File path of the comp to use") + + parser.add_argument("--asset_name", + type=str, + default=True, + help="Name of the asset (shot) to switch") + + parser.add_argument("--render", + default=False, + help="If set True the new composition file will be used" + "to render") + + args = parser.parse_args() + + process(file_path=args.file_path, + asset_name=args.asset_name, + deadline=args.render) From 025ac39917b8f7f490ad4aedc147e492767a4ef0 Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 22 Mar 2018 15:26:05 +0100 Subject: [PATCH 22/81] Fixed typo, fetch of machine limit from depending job --- colorbleed/plugins/maya/publish/submit_switch_job.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/colorbleed/plugins/maya/publish/submit_switch_job.py b/colorbleed/plugins/maya/publish/submit_switch_job.py index 5a83a716..b012f3b5 100644 --- a/colorbleed/plugins/maya/publish/submit_switch_job.py +++ b/colorbleed/plugins/maya/publish/submit_switch_job.py @@ -1,4 +1,5 @@ import os +import pprint from avalon import api from avalon.vendor import requests @@ -55,7 +56,7 @@ def process(self, context): comment = instance.context.data["comment"] scriptdir = _get_script_dir() - scriptfile = os.path.join(scriptdir, "deadline_swith_and_submi.py") + scriptfile = os.path.join(scriptdir, "deadline_swith_and_submit.py") args = '--file_path "{}" --asset_name "{}" --render 1'.format( filepath, shot) @@ -80,6 +81,10 @@ def process(self, context): "AuxFiles": [] } + # Update payload with machine limits + list_type = self._get_list_type(job) + payload["JobInfo"][list_type] = job["Props"]["ListedSlaves"] + environment = job["Props"].get("Env", {}) payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format( @@ -98,3 +103,6 @@ def process(self, context): instance.data["deadlineDependJob"] = response.json() self.log.info("Slap comp arguments: %s" % args) + + def _get_list_type(self, job): + return "Whitelist" if job["Props"]["White"]is True else "Blacklist" \ No newline at end of file From 8aa979978dbfc79bd54c5b695e1ee1fd7e14ade6 Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 22 Mar 2018 15:27:50 +0100 Subject: [PATCH 23/81] new line at end of file --- colorbleed/plugins/maya/publish/submit_switch_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/colorbleed/plugins/maya/publish/submit_switch_job.py b/colorbleed/plugins/maya/publish/submit_switch_job.py index b012f3b5..a9c20a49 100644 --- a/colorbleed/plugins/maya/publish/submit_switch_job.py +++ b/colorbleed/plugins/maya/publish/submit_switch_job.py @@ -105,4 +105,4 @@ def process(self, context): self.log.info("Slap comp arguments: %s" % args) def _get_list_type(self, job): - return "Whitelist" if job["Props"]["White"]is True else "Blacklist" \ No newline at end of file + return "Whitelist" if job["Props"]["White"]is True else "Blacklist" From 442eea56c3724ddbf16663c45d5110f0334c6f52 Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 22 Mar 2018 17:35:27 +0100 Subject: [PATCH 24/81] Refactored approach of getting machine limit --- .../plugins/maya/publish/submit_switch_job.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/colorbleed/plugins/maya/publish/submit_switch_job.py b/colorbleed/plugins/maya/publish/submit_switch_job.py index a9c20a49..6865048f 100644 --- a/colorbleed/plugins/maya/publish/submit_switch_job.py +++ b/colorbleed/plugins/maya/publish/submit_switch_job.py @@ -56,7 +56,7 @@ def process(self, context): comment = instance.context.data["comment"] scriptdir = _get_script_dir() - scriptfile = os.path.join(scriptdir, "deadline_swith_and_submit.py") + scriptfile = os.path.join(scriptdir, "deadline_switch_and_submit.py") args = '--file_path "{}" --asset_name "{}" --render 1'.format( filepath, shot) @@ -81,9 +81,8 @@ def process(self, context): "AuxFiles": [] } - # Update payload with machine limits - list_type = self._get_list_type(job) - payload["JobInfo"][list_type] = job["Props"]["ListedSlaves"] + machine_limit = self.get_machine_limit(instance) + payload["JobInfo"].update(machine_limit) environment = job["Props"].get("Env", {}) payload["JobInfo"].update({ @@ -99,10 +98,14 @@ def process(self, context): raise Exception(response.text) # Temporary key name, deadlineSubmissionJob was already taken - if instance.data("runSlapComp", False): + if instance.data.get("runSlapComp", False): instance.data["deadlineDependJob"] = response.json() self.log.info("Slap comp arguments: %s" % args) - def _get_list_type(self, job): - return "Whitelist" if job["Props"]["White"]is True else "Blacklist" + def get_machine_limit(self, instance): + renderglobals = instance.data.get("renderGlobals") + if "Whitelist" in renderglobals: + return {"Whitelist": renderglobals["Whitelise"]} + return {"Blacklist": renderglobals["Blacklist"]} + From 8d0b0c40346a7293a16296e5ecbdc10ba10ef047 Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 22 Mar 2018 17:36:26 +0100 Subject: [PATCH 25/81] Created backwards compatibility, added message if key is missing --- colorbleed/plugins/maya/publish/collect_renderlayers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/colorbleed/plugins/maya/publish/collect_renderlayers.py b/colorbleed/plugins/maya/publish/collect_renderlayers.py index 4292ec0f..b259048b 100644 --- a/colorbleed/plugins/maya/publish/collect_renderlayers.py +++ b/colorbleed/plugins/maya/publish/collect_renderlayers.py @@ -139,7 +139,11 @@ def parse_options(self, render_globals): options["suspendPublishJob"] = state # Check if the run slap comp - if attributes["runSlapComp"]: + if "runSlapComp" not in attributes: + self.log.warning("renderGlobals node might be out of data, missing" + "'runSlapComp' information.") + + if attributes.get("runSlapComp", False): self.log.info("Running render through slap comp as post ..") options["runSlapComp"] = True options["flowFile"] = attributes["flowFile"] From f890253e52b3c704df4b2e450e988fb91350f210 Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 22 Mar 2018 17:50:57 +0100 Subject: [PATCH 26/81] Improved the get_machine_limit function --- colorbleed/plugins/maya/publish/submit_switch_job.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/colorbleed/plugins/maya/publish/submit_switch_job.py b/colorbleed/plugins/maya/publish/submit_switch_job.py index 6865048f..f3b7aad5 100644 --- a/colorbleed/plugins/maya/publish/submit_switch_job.py +++ b/colorbleed/plugins/maya/publish/submit_switch_job.py @@ -104,8 +104,14 @@ def process(self, context): self.log.info("Slap comp arguments: %s" % args) def get_machine_limit(self, instance): - renderglobals = instance.data.get("renderGlobals") + """Retrieve the machine limit from the instance""" + + renderglobals = instance.data.get("renderGlobals", None) + if renderglobals is None: + return {} + if "Whitelist" in renderglobals: - return {"Whitelist": renderglobals["Whitelise"]} - return {"Blacklist": renderglobals["Blacklist"]} + return {"Whitelist": renderglobals["Whitelist"]} + + return {"Blacklist": renderglobals.get("Blacklist")} From 18afd8a377a2653ca15df62f1a00c7eacac2801a Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 22 Mar 2018 17:53:13 +0100 Subject: [PATCH 27/81] Updated docstrings --- colorbleed/plugins/maya/publish/submit_switch_job.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/colorbleed/plugins/maya/publish/submit_switch_job.py b/colorbleed/plugins/maya/publish/submit_switch_job.py index f3b7aad5..9bbb7993 100644 --- a/colorbleed/plugins/maya/publish/submit_switch_job.py +++ b/colorbleed/plugins/maya/publish/submit_switch_job.py @@ -104,7 +104,15 @@ def process(self, context): self.log.info("Slap comp arguments: %s" % args) def get_machine_limit(self, instance): - """Retrieve the machine limit from the instance""" + """Retrieve the machine limit from the instance + + Args: + instance: instance to retrieve the renderglobals from + + Returns: + dict: {list type: list of machine names + + """ renderglobals = instance.data.get("renderGlobals", None) if renderglobals is None: From ac602225e894bfd0a11a44d4ca5971ea28c8d970 Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 22 Mar 2018 17:56:36 +0100 Subject: [PATCH 28/81] Fixed error --- colorbleed/plugins/publish/submit_publish_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/colorbleed/plugins/publish/submit_publish_job.py b/colorbleed/plugins/publish/submit_publish_job.py index 63a24218..a89e1918 100644 --- a/colorbleed/plugins/publish/submit_publish_job.py +++ b/colorbleed/plugins/publish/submit_publish_job.py @@ -161,5 +161,5 @@ def process(self, instance): raise Exception(response.text) # Temporary key name, deadlineSubmissionJob was already taken - if instance.data("runSlapComp", False): + if instance.data.get("runSlapComp", False): instance.data["deadlineDependJob"] = response.json() From 87bb43f0c5cf619a15e1927bb5fc5164cd8cf56e Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 23 Mar 2018 10:03:36 +0100 Subject: [PATCH 29/81] Refactored function names, setting current comp to None when closing --- colorbleed/scripts/fusion/deadline_switch_and_submit.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/colorbleed/scripts/fusion/deadline_switch_and_submit.py b/colorbleed/scripts/fusion/deadline_switch_and_submit.py index 8eaf3d0c..1afa091f 100644 --- a/colorbleed/scripts/fusion/deadline_switch_and_submit.py +++ b/colorbleed/scripts/fusion/deadline_switch_and_submit.py @@ -144,7 +144,7 @@ def process(file_path, asset_name, deadline=False): loaded_comp = fusion.LoadComp(file_path) if not loaded_comp: raise RuntimeError("Comp could not be loaded. File '%s'" % file_path) - pipeline.force_current_comp(loaded_comp) + pipeline.set_current_comp(loaded_comp) current_comp = pipeline.get_current_comp() assert loaded_comp == current_comp, "Could not find the correct comp" @@ -162,11 +162,13 @@ def process(file_path, asset_name, deadline=False): result = switch.switch(asset_name=asset_name, deadline=deadline) except: tb = traceback.format_exc() + pipeline.set_current_comp(None) proc.terminate() # Ensure process closes when failing raise RuntimeError(tb) print("Success:", result is not None) print("Closing all running process ..") + pipeline.set_current_comp(None) proc.terminate() From 86f4c9cc7beb69b8351dedc23412758bb5c00a61 Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 23 Mar 2018 10:31:06 +0100 Subject: [PATCH 30/81] Removed __init__ modules from script folder --- colorbleed/scripts/__init__.py | 0 colorbleed/scripts/fusion/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 colorbleed/scripts/__init__.py delete mode 100644 colorbleed/scripts/fusion/__init__.py diff --git a/colorbleed/scripts/__init__.py b/colorbleed/scripts/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/colorbleed/scripts/fusion/__init__.py b/colorbleed/scripts/fusion/__init__.py deleted file mode 100644 index e69de29b..00000000 From 71f22936724e707dab199849b19d0d85700bb9ef Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 23 Mar 2018 11:24:18 +0100 Subject: [PATCH 31/81] Update environment to ensure session and environment are in sync --- colorbleed/scripts/fusion/switch_and_submit.py | 1 + 1 file changed, 1 insertion(+) diff --git a/colorbleed/scripts/fusion/switch_and_submit.py b/colorbleed/scripts/fusion/switch_and_submit.py index f92e420d..89e2f558 100644 --- a/colorbleed/scripts/fusion/switch_and_submit.py +++ b/colorbleed/scripts/fusion/switch_and_submit.py @@ -231,6 +231,7 @@ def switch(file_path=None, asset_name=None, new=True, deadline=True): if deadline: # Update session with correct asset name and comp path api.Session.update(switch_to_session) + os.environ.update(switch_to_session) # Set render mode to deadline current_comp.SetData("colorbleed.rendermode", "deadline") From 2f1b08a9f6ba375ebc9ad6e3bd1c2b0d72fa5bfd Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 26 Mar 2018 10:13:12 +0200 Subject: [PATCH 32/81] Updated docstring, fixed type, updated print --- .../scripts/fusion/deadline_switch_and_submit.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/colorbleed/scripts/fusion/deadline_switch_and_submit.py b/colorbleed/scripts/fusion/deadline_switch_and_submit.py index 1afa091f..9f45c30f 100644 --- a/colorbleed/scripts/fusion/deadline_switch_and_submit.py +++ b/colorbleed/scripts/fusion/deadline_switch_and_submit.py @@ -1,7 +1,11 @@ """ +This holds the logic to open a comp, switch the assets (shot), save it with +the correct naming. Optionally the new comp can be submitted to Deadline to +render and publish the output + This module is for a standalone approach for Fusion similar to Maya. Note that this will require FusionConsoleNode.exe and the BlackmagicFusion -module, +module. Deadline runs a python process, lets call it P @@ -43,9 +47,10 @@ key = "FUSION_PYTHON{0}_HOME".format(version) # Set Python 3.6 home for fusion, debug -print("Settings FUSION_PYTHON36_HOME ..") +print("Settings FUSION PYTHON HOME ..") os.environ[key] = os.path.dirname(sys.executable) +# TODO: define these paths somewhere else FUSCRIPT_EXE = r"C:/Program Files/Blackmagic Design/Fusion9/FuScript.exe" FUSION_CONSOLE_EXE = r"C:/Program Files/Blackmagic Design/Fusion Render Node 9/FusionConsoleNode.exe" @@ -113,6 +118,8 @@ def get_fusion_instance(pid, srv, timeout=10): def process(file_path, asset_name, deadline=False): """Run switch in a Fusion Console Node (cmd) + Open the comp (file_path) and switch to the asset (asset_name). + Args: file_path (str): File path of the comp to use asset_name (str): Name of the asset (shot) to switch From b6a1e69a011413dd1b4bbd229987625333b50151 Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 26 Mar 2018 10:42:20 +0200 Subject: [PATCH 33/81] Added more explicit comment and docstrings --- .../scripts/fusion/deadline_switch_and_submit.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/colorbleed/scripts/fusion/deadline_switch_and_submit.py b/colorbleed/scripts/fusion/deadline_switch_and_submit.py index 9f45c30f..ef13bdac 100644 --- a/colorbleed/scripts/fusion/deadline_switch_and_submit.py +++ b/colorbleed/scripts/fusion/deadline_switch_and_submit.py @@ -46,8 +46,10 @@ assert version in ["27", "36"], "Script only works in Python 2.7 or 3.6" key = "FUSION_PYTHON{0}_HOME".format(version) -# Set Python 3.6 home for fusion, debug -print("Settings FUSION PYTHON HOME ..") +# Importing BlackmagicFusion package in standalone Python interpreter +# crashes when not installed on default location but runs from, e.g. a +# network share. Forcing Fusion's Python home magically fixes it. +print("Setting %s to Python executable directory.." % key) os.environ[key] = os.path.dirname(sys.executable) # TODO: define these paths somewhere else @@ -118,7 +120,9 @@ def get_fusion_instance(pid, srv, timeout=10): def process(file_path, asset_name, deadline=False): """Run switch in a Fusion Console Node (cmd) - Open the comp (file_path) and switch to the asset (asset_name). + Open the comp (file_path) and switch to the asset (asset_name) and when + deadline is enabled it will submit the switched comp to Deadline to render + and publish the output. Args: file_path (str): File path of the comp to use From 28a779b165153cd708013b73b151f2c19a1d6568 Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 26 Mar 2018 14:04:00 +0200 Subject: [PATCH 34/81] Make switch logic more central --- colorbleed/fusion/lib.py | 158 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/colorbleed/fusion/lib.py b/colorbleed/fusion/lib.py index 7186cd2c..ef75db51 100644 --- a/colorbleed/fusion/lib.py +++ b/colorbleed/fusion/lib.py @@ -1,11 +1,18 @@ +import os +import re import sys +import logging import avalon.fusion +from avalon import api, io, pipeline +import colorbleed.lib as colorbleed self = sys.modules[__name__] self._project = None +log = logging.getLogger(__name__) + def update_frame_range(start, end, comp=None, set_render_range=True): """Set Fusion comp's start and end frame range @@ -38,3 +45,154 @@ def update_frame_range(start, end, comp=None, set_render_range=True): with avalon.fusion.comp_lock_and_undo_chunk(comp): comp.SetAttrs(attrs) + + +def get_next_version_folder(folder): + """Format a version folder based on the filepath + + Assumption here is made that, if the path does not exists the folder + will be "v001" + + Args: + folder: file path to a folder + + Returns: + str: new version folder name + """ + + version_int = 1 + if os.path.isdir(folder): + re_version = re.compile("v\d+") + versions = [i for i in os.listdir(folder) if re_version.match(i) + and os.path.isdir(os.path.join(folder, i))] + if versions: + # ensure the "v" is not included and convert to ints + int_versions = [int(v[1:]) for v in versions] + version_int += max(int_versions) + + return "v{:03d}".format(version_int) + + +def create_new_filepath(session): + + project = session["AVALON_PROJECT"] + asset = session["AVALON_ASSET"] + + # Save updated slap comp + template = self._project["config"]["template"]["work"] + template_work = pipeline._format_work_template(template, session) + + walk_to_dir = os.path.join(template_work, "scenes", "slapcomp") + slapcomp_dir = os.path.abspath(walk_to_dir) + + # Ensure destination exists + if not os.path.isdir(slapcomp_dir): + log.warning("Folder did not exist, creating folder structure") + os.makedirs(slapcomp_dir) + + # Compute output path + new_filename = "{}_{}_slapcomp_v001.comp".format(project, asset) + new_filepath = os.path.join(slapcomp_dir, new_filename) + + # Create new unqiue filepath + if os.path.exists(new_filepath): + new_filepath = colorbleed.version_up(new_filepath) + + return new_filepath + + +def update_savers(comp, session): + """Update all savers of the current comp to ensure the output is correct + + Args: + comp (object): current comp instance + session (dict): the current Avalon session + + Returns: + None + """ + + template = self._project["config"]["template"]["work"] + template_work = pipeline._format_work_template(template, session) + + render_dir = os.path.join(os.path.normpath(template_work), "renders") + version_folder = get_next_version_folder(render_dir) + renders_version = os.path.join(render_dir, version_folder) + + comp.Print("New renders to: %s\n" % render_dir) + + with avalon.fusion.comp_lock_and_undo_chunk(comp): + savers = comp.GetToolList(False, "Saver").values() + for saver in savers: + filepath = saver.GetAttrs("TOOLST_Clip_Name")[1.0] + filename = os.path.basename(filepath) + new_path = os.path.join(renders_version, filename) + saver["Clip"] = new_path + + +def switch(asset_name): + """Switch the current containers of the comp to the other asset (shot) + + Args: + asset_name (str): name of the asset (shot) + + Returns: + comp (PyObject): the comp instance + + """ + + assert asset_name, "Function requires asset name" + + host = api.registered_host() + assert host, "Host must be installed" + + # Get current project + self._project = io.find_one({"type": "project", + "name": api.Session["AVALON_PROJECT"]}) + + # Assert asset name exists + # It is better to do this here then to wait till switch_shot does it + asset = io.find_one({"type": "asset", "name": asset_name}) + assert asset, "Could not find '%s' in the database" % asset_name + + # Use the current open comp + current_comp = avalon.fusion.get_current_comp() + assert current_comp is not None, "Could not find current comp" + + containers = list(host.ls()) + assert containers, "Nothing to update" + + representations = [] + for container in containers: + try: + representation = colorbleed.switch_item(container, + asset_name=asset_name) + representations.append(representation) + log.debug(str(representation["_id"]) + "\n") + except Exception as e: + log.debug("Error in switching! %s\n" % e.message) + + log.info("Switched %i Loaders of the %i\n" % (len(representations), + len(containers))) + # Updating frame range + log.debug("\nUpdating frame range ..") + version_ids = [r["parent"] for r in representations] + versions = io.find({"type": "version", "_id": {"$in": version_ids}}) + versions = list(versions) + + start = min(v["data"]["startFrame"] for v in versions) + end = max(v["data"]["endFrame"] for v in versions) + + update_frame_range(start, end, comp=current_comp) + + # Build the session to switch to + switch_to_session = api.Session.copy() + switch_to_session["AVALON_ASSET"] = asset['name'] + + # Update session and environment + api.Session.update(switch_to_session) + os.environ.update(switch_to_session) + + self._project = None + + return current_comp From b0e8d44240d5388361a81c16dfb148ef9f70c6da Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 26 Mar 2018 14:04:52 +0200 Subject: [PATCH 35/81] Updated _on_switch, and cosmetics --- .../scripts/Comp/colorbleed/switch_ui.py | 45 +++++++------------ 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/setup/fusion/scripts/Comp/colorbleed/switch_ui.py b/setup/fusion/scripts/Comp/colorbleed/switch_ui.py index f5ab6dd3..40305bc9 100644 --- a/setup/fusion/scripts/Comp/colorbleed/switch_ui.py +++ b/setup/fusion/scripts/Comp/colorbleed/switch_ui.py @@ -1,16 +1,17 @@ import os -import site import glob import logging -import avalon.io as io -import avalon.api as api -import avalon.pipeline as pipeline +from avalon import io, api, pipeline import avalon.fusion + import avalon.style as style from avalon.vendor.Qt import QtWidgets, QtCore from avalon.vendor import qtawesome as qta +import colorbleed.lib as cblib +from colorbleed.fusion import flib + log = logging.getLogger("Fusion Switch Shot") @@ -147,36 +148,20 @@ def _refresh(self): def _on_switch(self): - def _get_script_dir(): - """Get path to the image sequence script""" - try: - import colorbleed - scriptdir = os.path.dirname(colorbleed.__file__) - fusion_scripts = os.path.join(scriptdir, - "scripts", - "fusion") - except: - raise RuntimeError("This is a bug") - - assert os.path.isdir(fusion_scripts), "Config is incomplete" - fusion_scripts = fusion_scripts.replace(os.sep, "/") - - return fusion_scripts - + asset = self._assets.currentText() + _comp = avalon.fusion.get_current_comp() if not self._use_current.isChecked(): file_name = self._comps.itemData(self._comps.currentIndex()) + # Get the current comp's App (FusionUI) + _fusion = _comp.GetApp() + # Open the selected comp + _fusion.LoadComp(file_name) else: - comp = avalon.fusion.get_current_comp() - file_name = comp.GetAttrs("COMPS_FileName") - - asset = self._assets.currentText() - - # Get switch and submit module - script_dir = _get_script_dir() - site.addsitedir(script_dir) + file_name = _comp.GetAttrs("COMPS_FileName") - import switch_and_submit as switch_shot - switch_shot.switch(asset_name=asset, file_path=file_name, new=True) + switched = flib.switch(asset) + new_filename = cblib.version_up(file_name) + switched.Save(new_filename) def _get_context_directory(self): From 75859b1dfe9c5fabd0ddda1e704cc1b6826a47b4 Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 26 Mar 2018 14:05:14 +0200 Subject: [PATCH 36/81] Moved switch logic to lib, module redundant --- .../scripts/fusion/switch_and_submit.py | 253 ------------------ 1 file changed, 253 deletions(-) delete mode 100644 colorbleed/scripts/fusion/switch_and_submit.py diff --git a/colorbleed/scripts/fusion/switch_and_submit.py b/colorbleed/scripts/fusion/switch_and_submit.py deleted file mode 100644 index 89e2f558..00000000 --- a/colorbleed/scripts/fusion/switch_and_submit.py +++ /dev/null @@ -1,253 +0,0 @@ -import os -import re -import sys -import logging - -# Pipeline imports -from avalon import api, io, pipeline -import avalon.fusion - -# Config imports -import colorbleed.lib as colorbleed -import colorbleed.fusion.lib as fusion_lib - -log = logging.getLogger("Update Slap Comp") - -self = sys.modules[__name__] -self._project = None - -error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" - - -def get_fusion_instance(): - """Try to get an instance of Fusion""" - fusion = getattr(sys.modules["__main__"], "fusion", None) - if fusion is None: - try: - # Support for FuScript.exe, BlackmagicFusion module for py2 only - import BlackmagicFusion as bmf - fusion = bmf.scriptapp("Fusion", "localhost", 2) - except ImportError: - raise RuntimeError("Could not find a Fusion instance") - return fusion - - -def format_version_folder(folder): - """Format a version folder based on the filepath - - Assumption here is made that, if the path does not exists the folder - will be "v001" - - Args: - folder: file path to a folder - - Returns: - str: new version folder name - """ - - version_int = 1 - if os.path.isdir(folder): - re_version = re.compile("v\d+") - versions = [i for i in os.listdir(folder) if re_version.match(i) - and os.path.isdir(os.path.join(folder, i))] - if versions: - # ensure the "v" is not included and convert to ints - int_versions = [int(v[1:]) for v in versions] - version_int += max(int_versions) - - return "v{:03d}".format(version_int) - - -def format_filepath(session): - - project = session["AVALON_PROJECT"] - asset = session["AVALON_ASSET"] - - # Save updated slap comp - work_path = get_work_folder(session) - walk_to_dir = os.path.join(work_path, "scenes", "slapcomp") - slapcomp_dir = os.path.abspath(walk_to_dir) - - # Ensure destination exists - if not os.path.isdir(slapcomp_dir): - log.warning("Folder did not exist, creating folder structure") - os.makedirs(slapcomp_dir) - - # Compute output path - new_filename = "{}_{}_slapcomp_v001.comp".format(project, asset) - new_filepath = os.path.join(slapcomp_dir, new_filename) - - # Create new unqiue filepath - if os.path.exists(new_filepath): - new_filepath = colorbleed.version_up(new_filepath) - - return new_filepath - - -def get_work_folder(session): - """Convenience function to get the work folder path of the current asset""" - - # Get new filename, create path based on asset and work template - template_work = self._project["config"]["template"]["work"] - work_path = pipeline._format_work_template(template_work, session) - - return os.path.normpath(work_path) - - -def update_savers(comp, session): - """Update all savers of the current comp to ensure the output is correct - - Args: - comp (object): current comp instance - session (dict): the current Avalon session - - Returns: - None - """ - - new_work = get_work_folder(session) - renders = os.path.join(new_work, "renders") - version_folder = format_version_folder(renders) - renders_version = os.path.join(renders, version_folder) - - comp.Print("New renders to: %s\n" % renders) - - with avalon.fusion.comp_lock_and_undo_chunk(comp): - savers = comp.GetToolList(False, "Saver").values() - for saver in savers: - filepath = saver.GetAttrs("TOOLST_Clip_Name")[1.0] - filename = os.path.basename(filepath) - new_path = os.path.join(renders_version, filename) - saver["Clip"] = new_path - - -def update_frame_range(comp, representations): - """Update the frame range of the comp and render length - - The start and end frame are based on the lowest start frame and the highest - end frame - - Args: - comp (object): current focused comp - representations (list) collection of dicts - - Returns: - None - - """ - - version_ids = [r["parent"] for r in representations] - versions = io.find({"type": "version", "_id": {"$in": version_ids}}) - versions = list(versions) - - start = min(v["data"]["startFrame"] for v in versions) - end = max(v["data"]["endFrame"] for v in versions) - - fusion_lib.update_frame_range(start, end, comp=comp) - - -def switch(file_path=None, asset_name=None, new=True, deadline=True): - """Switch the current containers of the file to the other asset (shot) - - Exitcode 1: No item found to publish - Exitcode 2: Errors during publish process - - Args: - file_path (str): file path of the comp file - asset_name (str): name of the asset (shot) - new (bool): Save updated comp under a different name - deadline (bool): if set to true - - Returns: - comp path (str): new filepath of the updated comp - - """ - - assert asset_name, "Function requires at least an asset name" - - # Ensure filename is absolute - if file_path: - file_path = os.path.abspath(file_path) - - api.install(avalon.fusion) - host = api.registered_host() - - # Get current project - self._project = io.find_one({"type": "project", - "name": api.Session["AVALON_PROJECT"]}) - - # Assert asset name exists - # It is better to do this here then to wait till switch_shot does it - asset = io.find_one({"type": "asset", "name": asset_name}) - assert asset, "Could not find '%s' in the database" % asset_name - - if not file_path: - # Assuming we use the current open comp - current_comp = avalon.fusion.get_current_comp() - assert current_comp is not None, "Could not find current comp" - file_path = current_comp.GetAttrs("COMPS_FileName") - else: - fusion = get_fusion_instance() - current_comp = fusion.LoadComp(file_path) - assert current_comp is not None, ("Fusion could not load '%s'" - % file_path) - - containers = list(host.ls()) - assert containers, "Nothing to update" - - representations = [] - for container in containers: - try: - representation = colorbleed.switch_item(container, - asset_name=asset_name) - representations.append(representation) - current_comp.Print(str(representation["_id"]) + "\n") - except Exception as e: - current_comp.Print("Error in switching! %s\n" % e.message) - - message = "Switched %i Loaders of the %i\n" % (len(representations), - len(containers)) - current_comp.Print(message) - - # Build the session to switch to - switch_to_session = api.Session.copy() - switch_to_session["AVALON_ASSET"] = asset['name'] - - if new: - comp_path = format_filepath(switch_to_session) - - # Update savers output based on new session - update_savers(current_comp, switch_to_session) - else: - comp_path = colorbleed.version_up(file_path) - - current_comp.Print("\nNew path: %s" % comp_path) - current_comp.Print("\nUpdating frame range ..") - update_frame_range(current_comp, representations) - - # Save changes - current_comp.Save(comp_path) - - if deadline: - # Update session with correct asset name and comp path - api.Session.update(switch_to_session) - os.environ.update(switch_to_session) - - # Set render mode to deadline - current_comp.SetData("colorbleed.rendermode", "deadline") - - # Run publisher - context = api.publish() - if not context: - log.warning("Nothing collected.") - sys.exit(1) - - # Collect errors, {plugin name: error}, if any - error_results = [r for r in context.data["results"] if r["error"]] - if error_results: - log.error("Errors occurred ...") - for result in error_results: - log.error(error_format.format(**result)) - sys.exit(2) - - return current_comp From 09911b2d1ca7ddcf3f003a43253b05ee2fb959a3 Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 26 Mar 2018 14:07:47 +0200 Subject: [PATCH 37/81] Updated switch logic, removed rendudant function, added submit function --- .../fusion/deadline_switch_and_submit.py | 74 ++++++++++--------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/colorbleed/scripts/fusion/deadline_switch_and_submit.py b/colorbleed/scripts/fusion/deadline_switch_and_submit.py index ef13bdac..3b6008bd 100644 --- a/colorbleed/scripts/fusion/deadline_switch_and_submit.py +++ b/colorbleed/scripts/fusion/deadline_switch_and_submit.py @@ -33,14 +33,15 @@ """ - import subprocess import traceback -import site +import logging import time import sys import os +log = logging.getLogger(__name__) + # This script only works with Python 2.7 and 3.6 version = "{0}{1}".format(*sys.version_info) # {major}{minor} assert version in ["27", "36"], "Script only works in Python 2.7 or 3.6" @@ -56,24 +57,13 @@ FUSCRIPT_EXE = r"C:/Program Files/Blackmagic Design/Fusion9/FuScript.exe" FUSION_CONSOLE_EXE = r"C:/Program Files/Blackmagic Design/Fusion Render Node 9/FusionConsoleNode.exe" -import BlackmagicFusion as bmf +# Pipeline and config imports +import avalon.fusion +from avalon import api +import colorbleed.fusion.lib as fusionlib - -def _get_script_dir(): - """Get path to the image sequence script""" - try: - import colorbleed - scriptdir = os.path.dirname(colorbleed.__file__) - fusion_scripts = os.path.join(scriptdir, - "scripts", - "fusion") - except: - raise RuntimeError("This is a bug") - - assert os.path.isdir(fusion_scripts), "Config is incomplete" - fusion_scripts = fusion_scripts.replace(os.sep, "/") - - return fusion_scripts +# Application related imports +import BlackmagicFusion as bmf def start_server(): @@ -117,6 +107,31 @@ def get_fusion_instance(pid, srv, timeout=10): return bmf.scriptapp(host["Name"], "localhost", 2, host["UUID"]) +def submit(current_comp): + """Set rendermode to deadline and publish / submit comp""" + + # Set comp render mode to deadline + current_comp.SetData("colorbleed.rendermode", "deadline") + + error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" + + # Publish + context = api.publish() + if not context: + log.warning("Nothing collected.") + sys.exit(1) + + # Collect errors, {plugin name: error}, if any + error_results = [r for r in context.data["results"] if r["error"]] + if error_results: + log.error("Errors occurred ...") + for result in error_results: + log.error(error_format.format(**result)) + raise RuntimeError + + return True + + def process(file_path, asset_name, deadline=False): """Run switch in a Fusion Console Node (cmd) @@ -139,16 +154,16 @@ def process(file_path, asset_name, deadline=False): srv = get_server() if not srv: - print("No server found, starting server ..") + log.info("No server found, starting server ..") srv = start_server() # Force fusion into main magical module so that host.ls() works fusion = get_fusion_instance(proc.pid, srv) assert fusion - print("Connected to: %s" % fusion) + log.info("Connected to: %s" % fusion) setattr(sys.modules["__main__"], "fusion", fusion) - # Get fusion.pipeline + api.install(avalon.fusion) from avalon.fusion import pipeline # This does not set @@ -159,25 +174,18 @@ def process(file_path, asset_name, deadline=False): current_comp = pipeline.get_current_comp() assert loaded_comp == current_comp, "Could not find the correct comp" - print("Loaded comp name: %s" % current_comp.GetAttrs("COMPS_FileName")) - # Get switch and submit script - scriptdir = _get_script_dir() - site.addsitedir(scriptdir) - import switch_and_submit as switch - - # Fusion host try: # Execute script in comp - result = switch.switch(asset_name=asset_name, deadline=deadline) + fusionlib.switch(asset_name=asset_name) + result = True if not deadline else submit(current_comp) except: - tb = traceback.format_exc() pipeline.set_current_comp(None) proc.terminate() # Ensure process closes when failing - raise RuntimeError(tb) + raise RuntimeError(traceback.format_exc()) # detailed traceback - print("Success:", result is not None) + print("Success:", result) print("Closing all running process ..") pipeline.set_current_comp(None) proc.terminate() From 0f8f03d7de6987b11453209131eb118add2177dc Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 26 Mar 2018 16:12:39 +0200 Subject: [PATCH 38/81] renamed variable --- setup/fusion/scripts/Comp/colorbleed/switch_ui.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/fusion/scripts/Comp/colorbleed/switch_ui.py b/setup/fusion/scripts/Comp/colorbleed/switch_ui.py index 40305bc9..6a1d844f 100644 --- a/setup/fusion/scripts/Comp/colorbleed/switch_ui.py +++ b/setup/fusion/scripts/Comp/colorbleed/switch_ui.py @@ -159,9 +159,9 @@ def _on_switch(self): else: file_name = _comp.GetAttrs("COMPS_FileName") - switched = flib.switch(asset) + switched_comp = flib.switch(asset) new_filename = cblib.version_up(file_name) - switched.Save(new_filename) + switched_comp.Save(new_filename) def _get_context_directory(self): From d5eacb2e8ef250ea768a20d44bfca6d33aebc3aa Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 26 Mar 2018 16:41:58 +0200 Subject: [PATCH 39/81] improved error handling, added create_new_filepath --- .../fusion/deadline_switch_and_submit.py | 69 +++++++++++++++---- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/colorbleed/scripts/fusion/deadline_switch_and_submit.py b/colorbleed/scripts/fusion/deadline_switch_and_submit.py index 3b6008bd..27009e4f 100644 --- a/colorbleed/scripts/fusion/deadline_switch_and_submit.py +++ b/colorbleed/scripts/fusion/deadline_switch_and_submit.py @@ -59,7 +59,9 @@ # Pipeline and config imports import avalon.fusion -from avalon import api +from avalon import io, api, pipeline + +import colorbleed.lib as cblib import colorbleed.fusion.lib as fusionlib # Application related imports @@ -107,6 +109,45 @@ def get_fusion_instance(pid, srv, timeout=10): return bmf.scriptapp(host["Name"], "localhost", 2, host["UUID"]) +def create_new_filepath(session): + """ + Create a new fil epath based on the session and the project's template + + Args: + session (dict): the Avalon session + + Returns: + file path (str) + + """ + + # Save updated slap comp + project = io.find_one({"type": "project", + "name": session["AVALON_PROJECT"]}) + + template = project["config"]["template"]["work"] + template_work = pipeline._format_work_template(template, session) + + walk_to_dir = os.path.join(template_work, "scenes", "slapcomp") + slapcomp_dir = os.path.abspath(walk_to_dir) + + # Ensure destination exists + if not os.path.isdir(slapcomp_dir): + log.warning("Folder did not exist, creating folder structure") + os.makedirs(slapcomp_dir) + + # Compute output path + new_filename = "{}_{}_slapcomp_v001.comp".format(session["AVALON_PROJECT"], + session["AVALON_ASSET"]) + new_filepath = os.path.join(slapcomp_dir, new_filename) + + # Create new unqiue filepath + if os.path.exists(new_filepath): + new_filepath = cblib.version_up(new_filepath) + + return new_filepath + + def submit(current_comp): """Set rendermode to deadline and publish / submit comp""" @@ -118,16 +159,14 @@ def submit(current_comp): # Publish context = api.publish() if not context: - log.warning("Nothing collected.") - sys.exit(1) + raise RuntimeError("Nothing collected for publish") # Collect errors, {plugin name: error}, if any error_results = [r for r in context.data["results"] if r["error"]] if error_results: - log.error("Errors occurred ...") for result in error_results: log.error(error_format.format(**result)) - raise RuntimeError + raise RuntimeError("Errors occured") return True @@ -170,6 +209,7 @@ def process(file_path, asset_name, deadline=False): loaded_comp = fusion.LoadComp(file_path) if not loaded_comp: raise RuntimeError("Comp could not be loaded. File '%s'" % file_path) + pipeline.set_current_comp(loaded_comp) current_comp = pipeline.get_current_comp() @@ -178,17 +218,18 @@ def process(file_path, asset_name, deadline=False): try: # Execute script in comp - fusionlib.switch(asset_name=asset_name) - result = True if not deadline else submit(current_comp) - except: + fusionlib.switch(asset_name=asset_name, new=deadline) + new_file_path = create_new_filepath(api.Session) + current_comp.Save(new_file_path) + if deadline: + submit(current_comp) + except Exception: + print(traceback.format_exc()) # ensure detailed traceback + raise + finally: pipeline.set_current_comp(None) + print("Closing running process ..") proc.terminate() # Ensure process closes when failing - raise RuntimeError(traceback.format_exc()) # detailed traceback - - print("Success:", result) - print("Closing all running process ..") - pipeline.set_current_comp(None) - proc.terminate() # Usability for deadline job submission From 6d9c0568d0fc5c64ce3fbc6a4812c17d6e461453 Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 26 Mar 2018 16:42:47 +0200 Subject: [PATCH 40/81] Removed the self variable, remvoed create_new_filepath --- colorbleed/fusion/lib.py | 42 ++++++---------------------------------- 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/colorbleed/fusion/lib.py b/colorbleed/fusion/lib.py index ef75db51..7f7d8d2d 100644 --- a/colorbleed/fusion/lib.py +++ b/colorbleed/fusion/lib.py @@ -1,6 +1,5 @@ import os import re -import sys import logging import avalon.fusion @@ -8,8 +7,6 @@ import colorbleed.lib as colorbleed -self = sys.modules[__name__] -self._project = None log = logging.getLogger(__name__) @@ -73,46 +70,19 @@ def get_next_version_folder(folder): return "v{:03d}".format(version_int) -def create_new_filepath(session): - - project = session["AVALON_PROJECT"] - asset = session["AVALON_ASSET"] - - # Save updated slap comp - template = self._project["config"]["template"]["work"] - template_work = pipeline._format_work_template(template, session) - - walk_to_dir = os.path.join(template_work, "scenes", "slapcomp") - slapcomp_dir = os.path.abspath(walk_to_dir) - - # Ensure destination exists - if not os.path.isdir(slapcomp_dir): - log.warning("Folder did not exist, creating folder structure") - os.makedirs(slapcomp_dir) - - # Compute output path - new_filename = "{}_{}_slapcomp_v001.comp".format(project, asset) - new_filepath = os.path.join(slapcomp_dir, new_filename) - - # Create new unqiue filepath - if os.path.exists(new_filepath): - new_filepath = colorbleed.version_up(new_filepath) - - return new_filepath - - -def update_savers(comp, session): +def update_savers(comp, session, project): """Update all savers of the current comp to ensure the output is correct Args: comp (object): current comp instance session (dict): the current Avalon session + project (dict): the project document from the database Returns: None """ - template = self._project["config"]["template"]["work"] + template = project["config"]["template"]["work"] template_work = pipeline._format_work_template(template, session) render_dir = os.path.join(os.path.normpath(template_work), "renders") @@ -147,8 +117,8 @@ def switch(asset_name): assert host, "Host must be installed" # Get current project - self._project = io.find_one({"type": "project", - "name": api.Session["AVALON_PROJECT"]}) + project = io.find_one({"type": "project", + "name": api.Session["AVALON_PROJECT"]}) # Assert asset name exists # It is better to do this here then to wait till switch_shot does it @@ -193,6 +163,6 @@ def switch(asset_name): api.Session.update(switch_to_session) os.environ.update(switch_to_session) - self._project = None + update_savers(current_comp, switch_to_session, project) return current_comp From a587b422e7d01be368c3a01ab4c39231190e1412 Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 26 Mar 2018 17:00:39 +0200 Subject: [PATCH 41/81] Fixed import error, added file filter in brows --- setup/fusion/scripts/Comp/colorbleed/switch_ui.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/fusion/scripts/Comp/colorbleed/switch_ui.py b/setup/fusion/scripts/Comp/colorbleed/switch_ui.py index 6a1d844f..bb88185f 100644 --- a/setup/fusion/scripts/Comp/colorbleed/switch_ui.py +++ b/setup/fusion/scripts/Comp/colorbleed/switch_ui.py @@ -10,7 +10,7 @@ from avalon.vendor import qtawesome as qta import colorbleed.lib as cblib -from colorbleed.fusion import flib +import colorbleed.fusion.lib as flib log = logging.getLogger("Fusion Switch Shot") @@ -127,7 +127,7 @@ def _on_open_from_dir(self): start_dir = self._get_context_directory() comp_file, _ = QtWidgets.QFileDialog.getOpenFileName( - self, "Choose comp", start_dir) + self, "Choose comp", start_dir, "*.comp") if not comp_file: return From 887ec7acdfe480a64d6b5146296e463150eda22e Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 26 Mar 2018 17:02:13 +0200 Subject: [PATCH 42/81] Removed unused argument --- colorbleed/scripts/fusion/deadline_switch_and_submit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/colorbleed/scripts/fusion/deadline_switch_and_submit.py b/colorbleed/scripts/fusion/deadline_switch_and_submit.py index 27009e4f..5b68c4d2 100644 --- a/colorbleed/scripts/fusion/deadline_switch_and_submit.py +++ b/colorbleed/scripts/fusion/deadline_switch_and_submit.py @@ -218,7 +218,7 @@ def process(file_path, asset_name, deadline=False): try: # Execute script in comp - fusionlib.switch(asset_name=asset_name, new=deadline) + fusionlib.switch(asset_name=asset_name) new_file_path = create_new_filepath(api.Session) current_comp.Save(new_file_path) if deadline: From a51c350c70824e6614f2d7f767b7a2065a508fe2 Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 26 Mar 2018 18:12:02 +0200 Subject: [PATCH 43/81] Raise error if file is invalid --- setup/fusion/scripts/Comp/colorbleed/switch_ui.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/fusion/scripts/Comp/colorbleed/switch_ui.py b/setup/fusion/scripts/Comp/colorbleed/switch_ui.py index bb88185f..2d060abb 100644 --- a/setup/fusion/scripts/Comp/colorbleed/switch_ui.py +++ b/setup/fusion/scripts/Comp/colorbleed/switch_ui.py @@ -155,7 +155,9 @@ def _on_switch(self): # Get the current comp's App (FusionUI) _fusion = _comp.GetApp() # Open the selected comp - _fusion.LoadComp(file_name) + loaded_comp = _fusion.LoadComp(file_name) + if not loaded_comp: + raise RuntimeError("Invalid file: '%s'" % file_name) else: file_name = _comp.GetAttrs("COMPS_FileName") From 155235e4197bc25606fbb441da36715815237561 Mon Sep 17 00:00:00 2001 From: wijnand Date: Tue, 27 Mar 2018 11:32:39 +0200 Subject: [PATCH 44/81] Fixed get_script, retrieve suspendPublishJob correct --- .../plugins/publish/submit_publish_job.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/colorbleed/plugins/publish/submit_publish_job.py b/colorbleed/plugins/publish/submit_publish_job.py index a89e1918..422781b5 100644 --- a/colorbleed/plugins/publish/submit_publish_job.py +++ b/colorbleed/plugins/publish/submit_publish_job.py @@ -10,17 +10,15 @@ def _get_script(): """Get path to the image sequence script""" - try: - from colorbleed.scripts import publish_filesequence - except Exception as e: - raise RuntimeError("Expected module 'publish_imagesequence'" - "to be available") - module_path = publish_filesequence.__file__ - if module_path.endswith(".pyc"): - module_path = module_path[:-len(".pyc")] + ".py" + import colorbleed + configpath = os.path.dirname(colorbleed.__file__) + script = os.path.join(configpath, "scripts", "publish_filesequence.py") + # verify script exists + assert os.path.isfile(script), ("Script `publish_filesequence.py` does not" + "exists") - return module_path + return script class SubmitDependentImageSequenceJobDeadline(pyblish.api.InstancePlugin): @@ -70,7 +68,7 @@ def process(self, instance): "submission prior to this plug-in.") subset = instance.data["subset"] - state = instance.data.get("publishJobState", "Suspended") + state = instance.data.get("suspendPublishJob", "Active") job_name = "{batch} - {subset} [publish image sequence]".format( batch=job["Props"]["Name"], subset=subset From 5b2708f909eec6b08c981c6df7da28599a017c5f Mon Sep 17 00:00:00 2001 From: wijnand Date: Tue, 27 Mar 2018 11:33:23 +0200 Subject: [PATCH 45/81] Set initialStatus active as it has a dependency, minor cosmetics --- colorbleed/plugins/maya/publish/submit_switch_job.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/colorbleed/plugins/maya/publish/submit_switch_job.py b/colorbleed/plugins/maya/publish/submit_switch_job.py index 9bbb7993..1775c62b 100644 --- a/colorbleed/plugins/maya/publish/submit_switch_job.py +++ b/colorbleed/plugins/maya/publish/submit_switch_job.py @@ -12,9 +12,7 @@ def _get_script_dir(): try: import colorbleed scriptdir = os.path.dirname(colorbleed.__file__) - fusion_scripts = os.path.join(scriptdir, - "scripts", - "fusion") + fusion_scripts = os.path.join(scriptdir, "scripts", "fusion") except: raise RuntimeError("This is a bug") @@ -71,7 +69,7 @@ def process(self, context): "JobDependency0": job["_id"], "UserName": job["Props"]["User"], "Comment": comment, - "InitialStatus": "Suspended"}, + "InitialStatus": "Active"}, # Set job pending "PluginInfo": { "Version": "3.6", "ScriptFile": scriptfile, From 5a8d93f1d3fe24307c199e844e689cc563df8600 Mon Sep 17 00:00:00 2001 From: wijnand Date: Wed, 28 Mar 2018 16:26:08 +0200 Subject: [PATCH 46/81] Added docstrings, removed slapcomp from new filename --- .../fusion/deadline_switch_and_submit.py | 38 ++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/colorbleed/scripts/fusion/deadline_switch_and_submit.py b/colorbleed/scripts/fusion/deadline_switch_and_submit.py index 5b68c4d2..9b50ff78 100644 --- a/colorbleed/scripts/fusion/deadline_switch_and_submit.py +++ b/colorbleed/scripts/fusion/deadline_switch_and_submit.py @@ -74,6 +74,15 @@ def start_server(): def get_server(tries=10, timeout=0.5): + """Retrieve the server instance + Args: + tries (int): number of attempts + timeout (float): how long the function needs to time out in between + attempts + + Return: + server (PyObject) + """ count = 0 srv = None @@ -89,7 +98,18 @@ def get_server(tries=10, timeout=0.5): def get_fusion_instance(pid, srv, timeout=10): - """Get the fusion instance which has been launched""" + """Get the fusion instance which has been launched + + Args: + pid (int, long): process id + srv (PyObject): fusion server instance + timeout (float): how long the function needs to time out in between + attempts + + Returns: + fusion + + """ count = 0 host = None @@ -111,7 +131,7 @@ def get_fusion_instance(pid, srv, timeout=10): def create_new_filepath(session): """ - Create a new fil epath based on the session and the project's template + Create a new file path based on the session and the project's template Args: session (dict): the Avalon session @@ -137,8 +157,8 @@ def create_new_filepath(session): os.makedirs(slapcomp_dir) # Compute output path - new_filename = "{}_{}_slapcomp_v001.comp".format(session["AVALON_PROJECT"], - session["AVALON_ASSET"]) + new_filename = "{}_{}_v001.comp".format(session["AVALON_PROJECT"], + session["AVALON_ASSET"]) new_filepath = os.path.join(slapcomp_dir, new_filename) # Create new unqiue filepath @@ -149,7 +169,15 @@ def create_new_filepath(session): def submit(current_comp): - """Set rendermode to deadline and publish / submit comp""" + """Set rende rmode to deadline and publish / submit comp + + Args: + current_comp (PyObject): current comp instance + + Returns: + bool (True) + + """ # Set comp render mode to deadline current_comp.SetData("colorbleed.rendermode", "deadline") From 290fa33a6f85d508642afd90ec31de726e7a1fda Mon Sep 17 00:00:00 2001 From: wijnand Date: Wed, 28 Mar 2018 16:47:21 +0200 Subject: [PATCH 47/81] refactored out "slapcomp", fixed typo --- .../scripts/fusion/deadline_switch_and_submit.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/colorbleed/scripts/fusion/deadline_switch_and_submit.py b/colorbleed/scripts/fusion/deadline_switch_and_submit.py index 9b50ff78..7338c9cc 100644 --- a/colorbleed/scripts/fusion/deadline_switch_and_submit.py +++ b/colorbleed/scripts/fusion/deadline_switch_and_submit.py @@ -148,18 +148,17 @@ def create_new_filepath(session): template = project["config"]["template"]["work"] template_work = pipeline._format_work_template(template, session) - walk_to_dir = os.path.join(template_work, "scenes", "slapcomp") - slapcomp_dir = os.path.abspath(walk_to_dir) + comp_dir = os.path.abspath(os.path.join(template_work, "scenes")) # Ensure destination exists - if not os.path.isdir(slapcomp_dir): + if not os.path.isdir(comp_dir): log.warning("Folder did not exist, creating folder structure") - os.makedirs(slapcomp_dir) + os.makedirs(comp_dir) # Compute output path new_filename = "{}_{}_v001.comp".format(session["AVALON_PROJECT"], session["AVALON_ASSET"]) - new_filepath = os.path.join(slapcomp_dir, new_filename) + new_filepath = os.path.join(comp_dir, new_filename) # Create new unqiue filepath if os.path.exists(new_filepath): @@ -169,7 +168,7 @@ def create_new_filepath(session): def submit(current_comp): - """Set rende rmode to deadline and publish / submit comp + """Set render mode to deadline and publish / submit comp Args: current_comp (PyObject): current comp instance From ab332327ea36f546ab1b00acc904bba33e06bf83 Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 29 Mar 2018 16:14:52 +0200 Subject: [PATCH 48/81] Added fusion 9 and fusion console node env jsons --- colorbleed/environments/fusion9.json | 29 ++++++++++++++++++ .../environments/fusionrconsolenode9.json | 30 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 colorbleed/environments/fusion9.json create mode 100644 colorbleed/environments/fusionrconsolenode9.json diff --git a/colorbleed/environments/fusion9.json b/colorbleed/environments/fusion9.json new file mode 100644 index 00000000..c81d5f59 --- /dev/null +++ b/colorbleed/environments/fusion9.json @@ -0,0 +1,29 @@ +{ + "FUSION_VERSION": "9", + "FUSION_LOCATION": { + "darwin": "/Applications/Blackmagic Design/Fusion ${FUSION_VERSION}/Fusion.app/Contents", + "linux": "/usr/blackmagic design/fusion ${FUSION_VERSION}", + "windows": "C:/Program Files/Blackmagic Design/Fusion ${FUSION_VERSION}" + }, + "PATH": { + "darwin": "${FUSION_LOCATION};{PATH}", + "linux": "${FUSION_LOCATION};{PATH}", + "windows": ["${FUSION_LOCATION}", "{PATH}"] + }, + + "FUSION_SHARED": "{CB_PIPELINE}/dev/apps/fusion_shared/{MAYA_VERSION}", + + "PYTHONPATH": [ + "{CB_PIPELINE}/2.1_dev/git/scriptsmenu/python", + "{CB_PIPELINE}/2.1_dev/git/maya-look-assigner", + "{CB_PIPELINE}/2.1_dev/git/maya_deadline_submission_settings/python", + "{CB_PIPELINE}/dev/git/maya-capture", + "{CB_PIPELINE}/dev/git/maya-capture-gui", + "{CB_PIPELINE}/dev/git/maya-capture-gui-cb", + "{CB_PIPELINE}/dev/apps/maya_shared/2018/scripts", + "{AVALON_CORE}/setup/maya", + "{PYTHONPATH}" + ], + + "FFMPEG_PATH": "{CB_PIPELINE}/dev/apps/ffmpeg/bin/ffmpeg.exe", +} \ No newline at end of file diff --git a/colorbleed/environments/fusionrconsolenode9.json b/colorbleed/environments/fusionrconsolenode9.json new file mode 100644 index 00000000..b8da1a30 --- /dev/null +++ b/colorbleed/environments/fusionrconsolenode9.json @@ -0,0 +1,30 @@ +{ + "FUSION_VERSION": "9", + "FUSION_LOCATION": { + "darwin": "/Applications/Blackmagic Design/Fusion ${FUSION_VERSION}/Fusion.app/Contents", + "linux": "/usr/blackmagic design/fusion ${FUSION_VERSION}", + "windows": "C:/Program Files/Blackmagic Design/Fusion ${FUSION_VERSION}" + }, + "PATH": { + "darwin": "${FUSION_LOCATION};{PATH}", + "linux": "${FUSION_LOCATION};{PATH}", + "windows": ["${FUSION_LOCATION}", "{PATH}"] + }, + + "FUSION9_MasterPrefs": "{CB_PIPELINE}/dev/apps/fusion_shared/{FUSION_VERSION}.0./fusion_shared.prefs", + + "OFX_PLUGIN_PATH" :[ + "P:/pipeline/dev/apps/ofx", + "P:/pipeline/dev/apps/ofx/Plugins", + ], + + "PYTHONPATH": [ + "P:/pipeline/dev/git/fusionless", + "P:/pipeline/dev/git/Qt.py", + "P:/pipeline/dev/apps/python/standalone/2.7/site-packages", + "{AVALON_CORE}/setup/fusion", + "{PYTHONPATH}" + ], + + "FFMPEG_PATH": "{CB_PIPELINE}/dev/apps/ffmpeg/bin/ffmpeg.exe", +} \ No newline at end of file From e22f95170edf3cf385f9e9e149d2bcbd3d520164 Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 29 Mar 2018 16:15:39 +0200 Subject: [PATCH 49/81] update environment through avalon.api --- colorbleed/fusion/lib.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/colorbleed/fusion/lib.py b/colorbleed/fusion/lib.py index 7f7d8d2d..3a6de9dd 100644 --- a/colorbleed/fusion/lib.py +++ b/colorbleed/fusion/lib.py @@ -156,13 +156,15 @@ def switch(asset_name): update_frame_range(start, end, comp=current_comp) # Build the session to switch to - switch_to_session = api.Session.copy() - switch_to_session["AVALON_ASSET"] = asset['name'] + api.update_current_task(task="comp", asset=asset_name, app="fusion") - # Update session and environment - api.Session.update(switch_to_session) - os.environ.update(switch_to_session) + # switch_to_session = api.Session.copy() - update_savers(current_comp, switch_to_session, project) + import pprint + print("START DEBUG") + pprint.pprint(api.Session) + print("END DEBUG") + + update_savers(current_comp, api.Session, project) return current_comp From 02013339b9bd86a73379cd53cc3908dae246eddb Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 29 Mar 2018 16:15:58 +0200 Subject: [PATCH 50/81] Removed debug prints --- colorbleed/fusion/lib.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/colorbleed/fusion/lib.py b/colorbleed/fusion/lib.py index 3a6de9dd..69e40fb3 100644 --- a/colorbleed/fusion/lib.py +++ b/colorbleed/fusion/lib.py @@ -158,13 +158,6 @@ def switch(asset_name): # Build the session to switch to api.update_current_task(task="comp", asset=asset_name, app="fusion") - # switch_to_session = api.Session.copy() - - import pprint - print("START DEBUG") - pprint.pprint(api.Session) - print("END DEBUG") - update_savers(current_comp, api.Session, project) return current_comp From 040853947e86c689add8434712859c6573f9036d Mon Sep 17 00:00:00 2001 From: wijnand Date: Thu, 29 Mar 2018 16:18:18 +0200 Subject: [PATCH 51/81] Added comment to job --- .../fusion/deadline_switch_and_submit.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/colorbleed/scripts/fusion/deadline_switch_and_submit.py b/colorbleed/scripts/fusion/deadline_switch_and_submit.py index 7338c9cc..dfa8ef0c 100644 --- a/colorbleed/scripts/fusion/deadline_switch_and_submit.py +++ b/colorbleed/scripts/fusion/deadline_switch_and_submit.py @@ -134,7 +134,7 @@ def create_new_filepath(session): Create a new file path based on the session and the project's template Args: - session (dict): the Avalon session + session (dict): a copy of the Avalon session Returns: file path (str) @@ -167,29 +167,36 @@ def create_new_filepath(session): return new_filepath -def submit(current_comp): +def submit(current_comp, source=""): """Set render mode to deadline and publish / submit comp Args: + source (str): file path current_comp (PyObject): current comp instance Returns: bool (True) """ + import pyblish + from pyblish import util # Set comp render mode to deadline current_comp.SetData("colorbleed.rendermode", "deadline") error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" + comment = "slapped using: {}".format(os.path.basename(source)) # Publish - context = api.publish() - if not context: + context = pyblish.api.Context() + context.data["comment"] = comment + + results = util.publish(context=context) + if not results: raise RuntimeError("Nothing collected for publish") # Collect errors, {plugin name: error}, if any - error_results = [r for r in context.data["results"] if r["error"]] + error_results = [r for r in results.data["results"] if r["error"]] if error_results: for result in error_results: log.error(error_format.format(**result)) @@ -249,7 +256,7 @@ def process(file_path, asset_name, deadline=False): new_file_path = create_new_filepath(api.Session) current_comp.Save(new_file_path) if deadline: - submit(current_comp) + submit(current_comp, source=file_path) except Exception: print(traceback.format_exc()) # ensure detailed traceback raise From ff8f3f755278c376321450c96174d33482ff6848 Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 30 Mar 2018 14:03:14 +0200 Subject: [PATCH 52/81] Moved api.update_current_task to script --- colorbleed/fusion/lib.py | 4 ---- colorbleed/scripts/fusion/deadline_switch_and_submit.py | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/colorbleed/fusion/lib.py b/colorbleed/fusion/lib.py index 69e40fb3..fe72f45d 100644 --- a/colorbleed/fusion/lib.py +++ b/colorbleed/fusion/lib.py @@ -154,10 +154,6 @@ def switch(asset_name): end = max(v["data"]["endFrame"] for v in versions) update_frame_range(start, end, comp=current_comp) - - # Build the session to switch to - api.update_current_task(task="comp", asset=asset_name, app="fusion") - update_savers(current_comp, api.Session, project) return current_comp diff --git a/colorbleed/scripts/fusion/deadline_switch_and_submit.py b/colorbleed/scripts/fusion/deadline_switch_and_submit.py index dfa8ef0c..894175bb 100644 --- a/colorbleed/scripts/fusion/deadline_switch_and_submit.py +++ b/colorbleed/scripts/fusion/deadline_switch_and_submit.py @@ -222,6 +222,9 @@ def process(file_path, asset_name, deadline=False): """ + # Build the session to switch to + api.update_current_task(task="comp", asset=asset_name, app="fusion") + # Start a fusion console node in "listen" mode proc = subprocess.Popen([FUSION_CONSOLE_EXE, "/listen"]) From 1a8c8d98e384af918df803f5fdbb989341253ecf Mon Sep 17 00:00:00 2001 From: wijnand Date: Wed, 4 Apr 2018 18:06:46 +0200 Subject: [PATCH 53/81] Script to override GlobalJobPreLoad of Deadline --- colorbleed/scripts/GlobalJobPreLoad.py | 42 ++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 colorbleed/scripts/GlobalJobPreLoad.py diff --git a/colorbleed/scripts/GlobalJobPreLoad.py b/colorbleed/scripts/GlobalJobPreLoad.py new file mode 100644 index 00000000..c7fb4cfd --- /dev/null +++ b/colorbleed/scripts/GlobalJobPreLoad.py @@ -0,0 +1,42 @@ +import os +import site + +# todo: refactore out the need of site +site.addsitedir(r"P:\pipeline\dev\git\env_prototype") + +try: + from env_prototype import api +except ImportError as exc: + # todo: nice error message + print(exc) + + +def __main__(deadlinePlugin): + deadlinePlugin.LogInfo("Setting up studio Environment!") + + # Compute + if "TOOL_ENV" not in os.environ: + print("Settings TOOL_ENV ..") + os.environ["TOOL_ENV"] = "P:/pipeline/dev/environments" + + tools = deadlinePlugin.GetProcessEnvironmentVariable("AVALON_TOOLS") + if not tools: + deadlinePlugin.LogInfo( + "Cannot set studio Environment without `AVALON_TOOLS`") + return + + deadlinePlugin.LogInfo("Setting environment for tools: %s" % tools) + + tools_env = api.get_tools(tools.split(";")) + env = api.compute(tools_env) + + # Get the merged environment for the local machine + merged = api.merge(env, current_env=os.environ.copy()) + + # Keep only the changed variables + env = {key: value for key, value in merged.items() if key in env} + + # Update + for key, value in sorted(env.items()): + deadlinePlugin.LogInfo("\t%s: %s" % (key, value)) + deadlinePlugin.SetProcessEnvironmentVariable(key, value) From 0125fc0b94f686c6ee71361db90c606d033839cf Mon Sep 17 00:00:00 2001 From: wijnand Date: Wed, 4 Apr 2018 18:08:33 +0200 Subject: [PATCH 54/81] Removed environment folder and jsons --- colorbleed/environments/fusion9.json | 29 ---------- .../environments/fusionrconsolenode9.json | 30 ---------- colorbleed/environments/global.json | 9 --- colorbleed/environments/maya2018.json | 58 ------------------- colorbleed/environments/yeti.2.2.4.json | 28 --------- 5 files changed, 154 deletions(-) delete mode 100644 colorbleed/environments/fusion9.json delete mode 100644 colorbleed/environments/fusionrconsolenode9.json delete mode 100644 colorbleed/environments/global.json delete mode 100644 colorbleed/environments/maya2018.json delete mode 100644 colorbleed/environments/yeti.2.2.4.json diff --git a/colorbleed/environments/fusion9.json b/colorbleed/environments/fusion9.json deleted file mode 100644 index c81d5f59..00000000 --- a/colorbleed/environments/fusion9.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "FUSION_VERSION": "9", - "FUSION_LOCATION": { - "darwin": "/Applications/Blackmagic Design/Fusion ${FUSION_VERSION}/Fusion.app/Contents", - "linux": "/usr/blackmagic design/fusion ${FUSION_VERSION}", - "windows": "C:/Program Files/Blackmagic Design/Fusion ${FUSION_VERSION}" - }, - "PATH": { - "darwin": "${FUSION_LOCATION};{PATH}", - "linux": "${FUSION_LOCATION};{PATH}", - "windows": ["${FUSION_LOCATION}", "{PATH}"] - }, - - "FUSION_SHARED": "{CB_PIPELINE}/dev/apps/fusion_shared/{MAYA_VERSION}", - - "PYTHONPATH": [ - "{CB_PIPELINE}/2.1_dev/git/scriptsmenu/python", - "{CB_PIPELINE}/2.1_dev/git/maya-look-assigner", - "{CB_PIPELINE}/2.1_dev/git/maya_deadline_submission_settings/python", - "{CB_PIPELINE}/dev/git/maya-capture", - "{CB_PIPELINE}/dev/git/maya-capture-gui", - "{CB_PIPELINE}/dev/git/maya-capture-gui-cb", - "{CB_PIPELINE}/dev/apps/maya_shared/2018/scripts", - "{AVALON_CORE}/setup/maya", - "{PYTHONPATH}" - ], - - "FFMPEG_PATH": "{CB_PIPELINE}/dev/apps/ffmpeg/bin/ffmpeg.exe", -} \ No newline at end of file diff --git a/colorbleed/environments/fusionrconsolenode9.json b/colorbleed/environments/fusionrconsolenode9.json deleted file mode 100644 index b8da1a30..00000000 --- a/colorbleed/environments/fusionrconsolenode9.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "FUSION_VERSION": "9", - "FUSION_LOCATION": { - "darwin": "/Applications/Blackmagic Design/Fusion ${FUSION_VERSION}/Fusion.app/Contents", - "linux": "/usr/blackmagic design/fusion ${FUSION_VERSION}", - "windows": "C:/Program Files/Blackmagic Design/Fusion ${FUSION_VERSION}" - }, - "PATH": { - "darwin": "${FUSION_LOCATION};{PATH}", - "linux": "${FUSION_LOCATION};{PATH}", - "windows": ["${FUSION_LOCATION}", "{PATH}"] - }, - - "FUSION9_MasterPrefs": "{CB_PIPELINE}/dev/apps/fusion_shared/{FUSION_VERSION}.0./fusion_shared.prefs", - - "OFX_PLUGIN_PATH" :[ - "P:/pipeline/dev/apps/ofx", - "P:/pipeline/dev/apps/ofx/Plugins", - ], - - "PYTHONPATH": [ - "P:/pipeline/dev/git/fusionless", - "P:/pipeline/dev/git/Qt.py", - "P:/pipeline/dev/apps/python/standalone/2.7/site-packages", - "{AVALON_CORE}/setup/fusion", - "{PYTHONPATH}" - ], - - "FFMPEG_PATH": "{CB_PIPELINE}/dev/apps/ffmpeg/bin/ffmpeg.exe", -} \ No newline at end of file diff --git a/colorbleed/environments/global.json b/colorbleed/environments/global.json deleted file mode 100644 index ea519691..00000000 --- a/colorbleed/environments/global.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "CB_PIPELINE": "P:/pipeline", - - "PYTHONPATH": [ - "{CB_PIPELINE}/dev/git/Qt.py", - "{CB_PIPELINE}/dev/git/cb", - "{PYTHONPATH}" - ] -} diff --git a/colorbleed/environments/maya2018.json b/colorbleed/environments/maya2018.json deleted file mode 100644 index 65fba760..00000000 --- a/colorbleed/environments/maya2018.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "MAYA_VERSION": "2018", - "MAYA_LOCATION": { - "darwin": "/Applications/Autodesk/maya${MAYA_VERSION}/Maya.app/Contents", - "linux": "/usr/autodesk/maya${MAYA_VERSION}", - "windows": "C:/Program Files/Autodesk/Maya${MAYA_VERSION}" - }, - "PATH": { - "darwin": "${MAYA_LOCATION}/bin;{PATH}", - "linux": "${MAYA_LOCATION}/bin;{PATH}", - "windows": [ - "${MAYA_LOCATION}/bin", - "C:/Program Files/Common Files/Autodesk Shared/", - "C:/Program Files (x86)/Autodesk/Backburner/", - "{PATH}" - ] - }, - "DYLD_LIBRARY_PATH": { - "darwin": "${MAYA_LOCATION}/MacOS" - }, - - "MAYA_DISABLE_CLIC_IPM": "Yes", - "MAYA_DISABLE_CIP": "Yes", - "MAYA_DISABLE_CER": "Yes", - "PYMEL_SKIP_MEL_INIT": "Yes", - "LC_ALL": "C", - - "CB_MAYA_SHARED": "{CB_PIPELINE}/dev/apps/maya_shared/{MAYA_VERSION}", - "MAYA_SCRIPT_PATH": [ - "{CB_MAYA_SHARED}/scripts", - "{MAYA_SCRIPT_PATH}" - ], - "MAYA_MODULE_PATH": [ - "{CB_MAYA_SHARED}/modules/cvshapeinverter", - "{MAYA_MODULE_PATH}" - ], - "XBMLANGPATH": [ - "{CB_MAYA_SHARED}/prefs/icons", - "{XBMLANGPATH}" - ], - - "PYTHONPATH": [ - "{CB_PIPELINE}/2.1_dev/git/scriptsmenu/python", - "{CB_PIPELINE}/2.1_dev/git/maya-look-assigner", - "{CB_PIPELINE}/2.1_dev/git/maya_deadline_submission_settings/python", - "{CB_PIPELINE}/dev/git/maya-capture", - "{CB_PIPELINE}/dev/git/maya-capture-gui", - "{CB_PIPELINE}/dev/git/maya-capture-gui-cb", - "{CB_PIPELINE}/dev/apps/maya_shared/2018/scripts", - "{AVALON_CORE}/setup/maya", - "{PYTHONPATH}" - ], - - "FFMPEG_PATH": "{CB_PIPELINE}/dev/apps/ffmpeg/bin/ffmpeg.exe", - "COLORBLEED_SCRIPTS": "{CB_PIPELINE}/dev/git/cbMayaScripts/cbMayaScripts", - - "VRAY_USE_THREAD_AFFINITY": "0" -} \ No newline at end of file diff --git a/colorbleed/environments/yeti.2.2.4.json b/colorbleed/environments/yeti.2.2.4.json deleted file mode 100644 index 0f3c4ba4..00000000 --- a/colorbleed/environments/yeti.2.2.4.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "PEREGRINEL_LICENSE": "5053@CBserver", - - "YETI_ROOT": "{CB_MAYA_SHARED}/modules/Yeti-v2.2.4_Maya2018-windows64", - "MAYA_MODULE_PATH": "{MAYA_MODULE_PATH};{YETI_ROOT}", - - "PATH": [ - "{YETI_ROOT}/bin", - "{PATH}" - ], - - "VRAY_PLUGINS_X64": [ - "{YETI_ROOT}/bin", - "{VRAY_PLUGINS_X64}" - ], - "VRAY_FOR_MAYA2018_PLUGINS_X64": [ - "{YETI_ROOT}/bin", - "{VRAY_FOR_MAYA2018_PLUGINS_X64}" - ], - - "REDSHIFT_MAYAEXTENSIONSPATH": [ - "{YETI_ROOT}/plug-ins", - "{REDSHIFT_MAYAEXTENSIONSPATH}" - ], - - "MTOA_EXTENSIONS_PATH": "{YETI_ROOT}/plug-ins;{MTOA_EXTENSIONS_PATH}", - "ARNOLD_PLUGIN_PATH": "{YETI_ROOT}/bin;{ARNOLD_PLUGIN_PATH}" -} \ No newline at end of file From ec4c164def87ad6aa7d2f673d97a9de4947f3d52 Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 6 Apr 2018 09:33:10 +0200 Subject: [PATCH 55/81] Added AVALON_TOOLS to launch in correct environment --- colorbleed/plugins/maya/publish/submit_switch_job.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/colorbleed/plugins/maya/publish/submit_switch_job.py b/colorbleed/plugins/maya/publish/submit_switch_job.py index 1775c62b..393544d0 100644 --- a/colorbleed/plugins/maya/publish/submit_switch_job.py +++ b/colorbleed/plugins/maya/publish/submit_switch_job.py @@ -1,5 +1,4 @@ import os -import pprint from avalon import api from avalon.vendor import requests @@ -82,7 +81,9 @@ def process(self, context): machine_limit = self.get_machine_limit(instance) payload["JobInfo"].update(machine_limit) - environment = job["Props"].get("Env", {}) + environment = api.Session.copy() + environment["AVALON_TOOLS"] = "global;python36" + payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format( key=key, @@ -120,4 +121,3 @@ def get_machine_limit(self, instance): return {"Whitelist": renderglobals["Whitelist"]} return {"Blacklist": renderglobals.get("Blacklist")} - From 1044cbc1a710c6f2473fc6ac966519cd1228976b Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 6 Apr 2018 09:58:44 +0200 Subject: [PATCH 56/81] Added function to collect maya version and yeti if loaded, added AVALON_TOOLS --- .../plugins/maya/publish/submit_deadline.py | 64 ++++++++++--------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/colorbleed/plugins/maya/publish/submit_deadline.py b/colorbleed/plugins/maya/publish/submit_deadline.py index 6be16402..fd7142a4 100644 --- a/colorbleed/plugins/maya/publish/submit_deadline.py +++ b/colorbleed/plugins/maya/publish/submit_deadline.py @@ -194,36 +194,17 @@ def process(self, instance): "AuxFiles": [] } - # Include critical environment variables with submission - keys = [ - # This will trigger `userSetup.py` on the slave - # such that proper initialisation happens the same - # way as it does on a local machine. - # TODO(marcus): This won't work if the slaves don't - # have accesss to these paths, such as if slaves are - # running Linux and the submitter is on Windows. - "PYTHONPATH", - - # todo: This is a temporary fix for yeti variables - "PEREGRINEL_LICENSE", - "REDSHIFT_MAYAEXTENSIONSPATH", - "VRAY_FOR_MAYA2018_PLUGINS_X64", - "VRAY_PLUGINS_X64", - "VRAY_USE_THREAD_AFFINITY", - "MAYA_MODULE_PATH" - ] - environment = dict({key: os.environ[key] for key in keys - if key in os.environ}, **api.Session) - - PATHS = os.environ["PATH"].split(";") - environment["PATH"] = ";".join([p for p in PATHS - if p.startswith("P:")]) + # Collects tools setup + # TODO: AVALON_TOOLS to be set at application launch, this is temp + env = api.Session.copy() + env["AVALON_TOOLS"] = self.find_loaded_tools() + # Ingest session in job environment payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format( key=key, - value=environment[key] - ) for index, key in enumerate(environment) + value=env[key] + ) for index, key in enumerate(env) }) # Include optional render globals @@ -254,7 +235,30 @@ def preflight_check(self, instance): if int(value) == value: continue - self.log.warning( - "%f=%d was rounded off to nearest integer" - % (value, int(value)) - ) + self.log.warning("%f=%d was rounded off to nearest integer" % + (value, int(value))) + + def find_loaded_tools(self): + """Temp function, remove after AVALON_TOOLS has been implemented + + Returns: + str + """ + + _maya_version = "maya" + cmds.about(version=True) + + # Check if Yeti is loaded + has_yeti = cmds.pluginInfo("pgYetiMaya", query=True, loaded=True) + if has_yeti: + # Get version and make it compatible with env_prototype + version = cmds.pluginInfo("pgYetiMaya", query=True, version=True) + yeti_version = "yeti{}".format(version) + else: + yeti_version = "" + + # Filter out empty strings + collected = ["global", _maya_version, yeti_version] + tool_names = [t for t in collected if t != ""] + tools = ";".join(tool_names) + + return tools From 3697a1f61b03c771d8d20c341fc7c4420d4fbe26 Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 6 Apr 2018 10:00:48 +0200 Subject: [PATCH 57/81] Removed fixed paths, start process through env_prototype, fix error in updating task --- .../fusion/deadline_switch_and_submit.py | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/colorbleed/scripts/fusion/deadline_switch_and_submit.py b/colorbleed/scripts/fusion/deadline_switch_and_submit.py index 894175bb..d0edd7d5 100644 --- a/colorbleed/scripts/fusion/deadline_switch_and_submit.py +++ b/colorbleed/scripts/fusion/deadline_switch_and_submit.py @@ -33,13 +33,17 @@ """ -import subprocess import traceback import logging import time +import site import sys import os + +site.addsitedir(r"P:\pipeline\dev\git\env_prototype") +import env_prototype.api as env_api + log = logging.getLogger(__name__) # This script only works with Python 2.7 and 3.6 @@ -53,13 +57,9 @@ print("Setting %s to Python executable directory.." % key) os.environ[key] = os.path.dirname(sys.executable) -# TODO: define these paths somewhere else -FUSCRIPT_EXE = r"C:/Program Files/Blackmagic Design/Fusion9/FuScript.exe" -FUSION_CONSOLE_EXE = r"C:/Program Files/Blackmagic Design/Fusion Render Node 9/FusionConsoleNode.exe" - # Pipeline and config imports +from avalon import io, api, pipeline, lib import avalon.fusion -from avalon import io, api, pipeline import colorbleed.lib as cblib import colorbleed.fusion.lib as fusionlib @@ -222,11 +222,19 @@ def process(file_path, asset_name, deadline=False): """ - # Build the session to switch to - api.update_current_task(task="comp", asset=asset_name, app="fusion") - # Start a fusion console node in "listen" mode - proc = subprocess.Popen([FUSION_CONSOLE_EXE, "/listen"]) + tools_env = env_api.get_tools(["global", "fusionnode9"]) + env = env_api.compute(tools_env) + env = env_api.merge(env, current_env=dict(os.environ)) + + # Search for the executable within the tool's environment + # by temporarily taking on its `PATH` settings + original = os.environ["PATH"] + os.environ["PATH"] = env.get("PATH", os.environ.get("PATH", "")) + exe = lib.which("fusionconsolenode") + os.environ["PATH"] = original + + proc = lib.launch(exe, environment=env, args=["/listen"]) srv = get_server() if not srv: @@ -242,6 +250,9 @@ def process(file_path, asset_name, deadline=False): api.install(avalon.fusion) from avalon.fusion import pipeline + # Build the session to switch to + api.update_current_task(task="comp", asset=asset_name, app="fusion") + # This does not set loaded_comp = fusion.LoadComp(file_path) if not loaded_comp: From 1a5cf732f5b8708acd418e2c00276a05215b82fa Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 6 Apr 2018 10:01:15 +0200 Subject: [PATCH 58/81] Added AVALON_TOOLS --- .../plugins/fusion/publish/submit_deadline.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/colorbleed/plugins/fusion/publish/submit_deadline.py b/colorbleed/plugins/fusion/publish/submit_deadline.py index bb69cc8a..4fa5e5f2 100644 --- a/colorbleed/plugins/fusion/publish/submit_deadline.py +++ b/colorbleed/plugins/fusion/publish/submit_deadline.py @@ -114,22 +114,14 @@ def process(self, instance): payload["JobInfo"]["OutputFilename%d" % index] = filename # Include critical variables with submission - keys = [ - # TODO: This won't work if the slaves don't have accesss to - # these paths, such as if slaves are running Linux and the - # submitter is on Windows. - "PYTHONPATH", - "OFX_PLUGIN_PATH", - "FUSION9_MasterPrefs" - ] - environment = dict({key: os.environ[key] for key in keys - if key in os.environ}, **api.Session) + env = api.Session.copy() + env["AVALON_TOOLS"] = "global;fusion{}".format(fusion_version) payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format( key=key, - value=environment[key] - ) for index, key in enumerate(environment) + value=env[key] + ) for index, key in enumerate(env) }) self.log.info("Submitting..") From 1d88db86657f7073fe7ed655d9a133cedd2a3773 Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 6 Apr 2018 10:02:28 +0200 Subject: [PATCH 59/81] Added AVALON_TOOLS --- colorbleed/plugins/publish/submit_publish_job.py | 1 + 1 file changed, 1 insertion(+) diff --git a/colorbleed/plugins/publish/submit_publish_job.py b/colorbleed/plugins/publish/submit_publish_job.py index 422781b5..02587b4d 100644 --- a/colorbleed/plugins/publish/submit_publish_job.py +++ b/colorbleed/plugins/publish/submit_publish_job.py @@ -143,6 +143,7 @@ def process(self, instance): # Transfer the environment from the original job to this dependent # job so they use the same environment environment = job["Props"].get("Env", {}) + environment["AVALON_TOOLS"] = "global;python36" payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format( key=key, From 75c08be2c12412cc005cf2bf2d700119c35256c0 Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 6 Apr 2018 14:35:06 +0200 Subject: [PATCH 60/81] removed workaround --- colorbleed/scripts/launch.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/colorbleed/scripts/launch.py b/colorbleed/scripts/launch.py index a9c3f5a2..a0f49c90 100644 --- a/colorbleed/scripts/launch.py +++ b/colorbleed/scripts/launch.py @@ -24,10 +24,8 @@ def launch(tools, executable, args): # Search for the executable within the tool's environment # by temporarily taking on its `PATH` settings - original = os.environ["PATH"] - os.environ["PATH"] = env.get("PATH", os.environ.get("PATH", "")) - exe = lib.which(executable) - os.environ["PATH"] = original + paths = env.get("PATH", os.environ.get("PATH", "")).split(os.pathsep) + exe = lib.which(executable, paths=paths) if not exe: raise ValueError("Unable to find executable: %s" % executable) From e49306cfc3d9b30dec7b5ab8f7a4fcd9dfd215b7 Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 6 Apr 2018 17:03:17 +0200 Subject: [PATCH 61/81] Moved \n ot end of string --- colorbleed/fusion/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/colorbleed/fusion/lib.py b/colorbleed/fusion/lib.py index fe72f45d..d89c296a 100644 --- a/colorbleed/fusion/lib.py +++ b/colorbleed/fusion/lib.py @@ -145,7 +145,7 @@ def switch(asset_name): log.info("Switched %i Loaders of the %i\n" % (len(representations), len(containers))) # Updating frame range - log.debug("\nUpdating frame range ..") + log.debug("Updating frame range ..\n") version_ids = [r["parent"] for r in representations] versions = io.find({"type": "version", "_id": {"$in": version_ids}}) versions = list(versions) From 0a5ad417b96aa787f1ad995db38c04a0c4f0ddaa Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 6 Apr 2018 17:38:55 +0200 Subject: [PATCH 62/81] Launch action through env_prototype --- colorbleed/launcher_actions.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/colorbleed/launcher_actions.py b/colorbleed/launcher_actions.py index dd5cf237..f338dd4e 100644 --- a/colorbleed/launcher_actions.py +++ b/colorbleed/launcher_actions.py @@ -1,10 +1,15 @@ import os -from avalon import api, lib, pipeline +import pprint +import avalon.pipeline as pipeline +import avalon.api as avalon -class FusionRenderNode(api.Action): +from env_prototype import api - name = "fusionrendernode9" + +class FusionRenderNode(avalon.Action): + + name = "fusionrendernode" label = "F9 Render Node" icon = "object-group" order = 997 @@ -27,18 +32,24 @@ def process(self, session, **kwargs): """ # Update environment with session - env = os.environ.copy() - env.update(session) + tools_env = api.get_tools(["global", "fusionnode9"]) + + env = api.compute(tools_env) + env = api.merge(env, current_env=session.copy()) + print("Environment %s" % pprint.pformat(env)) + + paths = env.get("PATH", os.environ.get("PATH", "")).split(os.pathsep) + exe = api.which(self.name, paths=paths) - # Get executable by na.e - app = lib.get_application(self.name) - executable = lib.which(app["executable"]) + if not exe: + raise ValueError("Unable to find executable: %s" % self.name) - return lib.launch(executable=executable, args=[], environment=env) + print("Launching: %s" % exe) + return api.execute(exe, environment=env, args=[]) def register_launcher_actions(): """Register specific actions which should be accessible in the launcher""" # Register fusion actions - pipeline.register_plugin(api.Action, FusionRenderNode) + pipeline.register_plugin(avalon.Action, FusionRenderNode) From fb2f1ced041c1fa76301bd8ffc9303ddc92e3d4a Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 6 Apr 2018 17:42:51 +0200 Subject: [PATCH 63/81] env prototype set in globals --- colorbleed/scripts/GlobalJobPreLoad.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/colorbleed/scripts/GlobalJobPreLoad.py b/colorbleed/scripts/GlobalJobPreLoad.py index c7fb4cfd..392b65f3 100644 --- a/colorbleed/scripts/GlobalJobPreLoad.py +++ b/colorbleed/scripts/GlobalJobPreLoad.py @@ -1,14 +1,9 @@ import os -import site - -# todo: refactore out the need of site -site.addsitedir(r"P:\pipeline\dev\git\env_prototype") try: from env_prototype import api except ImportError as exc: - # todo: nice error message - print(exc) + raise RuntimeError("EnvPrototype seems not to be available, this is a bug") def __main__(deadlinePlugin): From d7ec1db8721636e53b584e8fef8c92dd586b89c2 Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 6 Apr 2018 17:43:37 +0200 Subject: [PATCH 64/81] env prototype set in globals --- colorbleed/scripts/launch.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/colorbleed/scripts/launch.py b/colorbleed/scripts/launch.py index a0f49c90..7f1d191d 100644 --- a/colorbleed/scripts/launch.py +++ b/colorbleed/scripts/launch.py @@ -3,10 +3,6 @@ import avalon.lib as lib - -# TODO: Remove redundant hack -import sys -sys.path.append(r"P:\pipeline\dev\git\env_prototype") import env_prototype.api as api From 302cbfe7ee49c7a28be943e47fffe10d8a670544 Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 9 Apr 2018 15:00:37 +0200 Subject: [PATCH 65/81] fixed issue with incomplete environment --- colorbleed/launcher_actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/colorbleed/launcher_actions.py b/colorbleed/launcher_actions.py index f338dd4e..f290a53f 100644 --- a/colorbleed/launcher_actions.py +++ b/colorbleed/launcher_actions.py @@ -35,7 +35,7 @@ def process(self, session, **kwargs): tools_env = api.get_tools(["global", "fusionnode9"]) env = api.compute(tools_env) - env = api.merge(env, current_env=session.copy()) + env = api.merge(env, current_env=dict(os.environ)) print("Environment %s" % pprint.pformat(env)) paths = env.get("PATH", os.environ.get("PATH", "")).split(os.pathsep) From 38eeea8bfac6694f441eb2e865e62fb8647392db Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 9 Apr 2018 15:03:31 +0200 Subject: [PATCH 66/81] added assertion, get tools from environment --- colorbleed/plugins/fusion/publish/submit_deadline.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/colorbleed/plugins/fusion/publish/submit_deadline.py b/colorbleed/plugins/fusion/publish/submit_deadline.py index 4fa5e5f2..b2d2a9fa 100644 --- a/colorbleed/plugins/fusion/publish/submit_deadline.py +++ b/colorbleed/plugins/fusion/publish/submit_deadline.py @@ -114,8 +114,11 @@ def process(self, instance): payload["JobInfo"]["OutputFilename%d" % index] = filename # Include critical variables with submission + AVALON_TOOLS = os.getenv("AVALON_TOOLS") + assert AVALON_TOOLS, "No environment setup found" + env = api.Session.copy() - env["AVALON_TOOLS"] = "global;fusion{}".format(fusion_version) + env["AVALON_TOOLS"] = AVALON_TOOLS payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format( From 6176cf92596fb4f9f54837d8d6bb446e256d5555 Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 9 Apr 2018 15:04:45 +0200 Subject: [PATCH 67/81] Created more elegant way to retriev script file --- .../fusion/publish/publish_image_sequences.py | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/colorbleed/plugins/fusion/publish/publish_image_sequences.py b/colorbleed/plugins/fusion/publish/publish_image_sequences.py index 33d6d277..f60cf974 100644 --- a/colorbleed/plugins/fusion/publish/publish_image_sequences.py +++ b/colorbleed/plugins/fusion/publish/publish_image_sequences.py @@ -8,22 +8,19 @@ from colorbleed.action import get_errored_plugins_from_data -def _get_script(): +def _get_script_dir(): """Get path to the image sequence script""" - - # todo: use a more elegant way to get the python script - try: - from colorbleed.scripts import publish_filesequence - except Exception: - raise RuntimeError("Expected module 'publish_imagesequence'" - "to be available") + import colorbleed + config_dir = os.path.dirname(colorbleed.__file__) + script_dir = os.path.join(config_dir, "scripts") + except ImportError: + raise RuntimeError("This is a bug") - module_path = publish_filesequence.__file__ - if module_path.endswith(".pyc"): - module_path = module_path[:-len(".pyc")] + ".py" + assert os.path.isdir(script_dir), "Config is incomplete" + script_dir = script_dir.replace(os.sep, "/") - return module_path + return script_dir class PublishImageSequence(pyblish.api.InstancePlugin): @@ -73,7 +70,13 @@ def process(self, instance): startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW startupinfo.wShowWindow = subprocess.SW_HIDE - process = subprocess.Popen(["python", _get_script(), + # Get script + script_dir = _get_script_dir() + script = os.path.join(script_dir, "publish_imagesequence.py") + assert os.path.isfile(script), ("Config incomplete, missing " + "`script/publish_imagesequence.py`") + + process = subprocess.Popen(["python", script, "--paths", path], bufsize=1, stdout=subprocess.PIPE, From 67a3fe7c578768b90cce0fcaa49cccd61f3ba4b1 Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 9 Apr 2018 15:05:44 +0200 Subject: [PATCH 68/81] added assert for AVALON_TOOLS, removed redundant function --- .../plugins/maya/publish/submit_deadline.py | 34 ++++--------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/colorbleed/plugins/maya/publish/submit_deadline.py b/colorbleed/plugins/maya/publish/submit_deadline.py index fd7142a4..b857a7cb 100644 --- a/colorbleed/plugins/maya/publish/submit_deadline.py +++ b/colorbleed/plugins/maya/publish/submit_deadline.py @@ -195,9 +195,12 @@ def process(self, instance): } # Collects tools setup - # TODO: AVALON_TOOLS to be set at application launch, this is temp + + AVALON_TOOLS = os.getenv("AVALON_TOOLS") + assert AVALON_TOOLS, "No environment setup found" + env = api.Session.copy() - env["AVALON_TOOLS"] = self.find_loaded_tools() + env["AVALON_TOOLS"] = AVALON_TOOLS # Ingest session in job environment payload["JobInfo"].update({ @@ -222,7 +225,7 @@ def process(self, instance): if not response.ok: raise Exception(response.text) - # Store output dir for unified publisher (filesequence) + # Store output dir for unified publisher (file sequence) instance.data["outputDir"] = os.path.dirname(output_filename_0) instance.data["deadlineSubmissionJob"] = response.json() @@ -237,28 +240,3 @@ def preflight_check(self, instance): self.log.warning("%f=%d was rounded off to nearest integer" % (value, int(value))) - - def find_loaded_tools(self): - """Temp function, remove after AVALON_TOOLS has been implemented - - Returns: - str - """ - - _maya_version = "maya" + cmds.about(version=True) - - # Check if Yeti is loaded - has_yeti = cmds.pluginInfo("pgYetiMaya", query=True, loaded=True) - if has_yeti: - # Get version and make it compatible with env_prototype - version = cmds.pluginInfo("pgYetiMaya", query=True, version=True) - yeti_version = "yeti{}".format(version) - else: - yeti_version = "" - - # Filter out empty strings - collected = ["global", _maya_version, yeti_version] - tool_names = [t for t in collected if t != ""] - tools = ";".join(tool_names) - - return tools From 44e58c2606a934d6f367cebdaf2c36d549e6711d Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 9 Apr 2018 16:23:36 +0200 Subject: [PATCH 69/81] launch now fully relies on env_prototype --- colorbleed/scripts/launch.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/colorbleed/scripts/launch.py b/colorbleed/scripts/launch.py index 7f1d191d..e61544fd 100644 --- a/colorbleed/scripts/launch.py +++ b/colorbleed/scripts/launch.py @@ -1,8 +1,6 @@ import os import pprint -import avalon.lib as lib - import env_prototype.api as api @@ -20,16 +18,12 @@ def launch(tools, executable, args): # Search for the executable within the tool's environment # by temporarily taking on its `PATH` settings - paths = env.get("PATH", os.environ.get("PATH", "")).split(os.pathsep) - exe = lib.which(executable, paths=paths) - + exe = api.which(executable, env) if not exe: raise ValueError("Unable to find executable: %s" % executable) print("Launching: %s" % exe) - lib.launch(exe, - environment=env, - args=args) + api.execute(exe, environment=env, args=args, cwd=env.get("AVALON_WORKDIR")) if __name__ == '__main__': From a6b55d4257aaecd7ee20d6b6048a782281db9ca0 Mon Sep 17 00:00:00 2001 From: wijnand Date: Mon, 9 Apr 2018 16:24:15 +0200 Subject: [PATCH 70/81] Refactored to match signature of env_prototype.which --- colorbleed/launcher_actions.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/colorbleed/launcher_actions.py b/colorbleed/launcher_actions.py index f290a53f..99adaeee 100644 --- a/colorbleed/launcher_actions.py +++ b/colorbleed/launcher_actions.py @@ -38,9 +38,7 @@ def process(self, session, **kwargs): env = api.merge(env, current_env=dict(os.environ)) print("Environment %s" % pprint.pformat(env)) - paths = env.get("PATH", os.environ.get("PATH", "")).split(os.pathsep) - exe = api.which(self.name, paths=paths) - + exe = api.which(self.name, env) if not exe: raise ValueError("Unable to find executable: %s" % self.name) From fda2aaa8c3ec626a96025f4fa5b092ba852be62a Mon Sep 17 00:00:00 2001 From: wijnand Date: Tue, 10 Apr 2018 10:39:42 +0200 Subject: [PATCH 71/81] renamved env_prototype to acre --- colorbleed/launcher_actions.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/colorbleed/launcher_actions.py b/colorbleed/launcher_actions.py index 99adaeee..216dce59 100644 --- a/colorbleed/launcher_actions.py +++ b/colorbleed/launcher_actions.py @@ -1,13 +1,12 @@ import os import pprint -import avalon.pipeline as pipeline -import avalon.api as avalon +from avalon import api, pipeline, lib -from env_prototype import api +import acre -class FusionRenderNode(avalon.Action): +class FusionRenderNode(api.Action): name = "fusionrendernode" label = "F9 Render Node" @@ -32,22 +31,22 @@ def process(self, session, **kwargs): """ # Update environment with session - tools_env = api.get_tools(["global", "fusionnode9"]) + tools_env = acre.get_tools(["global", "fusionnode9"]) - env = api.compute(tools_env) - env = api.merge(env, current_env=dict(os.environ)) + env = acre.compute(tools_env) + env = acre.merge(env, current_env=dict(os.environ)) print("Environment %s" % pprint.pformat(env)) - exe = api.which(self.name, env) + exe = acre.which(self.name, env) if not exe: raise ValueError("Unable to find executable: %s" % self.name) print("Launching: %s" % exe) - return api.execute(exe, environment=env, args=[]) + return lib.launch(exe, environment=env, args=[]) def register_launcher_actions(): """Register specific actions which should be accessible in the launcher""" # Register fusion actions - pipeline.register_plugin(avalon.Action, FusionRenderNode) + pipeline.register_plugin(api.Action, FusionRenderNode) From 9e9edb051bd860ae30b490ec6d0baf017c6e1175 Mon Sep 17 00:00:00 2001 From: wijnand Date: Tue, 10 Apr 2018 10:40:42 +0200 Subject: [PATCH 72/81] renamved env_prototype to acre --- colorbleed/scripts/GlobalJobPreLoad.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/colorbleed/scripts/GlobalJobPreLoad.py b/colorbleed/scripts/GlobalJobPreLoad.py index 392b65f3..89f21779 100644 --- a/colorbleed/scripts/GlobalJobPreLoad.py +++ b/colorbleed/scripts/GlobalJobPreLoad.py @@ -1,9 +1,10 @@ import os try: - from env_prototype import api + import acre except ImportError as exc: - raise RuntimeError("EnvPrototype seems not to be available, this is a bug") + # todo: nice error message + print(exc) def __main__(deadlinePlugin): @@ -22,11 +23,11 @@ def __main__(deadlinePlugin): deadlinePlugin.LogInfo("Setting environment for tools: %s" % tools) - tools_env = api.get_tools(tools.split(";")) - env = api.compute(tools_env) + tools_env = acre.get_tools(tools.split(";")) + env = acre.compute(tools_env) # Get the merged environment for the local machine - merged = api.merge(env, current_env=os.environ.copy()) + merged = acre.merge(env, current_env=os.environ.copy()) # Keep only the changed variables env = {key: value for key, value in merged.items() if key in env} From 9d21bef84b76261edb8bebc175987f9959c8ee0b Mon Sep 17 00:00:00 2001 From: wijnand Date: Tue, 10 Apr 2018 14:43:57 +0200 Subject: [PATCH 73/81] renamved env_prototype to acre --- colorbleed/scripts/fusion/deadline_switch_and_submit.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/colorbleed/scripts/fusion/deadline_switch_and_submit.py b/colorbleed/scripts/fusion/deadline_switch_and_submit.py index d0edd7d5..41e674a9 100644 --- a/colorbleed/scripts/fusion/deadline_switch_and_submit.py +++ b/colorbleed/scripts/fusion/deadline_switch_and_submit.py @@ -42,7 +42,7 @@ site.addsitedir(r"P:\pipeline\dev\git\env_prototype") -import env_prototype.api as env_api +import acre log = logging.getLogger(__name__) @@ -223,9 +223,9 @@ def process(file_path, asset_name, deadline=False): """ # Start a fusion console node in "listen" mode - tools_env = env_api.get_tools(["global", "fusionnode9"]) - env = env_api.compute(tools_env) - env = env_api.merge(env, current_env=dict(os.environ)) + tools_env = acre.get_tools(["global", "fusionnode9"]) + env = acre.compute(tools_env) + env = acre.merge(env, current_env=dict(os.environ)) # Search for the executable within the tool's environment # by temporarily taking on its `PATH` settings From ea52a9a05b2be8bb3099ea78b10b1869508c50a8 Mon Sep 17 00:00:00 2001 From: wijnand Date: Tue, 10 Apr 2018 16:22:18 +0200 Subject: [PATCH 74/81] Refactored with env_prototype to acre --- colorbleed/scripts/launch.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/colorbleed/scripts/launch.py b/colorbleed/scripts/launch.py index e61544fd..e4d1ad74 100644 --- a/colorbleed/scripts/launch.py +++ b/colorbleed/scripts/launch.py @@ -1,7 +1,7 @@ import os import pprint -import env_prototype.api as api +import acre config_root = os.path.dirname(os.path.dirname(__file__)) @@ -10,20 +10,20 @@ def launch(tools, executable, args): - tools_env = api.get_tools(tools.split(";")) - env = api.compute(tools_env) + tools_env = acre.get_tools(tools.split(";")) + env = acre.compute(tools_env) - env = api.merge(env, current_env=dict(os.environ)) + env = acre.merge(env, current_env=dict(os.environ)) print("Environment:\n%s" % pprint.pformat(env, indent=4)) # Search for the executable within the tool's environment # by temporarily taking on its `PATH` settings - exe = api.which(executable, env) + exe = acre.which(executable, env) if not exe: raise ValueError("Unable to find executable: %s" % executable) print("Launching: %s" % exe) - api.execute(exe, environment=env, args=args, cwd=env.get("AVALON_WORKDIR")) + acre.launch(exe, environment=env, args=args, cwd=env.get("AVALON_WORKDIR")) if __name__ == '__main__': From b4e745b9ddf24a2f8100bd5dd078ff9a67b97511 Mon Sep 17 00:00:00 2001 From: wijnand Date: Wed, 11 Apr 2018 14:32:59 +0200 Subject: [PATCH 75/81] Refactored back acre, conflict was solved wrong --- colorbleed/launcher_actions.py | 38 +++++++++++++++++----------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/colorbleed/launcher_actions.py b/colorbleed/launcher_actions.py index 7d72cb2b..c5143e89 100644 --- a/colorbleed/launcher_actions.py +++ b/colorbleed/launcher_actions.py @@ -1,10 +1,13 @@ import os -from avalon import api, lib, pipeline + +from avalon import api, pipeline, lib + +import acre class FusionRenderNode(api.Action): - name = "fusionrendernode9" + name = "fusionrendernode" label = "F9 Render Node" icon = "object-group" order = 997 @@ -26,16 +29,17 @@ def process(self, session, **kwargs): """ - # Update environment with session - env = os.environ.copy() - env.update(session) + # Update environment with session and the current environment + tools_env = acre.get_tools(["global", "fusionnode9"]) + env = acre.compute(tools_env) + env = acre.merge(env, current_env=dict(os.environ)) # Get executable by name - app = lib.get_application(self.name) - env.update(app["environment"]) - executable = lib.which(app["executable"]) + executable = acre.which(self.name, env) + if not executable: + raise ValueError("Unable to find executable %s" % self.name) - return lib.launch(executable=executable, args=[], environment=env) + return acre.launch(executable=executable, args=[], environment=env) class VrayRenderSlave(api.Action): @@ -63,20 +67,16 @@ def process(self, session, **kwargs): """ # Update environment with session - env = os.environ.copy() - env.update(session) + tools_env = acre.get_tools(["global", "maya2018", "vrayrenderslave"]) + env = acre.compute(tools_env) + env = acre.merge(env, current_env=dict(os.environ)) # Get executable by name - app = lib.get_application(self.name) - env.update(app["environment"]) - executable = lib.which(app["executable"]) + executable = acre.which("vray", env) # Run as server - arguments = ["-server", "-portNumber=20207"] - - return lib.launch(executable=executable, - args=arguments, - environment=env) + args = ["-server", "-portNumber=20207"] + return acre.launch(executable=executable, args=args, environment=env) def register_launcher_actions(): From 9cc99efe9fb5bdda2f5ea270a0f8b9f0137ee1c2 Mon Sep 17 00:00:00 2001 From: wijnand Date: Wed, 11 Apr 2018 14:33:57 +0200 Subject: [PATCH 76/81] Added pretty error message --- colorbleed/scripts/GlobalJobPreLoad.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/colorbleed/scripts/GlobalJobPreLoad.py b/colorbleed/scripts/GlobalJobPreLoad.py index 89f21779..538036a2 100644 --- a/colorbleed/scripts/GlobalJobPreLoad.py +++ b/colorbleed/scripts/GlobalJobPreLoad.py @@ -3,8 +3,7 @@ try: import acre except ImportError as exc: - # todo: nice error message - print(exc) + raise ImportError("Module 'acre' cannot be found in the environment") def __main__(deadlinePlugin): From bceaa2518d2b5f6b073b020089f3940b6d7a1a0e Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 13 Apr 2018 12:20:04 +0200 Subject: [PATCH 77/81] Explicitlty set tool combination and TOOL_ENV --- .../plugins/fusion/publish/submit_deadline.py | 7 ++++--- .../global/publish/submit_publish_job.py | 5 +++++ .../plugins/maya/publish/submit_deadline.py | 3 +++ .../plugins/maya/publish/submit_switch_job.py | 17 +++++++++++++++++ 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/colorbleed/plugins/fusion/publish/submit_deadline.py b/colorbleed/plugins/fusion/publish/submit_deadline.py index 5a1b4b9b..b82b20d4 100644 --- a/colorbleed/plugins/fusion/publish/submit_deadline.py +++ b/colorbleed/plugins/fusion/publish/submit_deadline.py @@ -117,11 +117,12 @@ def process(self, instance): payload["JobInfo"]["OutputFilename%d" % index] = filename # Include critical variables with submission - AVALON_TOOLS = os.getenv("AVALON_TOOLS") - assert AVALON_TOOLS, "No environment setup found" + TOOL_ENV = os.getenv("TOOL_ENV") + assert TOOL_ENV, "No environment directory found" env = api.Session.copy() - env["AVALON_TOOLS"] = AVALON_TOOLS + env["AVALON_TOOLS"] = "global;fusion9" + env["TOOL_ENV"] = TOOL_ENV payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format( diff --git a/colorbleed/plugins/global/publish/submit_publish_job.py b/colorbleed/plugins/global/publish/submit_publish_job.py index 02587b4d..2a0f268b 100644 --- a/colorbleed/plugins/global/publish/submit_publish_job.py +++ b/colorbleed/plugins/global/publish/submit_publish_job.py @@ -142,8 +142,13 @@ def process(self, instance): # Transfer the environment from the original job to this dependent # job so they use the same environment + TOOL_ENV = os.getenv("TOOL_ENV") + assert TOOL_ENV, "No environment directory found" + environment = job["Props"].get("Env", {}) environment["AVALON_TOOLS"] = "global;python36" + environment["TOOL_ENV"] = TOOL_ENV + payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format( key=key, diff --git a/colorbleed/plugins/maya/publish/submit_deadline.py b/colorbleed/plugins/maya/publish/submit_deadline.py index b857a7cb..69c21025 100644 --- a/colorbleed/plugins/maya/publish/submit_deadline.py +++ b/colorbleed/plugins/maya/publish/submit_deadline.py @@ -196,11 +196,14 @@ def process(self, instance): # Collects tools setup + TOOL_ENV = os.getenv("TOOL_ENV") + assert TOOL_ENV, "No environment directory found" AVALON_TOOLS = os.getenv("AVALON_TOOLS") assert AVALON_TOOLS, "No environment setup found" env = api.Session.copy() env["AVALON_TOOLS"] = AVALON_TOOLS + env["TOOL_ENV"] = TOOL_ENV # Ingest session in job environment payload["JobInfo"].update({ diff --git a/colorbleed/plugins/maya/publish/submit_switch_job.py b/colorbleed/plugins/maya/publish/submit_switch_job.py index 393544d0..d16f543c 100644 --- a/colorbleed/plugins/maya/publish/submit_switch_job.py +++ b/colorbleed/plugins/maya/publish/submit_switch_job.py @@ -21,6 +21,18 @@ def _get_script_dir(): return fusion_scripts +def _get_acre_path(): + + import acre + moduledir = os.path.dirname(acre.__file__) + if not os.path.isdir(moduledir): + raise ValueError("Could not find 'acre'") + + package_dir = os.path.abspath(os.path.join(moduledir, "..")) + + return package_dir + + class SubmitDependentSwitchJobDeadline(pyblish.api.ContextPlugin): """Run Switch Shot on specified comp as depending job @@ -81,8 +93,13 @@ def process(self, context): machine_limit = self.get_machine_limit(instance) payload["JobInfo"].update(machine_limit) + TOOL_ENV = os.getenv("TOOL_ENV") + assert TOOL_ENV, "No environment directory found" + environment = api.Session.copy() + environment["PYTHONPATH"] = _get_acre_path() environment["AVALON_TOOLS"] = "global;python36" + environment["TOOL_ENV"] = TOOL_ENV payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format( From c4d703e6002e03268101261756f44d6a74508569 Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 13 Apr 2018 12:21:02 +0200 Subject: [PATCH 78/81] Replace env logic with acre, fixed type, print new comp name --- .../fusion/deadline_switch_and_submit.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/colorbleed/scripts/fusion/deadline_switch_and_submit.py b/colorbleed/scripts/fusion/deadline_switch_and_submit.py index 41e674a9..01b9894f 100644 --- a/colorbleed/scripts/fusion/deadline_switch_and_submit.py +++ b/colorbleed/scripts/fusion/deadline_switch_and_submit.py @@ -36,13 +36,13 @@ import traceback import logging import time -import site import sys import os - -site.addsitedir(r"P:\pipeline\dev\git\env_prototype") -import acre +try: + import acre +except ImportError: + raise ImportError("Module 'acre' cannot be found in the environment") log = logging.getLogger(__name__) @@ -160,7 +160,7 @@ def create_new_filepath(session): session["AVALON_ASSET"]) new_filepath = os.path.join(comp_dir, new_filename) - # Create new unqiue filepath + # Create new unique file path if os.path.exists(new_filepath): new_filepath = cblib.version_up(new_filepath) @@ -229,12 +229,8 @@ def process(file_path, asset_name, deadline=False): # Search for the executable within the tool's environment # by temporarily taking on its `PATH` settings - original = os.environ["PATH"] - os.environ["PATH"] = env.get("PATH", os.environ.get("PATH", "")) - exe = lib.which("fusionconsolenode") - os.environ["PATH"] = original - - proc = lib.launch(exe, environment=env, args=["/listen"]) + exe = acre.which("fusionconsolenode", env) + proc = acre.launch(exe, environment=env, args=["/listen"]) srv = get_server() if not srv: @@ -268,6 +264,7 @@ def process(file_path, asset_name, deadline=False): # Execute script in comp fusionlib.switch(asset_name=asset_name) new_file_path = create_new_filepath(api.Session) + print("Saving comp as: %s" % new_file_path) current_comp.Save(new_file_path) if deadline: submit(current_comp, source=file_path) From 3a1b1f6259d5d48609b2604618b2a79254d61f81 Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 13 Apr 2018 14:10:01 +0200 Subject: [PATCH 79/81] Using print instead of log, print server found and fusion found --- .../scripts/fusion/deadline_switch_and_submit.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/colorbleed/scripts/fusion/deadline_switch_and_submit.py b/colorbleed/scripts/fusion/deadline_switch_and_submit.py index 01b9894f..8e9e4e33 100644 --- a/colorbleed/scripts/fusion/deadline_switch_and_submit.py +++ b/colorbleed/scripts/fusion/deadline_switch_and_submit.py @@ -184,7 +184,7 @@ def submit(current_comp, source=""): # Set comp render mode to deadline current_comp.SetData("colorbleed.rendermode", "deadline") - error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" + error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}\n" comment = "slapped using: {}".format(os.path.basename(source)) # Publish @@ -198,9 +198,10 @@ def submit(current_comp, source=""): # Collect errors, {plugin name: error}, if any error_results = [r for r in results.data["results"] if r["error"]] if error_results: + custom_traceback = "" for result in error_results: - log.error(error_format.format(**result)) - raise RuntimeError("Errors occured") + custom_traceback += error_format.format(**result) + raise RuntimeError("Errors occured:\n%s" % custom_traceback) return True @@ -234,15 +235,18 @@ def process(file_path, asset_name, deadline=False): srv = get_server() if not srv: - log.info("No server found, starting server ..") + print("No server found, starting server ..") srv = start_server() + print("Found server: %s" % srv) + # Force fusion into main magical module so that host.ls() works fusion = get_fusion_instance(proc.pid, srv) assert fusion - log.info("Connected to: %s" % fusion) setattr(sys.modules["__main__"], "fusion", fusion) + print("Connected to: %s" % fusion) + api.install(avalon.fusion) from avalon.fusion import pipeline From 6911d560b2b8f77b05c58281beb07ba6c14ee7ab Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 13 Apr 2018 18:22:52 +0200 Subject: [PATCH 80/81] Improved traceback for deadline --- colorbleed/fusion/lib.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/colorbleed/fusion/lib.py b/colorbleed/fusion/lib.py index d89c296a..e1b81579 100644 --- a/colorbleed/fusion/lib.py +++ b/colorbleed/fusion/lib.py @@ -1,6 +1,7 @@ import os import re import logging +import traceback import avalon.fusion from avalon import api, io, pipeline @@ -140,7 +141,8 @@ def switch(asset_name): representations.append(representation) log.debug(str(representation["_id"]) + "\n") except Exception as e: - log.debug("Error in switching! %s\n" % e.message) + msg = traceback.format_stack(e) + log.debug("Error in switching! %s\n" % msg) log.info("Switched %i Loaders of the %i\n" % (len(representations), len(containers))) From c0c30112600cddbed8d8f26dfff0a996e694b216 Mon Sep 17 00:00:00 2001 From: wijnand Date: Fri, 13 Apr 2018 18:23:40 +0200 Subject: [PATCH 81/81] Delete launch files --- colorbleed/scripts/launch.py | 45 ------------------------------------ 1 file changed, 45 deletions(-) delete mode 100644 colorbleed/scripts/launch.py diff --git a/colorbleed/scripts/launch.py b/colorbleed/scripts/launch.py deleted file mode 100644 index e4d1ad74..00000000 --- a/colorbleed/scripts/launch.py +++ /dev/null @@ -1,45 +0,0 @@ -import os -import pprint - -import acre - - -config_root = os.path.dirname(os.path.dirname(__file__)) -os.environ["TOOL_ENV"] = os.path.join(config_root, "environments") - - -def launch(tools, executable, args): - - tools_env = acre.get_tools(tools.split(";")) - env = acre.compute(tools_env) - - env = acre.merge(env, current_env=dict(os.environ)) - print("Environment:\n%s" % pprint.pformat(env, indent=4)) - - # Search for the executable within the tool's environment - # by temporarily taking on its `PATH` settings - exe = acre.which(executable, env) - if not exe: - raise ValueError("Unable to find executable: %s" % executable) - - print("Launching: %s" % exe) - acre.launch(exe, environment=env, args=args, cwd=env.get("AVALON_WORKDIR")) - - -if __name__ == '__main__': - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("--tools", - help="The tool environments to include. " - "These should be separated by `;`", - required=True) - parser.add_argument("--executable", - help="The executable to run. ", - required=True) - - kwargs, args = parser.parse_known_args() - - launch(tools=kwargs.tools, - executable=kwargs.executable, - args=args)