Error occurred when executing Motionctrl-SVD Sample Simple:

No operator found for memory_efficient_attention_forward with inputs:
query : shape=(1, 9216, 1, 512) (torch.float32)
key : shape=(1, 9216, 1, 512) (torch.float32)
value : shape=(1, 9216, 1, 512) (torch.float32)
attn_bias :
p : 0.0
decoderF is not supported because:
max(query.shape[-1] != value.shape[-1]) > 128
xFormers wasn't build with CUDA support
attn_bias type is
operator wasn't built - see python -m xformers.info for more info
[email protected] is not supported because:
max(query.shape[-1] != value.shape[-1]) > 256
xFormers wasn't build with CUDA support
dtype=torch.float32 (supported: {torch.bfloat16, torch.float16})
operator wasn't built - see python -m xformers.info for more info
tritonflashattF is not supported because:
max(query.shape[-1] != value.shape[-1]) > 128
xFormers wasn't build with CUDA support
dtype=torch.float32 (supported: {torch.bfloat16, torch.float16})
operator wasn't built - see python -m xformers.info for more info
triton is not available
cutlassF is not supported because:
xFormers wasn't build with CUDA support
operator wasn't built - see python -m xformers.info for more info
smallkF is not supported because:
max(query.shape[-1] != value.shape[-1]) > 32
xFormers wasn't build with CUDA support
operator wasn't built - see python -m xformers.info for more info
unsupported embed per head: 512

File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\execution.py", line 152, in recursive_execute
output_data, output_ui = get_output_data(obj, input_data_all)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\execution.py", line 82, in get_output_data
return_values = map_node_over_list(obj, input_data_all, obj.FUNCTION, allow_interrupt=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\execution.py", line 75, in map_node_over_list
results.append(getattr(obj, func)(**slice_dict(input_data_all, i)))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\nodes.py", line 181, in run_inference
return motionctrl_sample(
^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\gradio_utils\motionctrl_cmcm_gradio.py", line 148, in motionctrl_sample
c, uc = model.conditioner.get_unconditional_conditioning(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\sgm\modules\encoders\modules.py", line 179, in get_unconditional_conditioning
c = self(batch_c, force_cond_zero_embeddings)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1511, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1520, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\sgm\modules\encoders\modules.py", line 132, in forward
emb_out = embedder(batch[embedder.input_key])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1511, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1520, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\sgm\modules\encoders\modules.py", line 1012, in forward
out = self.encoder.encode(vid[n * n_samples : (n + 1) * n_samples])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\sgm\models\autoencoder.py", line 472, in encode
z = self.encoder(x)
^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1511, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1520, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\sgm\modules\diffusionmodules\model.py", line 594, in forward
h = self.mid.attn_1(h)
^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1511, in wrapped_call_impl
return self.call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1520, in call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\sgm\modules\diffusionmodules\model.py", line 263, in forward
h = self.attention(h)
^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\sgm\modules\diffusionmodules\model.py", line 249, in attention
out = xformers.ops.memory_efficient_attention(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\python_embeded\Lib\site-packages\xformers\ops\fmha_init.py", line 223, in memory_efficient_attention
return memory_efficient_attention(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\python_embeded\Lib\site-packages\xformers\ops\fmha_init.py", line 321, in _memory_efficient_attention
return memory_efficient_attention_forward(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\python_embeded\Lib\site-packages\xformers\ops\fmha_init.py", line 337, in _memory_efficient_attention_forward
op = _dispatch_fw(inp, False)
^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\python_embeded\Lib\site-packages\xformers\ops\fmha\dispatch.py", line 120, in _dispatch_fw
return _run_priority_list(
^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\python_embeded\Lib\site-packages\xformers\ops\fmha\dispatch.py", line 63, in _run_priority_list
raise NotImplementedError(msg)

got prompt
Prompt executor has been patched by Job Iterator!
ERROR:root:!!! Exception during processing !!!
ERROR:root:Traceback (most recent call last):
File "C:\Downloads\ComfyUI_windows_portable_nvidia_cu118_or_cpu\ComfyUI_windows_portable\ComfyUI\execution.py", line 155, in recursive_execute
output_data, output_ui = get_output_data(obj, input_data_all)
File "C:\Downloads\ComfyUI_windows_portable_nvidia_cu118_or_cpu\ComfyUI_windows_portable\ComfyUI\execution.py", line 85, in get_output_data
return_values = map_node_over_list(obj, input_data_all, obj.FUNCTION, allow_interrupt=True)
File "C:\Downloads\ComfyUI_windows_portable_nvidia_cu118_or_cpu\ComfyUI_windows_portable\ComfyUI\execution.py", line 78, in map_node_over_list
results.append(getattr(obj, func)(**slice_dict(input_data_all, i)))
File "C:\Downloads\ComfyUI_windows_portable_nvidia_cu118_or_cpu\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl\nodes.py", line 214, in load_checkpoint
model = instantiate_from_config(model_config)
File "C:\Downloads\ComfyUI_windows_portable_nvidia_cu118_or_cpu\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl\utils\utils.py", line 37, in instantiate_from_config
return get_obj_from_str(config["target"])(**config.get("params", dict()))
File "C:\Downloads\ComfyUI_windows_portable_nvidia_cu118_or_cpu\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl\utils\utils.py", line 45, in get_obj_from_str
return getattr(importlib.import_module(module, package=None), cls)
File "importlib_init_.py", line 126, in import_module
File "", line 1050, in _gcd_import
File "", line 1027, in _find_and_load
File "", line 992, in _find_and_load_unlocked
File "", line 241, in _call_with_frames_removed
File "", line 1050, in _gcd_import
File "", line 1027, in _find_and_load
File "", line 992, in _find_and_load_unlocked
File "", line 241, in _call_with_frames_removed
File "", line 1050, in _gcd_import
File "", line 1027, in _find_and_load
File "", line 1004, in _find_and_load_unlocked
ModuleNotFoundError: No module named 'custom_nodes.ComfyUI-MotionCtrl'

Prompt executed in 0.02 seconds

requirements are installed, there was no red block. But still got this error.

FETCH DATA from: D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-Manager\extension-node-map.json
#read_workflow_json_files_all D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\custom_nodes\comfyui-mixlab-nodes-main\app
got prompt
Loading model from D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\models\checkpoints\motionctrl\motionctrl.pth
Using device cuda
Initialized embedder #0: FrozenOpenCLIPImagePredictionEmbedder with 683800065 params. Trainable: False
Initialized embedder #1: ConcatTimestepEmbedderND with 0 params. Trainable: False
Initialized embedder #2: ConcatTimestepEmbedderND with 0 params. Trainable: False
Initialized embedder #3: VideoPredictionEmbedderWithEncoder with 83653863 params. Trainable: False
Initialized embedder #4: ConcatTimestepEmbedderND with 0 params. Trainable: False
ERROR:root:!!! Exception during processing !!!
ERROR:root:Traceback (most recent call last):
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\execution.py", line 152, in recursive_execute
output_data, output_ui = get_output_data(obj, input_data_all)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\execution.py", line 82, in get_output_data
return_values = map_node_over_list(obj, input_data_all, obj.FUNCTION, allow_interrupt=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\execution.py", line 75, in map_node_over_list
results.append(getattr(obj, func)(**slice_dict(input_data_all, i)))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\nodes.py", line 152, in load_checkpoint
model = build_model(config_path, ckpt_path, device, frame_length, steps)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\gradio_utils\motionctrl_cmcm_gradio.py", line 55, in build_model
model, filter = load_model(
^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\gradio_utils\motionctrl_cmcm_gradio.py", line 279, in load_model
model = instantiate_from_config(config.model)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\sgm\util.py", line 175, in instantiate_from_config
return get_obj_from_str(config["target"])(**config.get("params", dict()))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\sgm\motionctrl\camera_motion_control.py", line 63, in init
self.init_from_ckpt(ckpt_path)
File "D:\Git\ComfyUI\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\sgm\models\diffusion.py", line 99, in init_from_ckpt
raise NotImplementedError
NotImplementedError

Prompt executed in 21.31 seconds

Is there something specific that needs to be done for this Motionctrl-SVD sample simple node, as when I add a new instance of this node to the canvas, there is just two inputs, model and image, but no camera, like in your example images. Also, if I open the sample project (base workflow), it has that node. I checked the right-click menu, and did not seem to find any option in there, either (inputs/outputs sections are greyed out.) So there is no way to get camera presets assingned. Am I missing something here? I did not look at the code yet.

Traceback (most recent call last):
File "F:\Blender_ComfyUI_aki\ComfyUI\nodes.py", line 1872, in load_custom_node
module_spec.loader.exec_module(module)
File "", line 883, in exec_module
File "", line 241, in call_with_frames_removed
File "F:\Blender_ComfyUI_aki\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD_init.py", line 1, in
from .nodes import NODE_CLASS_MAPPINGS
File "F:\Blender_ComfyUI_aki\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\nodes.py", line 17, in
from .gradio_utils.motionctrl_cmcm_gradio import build_model, motionctrl_sample
File "F:\Blender_ComfyUI_aki\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\gradio_utils\motionctrl_cmcm_gradio.py", line 17, in
from fire import Fire
File "F:\Blender_ComfyUI_aki\python_embeded\lib\site-packages\fire_init_.py", line 21, in
from fire.core import Fire
File "F:\Blender_ComfyUI_aki\python_embeded\lib\site-packages\fire\core.py", line 67, in
from fire import formatting
File "F:\Blender_ComfyUI_aki\python_embeded\lib\site-packages\fire\formatting.py", line 21, in
from fire import formatting_windows # pylint: disable=unused-import
File "F:\Blender_ComfyUI_aki\python_embeded\lib\site-packages\fire\formatting_windows.py", line 60, in
initialize_or_disable()
File "F:\Blender_ComfyUI_aki\python_embeded\lib\site-packages\fire\formatting_windows.py", line 38, in initialize_or_disable
if sys.stdout.isatty() and platform.release() == '10':
AttributeError: 'ComfyUIManagerLogger' object has no attribute 'isatty'

Cannot import F:\Blender_ComfyUI_aki\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD module for custom nodes: 'ComfyUIManagerLogger' object has no attribute 'isatty'

Can't get this running - followed all instructions - nodes won't show up with demo workflow - all red - says 'import failed' in comfyui manager.

Not sure how or why, but if I load this custom node successfully it causes comfyui-reactor-node (the one installed directly from Manager) to fail to load with the following error. Note that ComfyUI-MotionCtrl-SVD does still load successfully. If I temporarily remove this custom node from the custom_nodes folder then Reactor starts loading again correctly.

Traceback (most recent call last):
File "G:\AI-Content\ComfyUI_windows_portable\ComfyUI\nodes.py", line 1872, in load_custom_node
module_spec.loader.exec_module(module)
File "", line 940, in exec_module
File "", line 241, in call_with_frames_removed
File "G:\AI-Content\ComfyUI_windows_portable\ComfyUI\custom_nodes\comfyui-reactor-node_init.py", line 25, in
from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS
File "G:\AI-Content\ComfyUI_windows_portable\ComfyUI\custom_nodes\comfyui-reactor-node\nodes.py", line 4, in
from scripts.reactor_faceswap import FaceSwapScript, get_models, get_current_faces_model, analyze_faces
ModuleNotFoundError: No module named 'scripts.reactor_faceswap'

Cannot import G:\AI-Content\ComfyUI_windows_portable\ComfyUI\custom_nodes\comfyui-reactor-node module for custom nodes: No module named 'scripts.reactor_faceswap'

Hello, how do I add a new lens preset in Load Motionctrl-SVD Camera Preset?

Error occurred when executing Motionctrl-SVD Sample Simple:

name 'xformers' is not defined

File "F:\python\ComfyUI_windows_portable\ComfyUI\execution.py", line 152, in recursive_execute
output_data, output_ui = get_output_data(obj, input_data_all)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "F:\python\ComfyUI_windows_portable\ComfyUI\execution.py", line 82, in get_output_data
return_values = map_node_over_list(obj, input_data_all, obj.FUNCTION, allow_interrupt=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "F:\python\ComfyUI_windows_portable\ComfyUI\execution.py", line 75, in map_node_over_list
results.append(getattr(obj, func)(**slice_dict(input_data_all, i)))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "F:\python\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\nodes.py", line 181, in run_inference
return motionctrl_sample(
^^^^^^^^^^^^^^^^^^
File "F:\python\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\gradio_utils\motionctrl_cmcm_gradio.py", line 148, in motionctrl_sample
c, uc = model.conditioner.get_unconditional_conditioning(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "F:\python\ComfyUI_windows_portable\ComfyUI\comfy..\custom_nodes\ComfyUI-MotionCtrl-SVD\sgm\modules\encoders\modules.py", line 179, in get_unconditional_conditioning
c = self(batch_c, force_cond_zero_embeddings)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "F:\python\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1511, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "F:\python\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1520, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "F:\python\ComfyUI_windows_portable\ComfyUI\comfy..\custom_nodes\ComfyUI-MotionCtrl-SVD\sgm\modules\encoders\modules.py", line 132, in forward
emb_out = embedder(batch[embedder.input_key])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "F:\python\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1511, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "F:\python\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1520, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "F:\python\ComfyUI_windows_portable\ComfyUI\comfy..\custom_nodes\ComfyUI-MotionCtrl-SVD\sgm\modules\encoders\modules.py", line 1012, in forward
out = self.encoder.encode(vid[n * n_samples : (n + 1) * n_samples])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "F:\python\ComfyUI_windows_portable\ComfyUI\comfy..\custom_nodes\ComfyUI-MotionCtrl-SVD\sgm\models\autoencoder.py", line 472, in encode
z = self.encoder(x)
^^^^^^^^^^^^^^^
File "F:\python\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1511, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "F:\python\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1520, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "F:\python\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\sgm\modules\diffusionmodules\model.py", line 594, in forward
h = self.mid.attn_1(h)
^^^^^^^^^^^^^^^^^^
File "F:\python\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1511, in _wrapped_call_impl
return self.call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "F:\python\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1520, in call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "F:\python\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\sgm\modules\diffusionmodules\model.py", line 263, in forward
h = self.attention(h)
^^^^^^^^^^^^^^^^^^
File "F:\python\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\sgm\modules\diffusionmodules\model.py", line 249, in attention
out = xformers.ops.memory_efficient_attention(
^^^^^^^^

After installation, there is a red module

Hello, I want to animate clouds on static shot with MotionCtrl on SVD. I made static example with

[
    [
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0
    ],
   ... // 25 times
]

but camera was static only once...

Thanks for the model.

I have a Macbook M3 Max with 64 GB shared RAM.

When trying to run the default workflow in Comfy, I am running into CUDA errors.
Is there any official MPS support on the way?

Here is the terminal's output when I try to run the default workflow:

> $ python main.py --force-fp16 --use-split-cross-attention
** ComfyUI startup time: 2024-02-23 16:01:06.190700
** Platform: Darwin
** Python version: 3.12.1 | packaged by Anaconda, Inc. | (main, Jan 19 2024, 09:45:58) [Clang 14.0.6 ]
** Python executable: /Users/user/miniconda3/envs/comfy-svd/bin/python
** Log path: /Users/user/repos/Comfy-mt/ComfyUI/comfyui.log

Prestartup times for custom nodes:
   0.0 seconds: /Users/user/repos/Comfy-mt/ComfyUI/custom_nodes/ComfyUI-Manager

Total VRAM 65536 MB, total RAM 65536 MB
Forcing FP16.
Set vram state to: SHARED
Device: mps
VAE dtype: torch.float32
Using split optimization for cross attention
### Loading: ComfyUI-Manager (V2.7.2)
### ComfyUI Revision: 2011 [10847dfa] | Released on '2024-02-23'
### Loading: ComfyUI-Impact-Pack (V4.80)
[ComfyUI-Manager] default cache updated: https://raw.githubusercontent.com/ltdrdata/ComfyUI-Manager/main/alter-list.json
[ComfyUI-Manager] default cache updated: https://raw.githubusercontent.com/ltdrdata/ComfyUI-Manager/main/model-list.json
[ComfyUI-Manager] default cache updated: https://raw.githubusercontent.com/ltdrdata/ComfyUI-Manager/main/extension-node-map.json
[ComfyUI-Manager] default cache updated: https://raw.githubusercontent.com/ltdrdata/ComfyUI-Manager/main/custom-node-list.json
### Loading: ComfyUI-Impact-Pack (Subpack: V0.4)
[Impact Pack] Wildcards loading done.
no module 'xformers'. Processing without...
no module 'xformers'. Processing without...
Using device cpu
------------------------------------------
Comfyroll Studio v1.76 :  175 Nodes Loaded
------------------------------------------
** For changes, please see patch notes at https://github.com/Suzie1/ComfyUI_Comfyroll_CustomNodes/blob/main/Patch_Notes.md
** For help, please see the wiki at https://github.com/Suzie1/ComfyUI_Comfyroll_CustomNodes/wiki
------------------------------------------

Import times for custom nodes:
   0.0 seconds: /Users/user/repos/Comfy-mt/ComfyUI/custom_nodes/Derfuu_ComfyUI_ModdedNodes
   0.0 seconds: /Users/user/repos/Comfy-mt/ComfyUI/custom_nodes/ComfyUI-Frame-Interpolation
   0.0 seconds: /Users/user/repos/Comfy-mt/ComfyUI/custom_nodes/ComfyMath
   0.0 seconds: /Users/user/repos/Comfy-mt/ComfyUI/custom_nodes/ComfyUI_Comfyroll_CustomNodes
   0.1 seconds: /Users/user/repos/Comfy-mt/ComfyUI/custom_nodes/ComfyUI-Manager
   0.1 seconds: /Users/user/repos/Comfy-mt/ComfyUI/custom_nodes/ComfyUI-VideoHelperSuite
   0.7 seconds: /Users/user/repos/Comfy-mt/ComfyUI/custom_nodes/ComfyUI-MotionCtrl-SVD
   6.4 seconds: /Users/user/repos/Comfy-mt/ComfyUI/custom_nodes/ComfyUI-Impact-Pack

Starting server

To see the GUI go to: http://127.0.0.1:8188
got prompt
Loading model from /Users/user/repos/Comfy-mt/ComfyUI/models/checkpoints/motionctrl_svd.ckpt
Using device cpu
no module 'xformers'. Processing without...
no module 'xformers'. Processing without...
Attention mode 'softmax-xformers' is not available. Falling back to native attention. This is not a problem in Pytorch >= 2.0. FYI, you are running with PyTorch version 2.3.0.dev20240222.
Attention mode 'softmax-xformers' is not available. Falling back to native attention. This is not a problem in Pytorch >= 2.0. FYI, you are running with PyTorch version 2.3.0.dev20240222.
Attention mode 'softmax-xformers' is not available. Falling back to native attention. This is not a problem in Pytorch >= 2.0. FYI, you are running with PyTorch version 2.3.0.dev20240222.
Attention mode 'softmax-xformers' is not available. Falling back to native attention. This is not a problem in Pytorch >= 2.0. FYI, you are running with PyTorch version 2.3.0.dev20240222.
Attention mode 'softmax-xformers' is not available. Falling back to native attention. This is not a problem in Pytorch >= 2.0. FYI, you are running with PyTorch version 2.3.0.dev20240222.
Attention mode 'softmax-xformers' is not available. Falling back to native attention. This is not a problem in Pytorch >= 2.0. FYI, you are running with PyTorch version 2.3.0.dev20240222.
Attention mode 'softmax-xformers' is not available. Falling back to native attention. This is not a problem in Pytorch >= 2.0. FYI, you are running with PyTorch version 2.3.0.dev20240222.
Attention mode 'softmax-xformers' is not available. Falling back to native attention. This is not a problem in Pytorch >= 2.0. FYI, you are running with PyTorch version 2.3.0.dev20240222.
Attention mode 'softmax-xformers' is not available. Falling back to native attention. This is not a problem in Pytorch >= 2.0. FYI, you are running with PyTorch version 2.3.0.dev20240222.
Attention mode 'softmax-xformers' is not available. Falling back to native attention. This is not a problem in Pytorch >= 2.0. FYI, you are running with PyTorch version 2.3.0.dev20240222.
Attention mode 'softmax-xformers' is not available. Falling back to native attention. This is not a problem in Pytorch >= 2.0. FYI, you are running with PyTorch version 2.3.0.dev20240222.
Attention mode 'softmax-xformers' is not available. Falling back to native attention. This is not a problem in Pytorch >= 2.0. FYI, you are running with PyTorch version 2.3.0.dev20240222.
Attention mode 'softmax-xformers' is not available. Falling back to native attention. This is not a problem in Pytorch >= 2.0. FYI, you are running with PyTorch version 2.3.0.dev20240222.
Attention mode 'softmax-xformers' is not available. Falling back to native attention. This is not a problem in Pytorch >= 2.0. FYI, you are running with PyTorch version 2.3.0.dev20240222.
Attention mode 'softmax-xformers' is not available. Falling back to native attention. This is not a problem in Pytorch >= 2.0. FYI, you are running with PyTorch version 2.3.0.dev20240222.
Attention mode 'softmax-xformers' is not available. Falling back to native attention. This is not a problem in Pytorch >= 2.0. FYI, you are running with PyTorch version 2.3.0.dev20240222.
ERROR:root:!!! Exception during processing !!!
ERROR:root:Traceback (most recent call last):
  File "/Users/user/repos/Comfy-mt/ComfyUI/execution.py", line 152, in recursive_execute
    output_data, output_ui = get_output_data(obj, input_data_all)
                             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/repos/Comfy-mt/ComfyUI/execution.py", line 82, in get_output_data
    return_values = map_node_over_list(obj, input_data_all, obj.FUNCTION, allow_interrupt=True)
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/repos/Comfy-mt/ComfyUI/execution.py", line 75, in map_node_over_list
    results.append(getattr(obj, func)(**slice_dict(input_data_all, i)))
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/repos/Comfy-mt/ComfyUI/custom_nodes/ComfyUI-MotionCtrl-SVD/nodes.py", line 152, in load_checkpoint
    model = build_model(config_path, ckpt_path, device, frame_length, steps)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/repos/Comfy-mt/ComfyUI/custom_nodes/ComfyUI-MotionCtrl-SVD/gradio_utils/motionctrl_cmcm_gradio.py", line 55, in build_model
    model, filter = load_model(
                    ^^^^^^^^^^^
  File "/Users/user/repos/Comfy-mt/ComfyUI/custom_nodes/ComfyUI-MotionCtrl-SVD/gradio_utils/motionctrl_cmcm_gradio.py", line 279, in load_model
    model = instantiate_from_config(config.model)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/repos/Comfy-mt/ComfyUI/custom_nodes/ComfyUI-MotionCtrl-SVD/sgm/util.py", line 175, in instantiate_from_config
    return get_obj_from_str(config["target"])(**config.get("params", dict()))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/repos/Comfy-mt/ComfyUI/comfy/../custom_nodes/ComfyUI-MotionCtrl-SVD/sgm/motionctrl/camera_motion_control.py", line 24, in __init__
    super().__init__(*args, **kwargs)
  File "/Users/user/repos/Comfy-mt/ComfyUI/custom_nodes/ComfyUI-MotionCtrl-SVD/sgm/models/diffusion.py", line 60, in __init__
    self.conditioner = instantiate_from_config(
                       ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/repos/Comfy-mt/ComfyUI/custom_nodes/ComfyUI-MotionCtrl-SVD/sgm/util.py", line 175, in instantiate_from_config
    return get_obj_from_str(config["target"])(**config.get("params", dict()))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/repos/Comfy-mt/ComfyUI/comfy/../custom_nodes/ComfyUI-MotionCtrl-SVD/sgm/modules/encoders/modules.py", line 79, in __init__
    embedder = instantiate_from_config(embconfig)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/repos/Comfy-mt/ComfyUI/comfy/../custom_nodes/ComfyUI-MotionCtrl-SVD/sgm/util.py", line 175, in instantiate_from_config
    return get_obj_from_str(config["target"])(**config.get("params", dict()))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/repos/Comfy-mt/ComfyUI/comfy/../custom_nodes/ComfyUI-MotionCtrl-SVD/sgm/modules/encoders/modules.py", line 1038, in __init__
    self.open_clip = instantiate_from_config(open_clip_embedding_config)
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/repos/Comfy-mt/ComfyUI/comfy/../custom_nodes/ComfyUI-MotionCtrl-SVD/sgm/util.py", line 175, in instantiate_from_config
    return get_obj_from_str(config["target"])(**config.get("params", dict()))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/repos/Comfy-mt/ComfyUI/comfy/../custom_nodes/ComfyUI-MotionCtrl-SVD/sgm/modules/encoders/modules.py", line 591, in __init__
    model, _, _ = open_clip.create_model_and_transforms(
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/miniconda3/envs/comfy-svd/lib/python3.12/site-packages/open_clip/factory.py", line 384, in create_model_and_transforms
    model = create_model(
            ^^^^^^^^^^^^^
  File "/Users/user/miniconda3/envs/comfy-svd/lib/python3.12/site-packages/open_clip/factory.py", line 276, in create_model
    model.to(device=device)
  File "/Users/user/miniconda3/envs/comfy-svd/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1170, in to
    return self._apply(convert)
           ^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/miniconda3/envs/comfy-svd/lib/python3.12/site-packages/torch/nn/modules/module.py", line 778, in _apply
    module._apply(fn)
  File "/Users/user/miniconda3/envs/comfy-svd/lib/python3.12/site-packages/torch/nn/modules/module.py", line 778, in _apply
    module._apply(fn)
  File "/Users/user/miniconda3/envs/comfy-svd/lib/python3.12/site-packages/torch/nn/modules/module.py", line 803, in _apply
    param_applied = fn(param)
                    ^^^^^^^^^
  File "/Users/user/miniconda3/envs/comfy-svd/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1156, in convert
    return t.to(
           ^^^^^
  File "/Users/user/miniconda3/envs/comfy-svd/lib/python3.12/site-packages/torch/cuda/__init__.py", line 309, in _lazy_init
    raise AssertionError("Torch not compiled with CUDA enabled")
AssertionError: Torch not compiled with CUDA enabled

Prompt executed in 9.38 seconds

Error occurred when executing Load Motionctrl-SVD Checkpoint:

No module named 'sgm.motionctrl'

File "/home/ComfyUI/execution.py", line 154, in recursive_execute
output_data, output_ui = get_output_data(obj, input_data_all)
File "/home/ComfyUI/execution.py", line 84, in get_output_data
return_values = map_node_over_list(obj, input_data_all, obj.FUNCTION, allow_interrupt=True)

I have followed all installation steps but unable to run the model in Comfy. Any help?

Thanks for this node! The installation is a bit tricky but once running its very consistent.

I'm running the workflow with no problems on a 24GB 3090, but I get an OOM error on my 12GB 3060.

I tried converting the model to fp16 but I still get a OOM error.

Is there a way to run this on 12GB?

Appreciate your work, thanks in advance!

OOM log below

Error occurred when executing Load Motionctrl-SVD Checkpoint:

Allocation on device 0 would exceed allowed memory. (out of memory)
Currently allocated     : 11.27 GiB
Requested               : 112.50 MiB
Device limit            : 12.00 GiB
Free (according to CUDA): 0 bytes
PyTorch limit (set by user-supplied memory fraction)
                        : 17179869184.00 GiB

  File "E:\AI\Apps\StableDiffusionWebUI\ComfyUI\ComfyUI\execution.py", line 155, in recursive_execute
    output_data, output_ui = get_output_data(obj, input_data_all)
  File "E:\AI\Apps\StableDiffusionWebUI\ComfyUI\ComfyUI\execution.py", line 85, in get_output_data
    return_values = map_node_over_list(obj, input_data_all, obj.FUNCTION, allow_interrupt=True)
  File "E:\AI\Apps\StableDiffusionWebUI\ComfyUI\ComfyUI\execution.py", line 78, in map_node_over_list
    results.append(getattr(obj, func)(**slice_dict(input_data_all, i)))
  File "E:\AI\Apps\StableDiffusionWebUI\ComfyUI\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\nodes.py", line 152, in load_checkpoint
    model = build_model(config_path, ckpt_path, device, frame_length, steps)
  File "E:\AI\Apps\StableDiffusionWebUI\ComfyUI\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\gradio_utils\motionctrl_cmcm_gradio.py", line 55, in build_model
    model, filter = load_model(
  File "E:\AI\Apps\StableDiffusionWebUI\ComfyUI\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\gradio_utils\motionctrl_cmcm_gradio.py", line 281, in load_model
    model = model.to(device).eval()
  File "E:\AI\Apps\StableDiffusionWebUI\ComfyUI\python_embeded\lib\site-packages\lightning_fabric\utilities\device_dtype_mixin.py", line 54, in to
    return super().to(*args, **kwargs)
  File "E:\AI\Apps\StableDiffusionWebUI\ComfyUI\python_embeded\lib\site-packages\torch\nn\modules\module.py", line 1160, in to
    return self._apply(convert)
  File "E:\AI\Apps\StableDiffusionWebUI\ComfyUI\python_embeded\lib\site-packages\torch\nn\modules\module.py", line 810, in _apply
    module._apply(fn)
  File "E:\AI\Apps\StableDiffusionWebUI\ComfyUI\python_embeded\lib\site-packages\torch\nn\modules\module.py", line 810, in _apply
    module._apply(fn)
  File "E:\AI\Apps\StableDiffusionWebUI\ComfyUI\python_embeded\lib\site-packages\torch\nn\modules\module.py", line 810, in _apply
    module._apply(fn)
  [Previous line repeated 4 more times]
  File "E:\AI\Apps\StableDiffusionWebUI\ComfyUI\python_embeded\lib\site-packages\torch\nn\modules\module.py", line 833, in _apply
    param_applied = fn(param)
  File "E:\AI\Apps\StableDiffusionWebUI\ComfyUI\python_embeded\lib\site-packages\torch\nn\modules\module.py", line 1158, in convert
    return t.to(device, dtype if t.is_floating_point() or t.is_complex() else None, non_blocking)

Just clicked queue prompt,

Python 310, Win11, torch212+cu121

heres the log

got prompt
Loading model from C:\AI\ComfyUI\models\checkpoints\motionctrl_svd.ckpt
Using device cuda
Initialized embedder #0: FrozenOpenCLIPImagePredictionEmbedder with 683800065 params. Trainable: False
Initialized embedder #1: ConcatTimestepEmbedderND with 0 params. Trainable: False
Initialized embedder #2: ConcatTimestepEmbedderND with 0 params. Trainable: False
Initialized embedder #3: VideoPredictionEmbedderWithEncoder with 83653863 params. Trainable: False
Initialized embedder #4: ConcatTimestepEmbedderND with 0 params. Trainable: False
Restored from C:\AI\ComfyUI\models\checkpoints\motionctrl_svd.ckpt with 0 missing and 0 unexpected keys
ERROR:root:!!! Exception during processing !!!
ERROR:root:Traceback (most recent call last):
File "C:\AI\ComfyUI\execution.py", line 155, in recursive_execute
output_data, output_ui = get_output_data(obj, input_data_all)
File "C:\AI\ComfyUI\execution.py", line 85, in get_output_data
return_values = map_node_over_list(obj, input_data_all, obj.FUNCTION, allow_interrupt=True)
File "C:\AI\ComfyUI\execution.py", line 78, in map_node_over_list
results.append(getattr(obj, func)(**slice_dict(input_data_all, i)))
File "C:\AI\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\nodes.py", line 181, in run_inference
return motionctrl_sample(
File "C:\AI\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\gradio_utils\motionctrl_cmcm_gradio.py", line 185, in motionctrl_sample
samples_x = model.decode_first_stage(samples_z)
File "C:\AI\python_embeded\lib\site-packages\torch\utils_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
File "C:\AI\python_embeded\lib\site-packages\sgm\models\diffusion.py", line 130, in decode_first_stage
out = self.first_stage_model.decode(
File "C:\AI\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\sgm\models\autoencoder.py", line 211, in decode
x = self.decoder(z, **kwargs)
File "C:\AI\python_embeded\lib\site-packages\torch\nn\modules\module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "C:\AI\python_embeded\lib\site-packages\torch\nn\modules\module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "C:\AI\python_embeded\lib\site-packages\sgm\modules\diffusionmodules\model.py", line 726, in forward
h = self.mid.block_1(h, temb, **kwargs)
File "C:\AI\python_embeded\lib\site-packages\torch\nn\modules\module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "C:\AI\python_embeded\lib\site-packages\torch\nn\modules\module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "C:\AI\ComfyUI\custom_nodes\ComfyUI-MotionCtrl-SVD\sgm\modules\autoencoding\temporal_ae.py", line 66, in forward
timesteps = self.timesteps
File "C:\AI\python_embeded\lib\site-packages\torch\nn\modules\module.py", line 1695, in getattr
raise AttributeError(f"'{type(self).name}' object has no attribute '{name}'")
AttributeError: 'NewCls' object has no attribute 'timesteps'
Prompt executed in 58.97 seconds

torch.cuda.OutOfMemoryError: Allocation on device 0 would exceed allowed memory. (out of memory)
Currently allocated : 10.17 GiB
Requested : 1.23 GiB
Device limit : 12.00 GiB
Free (according to CUDA): 0 bytes
PyTorch limit (set by user-supplied memory fraction)
: 17179869184.00 GiB

It's just me or It's doesn't seems to works on windows ? thanks