Idefics-2-base model fine-tuning throws indexing error about transformers HOT 10 CLOSED

import torch
from transformers import AutoProcessor, Idefics2ForConditionalGeneration

processor_base = AutoProcessor.from_pretrained(
    "HuggingFaceM4/idefics2-8b-base"
)
processor_chat = AutoProcessor.from_pretrained(
    "HuggingFaceM4/idefics2-8b"
)

base = Idefics2ForConditionalGeneration.from_pretrained(
        "HuggingFaceM4/idefics2-8b-base",
        torch_dtype=torch.float16
    )
chat = Idefics2ForConditionalGeneration.from_pretrained(
        "HuggingFaceM4/idefics2-8b",
        torch_dtype=torch.float16
    )
    
print("Tokenizer chat max token:", max(processor_chat.tokenizer.get_vocab().values()))
print("Tokenizer base max token:", max(processor_base.tokenizer.get_vocab().values()))

print("chat embedding:", chat.base_model.get_submodule('text_model').get_submodule('embed_tokens'))
print("base embedding:", base.base_model.get_submodule('text_model').get_submodule('embed_tokens'))

print("last token:", processor_chat.tokenizer.convert_ids_to_tokens(max(processor_chat.tokenizer.get_vocab().values())))

Tokenizer chat max token: 32002
Tokenizer base max token: 32002
chat embedding: Embedding(32003, 4096, padding_idx=0)
base embedding: Embedding(32002, 4096, padding_idx=0)
last token: <end_of_utterance>

  text = processor.apply_chat_template(messages, add_generation_prompt=False)
  if "base" in args.model_name: # hack to remove the end of utterance token
      text = text.replace("<end_of_utterance>", "")

Traceback (most recent call last):
  File "/apps/arch/distro/python/3.8/lib/python3.8/runpy.py", line 193, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/apps/arch/distro/python/3.8/lib/python3.8/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "synth-diffuse/evals/idefics2_fine_tuning.py", line 299, in <module>
    main(args)
  File "synth-diffuse/evals/idefics2_fine_tuning.py", line 103, in main
    model.add_adapter(lora_config)
  File /lib/python3.8/site-packages/transformers/integrations/peft.py", line 264, in add_adapter
    inject_adapter_in_model(adapter_config, self, adapter_name)
  File "/lib/python3.8/site-packages/peft/mapping.py", line 166, in inject_adapter_in_model
    peft_model = tuner_cls(model, peft_config, adapter_name=adapter_name)
  File "/lib/python3.8/site-packages/peft/tuners/lora/model.py", line 136, in __init__
    super().__init__(model, config, adapter_name)
  File "/lib/python3.8/site-packages/peft/tuners/tuners_utils.py", line 148, in __init__
    self.inject_adapter(self.model, adapter_name)
  File "lib/python3.8/site-packages/peft/tuners/tuners_utils.py", line 325, in inject_adapter
    self._create_and_replace(peft_config, adapter_name, target, target_name, parent, current_key=key)
  File "/lib/python3.8/site-packages/peft/tuners/lora/model.py", line 220, in _create_and_replace
    new_module = self._create_new_module(lora_config, adapter_name, target, **kwargs)
  File "/lib/python3.8/site-packages/peft/tuners/lora/model.py", line 295, in _create_new_module
    new_module = dispatcher(target, adapter_name, lora_config=lora_config, **kwargs)
  File "/lib/python3.8/site-packages/peft/tuners/lora/layer.py", line 1056, in dispatch_default
    new_module = Linear(target, adapter_name, **kwargs)
  File "/lib/python3.8/site-packages/peft/tuners/lora/layer.py", line 356, in __init__
    self.update_layer(
  File "/s/lib/python3.8/site-packages/peft/tuners/lora/layer.py", line 126, in update_layer
    self.dora_init(adapter_name)
  File "/lib/python3.8/site-packages/peft/tuners/lora/layer.py", line 191, in dora_init
    lora_weight = lora_B.weight @ lora_A.weight
RuntimeError: "addmm_impl_cpu_" not implemented for 'Half'