facebookresearch / perfect Goto Github PK

When trying to run bash scripts/perfect.sh, training gets stuck somewhere in a Linear layer of PyTorch.

For context, I'm for now only interested in yielding some result training only for SST-5, even if it is totally useless, so I've reduced some parameters to lower values to reduce computing time on a laptop with limited resources. They are as follows (defined in configs/perfect.json:

{
"task": "sst-5",
"max_seq_length": 256,
"per_device_train_batch_size": 2,
"K": 8,
"data_dir": "datasets_processed",
"output_dir": "outputs",
"model_name_or_path": "roberta-large",
"do_train": true,
"do_eval": true,
"do_predict": true,
"learning_rate": 1e-4,
"max_steps": 4,
"eval_steps": 3,
"save_steps": 3,
"data_seed": 50,
"seed": 1,
"load_best_model_at_end": true,
"metric_for_best_model": "average",
"greater_is_better": true,
"evaluation_strategy": "steps",
"save_strategy": "steps",
"save_total_limit": 1,
"overwrite_output_dir": true,
"soft_pet": true,
"soft_pet_loss": "extra_tokens",
"extra_without_original": true,
"extra_tokens_init": "random",
"soft_mask_labels_learning_rate": 1e-1,
"adapter_tune": true,
"tune_layernorms": true,
"add_layer_norm_after_adapter": false,
"add_layer_norm_before_adapter": false,
"train_in_batch": true,
"add_adapter_after_attention": false,
"add_adapter_after_feedforward": true,
"extra_embd_initializer_range": 1e-4,
"overwrite_cache": true,
"per_device_eval_batch_size": 10,
"prototypical_eval": true,
"eval_soft_pet_aggregation": "max",
"prototypical_similarity": "euc",
"token_hinge_loss": true,
"mask_position": "1"
}

The training procedure starts and gets stuck at step three out of four, first yielding the WARNING:

WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/timdadum/.cache/huggingface/datasets/json/default-9463d57d37ff152e/0.0.0/c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426/cache-91b7584a2265b1f5.arrow

Then, training gets stuck. Upon aborting, the following traceback is present:

Traceback (most recent call last):
File "run_clm.py", line 516, in
main()
File "run_clm.py", line 426, in main
train_result = trainer.train(resume_from_checkpoint=checkpoint)
File "/home/timdadum/anaconda3/envs/perfect/lib/python3.8/site-packages/transformers/trainer.py", line 1340, in train
self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval)
File "/home/timdadum/anaconda3/envs/perfect/lib/python3.8/site-packages/transformers/trainer.py", line 1445, in _maybe_log_save_evaluate
metrics = self.evaluate(ignore_keys=ignore_keys_for_eval)
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/trainers/trainer.py", line 133, in evaluate
output = self.eval_loop(
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/trainers/trainer.py", line 185, in eval_loop
metrics = self.compute_pet_metrics(eval_datasets, model, self.extra_info[metric_key_prefix])
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/trainers/trainer.py", line 210, in compute_pet_metrics
centroids = self._compute_per_token_train_centroids(model)
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/trainers/trainer.py", line 287, in _compute_per_token_train_centroids
mask_embeds.append(self.get_masks_embeds(model, batch))
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/trainers/trainer.py", line 267, in get_masks_embeds
hidden_states = model.roberta(input_ids=input_ids, attention_mask=attention_mask)
File "/home/timdadum/anaconda3/envs/perfect/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/models/roberta/modeling_roberta.py", line 868, in forward
encoder_outputs = self.encoder(
File "/home/timdadum/anaconda3/envs/perfect/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/models/roberta/modeling_roberta.py", line 541, in forward
layer_outputs = layer_module(
File "/home/timdadum/anaconda3/envs/perfect/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/models/roberta/modeling_roberta.py", line 468, in forward
layer_output = apply_chunking_to_forward(
File "/home/timdadum/anaconda3/envs/perfect/lib/python3.8/site-packages/transformers/modeling_utils.py", line 2196, in apply_chunking_to_forward
return forward_fn(*input_tensors)
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/models/roberta/modeling_roberta.py", line 481, in feed_forward_chunk
layer_output = self.output(intermediate_output, attention_output)
File "/home/timdadum/anaconda3/envs/perfect/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/models/roberta/modeling_roberta.py", line 393, in forward
hidden_states = self.dense(hidden_states)
File "/home/timdadum/anaconda3/envs/perfect/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/timdadum/anaconda3/envs/perfect/lib/python3.8/site-packages/torch/nn/modules/linear.py", line 114, in forward
return F.linear(input, self.weight, self.bias)
KeyboardInterrupt

Which seems to suggest that somehow PyTorch can't comprehend the data processing done in Arrow. I'm not sure how to fix this, so any help would be welcome!

Thank you for sharing this work.
I want to try training and testing the model on my own dataset.
Could someone provide a general guide on how I could do this?

When I run perfect.sh after following all the instructions, I got an error in loading the superglue dataset:

raise FileNotFoundError(
FileNotFoundError: Couldn't find a dataset script at /userhome/cs/tyyim/perfect/fewshot/super_glue/super_glue.py or any data file in the same directory. Couldn't find 'super_glue' on the Hugging Face Hub either: FileNotFoundError: Couldn't find file at https://raw.githubusercontent.com/huggingface/datasets/master/datasets/super_glue/super_glue.py

I suppose for RTE and CB (which I tried), it should be download from huggingface?
I have tested that I can download RTE alright with Jupyter notebook.

Traceback (most recent call last):
File "run_clm.py", line 517, in
main()
File "run_clm.py", line 427, in main
train_result = trainer.train(resume_from_checkpoint=checkpoint)
File "/Applications/anaconda3/envs/perfect/lib/python3.8/site-packages/transformers/trainer.py", line 1340, in train
self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval)
File "/Applications/anaconda3/envs/perfect/lib/python3.8/site-packages/transformers/trainer.py", line 1445, in _maybe_log_save_evaluate
metrics = self.evaluate(ignore_keys=ignore_keys_for_eval)
File "/Users/01119378/Documents/2022/perfect-main/fewshot/third_party/trainers/trainer.py", line 133, in evaluate
output = self.eval_loop(
File "/Users/01119378/Documents/2022/perfect-main/fewshot/third_party/trainers/trainer.py", line 185, in eval_loop
metrics = self.compute_pet_metrics(eval_datasets, model, self.extra_info[metric_key_prefix])
File "/Users/01119378/Documents/2022/perfect-main/fewshot/third_party/trainers/trainer.py", line 210, in compute_pet_metrics
centroids = self._compute_per_token_train_centroids(model)
File "/Users/01119378/Documents/2022/perfect-main/fewshot/third_party/trainers/trainer.py", line 281, in _compute_per_token_train_centroids
data = get_label_samples(self.train_dataset, label)
File "/Users/01119378/Documents/2022/perfect-main/fewshot/third_party/trainers/trainer.py", line 278, in get_label_samples
return dataset.filter(lambda example: int(example['labels']) == label)
File "/Applications/anaconda3/envs/perfect/lib/python3.8/site-packages/datasets/arrow_dataset.py", line 470, in wrapper
out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
File "/Applications/anaconda3/envs/perfect/lib/python3.8/site-packages/datasets/fingerprint.py", line 406, in wrapper
out = func(self, *args, **kwargs)
File "/Applications/anaconda3/envs/perfect/lib/python3.8/site-packages/datasets/arrow_dataset.py", line 2519, in filter
indices = self.map(
File "/Applications/anaconda3/envs/perfect/lib/python3.8/site-packages/datasets/arrow_dataset.py", line 2036, in map
return self._map_single(
File "/Applications/anaconda3/envs/perfect/lib/python3.8/site-packages/datasets/arrow_dataset.py", line 503, in wrapper
out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
File "/Applications/anaconda3/envs/perfect/lib/python3.8/site-packages/datasets/arrow_dataset.py", line 470, in wrapper
out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
File "/Applications/anaconda3/envs/perfect/lib/python3.8/site-packages/datasets/fingerprint.py", line 406, in wrapper
out = func(self, *args, **kwargs)
File "/Applications/anaconda3/envs/perfect/lib/python3.8/site-packages/datasets/arrow_dataset.py", line 2248, in _map_single
return Dataset.from_file(cache_file_name, info=info, split=self.split)
File "/Applications/anaconda3/envs/perfect/lib/python3.8/site-packages/datasets/arrow_dataset.py", line 654, in from_file
return cls(
File "/Applications/anaconda3/envs/perfect/lib/python3.8/site-packages/datasets/arrow_dataset.py", line 593, in init
self.info.features = self.info.features.reorder_fields_as(inferred_features)
File "/Applications/anaconda3/envs/perfect/lib/python3.8/site-packages/datasets/features/features.py", line 1092, in reorder_fields_as
return Features(recursive_reorder(self, other))
File "/Applications/anaconda3/envs/perfect/lib/python3.8/site-packages/datasets/features/features.py", line 1081, in recursive_reorder
raise ValueError(f"Keys mismatch: between {source} and {target}" + stack_position)
ValueError: Keys mismatch: between {'indices': Value(dtype='uint64', id=None)} and {'candidates_ids': Sequence(feature=Sequence(feature=Value(dtype='int64', id=None), length=-1, id=None), length=-1, id=None), 'labels': Value(dtype='int64', id=None), 'attention_mask': Sequence(feature=Value(dtype='int8', id=None), length=-1, id=None), 'input_ids': Sequence(feature=Value(dtype='int32', id=None), length=-1, id=None), 'extra_fields': {}}
0%| | 2/6000 [08:36<430:29:39, 258.38s/it]