Clip Notebook not running

[rbunn80110]

I noticed you have a breakpoint in the code for the clip notebook. I'm assuming it isn't complete yet since you have a breakpoint in the code. I removed the breakpoint just to see what was going on, but couldn't figure out how to fix things from there. I replaced tpu with gpu in this code to see if it worked with ordinary GPUs, but unfortunately it did not

model = Model(1e-3)
trainer = pl.Trainer(
    max_epochs= 1,
    gpus=1,
    # gpus=torch.cuda.device_count(),
#     tpu_cores=1,
    gradient_clip_val=1.0,
    precision=16
)
trainer.fit(model, train_dl, valid_dl) # 

Traceback:

ValueError Traceback (most recent call last) in 8 precision=16 9 ) ---> 10 trainer.fit(model, train_dl, valid_dl) #

~/anaconda3/envs/keras/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in fit(self, model, train_dataloader, val_dataloaders, datamodule) 456 ) 457 --> 458 self._run(model) 459 460 assert self.state.stopped

~/anaconda3/envs/keras/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in _run(self, model) 754 755 # dispatch start_training or start_evaluating or start_predicting --> 756 self.dispatch() 757 758 # plugin will finalized fitting (e.g. ddp_spawn will load trained model)

~/anaconda3/envs/keras/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in dispatch(self) 795 self.accelerator.start_predicting(self) 796 else: --> 797 self.accelerator.start_training(self) 798 799 def run_stage(self):

~/anaconda3/envs/keras/lib/python3.7/site-packages/pytorch_lightning/accelerators/accelerator.py in start_training(self, trainer) 94 95 def start_training(self, trainer: 'pl.Trainer') -> None: ---> 96 self.training_type_plugin.start_training(trainer) 97 98 def start_evaluating(self, trainer: 'pl.Trainer') -> None:

~/anaconda3/envs/keras/lib/python3.7/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py in start_training(self, trainer) 142 def start_training(self, trainer: 'pl.Trainer') -> None: 143 # double dispatch to initiate the training loop --> 144 self._results = trainer.run_stage() 145 146 def start_evaluating(self, trainer: 'pl.Trainer') -> None:

~/anaconda3/envs/keras/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in run_stage(self) 805 if self.predicting: 806 return self.run_predict() --> 807 return self.run_train() 808 809 def _pre_training_routine(self):

~/anaconda3/envs/keras/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in run_train(self) 840 self.progress_bar_callback.disable() 841 --> 842 self.run_sanity_check(self.lightning_module) 843 844 self.checkpoint_connector.has_trained = False

~/anaconda3/envs/keras/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in run_sanity_check(self, ref_model) 1105 1106 # run eval step -> 1107 self.run_evaluation() 1108 1109 self.on_sanity_check_end()

~/anaconda3/envs/keras/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in run_evaluation(self, on_epoch) 960 # lightning module methods 961 with self.profiler.profile("evaluation_step_and_end"): --> 962 output = self.evaluation_loop.evaluation_step(batch, batch_idx, dataloader_idx) 963 output = self.evaluation_loop.evaluation_step_end(output) 964

~/anaconda3/envs/keras/lib/python3.7/site-packages/pytorch_lightning/trainer/evaluation_loop.py in evaluation_step(self, batch, batch_idx, dataloader_idx) 172 model_ref._current_fx_name = "validation_step" 173 with self.trainer.profiler.profile("validation_step"): --> 174 output = self.trainer.accelerator.validation_step(args) 175 176 # capture any logged information

~/anaconda3/envs/keras/lib/python3.7/site-packages/pytorch_lightning/accelerators/accelerator.py in validation_step(self, args) 224 225 with self.precision_plugin.val_step_context(), self.training_type_plugin.val_step_context(): --> 226 return self.training_type_plugin.validation_step(*args) 227 228 def test_step(self, args: List[Union[Any, int]]) -> Optional[STEP_OUTPUT]:

~/anaconda3/envs/keras/lib/python3.7/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py in validation_step(self, *args, **kwargs) 159 160 def validation_step(self, *args, **kwargs): --> 161 return self.lightning_module.validation_step(*args, **kwargs) 162 163 def test_step(self, *args, **kwargs):

in validation_step(self, batch, *args) 35 self, batch: Tuple[torch.Tensor, List[str]], *args: list 36 ) -> torch.Tensor: ---> 37 loss, img_acc, cap_acc = self.common_step(batch) 38 self.log("validation_loss", loss, on_step=True) 39 self.log("validation_img_acc", img_acc, on_step=True, prog_bar=True)

in common_step(self, batch) 16 17 image_embed = self.vision_encoder(images) ---> 18 caption_embed = self.caption_encoder(text) 19 similarity = caption_embed @ image_embed.T 20

~/anaconda3/envs/keras/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs) 887 result = self._slow_forward(*input, **kwargs) 888 else: --> 889 result = self.forward(*input, **kwargs) 890 for hook in itertools.chain( 891 _global_forward_hooks.values(),

in forward(self, x) 8 9 def forward(self, x): ---> 10 out = self.base(**x)[0] 11 out = out[:, 0, :] # get CLS token output 12 projected_vec = self.projection(out)

~/anaconda3/envs/keras/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs) 887 result = self._slow_forward(*input, **kwargs) 888 else: --> 889 result = self.forward(*input, **kwargs) 890 for hook in itertools.chain( 891 _global_forward_hooks.values(),

~/anaconda3/envs/keras/lib/python3.7/site-packages/transformers/models/distilbert/modeling_distilbert.py in forward(self, input_ids, attention_mask, head_mask, inputs_embeds, output_attentions, output_hidden_states, return_dict) 485 output_attentions=output_attentions, 486 output_hidden_states=output_hidden_states, --> 487 return_dict=return_dict, 488 ) 489

~/anaconda3/envs/keras/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs) 887 result = self._slow_forward(*input, **kwargs) 888 else: --> 889 result = self.forward(*input, **kwargs) 890 for hook in itertools.chain( 891 _global_forward_hooks.values(),

~/anaconda3/envs/keras/lib/python3.7/site-packages/transformers/models/distilbert/modeling_distilbert.py in forward(self, x, attn_mask, head_mask, output_attentions, output_hidden_states, return_dict) 305 306 layer_outputs = layer_module( --> 307 x=hidden_state, attn_mask=attn_mask, head_mask=head_mask[i], output_attentions=output_attentions 308 ) 309 hidden_state = layer_outputs[-1]

~/anaconda3/envs/keras/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs) 887 result = self._slow_forward(*input, **kwargs) 888 else: --> 889 result = self.forward(*input, **kwargs) 890 for hook in itertools.chain( 891 _global_forward_hooks.values(),

~/anaconda3/envs/keras/lib/python3.7/site-packages/transformers/models/distilbert/modeling_distilbert.py in forward(self, x, attn_mask, head_mask, output_attentions) 252 mask=attn_mask, 253 head_mask=head_mask, --> 254 output_attentions=output_attentions, 255 ) 256 if output_attentions:

~/anaconda3/envs/keras/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs) 887 result = self._slow_forward(*input, **kwargs) 888 else: --> 889 result = self.forward(*input, **kwargs) 890 for hook in itertools.chain( 891 _global_forward_hooks.values(),

~/anaconda3/envs/keras/lib/python3.7/site-packages/transformers/models/distilbert/modeling_distilbert.py in forward(self, query, key, value, mask, head_mask, output_attentions) 158 seq_length, dim) Contextualized layer. Optional: only if output_attentions=True 159 """ --> 160 bs, q_length, dim = query.size() 161 k_length = key.size(1) 162 # assert dim == self.dim, f'Dimensions do not match: {dim} input vs {self.dim} configured'

ValueError: too many values to unpack (expected 3)

[GuiQuQu]

I meet the same problem, and also don't know how to solve it

[GuiQuQu]

Now,I change the input shape of text_encoder to manage to debug. I write the cell and get the same bug: ''' text_encoder =TextEncoder(d_out=EMBED_DIM) for example in train_dl: img,texts = example print(img.shape) print(texts["input_ids"].shape) print(texts) #texts["input_ids"]=texts["input_ids"].view(BATCH_SIZE,-1) #texts["attention_mask"]=texts["attention_mask"].view(BATCH_SIZE,-1) output =text_encoder(texts) print("output shape:",output.shape) break ''' the wrong message : Some weights of the model checkpoint at distilbert-base-multilingual-cased were not used when initializing DistilBertModel: ['vocab_layer_norm.bias', 'vocab_transform.weight', 'vocab_layer_norm.weight', 'vocab_transform.bias', 'vocab_projector.weight', 'vocab_projector.bias']

• This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).

• This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model). /usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary. cpuset_checked)) torch.Size([64, 3, 128, 128]) torch.Size([64, 1, 32]) {'input_ids': tensor([[[ 101, 138, 71027, ..., 0, 0, 0]],

    [[  101,   138, 23840,  ...,     0,     0,     0]],
  
    [[  101,   138, 10189,  ...,     0,     0,     0]],
  
    ...,
  
    [[  101,   138, 11519,  ...,     0,     0,     0]],
  
    [[  101,   138, 38171,  ...,     0,     0,     0]],
  
    [[  101,   138, 13000,  ...,     0,     0,     0]]]), 'attention_mask': tensor([[[1, 1, 1,  ..., 0, 0, 0]],
  
    [[1, 1, 1,  ..., 0, 0, 0]],
  
    [[1, 1, 1,  ..., 0, 0, 0]],
  
    ...,
  
    [[1, 1, 1,  ..., 0, 0, 0]],
  
    [[1, 1, 1,  ..., 0, 0, 0]],
  
    [[1, 1, 1,  ..., 0, 0, 0]]])}
  

torch.Size([64, 1, 32])

ValueError Traceback (most recent call last) in () 7 #texts["input_ids"]=texts["input_ids"].view(BATCH_SIZE,-1) 8 #texts["attention_mask"]=texts["attention_mask"].view(BATCH_SIZE,-1) ----> 9 output =text_encoder(texts) 10 print("output shape:",output.shape) 11 break

9 frames /usr/local/lib/python3.7/dist-packages/transformers/models/distilbert/modeling_distilbert.py in forward(self, query, key, value, mask, head_mask, output_attentions) 181 seq_length, dim) Contextualized layer. Optional: only if output_attentions=True 182 """ --> 183 bs, q_length, dim = query.size() 184 k_length = key.size(1) 185 # assert dim == self.dim, f'Dimensions do not match: {dim} input vs {self.dim} configured'

ValueError: too many values to unpack (expected 3)

I find the shape of the input_ids and attention_mask is (BATCH_SIZE,1,MAX_LEN),I change their shape to (BATCH_SIZE,MAX_LEN) ,and get the output of text_encoder successfully(uncomment the caption in my code cell)