DataParallel is Supported for XPU?

[yash3056]

Describe the issue

I am facing error's with DataParallel.

[alexsin368]

Hi @yash3056 please describe your issue in detail and provide the code and steps to reproduce it.

[gujinghui]

@yash3056

The DP should be not fully supported by XPU for now. May I know why the DP is needed in your case, instead of DDP? I remember, DP will be obsoleted by PyTorch on GPU.

[yash3056]

@yash3056

The DP should be not fully supported by XPU for now. May I know why the DP is needed in your case, instead of DDP? I remember, DP will be obsoleted by PyTorch on GPU.

1. I wanted confirmation that DP is not supported. I am also facing error with DDP
2. I am facing engine error with XPU

Here is the code in which I am facing engine error

%%

#!pip install accelerate==1.0.0rc1 datasets

%%

from datasets import load_dataset from transformers import BertTokenizer, BertForSequenceClassification, AdamW from torch.utils.data import DataLoader from accelerate import Accelerator import torch from sklearn.metrics import accuracy_score

Load IMDB dataset

dataset = load_dataset("imdb")

Initialize the BERT tokenizer

tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

Tokenize the data

def tokenize_function(examples): return tokenizer(examples['text'], padding="max_length", truncation=True, max_length=512)

Tokenize the train and test dataset

tokenized_datasets = dataset.map(tokenize_function, batched=True) tokenized_datasets.set_format(type="torch", columns=['input_ids', 'attention_mask', 'label'])

train_dataset = tokenized_datasets['train'] test_dataset = tokenized_datasets['test']

Define DataLoader for batching

train_dataloader = DataLoader(train_dataset, batch_size=8, shuffle=True) test_dataloader = DataLoader(test_dataset, batch_size=8)

Load pre-trained BERT model with a classification head

model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)

Optimizer

optimizer = AdamW(model.parameters(), lr=5e-5)

Initialize Accelerator

accelerator = Accelerator() device = accelerator.device print(device)

Move model and optimizer to the appropriate device

model, optimizer, train_dataloader, test_dataloader = accelerator.prepare( model, optimizer, train_dataloader, test_dataloader )

%%

from tqdm.auto import tqdm

def train(model, dataloader, optimizer, accelerator): model.train() total_loss = 0

# Use tqdm for progress bar
loop = tqdm(dataloader, leave=True, desc="Training")

for batch in loop:
    # Forward pass
    outputs = model(**batch)
    loss = outputs.loss

    # Backward pass
    accelerator.backward(loss)

    # Optimization step
    optimizer.step()
    optimizer.zero_grad()

    total_loss += loss.item()

    # Update tqdm description with the current loss
    loop.set_description(f"Training Loss: {loss.item():.4f}")

avg_loss = total_loss / len(dataloader)
return avg_loss

%%

def evaluate(model, dataloader, accelerator): model.eval() predictions, labels = [], []

# Use tqdm for progress bar
loop = tqdm(dataloader, leave=True, desc="Evaluating")

with torch.no_grad():
    for batch in loop:
        # Forward pass
        outputs = model(**batch)
        logits = outputs.logits
        preds = torch.argmax(logits, dim=-1)

        predictions.extend(accelerator.gather(preds).cpu().numpy())
        labels.extend(accelerator.gather(batch['labels']).cpu().numpy())

# Calculate accuracy
accuracy = accuracy_score(labels, predictions)
return accuracy

%%

def train(model, dataloader, optimizer, accelerator): model.train() total_loss = 0

# Use tqdm for progress bar
loop = tqdm(dataloader, leave=True, desc="Training")

for batch in loop:
    # Forward pass
    # Only pass input_ids and attention_mask to the model
    outputs = model(input_ids=batch['input_ids'], attention_mask=batch['attention_mask'], labels=batch['label'])
    loss = outputs.loss

    # Backward pass
    accelerator.backward(loss)

    # Optimization step
    optimizer.step()
    optimizer.zero_grad()

    total_loss += loss.item()

    # Update tqdm description with the current loss
    loop.set_description(f"Training Loss: {loss.item():.4f}")

avg_loss = total_loss / len(dataloader)
return avg_loss

%%

epochs = 3

for epoch in range(epochs): # Train the model avg_train_loss = train(model, train_dataloader, optimizer, accelerator)

# Evaluate the model
accuracy = evaluate(model, test_dataloader, accelerator)

print(f"Epoch {epoch+1}/{epochs}")
print(f"Training Loss: {avg_train_loss:.4f}")
print(f"Validation Accuracy: {accuracy:.4f}")

here is the error

[WARNING] Failed to create Level Zero tracer: 2013265921 { "name": "RuntimeError", "message": "could not create an engine", "stack": "--------------------------------------------------------------------------- RuntimeError Traceback (most recent call last) Cell In[6], line 5 1 epochs = 3 3 for epoch in range(epochs): 4 # Train the model ----> 5 avg_train_loss = train(model, train_dataloader, optimizer, accelerator) 7 # Evaluate the model 8 accuracy = evaluate(model, test_dataloader, accelerator)

Cell In[5], line 11, in train(model, dataloader, optimizer, accelerator) 6 loop = tqdm(dataloader, leave=True, desc="Training") 8 for batch in loop: 9 # Forward pass 10 # Only pass input_ids and attention_mask to the model ---> 11 outputs = model(input_ids=batch['input_ids'], attention_mask=batch['attention_mask'], labels=batch['label']) 12 loss = outputs.loss 14 # Backward pass

File ~/.conda/envs/aza/lib/python3.10/site-packages/torch/nn/modules/module.py:1532, in Module._wrapped_call_impl(self, *args, **kwargs) 1530 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc] 1531 else: -> 1532 return self._call_impl(*args, **kwargs)

File ~/.conda/envs/aza/lib/python3.10/site-packages/torch/nn/modules/module.py:1541, in Module._call_impl(self, *args, **kwargs) 1536 # If we don't have any hooks, we want to skip the rest of the logic in 1537 # this function, and just call forward. 1538 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks 1539 or _global_backward_pre_hooks or _global_backward_hooks 1540 or _global_forward_hooks or _global_forward_pre_hooks): -> 1541 return forward_call(*args, **kwargs) 1543 try: 1544 result = None

File ~/.conda/envs/aza/lib/python3.10/site-packages/transformers/models/bert/modeling_bert.py:1695, in BertForSequenceClassification.forward(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict) 1687 r""" 1688 labels (torch.LongTensor of shape (batch_size,), optional): 1689 Labels for computing the sequence classification/regression loss. Indices should be in [0, ..., 1690 config.num_labels - 1]. If config.num_labels == 1 a regression loss is computed (Mean-Square loss), If 1691 config.num_labels > 1 a classification loss is computed (Cross-Entropy). 1692 """ 1693 return_dict = return_dict if return_dict is not None else self.config.use_return_dict -> 1695 outputs = self.bert( 1696 input_ids, 1697 attention_mask=attention_mask, 1698 token_type_ids=token_type_ids, 1699 position_ids=position_ids, 1700 head_mask=head_mask, 1701 inputs_embeds=inputs_embeds, 1702 output_attentions=output_attentions, 1703 output_hidden_states=output_hidden_states, 1704 return_dict=return_dict, 1705 ) 1707 pooled_output = outputs[1] 1709 pooled_output = self.dropout(pooled_output)

File ~/.conda/envs/aza/lib/python3.10/site-packages/torch/nn/modules/module.py:1532, in Module._wrapped_call_impl(self, *args, **kwargs) 1530 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc] 1531 else: -> 1532 return self._call_impl(*args, **kwargs)

File ~/.conda/envs/aza/lib/python3.10/site-packages/torch/nn/modules/module.py:1541, in Module._call_impl(self, *args, **kwargs) 1536 # If we don't have any hooks, we want to skip the rest of the logic in 1537 # this function, and just call forward. 1538 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks 1539 or _global_backward_pre_hooks or _global_backward_hooks 1540 or _global_forward_hooks or _global_forward_pre_hooks): -> 1541 return forward_call(*args, **kwargs) 1543 try: 1544 result = None

File ~/.conda/envs/aza/lib/python3.10/site-packages/transformers/models/bert/modeling_bert.py:1141, in BertModel.forward(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict) 1134 # Prepare head mask if needed 1135 # 1.0 in head_mask indicate we keep the head 1136 # attention_probs has shape bsz x n_heads x N x N 1137 # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] 1138 # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] 1139 head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers) -> 1141 encoder_outputs = self.encoder( 1142 embedding_output, 1143 attention_mask=extended_attention_mask, 1144 head_mask=head_mask, 1145 encoder_hidden_states=encoder_hidden_states, 1146 encoder_attention_mask=encoder_extended_attention_mask, 1147 past_key_values=past_key_values, 1148 use_cache=use_cache, 1149 output_attentions=output_attentions, 1150 output_hidden_states=output_hidden_states, 1151 return_dict=return_dict, 1152 ) 1153 sequence_output = encoder_outputs[0] 1154 pooled_output = self.pooler(sequence_output) if self.pooler is not None else None

File ~/.conda/envs/aza/lib/python3.10/site-packages/torch/nn/modules/module.py:1532, in Module._wrapped_call_impl(self, *args, **kwargs) 1530 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc] 1531 else: -> 1532 return self._call_impl(*args, **kwargs)

File ~/.conda/envs/aza/lib/python3.10/site-packages/torch/nn/modules/module.py:1541, in Module._call_impl(self, *args, **kwargs) 1536 # If we don't have any hooks, we want to skip the rest of the logic in 1537 # this function, and just call forward. 1538 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks 1539 or _global_backward_pre_hooks or _global_backward_hooks 1540 or _global_forward_hooks or _global_forward_pre_hooks): -> 1541 return forward_call(*args, **kwargs) 1543 try: 1544 result = None

File ~/.conda/envs/aza/lib/python3.10/site-packages/transformers/models/bert/modeling_bert.py:694, in BertEncoder.forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict) 683 layer_outputs = self._gradient_checkpointing_func( 684 layer_module.call, 685 hidden_states, (...) 691 output_attentions, 692 ) 693 else: --> 694 layer_outputs = layer_module( 695 hidden_states, 696 attention_mask, 697 layer_head_mask, 698 encoder_hidden_states, 699 encoder_attention_mask, 700 past_key_value, 701 output_attentions, 702 ) 704 hidden_states = layer_outputs[0] 705 if use_cache:

File ~/.conda/envs/aza/lib/python3.10/site-packages/torch/nn/modules/module.py:1532, in Module._wrapped_call_impl(self, *args, **kwargs) 1530 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc] 1531 else: -> 1532 return self._call_impl(*args, **kwargs)

File ~/.conda/envs/aza/lib/python3.10/site-packages/torch/nn/modules/module.py:1541, in Module._call_impl(self, *args, **kwargs) 1536 # If we don't have any hooks, we want to skip the rest of the logic in 1537 # this function, and just call forward. 1538 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks 1539 or _global_backward_pre_hooks or _global_backward_hooks 1540 or _global_forward_hooks or _global_forward_pre_hooks): -> 1541 return forward_call(*args, **kwargs) 1543 try: 1544 result = None

File ~/.conda/envs/aza/lib/python3.10/site-packages/transformers/models/bert/modeling_bert.py:584, in BertLayer.forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions) 572 def forward( 573 self, 574 hidden_states: torch.Tensor, (...) 581 ) -> Tuple[torch.Tensor]: 582 # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 583 self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None --> 584 self_attention_outputs = self.attention( 585 hidden_states, 586 attention_mask, 587 head_mask, 588 output_attentions=output_attentions, 589 past_key_value=self_attn_past_key_value, 590 ) 591 attention_output = self_attention_outputs[0] 593 # if decoder, the last output is tuple of self-attn cache

File ~/.conda/envs/aza/lib/python3.10/site-packages/torch/nn/modules/module.py:1532, in Module._wrapped_call_impl(self, *args, **kwargs) 1530 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc] 1531 else: -> 1532 return self._call_impl(*args, **kwargs)

File ~/.conda/envs/aza/lib/python3.10/site-packages/torch/nn/modules/module.py:1541, in Module._call_impl(self, *args, **kwargs) 1536 # If we don't have any hooks, we want to skip the rest of the logic in 1537 # this function, and just call forward. 1538 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks 1539 or _global_backward_pre_hooks or _global_backward_hooks 1540 or _global_forward_hooks or _global_forward_pre_hooks): -> 1541 return forward_call(*args, **kwargs) 1543 try: 1544 result = None

File ~/.conda/envs/aza/lib/python3.10/site-packages/transformers/models/bert/modeling_bert.py:514, in BertAttention.forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions) 504 def forward( 505 self, 506 hidden_states: torch.Tensor, (...) 512 output_attentions: Optional[bool] = False, 513 ) -> Tuple[torch.Tensor]: --> 514 self_outputs = self.self( 515 hidden_states, 516 attention_mask, 517 head_mask, 518 encoder_hidden_states, 519 encoder_attention_mask, 520 past_key_value, 521 output_attentions, 522 ) 523 attention_output = self.output(self_outputs[0], hidden_states) 524 outputs = (attention_output,) + self_outputs[1:] # add attentions if we output them

File ~/.conda/envs/aza/lib/python3.10/site-packages/torch/nn/modules/module.py:1532, in Module._wrapped_call_impl(self, *args, **kwargs) 1530 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc] 1531 else: -> 1532 return self._call_impl(*args, **kwargs)

File ~/.conda/envs/aza/lib/python3.10/site-packages/torch/nn/modules/module.py:1541, in Module._call_impl(self, *args, **kwargs) 1536 # If we don't have any hooks, we want to skip the rest of the logic in 1537 # this function, and just call forward. 1538 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks 1539 or _global_backward_pre_hooks or _global_backward_hooks 1540 or _global_forward_hooks or _global_forward_pre_hooks): -> 1541 return forward_call(*args, **kwargs) 1543 try: 1544 result = None

File ~/.conda/envs/aza/lib/python3.10/site-packages/transformers/models/bert/modeling_bert.py:394, in BertSdpaSelfAttention.forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions) 382 return super().forward( 383 hidden_states, 384 attention_mask, (...) 389 output_attentions, 390 ) 392 bsz, tgt_len, _ = hidden_states.size() --> 394 query_layer = self.transpose_for_scores(self.query(hidden_states)) 396 # If this is instantiated as a cross-attention module, the keys and values come from an encoder; the attention 397 # mask needs to be such that the encoder's padding tokens are not attended to. 398 is_cross_attention = encoder_hidden_states is not None

File ~/.conda/envs/aza/lib/python3.10/site-packages/torch/nn/modules/module.py:1532, in Module._wrapped_call_impl(self, *args, **kwargs) 1530 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc] 1531 else: -> 1532 return self._call_impl(*args, **kwargs)

File ~/.conda/envs/aza/lib/python3.10/site-packages/torch/nn/modules/module.py:1541, in Module._call_impl(self, *args, **kwargs) 1536 # If we don't have any hooks, we want to skip the rest of the logic in 1537 # this function, and just call forward. 1538 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks 1539 or _global_backward_pre_hooks or _global_backward_hooks 1540 or _global_forward_hooks or _global_forward_pre_hooks): -> 1541 return forward_call(*args, **kwargs) 1543 try: 1544 result = None

File ~/.conda/envs/aza/lib/python3.10/site-packages/torch/nn/modules/linear.py:116, in Linear.forward(self, input) 115 def forward(self, input: Tensor) -> Tensor: --> 116 return F.linear(input, self.weight, self.bias)

RuntimeError: could not create an engine" }

[yash3056]

@gujinghui @alexsin368 This code is running fine with pytorch 2.6 (mainline)