transformers

transformers copied to clipboard

System Info

• transformers version: 4.18.1
• Platform: Linux Jupyter Notebook, TF2.3 Python 3.6, 2 GPU
• Python version: '1.7.1+cu101'
• Using GPU in script?: yes
• Using distributed or parallel set-up in script?: no

Who can help?

@mf

Information

• [ ] The official example scripts
• [X] My own modified scripts

Tasks

• [ ] An officially supported task in the examples folder (such as GLUE/SQuAD, ...)
• [X] My own task or dataset (give details below)

Reproduction

model_checkpoint = "xlm-roberta-large-finetuned-conll03-english"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint,add_prefix_space=True)

train_examples ={'texts':[x[0] for x in train_set],'tag_names':[x[1] for x in train_set]}

def isin(a, b):
    return a[1] > b[0] and a[0] < b[1]

def tokenize_and_align_labels(examples, label2id, max_length=256):
    tokenized_inputs = tokenizer(examples["texts"], truncation=True, padding='max_length', max_length=max_length,
                                 return_offsets_mapping=True)
   

    labels = []
    for i, label_idx_for_single_input in enumerate(tqdm.tqdm(examples["tag_names"])):

     

        labels_for_single_input = ['O' for _ in range(max_length)]
       
        text_offsets = tokenized_inputs['offset_mapping'][i]
       
        for entity in label_idx_for_single_input:

            tag = entity['tag']
          
            tag_offset = [entity['start'], entity['end']]
          

           
            affected_token_ids = [j for j in range(max_length) if isin(tag_offset, text_offsets[j])]


            if len(affected_token_ids) < 1:
              
                continue
            if any(labels_for_single_input[j] != 'O' for j in affected_token_ids):
          
                continue

            for j in affected_token_ids:
                labels_for_single_input[j] = 'I_' + tag
            labels_for_single_input[affected_token_ids[-1]] = 'L_' + tag
            labels_for_single_input[affected_token_ids[0]] = 'B_' + tag

        label_ids = [label2id[x] for x in labels_for_single_input]
        labels.append(label_ids)

    tokenized_inputs["labels"] = labels
    print(tokenized_inputs.keys())
    return tokenized_inputs

class MyDataset(torch.utils.data.Dataset):
    def __init__(self, examples):
        self.encodings = examples
        self.labels = examples['labels']

    def __getitem__(self, idx):
        item = {k: torch.tensor(v[idx]) for k, v in self.encodings.items()}
        item["labels"] = torch.tensor([self.labels[idx]])
        return item

    def __len__(self):

        return len(self.labels)

train_data=MyDataset(train_data)

model = AutoModelForTokenClassification.from_pretrained(model_checkpoint,id2label=id2label,label2id=label2id,ignore_mismatched_sizes=True)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)

args = TrainingArguments(
    "xlmroberta-finetuned-ner",
#     evaluation_strategy="epoch",
    save_strategy="epoch",
    learning_rate=2e-5,
    num_train_epochs=2,
    weight_decay=0.01,
    per_device_train_batch_size=4,
    # per_device_eval_batch_size=32
    fp16=True
    # bf16=True #Ampere GPU
)

trainer = Trainer(
    model=model,
    args=args,
    train_dataset=train_data,
    # eval_dataset=train_data,
    # data_collator=data_collator,
    # compute_metrics=compute_metrics,
    tokenizer=tokenizer)
trainer.train()

Using amp half precision backend
 FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning
  FutureWarning,
***** Running training *****
  Num examples = 141648
  Num Epochs = 2
  Instantaneous batch size per device = 4
  Total train batch size (w. parallel, distributed & accumulation) = 8
  Gradient Accumulation steps = 1
  Total optimization steps = 35412
 MLflow's log_param() only accepts values no longer than 250 characters so we dropped this attribute.
TypeError: __init__() got an unexpected keyword argument 'has_model_config'

Expected behavior

To train NER model