intel-extension-for-transformers.ipynb

[ayttop]

!git clone https://github.com/intel/intel-extension-for-transformers.git

!pip install intel-extension-for-transformers

!pip install --upgrade neural_compressor==2.6

!python /content/intel-extension-for-transformers/examples/huggingface/pytorch/translation/quantization/run_translation.py
--model_name_or_path Helsinki-NLP/opus-mt-en-ro
--do_train
--do_eval
--source_lang en
--target_lang ro
--dataset_name wmt/wmt16
--dataset_config_name ro-en
--output_dir /tmp/tst-translation
--per_device_train_batch_size 4
--per_device_eval_batch

from datasets import load_dataset

اسم مجموعة البيانات التي تريد تحميلها (مثلاً: 'wmt16' للترجمة)

dataset_name = 'wmt16' dataset_config_name = 'ro-en' # تكوين مجموعة البيانات، إذا كان موجوداً

تحميل مجموعة البيانات

dataset = load_dataset(dataset_name, dataset_config_name)

عرض بعض الأمثلة من مجموعة البيانات

print("Train Examples:") for example in dataset['train'][:2]: print(example)

print("\nValidation Examples:") for example in dataset['validation'][:2]: print(example)

print("\nTest Examples:") for example in dataset['test'][:2]: print(example)

!pip install sacrebleu==2.2.1

!python /content/intel-extension-for-transformers/examples/huggingface/pytorch/translation/quantization/run_translation.py
--model_name_or_path Helsinki-NLP/opus-mt-en-ro
--do_train
--do_eval
--source_lang en
--target_lang ro
--dataset_name wmt16
--dataset_config_name ro-en
--output_dir /tmp/tst-translation
--per_device_train_batch_size 4
--per_device_eval_batch_size 4
--overwrite_output_dir
--tune
--predict_with_generate

!python /content/intel-extension-for-transformers/examples/huggingface/pytorch/translation/quantization/run_translation.py
--model_name_or_path t5-small
--do_train
--do_eval
--source_lang en
--target_lang ro
--source_prefix "translate English to Romanian:For system requirements and other installation tips please refer to Installation Guide "
--dataset_name wmt16
--dataset_config_name ro-en
--output_dir /tmp/tst-translation
--per_device_train_batch_size 4
--per_device_eval_batch_size 4
--overwrite_output_dir
--tune
--predict_with_generate

!pip uninstall tensorflow !pip install tensorflow # Make sure to install the version compatible with your CUDA version