opencompass
opencompass copied to clipboard
[Bug] Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.
Prerequisite
- [X] I have searched Issues and Discussions but cannot get the expected help.
- [X] The bug has not been fixed in the latest version.
Type
I have modified the code (config is not considered code), or I'm working on my own tasks/models/datasets.
Environment
{'CUDA available': True, “CUDA available”:True, 'CUDA_HOME': None, 'CUDA_HOME':无, 'GCC': 'gcc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0', 'GCC':'GCC(Ubuntu 11.4.0-1ubuntu1~22.04)11.4.0', 'GPU 0': 'NVIDIA GeForce RTX 4090 D', 'GPU 0':'NVIDIA GeForce RTX 4090 D', 'MMEngine': '0.10.4', 'MMCengine':'0.10.4', 'MUSA available': False, 'MUSA available':False, 'OpenCV': '4.9.0', 'OpenCV':'4.9.0', 'PyTorch': '2.3.0', 'PyTorch':' 2.3.0', 'PyTorch compiling details': 'PyTorch built with:\n' 'PyTorch编译详细信息':'PyTorch构建时使用:\n' ' - GCC 9.3\n' GCC 9.3\n的 ' - C++ Version: 201703\n' C++版本:201703\n ' - Intel(R) oneAPI Math Kernel Library Version ' ' -英特尔(R)oneAPI数学内核库版本' '2023.1-Product Build 20230303 for Intel(R) 64 ' '2023.1-英特尔® 64产品构建20230303' 'architecture applications\n' '体系结构应用程序\n' ' - Intel(R) MKL-DNN v3.3.6 (Git Hash ' ' -英特尔(R)MKL-DNN v3.3.6(Git Hash ' '86e6af5974177e513fd3fee58425e1063e7f1361)\n' '86e6af5974177e513fd3fee58425e1063e7f1361)\n' ' - OpenMP 201511 (a.k.a. OpenMP 4.5)\n' '—OpenMP 201511(a.k.a. OpenMP 4.5)\n ' ' - LAPACK is enabled (usually provided by ' ' - LAPACK已启用(通常由' 'MKL)\n' 'MKL)\n' ' - NNPACK is enabled\n' '—NNPACK已启用\n' ' - CPU capability usage: AVX2\n' '—CPU能力使用率:AVX2\n' ' - CUDA Runtime 12.1\n' ' - CUDA 12.1\n' ' - NVCC architecture flags: ' ' - NVCC体系结构标志:' '-gencode;arch=compute_50,code=sm_50;-gencode;arch=compute_60,code=sm_60;-gencode;arch=compute_61,code=sm_61;-gencode;arch=compute_70,code=sm_70;-gencode;arch=compute_75,code=sm_75;-gencode;arch=compute_80,code=sm_80;-gencode;arch=compute_86,code=sm_86;-gencode;arch=compute_90,code=sm_90\n' '-gencode; arch= compute_50,code =sm_50;-gencode;arch= compute_60,code =sm_60;-gencode;arch= compute_61,code =sm_61;-gencode;arch= compute_70,code =sm_70;-gencode;arch= compute_75,code =sm_75;-gencode;arch= compute_80,code =sm_80;-gencode;arch= compute_86,code =sm_86;-gencode;arch= compute_90,code =sm_90\n' ' - CuDNN 8.9.2\n' ' - Magma 2.6.1\n' ' - Build settings: BLAS_INFO=mkl, ' ' -构建设置:BLAS_INFO=mkl,' 'BUILD_TYPE=Release, CUDA_VERSION=12.1, ' 'BUILD_TYPE=Release,CUDA_VERSION=12.1,' 'CUDNN_VERSION=8.9.2, ' 'CUDNN_VERSION=8.9.2,' 'CXX_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/c++, ' 'CXX_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/c++,' 'CXX_FLAGS= -D_GLIBCXX_USE_CXX11_ABI=0 ' '-fabi-version=11 -fvisibility-inlines-hidden ' '-fabi-version=11 -fabability-inlines-hidden ' '-DUSE_PTHREADPOOL -DNDEBUG -DUSE_KINETO ' '-DLIBKINETO_NOROCTRACER -DUSE_FBGEMM ' '-DUSE_QNNPACK -DUSE_PYTORCH_QNNPACK ' '-DUSE_XNNPACK -DSYMBOLICATE_MOBILE_DEBUG_HANDLE ' '-DUSE_XNNPACK -DSYMBOLICATE_移动的_DEBUG_HANDLE ' '-O2 -fPIC -Wall -Wextra -Werror=return-type ' '-O2 -fPIC -Wall -Wextra -Werror=返回类型' '-Werror=non-virtual-dtor -Werror=bool-operation ' '-Werror=non-virtual-dtor -Werror= bool-operation' '-Wnarrowing -Wno-missing-field-initializers ' '-Wno-type-limits -Wno-array-bounds ' '-Wno-unknown-pragmas -Wno-unused-parameter ' '-Wno-unused-function -Wno-unused-result ' '-Wno-strict-overflow -Wno-strict-aliasing ' '-Wno-stringop-overflow -Wsuggest-override ' '-Wno-stringop-overflow -Wrest-override' '-Wno-psabi -Wno-error=pedantic ' '-Wno-psabi -Wno-error=迂腐' '-Wno-error=old-style-cast -Wno-missing-braces ' '-fdiagnostics-color=always -faligned-new ' '-Wno-unused-but-set-variable ' '-Wno-maybe-uninitialized -fno-math-errno ' '-fno-trapping-math -Werror=format ' '-Wno-stringop-overflow, LAPACK_INFO=mkl, ' '-Wno-stringop-overflow,LAPACK_INFO=mkl,' 'PERF_WITH_AVX=1, PERF_WITH_AVX2=1, ' PERF_WITH_AVX = 1,PERF_WITH_AVX2 = 1, 'PERF_WITH_AVX512=1, TORCH_VERSION=2.3.0, ' PERF_WITH_AVX512 = 1,TORCH_VERSION = 2.3.0,这是怎么回事? 'USE_CUDA=ON, USE_CUDNN=ON, USE_CUSPARSELT=1, ' USE_CUDA = ON,USE_CUDNN = ON,USE_CUPARSELT = 1,USE_CUDNN = ON,USE_CUDA = ON,USE_CUDNN = ON 'USE_EXCEPTION_PTR=1, USE_GFLAGS=OFF, ' 'USE_EXCEPTION_PTR=1,USE_GFLAGS=OFF,' 'USE_GLOG=OFF, USE_GLOO=ON, USE_MKL=ON, ' USE_GLOG = OFF,USE_GLOO = ON,USE_MKL = ON,这是怎么回事? 'USE_MKLDNN=ON, USE_MPI=OFF, USE_NCCL=ON, ' USE_MKLDNN = ON,USE_MPI = OFF,USE_NCCL = ON,这是怎么回事? 'USE_NNPACK=ON, USE_OPENMP=ON, USE_ROCM=OFF, ' USE_NNPACK = ON,USE_OPENMP = ON,USE_ROCM = OFF,使用: 'USE_ROCM_KERNEL_ASSERT=OFF, \n', 'USE_ROCM_KERNEL_ASSERT=OFF,\n', 'Python': '3.10.14 (main, May 6 2024, 19:42:50) [GCC 11.2.0]', 'Python':'3.10.14(main,May 6 2024,19:42:50)[GCC 11.2.0', 'TorchVision': '0.18.0', 'TorchVision':'0.18.0', 'numpy_random_seed': 2147483648, 'numpy_random_seed':2147483648, 'opencompass': '0.2.5+d656e81', 'opencompass':'0.2.5+ d656e81', 'sys.platform': 'linux'} "sys. platform":"linux"}
Reproduces the problem - code/configuration sample
Attempt to modify the "run(self)" function in tools/case_analyzer.py
def run(self):
filename = get_infer_output_path(
self.model_cfg, self.dataset_cfg,
osp.join(self.work_dir, 'predictions')
)
root, ext = osp.splitext(filename)
partial_filename = root + '_0' + ext
if not osp.exists(osp.realpath(filename)) and not osp.exists(
osp.realpath(partial_filename)):
print(f'{filename} not found')
return
dataset = build_dataset_from_cfg(self.dataset_cfg)
print("========== build-dataset ========")
print(dataset)
if 'dataset_postprocessor' in self.eval_cfg:
def postprocess(sample):
s = sample[self.ds_column]
proc = TEXT_POSTPROCESSORS.get(
self.eval_cfg['dataset_postprocessor']['type']
)
sample[self.ds_column] = proc(s)
print("========== sample ds_column ========")
print(sample)
print("self.ds_column")
return sample
dataset = dataset.map(postprocess)
print("========== postprocess dataset ========")
print(dataset)
if osp.exists(osp.realpath(filename)):
preds = mmengine.load(filename)
else:
filename = partial_filename
preds, offset = {}, 0
i = 1
while osp.exists(osp.realpath(filename)):
_preds = mmengine.load(filename)
filename = root + f'_{i}' + ext
i += 1
for _o in range(len(_preds)):
preds[str(offset)] = _preds[str(_o)]
offset += 1
pred_strs = [preds[str(i)]['prediction'] for i in range(len(preds))]
if 'in-context examples' in preds['0']:
print("___________ pred_strs ↓ ________")
print(pred_strs)
print("___________ pred_strs ↑ ________")
if 'pred_postprocessor' in self.eval_cfg:
print("=========== qwq ==============")
proc = TEXT_POSTPROCESSORS.get(
self.eval_cfg['pred_postprocessor']['type']
)
pred_strs = [proc(s) for s in pred_strs]
# if self.ds_split:
# references = dataset[self.ds_split][self.ds_column]
# else:
# references = dataset[self.ds_column]
# if len(pred_strs) != len(references):
# print('length mismatch')
# return
allcase, badcase = [], []
print("=========== gen origin_prompt ===============")
if 'in-context examples' in preds['0']:
for i, (pred_str) in enumerate(zip(tqdm(pred_strs))):
#ref_str = str(reference)
ref_str = str(i)
# print("=========== gen origin_prompt ===============")
# print("i: ",i,"ref_str: ",ref_str)
# print("")
try:
pred_prompt = preds[str(i)]['label: ' + str(pred_str)]['testing input']
pred_PPL = preds[str(i)]['label: ' + str(pred_str)]['PPL']
print("=========== gen origin_prompt ===============")
print(pred_prompt)
print("PPL: ",pred_PPL)
print("")
ref_prompt = preds[str(i)]['label: ' + ref_str]['testing input']
ref_PPL = preds[str(i)]['label: ' + ref_str]['PPL']
except KeyError:
continue
item = {
'prediction_prompt': pred_prompt,
'prediction': pred_str,
'prediction_PPL': pred_PPL,
'reference_prompt': ref_prompt,
'reference': ref_str,
'reference_PPL': ref_PPL
}
if pred_str != ref_str:
badcase.append(item)
allcase.append(item)
else:
allcase.append(item)
else:
print("=========== gen pred_str ===============")
pred_str = zip(tqdm(pred_strs))
# print(pred_str)
# print("")
# print(reference)
# print("")
for i, (pred_str) in enumerate(zip(tqdm(pred_strs))):
#ref_str = str(reference)
origin_prompt = preds[str(i)]['origin_prompt']
# print("=========== gen origin_prompt ===============")
# print(origin_prompt)
# print("")
item = {
'origin_prompt': origin_prompt,
'prediction': pred_str,
#'reference': ref_str
}
badcase.append(item)
allcase.append(item)
out_path = get_infer_output_path(
self.cfg['model'], self.cfg['dataset'],
osp.join(self.work_dir, 'case_analysis/bad')
)
mkdir_or_exist(osp.split(out_path)[0])
with open(out_path, 'w', encoding='utf-8') as f:
json.dump(badcase, f, indent=4, ensure_ascii=False)
# out_path = get_infer_output_path(
# self.cfg['model'], self.cfg['dataset'],
# osp.join(self.work_dir, 'case_analysis/all')
# )
# mkdir_or_exist(osp.split(out_path)[0])
with open(out_path, 'w', encoding='utf-8') as f:
json.dump(allcase, f, indent=4, ensure_ascii=False)
Reproduces the problem - command or script
python tools/case_analyzer.py configs/eval_demo.py
Reproduces the problem - error message
/home/qwq/anaconda3/envs/opencompass/lib/python3.10/site-packages/datasets/load.py:1486: FutureWarning: The repository for winograd_wsc contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/winograd_wsc
You can avoid this message in future by passing the argument trust_remote_code=True
.
Passing trust_remote_code=True
will be mandatory to load this dataset from the next major release of datasets
.
warnings.warn(
Other information
无法读取ppl类数据集内容,但gen类正常