arch_list[-1] += '+PTX'

[zyxcambridge]

`File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/utils/cpp_extension.py", line 1524, in _write_ninja_file_and_build_library _write_ninja_file_to_build_library( File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/utils/cpp_extension.py", line 1922, in _write_ninja_file_to_build_library cuda_flags = common_cflags + COMMON_NVCC_FLAGS + _get_cuda_arch_flags() File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/utils/cpp_extension.py", line 1694, in _get_cuda_arch_flags arch_list[-1] += '+PTX' IndexError: list index out of range ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 112484) of binary: /home/zyx/anaconda3/envs/FlashOcc/bin/python Traceback (most recent call last): File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/runpy.py", line 194, in _run_module_as_main return _run_code(code, main_globals, None, File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/runpy.py", line 87, in _run_code exec(code, run_globals) File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/distributed/launch.py", line 193, in main() File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/distributed/launch.py", line 189, in main launch(args) File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/distributed/launch.py", line 174, in launch run(args) File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run elastic_launch( File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in call return launch_agent(self._config, self._entrypoint, list(args)) File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent raise ChildFailedError( torch.distributed.elastic.multiprocessing.errors.ChildFailedError:

tools/test.py FAILED

Failures: <NO_OTHER_FAILURES>

Root Cause (first observed failure): [0]: time : 2024-09-12_13:58:51 host : zyx rank : 0 (local_rank: 0) exitcode : 1 (pid: 112484) error_file: <N/A> traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html

(FlashOcc) (base) zyx@zyx:~/code/FlashOCC$ hon3.8/site-packages/torch/utils/cpp_extension.py", line 1922, in _write_ninja_file_to_build_library cuda_flags = common_cflags + COMMON_NVCC_FLAGS + _get_cuda_arch_flags() File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/utils/cpp_extension.py", line 1694, in _get_cuda_arch_flags arch_list[-1] += '+PTX' IndexError: list index out of range ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 112484) of binary: /home/zyx/anaconda3/envs/FlashOcc/bin/python Traceback (most recent call last): File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/runpy.py", line 194, in _run_module_as_main return _run_code(code, main_globals, None, File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/runpy.py", line 87, in _run_code exec(code, run_globals) File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/distributed/launch.py", line 193, in main() File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/distributed/launch.py", line 189, in main launch(args) File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/distributed/launch.py", line 174, in launch run(args) File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/distributed/run.py", line 752, in run elastic_launch( File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in call return launch_agent(self._config, self._entrypoint, list(args)) File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 245, in launch_agent raise ChildFailedError( torch.distributed.elastic.multiprocessing.errors.ChildFailedError:

tools/test.py FAILED

Failures: <NO_OTHER_FAILURES>

Root Cause (first observed failure): [0]: time : 2024-09-12_13:58:51 host : zyx rank : 0 (local_rank: 0) exitcode : 1 (pid: 112484) error_file: <N/A> traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html

^C (FlashOcc) (base) zyx@zyx:~/code/FlashOCC$ ^C (FlashOcc) (base) zyx@zyx:~/code/FlashOCC$ ^C (FlashOcc) (base) zyx@zyx:~/code/FlashOCC$ bash tools/dist_test.sh projects/configs/flashocc/flashocc-r50.py ckpts/flashocc-r50-256x704.pth 1 --eval map --eval-options show_dir=work_dirs/flashocc_r50/results /home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/cuda/init.py:83: UserWarning: CUDA initialization: CUDA unknown error - this may be due to an incorrectly set up environment, e.g. changing env variable CUDA_VISIBLE_DEVICES after program start. Setting the available devices to be zero. (Triggered internally at ../c10/cuda/CUDAFunctions.cpp:109.) return torch._C._cuda_getDeviceCount() > 0 /home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/distributed/launch.py:178: FutureWarning: The module torch.distributed.launch is deprecated and will be removed in future. Use torchrun. Note that --use_env is set by default in torchrun. If your script expects --local_rank argument to be set, please change it to read from os.environ['LOCAL_RANK'] instead. See https://pytorch.org/docs/stable/distributed.html#launch-utility for further instructions

warnings.warn( /home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/cuda/init.py:83: UserWarning: CUDA initialization: CUDA unknown error - this may be due to an incorrectly set up environment, e.g. changing env variable CUDA_VISIBLE_DEVICES after program start. Setting the available devices to be zero. (Triggered internally at ../c10/cuda/CUDAFunctions.cpp:109.) return torch._C._cuda_getDeviceCount() > 0 No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda' /home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/mmdet/utils/setup_env.py:38: UserWarning: Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. warnings.warn( /home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. warnings.warn( projects.mmdet3d_plugin Using /home/zyx/.cache/torch_extensions/py38_cu113 as PyTorch extensions root... Detected CUDA files, patching ldflags Emitting ninja build file /home/zyx/.cache/torch_extensions/py38_cu113/dvr/build.ninja... Traceback (most recent call last): File "tools/test.py", line 290, in main() File "tools/test.py", line 171, in main plg_lib = importlib.import_module(_module_path) File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/importlib/init.py", line 127, in import_module return _bootstrap._gcd_import(name[level:], package, level) File "", line 1014, in _gcd_import File "", line 991, in _find_and_load File "", line 975, in _find_and_load_unlocked File "", line 671, in _load_unlocked File "", line 843, in exec_module File "", line 219, in _call_with_frames_removed File "/home/zyx/code/FlashOCC/projects/mmdet3d_plugin/init.py", line 1, in from .datasets import * File "/home/zyx/code/FlashOCC/projects/mmdet3d_plugin/datasets/init.py", line 2, in from .nuscenes_dataset_occ import NuScenesDatasetOccpancy File "/home/zyx/code/FlashOCC/projects/mmdet3d_plugin/datasets/nuscenes_dataset_occ.py", line 14, in from ..core.evaluation.ray_metrics import main as calc_rayiou File "/home/zyx/code/FlashOCC/projects/mmdet3d_plugin/core/evaluation/ray_metrics.py", line 13, in dvr = load("dvr", sources=["lib/dvr/dvr.cpp", "lib/dvr/dvr.cu"], verbose=True, extra_cuda_cflags=['-allow-unsupported-compiler']) File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/utils/cpp_extension.py", line 1202, in load return _jit_compile( File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/utils/cpp_extension.py", line 1425, in _jit_compile _write_ninja_file_and_build_library( File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/utils/cpp_extension.py", line 1524, in _write_ninja_file_and_build_library _write_ninja_file_to_build_library( File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/utils/cpp_extension.py", line 1922, in _write_ninja_file_to_build_library cuda_flags = common_cflags + COMMON_NVCC_FLAGS + _get_cuda_arch_flags() File "/home/zyx/anaconda3/envs/FlashOcc/lib/python3.8/site-packages/torch/utils/cpp_extension.py", line 1694, in _get_cuda_arch_flags arch_list[-1] += '+PTX' IndexError: list index out of range`