12_epoch484.yamlts/cityscapes/seg_hrnet_w48_train_512x1024_sgd_lr1e-2_wd5e-4_bs
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 949, in _build_extension_module
check=True)
File "/opt/conda/lib/python3.6/subprocess.py", line 438, in run
output=stdout, stderr=stderr)
subprocess.CalledProcessError: Command '['ninja', '-v']' returned non-zero exit status 1.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "tools/train.py", line 27, in
import models
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/init.py", line 11, in
import models.seg_hrnet
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/seg_hrnet.py", line 22, in
from .sync_bn.inplace_abn.bn import InPlaceABNSync
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/init.py", line 1, in
from .inplace_abn import bn
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/inplace_abn/init.py", line 1, in
from .bn import ABN, InPlaceABN, InPlaceABNSync
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/inplace_abn/bn.py", line 14, in
from functions import *
File "/home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/functions.py", line 16, in
extra_cuda_cflags=["--expt-extended-lambda"])
File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 644, in load
is_python_module)
File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 813, in jit_compile
with_cuda=with_cuda)
File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 866, in write_ninja_file_and_build
build_extension_module(name, build_directory, verbose)
File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 962, in build_extension_module
raise RuntimeError(message)
RuntimeError: Error building extension 'inplace_abn': b'[1/4] /usr/local/cuda/bin/nvcc -DTORCH_EXTENSION_NAME=inplace_abn -DTORCH_API_INCLUDE_EXTENSION_H -isystem /opt/conda/lib/python3.6/site-packages/torch/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.6/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /opt/conda/include/python3.6m -D_GLIBCXX_USE_CXX11_ABI=0 -D__CUDA_NO_HALF_OPERATORS -D__CUDA_NO_HALF_CONVERSIONS -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' --expt-extended-lambda -std=c++11 -c /home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn_cuda.cu -o inplace_abn_cuda.cuda.o\nFAILED: inplace_abn_cuda.cuda.o \n/usr/local/cuda/bin/nvcc -DTORCH_EXTENSION_NAME=inplace_abn -DTORCH_API_INCLUDE_EXTENSION_H -isystem /opt/conda/lib/python3.6/site-packages/torch/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.6/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /opt/conda/include/python3.6m -D_GLIBCXX_USE_CXX11_ABI=0 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' --expt-extended-lambda -std=c++11 -c /home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn_cuda.cu -o inplace_abn_cuda.cuda.o\nnvcc fatal : Could not open output file '/local_scratch/pbs.492439.pbs02/tmpxft_002ac4fd_00000000'\n[2/4] c++ -MMD -MF inplace_abn_cpu.o.d -DTORCH_EXTENSION_NAME=inplace_abn -DTORCH_API_INCLUDE_EXTENSION_H -isystem /opt/conda/lib/python3.6/site-packages/torch/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.6/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /opt/conda/include/python3.6m -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++11 -O3 -c /home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn_cpu.cpp -o inplace_abn_cpu.o\nFAILED: inplace_abn_cpu.o \nc++ -MMD -MF inplace_abn_cpu.o.d -DTORCH_EXTENSION_NAME=inplace_abn -DTORCH_API_INCLUDE_EXTENSION_H -isystem /opt/conda/lib/python3.6/site-packages/torch/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.6/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /opt/conda/include/python3.6m -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++11 -O3 -c /home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn_cpu.cpp -o inplace_abn_cpu.o\n/home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn_cpu.cpp: In function 'std::vectorat::Tensor backward_cpu(at::Tensor, at::Tensor, at::Tensor, at::Tensor, at::Tensor, at::Tensor, at::Tensor, bool, float)':\n/home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn_cpu.cpp:82:34: error: could not convert 'z.at::Tensor::type()' from 'at::DeprecatedTypeProperties' to 'c10::IntArrayRef {aka c10::ArrayRef}'\n auto dweight = at::empty(z.type(), {0});\n ^\n/home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn_cpu.cpp:83:32: error: could not convert 'z.at::Tensor::type()' from 'at::DeprecatedTypeProperties' to 'c10::IntArrayRef {aka c10::ArrayRef}'\n auto dbias = at::empty(z.type(), {0});\n ^\n/home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn_cpu.cpp:89:29: error: could not convert '{dx, dweight, dbias}' from '' to 'std::vectorat::Tensor'\n return {dx, dweight, dbias};\n ^\n[3/4] c++ -MMD -MF inplace_abn.o.d -DTORCH_EXTENSION_NAME=inplace_abn -DTORCH_API_INCLUDE_EXTENSION_H -isystem /opt/conda/lib/python3.6/site-packages/torch/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.6/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /opt/conda/include/python3.6m -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++11 -O3 -c /home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn.cpp -o inplace_abn.o\nIn file included from /home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn.cpp:1:0:\n/opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include/torch/torch.h:7:2: warning: #warning "Including torch/torch.h for C++ extensions is deprecated. Please include torch/extension.h" [-Wcpp]\n #warning \\n ^\nninja: build stopped: subcommand failed.\n'
Traceback (most recent call last):
File "tools/train.py", line 27, in
Traceback (most recent call last):
File "tools/train.py", line 27, in
import models
import models
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/init.py", line 11, in
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/init.py", line 11, in
import models.seg_hrnet
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/seg_hrnet.py", line 22, in
import models.seg_hrnet
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/seg_hrnet.py", line 22, in
from .sync_bn.inplace_abn.bn import InPlaceABNSync
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/init.py", line 1, in
from .sync_bn.inplace_abn.bn import InPlaceABNSync
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/init.py", line 1, in
from .inplace_abn import bn
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/inplace_abn/init.py", line 1, in
from .inplace_abn import bn
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/inplace_abn/init.py", line 1, in
from .bn import ABN, InPlaceABN, InPlaceABNSync
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/inplace_abn/bn.py", line 14, in
from .bn import ABN, InPlaceABN, InPlaceABNSync
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/inplace_abn/bn.py", line 14, in
from functions import *
File "/home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/functions.py", line 16, in
from functions import *
File "/home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/functions.py", line 16, in
extra_cuda_cflags=["--expt-extended-lambda"])
File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 644, in load
extra_cuda_cflags=["--expt-extended-lambda"])
File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 644, in load
is_python_module)
File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 824, in _jit_compile
is_python_module)
File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 824, in _jit_compile
return _import_module_from_library(name, build_directory, is_python_module)
File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 967, in _import_module_from_library
return _import_module_from_library(name, build_directory, is_python_module)
File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 967, in _import_module_from_library
file, path, description = imp.find_module(module_name, [path])
File "/opt/conda/lib/python3.6/imp.py", line 297, in find_module
file, path, description = imp.find_module(module_name, [path])
File "/opt/conda/lib/python3.6/imp.py", line 297, in find_module
Traceback (most recent call last):
File "tools/train.py", line 27, in
import models
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/init.py", line 11, in
import models.seg_hrnet
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/seg_hrnet.py", line 22, in
from .sync_bn.inplace_abn.bn import InPlaceABNSync
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/init.py", line 1, in
raise ImportError(_ERR_MSG.format(name), name=name)
ImportError: No module named 'inplace_abn'
raise ImportError(_ERR_MSG.format(name), name=name)
ImportError: No module named 'inplace_abn'
from .inplace_abn import bn
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/inplace_abn/init.py", line 1, in
from .bn import ABN, InPlaceABN, InPlaceABNSync
File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/inplace_abn/bn.py", line 14, in
from functions import *
File "/home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/functions.py", line 16, in
extra_cuda_cflags=["--expt-extended-lambda"])
File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 644, in load
is_python_module)
File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 824, in _jit_compile
return _import_module_from_library(name, build_directory, is_python_module)
File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 967, in _import_module_from_library
file, path, description = imp.find_module(module_name, [path])
File "/opt/conda/lib/python3.6/imp.py", line 297, in find_module
raise ImportError(_ERR_MSG.format(name), name=name)
ImportError: No module named 'inplace_abn'
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/runpy.py", line 193, in _run_module_as_main
"main", mod_spec)
File "/opt/conda/lib/python3.6/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/opt/conda/lib/python3.6/site-packages/torch/distributed/launch.py", line 235, in
main()
File "/opt/conda/lib/python3.6/site-packages/torch/distributed/launch.py", line 231, in main
cmd=process.args)
subprocess.CalledProcessError: Command '['/opt/conda/bin/python', '-u', 'tools/train.py', '--local_rank=0', '--cfg', 'experiments/cityscapes/seg_hrnet_w48_train_512x1024_sgd_lr1e-2_wd5e-4_bs_12_epoch484.yaml']' returned non-zero exit status 1.
Singularity> ninja version
HRNet-Semantic-Segmentation copied to clipboard
HRNet
