onnx-mlir
onnx-mlir copied to clipboard
Published
20 hours ago •
onnx
onnx
Add Benchmark Comparison Support to Perf CMakeLists.txt
@adriansion
I ran an example of your test like this:
python ~/onnx-mlir/test/perf/ParseBenchmarks.py --compare PerfGemm_Benchmark_1650564009 PerfGemm_Benchmark_1650564009 Pct --verbose
and got this
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatmulSquare/1024"
39 4
18.5847 149.555
18.5738 149.509
ms ms
2.68435e+08 2.14748e+09
1.44524e+10 1.43635e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatmulSquare/2048"
39 1
18.5847 4262.96
18.5738 4260.45
ms ms
2.68435e+08 1.71799e+10
1.44524e+10 4.03241e+09
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatmulSquare_BigO"
18.5847 0.490696
18.5738 0.490407
ms N^3
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatmulSquare_RMS"
18.5847 0.244589
18.5738 0.244559
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/16"
39 1225340
18.5847 0.000611784
18.5738 0.0006112
ms ms
2.68435e+08 7936
1.44524e+10 1.29843e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/32"
39 256126
18.5847 0.00276785
18.5738 0.00276213
ms ms
2.68435e+08 64512
1.44524e+10 2.33559e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/64"
39 29550
18.5847 0.0219624
18.5738 0.021894
ms ms
2.68435e+08 520192
1.44524e+10 2.37595e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/128"
39 4619
18.5847 0.149305
18.5738 0.149138
ms ms
2.68435e+08 4.17792e+06
1.44524e+10 2.80138e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/256"
39 469
18.5847 1.40377
18.5738 1.39973
ms ms
2.68435e+08 3.34889e+07
1.44524e+10 2.39252e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/512"
39 58
18.5847 10.9099
18.5738 10.8869
ms ms
2.68435e+08 2.68173e+08
1.44524e+10 2.46326e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/1024"
39 7
18.5847 94.0158
18.5738 93.8342
ms ms
2.68435e+08 2.14644e+09
1.44524e+10 2.28748e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/2048"
39 1
18.5847 749.031
18.5738 746.102
ms ms
2.68435e+08 1.71757e+10
1.44524e+10 2.30206e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare_BigO"
18.5847 0.0872028
18.5738 0.0868644
ms N^3
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare_RMS"
18.5847 0.00292568
18.5738 0.00318598
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare/16"
39 1006534
18.5847 0.00067036
18.5738 0.000670227
ms ms
2.68435e+08 8192
1.44524e+10 1.22227e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare/32"
39 214898
18.5847 0.00313916
18.5738 0.00313888
ms ms
2.68435e+08 65536
1.44524e+10 2.08788e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare/64"
39 29130
18.5847 0.022712
18.5738 0.0226865
ms ms
2.68435e+08 524288
1.44524e+10 2.31101e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare/128"
39 4387
18.5847 0.155923
18.5738 0.155919
ms ms
2.68435e+08 4.1943e+06
1.44524e+10 2.69005e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare/256"
39 474
18.5847 1.41001
18.5738 1.40963
ms ms
2.68435e+08 3.35544e+07
1.44524e+10 2.38037e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare/512"
39 58
18.5847 12.1311
18.5738 12.1112
ms ms
2.68435e+08 2.68435e+08
1.44524e+10 2.21643e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare/1024"
39 6
18.5847 94.4452
18.5738 94.2716
ms ms
2.68435e+08 2.14748e+09
1.44524e+10 2.27797e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare/2048"
39 1
18.5847 772.221
18.5738 771.601
ms ms
2.68435e+08 1.71799e+10
1.44524e+10 2.22652e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare_BigO"
18.5847 0.0898686
18.5738 0.089795
ms N^3
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare_RMS"
18.5847 0.0066506
18.5738 0.00696144
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatmulSquare/1024"
39 4
18.5847 149.555
18.5738 149.509
ms ms
2.68435e+08 2.14748e+09
1.44524e+10 1.43635e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatmulSquare/2048"
39 1
18.5847 4262.96
18.5738 4260.45
ms ms
2.68435e+08 1.71799e+10
1.44524e+10 4.03241e+09
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatmulSquare_BigO"
18.5847 0.490696
18.5738 0.490407
ms N^3
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatmulSquare_RMS"
18.5847 0.244589
18.5738 0.244559
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/16"
39 1225340
18.5847 0.000611784
18.5738 0.0006112
ms ms
2.68435e+08 7936
1.44524e+10 1.29843e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/32"
39 256126
18.5847 0.00276785
18.5738 0.00276213
ms ms
2.68435e+08 64512
1.44524e+10 2.33559e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/64"
39 29550
18.5847 0.0219624
18.5738 0.021894
ms ms
2.68435e+08 520192
1.44524e+10 2.37595e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/128"
39 4619
18.5847 0.149305
18.5738 0.149138
ms ms
2.68435e+08 4.17792e+06
1.44524e+10 2.80138e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/256"
39 469
18.5847 1.40377
18.5738 1.39973
ms ms
2.68435e+08 3.34889e+07
1.44524e+10 2.39252e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/512"
39 58
18.5847 10.9099
18.5738 10.8869
ms ms
2.68435e+08 2.68173e+08
1.44524e+10 2.46326e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/1024"
39 7
18.5847 94.0158
18.5738 93.8342
ms ms
2.68435e+08 2.14644e+09
1.44524e+10 2.28748e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/2048"
39 1
18.5847 749.031
18.5738 746.102
ms ms
2.68435e+08 1.71757e+10
1.44524e+10 2.30206e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare_BigO"
18.5847 0.0872028
18.5738 0.0868644
ms N^3
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_MatMulWithGemmSquare_RMS"
18.5847 0.00292568
18.5738 0.00318598
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare/16"
39 1006534
18.5847 0.00067036
18.5738 0.000670227
ms ms
2.68435e+08 8192
1.44524e+10 1.22227e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare/32"
39 214898
18.5847 0.00313916
18.5738 0.00313888
ms ms
2.68435e+08 65536
1.44524e+10 2.08788e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare/64"
39 29130
18.5847 0.022712
18.5738 0.0226865
ms ms
2.68435e+08 524288
1.44524e+10 2.31101e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare/128"
39 4387
18.5847 0.155923
18.5738 0.155919
ms ms
2.68435e+08 4.1943e+06
1.44524e+10 2.69005e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare/256"
39 474
18.5847 1.41001
18.5738 1.40963
ms ms
2.68435e+08 3.35544e+07
1.44524e+10 2.38037e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare/512"
39 58
18.5847 12.1311
18.5738 12.1112
ms ms
2.68435e+08 2.68435e+08
1.44524e+10 2.21643e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare/1024"
39 6
18.5847 94.4452
18.5738 94.2716
ms ms
2.68435e+08 2.14748e+09
1.44524e+10 2.27797e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare/2048"
39 1
18.5847 772.221
18.5738 771.601
ms ms
2.68435e+08 1.71799e+10
1.44524e+10 2.22652e+10
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare_BigO"
18.5847 0.0898686
18.5738 0.089795
ms N^3
----------------------------------------
# arg = PerfGemm_Benchmark_1650564009
"BM_MatmulSquare/512" "BM_GemmSquare_RMS"
18.5847 0.0066506
18.5738 0.00696144
----------------------------------------
Traceback (most recent call last):
File "/workdir/onnx-mlir/test/perf/ParseBenchmarks.py", line 480, in <module>
main()
File "/workdir/onnx-mlir/test/perf/ParseBenchmarks.py", line 472, in main
ComparisonOutput = CompareOutput(RawBenchmarkOutput1, RawBenchmarkOutput2, metric, maxrelativeslowdown)
File "/workdir/onnx-mlir/test/perf/ParseBenchmarks.py", line 305, in CompareOutput
dict1name = OutputDict1["name"]
KeyError: 'name'
root@1250b049ac2b:~/onnx-mlir/build/test/perf# python ~/onnx-mlir/test/perf/ParseBenchmarks.py --compare PerfGemm_Benchmark_1650564009 PerfGemm_Benchmark_1650564009 Pct --verbose
@adriansion
I ran an example of your test like this:
python ~/onnx-mlir/test/perf/ParseBenchmarks.py --compare PerfGemm_Benchmark_1650564009 PerfGemm_Benchmark_1650564009 Pct --verboseand got this
# arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatmulSquare/1024" 39 4 18.5847 149.555 18.5738 149.509 ms ms 2.68435e+08 2.14748e+09 1.44524e+10 1.43635e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatmulSquare/2048" 39 1 18.5847 4262.96 18.5738 4260.45 ms ms 2.68435e+08 1.71799e+10 1.44524e+10 4.03241e+09 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatmulSquare_BigO" 18.5847 0.490696 18.5738 0.490407 ms N^3 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatmulSquare_RMS" 18.5847 0.244589 18.5738 0.244559 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/16" 39 1225340 18.5847 0.000611784 18.5738 0.0006112 ms ms 2.68435e+08 7936 1.44524e+10 1.29843e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/32" 39 256126 18.5847 0.00276785 18.5738 0.00276213 ms ms 2.68435e+08 64512 1.44524e+10 2.33559e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/64" 39 29550 18.5847 0.0219624 18.5738 0.021894 ms ms 2.68435e+08 520192 1.44524e+10 2.37595e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/128" 39 4619 18.5847 0.149305 18.5738 0.149138 ms ms 2.68435e+08 4.17792e+06 1.44524e+10 2.80138e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/256" 39 469 18.5847 1.40377 18.5738 1.39973 ms ms 2.68435e+08 3.34889e+07 1.44524e+10 2.39252e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/512" 39 58 18.5847 10.9099 18.5738 10.8869 ms ms 2.68435e+08 2.68173e+08 1.44524e+10 2.46326e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/1024" 39 7 18.5847 94.0158 18.5738 93.8342 ms ms 2.68435e+08 2.14644e+09 1.44524e+10 2.28748e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/2048" 39 1 18.5847 749.031 18.5738 746.102 ms ms 2.68435e+08 1.71757e+10 1.44524e+10 2.30206e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare_BigO" 18.5847 0.0872028 18.5738 0.0868644 ms N^3 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare_RMS" 18.5847 0.00292568 18.5738 0.00318598 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare/16" 39 1006534 18.5847 0.00067036 18.5738 0.000670227 ms ms 2.68435e+08 8192 1.44524e+10 1.22227e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare/32" 39 214898 18.5847 0.00313916 18.5738 0.00313888 ms ms 2.68435e+08 65536 1.44524e+10 2.08788e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare/64" 39 29130 18.5847 0.022712 18.5738 0.0226865 ms ms 2.68435e+08 524288 1.44524e+10 2.31101e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare/128" 39 4387 18.5847 0.155923 18.5738 0.155919 ms ms 2.68435e+08 4.1943e+06 1.44524e+10 2.69005e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare/256" 39 474 18.5847 1.41001 18.5738 1.40963 ms ms 2.68435e+08 3.35544e+07 1.44524e+10 2.38037e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare/512" 39 58 18.5847 12.1311 18.5738 12.1112 ms ms 2.68435e+08 2.68435e+08 1.44524e+10 2.21643e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare/1024" 39 6 18.5847 94.4452 18.5738 94.2716 ms ms 2.68435e+08 2.14748e+09 1.44524e+10 2.27797e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare/2048" 39 1 18.5847 772.221 18.5738 771.601 ms ms 2.68435e+08 1.71799e+10 1.44524e+10 2.22652e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare_BigO" 18.5847 0.0898686 18.5738 0.089795 ms N^3 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare_RMS" 18.5847 0.0066506 18.5738 0.00696144 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatmulSquare/1024" 39 4 18.5847 149.555 18.5738 149.509 ms ms 2.68435e+08 2.14748e+09 1.44524e+10 1.43635e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatmulSquare/2048" 39 1 18.5847 4262.96 18.5738 4260.45 ms ms 2.68435e+08 1.71799e+10 1.44524e+10 4.03241e+09 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatmulSquare_BigO" 18.5847 0.490696 18.5738 0.490407 ms N^3 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatmulSquare_RMS" 18.5847 0.244589 18.5738 0.244559 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/16" 39 1225340 18.5847 0.000611784 18.5738 0.0006112 ms ms 2.68435e+08 7936 1.44524e+10 1.29843e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/32" 39 256126 18.5847 0.00276785 18.5738 0.00276213 ms ms 2.68435e+08 64512 1.44524e+10 2.33559e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/64" 39 29550 18.5847 0.0219624 18.5738 0.021894 ms ms 2.68435e+08 520192 1.44524e+10 2.37595e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/128" 39 4619 18.5847 0.149305 18.5738 0.149138 ms ms 2.68435e+08 4.17792e+06 1.44524e+10 2.80138e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/256" 39 469 18.5847 1.40377 18.5738 1.39973 ms ms 2.68435e+08 3.34889e+07 1.44524e+10 2.39252e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/512" 39 58 18.5847 10.9099 18.5738 10.8869 ms ms 2.68435e+08 2.68173e+08 1.44524e+10 2.46326e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/1024" 39 7 18.5847 94.0158 18.5738 93.8342 ms ms 2.68435e+08 2.14644e+09 1.44524e+10 2.28748e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare/2048" 39 1 18.5847 749.031 18.5738 746.102 ms ms 2.68435e+08 1.71757e+10 1.44524e+10 2.30206e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare_BigO" 18.5847 0.0872028 18.5738 0.0868644 ms N^3 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_MatMulWithGemmSquare_RMS" 18.5847 0.00292568 18.5738 0.00318598 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare/16" 39 1006534 18.5847 0.00067036 18.5738 0.000670227 ms ms 2.68435e+08 8192 1.44524e+10 1.22227e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare/32" 39 214898 18.5847 0.00313916 18.5738 0.00313888 ms ms 2.68435e+08 65536 1.44524e+10 2.08788e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare/64" 39 29130 18.5847 0.022712 18.5738 0.0226865 ms ms 2.68435e+08 524288 1.44524e+10 2.31101e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare/128" 39 4387 18.5847 0.155923 18.5738 0.155919 ms ms 2.68435e+08 4.1943e+06 1.44524e+10 2.69005e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare/256" 39 474 18.5847 1.41001 18.5738 1.40963 ms ms 2.68435e+08 3.35544e+07 1.44524e+10 2.38037e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare/512" 39 58 18.5847 12.1311 18.5738 12.1112 ms ms 2.68435e+08 2.68435e+08 1.44524e+10 2.21643e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare/1024" 39 6 18.5847 94.4452 18.5738 94.2716 ms ms 2.68435e+08 2.14748e+09 1.44524e+10 2.27797e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare/2048" 39 1 18.5847 772.221 18.5738 771.601 ms ms 2.68435e+08 1.71799e+10 1.44524e+10 2.22652e+10 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare_BigO" 18.5847 0.0898686 18.5738 0.089795 ms N^3 ---------------------------------------- # arg = PerfGemm_Benchmark_1650564009 "BM_MatmulSquare/512" "BM_GemmSquare_RMS" 18.5847 0.0066506 18.5738 0.00696144 ---------------------------------------- Traceback (most recent call last): File "/workdir/onnx-mlir/test/perf/ParseBenchmarks.py", line 480, in <module> main() File "/workdir/onnx-mlir/test/perf/ParseBenchmarks.py", line 472, in main ComparisonOutput = CompareOutput(RawBenchmarkOutput1, RawBenchmarkOutput2, metric, maxrelativeslowdown) File "/workdir/onnx-mlir/test/perf/ParseBenchmarks.py", line 305, in CompareOutput dict1name = OutputDict1["name"] KeyError: 'name' root@1250b049ac2b:~/onnx-mlir/build/test/perf# python ~/onnx-mlir/test/perf/ParseBenchmarks.py --compare PerfGemm_Benchmark_1650564009 PerfGemm_Benchmark_1650564009 Pct --verbose
Solved this problem