[WIP] wconstab/archive compare

[wconstab]

[pytorchbot]

Performance Bot: Warning: machine info mismatch old[node]=ip-172-16-73-165, new=ip-172-16-153-99 No change detected for test_train[LearningToPaint-cuda-jit]: old 0.02421516987879207, new 0.024371329484851725, diff/std -0.08875374674216603 No change detected for test_eval[BERT_pytorch-cpu-jit]: old 0.049298641450013746, new 0.04930400166668256, diff/std -0.003029245461925077 No change detected for test_train[Super_SloMo-cuda-eager]: old 1.4940141625998877, new 1.5046380766000311, diff/std -1.8196433516712018 No change detected for test_eval[Super_SloMo-cuda-eager]: old 0.580996050000067, new 0.5836499559999538, diff/std -1.379934668203965 No change detected for test_eval[BERT_pytorch-cpu-eager]: old 0.05808122977778238, new 0.058490116611121974, diff/std -0.20509933149425705 No change detected for test_train[Super_SloMo-cuda-jit]: old 1.4893182744001023, new 1.5020460444000492, diff/std -2.6880997684851504 No change detected for test_train[fastNLP-cpu-eager]: old 0.0050933750497096945, new 0.004811670553642931, diff/std 0.3636080302893565 No change detected for test_eval[pytorch_mobilenet_v3-cuda-eager]: old 0.027733320694450312, new 0.028051781777788973, diff/std -1.2717989974472395 No change detected for test_eval[fastNLP-cpu-eager]: old 0.000314802935275002, new 0.0003138845433784909, diff/std 0.12593032844172547 No change detected for test_eval[demucs-cpu-eager]: old 0.17118889771430595, new 0.15458387842863885, diff/std 0.4689738634093455 No change detected for test_eval[fastNLP-cuda-eager]: old 0.000264554611140666, new 0.0002697977429911338, diff/std -0.8545084650970491 Improvement detected for test_eval[LearningToPaint-cpu-jit]: old 0.022657553866651546, new 0.014744483695642924, diff/std 30.462359489931643 No change detected for test_eval[yolov3-cuda-eager]: old 0.5789186345999952, new 0.5787747157999548, diff/std 0.07315902080584785 Regression detected for test_eval[pytorch_mobilenet_v3-cpu-jit]: old 0.022566607977766277, new 0.023137435659123184, diff/std -5.6502623563337915 No change detected for test_train[pytorch_mobilenet_v3-cpu-jit]: old 0.24761654600001748, new 0.2498497762000625, diff/std -0.5326228448101474 No change detected for test_eval[pytorch_mobilenet_v3-cuda-jit]: old 0.022975691511141524, new 0.023326793522735235, diff/std -0.9422792243619594 Improvement detected for test_train[LearningToPaint-cpu-eager]: old 0.09873431181815497, new 0.06404537718749737, diff/std 7.974516589976378 Regression detected for test_train[pytorch_struct-cuda-jit]: old 0.2586545150000347, new 0.2602836553999623, diff/std -19.707986325493206 Regression detected for test_eval[pytorch_struct-cuda-eager]: old 0.001730117785463269, new 0.0018460481930162587, diff/std -7.633152487592447 Regression detected for test_train[tacotron2-cuda-eager]: old 0.8704008514000179, new 4.37578219700008, diff/std -1476.327529820428 Improvement detected for test_train[LearningToPaint-cpu-jit]: old 0.10013746445451943, new 0.07136599890910392, diff/std 6.195222769077925 Improvement detected for test_train[attension_is_all_you_nee...-cpu-eager]: old 2.945312515799924, new 2.8955824601998756, diff/std 3.549051247933885 No change detected for test_eval[Super_SloMo-cuda-jit]: old 0.5762050783999257, new 0.5857045979999839, diff/std -1.7526115150473287 Regression detected for test_train[fastNLP-cuda-jit]: old 0.002498663594711928, new 0.0026001629225131675, diff/std -7.627128588745777 No change detected for test_train[pytorch_struct-cpu-jit]: old 0.741462897800011, new 0.7346399301999554, diff/std 0.1749867910194537 No change detected for test_eval[attension_is_all_you_nee...-cpu-jit]: old 0.8817327056000522, new 0.8668819317998896, diff/std 2.879725313742739 No change detected for test_eval[demucs-cuda-eager]: old 0.08675343658334593, new 0.08693729816673113, diff/std -2.7896032678912728 No change detected for test_eval[pytorch_mobilenet_v3-cpu-eager]: old 0.027521218324317167, new 0.02829788962857752, diff/std -1.735341914075483 No change detected for test_train[Background_Matting-cuda-jit]: old 4.338049184400006, new 4.343523171799961, diff/std -0.07700578629288886 Regression detected for test_eval[LearningToPaint-cuda-eager]: old 0.00398531950597418, new 0.004065483085375816, diff/std -9.422021088700257 No change detected for test_train[demucs-cpu-eager]: old 0.7578278340000452, new 0.7200408444002278, diff/std 0.4356794266494244 Improvement detected for test_train[attension_is_all_you_nee...-cpu-jit]: old 2.95403769200002, new 2.8946904126000845, diff/std 3.168143323634564 Regression detected for test_eval[moco-cuda-eager]: old 0.6445039933999851, new 0.6479323282000223, diff/std -3.011804328647774 No change detected for test_train[attension_is_all_you_nee...-cuda-jit]: old 0.5997974462001366, new 0.5945494387999133, diff/std 2.976819248645519 No change detected for test_train[BERT_pytorch-cuda-eager]: old 0.12952533150001955, new 0.1300526978750156, diff/std -1.5867707678990752 No change detected for test_eval[demucs-cpu-jit]: old 0.15567824100003236, new 0.16504459714289105, diff/std -1.7149921899371536 No change detected for test_train[moco-cuda-jit]: old 1.2896711212000356, new 1.296868173999974, diff/std -2.9766010185212277 Regression detected for test_eval[pytorch_struct-cuda-jit]: old 0.001191956367859705, new 0.0012555696290653252, diff/std -4.186930609661339 No change detected for test_eval[fastNLP-cuda-jit]: old 0.0002115605456140089, new 0.0002179254767038681, diff/std -1.178066963751666 Regression detected for test_eval[BERT_pytorch-cuda-jit]: old 0.016670511852462593, new 0.01725384161015766, diff/std -5.376671263526533 No change detected for test_train[BERT_pytorch-cpu-jit]: old 0.29433024239999667, new 0.27034126980001927, diff/std 1.0466277250765064 No change detected for test_train[fastNLP-cpu-jit]: old 0.004935747082577151, new 0.005579756505611523, diff/std -0.8580327735558215 Improvement detected for test_eval[maskrcnn_benchmark-cuda-eager]: old 0.23169389680001587, new 0.22505156659981368, diff/std 4.3567605925823605 No change detected for test_eval[moco-cuda-jit]: old 0.6436821728000723, new 0.6464056035999419, diff/std -2.7129249289298403 No change detected for test_train[pytorch_mobilenet_v3-cuda-eager]: old 0.2619200013999944, new 0.26093498680002086, diff/std 0.4438926778899163 Regression detected for test_train[moco-cuda-eager]: old 1.2901278798000022, new 1.3001253203999112, diff/std -4.994457484759918 No change detected for test_eval[LearningToPaint-cpu-eager]: old 0.021624256800007263, new 0.014935733882345951, diff/std 2.3852469791795703 Regression detected for test_eval[LearningToPaint-cuda-jit]: old 0.004026266911642344, new 0.004105054512198019, diff/std -4.8980158240887395 Regression detected for test_train[pytorch_struct-cuda-eager]: old 0.2591258150000158, new 0.2608997132001605, diff/std -38.0405387523158 No change detected for test_train[demucs-cuda-eager]: old 0.21234260640003413, new 0.21297200360004354, diff/std -2.905014038100167 No change detected for test_train[fastNLP-cuda-eager]: old 0.0025507038778921826, new 0.0025639038046678456, diff/std -1.1039139581728163 No change detected for test_train[attension_is_all_you_nee...-cuda-eager]: old 0.5937980835999952, new 0.5946753360000002, diff/std -0.718314421111572 No change detected for test_train[Background_Matting-cuda-eager]: old 4.449714926599972, new 4.352193268400015, diff/std 1.1697728913384071 No change detected for test_eval[fastNLP-cpu-jit]: old 0.0002620378118724937, new 0.0002609388013520033, diff/std 0.17089348671533483 No change detected for test_train[BERT_pytorch-cuda-jit]: old 0.12014882050002029, new 0.11971289877778367, diff/std 0.7053376504197707 No change detected for test_eval[attension_is_all_you_nee...-cuda-jit]: old 0.18795252619997882, new 0.18929112320001878, diff/std -0.486923017377064 Regression detected for test_eval[pytorch_struct-cpu-jit]: old 0.002150302079917212, new 0.0025626209569607084, diff/std -21.17396659152883 Regression detected for test_train[LearningToPaint-cuda-eager]: old 0.021561977354153328, new 0.02170223177085499, diff/std -7.836870387736933 Regression detected for test_eval[BERT_pytorch-cuda-eager]: old 0.026342110743584248, new 0.027501631459447683, diff/std -4.976876128829333 No change detected for test_eval[demucs-cuda-jit]: old 0.08733853849997786, new 0.0869204945833341, diff/std 0.6945110643316094 No change detected for test_eval[attension_is_all_you_nee...-cuda-eager]: old 0.18648873499993593, new 0.18656862366674432, diff/std -0.037663766610253196 No change detected for test_train[pytorch_struct-cpu-eager]: old 0.8054847224000241, new 0.7006577634000678, diff/std 1.3331137172437 Regression detected for test_eval[tacotron2-cuda-eager]: old 1.2246978399999988, new 1.3091980586000318, diff/std -21.96759760721946 No change detected for test_train[demucs-cpu-jit]: old 0.7642197246000251, new 0.7136177778000274, diff/std 0.4734914864726845 No change detected for test_train[pytorch_mobilenet_v3-cuda-jit]: old 0.2502386925999417, new 0.2436142770000515, diff/std 1.4718213953905608 Regression detected for test_train[yolov3-cuda-eager]: old 6.580188702600026, new 6.800004817999889, diff/std -3.890373374412962 Improvement detected for test_eval[attension_is_all_you_nee...-cpu-eager]: old 0.8979000879999604, new 0.8805726082000547, diff/std 4.636345264530002 No change detected for test_train[BERT_pytorch-cpu-eager]: old 0.28985638800002106, new 0.279566690799993, diff/std 0.7490073200813705 No change detected for test_train[demucs-cuda-jit]: old 0.2124413622000702, new 0.21285937459997512, diff/std -1.4194014897894731 Regression detected for test_train[maskrcnn_benchmark-cuda-eager]: old 0.37252263179998407, new 0.37737548239992974, diff/std -17.271389890750818 No change detected for test_eval[yolov3-cpu-eager]: old 0.7377368989999923, new 0.74104736900008, diff/std -1.1576690817431352 No change detected for test_train[pytorch_mobilenet_v3-cpu-eager]: old 0.2644863808000082, new 0.26004186139989544, diff/std 0.8520394540207278 Regression detected for test_eval[pytorch_struct-cpu-eager]: old 0.002576141815854936, new 0.0027058167760066376, diff/std -5.227264844266714