V2 Performance Signal Detected by TorchBench CI on '2.5.0.dev20240820+cu124'

[xuzhao9]

TorchBench CI has detected a performance signal.

Base PyTorch version: 2.4.0.dev20240606+cu121

Base PyTorch commit: ffaea656b5d8ff6518669494cc8f664b94f8e8b1

Affected PyTorch version: 2.5.0.dev20240820+cu124

Affected PyTorch commit: 92151c814ba715fe7d1f5648b0ae6950dceee6b7

Affected Tests:

• test_train[Super_SloMo-cuda-eager]: +14.63489%
• test_eval[Super_SloMo-cuda-eager]: +11.05568%
• test_train[Super_SloMo-cuda-jit]: +14.75663%
• test_eval[Super_SloMo-cuda-jit]: +11.20618%
• test_eval[alexnet-cuda-eager]: +10.48116%
• test_eval[alexnet-cuda-jit]: +13.67153%
• test_train[dcgan-cuda-eager]: +63.45914%
• test_train[dcgan-cuda-jit]: +63.58688%
• test_eval[pyhpc_equation_of_state-cpu-jit]: +11.65174%
• test_eval[pytorch_stargan-cpu-eager]: -13.78277%
• test_eval[pytorch_stargan-cpu-jit]: -8.56469%
• test_eval[resnext50_32x4d-cpu-jit]: +16.41716%
• test_eval[shufflenet_v2_x1_0-cpu-eager]: +15.12995%
• test_eval[shufflenet_v2_x1_0-cuda-jit]: -27.58491%
• test_train[timm_nfnet-cpu-jit]: -7.30437%
• test_eval[timm_regnet-cuda-eager]: +170.86549%
• test_train[timm_regnet-cpu-eager]: +10.38223%
• test_eval[timm_regnet-cuda-jit]: +356.94699%
• test_train[yolov3-cuda-eager]: -9.93714%

cc @xuzhao9

Result json:

{
  "start": "ffaea656b5d8ff6518669494cc8f664b94f8e8b1",
  "end": "92151c814ba715fe7d1f5648b0ae6950dceee6b7",
  "threshold": 7,
  "timeout": 120,
  "torchbench_branch": "v2.0",
  "result": [
    {
      "commit1": "ffaea656b5d",
      "commit1_time": "2024-06-05 22:56:32 +0000",
      "commit1_digest": {
        "test_train[Super_SloMo-cuda-eager]": 1.338426807988435,
        "test_eval[Super_SloMo-cuda-eager]": 0.7831221119500696,
        "test_train[Super_SloMo-cuda-jit]": 1.3294725218787788,
        "test_eval[Super_SloMo-cuda-jit]": 0.773011963814497,
        "test_eval[alexnet-cuda-eager]": 0.0037493890403468805,
        "test_eval[alexnet-cuda-jit]": 0.0035936248865926566,
        "test_train[dcgan-cuda-eager]": 0.3330581451766193,
        "test_train[dcgan-cuda-jit]": 0.33322170078754426,
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09255130235105753,
        "test_eval[pytorch_stargan-cpu-eager]": 0.2572209872305393,
        "test_eval[pytorch_stargan-cpu-jit]": 0.24752906374633313,
        "test_eval[resnext50_32x4d-cpu-jit]": 0.04303841098832587,
        "test_eval[shufflenet_v2_x1_0-cpu-eager]": 0.15824245307594537,
        "test_eval[shufflenet_v2_x1_0-cuda-jit]": 0.012956801395958815,
        "test_train[timm_nfnet-cpu-jit]": 12.24168330449611,
        "test_eval[timm_regnet-cuda-eager]": 0.18943582335487008,
        "test_train[timm_regnet-cpu-eager]": 0.9492317086085678,
        "test_eval[timm_regnet-cuda-jit]": 0.213938781991601,
        "test_train[yolov3-cuda-eager]": 0.9178506722673774
      },
      "commit2": "e98662bed99",
      "commit2_time": "2024-06-05 23:44:54 +0000",
      "commit2_digest": {
        "test_train[timm_regnet-cpu-eager]": 0.8524301628582179
      }
    },
    {
      "commit1": "7bfd1db53a8",
      "commit1_time": "2024-06-09 03:08:53 +0000",
      "commit1_digest": {
        "test_eval[pytorch_stargan-cpu-jit]": 0.29908282998949287
      },
      "commit2": "31c3fa6cf57",
      "commit2_time": "2024-06-09 04:29:04 +0000",
      "commit2_digest": {
        "test_eval[pytorch_stargan-cpu-jit]": 0.24893751638010145
      }
    },
    {
      "commit1": "cba840fde9d",
      "commit1_time": "2024-06-12 21:14:04 +0000",
      "commit1_digest": {
        "test_eval[pytorch_stargan-cpu-jit]": 0.26865220805630086
      },
      "commit2": "1026b7cfbe1",
      "commit2_time": "2024-06-12 21:34:20 +0000",
      "commit2_digest": {
        "test_eval[pytorch_stargan-cpu-jit]": 0.2995990448631346
      }
    },
    {
      "commit1": "dcfa7702c3e",
      "commit1_time": "2024-06-08 18:16:33 +0000",
      "commit1_digest": {
        "test_eval[pytorch_stargan-cpu-jit]": 0.26757836537435653
      },
      "commit2": "ea614fb2b1c",
      "commit2_time": "2024-06-08 18:23:08 +0000",
      "commit2_digest": {
        "test_eval[pytorch_stargan-cpu-jit]": 0.2940203260630369
      }
    },
    {
      "commit1": "1026b7cfbe1",
      "commit1_time": "2024-06-12 21:34:20 +0000",
      "commit1_digest": {
        "test_eval[pytorch_stargan-cpu-jit]": 0.2995990448631346
      },
      "commit2": "ff3ba993209",
      "commit2_time": "2024-06-12 21:40:29 +0000",
      "commit2_digest": {
        "test_eval[pytorch_stargan-cpu-jit]": 0.26418569004163145
      }
    },
    {
      "commit1": "ff3ba993209",
      "commit1_time": "2024-06-12 21:40:29 +0000",
      "commit1_digest": {
        "test_eval[pytorch_stargan-cpu-jit]": 0.26418569004163145
      },
      "commit2": "c0ea8fc3a37",
      "commit2_time": "2024-06-12 21:40:29 +0000",
      "commit2_digest": {
        "test_eval[pytorch_stargan-cpu-jit]": 0.24675068277865647
      }
    },
    {
      "commit1": "87072dcfdb2",
      "commit1_time": "2024-06-12 21:57:12 +0000",
      "commit1_digest": {
        "test_eval[pytorch_stargan-cpu-jit]": 0.2536820833571255
      },
      "commit2": "6aef2052ea4",
      "commit2_time": "2024-06-12 21:58:34 +0000",
      "commit2_digest": {
        "test_eval[pytorch_stargan-cpu-jit]": 0.29720516335219144
      }
    },
    {
      "commit1": "0a976b88992",
      "commit1_time": "2024-08-19 09:18:12 +0000",
      "commit1_digest": {
        "test_eval[shufflenet_v2_x1_0-cpu-eager]": 0.1523409191519022
      },
      "commit2": "92151c814ba",
      "commit2_time": "2024-08-19 09:45:58 +0000",
      "commit2_digest": {
        "test_train[Super_SloMo-cuda-eager]": 1.3381493031978606,
        "test_eval[Super_SloMo-cuda-eager]": 0.7831633009016514,
        "test_train[Super_SloMo-cuda-jit]": 1.3293857318349183,
        "test_eval[Super_SloMo-cuda-jit]": 0.773026413936168,
        "test_eval[alexnet-cuda-eager]": 0.0037537254323076483,
        "test_eval[alexnet-cuda-jit]": 0.0036008351210114757,
        "test_train[dcgan-cuda-eager]": 0.33301299763843417,
        "test_train[dcgan-cuda-jit]": 0.3332995416596532,
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08122656056657433,
        "test_eval[pytorch_stargan-cpu-eager]": 0.24499883083626628,
        "test_eval[pytorch_stargan-cpu-jit]": 0.265600214432925,
        "test_eval[resnext50_32x4d-cpu-jit]": 0.043312149898459516,
        "test_eval[shufflenet_v2_x1_0-cpu-eager]": 0.1368943582288921,
        "test_eval[shufflenet_v2_x1_0-cuda-jit]": 0.012918812127258534,
        "test_train[timm_nfnet-cpu-jit]": 11.118164934404195,
        "test_eval[timm_regnet-cuda-eager]": 0.18923914283514023,
        "test_train[timm_regnet-cpu-eager]": 0.8597747129388154,
        "test_eval[timm_regnet-cuda-jit]": 0.21346155321225524,
        "test_train[yolov3-cuda-eager]": 0.9171416262164712
      }
    }
  ]
}

Bisection workflow link: https://github.com/pytorch/benchmark/actions/runs/10476104564