Test runner rendering next story before previous tests finish

[brynshanahan]

Describe the bug We're using test-runner to take Visual Regression snapshots of our components. Sometimes the test runner compares an the incorrect story to the snapshot. So our "default story" would sometimes be compared against a snapshot of the next story, "warning story" in this case.

This usually doesn't happen when running the test runner a second time which makes me think it might be related to Vite taking a while to bundle the story on the first view?

To Reproduce

test-runner.ts

import {
    TestRunnerConfig,
    getStoryContext,
    waitForPageReady,
} from '@storybook/test-runner'
import { injectAxe, checkA11y, configureAxe } from 'axe-playwright'
import { toMatchImageSnapshot } from 'jest-image-snapshot'
import path from 'path'
import fs from 'fs'
import { contract, gel2Themes, gel3Themes } from '@snsw-gel/theming'
import { Page } from 'playwright'

const themes = [...gel2Themes, ...gel3Themes]

async function waitForReady(page: Page) {
    await page.waitForTimeout(30)
    await page.waitForLoadState('networkidle')
    await waitForPageReady(page)

    await page.evaluate(async () => {
        while (true) {
            if (document.readyState === 'complete') {
                await new Promise(resolve => setTimeout(resolve, 100))
                if (document.readyState === 'complete') {
                    break
                }
            }
            await new Promise(resolve => {
                document.addEventListener('DOMContentLoaded', resolve, {
                    once: true,
                })
            })
        }
    })
    await page.evaluate(async () => {
        await document.fonts.ready
    })
}

async function removeThemes(page: Page) {
    await page.evaluate(
        themes => {
            document.body.classList.remove(...themes)
        },
        themes.map(t => t.className),
    )
}

async function enableTheme(page: Page, idx: number) {
    await page.evaluate(
        async ([idx, themes]) => {
            themes.forEach((cls, i) => {
                document.body.classList.toggle(cls, i === idx)
            })
        },
        [idx, themes.map(t => t.className)] as const,
    )
}

async function runAxeTest(page: Page, storyContext) {
    await removeThemes(page)

    // Apply story-level a11y rules
    await configureAxe(page, {
        rules: storyContext.parameters?.a11y?.config?.rules,
    })
    await checkA11y(page, '#storybook-root', {
        detailedReport: true,
        verbose: false,
        // pass axe options defined in @storybook/addon-a11y
        axeOptions: storyContext.parameters?.a11y?.options,
    })
}

async function runVisualRegressionTesting(page: Page, storyContext) {
    const browserName = page.context()?.browser()?.browserType().name()

    const breakpointsToTest = new Set(['smMobile', 'lgMobile', 'tablet'])
    let entries = Object.entries(contract.config.breakpoints).filter(([key]) =>
        breakpointsToTest.has(key),
    )

    let rootDir = path.resolve(storyContext.parameters.fileName)

    while (rootDir !== '/') {
        const packageJsonPath = path.resolve(rootDir, 'package.json')
        if (fs.existsSync(packageJsonPath)) {
            break
        }
        rootDir = path.resolve(rootDir, '..')
    }

    if (browserName !== 'webkit') {
        if (!storyContext.kind.includes('default')) return

        entries = [entries[entries.length - 1]]
    }

    for (let [breakpointKey, breakpoint] of entries) {
        let maxWidth = 'max' in breakpoint ? breakpoint.max : breakpoint.min + 1

        let pageHeight = 1080

        await page.setViewportSize({
            width: maxWidth - 1,
            height: pageHeight,
        })

        const height = await page.evaluate(() => {
            return document
                .querySelector('#storybook-root')
                ?.getBoundingClientRect().height
        })

        while (height && pageHeight < height) {
            pageHeight += 1080
        }

        await page.setViewportSize({
            width: maxWidth - 1,
            height: pageHeight,
        })

        for (let i = 0; i < themes.length; i++) {
            const theme = themes[i]
            await enableTheme(page, i)

            await waitForReady(page)

            const customSnapshotsDir = `${rootDir}/snapshots/${
                storyContext.kind
            }/${theme.className.replace('.', '')}/${breakpointKey}`

            const image = await page.screenshot()
            expect(image).toMatchImageSnapshot({
                customSnapshotsDir,
                customSnapshotIdentifier: storyContext.id,
            })
        }
    }
}

const config: TestRunnerConfig = {
    logLevel: 'none',
    setup() {
        // @ts-ignore
        expect.extend({ toMatchImageSnapshot })
    },

    async preVisit(page, context) {
        // Inject Axe utilities in the page before the story renders
        await injectAxe(page)
    },
    async postVisit(page, context) {
        // Get entire context of a story, including parameters, args, argTypes, etc.
        const storyContext = await getStoryContext(page, context)

        if (storyContext.parameters?.e2e?.enabled === false) {
            return
        }

        const browserName = page.context()?.browser()?.browserType().name()

        if (browserName !== 'webkit') {
            if (!storyContext.kind.includes('default')) return
        }

        await page.addStyleTag({
            content: `
                * {
                    transition: none !important;
                    -webkit-transition: none !important;
                    -moz-transition: none !important;
                    -o-transition: none !important;
                    -ms-transition: none !important;
                    animation: none !important;
                    -webkit-animation: none !important;
                    -moz-animation: none !important;
                    -o-animation: none !important;
                    -ms-animation: none !important;
                    transition-duration: 0s !important;
                    animation-duration: 0s !important;
                }

                svg animate {
                    display: none !important;
                }
            `,
        })

        await waitForPageReady(page)

        // Do not test a11y for stories that disable a11y
        if (storyContext.parameters?.a11y?.enabled !== false) {
            await runAxeTest(page, storyContext)
        }

        if (
            storyContext.parameters?.visual?.enabled !== false &&
            !process.env.CI
        ) {
            await runVisualRegressionTesting(page, storyContext)
        }
    },
}
export default config

main.tsx

import type { StorybookConfig } from '@storybook/react-vite'
import { InlineConfig, mergeConfig } from 'vite'
import fs from 'fs'
import os from 'os'

import { join, dirname, resolve } from 'path'

/**
 * This function is used to resolve the absolute path of a package.
 * It is needed in projects that use Yarn PnP or are set up within a monorepo.
 */
function getAbsolutePath(value: string): any {
    return dirname(require.resolve(join(value, 'package.json')))
}

const resolveCache = new Map<string, string>()
const requestersCache = new Map<string, Set<string>>()

const config: StorybookConfig = {
    stories: [
        '../../../packages/*/src/**/*.stories.@(js|ts|tsx|jsx)',
        '../../../packages/*/stories/**/*.stories.@(js|ts|tsx|jsx)',
        '../../../packages/*/stories/**/*.mdx',
    ],

    staticDirs: ['../public'],

    typescript: {
        check: true,
        reactDocgen: 'react-docgen-typescript',
        reactDocgenTypescriptOptions: {
            shouldExtractLiteralValuesFromEnum: true,
            shouldRemoveUndefinedFromOptional: true,
            include: ['../../**/src/**/*.{ts,tsx}'],
        },
    },

    addons: [
        getAbsolutePath('@storybook/addon-links'),
        getAbsolutePath('@storybook/addon-essentials'),
        getAbsolutePath('@storybook/addon-interactions'),
        '@storybook/addon-docs',
        getAbsolutePath('storybook-addon-jsx'),
        getAbsolutePath('@storybook/addon-a11y'),
        getAbsolutePath('@storybook/addon-mdx-gfm'),
    ],

    core: {},

    docs: {
        autodocs: true,
    },

    async viteFinal(config, { configType }) {
        if (configType === 'DEVELOPMENT') {
            // Your development configuration goes here
        }
        if (configType === 'PRODUCTION') {
            // Your production configuration goes here.
        }

        return mergeConfig<InlineConfig, InlineConfig>(config, {
            assetsInclude: ['**/*.md'],
            resolve: {
                alias: [],
            },

            optimizeDeps: {
                include: [
                    '@babel/parser',
                    'react-element-to-jsx-string',
                    '@babel/runtime/helpers/interopRequireWildcard',
                    '@mdx-js/react',
                    '@storybook/addon-docs',
                    '@storybook/react',
                    '@duetds/date-picker',
                    '@duetds/date-picker/dist/loader',
                    '@stencil/core',
                    '@base2/pretty-print-object',
                    '@storybook/client-api',
                    '@storybook/blocks',
                    '@storybook/client-logger',
                    'fast-deep-equal',
                    'lodash',
                    'styled-components',
                    'lodash-es',
                    'lodash/isPlainObject',
                    'lodash/mapValues',
                    'lodash/pickBy',
                    'lodash/pick',
                    'lodash/startCase',
                    'lodash/isFunction',
                    'lodash/isString',
                    'util-deprecate',
                    '@storybook/csf',
                    'react-router',
                    'react-router-dom',
                    'global',
                    'synchronous-promise',
                    'memoizerific',
                    'stable',
                    'doctrine',
                    'html-tags',
                    'escodegen',
                    'acorn',
                    'prettier',
                    '@prettier/sync',
                    'acorn-jsx',
                    '@base2/pretty-print-object',
                    'prop-types',
                    'react-dom',
                    'qs',
                    'uuid-browser',
                    'uuid-browser/v4',
                    'jest-mock',
                    // '@snsw-gel/react',
                ],
            },
            define: {
                'process.env.PATHNAME': JSON.stringify(process.env.PATHNAME || ""),
                'process.env.NODE_ENV': JSON.stringify(process.env.NODE_ENV),
                'process.env.STORYBOOK': JSON.stringify(true),
                'PKG_NAME': JSON.stringify(''),
                'PKG_VERSION': JSON.stringify(''),
                'GEL_NAME': JSON.stringify(
                    require('../../react/package.json').name,
                ),
                'GEL_VERSION': JSON.stringify(
                    require('../../react/package.json').version,
                ),
                'SNAPSHOT_RELEASE': JSON.stringify(
                    /\d+\.\d+.\d+-.*/.test(
                        require('../../react/package.json').version,
                    ),
                ),
            },
            // Your environment configuration here
            plugins: [
                {
                    enforce: 'post',
                    name: 'vite-plugin-resolve',
                    resolveId(id, requester, ...rest) {
                        if (id === 'package.json' && requester) {
                            let target = dirname(requester)
                            let resolved = ''
                            while (!resolved && target !== os.homedir()) {
                                let foundPackage = resolve(
                                    target,
                                    'package.json',
                                )
                                if (fs.existsSync(foundPackage)) {
                                    resolved = foundPackage
                                } else {
                                    target = dirname(target)
                                }
                            }
                            if (resolved) {
                                return resolved
                            }
                        }

                        if (id === '@snsw-gel/storybook') {
                            return require.resolve('../dist/esm/index.mjs')
                        }

                        let result

                        try {
                            result = require.resolve(id)
                        } catch (e) {
                            return null
                        }

                        const cachedResult = resolveCache.get(id)
                        let requesters = requestersCache.get(id)

                        if (!requesters) {
                            requesters = new Set()
                            requesters.add(requester!)
                            requestersCache.set(id, requesters)
                        }

                        if (cachedResult && cachedResult !== result) {
                            console.warn(
                                `Multiple requests resolving to different locations recieved for ${id} ${[
                                    ...requesters,
                                ].join(', ')}`,
                            )
                        }

                        return result
                    },
                },
            ],
        })
    },

    framework: {
        name: getAbsolutePath('@storybook/react-vite'),
        options: {},
    },
}
export default config

Expected behaviour Ideally the test runner would wait for the previous tests to finish before moving to the next story

Screenshots

In the above screenshot the test runner has rendered the next story "warning" before the the previous tests have finished

System

System: OS: macOS 13.6.2 CPU: (10) arm64 Apple M1 Pro Shell: 5.9 - /bin/zsh Binaries: Node: 18.19.0 - /private/var/folders/fn/6s5sc1b56pv0618wzc4k16v00000gq/T/xfs-0781bbb6/node Yarn: 4.0.2 - /private/var/folders/fn/6s5sc1b56pv0618wzc4k16v00000gq/T/xfs-0781bbb6/yarn <----- active npm: 10.2.3 - /usr/local/bin/npm Browsers: Chrome: 122.0.6261.129 Safari: 16.6

Additional context –