microsoft
[Bug]: Error: seed test not found when running Planner
Version
v1.56.1
Steps to reproduce
playwright.config.ts;
import * as path from 'path'
import * as dotenv from 'dotenv'
import { configAuth, configEnv } from './backbone/config'
import { defineConfig, devices } from '@playwright/test'
const TEST_ROOT = __dirname
dotenv.config({ path: path.join(TEST_ROOT, '.env') })
export default defineConfig({
testDir: TEST_ROOT,
retries: process.env.CI ? 1 : 1,
timeout: 35 * 1000,
expect: {
timeout: 15 * 1000,
},
reporter: process.env.CI
? [
['html', { outputFolder: process.env.REPORT_FOLDER }],
['json', { outputFile: `weekly-reports/report-${new Date().toISOString().slice(0, 10)}.json` }],
]
: [['html', { outputFolder: 'report' }]],
use: {
headless: true,
baseURL: configEnv.baseURL,
trace: 'retain-on-first-failure',
actionTimeout: 15 * 1000,
navigationTimeout: 30 * 1000,
ignoreHTTPSErrors: true,
},
projects: [
{ name: 'setup', testMatch: /setup\.ts/ },
{
name: 'api',
testMatch: /.*\.api\.spec\.ts$/,
use: {
...devices['Desktop Chrome'],
channel: 'chromium',
launchOptions: {
args: [
'--disable-web-security',
'--no-sandbox',
'--disable-gpu',
'--disable-blink-features=AutomationControlled',
],
},
},
dependencies: ['setup'],
workers: 2
},
{
name: 'ui',
fullyParallel: true,
testMatch: /.*\.ui\.spec\.ts$/,
use: {
...devices['Desktop Chrome'],
channel: 'chromium',
storageState: configAuth.regularUserAuth,
launchOptions: {
args: [
'--disable-web-security',
'--no-sandbox',
'--disable-gpu',
'--disable-blink-features=AutomationControlled',
],
},
},
dependencies: ['setup'],
workers: 4
}
]
})
config.ts;
import * as path from 'path'
const TEST_ROOT = path.resolve(__dirname, '..')
const ASSET_DIR = path.join(TEST_ROOT, 'common/assets')
let env = process.env.ENV || "aws-dev"
export const configEnv = {
...((env === 'aws-dev') && {
baseURL: 'https://test/beta/',
loginURL: 'https://test/beta/login',
apiBaseURL: 'https://test/beta/api',
docsUrl: 'https://test/beta/docs/'
}),
...((env === 'aws-staging') && {
baseURL: 'https://test/beta/',
loginURL: 'https://test/beta/login',
apiBaseURL: 'https://test/beta/api',
docsUrl: 'https://test/beta/docs/'
})
}
export const configAuth = {
regularUserAuth: path.join(ASSET_DIR, 'regular-user.json'),
secondaryRegularUserAuth: path.join(ASSET_DIR, 'secondary-regular-user.json'),
adminAuth: path.join(ASSET_DIR, 'admin-user.json'),
secondaryAdminUserAuth: path.join(ASSET_DIR, 'secondary-admin-user.json')
}
setup file
import { expect, test as setup } from '@playwright/test'
import { AuthPage } from "../playwright/pages/auth"
import { configEnv, configAuth } from "backbone/config"
dotenv.config()
setup('signin with regular user', async ({ page }) => {
let loginPage = new AuthPage(page)
await page.goto(`${configEnv.loginURL}`)
await expect(page.getByRole('link', { name: 'Create an account' })).toBeVisible()
await loginPage.loginUsernameInput.fill(process.env.REGULAR_USER!)
await loginPage.loginPasswordInput.fill(process.env.REGULAR_PASS!)
await loginPage.signInButton.click()
await page.waitForSelector('button#create')
await page.context().storageState({ path: configAuth.regularUserAuth })
})
setup('signin with secondary regular user', async ({ page }) => {
let loginPage = new AuthPage(page)
await page.goto(`${configEnv.loginURL}`)
await expect(page.getByRole('link', { name: 'Create an account' })).toBeVisible()
await loginPage.loginUsernameInput.fill(process.env.SECONDARY_REGULAR_USER!)
await loginPage.loginPasswordInput.fill(process.env.SECONDARY_REGULAR_PASS!)
await loginPage.signInButton.click()
await page.waitForSelector('button#create')
await page.context().storageState({ path: configAuth.secondaryRegularUserAuth })
})
setup('signin with admin user', async ({ page }) => {
let loginPage = new AuthPage(page)
await page.goto(`${configEnv.loginURL}`)
await expect(page.getByRole('link', { name: 'Create an account' })).toBeVisible()
await loginPage.loginUsernameInput.fill(process.env.ADMIN_USER!)
await loginPage.loginPasswordInput.fill(process.env.ADMIN_PASS!)
await loginPage.signInButton.click()
await page.waitForSelector('button#create')
await page.context().storageState({ path: configAuth.adminAuth })
})
setup('signin with secondary admin user', async ({ page }) => {
let loginPage = new AuthPage(page)
await page.goto(`${configEnv.loginURL}`)
await expect(page.getByRole('link', { name: 'Create an account' })).toBeVisible()
await loginPage.loginUsernameInput.fill(process.env.SECONDARY_ADMIN_USER!)
await loginPage.loginPasswordInput.fill(process.env.SECONDARY_ADMIN_PASS!)
await loginPage.signInButton.click()
await page.waitForSelector('button#create')
await page.context().storageState({ path: configAuth.secondaryAdminUserAuth })
})
.env file
[email protected]
REGULAR_PASS=test
[email protected]
SECONDARY_REGULAR_PASS=test
[email protected]
ADMIN_PASS=test
[email protected]
SECONDARY_ADMIN_PASS=test
package.json script;
"aws-dev:ui": "ENV=aws-dev npx playwright test --project=ui"
my spec file structures;
Expected behavior
Planner agent should initialize correctly using the provided seedFile and and project in playwright.config.ts
Actual behavior
Planner agent did not initialize correctly using the provided seedFile and and project in playwright.config.ts as below screenshoot....
Maybe it’s not a bug, but I might have configured something incorrectly — I’m not sure. If it’s a configuration issue, how can I fix it? It seems like it can’t detect my projects properly; instead, it automatically selects Chromium as the project. I manually wrote { "project": "ui" } and then clicked Allow. I need to fix this issue first. After that, do I really have to use the seed.spec.ts file, or can I just select one of my existing test files and make it generate a plan based on that file? I defined setup as dependancy ,does it read setup file automatically? I’m a bit confused.
Additional context
No response
Environment
PC: Chip Apple M2 macOS Sequoia 15.6.1
Playwright version:1.56.1
VsCode version: 1.105.1 (Universal)
Copilot is picking project.ui.spec.ts as the seed file, but you don't mention that file anywhere else. Just to double-check, does that file exist?
@Skn0tt yes project.ui.spec.ts is my file and have some tests in it ,it is exists
And is project.ui.spec.ts the full path to it, starting from the playwright.config.ts location? To me it looks like your Agent generated the wrong tool call input.
Also, are you using Playwright Test Agents (https://playwright.dev/docs/test-agents) with its subagents, or did you set up the MCP server alone and are trying to use it directly?
@Skn0tt I’m using Playwright Test Agents (https://playwright.dev/docs/test-agents) along with its subagents.The problem here is that the planner agent is running with { project: chromium } when it start, but I don’t have such a project defined in my Playwright config
We'll try improving the error message, so that LLM can figure it out.
Also we can try adding a config tool param.
Also, are you using Playwright Test Agents (https://playwright.dev/docs/test-agents) with its subagents, or did you set up the MCP server alone and are trying to use it directly?
I have a question that’s not directly related to this issue, but it caught my attention and I’m curious about it.
So, I was wondering if this matters? Is it not recommended to use run-test-mcp-server without its subagent?
