diff --git a/test/automation/src/playwrightBrowser.ts b/test/automation/src/playwrightBrowser.ts
index 5d85b0e910a..f4e28668ab0 100644
--- a/test/automation/src/playwrightBrowser.ts
+++ b/test/automation/src/playwrightBrowser.ts
@@ -6,8 +6,7 @@
 import * as playwright from '@playwright/test';
 import { ChildProcess, spawn } from 'child_process';
 import { join } from 'path';
-import { mkdir } from 'fs';
-import { promisify } from 'util';
+import * as mkdirp from 'mkdirp';
 import { URI } from 'vscode-uri';
 import { Logger, measureAndLog } from './logger';
 import type { LaunchOptions } from './code';
@@ -35,7 +34,7 @@ async function launchServer(options: LaunchOptions) {
 	const { userDataDir, codePath, extensionsPath, logger, logsPath } = options;
 	const codeServerPath = codePath ?? process.env.VSCODE_REMOTE_SERVER_PATH;
 	const agentFolder = userDataDir;
-	await measureAndLog(promisify(mkdir)(agentFolder), `mkdir(${agentFolder})`, logger);
+	await measureAndLog(mkdirp(agentFolder), `mkdirp(${agentFolder})`, logger);
 
 	const env = {
 		VSCODE_REMOTE_SERVER_PATH: codeServerPath,
diff --git a/test/smoke/src/main.ts b/test/smoke/src/main.ts
index 7f5accccec7..438ea351ff0 100644
--- a/test/smoke/src/main.ts
+++ b/test/smoke/src/main.ts
@@ -107,7 +107,7 @@ const testDataPath = path.join(os.tmpdir(), 'vscsmoke');
 if (fs.existsSync(testDataPath)) {
 	rimraf.sync(testDataPath);
 }
-fs.mkdirSync(testDataPath);
+mkdirp.sync(testDataPath);
 process.once('exit', () => {
 	try {
 		rimraf.sync(testDataPath);
diff --git a/test/smoke/src/utils.ts b/test/smoke/src/utils.ts
index bfcbd5f9e18..049f20535d5 100644
--- a/test/smoke/src/utils.ts
+++ b/test/smoke/src/utils.ts
@@ -150,9 +150,13 @@ export function timeout(i: number) {
 }
 
 export async function retryWithRestart(app: Application, testFn: () => Promise<unknown>, retries = 3, timeoutMs = 20000): Promise<unknown> {
+	let lastError: Error | undefined = undefined;
 	for (let i = 0; i < retries; i++) {
 		const result = await Promise.race([
-			testFn().then(() => true, error => { throw error; }),
+			testFn().then(() => true, error => {
+				lastError = error;
+				return false;
+			}),
 			timeout(timeoutMs).then(() => false)
 		]);
 
@@ -162,6 +166,8 @@ export async function retryWithRestart(app: Application, testFn: () => Promise<u
 
 		await app.restart();
 	}
+
+	throw lastError ?? new Error('retryWithRestart failed with an unknown error');
 }
 
 export interface ITask<T> {