testing: add more logical default autorun behavior

Fixes #117526

.vscode/settings.json

@@ -86,4 +86,5 @@
 	},
 	"typescript.tsc.autoDetect": "off",
 	"notebook.experimental.useMarkdownRenderer": true,
+  "testing.autoRun.mode": "onlyPreviouslyRun",
 }

src/vs/workbench/contrib/testing/common/configuration.ts

@@ -9,6 +9,7 @@ import { IConfigurationNode } from 'vs/platform/configuration/common/configurati
 
 export const enum TestingConfigKeys {
 	AutoRunDelay = 'testing.autoRun.delay',
+	AutoRunMode = 'testing.autoRun.mode',
 	AutoOpenPeekView = 'testing.automaticallyOpenPeekView',
 	AutoOpenPeekViewDuringAutoRun = 'testing.automaticallyOpenPeekViewDuringAutoRun',
 }
@@ -18,12 +19,29 @@ export const enum AutoOpenPeekViewWhen {
 	FailureAnywhere = 'failureAnywhere',
 }
 
+export const enum AutoRunMode {
+	AllInWorkspace = 'allInWorkspace',
+	OnlyPreviouslyRun = 'onlyPreviouslyRun',
+}
+
 export const testingConfiguation: IConfigurationNode = {
 	id: 'testing',
 	order: 21,
 	title: localize('testConfigurationTitle', "Testing"),
 	type: 'object',
 	properties: {
+		[TestingConfigKeys.AutoRunMode]: {
+			description: localize('testing.autoRun.mode', "Controls which tests are automatically run."),
+			enum: [
+				AutoRunMode.AllInWorkspace,
+				AutoRunMode.OnlyPreviouslyRun,
+			],
+			default: AutoRunMode.AllInWorkspace,
+			enumDescriptions: [
+				localize('testing.autoRun.mode.allInWorkspace', "Automatically run and then re-run all tests in the workspace."),
+				localize('testing.autoRun.mode.onlyPreviouslyRun', "Only re-run tests that have been run before, when they change.")
+			],
+		},
 		[TestingConfigKeys.AutoRunDelay]: {
 			type: 'integer',
 			minimum: 0,
@@ -46,11 +64,12 @@ export const testingConfiguation: IConfigurationNode = {
 			description: localize('testing.automaticallyOpenPeekViewDuringAutoRun', "Controls whether to automatically open the peek view during auto-run mode."),
 			type: 'boolean',
 			default: false,
-		}
+		},
 	}
 };
 
 export interface ITestingConfiguration {
+	[TestingConfigKeys.AutoRunMode]: AutoRunMode;
 	[TestingConfigKeys.AutoRunDelay]: number;
 	[TestingConfigKeys.AutoOpenPeekView]: AutoOpenPeekViewWhen;
 	[TestingConfigKeys.AutoOpenPeekViewDuringAutoRun]: boolean;

src/vs/workbench/contrib/testing/common/testingAutoRun.ts

@@ -4,15 +4,17 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { RunOnceScheduler } from 'vs/base/common/async';
-import { Disposable, DisposableStore, MutableDisposable } from 'vs/base/common/lifecycle';
+import { CancellationTokenSource } from 'vs/base/common/cancellation';
+import { Disposable, DisposableStore, MutableDisposable, toDisposable } from 'vs/base/common/lifecycle';
 import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
 import { IContextKey, IContextKeyService } from 'vs/platform/contextkey/common/contextkey';
 import { createDecorator } from 'vs/platform/instantiation/common/instantiation';
-import { getTestingConfiguration, TestingConfigKeys } from 'vs/workbench/contrib/testing/common/configuration';
-import { TestIdWithProvider } from 'vs/workbench/contrib/testing/common/testCollection';
+import { AutoRunMode, getTestingConfiguration, TestingConfigKeys } from 'vs/workbench/contrib/testing/common/configuration';
+import { InternalTestItem, TestDiffOpType, TestIdWithProvider } from 'vs/workbench/contrib/testing/common/testCollection';
 import { TestingContextKeys } from 'vs/workbench/contrib/testing/common/testingContextKeys';
 import { ITestResultService, TestResultItemChangeReason } from 'vs/workbench/contrib/testing/common/testResultService';
 import { ITestService } from 'vs/workbench/contrib/testing/common/testService';
+import { IWorkspaceTestCollectionService } from 'vs/workbench/contrib/testing/common/workspaceTestCollectionService';
 
 export interface ITestingAutoRun {
 	/**
@@ -32,9 +34,16 @@ export class TestingAutoRun extends Disposable implements ITestingAutoRun {
 		@ITestService private readonly testService: ITestService,
 		@ITestResultService private readonly results: ITestResultService,
 		@IConfigurationService private readonly configuration: IConfigurationService,
+		@IWorkspaceTestCollectionService private readonly workspaceTests: IWorkspaceTestCollectionService,
 	) {
 		super();
 		this.enabled = TestingContextKeys.autoRun.bindTo(contextKeyService);
+
+		this._register(configuration.onDidChangeConfiguration(evt => {
+			if (evt.affectsConfiguration(TestingConfigKeys.AutoRunMode) && this.enabled.get()) {
+				this.runner.value = this.makeRunner();
+			}
+		}));
 	}
 
 	/**
@@ -54,20 +63,17 @@ export class TestingAutoRun extends Disposable implements ITestingAutoRun {
 	/**
 	 * Creates the runner. Is triggered when tests are marked as retired.
 	 * Runs them on a debounce.
-	 *
-	 * We keep a workspace subscription open and try to find always find
-	 * tests in the workspace -- as opposed to document tests. This is needed
-	 * because a user could trigger a test run from a document, but close that
-	 * document and edit another file that would cause the test to be retired.
 	 */
 	private makeRunner() {
 		let isRunning = false;
 		const rerunIds = new Map<string, TestIdWithProvider>();
 		const store = new DisposableStore();
+		const cts = new CancellationTokenSource();
+		store.add(toDisposable(() => cts.dispose(true)));
 
 		let delay = getTestingConfiguration(this.configuration, TestingConfigKeys.AutoRunDelay);
 
-		store.add(this.configuration.onDidChangeConfiguration(evt => {
+		store.add(this.configuration.onDidChangeConfiguration(() => {
 			delay = getTestingConfiguration(this.configuration, TestingConfigKeys.AutoRunDelay);
 		}));
 
@@ -84,19 +90,43 @@ export class TestingAutoRun extends Disposable implements ITestingAutoRun {
 			}
 		}, delay));
 
-		store.add(this.results.onTestChanged(evt => {
-			if (evt.reason !== TestResultItemChangeReason.Retired) {
-				return;
-			}
-
-			const { extId } = evt.item.item;
-			rerunIds.set(`${extId}/${evt.item.providerId}`, ({ testId: extId, providerId: evt.item.providerId }));
-
+		const addToRerun = (test: InternalTestItem) => {
+			rerunIds.set(`${test.item.extId}/${test.providerId}`, ({ testId: test.item.extId, providerId: test.providerId }));
 			if (!isRunning) {
 				scheduler.schedule(delay);
 			}
+		};
+
+		store.add(this.results.onTestChanged(evt => {
+			if (evt.reason === TestResultItemChangeReason.Retired) {
+				addToRerun(evt.item);
+			}
 		}));
 
+		if (getTestingConfiguration(this.configuration, TestingConfigKeys.AutoRunMode) === AutoRunMode.AllInWorkspace) {
+			const sub = this.workspaceTests.subscribeToWorkspaceTests();
+			store.add(sub);
+
+			sub.waitForAllRoots(cts.token).then(() => {
+				if (!cts.token.isCancellationRequested) {
+					for (const [, collection] of sub.workspaceFolderCollections) {
+						for (const rootId of collection.rootIds) {
+							const root = collection.getNodeById(rootId);
+							if (root) { addToRerun(root); }
+						}
+					}
+				}
+			});
+
+			store.add(sub.onDiff(([, diff]) => {
+				for (const entry of diff) {
+					if (entry[0] === TestDiffOpType.Add) {
+						addToRerun(entry[1]);
+					}
+				}
+			}));
+		}
+
 		return store;
 	}
 }

src/vs/workbench/contrib/testing/common/workspaceTestCollectionService.ts

@@ -3,6 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import { CancellationToken } from 'vs/base/common/cancellation';
 import { Emitter } from 'vs/base/common/event';
 import { Iterable } from 'vs/base/common/iterator';
 import { Disposable, DisposableStore, IDisposable, toDisposable } from 'vs/base/common/lifecycle';
@@ -10,7 +11,7 @@ import { createDecorator, IInstantiationService } from 'vs/platform/instantiatio
 import { IWorkspaceContextService, IWorkspaceFolder, IWorkspaceFoldersChangeEvent } from 'vs/platform/workspace/common/workspace';
 import { ExtHostTestingResource } from 'vs/workbench/api/common/extHost.protocol';
 import { IncrementalTestCollectionItem, TestsDiff } from 'vs/workbench/contrib/testing/common/testCollection';
-import { IMainThreadTestCollection, ITestService } from 'vs/workbench/contrib/testing/common/testService';
+import { IMainThreadTestCollection, ITestService, waitForAllRoots } from 'vs/workbench/contrib/testing/common/testService';
 
 export interface ITestSubscriptionFolder {
 	folder: IWorkspaceFolder;
@@ -41,6 +42,10 @@ export class TestSubscriptionListener extends Disposable {
 		this._register(toDisposable(onDispose));
 	}
 
+	public async waitForAllRoots(token?: CancellationToken) {
+		await Promise.all(this.subscription.workspaceFolderCollections.map(([, c]) => waitForAllRoots(c, token)));
+	}
+
 	public publishFolderChange(evt: IWorkspaceFoldersChangeEvent) {
 		this.onFolderChangeEmitter.fire(evt);
 	}