files - add support for tracking bytes written and adopt for upload in web

src/vs/base/test/common/stream.test.ts

@@ -76,12 +76,6 @@ suite('Stream', () => {
 		};
 		stream.on('error', errorListener);
 
-		let end = false;
-		const endListener = () => {
-			end = true;
-		};
-		stream.on('end', endListener);
-
 		let data = false;
 		const dataListener = () => {
 			data = true;

src/vs/platform/files/common/fileService.ts

@@ -330,12 +330,12 @@ export class FileService extends Disposable implements IFileService {
 
 			// write file: unbuffered (only if data to write is a buffer, or the provider has no buffered write capability)
 			if (!hasOpenReadWriteCloseCapability(provider) || (hasReadWriteCapability(provider) && bufferOrReadableOrStream instanceof VSBuffer)) {
-				await this.doWriteUnbuffered(provider, resource, bufferOrReadableOrStream);
+				await this.doWriteUnbuffered(provider, resource, bufferOrReadableOrStream, options);
 			}
 
 			// write file: buffered
 			else {
-				await this.doWriteBuffered(provider, resource, bufferOrReadableOrStream instanceof VSBuffer ? bufferToReadable(bufferOrReadableOrStream) : bufferOrReadableOrStream);
+				await this.doWriteBuffered(provider, resource, bufferOrReadableOrStream instanceof VSBuffer ? bufferToReadable(bufferOrReadableOrStream) : bufferOrReadableOrStream, options);
 			}
 		} catch (error) {
 			throw new FileOperationError(localize('err.write', "Unable to write file '{0}' ({1})", this.resourceForError(resource), ensureFileSystemProviderError(error).toString()), toFileOperationResult(error), options);
@@ -953,7 +953,7 @@ export class FileService extends Disposable implements IFileService {
 		return isPathCaseSensitive ? resource.toString() : resource.toString().toLowerCase();
 	}
 
-	private async doWriteBuffered(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, resource: URI, readableOrStream: VSBufferReadable | VSBufferReadableStream): Promise<void> {
+	private async doWriteBuffered(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, resource: URI, readableOrStream: VSBufferReadable | VSBufferReadableStream, options?: IWriteFileOptions): Promise<void> {
 		return this.ensureWriteQueue(provider, resource).queue(async () => {
 
 			// open handle
@@ -962,9 +962,9 @@ export class FileService extends Disposable implements IFileService {
 			// write into handle until all bytes from buffer have been written
 			try {
 				if (isReadableStream(readableOrStream)) {
-					await this.doWriteStreamBufferedQueued(provider, handle, readableOrStream);
+					await this.doWriteStreamBufferedQueued(provider, handle, readableOrStream, options);
 				} else {
-					await this.doWriteReadableBufferedQueued(provider, handle, readableOrStream);
+					await this.doWriteReadableBufferedQueued(provider, handle, readableOrStream, options);
 				}
 			} catch (error) {
 				throw ensureFileSystemProviderError(error);
@@ -976,7 +976,7 @@ export class FileService extends Disposable implements IFileService {
 		});
 	}
 
-	private doWriteStreamBufferedQueued(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, stream: VSBufferReadableStream): Promise<void> {
+	private doWriteStreamBufferedQueued(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, stream: VSBufferReadableStream, options?: IWriteFileOptions): Promise<void> {
 		return new Promise((resolve, reject) => {
 			let posInFile = 0;
 
@@ -986,7 +986,7 @@ export class FileService extends Disposable implements IFileService {
 				stream.pause();
 
 				try {
-					await this.doWriteBuffer(provider, handle, chunk, chunk.byteLength, posInFile, 0);
+					await this.doWriteBuffer(provider, handle, chunk, chunk.byteLength, posInFile, 0, options);
 				} catch (error) {
 					return reject(error);
 				}
@@ -1005,30 +1005,35 @@ export class FileService extends Disposable implements IFileService {
 		});
 	}
 
-	private async doWriteReadableBufferedQueued(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, readable: VSBufferReadable): Promise<void> {
+	private async doWriteReadableBufferedQueued(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, readable: VSBufferReadable, options?: IWriteFileOptions): Promise<void> {
 		let posInFile = 0;
 
 		let chunk: VSBuffer | null;
 		while ((chunk = readable.read()) !== null) {
-			await this.doWriteBuffer(provider, handle, chunk, chunk.byteLength, posInFile, 0);
+			await this.doWriteBuffer(provider, handle, chunk, chunk.byteLength, posInFile, 0, options);
 
 			posInFile += chunk.byteLength;
 		}
 	}
 
-	private async doWriteBuffer(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, buffer: VSBuffer, length: number, posInFile: number, posInBuffer: number): Promise<void> {
+	private async doWriteBuffer(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, buffer: VSBuffer, length: number, posInFile: number, posInBuffer: number, options?: IWriteFileOptions): Promise<void> {
 		let totalBytesWritten = 0;
 		while (totalBytesWritten < length) {
+
+			// Write through the provider
 			const bytesWritten = await provider.write(handle, posInFile + totalBytesWritten, buffer.buffer, posInBuffer + totalBytesWritten, length - totalBytesWritten);
 			totalBytesWritten += bytesWritten;
+
+			// report progress as needed
+			options?.progress?.(bytesWritten);
 		}
 	}
 
-	private async doWriteUnbuffered(provider: IFileSystemProviderWithFileReadWriteCapability, resource: URI, bufferOrReadableOrStream: VSBuffer | VSBufferReadable | VSBufferReadableStream): Promise<void> {
+	private async doWriteUnbuffered(provider: IFileSystemProviderWithFileReadWriteCapability, resource: URI, bufferOrReadableOrStream: VSBuffer | VSBufferReadable | VSBufferReadableStream, options?: IWriteFileOptions): Promise<void> {
-		return this.ensureWriteQueue(provider, resource).queue(() => this.doWriteUnbufferedQueued(provider, resource, bufferOrReadableOrStream));
+		return this.ensureWriteQueue(provider, resource).queue(() => this.doWriteUnbufferedQueued(provider, resource, bufferOrReadableOrStream, options));
 	}
 
-	private async doWriteUnbufferedQueued(provider: IFileSystemProviderWithFileReadWriteCapability, resource: URI, bufferOrReadableOrStream: VSBuffer | VSBufferReadable | VSBufferReadableStream): Promise<void> {
+	private async doWriteUnbufferedQueued(provider: IFileSystemProviderWithFileReadWriteCapability, resource: URI, bufferOrReadableOrStream: VSBuffer | VSBufferReadable | VSBufferReadableStream, options?: IWriteFileOptions): Promise<void> {
 		let buffer: VSBuffer;
 		if (bufferOrReadableOrStream instanceof VSBuffer) {
 			buffer = bufferOrReadableOrStream;
@@ -1038,7 +1043,11 @@ export class FileService extends Disposable implements IFileService {
 			buffer = readableToBuffer(bufferOrReadableOrStream);
 		}
 
-		return provider.writeFile(resource, buffer.buffer, { create: true, overwrite: true });
+		// Write through the provider
+		await provider.writeFile(resource, buffer.buffer, { create: true, overwrite: true });
+
+		// Report progress as needed
+		options?.progress?.(buffer.byteLength);
 	}
 
 	private async doPipeBuffered(sourceProvider: IFileSystemProviderWithOpenReadWriteCloseCapability, source: URI, targetProvider: IFileSystemProviderWithOpenReadWriteCloseCapability, target: URI): Promise<void> {

src/vs/platform/files/common/files.ts

@@ -731,6 +731,13 @@ export interface IWriteFileOptions {
 	 * The etag of the file. This can be used to prevent dirty writes.
 	 */
 	readonly etag?: string;
+
+	/**
+	 * The progress callback can be used to get accurate information how many
+	 * bytes have been written. Each call carries the length of bytes written
+	 * since the last call was made.
+	 */
+	readonly progress?: (byteLength: number) => void;
 }
 
 export interface IResolveFileOptions {
@@ -865,3 +872,33 @@ export function whenProviderRegistered(file: URI, fileService: IFileService): Pr
  */
 export const MIN_MAX_MEMORY_SIZE_MB = 2048;
 export const FALLBACK_MAX_MEMORY_SIZE_MB = 4096;
+
+/**
+ * Helper to format a raw byte size into a human readable label.
+ */
+export class BinarySize {
+	static readonly KB = 1024;
+	static readonly MB = BinarySize.KB * BinarySize.KB;
+	static readonly GB = BinarySize.MB * BinarySize.KB;
+	static readonly TB = BinarySize.GB * BinarySize.KB;
+
+	static formatSize(size: number): string {
+		if (size < BinarySize.KB) {
+			return localize('sizeB', "{0}B", size);
+		}
+
+		if (size < BinarySize.MB) {
+			return localize('sizeKB', "{0}KB", (size / BinarySize.KB).toFixed(2));
+		}
+
+		if (size < BinarySize.GB) {
+			return localize('sizeMB', "{0}MB", (size / BinarySize.MB).toFixed(2));
+		}
+
+		if (size < BinarySize.TB) {
+			return localize('sizeGB', "{0}GB", (size / BinarySize.GB).toFixed(2));
+		}
+
+		return localize('sizeTB', "{0}TB", (size / BinarySize.TB).toFixed(2));
+	}
+}

src/vs/platform/files/test/electron-browser/diskFileService.test.ts

@@ -1681,9 +1681,11 @@ suite('Disk File Service', function () {
 		assert.equal(content, 'Small File');
 
 		const newContent = 'Updates to the small file';
-		await service.writeFile(resource, VSBuffer.fromString(newContent));
+		let totalBytes = 0;
+		await service.writeFile(resource, VSBuffer.fromString(newContent), { progress: byteLength => totalBytes += byteLength });
 
 		assert.equal(readFileSync(resource.fsPath), newContent);
+		assert.equal(totalBytes, newContent.length);
 	}
 
 	test('writeFile (large file) - default', async () => {
@@ -1708,10 +1710,12 @@ suite('Disk File Service', function () {
 		const content = readFileSync(resource.fsPath);
 		const newContent = content.toString() + content.toString();
 
-		const fileStat = await service.writeFile(resource, VSBuffer.fromString(newContent));
+		let totalBytes = 0;
+		const fileStat = await service.writeFile(resource, VSBuffer.fromString(newContent), { progress: byteLength => totalBytes += byteLength });
 		assert.equal(fileStat.name, 'lorem.txt');
 
 		assert.equal(readFileSync(resource.fsPath), newContent);
+		assert.equal(totalBytes, newContent.length);
 	}
 
 	test('writeFile - buffered - readonly throws', async () => {
@@ -1782,9 +1786,11 @@ suite('Disk File Service', function () {
 		assert.equal(content, 'Small File');
 
 		const newContent = 'Updates to the small file';
-		await service.writeFile(resource, toLineByLineReadable(newContent));
+		let totalBytes = 0;
+		await service.writeFile(resource, toLineByLineReadable(newContent), { progress: byteLength => totalBytes += byteLength });
 
 		assert.equal(readFileSync(resource.fsPath), newContent);
+		assert.equal(totalBytes, newContent.length);
 	}
 
 	test('writeFile (large file - readable) - default', async () => {
@@ -1809,10 +1815,12 @@ suite('Disk File Service', function () {
 		const content = readFileSync(resource.fsPath);
 		const newContent = content.toString() + content.toString();
 
-		const fileStat = await service.writeFile(resource, toLineByLineReadable(newContent));
+		let totalBytes = 0;
+		const fileStat = await service.writeFile(resource, toLineByLineReadable(newContent), { progress: byteLength => totalBytes += byteLength });
 		assert.equal(fileStat.name, 'lorem.txt');
 
 		assert.equal(readFileSync(resource.fsPath), newContent);
+		assert.equal(totalBytes, newContent.length);
 	}
 
 	test('writeFile (stream) - default', async () => {
@@ -1835,10 +1843,13 @@ suite('Disk File Service', function () {
 		const source = URI.file(join(testDir, 'small.txt'));
 		const target = URI.file(join(testDir, 'small-copy.txt'));
 
-		const fileStat = await service.writeFile(target, streamToBufferReadableStream(createReadStream(source.fsPath)));
+		let totalBytes = 0;
+		const fileStat = await service.writeFile(target, streamToBufferReadableStream(createReadStream(source.fsPath)), { progress: byteLength => totalBytes += byteLength });
 		assert.equal(fileStat.name, 'small-copy.txt');
 
-		assert.equal(readFileSync(source.fsPath).toString(), readFileSync(target.fsPath).toString());
+		const targetContents = readFileSync(target.fsPath).toString();
+		assert.equal(readFileSync(source.fsPath).toString(), targetContents);
+		assert.equal(totalBytes, targetContents.length);
 	}
 
 	test('writeFile (large file - stream) - default', async () => {
@@ -1861,10 +1872,13 @@ suite('Disk File Service', function () {
 		const source = URI.file(join(testDir, 'lorem.txt'));
 		const target = URI.file(join(testDir, 'lorem-copy.txt'));
 
-		const fileStat = await service.writeFile(target, streamToBufferReadableStream(createReadStream(source.fsPath)));
+		let totalBytes = 0;
+		const fileStat = await service.writeFile(target, streamToBufferReadableStream(createReadStream(source.fsPath)), { progress: byteLength => totalBytes += byteLength });
 		assert.equal(fileStat.name, 'lorem-copy.txt');
 
-		assert.equal(readFileSync(source.fsPath).toString(), readFileSync(target.fsPath).toString());
+		const targetContents = readFileSync(target.fsPath).toString();
+		assert.equal(readFileSync(source.fsPath).toString(), targetContents);
+		assert.equal(totalBytes, targetContents.length);
 	}
 
 	test('writeFile (file is created including parents)', async () => {

src/vs/workbench/browser/parts/editor/binaryEditor.ts

@@ -20,6 +20,7 @@ import { dispose, IDisposable, Disposable, DisposableStore } from 'vs/base/commo
 import { IStorageService } from 'vs/platform/storage/common/storage';
 import { IWorkbenchEnvironmentService } from 'vs/workbench/services/environment/common/environmentService';
 import { assertIsDefined, assertAllDefined } from 'vs/base/common/types';
+import { BinarySize } from 'vs/platform/files/common/files';
 
 export interface IOpenCallbacks {
 	openInternal: (input: EditorInput, options: EditorOptions | undefined) => Promise<void>;
@@ -169,33 +170,6 @@ export interface IResourceDescriptor {
 	readonly mime: string;
 }
 
-class BinarySize {
-	static readonly KB = 1024;
-	static readonly MB = BinarySize.KB * BinarySize.KB;
-	static readonly GB = BinarySize.MB * BinarySize.KB;
-	static readonly TB = BinarySize.GB * BinarySize.KB;
-
-	static formatSize(size: number): string {
-		if (size < BinarySize.KB) {
-			return nls.localize('sizeB', "{0}B", size);
-		}
-
-		if (size < BinarySize.MB) {
-			return nls.localize('sizeKB', "{0}KB", (size / BinarySize.KB).toFixed(2));
-		}
-
-		if (size < BinarySize.GB) {
-			return nls.localize('sizeMB', "{0}MB", (size / BinarySize.MB).toFixed(2));
-		}
-
-		if (size < BinarySize.TB) {
-			return nls.localize('sizeGB', "{0}GB", (size / BinarySize.GB).toFixed(2));
-		}
-
-		return nls.localize('sizeTB', "{0}TB", (size / BinarySize.TB).toFixed(2));
-	}
-}
-
 interface ResourceViewerContext extends IDisposable {
 	layout?(dimension: Dimension): void;
 }

src/vs/workbench/contrib/files/browser/views/explorerViewer.ts

@@ -9,7 +9,7 @@ import * as glob from 'vs/base/common/glob';
 import { IListVirtualDelegate, ListDragOverEffect } from 'vs/base/browser/ui/list/list';
 import { IProgressService, ProgressLocation, IProgressStep, IProgress } from 'vs/platform/progress/common/progress';
 import { INotificationService, Severity } from 'vs/platform/notification/common/notification';
-import { IFileService, FileKind, FileOperationError, FileOperationResult, FileSystemProviderCapabilities } from 'vs/platform/files/common/files';
+import { IFileService, FileKind, FileOperationError, FileOperationResult, FileSystemProviderCapabilities, BinarySize } from 'vs/platform/files/common/files';
 import { IWorkbenchLayoutService } from 'vs/workbench/services/layout/browser/layoutService';
 import { IWorkspaceContextService, WorkbenchState } from 'vs/platform/workspace/common/workspace';
 import { IDisposable, Disposable, dispose, toDisposable, DisposableStore } from 'vs/base/common/lifecycle';
@@ -1023,12 +1023,25 @@ export class FileDragAndDrop implements ITreeDragAndDrop<ExplorerItem> {
 		}
 
 		// Report progress
+		let totalBytesUploaded = 0;
+		const reportProgress = (fileSize: number, bytesUploaded: number): void => {
+			totalBytesUploaded += bytesUploaded;
+
+			let message: string;
+			if (operation.total === 1 && entry.name) {
+				message = entry.name;
+			} else {
+				message = localize('uploadProgress', "{0} of {1} files", operation.worked, operation.total);
+			}
+
+			if (fileSize > BinarySize.MB) {
+				message = localize('uploadProgressDetail', "{0} ({1} of {2})", message, BinarySize.formatSize(totalBytesUploaded), BinarySize.formatSize(fileSize));
+			}
+
+			progress.report({ message });
+		};
 		operation.worked++;
-		if (operation.total === 1) {
+		reportProgress(0, 0);
-			progress.report({ message: entry.name });
-		} else {
-			progress.report({ message: localize('uploadProgress', "{0} of {1} files", operation.worked, operation.total) });
-		}
 
 		// Handle file upload
 		if (entry.isFile) {
@@ -1040,12 +1053,12 @@ export class FileDragAndDrop implements ITreeDragAndDrop<ExplorerItem> {
 
 			// Chrome/Edge/Firefox support stream method
 			if (typeof file.stream === 'function') {
-				await this.doUploadWebFileEntryBuffered(resource, file);
+				await this.doUploadWebFileEntryBuffered(resource, file, reportProgress);
 			}
 
 			// Fallback to unbuffered upload for other browsers
 			else {
-				await this.doUploadWebFileEntryUnbuffered(resource, file);
+				await this.doUploadWebFileEntryUnbuffered(resource, file, reportProgress);
 			}
 
 			return { isFile: true, resource };
@@ -1087,9 +1100,9 @@ export class FileDragAndDrop implements ITreeDragAndDrop<ExplorerItem> {
 		}
 	}
 
-	private async doUploadWebFileEntryBuffered(resource: URI, file: File): Promise<void> {
+	private async doUploadWebFileEntryBuffered(resource: URI, file: File, progressReporter: (fileSize: number, bytesUploaded: number) => void): Promise<void> {
 		const writeableStream = newWriteableBufferStream();
-		const writeFilePromise = this.fileService.writeFile(resource, writeableStream);
+		const writeFilePromise = this.fileService.writeFile(resource, writeableStream, { progress: byteLength => progressReporter(file.size, byteLength) });
 
 		// Read the file in chunks using File.stream() web APIs
 		try {
@@ -1110,13 +1123,13 @@ export class FileDragAndDrop implements ITreeDragAndDrop<ExplorerItem> {
 		await writeFilePromise;
 	}
 
-	private doUploadWebFileEntryUnbuffered(resource: URI, file: File): Promise<void> {
+	private doUploadWebFileEntryUnbuffered(resource: URI, file: File, progressReporter: (fileSize: number, bytesUploaded: number) => void): Promise<void> {
 		return new Promise<void>((resolve, reject) => {
 			const reader = new FileReader();
 			reader.onload = async event => {
 				try {
 					if (event.target?.result instanceof ArrayBuffer) {
-						await this.fileService.writeFile(resource, VSBuffer.wrap(new Uint8Array(event.target.result)));
+						await this.fileService.writeFile(resource, VSBuffer.wrap(new Uint8Array(event.target.result)), { progress: byteLength => progressReporter(file.size, byteLength) });
 					} else {
 						throw new Error('Could not read from dropped file.');
 					}