From cbb1610c1257afdfecdf1e307dd5133d7ab5e89d Mon Sep 17 00:00:00 2001 From: Benjamin Pasero Date: Mon, 8 Jun 2020 12:46:15 +0200 Subject: [PATCH] files - add support for tracking bytes written and adopt for upload in web --- src/vs/base/test/common/stream.test.ts | 6 --- src/vs/platform/files/common/fileService.ts | 37 ++++++++++++------- src/vs/platform/files/common/files.ts | 37 +++++++++++++++++++ .../electron-browser/diskFileService.test.ts | 30 +++++++++++---- .../browser/parts/editor/binaryEditor.ts | 28 +------------- .../files/browser/views/explorerViewer.ts | 37 +++++++++++++------ 6 files changed, 108 insertions(+), 67 deletions(-) diff --git a/src/vs/base/test/common/stream.test.ts b/src/vs/base/test/common/stream.test.ts index 9a54cf6c423..443dc783978 100644 --- a/src/vs/base/test/common/stream.test.ts +++ b/src/vs/base/test/common/stream.test.ts @@ -76,12 +76,6 @@ suite('Stream', () => { }; stream.on('error', errorListener); - let end = false; - const endListener = () => { - end = true; - }; - stream.on('end', endListener); - let data = false; const dataListener = () => { data = true; diff --git a/src/vs/platform/files/common/fileService.ts b/src/vs/platform/files/common/fileService.ts index 76ca9918e3e..53bab8f1faf 100644 --- a/src/vs/platform/files/common/fileService.ts +++ b/src/vs/platform/files/common/fileService.ts @@ -330,12 +330,12 @@ export class FileService extends Disposable implements IFileService { // write file: unbuffered (only if data to write is a buffer, or the provider has no buffered write capability) if (!hasOpenReadWriteCloseCapability(provider) || (hasReadWriteCapability(provider) && bufferOrReadableOrStream instanceof VSBuffer)) { - await this.doWriteUnbuffered(provider, resource, bufferOrReadableOrStream); + await this.doWriteUnbuffered(provider, resource, bufferOrReadableOrStream, options); } // write file: buffered else { - await this.doWriteBuffered(provider, resource, bufferOrReadableOrStream instanceof VSBuffer ? bufferToReadable(bufferOrReadableOrStream) : bufferOrReadableOrStream); + await this.doWriteBuffered(provider, resource, bufferOrReadableOrStream instanceof VSBuffer ? bufferToReadable(bufferOrReadableOrStream) : bufferOrReadableOrStream, options); } } catch (error) { throw new FileOperationError(localize('err.write', "Unable to write file '{0}' ({1})", this.resourceForError(resource), ensureFileSystemProviderError(error).toString()), toFileOperationResult(error), options); @@ -953,7 +953,7 @@ export class FileService extends Disposable implements IFileService { return isPathCaseSensitive ? resource.toString() : resource.toString().toLowerCase(); } - private async doWriteBuffered(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, resource: URI, readableOrStream: VSBufferReadable | VSBufferReadableStream): Promise { + private async doWriteBuffered(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, resource: URI, readableOrStream: VSBufferReadable | VSBufferReadableStream, options?: IWriteFileOptions): Promise { return this.ensureWriteQueue(provider, resource).queue(async () => { // open handle @@ -962,9 +962,9 @@ export class FileService extends Disposable implements IFileService { // write into handle until all bytes from buffer have been written try { if (isReadableStream(readableOrStream)) { - await this.doWriteStreamBufferedQueued(provider, handle, readableOrStream); + await this.doWriteStreamBufferedQueued(provider, handle, readableOrStream, options); } else { - await this.doWriteReadableBufferedQueued(provider, handle, readableOrStream); + await this.doWriteReadableBufferedQueued(provider, handle, readableOrStream, options); } } catch (error) { throw ensureFileSystemProviderError(error); @@ -976,7 +976,7 @@ export class FileService extends Disposable implements IFileService { }); } - private doWriteStreamBufferedQueued(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, stream: VSBufferReadableStream): Promise { + private doWriteStreamBufferedQueued(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, stream: VSBufferReadableStream, options?: IWriteFileOptions): Promise { return new Promise((resolve, reject) => { let posInFile = 0; @@ -986,7 +986,7 @@ export class FileService extends Disposable implements IFileService { stream.pause(); try { - await this.doWriteBuffer(provider, handle, chunk, chunk.byteLength, posInFile, 0); + await this.doWriteBuffer(provider, handle, chunk, chunk.byteLength, posInFile, 0, options); } catch (error) { return reject(error); } @@ -1005,30 +1005,35 @@ export class FileService extends Disposable implements IFileService { }); } - private async doWriteReadableBufferedQueued(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, readable: VSBufferReadable): Promise { + private async doWriteReadableBufferedQueued(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, readable: VSBufferReadable, options?: IWriteFileOptions): Promise { let posInFile = 0; let chunk: VSBuffer | null; while ((chunk = readable.read()) !== null) { - await this.doWriteBuffer(provider, handle, chunk, chunk.byteLength, posInFile, 0); + await this.doWriteBuffer(provider, handle, chunk, chunk.byteLength, posInFile, 0, options); posInFile += chunk.byteLength; } } - private async doWriteBuffer(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, buffer: VSBuffer, length: number, posInFile: number, posInBuffer: number): Promise { + private async doWriteBuffer(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, buffer: VSBuffer, length: number, posInFile: number, posInBuffer: number, options?: IWriteFileOptions): Promise { let totalBytesWritten = 0; while (totalBytesWritten < length) { + + // Write through the provider const bytesWritten = await provider.write(handle, posInFile + totalBytesWritten, buffer.buffer, posInBuffer + totalBytesWritten, length - totalBytesWritten); totalBytesWritten += bytesWritten; + + // report progress as needed + options?.progress?.(bytesWritten); } } - private async doWriteUnbuffered(provider: IFileSystemProviderWithFileReadWriteCapability, resource: URI, bufferOrReadableOrStream: VSBuffer | VSBufferReadable | VSBufferReadableStream): Promise { - return this.ensureWriteQueue(provider, resource).queue(() => this.doWriteUnbufferedQueued(provider, resource, bufferOrReadableOrStream)); + private async doWriteUnbuffered(provider: IFileSystemProviderWithFileReadWriteCapability, resource: URI, bufferOrReadableOrStream: VSBuffer | VSBufferReadable | VSBufferReadableStream, options?: IWriteFileOptions): Promise { + return this.ensureWriteQueue(provider, resource).queue(() => this.doWriteUnbufferedQueued(provider, resource, bufferOrReadableOrStream, options)); } - private async doWriteUnbufferedQueued(provider: IFileSystemProviderWithFileReadWriteCapability, resource: URI, bufferOrReadableOrStream: VSBuffer | VSBufferReadable | VSBufferReadableStream): Promise { + private async doWriteUnbufferedQueued(provider: IFileSystemProviderWithFileReadWriteCapability, resource: URI, bufferOrReadableOrStream: VSBuffer | VSBufferReadable | VSBufferReadableStream, options?: IWriteFileOptions): Promise { let buffer: VSBuffer; if (bufferOrReadableOrStream instanceof VSBuffer) { buffer = bufferOrReadableOrStream; @@ -1038,7 +1043,11 @@ export class FileService extends Disposable implements IFileService { buffer = readableToBuffer(bufferOrReadableOrStream); } - return provider.writeFile(resource, buffer.buffer, { create: true, overwrite: true }); + // Write through the provider + await provider.writeFile(resource, buffer.buffer, { create: true, overwrite: true }); + + // Report progress as needed + options?.progress?.(buffer.byteLength); } private async doPipeBuffered(sourceProvider: IFileSystemProviderWithOpenReadWriteCloseCapability, source: URI, targetProvider: IFileSystemProviderWithOpenReadWriteCloseCapability, target: URI): Promise { diff --git a/src/vs/platform/files/common/files.ts b/src/vs/platform/files/common/files.ts index da896bca24f..fa02a4b2520 100644 --- a/src/vs/platform/files/common/files.ts +++ b/src/vs/platform/files/common/files.ts @@ -731,6 +731,13 @@ export interface IWriteFileOptions { * The etag of the file. This can be used to prevent dirty writes. */ readonly etag?: string; + + /** + * The progress callback can be used to get accurate information how many + * bytes have been written. Each call carries the length of bytes written + * since the last call was made. + */ + readonly progress?: (byteLength: number) => void; } export interface IResolveFileOptions { @@ -865,3 +872,33 @@ export function whenProviderRegistered(file: URI, fileService: IFileService): Pr */ export const MIN_MAX_MEMORY_SIZE_MB = 2048; export const FALLBACK_MAX_MEMORY_SIZE_MB = 4096; + +/** + * Helper to format a raw byte size into a human readable label. + */ +export class BinarySize { + static readonly KB = 1024; + static readonly MB = BinarySize.KB * BinarySize.KB; + static readonly GB = BinarySize.MB * BinarySize.KB; + static readonly TB = BinarySize.GB * BinarySize.KB; + + static formatSize(size: number): string { + if (size < BinarySize.KB) { + return localize('sizeB', "{0}B", size); + } + + if (size < BinarySize.MB) { + return localize('sizeKB', "{0}KB", (size / BinarySize.KB).toFixed(2)); + } + + if (size < BinarySize.GB) { + return localize('sizeMB', "{0}MB", (size / BinarySize.MB).toFixed(2)); + } + + if (size < BinarySize.TB) { + return localize('sizeGB', "{0}GB", (size / BinarySize.GB).toFixed(2)); + } + + return localize('sizeTB', "{0}TB", (size / BinarySize.TB).toFixed(2)); + } +} diff --git a/src/vs/platform/files/test/electron-browser/diskFileService.test.ts b/src/vs/platform/files/test/electron-browser/diskFileService.test.ts index 0ea2e2b1c8b..1ec2b39e92a 100644 --- a/src/vs/platform/files/test/electron-browser/diskFileService.test.ts +++ b/src/vs/platform/files/test/electron-browser/diskFileService.test.ts @@ -1681,9 +1681,11 @@ suite('Disk File Service', function () { assert.equal(content, 'Small File'); const newContent = 'Updates to the small file'; - await service.writeFile(resource, VSBuffer.fromString(newContent)); + let totalBytes = 0; + await service.writeFile(resource, VSBuffer.fromString(newContent), { progress: byteLength => totalBytes += byteLength }); assert.equal(readFileSync(resource.fsPath), newContent); + assert.equal(totalBytes, newContent.length); } test('writeFile (large file) - default', async () => { @@ -1708,10 +1710,12 @@ suite('Disk File Service', function () { const content = readFileSync(resource.fsPath); const newContent = content.toString() + content.toString(); - const fileStat = await service.writeFile(resource, VSBuffer.fromString(newContent)); + let totalBytes = 0; + const fileStat = await service.writeFile(resource, VSBuffer.fromString(newContent), { progress: byteLength => totalBytes += byteLength }); assert.equal(fileStat.name, 'lorem.txt'); assert.equal(readFileSync(resource.fsPath), newContent); + assert.equal(totalBytes, newContent.length); } test('writeFile - buffered - readonly throws', async () => { @@ -1782,9 +1786,11 @@ suite('Disk File Service', function () { assert.equal(content, 'Small File'); const newContent = 'Updates to the small file'; - await service.writeFile(resource, toLineByLineReadable(newContent)); + let totalBytes = 0; + await service.writeFile(resource, toLineByLineReadable(newContent), { progress: byteLength => totalBytes += byteLength }); assert.equal(readFileSync(resource.fsPath), newContent); + assert.equal(totalBytes, newContent.length); } test('writeFile (large file - readable) - default', async () => { @@ -1809,10 +1815,12 @@ suite('Disk File Service', function () { const content = readFileSync(resource.fsPath); const newContent = content.toString() + content.toString(); - const fileStat = await service.writeFile(resource, toLineByLineReadable(newContent)); + let totalBytes = 0; + const fileStat = await service.writeFile(resource, toLineByLineReadable(newContent), { progress: byteLength => totalBytes += byteLength }); assert.equal(fileStat.name, 'lorem.txt'); assert.equal(readFileSync(resource.fsPath), newContent); + assert.equal(totalBytes, newContent.length); } test('writeFile (stream) - default', async () => { @@ -1835,10 +1843,13 @@ suite('Disk File Service', function () { const source = URI.file(join(testDir, 'small.txt')); const target = URI.file(join(testDir, 'small-copy.txt')); - const fileStat = await service.writeFile(target, streamToBufferReadableStream(createReadStream(source.fsPath))); + let totalBytes = 0; + const fileStat = await service.writeFile(target, streamToBufferReadableStream(createReadStream(source.fsPath)), { progress: byteLength => totalBytes += byteLength }); assert.equal(fileStat.name, 'small-copy.txt'); - assert.equal(readFileSync(source.fsPath).toString(), readFileSync(target.fsPath).toString()); + const targetContents = readFileSync(target.fsPath).toString(); + assert.equal(readFileSync(source.fsPath).toString(), targetContents); + assert.equal(totalBytes, targetContents.length); } test('writeFile (large file - stream) - default', async () => { @@ -1861,10 +1872,13 @@ suite('Disk File Service', function () { const source = URI.file(join(testDir, 'lorem.txt')); const target = URI.file(join(testDir, 'lorem-copy.txt')); - const fileStat = await service.writeFile(target, streamToBufferReadableStream(createReadStream(source.fsPath))); + let totalBytes = 0; + const fileStat = await service.writeFile(target, streamToBufferReadableStream(createReadStream(source.fsPath)), { progress: byteLength => totalBytes += byteLength }); assert.equal(fileStat.name, 'lorem-copy.txt'); - assert.equal(readFileSync(source.fsPath).toString(), readFileSync(target.fsPath).toString()); + const targetContents = readFileSync(target.fsPath).toString(); + assert.equal(readFileSync(source.fsPath).toString(), targetContents); + assert.equal(totalBytes, targetContents.length); } test('writeFile (file is created including parents)', async () => { diff --git a/src/vs/workbench/browser/parts/editor/binaryEditor.ts b/src/vs/workbench/browser/parts/editor/binaryEditor.ts index 4fc47fa281b..ca270875dfb 100644 --- a/src/vs/workbench/browser/parts/editor/binaryEditor.ts +++ b/src/vs/workbench/browser/parts/editor/binaryEditor.ts @@ -20,6 +20,7 @@ import { dispose, IDisposable, Disposable, DisposableStore } from 'vs/base/commo import { IStorageService } from 'vs/platform/storage/common/storage'; import { IWorkbenchEnvironmentService } from 'vs/workbench/services/environment/common/environmentService'; import { assertIsDefined, assertAllDefined } from 'vs/base/common/types'; +import { BinarySize } from 'vs/platform/files/common/files'; export interface IOpenCallbacks { openInternal: (input: EditorInput, options: EditorOptions | undefined) => Promise; @@ -169,33 +170,6 @@ export interface IResourceDescriptor { readonly mime: string; } -class BinarySize { - static readonly KB = 1024; - static readonly MB = BinarySize.KB * BinarySize.KB; - static readonly GB = BinarySize.MB * BinarySize.KB; - static readonly TB = BinarySize.GB * BinarySize.KB; - - static formatSize(size: number): string { - if (size < BinarySize.KB) { - return nls.localize('sizeB', "{0}B", size); - } - - if (size < BinarySize.MB) { - return nls.localize('sizeKB', "{0}KB", (size / BinarySize.KB).toFixed(2)); - } - - if (size < BinarySize.GB) { - return nls.localize('sizeMB', "{0}MB", (size / BinarySize.MB).toFixed(2)); - } - - if (size < BinarySize.TB) { - return nls.localize('sizeGB', "{0}GB", (size / BinarySize.GB).toFixed(2)); - } - - return nls.localize('sizeTB', "{0}TB", (size / BinarySize.TB).toFixed(2)); - } -} - interface ResourceViewerContext extends IDisposable { layout?(dimension: Dimension): void; } diff --git a/src/vs/workbench/contrib/files/browser/views/explorerViewer.ts b/src/vs/workbench/contrib/files/browser/views/explorerViewer.ts index a9f1462f624..1a2c63e6955 100644 --- a/src/vs/workbench/contrib/files/browser/views/explorerViewer.ts +++ b/src/vs/workbench/contrib/files/browser/views/explorerViewer.ts @@ -9,7 +9,7 @@ import * as glob from 'vs/base/common/glob'; import { IListVirtualDelegate, ListDragOverEffect } from 'vs/base/browser/ui/list/list'; import { IProgressService, ProgressLocation, IProgressStep, IProgress } from 'vs/platform/progress/common/progress'; import { INotificationService, Severity } from 'vs/platform/notification/common/notification'; -import { IFileService, FileKind, FileOperationError, FileOperationResult, FileSystemProviderCapabilities } from 'vs/platform/files/common/files'; +import { IFileService, FileKind, FileOperationError, FileOperationResult, FileSystemProviderCapabilities, BinarySize } from 'vs/platform/files/common/files'; import { IWorkbenchLayoutService } from 'vs/workbench/services/layout/browser/layoutService'; import { IWorkspaceContextService, WorkbenchState } from 'vs/platform/workspace/common/workspace'; import { IDisposable, Disposable, dispose, toDisposable, DisposableStore } from 'vs/base/common/lifecycle'; @@ -1023,12 +1023,25 @@ export class FileDragAndDrop implements ITreeDragAndDrop { } // Report progress + let totalBytesUploaded = 0; + const reportProgress = (fileSize: number, bytesUploaded: number): void => { + totalBytesUploaded += bytesUploaded; + + let message: string; + if (operation.total === 1 && entry.name) { + message = entry.name; + } else { + message = localize('uploadProgress', "{0} of {1} files", operation.worked, operation.total); + } + + if (fileSize > BinarySize.MB) { + message = localize('uploadProgressDetail', "{0} ({1} of {2})", message, BinarySize.formatSize(totalBytesUploaded), BinarySize.formatSize(fileSize)); + } + + progress.report({ message }); + }; operation.worked++; - if (operation.total === 1) { - progress.report({ message: entry.name }); - } else { - progress.report({ message: localize('uploadProgress', "{0} of {1} files", operation.worked, operation.total) }); - } + reportProgress(0, 0); // Handle file upload if (entry.isFile) { @@ -1040,12 +1053,12 @@ export class FileDragAndDrop implements ITreeDragAndDrop { // Chrome/Edge/Firefox support stream method if (typeof file.stream === 'function') { - await this.doUploadWebFileEntryBuffered(resource, file); + await this.doUploadWebFileEntryBuffered(resource, file, reportProgress); } // Fallback to unbuffered upload for other browsers else { - await this.doUploadWebFileEntryUnbuffered(resource, file); + await this.doUploadWebFileEntryUnbuffered(resource, file, reportProgress); } return { isFile: true, resource }; @@ -1087,9 +1100,9 @@ export class FileDragAndDrop implements ITreeDragAndDrop { } } - private async doUploadWebFileEntryBuffered(resource: URI, file: File): Promise { + private async doUploadWebFileEntryBuffered(resource: URI, file: File, progressReporter: (fileSize: number, bytesUploaded: number) => void): Promise { const writeableStream = newWriteableBufferStream(); - const writeFilePromise = this.fileService.writeFile(resource, writeableStream); + const writeFilePromise = this.fileService.writeFile(resource, writeableStream, { progress: byteLength => progressReporter(file.size, byteLength) }); // Read the file in chunks using File.stream() web APIs try { @@ -1110,13 +1123,13 @@ export class FileDragAndDrop implements ITreeDragAndDrop { await writeFilePromise; } - private doUploadWebFileEntryUnbuffered(resource: URI, file: File): Promise { + private doUploadWebFileEntryUnbuffered(resource: URI, file: File, progressReporter: (fileSize: number, bytesUploaded: number) => void): Promise { return new Promise((resolve, reject) => { const reader = new FileReader(); reader.onload = async event => { try { if (event.target?.result instanceof ArrayBuffer) { - await this.fileService.writeFile(resource, VSBuffer.wrap(new Uint8Array(event.target.result))); + await this.fileService.writeFile(resource, VSBuffer.wrap(new Uint8Array(event.target.result)), { progress: byteLength => progressReporter(file.size, byteLength) }); } else { throw new Error('Could not read from dropped file.'); }