mirror of
https://github.com/Microsoft/vscode
synced 2024-07-17 02:57:19 +00:00
files - add support for tracking bytes written and adopt for upload in web
This commit is contained in:
Benjamin Pasero
parent
0c73b69495
commit
cbb1610c12
|
|
@ -76,12 +76,6 @@ suite('Stream', () => {
|
|||
};
|
||||
stream.on('error', errorListener);
|
||||
|
||||
let end = false;
|
||||
const endListener = () => {
|
||||
end = true;
|
||||
};
|
||||
stream.on('end', endListener);
|
||||
|
||||
let data = false;
|
||||
const dataListener = () => {
|
||||
data = true;
|
||||
|
|
|
|||
|
|
@ -330,12 +330,12 @@ export class FileService extends Disposable implements IFileService {
|
|||
|
||||
// write file: unbuffered (only if data to write is a buffer, or the provider has no buffered write capability)
|
||||
if (!hasOpenReadWriteCloseCapability(provider) || (hasReadWriteCapability(provider) && bufferOrReadableOrStream instanceof VSBuffer)) {
|
||||
await this.doWriteUnbuffered(provider, resource, bufferOrReadableOrStream);
|
||||
await this.doWriteUnbuffered(provider, resource, bufferOrReadableOrStream, options);
|
||||
}
|
||||
|
||||
// write file: buffered
|
||||
else {
|
||||
await this.doWriteBuffered(provider, resource, bufferOrReadableOrStream instanceof VSBuffer ? bufferToReadable(bufferOrReadableOrStream) : bufferOrReadableOrStream);
|
||||
await this.doWriteBuffered(provider, resource, bufferOrReadableOrStream instanceof VSBuffer ? bufferToReadable(bufferOrReadableOrStream) : bufferOrReadableOrStream, options);
|
||||
}
|
||||
} catch (error) {
|
||||
throw new FileOperationError(localize('err.write', "Unable to write file '{0}' ({1})", this.resourceForError(resource), ensureFileSystemProviderError(error).toString()), toFileOperationResult(error), options);
|
||||
|
|
@ -953,7 +953,7 @@ export class FileService extends Disposable implements IFileService {
|
|||
return isPathCaseSensitive ? resource.toString() : resource.toString().toLowerCase();
|
||||
}
|
||||
|
||||
private async doWriteBuffered(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, resource: URI, readableOrStream: VSBufferReadable | VSBufferReadableStream): Promise<void> {
|
||||
private async doWriteBuffered(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, resource: URI, readableOrStream: VSBufferReadable | VSBufferReadableStream, options?: IWriteFileOptions): Promise<void> {
|
||||
return this.ensureWriteQueue(provider, resource).queue(async () => {
|
||||
|
||||
// open handle
|
||||
|
|
@ -962,9 +962,9 @@ export class FileService extends Disposable implements IFileService {
|
|||
// write into handle until all bytes from buffer have been written
|
||||
try {
|
||||
if (isReadableStream(readableOrStream)) {
|
||||
await this.doWriteStreamBufferedQueued(provider, handle, readableOrStream);
|
||||
await this.doWriteStreamBufferedQueued(provider, handle, readableOrStream, options);
|
||||
} else {
|
||||
await this.doWriteReadableBufferedQueued(provider, handle, readableOrStream);
|
||||
await this.doWriteReadableBufferedQueued(provider, handle, readableOrStream, options);
|
||||
}
|
||||
} catch (error) {
|
||||
throw ensureFileSystemProviderError(error);
|
||||
|
|
@ -976,7 +976,7 @@ export class FileService extends Disposable implements IFileService {
|
|||
});
|
||||
}
|
||||
|
||||
private doWriteStreamBufferedQueued(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, stream: VSBufferReadableStream): Promise<void> {
|
||||
private doWriteStreamBufferedQueued(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, stream: VSBufferReadableStream, options?: IWriteFileOptions): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let posInFile = 0;
|
||||
|
||||
|
|
@ -986,7 +986,7 @@ export class FileService extends Disposable implements IFileService {
|
|||
stream.pause();
|
||||
|
||||
try {
|
||||
await this.doWriteBuffer(provider, handle, chunk, chunk.byteLength, posInFile, 0);
|
||||
await this.doWriteBuffer(provider, handle, chunk, chunk.byteLength, posInFile, 0, options);
|
||||
} catch (error) {
|
||||
return reject(error);
|
||||
}
|
||||
|
|
@ -1005,30 +1005,35 @@ export class FileService extends Disposable implements IFileService {
|
|||
});
|
||||
}
|
||||
|
||||
private async doWriteReadableBufferedQueued(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, readable: VSBufferReadable): Promise<void> {
|
||||
private async doWriteReadableBufferedQueued(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, readable: VSBufferReadable, options?: IWriteFileOptions): Promise<void> {
|
||||
let posInFile = 0;
|
||||
|
||||
let chunk: VSBuffer | null;
|
||||
while ((chunk = readable.read()) !== null) {
|
||||
await this.doWriteBuffer(provider, handle, chunk, chunk.byteLength, posInFile, 0);
|
||||
await this.doWriteBuffer(provider, handle, chunk, chunk.byteLength, posInFile, 0, options);
|
||||
|
||||
posInFile += chunk.byteLength;
|
||||
}
|
||||
}
|
||||
|
||||
private async doWriteBuffer(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, buffer: VSBuffer, length: number, posInFile: number, posInBuffer: number): Promise<void> {
|
||||
private async doWriteBuffer(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, buffer: VSBuffer, length: number, posInFile: number, posInBuffer: number, options?: IWriteFileOptions): Promise<void> {
|
||||
let totalBytesWritten = 0;
|
||||
while (totalBytesWritten < length) {
|
||||
|
||||
// Write through the provider
|
||||
const bytesWritten = await provider.write(handle, posInFile + totalBytesWritten, buffer.buffer, posInBuffer + totalBytesWritten, length - totalBytesWritten);
|
||||
totalBytesWritten += bytesWritten;
|
||||
|
||||
// report progress as needed
|
||||
options?.progress?.(bytesWritten);
|
||||
}
|
||||
}
|
||||
|
||||
private async doWriteUnbuffered(provider: IFileSystemProviderWithFileReadWriteCapability, resource: URI, bufferOrReadableOrStream: VSBuffer | VSBufferReadable | VSBufferReadableStream): Promise<void> {
|
||||
return this.ensureWriteQueue(provider, resource).queue(() => this.doWriteUnbufferedQueued(provider, resource, bufferOrReadableOrStream));
|
||||
private async doWriteUnbuffered(provider: IFileSystemProviderWithFileReadWriteCapability, resource: URI, bufferOrReadableOrStream: VSBuffer | VSBufferReadable | VSBufferReadableStream, options?: IWriteFileOptions): Promise<void> {
|
||||
return this.ensureWriteQueue(provider, resource).queue(() => this.doWriteUnbufferedQueued(provider, resource, bufferOrReadableOrStream, options));
|
||||
}
|
||||
|
||||
private async doWriteUnbufferedQueued(provider: IFileSystemProviderWithFileReadWriteCapability, resource: URI, bufferOrReadableOrStream: VSBuffer | VSBufferReadable | VSBufferReadableStream): Promise<void> {
|
||||
private async doWriteUnbufferedQueued(provider: IFileSystemProviderWithFileReadWriteCapability, resource: URI, bufferOrReadableOrStream: VSBuffer | VSBufferReadable | VSBufferReadableStream, options?: IWriteFileOptions): Promise<void> {
|
||||
let buffer: VSBuffer;
|
||||
if (bufferOrReadableOrStream instanceof VSBuffer) {
|
||||
buffer = bufferOrReadableOrStream;
|
||||
|
|
@ -1038,7 +1043,11 @@ export class FileService extends Disposable implements IFileService {
|
|||
buffer = readableToBuffer(bufferOrReadableOrStream);
|
||||
}
|
||||
|
||||
return provider.writeFile(resource, buffer.buffer, { create: true, overwrite: true });
|
||||
// Write through the provider
|
||||
await provider.writeFile(resource, buffer.buffer, { create: true, overwrite: true });
|
||||
|
||||
// Report progress as needed
|
||||
options?.progress?.(buffer.byteLength);
|
||||
}
|
||||
|
||||
private async doPipeBuffered(sourceProvider: IFileSystemProviderWithOpenReadWriteCloseCapability, source: URI, targetProvider: IFileSystemProviderWithOpenReadWriteCloseCapability, target: URI): Promise<void> {
|
||||
|
|
|
|||
|
|
@ -731,6 +731,13 @@ export interface IWriteFileOptions {
|
|||
* The etag of the file. This can be used to prevent dirty writes.
|
||||
*/
|
||||
readonly etag?: string;
|
||||
|
||||
/**
|
||||
* The progress callback can be used to get accurate information how many
|
||||
* bytes have been written. Each call carries the length of bytes written
|
||||
* since the last call was made.
|
||||
*/
|
||||
readonly progress?: (byteLength: number) => void;
|
||||
}
|
||||
|
||||
export interface IResolveFileOptions {
|
||||
|
|
@ -865,3 +872,33 @@ export function whenProviderRegistered(file: URI, fileService: IFileService): Pr
|
|||
*/
|
||||
export const MIN_MAX_MEMORY_SIZE_MB = 2048;
|
||||
export const FALLBACK_MAX_MEMORY_SIZE_MB = 4096;
|
||||
|
||||
/**
|
||||
* Helper to format a raw byte size into a human readable label.
|
||||
*/
|
||||
export class BinarySize {
|
||||
static readonly KB = 1024;
|
||||
static readonly MB = BinarySize.KB * BinarySize.KB;
|
||||
static readonly GB = BinarySize.MB * BinarySize.KB;
|
||||
static readonly TB = BinarySize.GB * BinarySize.KB;
|
||||
|
||||
static formatSize(size: number): string {
|
||||
if (size < BinarySize.KB) {
|
||||
return localize('sizeB', "{0}B", size);
|
||||
}
|
||||
|
||||
if (size < BinarySize.MB) {
|
||||
return localize('sizeKB', "{0}KB", (size / BinarySize.KB).toFixed(2));
|
||||
}
|
||||
|
||||
if (size < BinarySize.GB) {
|
||||
return localize('sizeMB', "{0}MB", (size / BinarySize.MB).toFixed(2));
|
||||
}
|
||||
|
||||
if (size < BinarySize.TB) {
|
||||
return localize('sizeGB', "{0}GB", (size / BinarySize.GB).toFixed(2));
|
||||
}
|
||||
|
||||
return localize('sizeTB', "{0}TB", (size / BinarySize.TB).toFixed(2));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1681,9 +1681,11 @@ suite('Disk File Service', function () {
|
|||
assert.equal(content, 'Small File');
|
||||
|
||||
const newContent = 'Updates to the small file';
|
||||
await service.writeFile(resource, VSBuffer.fromString(newContent));
|
||||
let totalBytes = 0;
|
||||
await service.writeFile(resource, VSBuffer.fromString(newContent), { progress: byteLength => totalBytes += byteLength });
|
||||
|
||||
assert.equal(readFileSync(resource.fsPath), newContent);
|
||||
assert.equal(totalBytes, newContent.length);
|
||||
}
|
||||
|
||||
test('writeFile (large file) - default', async () => {
|
||||
|
|
@ -1708,10 +1710,12 @@ suite('Disk File Service', function () {
|
|||
const content = readFileSync(resource.fsPath);
|
||||
const newContent = content.toString() + content.toString();
|
||||
|
||||
const fileStat = await service.writeFile(resource, VSBuffer.fromString(newContent));
|
||||
let totalBytes = 0;
|
||||
const fileStat = await service.writeFile(resource, VSBuffer.fromString(newContent), { progress: byteLength => totalBytes += byteLength });
|
||||
assert.equal(fileStat.name, 'lorem.txt');
|
||||
|
||||
assert.equal(readFileSync(resource.fsPath), newContent);
|
||||
assert.equal(totalBytes, newContent.length);
|
||||
}
|
||||
|
||||
test('writeFile - buffered - readonly throws', async () => {
|
||||
|
|
@ -1782,9 +1786,11 @@ suite('Disk File Service', function () {
|
|||
assert.equal(content, 'Small File');
|
||||
|
||||
const newContent = 'Updates to the small file';
|
||||
await service.writeFile(resource, toLineByLineReadable(newContent));
|
||||
let totalBytes = 0;
|
||||
await service.writeFile(resource, toLineByLineReadable(newContent), { progress: byteLength => totalBytes += byteLength });
|
||||
|
||||
assert.equal(readFileSync(resource.fsPath), newContent);
|
||||
assert.equal(totalBytes, newContent.length);
|
||||
}
|
||||
|
||||
test('writeFile (large file - readable) - default', async () => {
|
||||
|
|
@ -1809,10 +1815,12 @@ suite('Disk File Service', function () {
|
|||
const content = readFileSync(resource.fsPath);
|
||||
const newContent = content.toString() + content.toString();
|
||||
|
||||
const fileStat = await service.writeFile(resource, toLineByLineReadable(newContent));
|
||||
let totalBytes = 0;
|
||||
const fileStat = await service.writeFile(resource, toLineByLineReadable(newContent), { progress: byteLength => totalBytes += byteLength });
|
||||
assert.equal(fileStat.name, 'lorem.txt');
|
||||
|
||||
assert.equal(readFileSync(resource.fsPath), newContent);
|
||||
assert.equal(totalBytes, newContent.length);
|
||||
}
|
||||
|
||||
test('writeFile (stream) - default', async () => {
|
||||
|
|
@ -1835,10 +1843,13 @@ suite('Disk File Service', function () {
|
|||
const source = URI.file(join(testDir, 'small.txt'));
|
||||
const target = URI.file(join(testDir, 'small-copy.txt'));
|
||||
|
||||
const fileStat = await service.writeFile(target, streamToBufferReadableStream(createReadStream(source.fsPath)));
|
||||
let totalBytes = 0;
|
||||
const fileStat = await service.writeFile(target, streamToBufferReadableStream(createReadStream(source.fsPath)), { progress: byteLength => totalBytes += byteLength });
|
||||
assert.equal(fileStat.name, 'small-copy.txt');
|
||||
|
||||
assert.equal(readFileSync(source.fsPath).toString(), readFileSync(target.fsPath).toString());
|
||||
const targetContents = readFileSync(target.fsPath).toString();
|
||||
assert.equal(readFileSync(source.fsPath).toString(), targetContents);
|
||||
assert.equal(totalBytes, targetContents.length);
|
||||
}
|
||||
|
||||
test('writeFile (large file - stream) - default', async () => {
|
||||
|
|
@ -1861,10 +1872,13 @@ suite('Disk File Service', function () {
|
|||
const source = URI.file(join(testDir, 'lorem.txt'));
|
||||
const target = URI.file(join(testDir, 'lorem-copy.txt'));
|
||||
|
||||
const fileStat = await service.writeFile(target, streamToBufferReadableStream(createReadStream(source.fsPath)));
|
||||
let totalBytes = 0;
|
||||
const fileStat = await service.writeFile(target, streamToBufferReadableStream(createReadStream(source.fsPath)), { progress: byteLength => totalBytes += byteLength });
|
||||
assert.equal(fileStat.name, 'lorem-copy.txt');
|
||||
|
||||
assert.equal(readFileSync(source.fsPath).toString(), readFileSync(target.fsPath).toString());
|
||||
const targetContents = readFileSync(target.fsPath).toString();
|
||||
assert.equal(readFileSync(source.fsPath).toString(), targetContents);
|
||||
assert.equal(totalBytes, targetContents.length);
|
||||
}
|
||||
|
||||
test('writeFile (file is created including parents)', async () => {
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import { dispose, IDisposable, Disposable, DisposableStore } from 'vs/base/commo
|
|||
import { IStorageService } from 'vs/platform/storage/common/storage';
|
||||
import { IWorkbenchEnvironmentService } from 'vs/workbench/services/environment/common/environmentService';
|
||||
import { assertIsDefined, assertAllDefined } from 'vs/base/common/types';
|
||||
import { BinarySize } from 'vs/platform/files/common/files';
|
||||
|
||||
export interface IOpenCallbacks {
|
||||
openInternal: (input: EditorInput, options: EditorOptions | undefined) => Promise<void>;
|
||||
|
|
@ -169,33 +170,6 @@ export interface IResourceDescriptor {
|
|||
readonly mime: string;
|
||||
}
|
||||
|
||||
class BinarySize {
|
||||
static readonly KB = 1024;
|
||||
static readonly MB = BinarySize.KB * BinarySize.KB;
|
||||
static readonly GB = BinarySize.MB * BinarySize.KB;
|
||||
static readonly TB = BinarySize.GB * BinarySize.KB;
|
||||
|
||||
static formatSize(size: number): string {
|
||||
if (size < BinarySize.KB) {
|
||||
return nls.localize('sizeB', "{0}B", size);
|
||||
}
|
||||
|
||||
if (size < BinarySize.MB) {
|
||||
return nls.localize('sizeKB', "{0}KB", (size / BinarySize.KB).toFixed(2));
|
||||
}
|
||||
|
||||
if (size < BinarySize.GB) {
|
||||
return nls.localize('sizeMB', "{0}MB", (size / BinarySize.MB).toFixed(2));
|
||||
}
|
||||
|
||||
if (size < BinarySize.TB) {
|
||||
return nls.localize('sizeGB', "{0}GB", (size / BinarySize.GB).toFixed(2));
|
||||
}
|
||||
|
||||
return nls.localize('sizeTB', "{0}TB", (size / BinarySize.TB).toFixed(2));
|
||||
}
|
||||
}
|
||||
|
||||
interface ResourceViewerContext extends IDisposable {
|
||||
layout?(dimension: Dimension): void;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import * as glob from 'vs/base/common/glob';
|
|||
import { IListVirtualDelegate, ListDragOverEffect } from 'vs/base/browser/ui/list/list';
|
||||
import { IProgressService, ProgressLocation, IProgressStep, IProgress } from 'vs/platform/progress/common/progress';
|
||||
import { INotificationService, Severity } from 'vs/platform/notification/common/notification';
|
||||
import { IFileService, FileKind, FileOperationError, FileOperationResult, FileSystemProviderCapabilities } from 'vs/platform/files/common/files';
|
||||
import { IFileService, FileKind, FileOperationError, FileOperationResult, FileSystemProviderCapabilities, BinarySize } from 'vs/platform/files/common/files';
|
||||
import { IWorkbenchLayoutService } from 'vs/workbench/services/layout/browser/layoutService';
|
||||
import { IWorkspaceContextService, WorkbenchState } from 'vs/platform/workspace/common/workspace';
|
||||
import { IDisposable, Disposable, dispose, toDisposable, DisposableStore } from 'vs/base/common/lifecycle';
|
||||
|
|
@ -1023,12 +1023,25 @@ export class FileDragAndDrop implements ITreeDragAndDrop<ExplorerItem> {
|
|||
}
|
||||
|
||||
// Report progress
|
||||
let totalBytesUploaded = 0;
|
||||
const reportProgress = (fileSize: number, bytesUploaded: number): void => {
|
||||
totalBytesUploaded += bytesUploaded;
|
||||
|
||||
let message: string;
|
||||
if (operation.total === 1 && entry.name) {
|
||||
message = entry.name;
|
||||
} else {
|
||||
message = localize('uploadProgress', "{0} of {1} files", operation.worked, operation.total);
|
||||
}
|
||||
|
||||
if (fileSize > BinarySize.MB) {
|
||||
message = localize('uploadProgressDetail', "{0} ({1} of {2})", message, BinarySize.formatSize(totalBytesUploaded), BinarySize.formatSize(fileSize));
|
||||
}
|
||||
|
||||
progress.report({ message });
|
||||
};
|
||||
operation.worked++;
|
||||
if (operation.total === 1) {
|
||||
progress.report({ message: entry.name });
|
||||
} else {
|
||||
progress.report({ message: localize('uploadProgress', "{0} of {1} files", operation.worked, operation.total) });
|
||||
}
|
||||
reportProgress(0, 0);
|
||||
|
||||
// Handle file upload
|
||||
if (entry.isFile) {
|
||||
|
|
@ -1040,12 +1053,12 @@ export class FileDragAndDrop implements ITreeDragAndDrop<ExplorerItem> {
|
|||
|
||||
// Chrome/Edge/Firefox support stream method
|
||||
if (typeof file.stream === 'function') {
|
||||
await this.doUploadWebFileEntryBuffered(resource, file);
|
||||
await this.doUploadWebFileEntryBuffered(resource, file, reportProgress);
|
||||
}
|
||||
|
||||
// Fallback to unbuffered upload for other browsers
|
||||
else {
|
||||
await this.doUploadWebFileEntryUnbuffered(resource, file);
|
||||
await this.doUploadWebFileEntryUnbuffered(resource, file, reportProgress);
|
||||
}
|
||||
|
||||
return { isFile: true, resource };
|
||||
|
|
@ -1087,9 +1100,9 @@ export class FileDragAndDrop implements ITreeDragAndDrop<ExplorerItem> {
|
|||
}
|
||||
}
|
||||
|
||||
private async doUploadWebFileEntryBuffered(resource: URI, file: File): Promise<void> {
|
||||
private async doUploadWebFileEntryBuffered(resource: URI, file: File, progressReporter: (fileSize: number, bytesUploaded: number) => void): Promise<void> {
|
||||
const writeableStream = newWriteableBufferStream();
|
||||
const writeFilePromise = this.fileService.writeFile(resource, writeableStream);
|
||||
const writeFilePromise = this.fileService.writeFile(resource, writeableStream, { progress: byteLength => progressReporter(file.size, byteLength) });
|
||||
|
||||
// Read the file in chunks using File.stream() web APIs
|
||||
try {
|
||||
|
|
@ -1110,13 +1123,13 @@ export class FileDragAndDrop implements ITreeDragAndDrop<ExplorerItem> {
|
|||
await writeFilePromise;
|
||||
}
|
||||
|
||||
private doUploadWebFileEntryUnbuffered(resource: URI, file: File): Promise<void> {
|
||||
private doUploadWebFileEntryUnbuffered(resource: URI, file: File, progressReporter: (fileSize: number, bytesUploaded: number) => void): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = async event => {
|
||||
try {
|
||||
if (event.target?.result instanceof ArrayBuffer) {
|
||||
await this.fileService.writeFile(resource, VSBuffer.wrap(new Uint8Array(event.target.result)));
|
||||
await this.fileService.writeFile(resource, VSBuffer.wrap(new Uint8Array(event.target.result)), { progress: byteLength => progressReporter(file.size, byteLength) });
|
||||
} else {
|
||||
throw new Error('Could not read from dropped file.');
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue