development/vscode
development/vscode
1
0
Fork 0
mirror of https://github.com/Microsoft/vscode synced 2024-08-27 04:49:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add a download util that uses a separate process

Browse source
This commit is contained in:
Alex Dima 2019-03-06 10:14:38 +01:00
parent ec878903e9
commit c4bed46784
5 changed files with 278 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

91
build/download/download.js Normal file
View file

@ -0,0 +1,91 @@
"use strict";
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
Object.defineProperty(exports, "__esModule", { value: true });
const https = require("https");
const fs = require("fs");
const path = require("path");
const cp = require("child_process");
function ensureDir(filepath) {
if (!fs.existsSync(filepath)) {
ensureDir(path.dirname(filepath));
fs.mkdirSync(filepath);
}
}
function download(options, destination) {
ensureDir(path.dirname(destination));
return new Promise((c, e) => {
const fd = fs.openSync(destination, 'w');
const req = https.get(options, (res) => {
res.on('data', (chunk) => {
fs.writeSync(fd, chunk);
});
res.on('end', () => {
fs.closeSync(fd);
c();
});
});
req.on('error', (reqErr) => {
console.error(`request to ${options.host}${options.path} failed.`);
console.error(reqErr);
e(reqErr);
});
});
}
const MARKER_ARGUMENT = `_download_fork_`;
function base64encode(str) {
return Buffer.from(str, 'utf8').toString('base64');
}
function base64decode(str) {
return Buffer.from(str, 'base64').toString('utf8');
}
function downloadInExternalProcess(options) {
const url = `https://${options.requestOptions.host}${options.requestOptions.path}`;
console.log(`Downloading ${url}...`);
return new Promise((c, e) => {
const child = cp.fork(__filename, [MARKER_ARGUMENT, base64encode(JSON.stringify(options))], {
stdio: ['pipe', 'pipe', 'pipe', 'ipc']
});
let stderr = [];
child.stderr.on('data', (chunk) => {
stderr.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk);
});
child.on('exit', (code) => {
if (code === 0) {
// normal termination
console.log(`Finished downloading ${url}.`);
c();
}
else {
// abnormal termination
console.error(Buffer.concat(stderr).toString());
e(new Error(`Download of ${url} failed.`));
}
});
});
}
exports.downloadInExternalProcess = downloadInExternalProcess;
function _downloadInExternalProcess() {
let options;
try {
options = JSON.parse(base64decode(process.argv[3]));
}
catch (err) {
console.error(`Cannot read arguments`);
console.error(err);
process.exit(-1);
return;
}
download(options.requestOptions, options.destinationPath).then(() => {
process.exit(0);
}, (err) => {
console.error(err);
process.exit(-2);
});
}
if (process.argv.length >= 4 && process.argv[2] === MARKER_ARGUMENT) {
// running as forked download script
_downloadInExternalProcess();
}

111
build/download/download.ts Normal file
View file

@ -0,0 +1,111 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as https from 'https';
import * as fs from 'fs';
import * as path from 'path';
import * as cp from 'child_process';
function ensureDir(filepath: string) {
if (!fs.existsSync(filepath)) {
ensureDir(path.dirname(filepath));
fs.mkdirSync(filepath);
}
}
function download(options: https.RequestOptions, destination: string): Promise<void> {
ensureDir(path.dirname(destination));
return new Promise<void>((c, e) => {
const fd = fs.openSync(destination, 'w');
const req = https.get(options, (res) => {
res.on('data', (chunk) => {
fs.writeSync(fd, chunk);
});
res.on('end', () => {
fs.closeSync(fd);
c();
});
});
req.on('error', (reqErr) => {
console.error(`request to ${options.host}${options.path} failed.`);
console.error(reqErr);
e(reqErr);
});
});
}
const MARKER_ARGUMENT = `_download_fork_`;
function base64encode(str: string): string {
return Buffer.from(str, 'utf8').toString('base64');
}
function base64decode(str: string): string {
return Buffer.from(str, 'base64').toString('utf8');
}
export interface IDownloadRequestOptions {
host: string;
path: string;
}
export interface IDownloadOptions {
requestOptions: IDownloadRequestOptions;
destinationPath: string;
}
export function downloadInExternalProcess(options: IDownloadOptions): Promise<void> {
const url = `https://${options.requestOptions.host}${options.requestOptions.path}`;
console.log(`Downloading ${url}...`);
return new Promise<void>((c, e) => {
const child = cp.fork(
__filename,
[MARKER_ARGUMENT, base64encode(JSON.stringify(options))],
{
stdio: ['pipe', 'pipe', 'pipe', 'ipc']
}
);
let stderr: Buffer[] = [];
child.stderr.on('data', (chunk) => {
stderr.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk);
});
child.on('exit', (code) => {
if (code === 0) {
// normal termination
console.log(`Finished downloading ${url}.`);
c();
} else {
// abnormal termination
console.error(Buffer.concat(stderr).toString());
e(new Error(`Download of ${url} failed.`));
}
});
});
}
function _downloadInExternalProcess() {
let options: IDownloadOptions;
try {
options = JSON.parse(base64decode(process.argv[3]));
} catch (err) {
console.error(`Cannot read arguments`);
console.error(err);
process.exit(-1);
return;
}
download(options.requestOptions, options.destinationPath).then(() => {
process.exit(0);
}, (err) => {
console.error(err);
process.exit(-2);
});
}
if (process.argv.length >= 4 && process.argv[2] === MARKER_ARGUMENT) {
// running as forked download script
_downloadInExternalProcess();
}

2
build/gulpfile.hygiene.js
View file

@ -81,7 +81,7 @@ const indentationFilter = [
'!src/typings/**/*.d.ts',
'!extensions/**/*.d.ts',
'!**/*.{svg,exe,png,bmp,scpt,bat,cmd,cur,ttf,woff,eot,md,ps1,template,yaml,yml,d.ts.recipe,ico,icns}',
'!build/{lib,tslintRules}/**/*.js',
'!build/{lib,tslintRules,download}/**/*.js',
'!build/**/*.sh',
'!build/azure-pipelines/**/*.js',
'!build/azure-pipelines/**/*.config',

37
build/lib/util.js
View file

@ -14,6 +14,8 @@ const fs = require("fs");
const _rimraf = require("rimraf");
const git = require("./git");
const VinylFile = require("vinyl");
const download_1 = require("../download/download");
const REPO_ROOT = path.join(__dirname, '../../');
const NoCancellationToken = { isCancellationRequested: () => false };
function incremental(streamProvider, initial, supportsCancellation) {
const input = es.through();
@ -221,3 +223,38 @@ function versionStringToNumber(versionStr) {
return parseInt(match[1], 10) * 1e4 + parseInt(match[2], 10) * 1e2 + parseInt(match[3], 10);
}
exports.versionStringToNumber = versionStringToNumber;
function download(requestOptions) {
const result = es.through();
const filename = path.join(REPO_ROOT, `.build/tmp-${Date.now()}-${path.posix.basename(requestOptions.path)}`);
const opts = {
requestOptions: requestOptions,
destinationPath: filename
};
download_1.downloadInExternalProcess(opts).then(() => {
fs.stat(filename, (err, stat) => {
if (err) {
result.emit('error', err);
return;
}
fs.readFile(filename, (err, data) => {
if (err) {
result.emit('error', err);
return;
}
fs.unlink(filename, () => {
result.emit('data', new VinylFile({
path: path.normalize(requestOptions.path),
stat: stat,
base: path.normalize(requestOptions.path),
contents: data
}));
result.emit('end');
});
});
});
}, (err) => {
result.emit('error', err);
});
return result;
}
exports.download = download;

38
build/lib/util.ts
View file

@ -17,6 +17,9 @@ import * as git from './git';
import * as VinylFile from 'vinyl';
import { ThroughStream } from 'through';
import * as sm from 'source-map';
import { IDownloadOptions, downloadInExternalProcess, IDownloadRequestOptions } from '../download/download';
const REPO_ROOT = path.join(__dirname, '../../');
export interface ICancellationToken {
isCancellationRequested(): boolean;
@ -280,3 +283,38 @@ export function versionStringToNumber(versionStr: string) {
return parseInt(match[1], 10) * 1e4 + parseInt(match[2], 10) * 1e2 + parseInt(match[3], 10);
}
export function download(requestOptions: IDownloadRequestOptions): NodeJS.ReadWriteStream {
const result = es.through();
const filename = path.join(REPO_ROOT, `.build/tmp-${Date.now()}-${path.posix.basename(requestOptions.path)}`);
const opts: IDownloadOptions = {
requestOptions: requestOptions,
destinationPath: filename
};
downloadInExternalProcess(opts).then(() => {
fs.stat(filename, (err, stat) => {
if (err) {
result.emit('error', err);
return;
}
fs.readFile(filename, (err, data) => {
if (err) {
result.emit('error', err);
return;
}
fs.unlink(filename, () => {
result.emit('data', new VinylFile({
path: path.normalize(requestOptions.path),
stat: stat,
base: path.normalize(requestOptions.path),
contents: data
}));
result.emit('end');
});
});
});
}, (err) => {
result.emit('error', err);
});
return result;
}