mirror of
https://github.com/Microsoft/vscode
synced 2024-09-13 13:46:13 +00:00
Add a download util that uses a separate process
This commit is contained in:
parent
ec878903e9
commit
c4bed46784
91
build/download/download.js
Normal file
91
build/download/download.js
Normal file
|
@ -0,0 +1,91 @@
|
|||
"use strict";
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const https = require("https");
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const cp = require("child_process");
|
||||
function ensureDir(filepath) {
|
||||
if (!fs.existsSync(filepath)) {
|
||||
ensureDir(path.dirname(filepath));
|
||||
fs.mkdirSync(filepath);
|
||||
}
|
||||
}
|
||||
function download(options, destination) {
|
||||
ensureDir(path.dirname(destination));
|
||||
return new Promise((c, e) => {
|
||||
const fd = fs.openSync(destination, 'w');
|
||||
const req = https.get(options, (res) => {
|
||||
res.on('data', (chunk) => {
|
||||
fs.writeSync(fd, chunk);
|
||||
});
|
||||
res.on('end', () => {
|
||||
fs.closeSync(fd);
|
||||
c();
|
||||
});
|
||||
});
|
||||
req.on('error', (reqErr) => {
|
||||
console.error(`request to ${options.host}${options.path} failed.`);
|
||||
console.error(reqErr);
|
||||
e(reqErr);
|
||||
});
|
||||
});
|
||||
}
|
||||
const MARKER_ARGUMENT = `_download_fork_`;
|
||||
function base64encode(str) {
|
||||
return Buffer.from(str, 'utf8').toString('base64');
|
||||
}
|
||||
function base64decode(str) {
|
||||
return Buffer.from(str, 'base64').toString('utf8');
|
||||
}
|
||||
function downloadInExternalProcess(options) {
|
||||
const url = `https://${options.requestOptions.host}${options.requestOptions.path}`;
|
||||
console.log(`Downloading ${url}...`);
|
||||
return new Promise((c, e) => {
|
||||
const child = cp.fork(__filename, [MARKER_ARGUMENT, base64encode(JSON.stringify(options))], {
|
||||
stdio: ['pipe', 'pipe', 'pipe', 'ipc']
|
||||
});
|
||||
let stderr = [];
|
||||
child.stderr.on('data', (chunk) => {
|
||||
stderr.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk);
|
||||
});
|
||||
child.on('exit', (code) => {
|
||||
if (code === 0) {
|
||||
// normal termination
|
||||
console.log(`Finished downloading ${url}.`);
|
||||
c();
|
||||
}
|
||||
else {
|
||||
// abnormal termination
|
||||
console.error(Buffer.concat(stderr).toString());
|
||||
e(new Error(`Download of ${url} failed.`));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
exports.downloadInExternalProcess = downloadInExternalProcess;
|
||||
function _downloadInExternalProcess() {
|
||||
let options;
|
||||
try {
|
||||
options = JSON.parse(base64decode(process.argv[3]));
|
||||
}
|
||||
catch (err) {
|
||||
console.error(`Cannot read arguments`);
|
||||
console.error(err);
|
||||
process.exit(-1);
|
||||
return;
|
||||
}
|
||||
download(options.requestOptions, options.destinationPath).then(() => {
|
||||
process.exit(0);
|
||||
}, (err) => {
|
||||
console.error(err);
|
||||
process.exit(-2);
|
||||
});
|
||||
}
|
||||
if (process.argv.length >= 4 && process.argv[2] === MARKER_ARGUMENT) {
|
||||
// running as forked download script
|
||||
_downloadInExternalProcess();
|
||||
}
|
111
build/download/download.ts
Normal file
111
build/download/download.ts
Normal file
|
@ -0,0 +1,111 @@
|
|||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import * as https from 'https';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as cp from 'child_process';
|
||||
|
||||
function ensureDir(filepath: string) {
|
||||
if (!fs.existsSync(filepath)) {
|
||||
ensureDir(path.dirname(filepath));
|
||||
fs.mkdirSync(filepath);
|
||||
}
|
||||
}
|
||||
|
||||
function download(options: https.RequestOptions, destination: string): Promise<void> {
|
||||
ensureDir(path.dirname(destination));
|
||||
|
||||
return new Promise<void>((c, e) => {
|
||||
const fd = fs.openSync(destination, 'w');
|
||||
const req = https.get(options, (res) => {
|
||||
res.on('data', (chunk) => {
|
||||
fs.writeSync(fd, chunk);
|
||||
});
|
||||
res.on('end', () => {
|
||||
fs.closeSync(fd);
|
||||
c();
|
||||
});
|
||||
});
|
||||
req.on('error', (reqErr) => {
|
||||
console.error(`request to ${options.host}${options.path} failed.`);
|
||||
console.error(reqErr);
|
||||
e(reqErr);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
const MARKER_ARGUMENT = `_download_fork_`;
|
||||
|
||||
function base64encode(str: string): string {
|
||||
return Buffer.from(str, 'utf8').toString('base64');
|
||||
}
|
||||
|
||||
function base64decode(str: string): string {
|
||||
return Buffer.from(str, 'base64').toString('utf8');
|
||||
}
|
||||
|
||||
export interface IDownloadRequestOptions {
|
||||
host: string;
|
||||
path: string;
|
||||
}
|
||||
|
||||
export interface IDownloadOptions {
|
||||
requestOptions: IDownloadRequestOptions;
|
||||
destinationPath: string;
|
||||
}
|
||||
|
||||
export function downloadInExternalProcess(options: IDownloadOptions): Promise<void> {
|
||||
const url = `https://${options.requestOptions.host}${options.requestOptions.path}`;
|
||||
console.log(`Downloading ${url}...`);
|
||||
return new Promise<void>((c, e) => {
|
||||
const child = cp.fork(
|
||||
__filename,
|
||||
[MARKER_ARGUMENT, base64encode(JSON.stringify(options))],
|
||||
{
|
||||
stdio: ['pipe', 'pipe', 'pipe', 'ipc']
|
||||
}
|
||||
);
|
||||
let stderr: Buffer[] = [];
|
||||
child.stderr.on('data', (chunk) => {
|
||||
stderr.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk);
|
||||
});
|
||||
child.on('exit', (code) => {
|
||||
if (code === 0) {
|
||||
// normal termination
|
||||
console.log(`Finished downloading ${url}.`);
|
||||
c();
|
||||
} else {
|
||||
// abnormal termination
|
||||
console.error(Buffer.concat(stderr).toString());
|
||||
e(new Error(`Download of ${url} failed.`));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function _downloadInExternalProcess() {
|
||||
let options: IDownloadOptions;
|
||||
try {
|
||||
options = JSON.parse(base64decode(process.argv[3]));
|
||||
} catch (err) {
|
||||
console.error(`Cannot read arguments`);
|
||||
console.error(err);
|
||||
process.exit(-1);
|
||||
return;
|
||||
}
|
||||
|
||||
download(options.requestOptions, options.destinationPath).then(() => {
|
||||
process.exit(0);
|
||||
}, (err) => {
|
||||
console.error(err);
|
||||
process.exit(-2);
|
||||
});
|
||||
}
|
||||
|
||||
if (process.argv.length >= 4 && process.argv[2] === MARKER_ARGUMENT) {
|
||||
// running as forked download script
|
||||
_downloadInExternalProcess();
|
||||
}
|
|
@ -81,7 +81,7 @@ const indentationFilter = [
|
|||
'!src/typings/**/*.d.ts',
|
||||
'!extensions/**/*.d.ts',
|
||||
'!**/*.{svg,exe,png,bmp,scpt,bat,cmd,cur,ttf,woff,eot,md,ps1,template,yaml,yml,d.ts.recipe,ico,icns}',
|
||||
'!build/{lib,tslintRules}/**/*.js',
|
||||
'!build/{lib,tslintRules,download}/**/*.js',
|
||||
'!build/**/*.sh',
|
||||
'!build/azure-pipelines/**/*.js',
|
||||
'!build/azure-pipelines/**/*.config',
|
||||
|
|
|
@ -14,6 +14,8 @@ const fs = require("fs");
|
|||
const _rimraf = require("rimraf");
|
||||
const git = require("./git");
|
||||
const VinylFile = require("vinyl");
|
||||
const download_1 = require("../download/download");
|
||||
const REPO_ROOT = path.join(__dirname, '../../');
|
||||
const NoCancellationToken = { isCancellationRequested: () => false };
|
||||
function incremental(streamProvider, initial, supportsCancellation) {
|
||||
const input = es.through();
|
||||
|
@ -221,3 +223,38 @@ function versionStringToNumber(versionStr) {
|
|||
return parseInt(match[1], 10) * 1e4 + parseInt(match[2], 10) * 1e2 + parseInt(match[3], 10);
|
||||
}
|
||||
exports.versionStringToNumber = versionStringToNumber;
|
||||
function download(requestOptions) {
|
||||
const result = es.through();
|
||||
const filename = path.join(REPO_ROOT, `.build/tmp-${Date.now()}-${path.posix.basename(requestOptions.path)}`);
|
||||
const opts = {
|
||||
requestOptions: requestOptions,
|
||||
destinationPath: filename
|
||||
};
|
||||
download_1.downloadInExternalProcess(opts).then(() => {
|
||||
fs.stat(filename, (err, stat) => {
|
||||
if (err) {
|
||||
result.emit('error', err);
|
||||
return;
|
||||
}
|
||||
fs.readFile(filename, (err, data) => {
|
||||
if (err) {
|
||||
result.emit('error', err);
|
||||
return;
|
||||
}
|
||||
fs.unlink(filename, () => {
|
||||
result.emit('data', new VinylFile({
|
||||
path: path.normalize(requestOptions.path),
|
||||
stat: stat,
|
||||
base: path.normalize(requestOptions.path),
|
||||
contents: data
|
||||
}));
|
||||
result.emit('end');
|
||||
});
|
||||
});
|
||||
});
|
||||
}, (err) => {
|
||||
result.emit('error', err);
|
||||
});
|
||||
return result;
|
||||
}
|
||||
exports.download = download;
|
||||
|
|
|
@ -17,6 +17,9 @@ import * as git from './git';
|
|||
import * as VinylFile from 'vinyl';
|
||||
import { ThroughStream } from 'through';
|
||||
import * as sm from 'source-map';
|
||||
import { IDownloadOptions, downloadInExternalProcess, IDownloadRequestOptions } from '../download/download';
|
||||
|
||||
const REPO_ROOT = path.join(__dirname, '../../');
|
||||
|
||||
export interface ICancellationToken {
|
||||
isCancellationRequested(): boolean;
|
||||
|
@ -280,3 +283,38 @@ export function versionStringToNumber(versionStr: string) {
|
|||
|
||||
return parseInt(match[1], 10) * 1e4 + parseInt(match[2], 10) * 1e2 + parseInt(match[3], 10);
|
||||
}
|
||||
|
||||
export function download(requestOptions: IDownloadRequestOptions): NodeJS.ReadWriteStream {
|
||||
const result = es.through();
|
||||
const filename = path.join(REPO_ROOT, `.build/tmp-${Date.now()}-${path.posix.basename(requestOptions.path)}`);
|
||||
const opts: IDownloadOptions = {
|
||||
requestOptions: requestOptions,
|
||||
destinationPath: filename
|
||||
};
|
||||
downloadInExternalProcess(opts).then(() => {
|
||||
fs.stat(filename, (err, stat) => {
|
||||
if (err) {
|
||||
result.emit('error', err);
|
||||
return;
|
||||
}
|
||||
fs.readFile(filename, (err, data) => {
|
||||
if (err) {
|
||||
result.emit('error', err);
|
||||
return;
|
||||
}
|
||||
fs.unlink(filename, () => {
|
||||
result.emit('data', new VinylFile({
|
||||
path: path.normalize(requestOptions.path),
|
||||
stat: stat,
|
||||
base: path.normalize(requestOptions.path),
|
||||
contents: data
|
||||
}));
|
||||
result.emit('end');
|
||||
});
|
||||
});
|
||||
});
|
||||
}, (err) => {
|
||||
result.emit('error', err);
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue