extensions: pull from github in OSS build (#146863)

* extensions: pull from github in OSS build

* fixup! address pr comment
This commit is contained in:
Connor Peet 2022-04-10 22:36:12 -07:00 committed by GitHub
parent b84feecf92
commit 5e86e40fab
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 149 additions and 29 deletions

View file

@ -103,6 +103,8 @@ steps:
- script: |
set -e
yarn npm-run-all -lp core-ci extensions-ci hygiene eslint valid-layers-check
env:
GITHUB_TOKEN: "$(github-distro-mixin-password)"
displayName: Compile & Hygiene
- script: |

View file

@ -20,6 +20,7 @@ const root = path.dirname(__dirname);
const commit = util.getVersion(root);
const plumber = require('gulp-plumber');
const ext = require('./lib/extensions');
const product = require('../product.json');
const extensionsPath = path.join(path.dirname(__dirname), 'extensions');
@ -221,7 +222,7 @@ const cleanExtensionsBuildTask = task.define('clean-extensions-build', util.rimr
const compileExtensionsBuildTask = task.define('compile-extensions-build', task.series(
cleanExtensionsBuildTask,
task.define('bundle-extensions-build', () => ext.packageLocalExtensionsStream(false).pipe(gulp.dest('.build'))),
task.define('bundle-marketplace-extensions-build', () => ext.packageMarketplaceExtensionsStream(false).pipe(gulp.dest('.build'))),
task.define('bundle-marketplace-extensions-build', () => ext.packageMarketplaceExtensionsStream(false, product.extensionsGallery?.serviceUrl).pipe(gulp.dest('.build'))),
));
gulp.task(compileExtensionsBuildTask);

View file

@ -229,7 +229,7 @@ function packageTask(sourceFolderName, destinationFolderName) {
const compileWebExtensionsBuildTask = task.define('compile-web-extensions-build', task.series(
task.define('clean-web-extensions-build', util.rimraf('.build/web/extensions')),
task.define('bundle-web-extensions-build', () => extensions.packageLocalExtensionsStream(true).pipe(gulp.dest('.build/web'))),
task.define('bundle-marketplace-web-extensions-build', () => extensions.packageMarketplaceExtensionsStream(true).pipe(gulp.dest('.build/web'))),
task.define('bundle-marketplace-web-extensions-build', () => extensions.packageMarketplaceExtensionsStream(true, product.extensionsGallery?.serviceUrl).pipe(gulp.dest('.build/web'))),
task.define('bundle-web-extension-media-build', () => extensions.buildExtensionMedia(false, '.build/web/extensions')),
));
gulp.task(compileWebExtensionsBuildTask);

View file

@ -45,15 +45,17 @@ function isUpToDate(extension) {
}
}
function syncMarketplaceExtension(extension) {
const galleryServiceUrl = productjson.extensionsGallery?.serviceUrl;
const source = ansiColors.blue(galleryServiceUrl ? '[marketplace]' : '[github]');
if (isUpToDate(extension)) {
log(ansiColors.blue('[marketplace]'), `${extension.name}@${extension.version}`, ansiColors.green('✔︎'));
log(source, `${extension.name}@${extension.version}`, ansiColors.green('✔︎'));
return es.readArray([]);
}
rimraf.sync(getExtensionPath(extension));
return ext.fromMarketplace(extension.name, extension.version, extension.metadata)
return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension))
.pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`))
.pipe(vfs.dest('.build/builtInExtensions'))
.on('end', () => log(ansiColors.blue('[marketplace]'), extension.name, ansiColors.green('✔︎')));
.on('end', () => log(source, extension.name, ansiColors.green('✔︎')));
}
function syncExtension(extension, controlState) {
if (extension.platforms) {

View file

@ -69,17 +69,20 @@ function isUpToDate(extension: IExtensionDefinition): boolean {
}
function syncMarketplaceExtension(extension: IExtensionDefinition): Stream {
const galleryServiceUrl = productjson.extensionsGallery?.serviceUrl;
const source = ansiColors.blue(galleryServiceUrl ? '[marketplace]' : '[github]');
if (isUpToDate(extension)) {
log(ansiColors.blue('[marketplace]'), `${extension.name}@${extension.version}`, ansiColors.green('✔︎'));
log(source, `${extension.name}@${extension.version}`, ansiColors.green('✔︎'));
return es.readArray([]);
}
rimraf.sync(getExtensionPath(extension));
return ext.fromMarketplace(extension.name, extension.version, extension.metadata)
return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension))
.pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`))
.pipe(vfs.dest('.build/builtInExtensions'))
.on('end', () => log(ansiColors.blue('[marketplace]'), extension.name, ansiColors.green('✔︎')));
.on('end', () => log(source, extension.name, ansiColors.green('✔︎')));
}
function syncExtension(extension: IExtensionDefinition, controlState: 'disabled' | 'marketplace'): Stream {

View file

@ -4,13 +4,15 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.buildExtensionMedia = exports.webpackExtensions = exports.translatePackageJSON = exports.scanBuiltinExtensions = exports.packageMarketplaceExtensionsStream = exports.packageLocalExtensionsStream = exports.fromMarketplace = void 0;
exports.buildExtensionMedia = exports.webpackExtensions = exports.translatePackageJSON = exports.scanBuiltinExtensions = exports.packageMarketplaceExtensionsStream = exports.packageLocalExtensionsStream = exports.fromGithub = exports.fromMarketplace = void 0;
const es = require("event-stream");
const fs = require("fs");
const cp = require("child_process");
const glob = require("glob");
const gulp = require("gulp");
const path = require("path");
const through2 = require("through2");
const got_1 = require("got");
const File = require("vinyl");
const stats_1 = require("./stats");
const util2 = require("./util");
@ -169,16 +171,17 @@ function fromLocalNormal(extensionPath) {
.catch(err => result.emit('error', err));
return result.pipe((0, stats_1.createStatsStream)(path.basename(extensionPath)));
}
const userAgent = 'VSCode Build';
const baseHeaders = {
'X-Market-Client-Id': 'VSCode Build',
'User-Agent': 'VSCode Build',
'User-Agent': userAgent,
'X-Market-User-Id': '291C1CD0-051A-4123-9B4B-30D60EF52EE2',
};
function fromMarketplace(extensionName, version, metadata) {
function fromMarketplace(serviceUrl, { name: extensionName, version, metadata }) {
const remote = require('gulp-remote-retry-src');
const json = require('gulp-json-editor');
const [publisher, name] = extensionName.split('.');
const url = `https://marketplace.visualstudio.com/_apis/public/gallery/publishers/${publisher}/vsextensions/${name}/${version}/vspackage`;
const url = `${serviceUrl}/publishers/${publisher}/vsextensions/${name}/${version}/vspackage`;
fancyLog('Downloading extension:', ansiColors.yellow(`${extensionName}@${version}`), '...');
const options = {
base: url,
@ -198,6 +201,44 @@ function fromMarketplace(extensionName, version, metadata) {
.pipe(packageJsonFilter.restore);
}
exports.fromMarketplace = fromMarketplace;
const ghApiHeaders = {
Accept: 'application/vnd.github.v3+json',
'User-Agent': userAgent,
};
if (process.env.GITHUB_TOKEN) {
ghApiHeaders.Authorization = 'Basic ' + Buffer.from(process.env.GITHUB_TOKEN).toString('base64');
}
const ghDownloadHeaders = {
...ghApiHeaders,
Accept: 'application/octet-stream',
};
function fromGithub({ name, version, repo, metadata }) {
const remote = require('gulp-remote-retry-src');
const json = require('gulp-json-editor');
fancyLog('Downloading extension from GH:', ansiColors.yellow(`${name}@${version}`), '...');
const packageJsonFilter = filter('package.json', { restore: true });
return remote([`/repos${new URL(repo).pathname}/releases/tags/v${version}`], {
base: 'https://api.github.com',
requestOptions: { headers: ghApiHeaders }
}).pipe(through2.obj(function (file, _enc, callback) {
const asset = JSON.parse(file.contents.toString()).assets.find((a) => a.name.endsWith('.vsix'));
if (!asset) {
return callback(new Error(`Could not find vsix in release of ${repo} @ ${version}`));
}
const res = got_1.default.stream(asset.url, { headers: ghDownloadHeaders, followRedirect: true });
file.contents = res.pipe(through2());
callback(null, file);
}))
.pipe(buffer())
.pipe(vzip.src())
.pipe(filter('extension/**'))
.pipe(rename(p => p.dirname = p.dirname.replace(/^extension\/?/, '')))
.pipe(packageJsonFilter)
.pipe(buffer())
.pipe(json({ __metadata: metadata }))
.pipe(packageJsonFilter.restore);
}
exports.fromGithub = fromGithub;
const excludedExtensions = [
'vscode-api-tests',
'vscode-colorize-tests',
@ -272,14 +313,14 @@ function packageLocalExtensionsStream(forWeb) {
.pipe(util2.setExecutableBit(['**/*.sh'])));
}
exports.packageLocalExtensionsStream = packageLocalExtensionsStream;
function packageMarketplaceExtensionsStream(forWeb) {
function packageMarketplaceExtensionsStream(forWeb, galleryServiceUrl) {
const marketplaceExtensionsDescriptions = [
...builtInExtensions.filter(({ name }) => (forWeb ? !marketplaceWebExtensionsExclude.has(name) : true)),
...(forWeb ? webBuiltInExtensions : [])
];
const marketplaceExtensionsStream = minifyExtensionResources(es.merge(...marketplaceExtensionsDescriptions
.map(extension => {
const input = fromMarketplace(extension.name, extension.version, extension.metadata)
const input = (galleryServiceUrl ? fromMarketplace(galleryServiceUrl, extension) : fromGithub(extension))
.pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
return updateExtensionPackageJSON(input, (data) => {
delete data.scripts;

View file

@ -9,6 +9,8 @@ import * as cp from 'child_process';
import * as glob from 'glob';
import * as gulp from 'gulp';
import * as path from 'path';
import * as through2 from 'through2';
import got from 'got';
import { Stream } from 'stream';
import * as File from 'vinyl';
import { createStatsStream } from './stats';
@ -198,18 +200,19 @@ function fromLocalNormal(extensionPath: string): Stream {
return result.pipe(createStatsStream(path.basename(extensionPath)));
}
const userAgent = 'VSCode Build';
const baseHeaders = {
'X-Market-Client-Id': 'VSCode Build',
'User-Agent': 'VSCode Build',
'User-Agent': userAgent,
'X-Market-User-Id': '291C1CD0-051A-4123-9B4B-30D60EF52EE2',
};
export function fromMarketplace(extensionName: string, version: string, metadata: any): Stream {
export function fromMarketplace(serviceUrl: string, { name: extensionName, version, metadata }: IBuiltInExtension): Stream {
const remote = require('gulp-remote-retry-src');
const json = require('gulp-json-editor') as typeof import('gulp-json-editor');
const [publisher, name] = extensionName.split('.');
const url = `https://marketplace.visualstudio.com/_apis/public/gallery/publishers/${publisher}/vsextensions/${name}/${version}/vspackage`;
const url = `${serviceUrl}/publishers/${publisher}/vsextensions/${name}/${version}/vspackage`;
fancyLog('Downloading extension:', ansiColors.yellow(`${extensionName}@${version}`), '...');
@ -232,6 +235,50 @@ export function fromMarketplace(extensionName: string, version: string, metadata
.pipe(json({ __metadata: metadata }))
.pipe(packageJsonFilter.restore);
}
const ghApiHeaders: Record<string, string> = {
Accept: 'application/vnd.github.v3+json',
'User-Agent': userAgent,
};
if (process.env.GITHUB_TOKEN) {
ghApiHeaders.Authorization = 'Basic ' + Buffer.from(process.env.GITHUB_TOKEN).toString('base64');
}
const ghDownloadHeaders = {
...ghApiHeaders,
Accept: 'application/octet-stream',
};
export function fromGithub({ name, version, repo, metadata }: IBuiltInExtension): Stream {
const remote = require('gulp-remote-retry-src');
const json = require('gulp-json-editor') as typeof import('gulp-json-editor');
fancyLog('Downloading extension from GH:', ansiColors.yellow(`${name}@${version}`), '...');
const packageJsonFilter = filter('package.json', { restore: true });
return remote([`/repos${new URL(repo).pathname}/releases/tags/v${version}`], {
base: 'https://api.github.com',
requestOptions: { headers: ghApiHeaders }
}).pipe(through2.obj(function (file, _enc, callback) {
const asset = JSON.parse(file.contents.toString()).assets.find((a: any) => a.name.endsWith('.vsix'));
if (!asset) {
return callback(new Error(`Could not find vsix in release of ${repo} @ ${version}`));
}
const res = got.stream(asset.url, { headers: ghDownloadHeaders, followRedirect: true });
file.contents = res.pipe(through2());
callback(null, file);
}))
.pipe(buffer())
.pipe(vzip.src())
.pipe(filter('extension/**'))
.pipe(rename(p => p.dirname = p.dirname!.replace(/^extension\/?/, '')))
.pipe(packageJsonFilter)
.pipe(buffer())
.pipe(json({ __metadata: metadata }))
.pipe(packageJsonFilter.restore);
}
const excludedExtensions = [
'vscode-api-tests',
'vscode-colorize-tests',
@ -335,7 +382,7 @@ export function packageLocalExtensionsStream(forWeb: boolean): Stream {
);
}
export function packageMarketplaceExtensionsStream(forWeb: boolean): Stream {
export function packageMarketplaceExtensionsStream(forWeb: boolean, galleryServiceUrl?: string): Stream {
const marketplaceExtensionsDescriptions = [
...builtInExtensions.filter(({ name }) => (forWeb ? !marketplaceWebExtensionsExclude.has(name) : true)),
...(forWeb ? webBuiltInExtensions : [])
@ -344,7 +391,7 @@ export function packageMarketplaceExtensionsStream(forWeb: boolean): Stream {
es.merge(
...marketplaceExtensionsDescriptions
.map(extension => {
const input = fromMarketplace(extension.name, extension.version, extension.metadata)
const input = (galleryServiceUrl ? fromMarketplace(galleryServiceUrl, extension) : fromGithub(extension))
.pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
return updateExtensionPackageJSON(input, (data: any) => {
delete data.scripts;

View file

@ -35,7 +35,7 @@
"@types/request": "^2.47.0",
"@types/rimraf": "^2.0.4",
"@types/through": "^0.0.29",
"@types/through2": "^2.0.34",
"@types/through2": "^2.0.36",
"@types/tmp": "^0.2.1",
"@types/underscore": "^1.8.9",
"@types/webpack": "^4.41.25",
@ -59,6 +59,7 @@
"mkdirp": "^1.0.4",
"p-limit": "^3.1.0",
"source-map": "0.6.1",
"through2": "^4.0.2",
"tmp": "^0.2.1",
"vsce": "^1.100.0",
"vscode-universal-bundler": "^0.0.2"

View file

@ -576,10 +576,10 @@
resolved "https://registry.yarnpkg.com/@types/tapable/-/tapable-1.0.8.tgz#b94a4391c85666c7b73299fd3ad79d4faa435310"
integrity sha512-ipixuVrh2OdNmauvtT51o3d8z12p6LtFW9in7U79der/kwejjdNchQC5UMn5u/KxNoM7VHHOs/l8KS8uHxhODQ==
"@types/through2@^2.0.34":
version "2.0.34"
resolved "https://registry.yarnpkg.com/@types/through2/-/through2-2.0.34.tgz#9c2a259a238dace2a05a2f8e94b786961bc27ac4"
integrity sha512-nhRG8+RuG/L+0fAZBQYaRflXKjTrHOKH8MFTChnf+dNVMxA3wHYYrfj0tztK0W51ABXjGfRCDc0vRkecCOrsow==
"@types/through2@^2.0.36":
version "2.0.36"
resolved "https://registry.yarnpkg.com/@types/through2/-/through2-2.0.36.tgz#35fda0db635827d44c0e08e2c94653e647574a00"
integrity sha512-vuifQksQHJXhV9McpVsXKuhnf3lsoX70PnhcqIAbs9dqLH2NgrGz0DzZPDY3+Yh6eaRqcE1gnCQ6QhBn1/PT5A==
dependencies:
"@types/node" "*"
@ -1909,7 +1909,7 @@ inflight@^1.0.4:
once "^1.3.0"
wrappy "1"
inherits@2, inherits@^2.0.1, inherits@~2.0.3:
inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.3:
version "2.0.4"
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
@ -2533,6 +2533,15 @@ read@^1.0.7:
dependencies:
mute-stream "~0.0.4"
readable-stream@3:
version "3.6.0"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
dependencies:
inherits "^2.0.3"
string_decoder "^1.1.1"
util-deprecate "^1.0.1"
readable-stream@^2.3.5:
version "2.3.7"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
@ -2606,7 +2615,7 @@ run-parallel@^1.1.9:
dependencies:
queue-microtask "^1.2.2"
safe-buffer@^5.0.1:
safe-buffer@^5.0.1, safe-buffer@~5.2.0:
version "5.2.1"
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
@ -2728,6 +2737,13 @@ stoppable@^1.1.0:
resolved "https://registry.yarnpkg.com/stoppable/-/stoppable-1.1.0.tgz#32da568e83ea488b08e4d7ea2c3bcc9d75015d5b"
integrity sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==
string_decoder@^1.1.1:
version "1.3.0"
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"
integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==
dependencies:
safe-buffer "~5.2.0"
string_decoder@~1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8"
@ -2756,6 +2772,13 @@ supports-color@^6.1.0:
dependencies:
has-flag "^3.0.0"
through2@^4.0.2:
version "4.0.2"
resolved "https://registry.yarnpkg.com/through2/-/through2-4.0.2.tgz#a7ce3ac2a7a8b0b966c80e7c49f0484c3b239764"
integrity sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==
dependencies:
readable-stream "3"
through@^2.3.8:
version "2.3.8"
resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
@ -2872,7 +2895,7 @@ url-parse-lax@^3.0.0:
dependencies:
prepend-http "^2.0.0"
util-deprecate@~1.0.1:
util-deprecate@^1.0.1, util-deprecate@~1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=

View file

@ -46,7 +46,7 @@
},
{
"name": "ms-vscode.js-debug-companion",
"version": "1.0.16",
"version": "1.0.17",
"repo": "https://github.com/microsoft/vscode-js-debug-companion",
"metadata": {
"id": "99cb0b7f-7354-4278-b8da-6cc79972169d",
@ -76,7 +76,7 @@
},
{
"name": "ms-vscode.vscode-js-profile-table",
"version": "1.0.0",
"version": "1.0.1",
"repo": "https://github.com/microsoft/vscode-js-profile-visualizer",
"metadata": {
"id": "7e52b41b-71ad-457b-ab7e-0620f1fc4feb",